Finn Thain <fthain@linux-m68k.org> <fthain@telegraphics.com.au>
Franck Bui-Huu <vagabon.xyz@gmail.com>
Frank Rowand <frowand.list@gmail.com> <frank.rowand@am.sony.com>
+Frank Rowand <frowand.list@gmail.com> <frank.rowand@sony.com>
Frank Rowand <frowand.list@gmail.com> <frank.rowand@sonymobile.com>
Frank Rowand <frowand.list@gmail.com> <frowand@mvista.com>
Frank Zago <fzago@systemfabricworks.com>
The file is writable if the PF is bound to a driver that
implements ->sriov_set_msix_vec_count().
+
+What: /sys/bus/pci/devices/.../resourceN_resize
+Date: September 2022
+Contact: Alex Williamson <alex.williamson@redhat.com>
+Description:
+ These files provide an interface to PCIe Resizable BAR support.
+ A file is created for each BAR resource (N) supported by the
+ PCIe Resizable BAR extended capability of the device. Reading
+ each file exposes the bitmap of available resource sizes:
+
+ # cat resource1_resize
+ 00000000000001c0
+
+ The bitmap represents supported resource sizes for the BAR,
+ where bit0 = 1MB, bit1 = 2MB, bit2 = 4MB, etc. In the above
+ example the device supports 64MB, 128MB, and 256MB BAR sizes.
+
+ When writing the file, the user provides the bit position of
+ the desired resource size, for example:
+
+ # echo 7 > resource1_resize
+
+ This indicates to set the size value corresponding to bit 7,
+ 128MB. The resulting size is 2 ^ (bit# + 20). This definition
+ matches the PCIe specification of this capability.
+
+ In order to make use of resource resizing, all PCI drivers must
+ be unbound from the device and peer devices under the same
+ parent bridge may need to be soft removed. In the case of
+ VGA devices, writing a resize value will remove low level
+ console drivers from the device. Raw users of pci-sysfs
+ resourceN attributes must be terminated prior to resizing.
+ Success of the resizing operation is not guaranteed.
--- /dev/null
+What: /sys/.../<device>/vfio-dev/vfioX/
+Date: September 2022
+Contact: Yi Liu <yi.l.liu@intel.com>
+Description:
+ This directory is created when the device is bound to a
+ vfio driver. The layout under this directory matches what
+ exists for a standard 'struct device'. 'X' is a unique
+ index marking this device in vfio.
0x4000 SBI_IS_FREEZING freefs is in process
====== ===================== =================================
+What: /sys/fs/f2fs/<disk>/stat/cp_status
+Date: September 2022
+Contact: "Chao Yu" <chao.yu@oppo.com>
+Description: Show status of f2fs checkpoint in real time.
+
+ =============================== ==============================
+ cp flag value
+ CP_UMOUNT_FLAG 0x00000001
+ CP_ORPHAN_PRESENT_FLAG 0x00000002
+ CP_COMPACT_SUM_FLAG 0x00000004
+ CP_ERROR_FLAG 0x00000008
+ CP_FSCK_FLAG 0x00000010
+ CP_FASTBOOT_FLAG 0x00000020
+ CP_CRC_RECOVERY_FLAG 0x00000040
+ CP_NAT_BITS_FLAG 0x00000080
+ CP_TRIMMED_FLAG 0x00000100
+ CP_NOCRC_RECOVERY_FLAG 0x00000200
+ CP_LARGE_NAT_BITMAP_FLAG 0x00000400
+ CP_QUOTA_NEED_FSCK_FLAG 0x00000800
+ CP_DISABLED_FLAG 0x00001000
+ CP_DISABLED_QUICK_FLAG 0x00002000
+ CP_RESIZEFS_FLAG 0x00004000
+ =============================== ==============================
+
What: /sys/fs/f2fs/<disk>/ckpt_thread_ioprio
Date: January 2021
Contact: "Daeho Jeong" <daehojeong@google.com>
--- /dev/null
+What: /sys/devices/virtual/memory_tiering/
+Date: August 2022
+Contact: Linux memory management mailing list <linux-mm@kvack.org>
+Description: A collection of all the memory tiers allocated.
+
+ Individual memory tier details are contained in subdirectories
+ named by the abstract distance of the memory tier.
+
+ /sys/devices/virtual/memory_tiering/memory_tierN/
+
+
+What: /sys/devices/virtual/memory_tiering/memory_tierN/
+ /sys/devices/virtual/memory_tiering/memory_tierN/nodes
+Date: August 2022
+Contact: Linux memory management mailing list <linux-mm@kvack.org>
+Description: Directory with details of a specific memory tier
+
+ This is the directory containing information about a particular
+ memory tier, memtierN, where N is derived based on abstract distance.
+
+ A smaller value of N implies a higher (faster) memory tier in the
+ hierarchy.
+
+ nodes: NUMA nodes that are part of this memory tier.
+
b) completion of synchronous block I/O initiated by the task
c) swapping in pages
d) memory reclaim
-e) thrashing page cache
+e) thrashing
f) direct compact
g) write-protect copy
:maxdepth: 1
initrd_table_override
- dsdt-override
ssdt-overlays
cppc_sysfs
fan_performance_states
lruvec->lru_lock; PG_lru bit of page->flags is cleared before
isolating a page from its LRU under lruvec->lru_lock.
-2.7 Kernel Memory Extension (CONFIG_MEMCG_KMEM)
+2.7 Kernel Memory Extension
-----------------------------------------------
With the Kernel memory extension, the Memory Controller is able to limit
a. Enable CONFIG_CGROUPS
b. Enable CONFIG_MEMCG
-c. Enable CONFIG_MEMCG_SWAP (to use swap extension)
-d. Enable CONFIG_MEMCG_KMEM (to use kmem extension)
3.1. Prepare the cgroups (see cgroups.txt, Why are cgroups needed?)
-------------------------------------------------------------------
killing cgroups is a process directed operation, i.e. it affects
the whole thread-group.
+ cgroup.pressure
+ A read-write single value file that allowed values are "0" and "1".
+ The default is "1".
+
+ Writing "0" to the file will disable the cgroup PSI accounting.
+ Writing "1" to the file will re-enable the cgroup PSI accounting.
+
+ This control attribute is not hierarchical, so disable or enable PSI
+ accounting in a cgroup does not affect PSI accounting in descendants
+ and doesn't need pass enablement via ancestors from root.
+
+ The reason this control attribute exists is that PSI accounts stalls for
+ each cgroup separately and aggregates it at each level of the hierarchy.
+ This may cause non-negligible overhead for some workloads when under
+ deep level of the hierarchy, in which case this control attribute can
+ be used to disable PSI accounting in the non-leaf cgroups.
+
+ irq.pressure
+ A read-write nested-keyed file.
+
+ Shows pressure stall information for IRQ/SOFTIRQ. See
+ :ref:`Documentation/accounting/psi.rst <psi>` for details.
+
Controllers
===========
Permit 'security.evm' to be updated regardless of
current integrity status.
+ early_page_ext [KNL] Enforces page_ext initialization to earlier
+ stages so cover more early boot allocations.
+ Please note that as side effect some optimizations
+ might be disabled to achieve that (e.g. parallelized
+ memory initialization is disabled) so the boot process
+ might take longer, especially on systems with a lot of
+ memory. Available with CONFIG_PAGE_EXTENSION=y.
+
failslab=
fail_usercopy=
fail_page_alloc=
This parameter controls use of the Protected
Execution Facility on pSeries.
- swapaccount= [KNL]
- Format: [0|1]
- Enable accounting of swap in memory resource
- controller if no parameter or 1 is given or disable
- it if 0 is given (See Documentation/admin-guide/cgroup-v1/memory.rst)
-
swiotlb= [ARM,IA-64,PPC,MIPS,X86]
Format: { <int> [,<int>] | force | noforce }
<int> -- Number of I/O TLB slabs
Crash from Xen panic notifier, without executing late
panic() code such as dumping handler.
+ xen_msr_safe= [X86,XEN]
+ Format: <bool>
+ Select whether to always use non-faulting (safe) MSR
+ access functions when running as Xen PV guest. The
+ default value is controlled by CONFIG_XEN_PV_MSR_SAFE.
+
xen_nopvspin [X86,XEN]
Disables the qspinlock slowpath using Xen PV optimizations.
This parameter is obsoleted by "nopvspin" parameter, which
The CMA debugfs interface is useful to retrieve basic information out of the
different CMA areas and to test allocation/release in each of the areas.
-Each CMA zone represents a directory under <debugfs>/cma/, indexed by the
-kernel's CMA index. So the first CMA zone would be:
+Each CMA area represents a directory under <debugfs>/cma/, represented by
+its CMA name like below:
- <debugfs>/cma/cma-0
+ <debugfs>/cma/<cma_name>
The structure of the files created under that directory is as follows:
- [RO] bitmap: The bitmap of page states in the zone.
- [WO] alloc: Allocate N pages from that CMA area. For example::
- echo 5 > <debugfs>/cma/cma-2/alloc
+ echo 5 > <debugfs>/cma/<cma_name>/alloc
-would try to allocate 5 pages from the cma-2 area.
+would try to allocate 5 pages from the 'cma_name' area.
- [WO] free: Free N pages from that CMA area, similar to the above.
.. SPDX-License-Identifier: GPL-2.0
-========================
-Monitoring Data Accesses
-========================
+==========================
+DAMON: Data Access MONitor
+==========================
:doc:`DAMON </mm/damon/index>` allows light-weight data access monitoring.
Using DAMON, users can analyze the memory access patterns of their systems and
https://github.com/awslabs/damo. The examples below assume that ``damo`` is on
your ``$PATH``. It's not mandatory, though.
-Because DAMO is using the debugfs interface (refer to :doc:`usage` for the
-detail) of DAMON, you should ensure debugfs is mounted. Mount it manually as
-below::
-
- # mount -t debugfs none /sys/kernel/debug/
-
-or append the following line to your ``/etc/fstab`` file so that your system
-can automatically mount debugfs upon booting::
-
- debugfs /sys/kernel/debug debugfs defaults 0 0
+Because DAMO is using the sysfs interface (refer to :doc:`usage` for the
+detail) of DAMON, you should ensure :doc:`sysfs </filesystems/sysfs>` is
+mounted.
Recording Data Access Patterns
debugfs Interface
=================
+.. note::
+
+ DAMON debugfs interface will be removed after next LTS kernel is released, so
+ users should move to the :ref:`sysfs interface <sysfs_interface>`.
+
DAMON exports eight files, ``attrs``, ``target_ids``, ``init_regions``,
``schemes``, ``monitor_on``, ``kdamond_pid``, ``mk_contexts`` and
``rm_contexts`` under its debugfs directory, ``<debugfs>/damon/``.
idle_page_tracking
ksm
memory-hotplug
+ multigen_lru
nommu-mmap
numa_memory_policy
numaperf
``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must
be increased accordingly.
+Monitoring KSM profit
+=====================
+
+KSM can save memory by merging identical pages, but also can consume
+additional memory, because it needs to generate a number of rmap_items to
+save each scanned page's brief rmap information. Some of these pages may
+be merged, but some may not be abled to be merged after being checked
+several times, which are unprofitable memory consumed.
+
+1) How to determine whether KSM save memory or consume memory in system-wide
+ range? Here is a simple approximate calculation for reference::
+
+ general_profit =~ pages_sharing * sizeof(page) - (all_rmap_items) *
+ sizeof(rmap_item);
+
+ where all_rmap_items can be easily obtained by summing ``pages_sharing``,
+ ``pages_shared``, ``pages_unshared`` and ``pages_volatile``.
+
+2) The KSM profit inner a single process can be similarly obtained by the
+ following approximate calculation::
+
+ process_profit =~ ksm_merging_pages * sizeof(page) -
+ ksm_rmap_items * sizeof(rmap_item).
+
+ where ksm_merging_pages is shown under the directory ``/proc/<pid>/``,
+ and ksm_rmap_items is shown in ``/proc/<pid>/ksm_stat``.
+
+From the perspective of application, a high ratio of ``ksm_rmap_items`` to
+``ksm_merging_pages`` means a bad madvise-applied policy, so developers or
+administrators have to rethink how to change madvise policy. Giving an example
+for reference, a page's size is usually 4K, and the rmap_item's size is
+separately 32B on 32-bit CPU architecture and 64B on 64-bit CPU architecture.
+so if the ``ksm_rmap_items/ksm_merging_pages`` ratio exceeds 64 on 64-bit CPU
+or exceeds 128 on 32-bit CPU, then the app's madvise policy should be dropped,
+because the ksm profit is approximately zero or negative.
+
Monitoring KSM events
=====================
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+=============
+Multi-Gen LRU
+=============
+The multi-gen LRU is an alternative LRU implementation that optimizes
+page reclaim and improves performance under memory pressure. Page
+reclaim decides the kernel's caching policy and ability to overcommit
+memory. It directly impacts the kswapd CPU usage and RAM efficiency.
+
+Quick start
+===========
+Build the kernel with the following configurations.
+
+* ``CONFIG_LRU_GEN=y``
+* ``CONFIG_LRU_GEN_ENABLED=y``
+
+All set!
+
+Runtime options
+===============
+``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the
+following subsections.
+
+Kill switch
+-----------
+``enabled`` accepts different values to enable or disable the
+following components. Its default value depends on
+``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled
+unless some of them have unforeseen side effects. Writing to
+``enabled`` has no effect when a component is not supported by the
+hardware, and valid values will be accepted even when the main switch
+is off.
+
+====== ===============================================================
+Values Components
+====== ===============================================================
+0x0001 The main switch for the multi-gen LRU.
+0x0002 Clearing the accessed bit in leaf page table entries in large
+ batches, when MMU sets it (e.g., on x86). This behavior can
+ theoretically worsen lock contention (mmap_lock). If it is
+ disabled, the multi-gen LRU will suffer a minor performance
+ degradation for workloads that contiguously map hot pages,
+ whose accessed bits can be otherwise cleared by fewer larger
+ batches.
+0x0004 Clearing the accessed bit in non-leaf page table entries as
+ well, when MMU sets it (e.g., on x86). This behavior was not
+ verified on x86 varieties other than Intel and AMD. If it is
+ disabled, the multi-gen LRU will suffer a negligible
+ performance degradation.
+[yYnN] Apply to all the components above.
+====== ===============================================================
+
+E.g.,
+::
+
+ echo y >/sys/kernel/mm/lru_gen/enabled
+ cat /sys/kernel/mm/lru_gen/enabled
+ 0x0007
+ echo 5 >/sys/kernel/mm/lru_gen/enabled
+ cat /sys/kernel/mm/lru_gen/enabled
+ 0x0005
+
+Thrashing prevention
+--------------------
+Personal computers are more sensitive to thrashing because it can
+cause janks (lags when rendering UI) and negatively impact user
+experience. The multi-gen LRU offers thrashing prevention to the
+majority of laptop and desktop users who do not have ``oomd``.
+
+Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of
+``N`` milliseconds from getting evicted. The OOM killer is triggered
+if this working set cannot be kept in memory. In other words, this
+option works as an adjustable pressure relief valve, and when open, it
+terminates applications that are hopefully not being used.
+
+Based on the average human detectable lag (~100ms), ``N=1000`` usually
+eliminates intolerable janks due to thrashing. Larger values like
+``N=3000`` make janks less noticeable at the risk of premature OOM
+kills.
+
+The default value ``0`` means disabled.
+
+Experimental features
+=====================
+``/sys/kernel/debug/lru_gen`` accepts commands described in the
+following subsections. Multiple command lines are supported, so does
+concatenation with delimiters ``,`` and ``;``.
+
+``/sys/kernel/debug/lru_gen_full`` provides additional stats for
+debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from
+evicted generations in this file.
+
+Working set estimation
+----------------------
+Working set estimation measures how much memory an application needs
+in a given time interval, and it is usually done with little impact on
+the performance of the application. E.g., data centers want to
+optimize job scheduling (bin packing) to improve memory utilizations.
+When a new job comes in, the job scheduler needs to find out whether
+each server it manages can allocate a certain amount of memory for
+this new job before it can pick a candidate. To do so, the job
+scheduler needs to estimate the working sets of the existing jobs.
+
+When it is read, ``lru_gen`` returns a histogram of numbers of pages
+accessed over different time intervals for each memcg and node.
+``MAX_NR_GENS`` decides the number of bins for each histogram. The
+histograms are noncumulative.
+::
+
+ memcg memcg_id memcg_path
+ node node_id
+ min_gen_nr age_in_ms nr_anon_pages nr_file_pages
+ ...
+ max_gen_nr age_in_ms nr_anon_pages nr_file_pages
+
+Each bin contains an estimated number of pages that have been accessed
+within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages
+and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of
+the former is the largest and that of the latter is the smallest.
+
+Users can write the following command to ``lru_gen`` to create a new
+generation ``max_gen_nr+1``:
+
+ ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]``
+
+``can_swap`` defaults to the swap setting and, if it is set to ``1``,
+it forces the scan of anon pages when swap is off, and vice versa.
+``force_scan`` defaults to ``1`` and, if it is set to ``0``, it
+employs heuristics to reduce the overhead, which is likely to reduce
+the coverage as well.
+
+A typical use case is that a job scheduler runs this command at a
+certain time interval to create new generations, and it ranks the
+servers it manages based on the sizes of their cold pages defined by
+this time interval.
+
+Proactive reclaim
+-----------------
+Proactive reclaim induces page reclaim when there is no memory
+pressure. It usually targets cold pages only. E.g., when a new job
+comes in, the job scheduler wants to proactively reclaim cold pages on
+the server it selected, to improve the chance of successfully landing
+this new job.
+
+Users can write the following command to ``lru_gen`` to evict
+generations less than or equal to ``min_gen_nr``.
+
+ ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]``
+
+``min_gen_nr`` should be less than ``max_gen_nr-1``, since
+``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to
+the active list) and therefore cannot be evicted. ``swappiness``
+overrides the default value in ``/proc/sys/vm/swappiness``.
+``nr_to_reclaim`` limits the number of pages to evict.
+
+A typical use case is that a job scheduler runs this command before it
+tries to land a new job on a server. If it fails to materialize enough
+cold pages because of the overestimation, it retries on the next
+server according to the ranking result obtained from the working set
+estimation step. This less forceful approach limits the impacts on the
+existing jobs.
/sys/kernel/mm/transparent_hugepage/khugepaged/alloc_sleep_millisecs
-The khugepaged progress can be seen in the number of pages collapsed::
+The khugepaged progress can be seen in the number of pages collapsed (note
+that this counter may not be an exact count of the number of pages
+collapsed, since "collapsed" could mean multiple things: (1) A PTE mapping
+being replaced by a PMD mapping, or (2) All 4K physical pages replaced by
+one 2M hugepage. Each may happen independently, or together, depending on
+the type of memory and the failures that occur. As such, this value should
+be interpreted roughly as a sign of progress, and counters in /proc/vmstat
+consulted for more accurate accounting)::
/sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed
page table entry.
thp_zero_page_alloc
- is incremented every time a huge zero page is
- successfully allocated. It includes allocations which where
- dropped due race with other allocation. Note, it doesn't count
- every map of the huge zero page, only its allocation.
+ is incremented every time a huge zero page used for thp is
+ successfully allocated. Note, it doesn't count every map of
+ the huge zero page, only its allocation.
thp_zero_page_alloc_failed
is incremented if kernel fails to allocate
Design
======
-Userfaults are delivered and resolved through the ``userfaultfd`` syscall.
+Userspace creates a new userfaultfd, initializes it, and registers one or more
+regions of virtual memory with it. Then, any page faults which occur within the
+region(s) result in a message being delivered to the userfaultfd, notifying
+userspace of the fault.
The ``userfaultfd`` (aside from registering and unregistering virtual
memory ranges) provides two primary functionalities:
management of mremap/mprotect is that the userfaults in all their
operations never involve heavyweight structures like vmas (in fact the
``userfaultfd`` runtime load never takes the mmap_lock for writing).
-
Vmas are not suitable for page- (or hugepage) granular fault tracking
when dealing with virtual address spaces that could span
Terabytes. Too many vmas would be needed for that.
-The ``userfaultfd`` once opened by invoking the syscall, can also be
+The ``userfaultfd``, once created, can also be
passed using unix domain sockets to a manager process, so the same
manager process could handle the userfaults of a multitude of
different processes without them being aware about what is going on
API
===
+Creating a userfaultfd
+----------------------
+
+There are two ways to create a new userfaultfd, each of which provide ways to
+restrict access to this functionality (since historically userfaultfds which
+handle kernel page faults have been a useful tool for exploiting the kernel).
+
+The first way, supported since userfaultfd was introduced, is the
+userfaultfd(2) syscall. Access to this is controlled in several ways:
+
+- Any user can always create a userfaultfd which traps userspace page faults
+ only. Such a userfaultfd can be created using the userfaultfd(2) syscall
+ with the flag UFFD_USER_MODE_ONLY.
+
+- In order to also trap kernel page faults for the address space, either the
+ process needs the CAP_SYS_PTRACE capability, or the system must have
+ vm.unprivileged_userfaultfd set to 1. By default, vm.unprivileged_userfaultfd
+ is set to 0.
+
+The second way, added to the kernel more recently, is by opening
+/dev/userfaultfd and issuing a USERFAULTFD_IOC_NEW ioctl to it. This method
+yields equivalent userfaultfds to the userfaultfd(2) syscall.
+
+Unlike userfaultfd(2), access to /dev/userfaultfd is controlled via normal
+filesystem permissions (user/group/mode), which gives fine grained access to
+userfaultfd specifically, without also granting other unrelated privileges at
+the same time (as e.g. granting CAP_SYS_PTRACE would do). Users who have access
+to /dev/userfaultfd can always create userfaultfds that trap kernel page faults;
+vm.unprivileged_userfaultfd is not considered.
+
+Initializing a userfaultfd
+--------------------------
+
When first opened the ``userfaultfd`` must be enabled invoking the
``UFFDIO_API`` ioctl specifying a ``uffdio_api.api`` value set to ``UFFD_API`` (or
a later API version) which will specify the ``read/POLLIN`` protocol
4 s3_beep
= =======
+arch
+====
+
+The machine hardware name, the same output as ``uname -m``
+(e.g. ``x86_64`` or ``aarch64``).
auto_msgmni
===========
place the hot pages in the fast memory. This is implemented based on
unmapping and page fault too.
+numa_balancing_promote_rate_limit_MBps
+======================================
+
+Too high promotion/demotion throughput between different memory types
+may hurt application latency. This can be used to rate limit the
+promotion throughput. The per-node max promotion throughput in MB/s
+will be limited to be no more than the set value.
+
+A rule of thumb is to set this to less than 1/10 of the PMEM node
+write bandwidth.
+
oops_all_cpu_backtrace
======================
The default value is 0.
+Another way to control permissions for userfaultfd is to use
+/dev/userfaultfd instead of userfaultfd(2). See
+Documentation/admin-guide/mm/userfaultfd.rst.
user_reserve_kbytes
===================
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A55 | #1530923 | ARM64_ERRATUM_1530923 |
+----------------+-----------------+-----------------+-----------------------------+
+| ARM | Cortex-A55 | #2441007 | ARM64_ERRATUM_2441007 |
++----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A57 | #832075 | ARM64_ERRATUM_832075 |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | Cortex-A57 | #852523 | N/A |
kref
assoc_array
xarray
+ maple_tree
idr
circular-buffers
rbtree
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0+
+
+
+==========
+Maple Tree
+==========
+
+:Author: Liam R. Howlett
+
+Overview
+========
+
+The Maple Tree is a B-Tree data type which is optimized for storing
+non-overlapping ranges, including ranges of size 1. The tree was designed to
+be simple to use and does not require a user written search method. It
+supports iterating over a range of entries and going to the previous or next
+entry in a cache-efficient manner. The tree can also be put into an RCU-safe
+mode of operation which allows reading and writing concurrently. Writers must
+synchronize on a lock, which can be the default spinlock, or the user can set
+the lock to an external lock of a different type.
+
+The Maple Tree maintains a small memory footprint and was designed to use
+modern processor cache efficiently. The majority of the users will be able to
+use the normal API. An :ref:`maple-tree-advanced-api` exists for more complex
+scenarios. The most important usage of the Maple Tree is the tracking of the
+virtual memory areas.
+
+The Maple Tree can store values between ``0`` and ``ULONG_MAX``. The Maple
+Tree reserves values with the bottom two bits set to '10' which are below 4096
+(ie 2, 6, 10 .. 4094) for internal use. If the entries may use reserved
+entries then the users can convert the entries using xa_mk_value() and convert
+them back by calling xa_to_value(). If the user needs to use a reserved
+value, then the user can convert the value when using the
+:ref:`maple-tree-advanced-api`, but are blocked by the normal API.
+
+The Maple Tree can also be configured to support searching for a gap of a given
+size (or larger).
+
+Pre-allocating of nodes is also supported using the
+:ref:`maple-tree-advanced-api`. This is useful for users who must guarantee a
+successful store operation within a given
+code segment when allocating cannot be done. Allocations of nodes are
+relatively small at around 256 bytes.
+
+.. _maple-tree-normal-api:
+
+Normal API
+==========
+
+Start by initialising a maple tree, either with DEFINE_MTREE() for statically
+allocated maple trees or mt_init() for dynamically allocated ones. A
+freshly-initialised maple tree contains a ``NULL`` pointer for the range ``0``
+- ``ULONG_MAX``. There are currently two types of maple trees supported: the
+allocation tree and the regular tree. The regular tree has a higher branching
+factor for internal nodes. The allocation tree has a lower branching factor
+but allows the user to search for a gap of a given size or larger from either
+``0`` upwards or ``ULONG_MAX`` down. An allocation tree can be used by
+passing in the ``MT_FLAGS_ALLOC_RANGE`` flag when initialising the tree.
+
+You can then set entries using mtree_store() or mtree_store_range().
+mtree_store() will overwrite any entry with the new entry and return 0 on
+success or an error code otherwise. mtree_store_range() works in the same way
+but takes a range. mtree_load() is used to retrieve the entry stored at a
+given index. You can use mtree_erase() to erase an entire range by only
+knowing one value within that range, or mtree_store() call with an entry of
+NULL may be used to partially erase a range or many ranges at once.
+
+If you want to only store a new entry to a range (or index) if that range is
+currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which
+return -EEXIST if the range is not empty.
+
+You can search for an entry from an index upwards by using mt_find().
+
+You can walk each entry within a range by calling mt_for_each(). You must
+provide a temporary variable to store a cursor. If you want to walk each
+element of the tree then ``0`` and ``ULONG_MAX`` may be used as the range. If
+the caller is going to hold the lock for the duration of the walk then it is
+worth looking at the mas_for_each() API in the :ref:`maple-tree-advanced-api`
+section.
+
+Sometimes it is necessary to ensure the next call to store to a maple tree does
+not allocate memory, please see :ref:`maple-tree-advanced-api` for this use case.
+
+Finally, you can remove all entries from a maple tree by calling
+mtree_destroy(). If the maple tree entries are pointers, you may wish to free
+the entries first.
+
+Allocating Nodes
+----------------
+
+The allocations are handled by the internal tree code. See
+:ref:`maple-tree-advanced-alloc` for other options.
+
+Locking
+-------
+
+You do not have to worry about locking. See :ref:`maple-tree-advanced-locks`
+for other options.
+
+The Maple Tree uses RCU and an internal spinlock to synchronise access:
+
+Takes RCU read lock:
+ * mtree_load()
+ * mt_find()
+ * mt_for_each()
+ * mt_next()
+ * mt_prev()
+
+Takes ma_lock internally:
+ * mtree_store()
+ * mtree_store_range()
+ * mtree_insert()
+ * mtree_insert_range()
+ * mtree_erase()
+ * mtree_destroy()
+ * mt_set_in_rcu()
+ * mt_clear_in_rcu()
+
+If you want to take advantage of the internal lock to protect the data
+structures that you are storing in the Maple Tree, you can call mtree_lock()
+before calling mtree_load(), then take a reference count on the object you
+have found before calling mtree_unlock(). This will prevent stores from
+removing the object from the tree between looking up the object and
+incrementing the refcount. You can also use RCU to avoid dereferencing
+freed memory, but an explanation of that is beyond the scope of this
+document.
+
+.. _maple-tree-advanced-api:
+
+Advanced API
+============
+
+The advanced API offers more flexibility and better performance at the
+cost of an interface which can be harder to use and has fewer safeguards.
+You must take care of your own locking while using the advanced API.
+You can use the ma_lock, RCU or an external lock for protection.
+You can mix advanced and normal operations on the same array, as long
+as the locking is compatible. The :ref:`maple-tree-normal-api` is implemented
+in terms of the advanced API.
+
+The advanced API is based around the ma_state, this is where the 'mas'
+prefix originates. The ma_state struct keeps track of tree operations to make
+life easier for both internal and external tree users.
+
+Initialising the maple tree is the same as in the :ref:`maple-tree-normal-api`.
+Please see above.
+
+The maple state keeps track of the range start and end in mas->index and
+mas->last, respectively.
+
+mas_walk() will walk the tree to the location of mas->index and set the
+mas->index and mas->last according to the range for the entry.
+
+You can set entries using mas_store(). mas_store() will overwrite any entry
+with the new entry and return the first existing entry that is overwritten.
+The range is passed in as members of the maple state: index and last.
+
+You can use mas_erase() to erase an entire range by setting index and
+last of the maple state to the desired range to erase. This will erase
+the first range that is found in that range, set the maple state index
+and last as the range that was erased and return the entry that existed
+at that location.
+
+You can walk each entry within a range by using mas_for_each(). If you want
+to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as
+the range. If the lock needs to be periodically dropped, see the locking
+section mas_pause().
+
+Using a maple state allows mas_next() and mas_prev() to function as if the
+tree was a linked list. With such a high branching factor the amortized
+performance penalty is outweighed by cache optimization. mas_next() will
+return the next entry which occurs after the entry at index. mas_prev()
+will return the previous entry which occurs before the entry at index.
+
+mas_find() will find the first entry which exists at or above index on
+the first call, and the next entry from every subsequent calls.
+
+mas_find_rev() will find the fist entry which exists at or below the last on
+the first call, and the previous entry from every subsequent calls.
+
+If the user needs to yield the lock during an operation, then the maple state
+must be paused using mas_pause().
+
+There are a few extra interfaces provided when using an allocation tree.
+If you wish to search for a gap within a range, then mas_empty_area()
+or mas_empty_area_rev() can be used. mas_empty_area() searches for a gap
+starting at the lowest index given up to the maximum of the range.
+mas_empty_area_rev() searches for a gap starting at the highest index given
+and continues downward to the lower bound of the range.
+
+.. _maple-tree-advanced-alloc:
+
+Advanced Allocating Nodes
+-------------------------
+
+Allocations are usually handled internally to the tree, however if allocations
+need to occur before a write occurs then calling mas_expected_entries() will
+allocate the worst-case number of needed nodes to insert the provided number of
+ranges. This also causes the tree to enter mass insertion mode. Once
+insertions are complete calling mas_destroy() on the maple state will free the
+unused allocations.
+
+.. _maple-tree-advanced-locks:
+
+Advanced Locking
+----------------
+
+The maple tree uses a spinlock by default, but external locks can be used for
+tree updates as well. To use an external lock, the tree must be initialized
+with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the
+MTREE_INIT_EXT() #define, which takes an external lock as an argument.
+
+Functions and structures
+========================
+
+.. kernel-doc:: include/linux/maple_tree.h
+.. kernel-doc:: lib/maple_tree.c
Memory Allocation Controls
==========================
-.. kernel-doc:: include/linux/gfp.h
- :internal:
-
.. kernel-doc:: include/linux/gfp_types.h
:doc: Page mobility and placement hints
See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes
+ **BAD_FIXES_TAG**
+ The Fixes: tag is malformed or does not follow the community conventions.
+ This can occur if the tag have been split into multiple lines (e.g., when
+ pasted in an email program with word wrapping enabled).
+
+ See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes
+
Comparison style
----------------
kcov
gcov
kasan
+ kmsan
ubsan
kmemleak
kcsan
report or also panic the kernel (default: ``report``). The panic happens even
if ``kasan_multi_shot`` is enabled.
-Hardware Tag-Based KASAN mode (see the section about various modes below) is
-intended for use in production as a security mitigation. Therefore, it supports
-additional boot parameters that allow disabling KASAN or controlling features:
+Software and Hardware Tag-Based KASAN modes (see the section about various
+modes below) support altering stack trace collection behavior:
+
+- ``kasan.stacktrace=off`` or ``=on`` disables or enables alloc and free stack
+ traces collection (default: ``on``).
+- ``kasan.stack_ring_size=<number of entries>`` specifies the number of entries
+ in the stack ring (default: ``32768``).
+
+Hardware Tag-Based KASAN mode is intended for use in production as a security
+mitigation. Therefore, it supports additional boot parameters that allow
+disabling KASAN altogether or controlling its features:
- ``kasan=off`` or ``=on`` controls whether KASAN is enabled (default: ``on``).
- ``kasan.vmalloc=off`` or ``=on`` disables or enables tagging of vmalloc
allocations (default: ``on``).
-- ``kasan.stacktrace=off`` or ``=on`` disables or enables alloc and free stack
- traces collection (default: ``on``).
-
Error reports
~~~~~~~~~~~~~
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+.. Copyright (C) 2022, Google LLC.
+
+===================================
+The Kernel Memory Sanitizer (KMSAN)
+===================================
+
+KMSAN is a dynamic error detector aimed at finding uses of uninitialized
+values. It is based on compiler instrumentation, and is quite similar to the
+userspace `MemorySanitizer tool`_.
+
+An important note is that KMSAN is not intended for production use, because it
+drastically increases kernel memory footprint and slows the whole system down.
+
+Usage
+=====
+
+Building the kernel
+-------------------
+
+In order to build a kernel with KMSAN you will need a fresh Clang (14.0.6+).
+Please refer to `LLVM documentation`_ for the instructions on how to build Clang.
+
+Now configure and build the kernel with CONFIG_KMSAN enabled.
+
+Example report
+--------------
+
+Here is an example of a KMSAN report::
+
+ =====================================================
+ BUG: KMSAN: uninit-value in test_uninit_kmsan_check_memory+0x1be/0x380 [kmsan_test]
+ test_uninit_kmsan_check_memory+0x1be/0x380 mm/kmsan/kmsan_test.c:273
+ kunit_run_case_internal lib/kunit/test.c:333
+ kunit_try_run_case+0x206/0x420 lib/kunit/test.c:374
+ kunit_generic_run_threadfn_adapter+0x6d/0xc0 lib/kunit/try-catch.c:28
+ kthread+0x721/0x850 kernel/kthread.c:327
+ ret_from_fork+0x1f/0x30 ??:?
+
+ Uninit was stored to memory at:
+ do_uninit_local_array+0xfa/0x110 mm/kmsan/kmsan_test.c:260
+ test_uninit_kmsan_check_memory+0x1a2/0x380 mm/kmsan/kmsan_test.c:271
+ kunit_run_case_internal lib/kunit/test.c:333
+ kunit_try_run_case+0x206/0x420 lib/kunit/test.c:374
+ kunit_generic_run_threadfn_adapter+0x6d/0xc0 lib/kunit/try-catch.c:28
+ kthread+0x721/0x850 kernel/kthread.c:327
+ ret_from_fork+0x1f/0x30 ??:?
+
+ Local variable uninit created at:
+ do_uninit_local_array+0x4a/0x110 mm/kmsan/kmsan_test.c:256
+ test_uninit_kmsan_check_memory+0x1a2/0x380 mm/kmsan/kmsan_test.c:271
+
+ Bytes 4-7 of 8 are uninitialized
+ Memory access of size 8 starts at ffff888083fe3da0
+
+ CPU: 0 PID: 6731 Comm: kunit_try_catch Tainted: G B E 5.16.0-rc3+ #104
+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014
+ =====================================================
+
+The report says that the local variable ``uninit`` was created uninitialized in
+``do_uninit_local_array()``. The third stack trace corresponds to the place
+where this variable was created.
+
+The first stack trace shows where the uninit value was used (in
+``test_uninit_kmsan_check_memory()``). The tool shows the bytes which were left
+uninitialized in the local variable, as well as the stack where the value was
+copied to another memory location before use.
+
+A use of uninitialized value ``v`` is reported by KMSAN in the following cases:
+ - in a condition, e.g. ``if (v) { ... }``;
+ - in an indexing or pointer dereferencing, e.g. ``array[v]`` or ``*v``;
+ - when it is copied to userspace or hardware, e.g. ``copy_to_user(..., &v, ...)``;
+ - when it is passed as an argument to a function, and
+ ``CONFIG_KMSAN_CHECK_PARAM_RETVAL`` is enabled (see below).
+
+The mentioned cases (apart from copying data to userspace or hardware, which is
+a security issue) are considered undefined behavior from the C11 Standard point
+of view.
+
+Disabling the instrumentation
+-----------------------------
+
+A function can be marked with ``__no_kmsan_checks``. Doing so makes KMSAN
+ignore uninitialized values in that function and mark its output as initialized.
+As a result, the user will not get KMSAN reports related to that function.
+
+Another function attribute supported by KMSAN is ``__no_sanitize_memory``.
+Applying this attribute to a function will result in KMSAN not instrumenting
+it, which can be helpful if we do not want the compiler to interfere with some
+low-level code (e.g. that marked with ``noinstr`` which implicitly adds
+``__no_sanitize_memory``).
+
+This however comes at a cost: stack allocations from such functions will have
+incorrect shadow/origin values, likely leading to false positives. Functions
+called from non-instrumented code may also receive incorrect metadata for their
+parameters.
+
+As a rule of thumb, avoid using ``__no_sanitize_memory`` explicitly.
+
+It is also possible to disable KMSAN for a single file (e.g. main.o)::
+
+ KMSAN_SANITIZE_main.o := n
+
+or for the whole directory::
+
+ KMSAN_SANITIZE := n
+
+in the Makefile. Think of this as applying ``__no_sanitize_memory`` to every
+function in the file or directory. Most users won't need KMSAN_SANITIZE, unless
+their code gets broken by KMSAN (e.g. runs at early boot time).
+
+Support
+=======
+
+In order for KMSAN to work the kernel must be built with Clang, which so far is
+the only compiler that has KMSAN support. The kernel instrumentation pass is
+based on the userspace `MemorySanitizer tool`_.
+
+The runtime library only supports x86_64 at the moment.
+
+How KMSAN works
+===============
+
+KMSAN shadow memory
+-------------------
+
+KMSAN associates a metadata byte (also called shadow byte) with every byte of
+kernel memory. A bit in the shadow byte is set iff the corresponding bit of the
+kernel memory byte is uninitialized. Marking the memory uninitialized (i.e.
+setting its shadow bytes to ``0xff``) is called poisoning, marking it
+initialized (setting the shadow bytes to ``0x00``) is called unpoisoning.
+
+When a new variable is allocated on the stack, it is poisoned by default by
+instrumentation code inserted by the compiler (unless it is a stack variable
+that is immediately initialized). Any new heap allocation done without
+``__GFP_ZERO`` is also poisoned.
+
+Compiler instrumentation also tracks the shadow values as they are used along
+the code. When needed, instrumentation code invokes the runtime library in
+``mm/kmsan/`` to persist shadow values.
+
+The shadow value of a basic or compound type is an array of bytes of the same
+length. When a constant value is written into memory, that memory is unpoisoned.
+When a value is read from memory, its shadow memory is also obtained and
+propagated into all the operations which use that value. For every instruction
+that takes one or more values the compiler generates code that calculates the
+shadow of the result depending on those values and their shadows.
+
+Example::
+
+ int a = 0xff; // i.e. 0x000000ff
+ int b;
+ int c = a | b;
+
+In this case the shadow of ``a`` is ``0``, shadow of ``b`` is ``0xffffffff``,
+shadow of ``c`` is ``0xffffff00``. This means that the upper three bytes of
+``c`` are uninitialized, while the lower byte is initialized.
+
+Origin tracking
+---------------
+
+Every four bytes of kernel memory also have a so-called origin mapped to them.
+This origin describes the point in program execution at which the uninitialized
+value was created. Every origin is associated with either the full allocation
+stack (for heap-allocated memory), or the function containing the uninitialized
+variable (for locals).
+
+When an uninitialized variable is allocated on stack or heap, a new origin
+value is created, and that variable's origin is filled with that value. When a
+value is read from memory, its origin is also read and kept together with the
+shadow. For every instruction that takes one or more values, the origin of the
+result is one of the origins corresponding to any of the uninitialized inputs.
+If a poisoned value is written into memory, its origin is written to the
+corresponding storage as well.
+
+Example 1::
+
+ int a = 42;
+ int b;
+ int c = a + b;
+
+In this case the origin of ``b`` is generated upon function entry, and is
+stored to the origin of ``c`` right before the addition result is written into
+memory.
+
+Several variables may share the same origin address, if they are stored in the
+same four-byte chunk. In this case every write to either variable updates the
+origin for all of them. We have to sacrifice precision in this case, because
+storing origins for individual bits (and even bytes) would be too costly.
+
+Example 2::
+
+ int combine(short a, short b) {
+ union ret_t {
+ int i;
+ short s[2];
+ } ret;
+ ret.s[0] = a;
+ ret.s[1] = b;
+ return ret.i;
+ }
+
+If ``a`` is initialized and ``b`` is not, the shadow of the result would be
+0xffff0000, and the origin of the result would be the origin of ``b``.
+``ret.s[0]`` would have the same origin, but it will never be used, because
+that variable is initialized.
+
+If both function arguments are uninitialized, only the origin of the second
+argument is preserved.
+
+Origin chaining
+~~~~~~~~~~~~~~~
+
+To ease debugging, KMSAN creates a new origin for every store of an
+uninitialized value to memory. The new origin references both its creation stack
+and the previous origin the value had. This may cause increased memory
+consumption, so we limit the length of origin chains in the runtime.
+
+Clang instrumentation API
+-------------------------
+
+Clang instrumentation pass inserts calls to functions defined in
+``mm/kmsan/nstrumentation.c`` into the kernel code.
+
+Shadow manipulation
+~~~~~~~~~~~~~~~~~~~
+
+For every memory access the compiler emits a call to a function that returns a
+pair of pointers to the shadow and origin addresses of the given memory::
+
+ typedef struct {
+ void *shadow, *origin;
+ } shadow_origin_ptr_t
+
+ shadow_origin_ptr_t __msan_metadata_ptr_for_load_{1,2,4,8}(void *addr)
+ shadow_origin_ptr_t __msan_metadata_ptr_for_store_{1,2,4,8}(void *addr)
+ shadow_origin_ptr_t __msan_metadata_ptr_for_load_n(void *addr, uintptr_t size)
+ shadow_origin_ptr_t __msan_metadata_ptr_for_store_n(void *addr, uintptr_t size)
+
+The function name depends on the memory access size.
+
+The compiler makes sure that for every loaded value its shadow and origin
+values are read from memory. When a value is stored to memory, its shadow and
+origin are also stored using the metadata pointers.
+
+Handling locals
+~~~~~~~~~~~~~~~
+
+A special function is used to create a new origin value for a local variable and
+set the origin of that variable to that value::
+
+ void __msan_poison_alloca(void *addr, uintptr_t size, char *descr)
+
+Access to per-task data
+~~~~~~~~~~~~~~~~~~~~~~~
+
+At the beginning of every instrumented function KMSAN inserts a call to
+``__msan_get_context_state()``::
+
+ kmsan_context_state *__msan_get_context_state(void)
+
+``kmsan_context_state`` is declared in ``include/linux/kmsan.h``::
+
+ struct kmsan_context_state {
+ char param_tls[KMSAN_PARAM_SIZE];
+ char retval_tls[KMSAN_RETVAL_SIZE];
+ char va_arg_tls[KMSAN_PARAM_SIZE];
+ char va_arg_origin_tls[KMSAN_PARAM_SIZE];
+ u64 va_arg_overflow_size_tls;
+ char param_origin_tls[KMSAN_PARAM_SIZE];
+ depot_stack_handle_t retval_origin_tls;
+ };
+
+This structure is used by KMSAN to pass parameter shadows and origins between
+instrumented functions (unless the parameters are checked immediately by
+``CONFIG_KMSAN_CHECK_PARAM_RETVAL``).
+
+Passing uninitialized values to functions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Clang's MemorySanitizer instrumentation has an option,
+``-fsanitize-memory-param-retval``, which makes the compiler check function
+parameters passed by value, as well as function return values.
+
+The option is controlled by ``CONFIG_KMSAN_CHECK_PARAM_RETVAL``, which is
+enabled by default to let KMSAN report uninitialized values earlier.
+Please refer to the `LKML discussion`_ for more details.
+
+Because of the way the checks are implemented in LLVM (they are only applied to
+parameters marked as ``noundef``), not all parameters are guaranteed to be
+checked, so we cannot give up the metadata storage in ``kmsan_context_state``.
+
+String functions
+~~~~~~~~~~~~~~~~
+
+The compiler replaces calls to ``memcpy()``/``memmove()``/``memset()`` with the
+following functions. These functions are also called when data structures are
+initialized or copied, making sure shadow and origin values are copied alongside
+with the data::
+
+ void *__msan_memcpy(void *dst, void *src, uintptr_t n)
+ void *__msan_memmove(void *dst, void *src, uintptr_t n)
+ void *__msan_memset(void *dst, int c, uintptr_t n)
+
+Error reporting
+~~~~~~~~~~~~~~~
+
+For each use of a value the compiler emits a shadow check that calls
+``__msan_warning()`` in the case that value is poisoned::
+
+ void __msan_warning(u32 origin)
+
+``__msan_warning()`` causes KMSAN runtime to print an error report.
+
+Inline assembly instrumentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+KMSAN instruments every inline assembly output with a call to::
+
+ void __msan_instrument_asm_store(void *addr, uintptr_t size)
+
+, which unpoisons the memory region.
+
+This approach may mask certain errors, but it also helps to avoid a lot of
+false positives in bitwise operations, atomics etc.
+
+Sometimes the pointers passed into inline assembly do not point to valid memory.
+In such cases they are ignored at runtime.
+
+
+Runtime library
+---------------
+
+The code is located in ``mm/kmsan/``.
+
+Per-task KMSAN state
+~~~~~~~~~~~~~~~~~~~~
+
+Every task_struct has an associated KMSAN task state that holds the KMSAN
+context (see above) and a per-task flag disallowing KMSAN reports::
+
+ struct kmsan_context {
+ ...
+ bool allow_reporting;
+ struct kmsan_context_state cstate;
+ ...
+ }
+
+ struct task_struct {
+ ...
+ struct kmsan_context kmsan;
+ ...
+ }
+
+KMSAN contexts
+~~~~~~~~~~~~~~
+
+When running in a kernel task context, KMSAN uses ``current->kmsan.cstate`` to
+hold the metadata for function parameters and return values.
+
+But in the case the kernel is running in the interrupt, softirq or NMI context,
+where ``current`` is unavailable, KMSAN switches to per-cpu interrupt state::
+
+ DEFINE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx);
+
+Metadata allocation
+~~~~~~~~~~~~~~~~~~~
+
+There are several places in the kernel for which the metadata is stored.
+
+1. Each ``struct page`` instance contains two pointers to its shadow and
+origin pages::
+
+ struct page {
+ ...
+ struct page *shadow, *origin;
+ ...
+ };
+
+At boot-time, the kernel allocates shadow and origin pages for every available
+kernel page. This is done quite late, when the kernel address space is already
+fragmented, so normal data pages may arbitrarily interleave with the metadata
+pages.
+
+This means that in general for two contiguous memory pages their shadow/origin
+pages may not be contiguous. Consequently, if a memory access crosses the
+boundary of a memory block, accesses to shadow/origin memory may potentially
+corrupt other pages or read incorrect values from them.
+
+In practice, contiguous memory pages returned by the same ``alloc_pages()``
+call will have contiguous metadata, whereas if these pages belong to two
+different allocations their metadata pages can be fragmented.
+
+For the kernel data (``.data``, ``.bss`` etc.) and percpu memory regions
+there also are no guarantees on metadata contiguity.
+
+In the case ``__msan_metadata_ptr_for_XXX_YYY()`` hits the border between two
+pages with non-contiguous metadata, it returns pointers to fake shadow/origin regions::
+
+ char dummy_load_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+ char dummy_store_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+
+``dummy_load_page`` is zero-initialized, so reads from it always yield zeroes.
+All stores to ``dummy_store_page`` are ignored.
+
+2. For vmalloc memory and modules, there is a direct mapping between the memory
+range, its shadow and origin. KMSAN reduces the vmalloc area by 3/4, making only
+the first quarter available to ``vmalloc()``. The second quarter of the vmalloc
+area contains shadow memory for the first quarter, the third one holds the
+origins. A small part of the fourth quarter contains shadow and origins for the
+kernel modules. Please refer to ``arch/x86/include/asm/pgtable_64_types.h`` for
+more details.
+
+When an array of pages is mapped into a contiguous virtual memory space, their
+shadow and origin pages are similarly mapped into contiguous regions.
+
+References
+==========
+
+E. Stepanov, K. Serebryany. `MemorySanitizer: fast detector of uninitialized
+memory use in C++
+<https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43308.pdf>`_.
+In Proceedings of CGO 2015.
+
+.. _MemorySanitizer tool: https://clang.llvm.org/docs/MemorySanitizer.html
+.. _LLVM documentation: https://llvm.org/docs/GettingStarted.html
+.. _LKML discussion: https://lore.kernel.org/all/20220614144853.3693273-1-glider@google.com/
compiling a kernel (using ``build`` or ``run`` commands). For example:
to enable compiler warnings, we can pass ``--make_options W=1``.
-- ``--alltests``: Builds a UML kernel with all config options enabled
- using ``make allyesconfig``. This allows us to run as many tests as
- possible.
-
- .. note:: It is slow and prone to breakage as new options are
- added or modified. Instead, enable all tests
- which have satisfied dependencies by adding
- ``CONFIG_KUNIT_ALL_TESTS=y`` to your ``.kunitconfig``.
+- ``--alltests``: Enable a predefined set of options in order to build
+ as many tests as possible.
+
+ .. note:: The list of enabled options can be found in
+ ``tools/testing/kunit/configs/all_tests.config``.
+
+ If you only want to enable all tests with otherwise satisfied
+ dependencies, instead add ``CONFIG_KUNIT_ALL_TESTS=y`` to your
+ ``.kunitconfig``.
- ``--kunitconfig``: Specifies the path or the directory of the ``.kunitconfig``
file. For example:
properties:
compatible:
enum:
+ - qcom,msm8226-cci
- qcom,msm8916-cci
- qcom,msm8974-cci
- qcom,msm8996-cci
const: 0
clocks:
- minItems: 4
+ minItems: 3
maxItems: 6
clock-names:
- minItems: 4
+ minItems: 3
maxItems: 6
interrupts:
compatible:
contains:
enum:
+ - qcom,msm8226-cci
- qcom,msm8916-cci
then:
properties:
i2c-bus@1: false
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,msm8226-cci
+ - qcom,msm8974-cci
+ then:
+ properties:
+ clocks:
+ maxItems: 3
+ clock-names:
+ items:
+ - const: camss_top_ahb
+ - const: cci_ahb
+ - const: cci
+
- if:
properties:
compatible:
Bindings for joystick devices connected to ADC controllers supporting
the Industrial I/O subsystem.
+allOf:
+ - $ref: input.yaml#
+
properties:
compatible:
const: adc-joystick
https://github.com/devicetree-org/dt-schema/blob/master/schemas/iio/iio-consumer.yaml
for details.
+ poll-interval: true
+
'#address-cells':
const: 1
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/adi,adp5588.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices ADP5588 Keypad Controller
+
+maintainers:
+ - Nuno Sá <nuno.sa@analog.com>
+
+description: |
+ Analog Devices Mobile I/O Expander and QWERTY Keypad Controller
+ https://www.analog.com/media/en/technical-documentation/data-sheets/ADP5588.pdf
+
+allOf:
+ - $ref: matrix-keymap.yaml#
+ - $ref: input.yaml#
+
+properties:
+ compatible:
+ enum:
+ - adi,adp5587
+ - adi,adp5588
+
+ reg:
+ maxItems: 1
+
+ vcc-supply:
+ description: Supply Voltage Input
+
+ reset-gpios:
+ description:
+ If specified, it will be asserted during driver probe. As the line is
+ active low, it should be marked GPIO_ACTIVE_LOW.
+ maxItems: 1
+
+ interrupts:
+ maxItems: 1
+
+ gpio-controller:
+ description:
+ This property applies if either keypad,num-rows lower than 8 or
+ keypad,num-columns lower than 10.
+
+ '#gpio-cells':
+ const: 2
+
+ interrupt-controller:
+ description:
+ This property applies if either keypad,num-rows lower than 8 or
+ keypad,num-columns lower than 10.
+
+ '#interrupt-cells':
+ const: 2
+
+ adi,unlock-keys:
+ description:
+ Specifies a maximum of 2 keys that can be used to unlock the keypad.
+ If this property is set, the keyboard will be locked and only unlocked
+ after these keys are pressed. If only one key is set, a double click is
+ needed to unlock the keypad. The value of this property cannot be bigger
+ or equal than keypad,num-rows * keypad,num-columns.
+ $ref: /schemas/types.yaml#/definitions/uint32-array
+ minItems: 1
+ maxItems: 2
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - keypad,num-rows
+ - keypad,num-columns
+ - linux,keymap
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+ #include <dt-bindings/input/input.h>
+ #include <dt-bindings/gpio/gpio.h>
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ keys@34 {
+ compatible = "adi,adp5588";
+ reg = <0x34>;
+
+ vcc-supply = <&vcc>;
+ interrupts = <21 IRQ_TYPE_EDGE_FALLING>;
+ interrupt-parent = <&gpio>;
+ reset-gpios = <&gpio 20 GPIO_ACTIVE_LOW>;
+
+ keypad,num-rows = <1>;
+ keypad,num-columns = <9>;
+ linux,keymap = <
+ MATRIX_KEY(0x00, 0x00, KEY_1)
+ MATRIX_KEY(0x00, 0x01, KEY_2)
+ MATRIX_KEY(0x00, 0x02, KEY_3)
+ MATRIX_KEY(0x00, 0x03, KEY_4)
+ MATRIX_KEY(0x00, 0x04, KEY_5)
+ MATRIX_KEY(0x00, 0x05, KEY_6)
+ MATRIX_KEY(0x00, 0x06, KEY_7)
+ MATRIX_KEY(0x00, 0x07, KEY_8)
+ MATRIX_KEY(0x00, 0x08, KEY_9)
+ >;
+ };
+ };
+...
+++ /dev/null
-* HID over I2C Device-Tree bindings
-
-HID over I2C provides support for various Human Interface Devices over the
-I2C bus. These devices can be for example touchpads, keyboards, touch screens
-or sensors.
-
-The specification has been written by Microsoft and is currently available here:
-http://msdn.microsoft.com/en-us/library/windows/hardware/hh852380.aspx
-
-If this binding is used, the kernel module i2c-hid will handle the communication
-with the device and the generic hid core layer will handle the protocol.
-
-Required properties:
-- compatible: must be "hid-over-i2c"
-- reg: i2c slave address
-- hid-descr-addr: HID descriptor address
-- interrupts: interrupt line
-
-Additional optional properties:
-
-Some devices may support additional optional properties to help with, e.g.,
-power sequencing. The following properties can be supported by one or more
-device-specific compatible properties, which should be used in addition to the
-"hid-over-i2c" string.
-
-- compatible:
- * "wacom,w9013" (Wacom W9013 digitizer). Supports:
- - vdd-supply (3.3V)
- - vddl-supply (1.8V)
- - post-power-on-delay-ms
-
-- vdd-supply: phandle of the regulator that provides the supply voltage.
-- post-power-on-delay-ms: time required by the device after enabling its regulators
- or powering it on, before it is ready for communication.
-- touchscreen-inverted-x: See touchscreen.txt
-- touchscreen-inverted-y: See touchscreen.txt
-
-Example:
-
- i2c-hid-dev@2c {
- compatible = "hid-over-i2c";
- reg = <0x2c>;
- hid-descr-addr = <0x0020>;
- interrupt-parent = <&gpx3>;
- interrupts = <3 2>;
- };
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/hid-over-i2c.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: HID over I2C Devices
+
+maintainers:
+ - Benjamin Tissoires <benjamin.tissoires@redhat.com>
+ - Jiri Kosina <jkosina@suse.cz>
+
+description: |+
+ HID over I2C provides support for various Human Interface Devices over the
+ I2C bus. These devices can be for example touchpads, keyboards, touch screens
+ or sensors.
+
+ The specification has been written by Microsoft and is currently available here:
+ https://msdn.microsoft.com/en-us/library/windows/hardware/hh852380.aspx
+
+ If this binding is used, the kernel module i2c-hid will handle the communication
+ with the device and the generic hid core layer will handle the protocol.
+
+allOf:
+ - $ref: /schemas/input/touchscreen/touchscreen.yaml#
+
+properties:
+ compatible:
+ oneOf:
+ - items:
+ - enum:
+ - wacom,w9013
+ - const: hid-over-i2c
+ - description: Just "hid-over-i2c" alone is allowed, but not recommended.
+ const: hid-over-i2c
+
+ reg:
+ maxItems: 1
+
+ interrupts:
+ maxItems: 1
+
+ hid-descr-addr:
+ description: HID descriptor address
+ $ref: /schemas/types.yaml#/definitions/uint32
+
+ post-power-on-delay-ms:
+ description: Time required by the device after enabling its regulators
+ or powering it on, before it is ready for communication.
+
+ touchscreen-inverted-x: true
+
+ touchscreen-inverted-y: true
+
+ vdd-supply:
+ description: 3.3V supply
+
+ vddl-supply:
+ description: 1.8V supply
+
+ wakeup-source: true
+
+required:
+ - compatible
+ - reg
+ - interrupts
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ hid@2c {
+ compatible = "hid-over-i2c";
+ reg = <0x2c>;
+ hid-descr-addr = <0x0020>;
+ interrupts = <3 2>;
+ };
+ };
+...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/ibm,op-panel.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: IBM Operation Panel
+
+maintainers:
+ - Eddie James <eajames@linux.ibm.com>
+
+allOf:
+ - $ref: input.yaml#
+
+description: |
+ The IBM Operation Panel provides a simple interface to control the connected
+ server. It has a display and three buttons: two directional arrows and one
+ 'Enter' button.
+
+properties:
+ compatible:
+ const: ibm,op-panel
+
+ reg:
+ maxItems: 1
+
+ linux,keycodes:
+ minItems: 1
+ maxItems: 3
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/i2c/i2c.h>
+ #include <dt-bindings/input/input.h>
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ ibm-op-panel@62 {
+ compatible = "ibm,op-panel";
+ reg = <(0x62 | I2C_OWN_SLAVE_ADDRESS)>;
+ linux,keycodes = <KEY_UP>, <KEY_DOWN>, <KEY_ENTER>;
+ };
+ };
maximum: 256
default: 16
+ mediatek,keys-per-group:
+ description: each (row, column) group has multiple keys
+ $ref: /schemas/types.yaml#/definitions/uint32
+ default: 1
+ maximum: 2
+
required:
- compatible
- reg
- clocks
- clock-names
-additionalProperties: false
+unevaluatedProperties: false
examples:
- |
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/mediatek,pmic-keys.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek PMIC Keys
+
+maintainers:
+ - Chen Zhong <chen.zhong@mediatek.com>
+
+allOf:
+ - $ref: input.yaml#
+
+description: |
+ There are two key functions provided by MT6397, MT6323 and other MediaTek
+ PMICs: pwrkey and homekey.
+ The key functions are defined as the subnode of the function node provided
+ by the PMIC that is defined as a Multi-Function Device (MFD).
+
+ For MediaTek MT6323/MT6397 PMIC bindings see
+ Documentation/devicetree/bindings/mfd/mt6397.txt
+
+properties:
+ compatible:
+ enum:
+ - mediatek,mt6323-keys
+ - mediatek,mt6331-keys
+ - mediatek,mt6358-keys
+ - mediatek,mt6397-keys
+
+ power-off-time-sec: true
+
+ mediatek,long-press-mode:
+ description: |
+ Key long-press force shutdown setting
+ 0 - disabled
+ 1 - pwrkey
+ 2 - pwrkey+homekey
+ $ref: /schemas/types.yaml#/definitions/uint32
+ default: 0
+ maximum: 2
+
+patternProperties:
+ "^((power|home)|(key-[a-z0-9-]+|[a-z0-9-]+-key))$":
+ $ref: input.yaml#
+
+ properties:
+ interrupts:
+ minItems: 1
+ items:
+ - description: Key press interrupt
+ - description: Key release interrupt
+
+ interrupt-names: true
+
+ linux-keycodes:
+ maxItems: 1
+
+ wakeup-source: true
+
+ required:
+ - linux,keycodes
+
+ if:
+ properties:
+ interrupt-names:
+ contains:
+ const: powerkey
+ then:
+ properties:
+ interrupt-names:
+ minItems: 1
+ items:
+ - const: powerkey
+ - const: powerkey_r
+ else:
+ properties:
+ interrupt-names:
+ minItems: 1
+ items:
+ - const: homekey
+ - const: homekey_r
+
+ unevaluatedProperties: false
+
+required:
+ - compatible
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/input/input.h>
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ pmic {
+ compatible = "mediatek,mt6397";
+
+ keys {
+ compatible = "mediatek,mt6397-keys";
+ mediatek,long-press-mode = <1>;
+ power-off-time-sec = <0>;
+
+ key-power {
+ linux,keycodes = <KEY_POWER>;
+ wakeup-source;
+ };
+
+ key-home {
+ linux,keycodes = <KEY_VOLUMEDOWN>;
+ };
+ };
+ };
+++ /dev/null
-MediaTek MT6397/MT6323 PMIC Keys Device Driver
-
-There are two key functions provided by MT6397/MT6323 PMIC, pwrkey
-and homekey. The key functions are defined as the subnode of the function
-node provided by MT6397/MT6323 PMIC that is being defined as one kind
-of Muti-Function Device (MFD)
-
-For MT6397/MT6323 MFD bindings see:
-Documentation/devicetree/bindings/mfd/mt6397.txt
-
-Required properties:
-- compatible: Should be one of:
- - "mediatek,mt6397-keys"
- - "mediatek,mt6323-keys"
- - "mediatek,mt6358-keys"
-- linux,keycodes: See Documentation/devicetree/bindings/input/input.yaml
-
-Optional Properties:
-- wakeup-source: See Documentation/devicetree/bindings/power/wakeup-source.txt
-- mediatek,long-press-mode: Long press key shutdown setting, 1 for
- pwrkey only, 2 for pwrkey/homekey together, others for disabled.
-- power-off-time-sec: See Documentation/devicetree/bindings/input/input.yaml
-
-Example:
-
- pmic: mt6397 {
- compatible = "mediatek,mt6397";
-
- ...
-
- mt6397keys: mt6397keys {
- compatible = "mediatek,mt6397-keys";
- mediatek,long-press-mode = <1>;
- power-off-time-sec = <0>;
-
- power {
- linux,keycodes = <116>;
- wakeup-source;
- };
-
- home {
- linux,keycodes = <114>;
- };
- };
-
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/pine64,pinephone-keyboard.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Pine64 PinePhone keyboard device tree bindings
+
+maintainers:
+ - Samuel Holland <samuel@sholland.org>
+
+description:
+ A keyboard accessory is available for the Pine64 PinePhone and PinePhone Pro.
+ It connects via I2C, providing a raw scan matrix, a flashing interface, and a
+ subordinate I2C bus for communication with a battery charger IC.
+
+properties:
+ compatible:
+ const: pine64,pinephone-keyboard
+
+ reg:
+ const: 0x15
+
+ interrupts:
+ maxItems: 1
+
+ vbat-supply:
+ description: Supply for the keyboard MCU
+
+ wakeup-source: true
+
+ i2c:
+ $ref: /schemas/i2c/i2c-controller.yaml#
+
+required:
+ - compatible
+ - reg
+ - interrupts
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/input/input.h>
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ keyboard@15 {
+ compatible = "pine64,pinephone-keyboard";
+ reg = <0x15>;
+ interrupt-parent = <&r_pio>;
+ interrupts = <0 12 IRQ_TYPE_EDGE_FALLING>; /* PL12 */
+
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ charger@75 {
+ reg = <0x75>;
+ };
+ };
+ };
+ };
+++ /dev/null
-Qualcomm PM8xxx PMIC Vibrator
-
-PROPERTIES
-
-- compatible:
- Usage: required
- Value type: <string>
- Definition: must be one of:
- "qcom,pm8058-vib"
- "qcom,pm8916-vib"
- "qcom,pm8921-vib"
-
-- reg:
- Usage: required
- Value type: <prop-encoded-array>
- Definition: address of vibration control register
-
-EXAMPLE
-
- vibrator@4a {
- compatible = "qcom,pm8058-vib";
- reg = <0x4a>;
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/qcom,pm8xxx-vib.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm PM8xxx PMIC Vibrator
+
+maintainers:
+ - Bjorn Andersson <andersson@kernel.org>
+
+properties:
+ compatible:
+ enum:
+ - qcom,pm8058-vib
+ - qcom,pm8916-vib
+ - qcom,pm8921-vib
+
+ reg:
+ maxItems: 1
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ pmic {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ vibrator@4a {
+ compatible = "qcom,pm8058-vib";
+ reg = <0x4a>;
+ };
+ };
auo_pixcir_ts@5c {
compatible = "auo,auo_pixcir_ts";
reg = <0x5c>;
- interrupts = <2 0>;
+ interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
- gpios = <&gpf 2 0 2>, /* INT */
- <&gpf 5 1 0>; /* RST */
+ gpios = <&gpf 2 0 GPIO_LEVEL_HIGH>, /* INT */
+ <&gpf 5 1 GPIO_LEVEL_LOW>; /* RST */
x-size = <800>;
y-size = <600>;
Required Properties:
- compatible must be toradex,vf50-touchscreen
- io-channels: adc channels being used by the Colibri VF50 module
+ IIO ADC for Y-, X-, Y+, X+ connections
- xp-gpios: FET gate driver for input of X+
- xm-gpios: FET gate driver for input of X-
- yp-gpios: FET gate driver for input of Y+
- ym-gpios: FET gate driver for input of Y-
-- interrupts: pen irq interrupt for touch detection
-- pinctrl-names: "idle", "default", "gpios"
-- pinctrl-0: pinctrl node for pen/touch detection state pinmux
+- interrupts: pen irq interrupt for touch detection, signal from X plate
+- pinctrl-names: "idle", "default"
+- pinctrl-0: pinctrl node for pen/touch detection, pinctrl must provide
+ pull-up resistor on X+, X-.
- pinctrl-1: pinctrl node for X/Y and pressure measurement (ADC) state pinmux
-- pinctrl-2: pinctrl node for gpios functioning as FET gate drivers
- vf50-ts-min-pressure: pressure level at which to stop measuring X/Y values
Example:
ym-gpios = <&gpio0 4 GPIO_ACTIVE_HIGH>;
interrupt-parent = <&gpio0>;
interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
- pinctrl-names = "idle","default","gpios";
- pinctrl-0 = <&pinctrl_touchctrl_idle>;
- pinctrl-1 = <&pinctrl_touchctrl_default>;
- pinctrl-2 = <&pinctrl_touchctrl_gpios>;
+ pinctrl-names = "idle","default";
+ pinctrl-0 = <&pinctrl_touchctrl_idle>, <&pinctrl_touchctrl_gpios>;
+ pinctrl-1 = <&pinctrl_touchctrl_default>, <&pinctrl_touchctrl_gpios>;
vf50-ts-min-pressure = <200>;
};
properties:
compatible:
- enum:
- - elan,ektf3624
- - elan,ekth3500
+ oneOf:
+ - enum:
+ - elan,ektf3624
+ - elan,ekth3500
+ - items:
+ - const: elan,ekth3915
+ - const: elan,ekth3500
reg:
maxItems: 1
1 -> 3.25 MHz
2 || 3 -> 6.5 MHz
-Node name must be stmpe_touchscreen and should be child node of stmpe node to
-which it belongs.
+Node should be child node of stmpe node to which it belongs.
Note that common ADC settings of stmpe_touchscreen (child) will take precedence
over the settings done in MFD.
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/fsl,mu-msi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Freescale/NXP i.MX Messaging Unit (MU) work as msi controller
+
+maintainers:
+ - Frank Li <Frank.Li@nxp.com>
+
+description: |
+ The Messaging Unit module enables two processors within the SoC to
+ communicate and coordinate by passing messages (e.g. data, status
+ and control) through the MU interface. The MU also provides the ability
+ for one processor (A side) to signal the other processor (B side) using
+ interrupts.
+
+ Because the MU manages the messaging between processors, the MU uses
+ different clocks (from each side of the different peripheral buses).
+ Therefore, the MU must synchronize the accesses from one side to the
+ other. The MU accomplishes synchronization using two sets of matching
+ registers (Processor A-side, Processor B-side).
+
+ MU can work as msi interrupt controller to do doorbell
+
+allOf:
+ - $ref: /schemas/interrupt-controller/msi-controller.yaml#
+
+properties:
+ compatible:
+ enum:
+ - fsl,imx6sx-mu-msi
+ - fsl,imx7ulp-mu-msi
+ - fsl,imx8ulp-mu-msi
+ - fsl,imx8ulp-mu-msi-s4
+
+ reg:
+ items:
+ - description: a side register base address
+ - description: b side register base address
+
+ reg-names:
+ items:
+ - const: processor-a-side
+ - const: processor-b-side
+
+ interrupts:
+ description: a side interrupt number.
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+
+ power-domains:
+ items:
+ - description: a side power domain
+ - description: b side power domain
+
+ power-domain-names:
+ items:
+ - const: processor-a-side
+ - const: processor-b-side
+
+ interrupt-controller: true
+
+ msi-controller: true
+
+ "#msi-cells":
+ const: 0
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - interrupt-controller
+ - msi-controller
+ - "#msi-cells"
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ #include <dt-bindings/firmware/imx/rsrc.h>
+
+ msi-controller@5d270000 {
+ compatible = "fsl,imx6sx-mu-msi";
+ msi-controller;
+ #msi-cells = <0>;
+ interrupt-controller;
+ reg = <0x5d270000 0x10000>, /* A side */
+ <0x5d300000 0x10000>; /* B side */
+ reg-names = "processor-a-side", "processor-b-side";
+ interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+ power-domains = <&pd IMX_SC_R_MU_12A>,
+ <&pd IMX_SC_R_MU_12B>;
+ power-domain-names = "processor-a-side", "processor-b-side";
+ };
title: Realtek RTL SoC interrupt controller devicetree bindings
+description:
+ Interrupt controller and router for Realtek MIPS SoCs, allowing each SoC
+ interrupt to be routed to one parent CPU (hardware) interrupt, or left
+ disconnected.
+ All connected input lines from SoC peripherals can be masked individually,
+ and an interrupt status register is present to indicate which interrupts are
+ pending.
+
maintainers:
- Birger Koblitz <mail@birger-koblitz.de>
- Bert Vermeulen <bert@biot.com>
properties:
compatible:
- const: realtek,rtl-intc
+ oneOf:
+ - items:
+ - enum:
+ - realtek,rtl8380-intc
+ - const: realtek,rtl-intc
+ - const: realtek,rtl-intc
+ deprecated: true
"#interrupt-cells":
+ description:
+ SoC interrupt line index.
const: 1
reg:
maxItems: 1
interrupts:
- maxItems: 1
+ minItems: 1
+ maxItems: 15
+ description:
+ List of parent interrupts, in the order that they are connected to this
+ interrupt router's outputs, starting at the first output.
interrupt-controller: true
- "#address-cells":
- const: 0
-
interrupt-map:
+ deprecated: true
description: Describes mapping from SoC interrupts to CPU interrupts
required:
- reg
- "#interrupt-cells"
- interrupt-controller
- - "#address-cells"
- - interrupt-map
+
+allOf:
+ - if:
+ properties:
+ compatible:
+ const: realtek,rtl-intc
+ then:
+ properties:
+ "#address-cells":
+ const: 0
+ required:
+ - "#address-cells"
+ - interrupt-map
+ else:
+ required:
+ - interrupts
additionalProperties: false
examples:
- |
- intc: interrupt-controller@3000 {
- compatible = "realtek,rtl-intc";
+ interrupt-controller@3000 {
+ compatible = "realtek,rtl8380-intc", "realtek,rtl-intc";
#interrupt-cells = <1>;
interrupt-controller;
- reg = <0x3000 0x20>;
- #address-cells = <0>;
- interrupt-map =
- <31 &cpuintc 2>,
- <30 &cpuintc 1>,
- <29 &cpuintc 5>;
+ reg = <0x3000 0x18>;
+
+ interrupt-parent = <&cpuintc>;
+ interrupts = <2>, <3>, <4>, <5>, <6>;
};
- renesas,intc-ex-r8a77990 # R-Car E3
- renesas,intc-ex-r8a77995 # R-Car D3
- renesas,intc-ex-r8a779a0 # R-Car V3U
+ - renesas,intc-ex-r8a779g0 # R-Car V4H
- const: renesas,irqc
'#interrupt-cells':
- enum:
- allwinner,sun20i-d1-plic
- const: thead,c900-plic
+ - items:
+ - const: sifive,plic-1.0.0
+ - const: riscv,plic0
+ deprecated: true
+ description: For the QEMU virt machine only
reg:
maxItems: 1
interrupt-controller: true
+ '#interrupt-cells':
+ const: 0
+
msi-controller: true
ti,interrupt-ranges:
1 = If intr supports edge triggered interrupts.
4 = If intr supports level triggered interrupts.
+ reg:
+ maxItems: 1
+
interrupt-controller: true
'#interrupt-cells':
-* Nuvoton NPCM7xx KCS (Keyboard Controller Style) IPMI interface
+* Nuvoton NPCM KCS (Keyboard Controller Style) IPMI interface
-The Nuvoton SOCs (NPCM7xx) are commonly used as BMCs
+The Nuvoton SOCs (NPCM) are commonly used as BMCs
(Baseboard Management Controllers) and the KCS interface can be
used to perform in-band IPMI communication with their host.
Required properties:
- compatible : should be one of
"nuvoton,npcm750-kcs-bmc"
+ "nuvoton,npcm845-kcs-bmc", "nuvoton,npcm750-kcs-bmc"
- interrupts : interrupt generated by the controller
- kcs_chan : The KCS channel number in the controller
the LED.
$ref: /schemas/types.yaml#/definitions/string
- enum:
- # LED will act as a back-light, controlled by the framebuffer system
- - backlight
- # LED will turn on (but for leds-gpio see "default-state" property in
- # Documentation/devicetree/bindings/leds/leds-gpio.yaml)
- - default-on
- # LED "double" flashes at a load average based rate
- - heartbeat
- # LED indicates disk activity
- - disk-activity
- # LED indicates IDE disk activity (deprecated), in new implementations
- # use "disk-activity"
- - ide-disk
- # LED flashes at a fixed, configurable rate
- - timer
- # LED alters the brightness for the specified duration with one software
- # timer (requires "led-pattern" property)
- - pattern
+ oneOf:
+ - enum:
+ # LED will act as a back-light, controlled by the framebuffer system
+ - backlight
+ # LED will turn on (but for leds-gpio see "default-state" property in
+ # Documentation/devicetree/bindings/leds/leds-gpio.yaml)
+ - default-on
+ # LED "double" flashes at a load average based rate
+ - heartbeat
+ # LED indicates disk activity
+ - disk-activity
+ # LED indicates IDE disk activity (deprecated), in new implementations
+ # use "disk-activity"
+ - ide-disk
+ # LED flashes at a fixed, configurable rate
+ - timer
+ # LED alters the brightness for the specified duration with one software
+ # timer (requires "led-pattern" property)
+ - pattern
+ # LED is triggered by SD/MMC activity
+ - pattern: "^mmc[0-9]+$"
led-pattern:
description: |
This module is part of the MT6370 MFD device.
Add MT6370 LED driver include 4-channel RGB LED support Register/PWM/Breath Mode
-allOf:
- - $ref: leds-class-multicolor.yaml#
-
properties:
compatible:
const: mediatek,mt6370-indicator
patternProperties:
"^multi-led@[0-3]$":
type: object
+ $ref: leds-class-multicolor.yaml#
+ unevaluatedProperties: false
properties:
reg:
charger {
compatible = "mediatek,mt6370-charger";
- interrupts = <48>, <68>, <6>;
- interrupt-names = "attach_i", "uvp_d_evt", "mivr";
+ interrupts = <68>, <48>, <6>;
+ interrupt-names = "uvp_d_evt", "attach_i", "mivr";
io-channels = <&mt6370_adc MT6370_CHAN_IBUS>;
mt6370_otg_vbus: usb-otg-vbus-regulator {
properties:
compatible:
- const: mediatek,mt8192-pcie
+ oneOf:
+ - items:
+ - enum:
+ - mediatek,mt8188-pcie
+ - mediatek,mt8195-pcie
+ - const: mediatek,mt8192-pcie
+ - const: mediatek,mt8192-pcie
reg:
maxItems: 1
- const: tl_96m
- const: tl_32k
- const: peri_26m
- - const: top_133m
+ - enum:
+ - top_133m # for MT8192
+ - peri_mem # for MT8188/MT8195
assigned-clocks:
maxItems: 1
- interrupts
- ranges
- clocks
+ - clock-names
- '#interrupt-cells'
- interrupt-controller
- const: cfg
- const: apb
+ clocks:
+ description:
+ Fabric Interface Controllers, FICs, are the interface between the FPGA
+ fabric and the core complex on PolarFire SoC. The FICs require two clocks,
+ one from each side of the interface. The "FIC clocks" described by this
+ property are on the core complex side & communication through a FIC is not
+ possible unless it's corresponding clock is enabled. A clock must be
+ enabled for each of the interfaces the root port is connected through.
+ This could in theory be all 4 interfaces, one interface or any combination
+ in between.
+ minItems: 1
+ items:
+ - description: FIC0's clock
+ - description: FIC1's clock
+ - description: FIC2's clock
+ - description: FIC3's clock
+
+ clock-names:
+ description:
+ As any FIC connection combination is possible, the names should match the
+ order in the clocks property and take the form "ficN" where N is a number
+ 0-3
+ minItems: 1
+ maxItems: 4
+ items:
+ pattern: '^fic[0-3]$'
+
interrupts:
minItems: 1
items:
ranges:
maxItems: 1
+ dma-ranges:
+ minItems: 1
+ maxItems: 6
+
msi-controller:
description: Identifies the node as an MSI controller.
maintainers:
- Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
-allOf:
- - $ref: "pci-ep.yaml#"
-
properties:
compatible:
- const: qcom,sdx55-pcie-ep
+ enum:
+ - qcom,sdx55-pcie-ep
+ - qcom,sm8450-pcie-ep
reg:
items:
- const: mmio
clocks:
- items:
- - description: PCIe Auxiliary clock
- - description: PCIe CFG AHB clock
- - description: PCIe Master AXI clock
- - description: PCIe Slave AXI clock
- - description: PCIe Slave Q2A AXI clock
- - description: PCIe Sleep clock
- - description: PCIe Reference clock
+ minItems: 7
+ maxItems: 8
clock-names:
- items:
- - const: aux
- - const: cfg
- - const: bus_master
- - const: bus_slave
- - const: slave_q2a
- - const: sleep
- - const: ref
+ minItems: 7
+ maxItems: 8
qcom,perst-regs:
description: Reference to a syscon representing TCSR followed by the two
- reg-names
- clocks
- clock-names
- - qcom,perst-regs
- interrupts
- interrupt-names
- reset-gpios
- reset-names
- power-domains
+allOf:
+ - $ref: pci-ep.yaml#
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,sdx55-pcie-ep
+ then:
+ properties:
+ clocks:
+ items:
+ - description: PCIe Auxiliary clock
+ - description: PCIe CFG AHB clock
+ - description: PCIe Master AXI clock
+ - description: PCIe Slave AXI clock
+ - description: PCIe Slave Q2A AXI clock
+ - description: PCIe Sleep clock
+ - description: PCIe Reference clock
+ clock-names:
+ items:
+ - const: aux
+ - const: cfg
+ - const: bus_master
+ - const: bus_slave
+ - const: slave_q2a
+ - const: sleep
+ - const: ref
+
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,sm8450-pcie-ep
+ then:
+ properties:
+ clocks:
+ items:
+ - description: PCIe Auxiliary clock
+ - description: PCIe CFG AHB clock
+ - description: PCIe Master AXI clock
+ - description: PCIe Slave AXI clock
+ - description: PCIe Slave Q2A AXI clock
+ - description: PCIe Reference clock
+ - description: PCIe DDRSS SF TBU clock
+ - description: PCIe AGGRE NOC AXI clock
+ clock-names:
+ items:
+ - const: aux
+ - const: cfg
+ - const: bus_master
+ - const: bus_slave
+ - const: slave_q2a
+ - const: ref
+ - const: ddrss_sf_tbu
+ - const: aggre_noc_axi
+
unevaluatedProperties: false
examples:
- qcom,pcie-ipq4019
- qcom,pcie-ipq8074
- qcom,pcie-qcs404
+ - qcom,pcie-sa8540p
- qcom,pcie-sc7280
- qcom,pcie-sc8180x
+ - qcom,pcie-sc8280xp
- qcom,pcie-sdm845
- qcom,pcie-sm8150
- qcom,pcie-sm8250
enum:
- qcom,pcie-sc7280
- qcom,pcie-sc8180x
+ - qcom,pcie-sc8280xp
- qcom,pcie-sm8250
- qcom,pcie-sm8450-pcie0
- qcom,pcie-sm8450-pcie1
items:
- const: pci # PCIe core reset
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,pcie-sa8540p
+ - qcom,pcie-sc8280xp
+ then:
+ properties:
+ clocks:
+ minItems: 8
+ maxItems: 9
+ clock-names:
+ minItems: 8
+ items:
+ - const: aux # Auxiliary clock
+ - const: cfg # Configuration clock
+ - const: bus_master # Master AXI clock
+ - const: bus_slave # Slave AXI clock
+ - const: slave_q2a # Slave Q2A clock
+ - const: ddrss_sf_tbu # PCIe SF TBU clock
+ - const: noc_aggr_4 # NoC aggregate 4 clock
+ - const: noc_aggr_south_sf # NoC aggregate South SF clock
+ - const: cnoc_qx # Configuration NoC QX clock
+ resets:
+ maxItems: 1
+ reset-names:
+ items:
+ - const: pci # PCIe core reset
+
- if:
not:
properties:
- resets
- reset-names
- # Newer chipsets support either 1 or 8 MSI vectors
- # On older chipsets it's always 1 MSI vector
- if:
properties:
compatible:
- const: msi5
- const: msi6
- const: msi7
- else:
+
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,pcie-sc8280xp
+ then:
+ properties:
+ interrupts:
+ minItems: 4
+ maxItems: 4
+ interrupt-names:
+ items:
+ - const: msi0
+ - const: msi1
+ - const: msi2
+ - const: msi3
+
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - qcom,pcie-apq8064
+ - qcom,pcie-apq8084
+ - qcom,pcie-ipq4019
+ - qcom,pcie-ipq6018
+ - qcom,pcie-ipq8064
+ - qcom,pcie-ipq8064-v2
+ - qcom,pcie-ipq8074
+ - qcom,pcie-qcs404
+ - qcom,pcie-sa8540p
+ then:
properties:
interrupts:
maxItems: 1
description: A phandle to the PCIe power up reset line.
maxItems: 1
+ clocks:
+ maxItems: 1
+
+ clock-names:
+ const: pcie_aux
+
pwren-gpios:
description: Should specify the GPIO for controlling the PCI bus device power on.
maxItems: 1
- interrupt-map-mask
- interrupt-map
- clocks
+ - clock-names
- resets
- pwren-gpios
- reset-gpios
<0x0 0x0 0x0 0x2 &plic0 58>,
<0x0 0x0 0x0 0x3 &plic0 59>,
<0x0 0x0 0x0 0x4 &plic0 60>;
+ clock-names = "pcie_aux";
clocks = <&prci FU740_PRCI_CLK_PCIE_AUX>;
resets = <&prci 4>;
pwren-gpios = <&gpio 5 0>;
syscon: scu@1e6e2000 {
compatible = "aspeed,ast2400-scu", "syscon", "simple-mfd";
reg = <0x1e6e2000 0x1a8>;
+ #clock-cells = <1>;
+ #reset-cells = <1>;
+
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges = <0x0 0x1e6e2000 0x1000>;
pinctrl: pinctrl {
compatible = "aspeed,ast2400-pinctrl";
#clock-cells = <1>;
#reset-cells = <1>;
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges = <0x0 0x1e6e2000 0x1000>;
+
pinctrl: pinctrl {
compatible = "aspeed,ast2500-pinctrl";
aspeed,external-nodes = <&gfx>, <&lhc>;
syscon: scu@1e6e2000 {
compatible = "aspeed,ast2600-scu", "syscon", "simple-mfd";
reg = <0x1e6e2000 0xf6c>;
+ #clock-cells = <1>;
+ #reset-cells = <1>;
+
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges = <0x0 0x1e6e2000 0x1000>;
pinctrl: pinctrl {
compatible = "aspeed,ast2600-pinctrl";
'-pins$':
type: object
$ref: pinmux-node.yaml#
+ additionalProperties: false
properties:
function:
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/cypress,cy8c95x0.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Cypress CY8C95X0 I2C GPIO expander
+
+maintainers:
+ - Patrick Rudolph <patrick.rudolph@9elements.com>
+
+description: |
+ This supports the 20/40/60 pin Cypress CYC95x0 GPIO I2C expanders.
+ Pin function configuration is performed on a per-pin basis.
+
+properties:
+ compatible:
+ enum:
+ - cypress,cy8c9520
+ - cypress,cy8c9540
+ - cypress,cy8c9560
+
+ reg:
+ maxItems: 1
+
+ gpio-controller: true
+
+ '#gpio-cells':
+ description:
+ The first cell is the GPIO number and the second cell specifies GPIO
+ flags, as defined in <dt-bindings/gpio/gpio.h>.
+ const: 2
+
+ interrupts:
+ maxItems: 1
+
+ interrupt-controller: true
+
+ '#interrupt-cells':
+ const: 2
+
+ gpio-line-names: true
+
+ gpio-ranges:
+ maxItems: 1
+
+ gpio-reserved-ranges:
+ maxItems: 1
+
+ vdd-supply:
+ description:
+ Optional power supply.
+
+patternProperties:
+ '-pins$':
+ type: object
+ description:
+ Pinctrl node's client devices use subnodes for desired pin configuration.
+ Client device subnodes use below standard properties.
+ $ref: pincfg-node.yaml#
+
+ properties:
+ pins:
+ description:
+ List of gpio pins affected by the properties specified in this
+ subnode.
+ items:
+ pattern: '^gp([0-7][0-7])$'
+ minItems: 1
+ maxItems: 60
+
+ function:
+ description:
+ Specify the alternative function to be configured for the specified
+ pins.
+ enum: [ gpio, pwm ]
+
+ bias-pull-down: true
+
+ bias-pull-up: true
+
+ bias-disable: true
+
+ output-high: true
+
+ output-low: true
+
+ drive-push-pull: true
+
+ drive-open-drain: true
+
+ drive-open-source: true
+
+ required:
+ - pins
+ - function
+
+ additionalProperties: false
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - interrupt-controller
+ - '#interrupt-cells'
+ - gpio-controller
+ - '#gpio-cells'
+
+additionalProperties: false
+
+allOf:
+ - $ref: "pinctrl.yaml#"
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ pinctrl@20 {
+ compatible = "cypress,cy8c9520";
+ reg = <0x20>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ #interrupt-cells = <2>;
+ interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-controller;
+ vdd-supply = <&p3v3>;
+ gpio-reserved-ranges = <5 1>;
+ };
+ };
patternProperties:
'^gpio@[0-9a-f]*$':
type: object
+ additionalProperties: false
description:
Child nodes can be specified to contain pin configuration information,
patternProperties:
'^gpio@[0-9a-f]*$':
type: object
+ additionalProperties: false
description:
Child nodes can be specified to contain pin configuration information,
'-pins$':
type: object
$ref: pinmux-node.yaml#
+ additionalProperties: false
properties:
marvell,function:
patternProperties:
'-[0-9]*$':
type: object
+ additionalProperties: false
+
patternProperties:
'-pins*$':
type: object
"i2s" "audio" 62, 63, 64, 65
"switch_int" "eth" 66
"mdc_mdio" "eth" 67
+ "wf_2g" "wifi" 74, 75, 76, 77, 78, 79, 80, 81, 82, 83
+ "wf_5g" "wifi" 91, 92, 93, 94, 95, 96, 97, 98, 99, 100
+ "wf_dbdc" "wifi" 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,
+ 84, 85
$ref: "/schemas/pinctrl/pinmux-node.yaml"
properties:
then:
properties:
groups:
- enum: [wf_2g, wf_5g, wf_dbdc]
+ items:
+ enum: [wf_2g, wf_5g, wf_dbdc]
+ maxItems: 3
'.*conf.*':
type: object
additionalProperties: false
An array of strings. Each string contains the name of a pin.
There is no PIN 41 to PIN 65 above on mt7686b, you can only use
those pins on mt7986a.
- enum: [SYS_WATCHDOG, WF2G_LED, WF5G_LED, I2C_SCL, I2C_SDA, GPIO_0,
- GPIO_1, GPIO_2, GPIO_3, GPIO_4, GPIO_5, GPIO_6, GPIO_7,
- GPIO_8, GPIO_9, GPIO_10, GPIO_11, GPIO_12, GPIO_13, GPIO_14,
- GPIO_15, PWM0, PWM1, SPI0_CLK, SPI0_MOSI, SPI0_MISO, SPI0_CS,
- SPI0_HOLD, SPI0_WP, SPI1_CLK, SPI1_MOSI, SPI1_MISO, SPI1_CS,
- SPI2_CLK, SPI2_MOSI, SPI2_MISO, SPI2_CS, SPI2_HOLD, SPI2_WP,
- UART0_RXD, UART0_TXD, PCIE_PERESET_N, UART1_RXD, UART1_TXD,
- UART1_CTS, UART1_RTS, UART2_RXD, UART2_TXD, UART2_CTS,
- UART2_RTS, EMMC_DATA_0, EMMC_DATA_1, EMMC_DATA_2,
- EMMC_DATA_3, EMMC_DATA_4, EMMC_DATA_5, EMMC_DATA_6,
- EMMC_DATA_7, EMMC_CMD, EMMC_CK, EMMC_DSL, EMMC_RSTB, PCM_DTX,
- PCM_DRX, PCM_CLK, PCM_FS, MT7531_INT, SMI_MDC, SMI_MDIO,
- WF0_DIG_RESETB, WF0_CBA_RESETB, WF0_XO_REQ, WF0_TOP_CLK,
- WF0_TOP_DATA, WF0_HB1, WF0_HB2, WF0_HB3, WF0_HB4, WF0_HB0,
- WF0_HB0_B, WF0_HB5, WF0_HB6, WF0_HB7, WF0_HB8, WF0_HB9,
- WF0_HB10, WF1_DIG_RESETB, WF1_CBA_RESETB, WF1_XO_REQ,
- WF1_TOP_CLK, WF1_TOP_DATA, WF1_HB1, WF1_HB2, WF1_HB3,
- WF1_HB4, WF1_HB0, WF1_HB0_B, WF1_HB5, WF1_HB6, WF1_HB7,
- WF1_HB8]
+ items:
+ enum: [SYS_WATCHDOG, WF2G_LED, WF5G_LED, I2C_SCL, I2C_SDA, GPIO_0,
+ GPIO_1, GPIO_2, GPIO_3, GPIO_4, GPIO_5, GPIO_6, GPIO_7,
+ GPIO_8, GPIO_9, GPIO_10, GPIO_11, GPIO_12, GPIO_13, GPIO_14,
+ GPIO_15, PWM0, PWM1, SPI0_CLK, SPI0_MOSI, SPI0_MISO, SPI0_CS,
+ SPI0_HOLD, SPI0_WP, SPI1_CLK, SPI1_MOSI, SPI1_MISO, SPI1_CS,
+ SPI2_CLK, SPI2_MOSI, SPI2_MISO, SPI2_CS, SPI2_HOLD, SPI2_WP,
+ UART0_RXD, UART0_TXD, PCIE_PERESET_N, UART1_RXD, UART1_TXD,
+ UART1_CTS, UART1_RTS, UART2_RXD, UART2_TXD, UART2_CTS,
+ UART2_RTS, EMMC_DATA_0, EMMC_DATA_1, EMMC_DATA_2,
+ EMMC_DATA_3, EMMC_DATA_4, EMMC_DATA_5, EMMC_DATA_6,
+ EMMC_DATA_7, EMMC_CMD, EMMC_CK, EMMC_DSL, EMMC_RSTB, PCM_DTX,
+ PCM_DRX, PCM_CLK, PCM_FS, MT7531_INT, SMI_MDC, SMI_MDIO,
+ WF0_DIG_RESETB, WF0_CBA_RESETB, WF0_XO_REQ, WF0_TOP_CLK,
+ WF0_TOP_DATA, WF0_HB1, WF0_HB2, WF0_HB3, WF0_HB4, WF0_HB0,
+ WF0_HB0_B, WF0_HB5, WF0_HB6, WF0_HB7, WF0_HB8, WF0_HB9,
+ WF0_HB10, WF1_DIG_RESETB, WF1_CBA_RESETB, WF1_XO_REQ,
+ WF1_TOP_CLK, WF1_TOP_DATA, WF1_HB1, WF1_HB2, WF1_HB3,
+ WF1_HB4, WF1_HB0, WF1_HB0_B, WF1_HB5, WF1_HB6, WF1_HB7,
+ WF1_HB8]
+ maxItems: 101
bias-disable: true
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/mediatek,mt8188-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek MT8188 Pin Controller
+
+maintainers:
+ - Hui Liu <hui.liu@mediatek.com>
+
+description: |
+ The MediaTek's MT8188 Pin controller is used to control SoC pins.
+
+properties:
+ compatible:
+ const: mediatek,mt8188-pinctrl
+
+ gpio-controller: true
+
+ '#gpio-cells':
+ description: |
+ Number of cells in GPIO specifier, should be two. The first cell
+ is the pin number, the second cell is used to specify optional
+ parameters which are defined in <dt-bindings/gpio/gpio.h>.
+ const: 2
+
+ gpio-ranges:
+ maxItems: 1
+
+ gpio-line-names: true
+
+ reg:
+ items:
+ - description: gpio registers base address
+ - description: rm group io configuration registers base address
+ - description: lt group io configuration registers base address
+ - description: lm group io configuration registers base address
+ - description: rt group io configuration registers base address
+ - description: eint registers base address
+
+ reg-names:
+ items:
+ - const: iocfg0
+ - const: iocfg_rm
+ - const: iocfg_lt
+ - const: iocfg_lm
+ - const: iocfg_rt
+ - const: eint
+
+ interrupt-controller: true
+
+ '#interrupt-cells':
+ const: 2
+
+ interrupts:
+ description: The interrupt outputs to sysirq.
+ maxItems: 1
+
+ mediatek,rsel-resistance-in-si-unit:
+ type: boolean
+ description: |
+ We provide two methods to select the resistance for I2C when pull up or pull down.
+ The first is by RSEL definition value, another one is by resistance value(ohm).
+ This flag is used to identify if the method is resistance(si unit) value.
+
+# PIN CONFIGURATION NODES
+patternProperties:
+ '-pins$':
+ type: object
+ additionalProperties: false
+
+ patternProperties:
+ '^pins':
+ type: object
+ $ref: "/schemas/pinctrl/pincfg-node.yaml"
+ additionalProperties: false
+ description: |
+ A pinctrl node should contain at least one subnode representing the
+ pinctrl groups available on the machine. Each subnode will list the
+ pins it needs, and how they should be configured, with regard to muxer
+ configuration, pullups, drive strength, input enable/disable and
+ input schmitt.
+
+ properties:
+ pinmux:
+ description: |
+ Integer array, represents gpio pin number and mux setting.
+ Supported pin number and mux varies for different SoCs, and are
+ defined as macros in dt-bindings/pinctrl/mediatek,<soc>-pinfunc.h
+ directly.
+
+ drive-strength:
+ enum: [2, 4, 6, 8, 10, 12, 14, 16]
+
+ drive-strength-microamp:
+ enum: [125, 250, 500, 1000]
+
+ bias-pull-down:
+ oneOf:
+ - type: boolean
+ - enum: [100, 101, 102, 103]
+ description: mt8188 pull down PUPD/R0/R1 type define value.
+ - enum: [200, 201, 202, 203, 204, 205, 206, 207]
+ description: mt8188 pull down RSEL type define value.
+ - enum: [75000, 5000]
+ description: mt8188 pull down RSEL type si unit value(ohm).
+ description: |
+ For pull down type is normal, it doesn't need add RSEL & R1R0 define
+ and resistance value.
+ For pull down type is PUPD/R0/R1 type, it can add R1R0 define to
+ set different resistance. It can support "MTK_PUPD_SET_R1R0_00" &
+ "MTK_PUPD_SET_R1R0_01" & "MTK_PUPD_SET_R1R0_10" & "MTK_PUPD_SET_R1R0_11"
+ define in mt8188.
+ For pull down type is RSEL, it can add RSEL define & resistance value(ohm)
+ to set different resistance by identifying property "mediatek,rsel-resistance-in-si-unit".
+ It can support "MTK_PULL_SET_RSEL_000" & "MTK_PULL_SET_RSEL_001"
+ & "MTK_PULL_SET_RSEL_010" & "MTK_PULL_SET_RSEL_011" & "MTK_PULL_SET_RSEL_100"
+ & "MTK_PULL_SET_RSEL_101" & "MTK_PULL_SET_RSEL_110" & "MTK_PULL_SET_RSEL_111"
+ define in mt8188. It can also support resistance value(ohm) "75000" & "5000" in mt8188.
+
+ bias-pull-up:
+ oneOf:
+ - type: boolean
+ - enum: [100, 101, 102, 103]
+ description: mt8188 pull up PUPD/R0/R1 type define value.
+ - enum: [200, 201, 202, 203, 204, 205, 206, 207]
+ description: mt8188 pull up RSEL type define value.
+ - enum: [1000, 1500, 2000, 3000, 4000, 5000, 10000, 75000]
+ description: mt8188 pull up RSEL type si unit value(ohm).
+ description: |
+ For pull up type is normal, it don't need add RSEL & R1R0 define
+ and resistance value.
+ For pull up type is PUPD/R0/R1 type, it can add R1R0 define to
+ set different resistance. It can support "MTK_PUPD_SET_R1R0_00" &
+ "MTK_PUPD_SET_R1R0_01" & "MTK_PUPD_SET_R1R0_10" & "MTK_PUPD_SET_R1R0_11"
+ define in mt8188.
+ For pull up type is RSEL, it can add RSEL define & resistance value(ohm)
+ to set different resistance by identifying property "mediatek,rsel-resistance-in-si-unit".
+ It can support "MTK_PULL_SET_RSEL_000" & "MTK_PULL_SET_RSEL_001"
+ & "MTK_PULL_SET_RSEL_010" & "MTK_PULL_SET_RSEL_011" & "MTK_PULL_SET_RSEL_100"
+ & "MTK_PULL_SET_RSEL_101" & "MTK_PULL_SET_RSEL_110" & "MTK_PULL_SET_RSEL_111"
+ define in mt8188. It can also support resistance value(ohm)
+ "1000" & "1500" & "2000" & "3000" & "4000" & "5000" & "10000" & "75000" in mt8188.
+
+ bias-disable: true
+
+ output-high: true
+
+ output-low: true
+
+ input-enable: true
+
+ input-disable: true
+
+ input-schmitt-enable: true
+
+ input-schmitt-disable: true
+
+ required:
+ - pinmux
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - interrupt-controller
+ - '#interrupt-cells'
+ - gpio-controller
+ - '#gpio-cells'
+ - gpio-ranges
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h>
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ pio: pinctrl@10005000 {
+ compatible = "mediatek,mt8188-pinctrl";
+ reg = <0x10005000 0x1000>,
+ <0x11c00000 0x1000>,
+ <0x11e10000 0x1000>,
+ <0x11e20000 0x1000>,
+ <0x11ea0000 0x1000>,
+ <0x1000b000 0x1000>;
+ reg-names = "iocfg0", "iocfg_rm",
+ "iocfg_lt", "iocfg_lm", "iocfg_rt",
+ "eint";
+ gpio-controller;
+ #gpio-cells = <2>;
+ gpio-ranges = <&pio 0 0 176>;
+ interrupt-controller;
+ interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH 0>;
+ #interrupt-cells = <2>;
+
+ pio-pins {
+ pins {
+ pinmux = <PINMUX_GPIO0__FUNC_B_GPIO0>;
+ output-low;
+ };
+ };
+
+ spi0-pins {
+ pins-spi {
+ pinmux = <PINMUX_GPIO75__FUNC_O_SPIM1_CSB>,
+ <PINMUX_GPIO76__FUNC_O_SPIM1_CLK>,
+ <PINMUX_GPIO77__FUNC_B0_SPIM1_MOSI>;
+ drive-strength = <6>;
+ };
+ pins-spi-mi {
+ pinmux = <PINMUX_GPIO78__FUNC_B0_SPIM1_MISO>;
+ bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
+ };
+ };
+
+ i2c0-pins {
+ pins {
+ pinmux = <PINMUX_GPIO55__FUNC_B1_SCL0>,
+ <PINMUX_GPIO56__FUNC_B1_SDA0>;
+ bias-disable;
+ drive-strength-microamp = <1000>;
+ };
+ };
+ };
"^gpio@[0-7]$":
type: object
+ additionalProperties: false
description:
Eight GPIO banks (gpio@0 to gpio@7), that each contain between 14 and 18
Gpio base register names.
items:
- const: iocfg0
- - const: iocfg_bm
- - const: iocfg_bl
- - const: iocfg_br
+ - const: iocfg_lt
- const: iocfg_lm
+ - const: iocfg_lb
+ - const: iocfg_bl
- const: iocfg_rb
- - const: iocfg_tl
+ - const: iocfg_rt
- const: eint
interrupt-controller: true
<0x10002A00 0x0200>,
<0x10002c00 0x0200>,
<0x1000b000 0x1000>;
- reg-names = "iocfg0", "iocfg_bm", "iocfg_bl",
- "iocfg_br", "iocfg_lm", "iocfg_rb",
- "iocfg_tl", "eint";
+ reg-names = "iocfg0", "iocfg_lt", "iocfg_lm",
+ "iocfg_lb", "iocfg_bl", "iocfg_rb",
+ "iocfg_rt", "eint";
gpio-controller;
#gpio-cells = <2>;
gpio-ranges = <&pio 0 0 185>;
- qcom,pm6150-gpio
- qcom,pm6150l-gpio
- qcom,pm6350-gpio
+ - qcom,pm7250b-gpio
- qcom,pm7325-gpio
- qcom,pm8005-gpio
- qcom,pm8008-gpio
enum:
- qcom,pm660l-gpio
- qcom,pm6150l-gpio
+ - qcom,pm7250b-gpio
- qcom,pm8038-gpio
- qcom,pm8150b-gpio
- qcom,pm8150l-gpio
- gpio1-gpio10 for pm6150
- gpio1-gpio12 for pm6150l
- gpio1-gpio9 for pm6350
+ - gpio1-gpio12 for pm7250b
- gpio1-gpio10 for pm7325
- gpio1-gpio4 for pm8005
- gpio1-gpio2 for pm8008
- gpio1-gpio10 for pm8350
- gpio1-gpio8 for pm8350b
- gpio1-gpio9 for pm8350c
+ - gpio1-gpio4 for pm8450
- gpio1-gpio38 for pm8917
- gpio1-gpio44 for pm8921
- gpio1-gpio36 for pm8941
gpio-ranges:
maxItems: 1
+ gpio-line-names:
+ maxItems: 174
+
wakeup-parent: true
#PIN CONFIGURATION NODES
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "/schemas/pinctrl/pincfg-node.yaml"
properties:
pins:
subnode.
items:
oneOf:
- - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-4])$"
+ - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9]|18[0-2])$"
- enum: [ sdc1_rclk, sdc1_clk, sdc1_cmd, sdc1_data, sdc2_clk,
sdc2_cmd, sdc2_data, ufs_reset ]
minItems: 1
required:
- pins
- - function
+
+ allOf:
+ - $ref: /schemas/pinctrl/pincfg-node.yaml
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9]|18[0-2])$"
+ then:
+ required:
+ - function
additionalProperties: false
allOf:
- - $ref: "pinctrl.yaml#"
+ - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
required:
- compatible
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- tlmm: pinctrl@f000000 {
- compatible = "qcom,sc7280-pinctrl";
- reg = <0xf000000 0x1000000>;
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 175>;
- wakeup-parent = <&pdc>;
-
- qup_uart5_default: qup-uart5-pins {
- pins = "gpio46", "gpio47";
- function = "qup13";
- drive-strength = <2>;
- bias-disable;
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ tlmm: pinctrl@f000000 {
+ compatible = "qcom,sc7280-pinctrl";
+ reg = <0xf000000 0x1000000>;
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 175>;
+ wakeup-parent = <&pdc>;
+
+ qup_uart5_default: qup-uart5-pins {
+ pins = "gpio46", "gpio47";
+ function = "qup13";
+ drive-strength = <2>;
+ bias-disable;
};
+ };
oneOf:
- $ref: "#/$defs/qcom-sc8180x-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sc8180x-tlmm-state"
+ additionalProperties: false
'$defs':
qcom-sc8180x-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-8][0-9])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@3100000 {
- compatible = "qcom,sc8180x-tlmm";
- reg = <0x03100000 0x300000>,
- <0x03500000 0x700000>,
- <0x03d00000 0x300000>;
- reg-names = "west", "east", "south";
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 190>;
-
- gpio-wo-subnode-state {
- pins = "gpio1";
- function = "gpio";
- };
-
- uart-w-subnodes-state {
- rx {
- pins = "gpio4";
- function = "qup6";
- bias-pull-up;
- };
-
- tx {
- pins = "gpio5";
- function = "qup6";
- bias-disable;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@3100000 {
+ compatible = "qcom,sc8180x-tlmm";
+ reg = <0x03100000 0x300000>,
+ <0x03500000 0x700000>,
+ <0x03d00000 0x300000>;
+ reg-names = "west", "east", "south";
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 190>;
+
+ gpio-wo-subnode-state {
+ pins = "gpio1";
+ function = "gpio";
+ };
+
+ uart-w-subnodes-state {
+ rx-pins {
+ pins = "gpio4";
+ function = "qup6";
+ bias-pull-up;
+ };
+
+ tx-pins {
+ pins = "gpio5";
+ function = "qup6";
+ bias-disable;
+ };
};
+ };
...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,sc8280xp-lpass-lpi-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. Low Power Audio SubSystem (LPASS)
+ Low Power Island (LPI) TLMM block
+
+maintainers:
+ - Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
+
+description: |
+ This binding describes the Top Level Mode Multiplexer block found in the
+ LPASS LPI IP on most Qualcomm SoCs
+
+properties:
+ compatible:
+ const: qcom,sc8280xp-lpass-lpi-pinctrl
+
+ reg:
+ items:
+ - description: LPASS LPI TLMM Control and Status registers
+ - description: LPASS LPI pins SLEW registers
+
+ clocks:
+ items:
+ - description: LPASS Core voting clock
+ - description: LPASS Audio voting clock
+
+ clock-names:
+ items:
+ - const: core
+ - const: audio
+
+ gpio-controller: true
+
+ '#gpio-cells':
+ description: Specifying the pin number and flags, as defined in
+ include/dt-bindings/gpio/gpio.h
+ const: 2
+
+ gpio-ranges:
+ maxItems: 1
+
+#PIN CONFIGURATION NODES
+patternProperties:
+ '-pins$':
+ type: object
+ description:
+ Pinctrl node's client devices use subnodes for desired pin configuration.
+ Client device subnodes use below standard properties.
+ $ref: /schemas/pinctrl/pincfg-node.yaml
+
+ properties:
+ pins:
+ description:
+ List of gpio pins affected by the properties specified in this
+ subnode.
+ items:
+ pattern: "^gpio([0-1]|1[0-8]])$"
+
+ function:
+ enum: [ swr_tx_clk, swr_tx_data, swr_rx_clk, swr_rx_data,
+ dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic4_clk,
+ dmic4_data, i2s2_clk, i2s2_ws, dmic3_clk, dmic3_data,
+ qua_mi2s_sclk, qua_mi2s_ws, qua_mi2s_data, i2s1_clk, i2s1_ws,
+ i2s1_data, wsa_swr_clk, wsa_swr_data, wsa2_swr_clk,
+ wsa2_swr_data, i2s2_data, i2s3_clk, i2s3_ws, i2s3_data,
+ ext_mclk1_c, ext_mclk1_b, ext_mclk1_a ]
+ description:
+ Specify the alternative function to be configured for the specified
+ pins.
+
+ drive-strength:
+ enum: [2, 4, 6, 8, 10, 12, 14, 16]
+ default: 2
+ description:
+ Selects the drive strength for the specified pins, in mA.
+
+ slew-rate:
+ enum: [0, 1, 2, 3]
+ default: 0
+ description: |
+ 0: No adjustments
+ 1: Higher Slew rate (faster edges)
+ 2: Lower Slew rate (slower edges)
+ 3: Reserved (No adjustments)
+
+ bias-pull-down: true
+
+ bias-pull-up: true
+
+ bias-disable: true
+
+ output-high: true
+
+ output-low: true
+
+ required:
+ - pins
+ - function
+
+ additionalProperties: false
+
+allOf:
+ - $ref: pinctrl.yaml#
+
+required:
+ - compatible
+ - reg
+ - clocks
+ - clock-names
+ - gpio-controller
+ - '#gpio-cells'
+ - gpio-ranges
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/sound/qcom,q6afe.h>
+ pinctrl@33c0000 {
+ compatible = "qcom,sc8280xp-lpass-lpi-pinctrl";
+ reg = <0x33c0000 0x20000>,
+ <0x3550000 0x10000>;
+ clocks = <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+ <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+ clock-names = "core", "audio";
+ gpio-controller;
+ #gpio-cells = <2>;
+ gpio-ranges = <&lpi_tlmm 0 0 18>;
+ };
oneOf:
- $ref: "#/$defs/qcom-sc8280xp-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sc8280xp-tlmm-state"
+ additionalProperties: false
'$defs':
qcom-sc8280xp-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-1][0-9]|22[0-7])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@f100000 {
- compatible = "qcom,sc8280xp-tlmm";
- reg = <0x0f100000 0x300000>;
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 230>;
-
- gpio-wo-subnode-state {
- pins = "gpio1";
- function = "gpio";
- };
-
- uart-w-subnodes-state {
- rx {
- pins = "gpio4";
- function = "qup14";
- bias-pull-up;
- };
-
- tx {
- pins = "gpio5";
- function = "qup14";
- bias-disable;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@f100000 {
+ compatible = "qcom,sc8280xp-tlmm";
+ reg = <0x0f100000 0x300000>;
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 230>;
+
+ gpio-wo-subnode-state {
+ pins = "gpio1";
+ function = "gpio";
+ };
+
+ uart-w-subnodes-state {
+ rx-pins {
+ pins = "gpio4";
+ function = "qup14";
+ bias-pull-up;
+ };
+
+ tx-pins {
+ pins = "gpio5";
+ function = "qup14";
+ bias-disable;
+ };
};
+ };
...
gpio-ranges:
maxItems: 1
+ gpio-reserved-ranges: true
+
wakeup-parent: true
#PIN CONFIGURATION NODES
oneOf:
- $ref: "#/$defs/qcom-sm6115-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sm6115-tlmm-state"
+ additionalProperties: false
'$defs':
qcom-sm6115-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|10[0-9]|11[0-2])$"
+ then:
+ required:
+ - function
+
additionalProperties: false
allOf:
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- tlmm: pinctrl@500000 {
- compatible = "qcom,sm6115-tlmm";
- reg = <0x500000 0x400000>,
- <0x900000 0x400000>,
- <0xd00000 0x400000>;
- reg-names = "west", "south", "east";
- interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 114>;
-
- sdc2_on_state: sdc2-on-state {
- clk {
- pins = "sdc2_clk";
- bias-disable;
- drive-strength = <16>;
- };
-
- cmd {
- pins = "sdc2_cmd";
- bias-pull-up;
- drive-strength = <10>;
- };
-
- data {
- pins = "sdc2_data";
- bias-pull-up;
- drive-strength = <10>;
- };
-
- sd-cd {
- pins = "gpio88";
- function = "gpio";
- bias-pull-up;
- drive-strength = <2>;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ tlmm: pinctrl@500000 {
+ compatible = "qcom,sm6115-tlmm";
+ reg = <0x500000 0x400000>,
+ <0x900000 0x400000>,
+ <0xd00000 0x400000>;
+ reg-names = "west", "south", "east";
+ interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 114>;
+
+ sdc2_on_state: sdc2-on-state {
+ clk-pins {
+ pins = "sdc2_clk";
+ bias-disable;
+ drive-strength = <16>;
+ };
+
+ cmd-pins {
+ pins = "sdc2_cmd";
+ bias-pull-up;
+ drive-strength = <10>;
+ };
+
+ data-pins {
+ pins = "sdc2_data";
+ bias-pull-up;
+ drive-strength = <10>;
+ };
+
+ sd-cd-pins {
+ pins = "gpio88";
+ function = "gpio";
+ bias-pull-up;
+ drive-strength = <2>;
+ };
};
+ };
oneOf:
- $ref: "#/$defs/qcom-sm6125-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sm6125-tlmm-state"
+ additionalProperties: false
$defs:
qcom-sm6125-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio[0-9]|[1-9][0-9]|1[0-2][0-9]|13[0-2]$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@500000 {
- compatible = "qcom,sm6125-tlmm";
- reg = <0x00500000 0x400000>,
- <0x00900000 0x400000>,
- <0x00d00000 0x400000>;
- reg-names = "west", "south", "east";
- interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- gpio-ranges = <&tlmm 0 0 134>;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@500000 {
+ compatible = "qcom,sm6125-tlmm";
+ reg = <0x00500000 0x400000>,
+ <0x00900000 0x400000>,
+ <0x00d00000 0x400000>;
+ reg-names = "west", "south", "east";
+ interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ gpio-ranges = <&tlmm 0 0 134>;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+
+ sdc2-off-state {
+ clk-pins {
+ pins = "sdc2_clk";
+ drive-strength = <2>;
+ bias-disable;
+ };
+
+ cmd-pins {
+ pins = "sdc2_cmd";
+ drive-strength = <2>;
+ bias-pull-up;
+ };
+
+ data-pins {
+ pins = "sdc2_data";
+ drive-strength = <2>;
+ bias-pull-up;
+ };
};
+ };
oneOf:
- $ref: "#/$defs/qcom-sm6350-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sm6350-tlmm-state"
+ additionalProperties: false
$defs:
qcom-sm6350-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-4][0-9]|15[0-7])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@f100000 {
- compatible = "qcom,sm6350-tlmm";
- reg = <0x0f100000 0x300000>;
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 157>;
-
- gpio-wo-subnode-state {
- pins = "gpio1";
- function = "gpio";
- };
-
- uart-w-subnodes-state {
- rx {
- pins = "gpio25";
- function = "qup13_f2";
- bias-disable;
- };
-
- tx {
- pins = "gpio26";
- function = "qup13_f2";
- bias-disable;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@f100000 {
+ compatible = "qcom,sm6350-tlmm";
+ reg = <0x0f100000 0x300000>;
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 157>;
+
+ gpio-wo-subnode-state {
+ pins = "gpio1";
+ function = "gpio";
+ };
+
+ uart-w-subnodes-state {
+ rx-pins {
+ pins = "gpio25";
+ function = "qup13_f2";
+ bias-disable;
+ };
+
+ tx-pins {
+ pins = "gpio26";
+ function = "qup13_f2";
+ bias-disable;
+ };
};
+ };
...
oneOf:
- $ref: "#/$defs/qcom-sm6375-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sm6375-tlmm-state"
+ additionalProperties: false
$defs:
qcom-sm6375-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-4][0-9]|15[0-6])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@500000 {
- compatible = "qcom,sm6375-tlmm";
- reg = <0x00500000 0x800000>;
- interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 157>;
-
- gpio-wo-subnode-state {
- pins = "gpio1";
- function = "gpio";
- };
-
- uart-w-subnodes-state {
- rx {
- pins = "gpio18";
- function = "qup13_f2";
- bias-pull-up;
- };
-
- tx {
- pins = "gpio19";
- function = "qup13_f2";
- bias-disable;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@500000 {
+ compatible = "qcom,sm6375-tlmm";
+ reg = <0x00500000 0x800000>;
+ interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 157>;
+
+ gpio-wo-subnode-state {
+ pins = "gpio1";
+ function = "gpio";
+ };
+
+ uart-w-subnodes-state {
+ rx-pins {
+ pins = "gpio18";
+ function = "qup13_f2";
+ bias-pull-up;
+ };
+
+ tx-pins {
+ pins = "gpio19";
+ function = "qup13_f2";
+ bias-disable;
+ };
};
+ };
...
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@1f00000 {
- compatible = "qcom,sm8250-pinctrl";
- reg = <0x0f100000 0x300000>,
- <0x0f500000 0x300000>,
- <0x0f900000 0x300000>;
- reg-names = "west", "south", "north";
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 180>;
- wakeup-parent = <&pdc>;
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@1f00000 {
+ compatible = "qcom,sm8250-pinctrl";
+ reg = <0x0f100000 0x300000>,
+ <0x0f500000 0x300000>,
+ <0x0f900000 0x300000>;
+ reg-names = "west", "south", "north";
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 180>;
+ wakeup-parent = <&pdc>;
+ };
oneOf:
- $ref: "#/$defs/qcom-sm8350-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sm8350-tlmm-state"
+ additionalProperties: false
$defs:
qcom-sm8350-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-3])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@f100000 {
- compatible = "qcom,sm8350-tlmm";
- reg = <0x0f100000 0x300000>;
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- gpio-ranges = <&tlmm 0 0 203>;
-
- gpio-wo-subnode-state {
- pins = "gpio1";
- function = "gpio";
- };
-
- uart-w-subnodes-state {
- rx {
- pins = "gpio18";
- function = "qup3";
- bias-pull-up;
- };
-
- tx {
- pins = "gpio19";
- function = "qup3";
- bias-disable;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@f100000 {
+ compatible = "qcom,sm8350-tlmm";
+ reg = <0x0f100000 0x300000>;
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 203>;
+
+ gpio-wo-subnode-state {
+ pins = "gpio1";
+ function = "gpio";
+ };
+
+ uart-w-subnodes-state {
+ rx-pins {
+ pins = "gpio18";
+ function = "qup3";
+ bias-pull-up;
+ };
+
+ tx-pins {
+ pins = "gpio19";
+ function = "qup3";
+ bias-disable;
+ };
};
+ };
...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. Low Power Audio SubSystem (LPASS)
+ Low Power Island (LPI) TLMM block
+
+maintainers:
+ - Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
+
+description: |
+ This binding describes the Top Level Mode Multiplexer block found in the
+ LPASS LPI IP on most Qualcomm SoCs
+
+properties:
+ compatible:
+ const: qcom,sm8450-lpass-lpi-pinctrl
+
+ reg:
+ items:
+ - description: LPASS LPI TLMM Control and Status registers
+ - description: LPASS LPI pins SLEW registers
+
+ clocks:
+ items:
+ - description: LPASS Core voting clock
+ - description: LPASS Audio voting clock
+
+ clock-names:
+ items:
+ - const: core
+ - const: audio
+
+ gpio-controller: true
+
+ '#gpio-cells':
+ description: Specifying the pin number and flags, as defined in
+ include/dt-bindings/gpio/gpio.h
+ const: 2
+
+ gpio-ranges:
+ maxItems: 1
+
+#PIN CONFIGURATION NODES
+patternProperties:
+ '-pins$':
+ type: object
+ description:
+ Pinctrl node's client devices use subnodes for desired pin configuration.
+ Client device subnodes use below standard properties.
+ $ref: /schemas/pinctrl/pincfg-node.yaml
+
+ properties:
+ pins:
+ description:
+ List of gpio pins affected by the properties specified in this
+ subnode.
+ items:
+ pattern: "^gpio([0-9]|[1-2][0-9]])$"
+
+ function:
+ enum: [ swr_tx_clk, swr_tx_data, swr_rx_clk, swr_rx_data,
+ dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic4_clk,
+ dmic4_data, i2s2_clk, i2s2_ws, dmic3_clk, dmic3_data,
+ qua_mi2s_sclk, qua_mi2s_ws, qua_mi2s_data, i2s1_clk, i2s1_ws,
+ i2s1_data, wsa_swr_clk, wsa_swr_data, wsa2_swr_clk,
+ wsa2_swr_data, i2s2_data, i2s4_ws, i2s4_clk, i2s4_data,
+ slimbus_clk, i2s3_clk, i2s3_ws, i2s3_data, slimbus_data,
+ ext_mclk1_c, ext_mclk1_b, ext_mclk1_a, ext_mclk1_d,
+ ext_mclk1_e ]
+ description:
+ Specify the alternative function to be configured for the specified
+ pins.
+
+ drive-strength:
+ enum: [2, 4, 6, 8, 10, 12, 14, 16]
+ default: 2
+ description:
+ Selects the drive strength for the specified pins, in mA.
+
+ slew-rate:
+ enum: [0, 1, 2, 3]
+ default: 0
+ description: |
+ 0: No adjustments
+ 1: Higher Slew rate (faster edges)
+ 2: Lower Slew rate (slower edges)
+ 3: Reserved (No adjustments)
+
+ bias-pull-down: true
+
+ bias-pull-up: true
+
+ bias-disable: true
+
+ output-high: true
+
+ output-low: true
+
+ required:
+ - pins
+ - function
+
+ additionalProperties: false
+
+allOf:
+ - $ref: pinctrl.yaml#
+
+required:
+ - compatible
+ - reg
+ - clocks
+ - clock-names
+ - gpio-controller
+ - '#gpio-cells'
+ - gpio-ranges
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/sound/qcom,q6afe.h>
+ pinctrl@3440000 {
+ compatible = "qcom,sm8450-lpass-lpi-pinctrl";
+ reg = <0x3440000 0x20000>,
+ <0x34d0000 0x10000>;
+ clocks = <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+ <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+ clock-names = "core", "audio";
+ gpio-controller;
+ #gpio-cells = <2>;
+ gpio-ranges = <&lpi_tlmm 0 0 23>;
+ };
interrupt-controller: true
'#interrupt-cells': true
gpio-controller: true
- gpio-reserved-ranges: true
+
+ gpio-reserved-ranges:
+ minItems: 1
+ maxItems: 105
+
+ gpio-line-names:
+ maxItems: 209
+
'#gpio-cells': true
gpio-ranges: true
wakeup-parent: true
oneOf:
- $ref: "#/$defs/qcom-sm8450-tlmm-state"
- patternProperties:
- ".*":
+ "-pins$":
$ref: "#/$defs/qcom-sm8450-tlmm-state"
+ additionalProperties: false
$defs:
qcom-sm8450-tlmm-state:
description:
Pinctrl node's client devices use subnodes for desired pin configuration.
Client device subnodes use below standard properties.
- $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
properties:
pins:
required:
- pins
- - function
+
+ allOf:
+ - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state"
+ - if:
+ properties:
+ pins:
+ pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-9])$"
+ then:
+ required:
+ - function
additionalProperties: false
examples:
- |
- #include <dt-bindings/interrupt-controller/arm-gic.h>
- pinctrl@f100000 {
- compatible = "qcom,sm8450-tlmm";
- reg = <0x0f100000 0x300000>;
- gpio-controller;
- #gpio-cells = <2>;
- gpio-ranges = <&tlmm 0 0 211>;
- interrupt-controller;
- #interrupt-cells = <2>;
- interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
-
- gpio-wo-subnode-state {
- pins = "gpio1";
- function = "gpio";
- };
-
- uart-w-subnodes-state {
- rx {
- pins = "gpio26";
- function = "qup7";
- bias-pull-up;
- };
-
- tx {
- pins = "gpio27";
- function = "qup7";
- bias-disable;
- };
- };
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pinctrl@f100000 {
+ compatible = "qcom,sm8450-tlmm";
+ reg = <0x0f100000 0x300000>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ gpio-ranges = <&tlmm 0 0 211>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+
+ gpio-wo-state {
+ pins = "gpio1";
+ function = "gpio";
+ };
+
+ uart-w-state {
+ rx-pins {
+ pins = "gpio26";
+ function = "qup7";
+ bias-pull-up;
+ };
+
+ tx-pins {
+ pins = "gpio27";
+ function = "qup7";
+ bias-disable;
+ };
};
+ };
...
patternProperties:
"^gpio-[0-9]*$":
type: object
+ additionalProperties: false
description:
Each port of the r7s72100 pin controller hardware is itself a GPIO
oneOf:
- items:
- enum:
- - renesas,r9a07g043-pinctrl # RZ/G2UL{Type-1,Type-2}
+ - renesas,r9a07g043-pinctrl # RZ/G2UL{Type-1,Type-2} and RZ/Five
- renesas,r9a07g044-pinctrl # RZ/G2{L,LC}
- items:
- rockchip,rk3568-pinctrl
- rockchip,rk3588-pinctrl
- rockchip,rv1108-pinctrl
+ - rockchip,rv1126-pinctrl
rockchip,grf:
$ref: "/schemas/types.yaml#/definitions/phandle"
The values used for config properties should be derived from the hardware
manual and these values are programmed as-is into the pin pull up/down and
driver strength register of the pin-controller.
- See also include/dt-bindings/pinctrl/samsung.h with useful constants.
See also Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml for
additional information and example.
This is a part of device tree bindings for Samsung S3C/S5P/Exynos SoC pin
controller.
- Pin group settings (like drive strength, pull up/down) are available as
- macros in include/dt-bindings/pinctrl/samsung.h.
-
All the pin controller nodes should be represented in the aliases node using
the following format 'pinctrl{n}' where n is a unique number for the alias.
additionalProperties: false
"^(initial|sleep)-state$":
+ type: object
+ additionalProperties: false
+
patternProperties:
"^(pin-[a-z0-9-]+|[a-z0-9-]+-pin)$":
$ref: samsung,pinctrl-pins-cfg.yaml
examples:
- |
- #include <dt-bindings/pinctrl/samsung.h>
-
pinctrl@7f008000 {
compatible = "samsung,s3c64xx-pinctrl";
reg = <0x7f008000 0x1000>;
uart0-data-pins {
samsung,pins = "gpa-0", "gpa-1";
- samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
- samsung,pin-pud = <S3C64XX_PIN_PULL_NONE>;
+ samsung,pin-function = <2>;
+ samsung,pin-pud = <0>;
};
// ...
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
- #include <dt-bindings/pinctrl/samsung.h>
pinctrl@11400000 {
compatible = "samsung,exynos4210-pinctrl";
uart0-data-pins {
samsung,pins = "gpa0-0", "gpa0-1";
- samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
- samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+ samsung,pin-function = <2>;
+ samsung,pin-pud = <0>;
+ samsung,pin-drv = <0>;
};
// ...
sleep0: sleep-state {
gpa0-0-pin {
samsung,pins = "gpa0-0";
- samsung,pin-con-pdn = <EXYNOS_PIN_PDN_INPUT>;
- samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-con-pdn = <2>;
+ samsung,pin-pud-pdn = <0>;
};
gpa0-1-pin {
samsung,pins = "gpa0-1";
- samsung,pin-con-pdn = <EXYNOS_PIN_PDN_OUT0>;
- samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+ samsung,pin-con-pdn = <0>;
+ samsung,pin-pud-pdn = <0>;
};
// ...
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
- #include <dt-bindings/pinctrl/samsung.h>
pinctrl@11000000 {
compatible = "samsung,exynos4210-pinctrl";
sd0-clk-pins {
samsung,pins = "gpk0-0";
- samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
- samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
+ samsung,pin-function = <2>;
+ samsung,pin-pud = <0>;
+ samsung,pin-drv = <3>;
};
sd4-bus-width8-pins {
part-1-pins {
samsung,pins = "gpk0-3", "gpk0-4",
"gpk0-5", "gpk0-6";
- samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
- samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
+ samsung,pin-function = <3>;
+ samsung,pin-pud = <3>;
+ samsung,pin-drv = <3>;
};
part-2-pins {
samsung,pins = "gpk1-3", "gpk1-4",
"gpk1-5", "gpk1-6";
- samsung,pin-function = <EXYNOS_PIN_FUNC_4>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
- samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
+ samsung,pin-function = <4>;
+ samsung,pin-pud = <3>;
+ samsung,pin-drv = <3>;
};
};
otg-gp-pins {
samsung,pins = "gpx3-3";
- samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
- samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+ samsung,pin-function = <1>;
+ samsung,pin-pud = <0>;
+ samsung,pin-drv = <0>;
samsung,pin-val = <0>;
};
};
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
- #include <dt-bindings/pinctrl/samsung.h>
pinctrl@10580000 {
compatible = "samsung,exynos5433-pinctrl";
initial_alive: initial-state {
gpa0-0-pin {
samsung,pins = "gpa0-0";
- samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_DOWN>;
- samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+ samsung,pin-function = <0>;
+ samsung,pin-pud = <1>;
+ samsung,pin-drv = <0>;
};
// ...
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
- #include <dt-bindings/pinctrl/samsung.h>
pinctrl@114b0000 {
compatible = "samsung,exynos5433-pinctrl";
i2s0-bus-pins {
samsung,pins = "gpz0-0", "gpz0-1", "gpz0-2", "gpz0-3",
"gpz0-4", "gpz0-5", "gpz0-6";
- samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
- samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
- samsung,pin-drv = <EXYNOS5433_PIN_DRV_FAST_SR1>;
+ samsung,pin-function = <2>;
+ samsung,pin-pud = <0>;
+ samsung,pin-drv = <0>;
};
// ...
gpio-controller: true
'#gpio-cells':
const: 2
+ interrupt-controller: true
+ '#interrupt-cells':
+ const: 2
reg:
maxItems: 1
maxItems: 1
resets:
maxItems: 1
+ gpio-line-names: true
gpio-ranges:
minItems: 1
maxItems: 16
minimum: 0
maximum: 11
+ patternProperties:
+ "^(.+-hog(-[0-9]+)?)$":
+ type: object
+ required:
+ - gpio-hog
+
required:
- gpio-controller
- '#gpio-cells'
'-[0-9]*$':
type: object
+ additionalProperties: false
+
patternProperties:
'^pins':
type: object
+ additionalProperties: false
description: |
A pinctrl node should contain at least one subnode representing the
pinctrl group available on the machine. Each subnode will list the
- |
#include <dt-bindings/clock/starfive-jh7100.h>
#include <dt-bindings/reset/starfive-jh7100.h>
- #include <dt-bindings/pinctrl/pinctrl-starfive.h>
+ #include <dt-bindings/pinctrl/pinctrl-starfive-jh7100.h>
soc {
#address-cells = <2>;
pins it needs, and how they should be configured, with regard to muxer
configuration, pullups, drive strength.
$ref: "pinmux-node.yaml"
+ additionalProperties: false
properties:
function:
maintainers:
- Paul Walmsley <paul.walmsley@sifive.com>
- Palmer Dabbelt <palmer@sifive.com>
+ - Conor Dooley <conor@kernel.org>
description: |
This document uses some terminology common to the RISC-V community
insensitive, letters in the riscv,isa string must be all
lowercase to simplify parsing.
$ref: "/schemas/types.yaml#/definitions/string"
- enum:
- - rv64imac
- - rv64imafdc
+ pattern: ^rv(?:64|32)imaf?d?q?c?b?v?k?h?(?:_[hsxz](?:[a-z])+)*$
# RISC-V requires 'timebase-frequency' in /cpus, so disallow it here
timebase-frequency: false
title: Microchip PolarFire SoC-based boards
maintainers:
- - Cyril Jean <Cyril.Jean@microchip.com>
- - Lewis Hanly <lewis.hanly@microchip.com>
+ - Conor Dooley <conor.dooley@microchip.com>
+ - Daire McNamara <daire.mcnamara@microchip.com>
description:
Microchip PolarFire SoC-based boards
$nodename:
const: '/'
compatible:
- items:
- - enum:
- - microchip,mpfs-icicle-kit
- - microchip,mpfs-icicle-reference-rtlv2203
- - sundance,polarberry
- - const: microchip,mpfs
+ oneOf:
+ - items:
+ - enum:
+ - microchip,mpfs-icicle-reference-rtlv2203
+ - microchip,mpfs-icicle-reference-rtlv2210
+ - const: microchip,mpfs-icicle-kit
+ - const: microchip,mpfs
+
+ - items:
+ - enum:
+ - aries,m100pfsevp
+ - microchip,mpfs-sev-kit
+ - sundance,polarberry
+ - const: microchip,mpfs
additionalProperties: true
# Copyright (C) 2020 SiFive, Inc.
%YAML 1.2
---
-$id: http://devicetree.org/schemas/riscv/sifive-l2-cache.yaml#
+$id: http://devicetree.org/schemas/riscv/sifive,ccache0.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
-title: SiFive L2 Cache Controller
+title: SiFive Composable Cache Controller
maintainers:
- Sagar Kadam <sagar.kadam@sifive.com>
- Paul Walmsley <paul.walmsley@sifive.com>
description:
- The SiFive Level 2 Cache Controller is used to provide access to fast copies
- of memory for masters in a Core Complex. The Level 2 Cache Controller also
+ The SiFive Composable Cache Controller is used to provide access to fast copies
+ of memory for masters in a Core Complex. The Composable Cache Controller also
acts as directory-based coherency manager.
All the properties in ePAPR/DeviceTree specification applies for this platform.
compatible:
contains:
enum:
+ - sifive,ccache0
- sifive,fu540-c000-ccache
- sifive,fu740-c000-ccache
oneOf:
- items:
- enum:
+ - sifive,ccache0
- sifive,fu540-c000-ccache
- sifive,fu740-c000-ccache
- const: cache
const: 64
cache-level:
- const: 2
+ enum: [2, 3]
cache-sets:
enum: [1024, 2048]
cache-sets:
const: 1024
+ - if:
+ properties:
+ compatible:
+ contains:
+ const: sifive,ccache0
+
+ then:
+ properties:
+ cache-level:
+ enum: [2, 3]
+
+ else:
+ properties:
+ cache-level:
+ const: 2
+
additionalProperties: false
required:
properties:
compatible:
- items:
- - enum:
- - sifive,fu540-c000-clint
- - starfive,jh7100-clint
- - canaan,k210-clint
- - const: sifive,clint0
+ oneOf:
+ - items:
+ - enum:
+ - sifive,fu540-c000-clint
+ - starfive,jh7100-clint
+ - canaan,k210-clint
+ - const: sifive,clint0
+ - items:
+ - const: sifive,clint0
+ - const: riscv,clint0
+ deprecated: true
+ description: For the QEMU virt machine only
description:
Should be "<vendor>,<chip>-clint" and "sifive,clint<version>".
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright (C) 2022 Microchip Technology, Inc. and its subsidiaries
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/atmel,at91sam9-wdt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Atmel Watchdog Timers
+
+maintainers:
+ - Eugen Hristev <eugen.hristev@microchip.com>
+
+properties:
+ compatible:
+ const: atmel,at91sam9260-wdt
+
+ reg:
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+
+ interrupts:
+ maxItems: 1
+
+ atmel,max-heartbeat-sec:
+ description:
+ Should contain the maximum heartbeat value in seconds. This value
+ should be less or equal to 16. It is used to compute the WDV field.
+ maximum: 16
+
+ atmel,min-heartbeat-sec:
+ description:
+ Should contain the minimum heartbeat value in seconds. This value
+ must be smaller than the max-heartbeat-sec value. It is used to
+ compute the WDD field.
+ maximum: 16
+
+ atmel,watchdog-type:
+ $ref: /schemas/types.yaml#/definitions/string
+ description: |
+ Should be hardware or software.
+ oneOf:
+ - description:
+ Hardware watchdog uses the at91 watchdog reset.
+ const: hardware
+ - description: |
+ Software watchdog uses the watchdog interrupt
+ to trigger a software reset.
+ const: software
+ default: hardware
+
+ atmel,reset-type:
+ $ref: /schemas/types.yaml#/definitions/string
+ description: |
+ Should be proc or all. This is valid only when using hardware watchdog.
+ oneOf:
+ - description:
+ Assert peripherals and processor reset signals.
+ const: all
+ - description:
+ Assert the processor reset signal.
+ const: proc
+ default: all
+
+ atmel,disable:
+ $ref: /schemas/types.yaml#/definitions/flag
+ description:
+ Should be present if you want to stop the watchdog.
+
+ atmel,idle-halt:
+ $ref: /schemas/types.yaml#/definitions/flag
+ description: |
+ Should be present if you want to stop the watchdog when
+ entering idle state.
+ CAUTION: This property should be used with care, it actually makes the
+ watchdog not counting when the CPU is in idle state, therefore the
+ watchdog reset time depends on mean CPU usage and will not reset at all
+ if the CPU stops working while it is in idle state, which is probably
+ not what you want.
+
+ atmel,dbg-halt:
+ $ref: /schemas/types.yaml#/definitions/flag
+ description: |
+ Should be present if you want to stop the watchdog when
+ entering debug state.
+
+required:
+ - compatible
+ - reg
+ - clocks
+
+allOf:
+ - $ref: watchdog.yaml#
+ - if:
+ properties:
+ atmel,reset-type:
+ enum:
+ - all
+ - proc
+ then:
+ properties:
+ atmel,watchdog-type:
+ const: hardware
+
+dependencies:
+ atmel,reset-type: ['atmel,watchdog-type']
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ watchdog@fffffd40 {
+ compatible = "atmel,at91sam9260-wdt";
+ reg = <0xfffffd40 0x10>;
+ interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
+ clocks = <&clk32k>;
+ timeout-sec = <15>;
+ atmel,watchdog-type = "hardware";
+ atmel,reset-type = "all";
+ atmel,dbg-halt;
+ atmel,idle-halt;
+ atmel,max-heartbeat-sec = <16>;
+ atmel,min-heartbeat-sec = <0>;
+ };
+++ /dev/null
-* Atmel Watchdog Timers
-
-** at91sam9-wdt
-
-Required properties:
-- compatible: must be "atmel,at91sam9260-wdt".
-- reg: physical base address of the controller and length of memory mapped
- region.
-- clocks: phandle to input clock.
-
-Optional properties:
-- timeout-sec: contains the watchdog timeout in seconds.
-- interrupts : Should contain WDT interrupt.
-- atmel,max-heartbeat-sec : Should contain the maximum heartbeat value in
- seconds. This value should be less or equal to 16. It is used to
- compute the WDV field.
-- atmel,min-heartbeat-sec : Should contain the minimum heartbeat value in
- seconds. This value must be smaller than the max-heartbeat-sec value.
- It is used to compute the WDD field.
-- atmel,watchdog-type : Should be "hardware" or "software". Hardware watchdog
- use the at91 watchdog reset. Software watchdog use the watchdog
- interrupt to trigger a software reset.
-- atmel,reset-type : Should be "proc" or "all".
- "all" : assert peripherals and processor reset signals
- "proc" : assert the processor reset signal
- This is valid only when using "hardware" watchdog.
-- atmel,disable : Should be present if you want to disable the watchdog.
-- atmel,idle-halt : Should be present if you want to stop the watchdog when
- entering idle state.
- CAUTION: This property should be used with care, it actually makes the
- watchdog not counting when the CPU is in idle state, therefore the
- watchdog reset time depends on mean CPU usage and will not reset at all
- if the CPU stop working while it is in idle state, which is probably
- not what you want.
-- atmel,dbg-halt : Should be present if you want to stop the watchdog when
- entering debug state.
-
-Example:
- watchdog@fffffd40 {
- compatible = "atmel,at91sam9260-wdt";
- reg = <0xfffffd40 0x10>;
- interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
- clocks = <&clk32k>;
- timeout-sec = <15>;
- atmel,watchdog-type = "hardware";
- atmel,reset-type = "all";
- atmel,dbg-halt;
- atmel,idle-halt;
- atmel,max-heartbeat-sec = <16>;
- atmel,min-heartbeat-sec = <0>;
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/mediatek,mt7621-wdt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Ralink Watchdog Timers
+
+maintainers:
+ - Sergio Paracuellos <sergio.paracuellos@gmail.com>
+
+allOf:
+ - $ref: watchdog.yaml#
+
+properties:
+ compatible:
+ const: mediatek,mt7621-wdt
+
+ reg:
+ maxItems: 1
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ watchdog@100 {
+ compatible = "mediatek,mt7621-wdt";
+ reg = <0x100 0x100>;
+ };
+++ /dev/null
-Ralink Watchdog Timers
-
-Required properties:
-- compatible: must be "mediatek,mt7621-wdt"
-- reg: physical base address of the controller and length of the register range
-
-Example:
-
- watchdog@100 {
- compatible = "mediatek,mt7621-wdt";
- reg = <0x100 0x10>;
- };
+++ /dev/null
-Xilinx AXI/PLB soft-core watchdog Device Tree Bindings
----------------------------------------------------------
-
-Required properties:
-- compatible : Should be "xlnx,xps-timebase-wdt-1.00.a" or
- "xlnx,xps-timebase-wdt-1.01.a".
-- reg : Physical base address and size
-
-Optional properties:
-- clocks : Input clock specifier. Refer to common clock
- bindings.
-- clock-frequency : Frequency of clock in Hz
-- xlnx,wdt-enable-once : 0 - Watchdog can be restarted
- 1 - Watchdog can be enabled just once
-- xlnx,wdt-interval : Watchdog timeout interval in 2^<val> clock cycles,
- <val> is integer from 8 to 31.
-
-Example:
-axi-timebase-wdt@40100000 {
- clock-frequency = <50000000>;
- compatible = "xlnx,xps-timebase-wdt-1.00.a";
- clocks = <&clkc 15>;
- reg = <0x40100000 0x10000>;
- xlnx,wdt-enable-once = <0x0>;
- xlnx,wdt-interval = <0x1b>;
-} ;
- renesas,r9a07g054-wdt # RZ/V2L
- const: renesas,rzg2l-wdt
+ - items:
+ - enum:
+ - renesas,r9a09g011-wdt # RZ/V2M
+ - const: renesas,rzv2m-wdt # RZ/V2M
+
- items:
- enum:
- renesas,r8a7742-wdt # RZ/G1H
- enum:
- renesas,r8a779a0-wdt # R-Car V3U
- renesas,r8a779f0-wdt # R-Car S4-8
+ - renesas,r8a779g0-wdt # R-Car V4H
- const: renesas,rcar-gen4-wdt # R-Car Gen4
reg:
maxItems: 1
- interrupts: true
-
- interrupt-names: true
-
- clocks: true
-
- clock-names: true
+ interrupts:
+ minItems: 1
+ items:
+ - description: Timeout
+ - description: Parity error
+
+ interrupt-names:
+ minItems: 1
+ items:
+ - const: wdt
+ - const: perrout
+
+ clocks:
+ minItems: 1
+ items:
+ - description: Register access clock
+ - description: Main clock
+
+ clock-names:
+ minItems: 1
+ items:
+ - const: pclk
+ - const: oscclk
power-domains:
maxItems: 1
required:
- compatible
- reg
+ - interrupts
- clocks
allOf:
contains:
enum:
- renesas,rzg2l-wdt
+ - renesas,rzv2m-wdt
then:
properties:
- interrupts:
- maxItems: 2
- interrupt-names:
- items:
- - const: wdt
- - const: perrout
clocks:
- items:
- - description: Register access clock
- - description: Main clock
+ minItems: 2
clock-names:
- items:
- - const: pclk
- - const: oscclk
+ minItems: 2
required:
- clock-names
+ else:
+ properties:
+ clocks:
+ maxItems: 1
+
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - renesas,rzg2l-wdt
+ then:
+ properties:
+ interrupts:
+ minItems: 2
+ interrupt-names:
+ minItems: 2
+ required:
- interrupt-names
else:
properties:
interrupts:
maxItems: 1
- clocks:
- maxItems: 1
additionalProperties: false
- |
#include <dt-bindings/clock/r8a7795-cpg-mssr.h>
#include <dt-bindings/power/r8a7795-sysc.h>
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
wdt0: watchdog@e6020000 {
compatible = "renesas,r8a7795-wdt", "renesas,rcar-gen3-wdt";
reg = <0xe6020000 0x0c>;
+ interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
clocks = <&cpg CPG_MOD 402>;
power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
resets = <&cpg 402>;
- samsung,exynos5420-wdt # for Exynos5420
- samsung,exynos7-wdt # for Exynos7
- samsung,exynos850-wdt # for Exynos850
+ - samsung,exynosautov9-wdt # for Exynosautov9
reg:
maxItems: 1
- samsung,exynos5420-wdt
- samsung,exynos7-wdt
- samsung,exynos850-wdt
+ - samsung,exynosautov9-wdt
then:
required:
- samsung,syscon-phandle
contains:
enum:
- samsung,exynos850-wdt
+ - samsung,exynosautov9-wdt
then:
properties:
clocks:
- enum:
- rockchip,px30-wdt
- rockchip,rk3066-wdt
+ - rockchip,rk3128-wdt
- rockchip,rk3188-wdt
- rockchip,rk3228-wdt
- rockchip,rk3288-wdt
examples:
- |
+ #include <dt-bindings/clock/toshiba,tmpv770x.h>
+
soc {
#address-cells = <2>;
#size-cells = <2>;
- wdt_clk: wdt-clk {
- compatible = "fixed-clock";
- clock-frequency = <150000000>;
- #clock-cells = <0>;
- };
-
- watchdog@28330000 {
+ wdt: watchdog@28330000 {
compatible = "toshiba,visconti-wdt";
reg = <0 0x28330000 0 0x1000>;
- clocks = <&wdt_clk>;
timeout-sec = <20>;
+ clocks = <&pismu TMPV770X_CLK_WDTCLK>;
};
};
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-or-later OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/xlnx,xps-timebase-wdt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Xilinx AXI/PLB softcore and window Watchdog Timer
+
+maintainers:
+ - Shubhrajyoti Datta <shubhrajyoti.datta@xilinx.com>
+ - Srinivas Neeli <srinivas.neeli@xilinx.com>
+
+description:
+ The Timebase watchdog timer(WDT) is a free-running 32 bit counter.
+ WDT uses a dual-expiration architecture. After one expiration of
+ the timeout interval, an interrupt is generated and the WDT state
+ bit is set to one in the status register. If the state bit is not
+ cleared (by writing a one to the state bit) before the next
+ expiration of the timeout interval, a WDT reset is generated.
+
+allOf:
+ - $ref: watchdog.yaml#
+
+properties:
+ compatible:
+ enum:
+ - xlnx,xps-timebase-wdt-1.01.a
+ - xlnx,xps-timebase-wdt-1.00.a
+
+ reg:
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+
+ clock-frequency:
+ description: Frequency of clock in Hz
+
+ xlnx,wdt-interval:
+ $ref: /schemas/types.yaml#/definitions/uint32
+ description: Watchdog timeout interval
+ minimum: 8
+ maximum: 32
+
+ xlnx,wdt-enable-once:
+ $ref: /schemas/types.yaml#/definitions/uint32
+ enum: [0, 1]
+ description: If watchdog is configured as enable once,
+ then the watchdog cannot be disabled after
+ it has been enabled.
+
+required:
+ - compatible
+ - reg
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ watchdog@40100000 {
+ compatible = "xlnx,xps-timebase-wdt-1.00.a";
+ reg = <0x40100000 0x1000>;
+ clock-frequency = <50000000>;
+ clocks = <&clkc 15>;
+ xlnx,wdt-enable-once = <0x0>;
+ xlnx,wdt-interval = <0x1b>;
+ };
+...
| MDEV CORE |
| MODULE |
| mdev.ko |
- | +-----------+ | mdev_register_device() +--------------+
+ | +-----------+ | mdev_register_parent() +--------------+
| | | +<------------------------+ |
| | | | | nvidia.ko |<-> physical
| | | +------------------------>+ | device
| | | | callbacks +--------------+
| | Physical | |
- | | device | | mdev_register_device() +--------------+
+ | | device | | mdev_register_parent() +--------------+
| | interface | |<------------------------+ |
| | | | | i915.ko |<-> physical
| | | +------------------------>+ | device
| | | | callbacks +--------------+
| | | |
- | | | | mdev_register_device() +--------------+
+ | | | | mdev_register_parent() +--------------+
| | | +<------------------------+ |
| | | | | ccw_device.ko|<-> physical
| | | +------------------------>+ | device
struct mdev_driver {
int (*probe) (struct mdev_device *dev);
void (*remove) (struct mdev_device *dev);
- struct attribute_group **supported_type_groups;
+ unsigned int (*get_available)(struct mdev_type *mtype);
+ ssize_t (*show_description)(struct mdev_type *mtype, char *buf);
struct device_driver driver;
};
When a driver wants to add the GUID creation sysfs to an existing device it has
probe'd to then it should call::
- int mdev_register_device(struct device *dev,
- struct mdev_driver *mdev_driver);
+ int mdev_register_parent(struct mdev_parent *parent, struct device *dev,
+ struct mdev_driver *mdev_driver);
This will provide the 'mdev_supported_types/XX/create' files which can then be
used to trigger the creation of a mdev_device. The created mdev_device will be
When the driver needs to remove itself it calls::
- void mdev_unregister_device(struct device *dev);
+ void mdev_unregister_parent(struct mdev_parent *parent);
Which will unbind and destroy all the created mdevs and remove the sysfs files.
sprintf(buf, "%s-%s", dev_driver_string(parent->dev), group->name);
- (or using mdev_parent_dev(mdev) to arrive at the parent device outside
- of the core mdev code)
-
* device_api
- This attribute should show which device API is being created, for example,
+ This attribute shows which device API is being created, for example,
"vfio-pci" for a PCI device.
* available_instances
- This attribute should show the number of devices of type <type-id> that can be
+ This attribute shows the number of devices of type <type-id> that can be
created.
* [device]
* name
- This attribute should show human readable name. This is optional attribute.
+ This attribute shows a human readable name.
* description
- This attribute should show brief features/description of the type. This is
+ This attribute can show brief features/description of the type. This is an
optional attribute.
Directories and Files Under the sysfs for Each mdev Device
There are tools/testing/selftests using the notifier error injection features
for CPU and memory notifiers.
- * tools/testing/selftests/cpu-hotplug/on-off-test.sh
- * tools/testing/selftests/memory-hotplug/on-off-test.sh
+ * tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh
+ * tools/testing/selftests/memory-hotplug/mem-on-off-test.sh
These scripts first do simple online and offline tests and then do fault
injection tests if notifier error injection module is available.
The Linux kernel client source tree is available at
- https://github.com/ceph/ceph-client.git
- - git://git.kernel.org/pub/scm/linux/kernel/git/sage/ceph-client.git
and the source for the full system is at
https://github.com/ceph/ceph.git
int (*atomic_open)(struct inode *, struct dentry *,
struct file *, unsigned open_flag,
umode_t create_mode);
- int (*tmpfile) (struct inode *, struct dentry *, umode_t);
+ int (*tmpfile) (struct user_namespace *, struct inode *,
+ struct file *, umode_t);
int (*fileattr_set)(struct user_namespace *mnt_userns,
struct dentry *dentry, struct fileattr *fa);
int (*fileattr_get)(struct dentry *dentry, struct fileattr *fa);
callers never looked at specific -E... values anyway. ->iterate() and
->iterate_shared() instance require no changes at all, all filldir_t ones in
the tree converted.
+
+---
+
+**mandatory**
+
+Calling conventions for ->tmpfile() have changed. It now takes a struct
+file pointer instead of struct dentry pointer. d_tmpfile() is similarly
+changed to simplify callers. The passed file is in a non-open state and on
+success must be opened before returning (e.g. by calling
+finish_open_simple()).
* JFFS2 is a write-through file-system, while UBIFS supports write-back,
which makes UBIFS much faster on writes.
-Similarly to JFFS2, UBIFS supports on-the-flight compression which makes
+Similarly to JFFS2, UBIFS supports on-the-fly compression which makes
it possible to fit quite a lot of data to the flash.
Similarly to JFFS2, UBIFS is tolerant of unclean reboots and power-cuts.
void (*update_time)(struct inode *, struct timespec *, int);
int (*atomic_open)(struct inode *, struct dentry *, struct file *,
unsigned open_flag, umode_t create_mode);
- int (*tmpfile) (struct user_namespace *, struct inode *, struct dentry *, umode_t);
+ int (*tmpfile) (struct user_namespace *, struct inode *, struct file *, umode_t);
int (*set_acl)(struct user_namespace *, struct inode *, struct posix_acl *, int);
int (*fileattr_set)(struct user_namespace *mnt_userns,
struct dentry *dentry, struct fileattr *fa);
``tmpfile``
called in the end of O_TMPFILE open(). Optional, equivalent to
atomically creating, opening and unlinking a file in given
- directory.
+ directory. On success needs to return with the file already
+ open; this can be done by calling finish_open_simple() right at
+ the end.
``fileattr_get``
called on ioctl(FS_IOC_GETFLAGS) and ioctl(FS_IOC_FSGETXATTR) to
BTN_TOOL_<name> signals the type of tool that is currently detected by the
hardware and is otherwise independent of ABS_DISTANCE and/or BTN_TOUCH.
+* ABS_PROFILE:
+
+ - Used to describe the state of a multi-value profile switch. An event is
+ emitted only when the selected profile changes, indicating the newly
+ selected profile value.
+
* ABS_MT_<name>:
- Used to describe multitouch input events. Please see
- Rumble:
Rumble is advertised as FF_RUMBLE.
+
+- Profile:
+
+ Some pads provide a multi-value profile selection switch. An example is the
+ XBox Adaptive and the XBox Elite 2 controllers. When the active profile is
+ switched, its newly selected value is emitted as an ABS_PROFILE event.
ksm
memory-model
mmu_notifier
+ multigen_lru
numa
overcommit-accounting
page_migration
If a KSM page is shared between less than ``max_page_sharing`` VMAs,
the node of the stable tree that represents such KSM page points to a
-list of struct rmap_item and the ``page->mapping`` of the
+list of struct ksm_rmap_item and the ``page->mapping`` of the
KSM page points to the stable tree node.
When the sharing passes this threshold, KSM adds a second dimension to
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+=============
+Multi-Gen LRU
+=============
+The multi-gen LRU is an alternative LRU implementation that optimizes
+page reclaim and improves performance under memory pressure. Page
+reclaim decides the kernel's caching policy and ability to overcommit
+memory. It directly impacts the kswapd CPU usage and RAM efficiency.
+
+Design overview
+===============
+Objectives
+----------
+The design objectives are:
+
+* Good representation of access recency
+* Try to profit from spatial locality
+* Fast paths to make obvious choices
+* Simple self-correcting heuristics
+
+The representation of access recency is at the core of all LRU
+implementations. In the multi-gen LRU, each generation represents a
+group of pages with similar access recency. Generations establish a
+(time-based) common frame of reference and therefore help make better
+choices, e.g., between different memcgs on a computer or different
+computers in a data center (for job scheduling).
+
+Exploiting spatial locality improves efficiency when gathering the
+accessed bit. A rmap walk targets a single page and does not try to
+profit from discovering a young PTE. A page table walk can sweep all
+the young PTEs in an address space, but the address space can be too
+sparse to make a profit. The key is to optimize both methods and use
+them in combination.
+
+Fast paths reduce code complexity and runtime overhead. Unmapped pages
+do not require TLB flushes; clean pages do not require writeback.
+These facts are only helpful when other conditions, e.g., access
+recency, are similar. With generations as a common frame of reference,
+additional factors stand out. But obvious choices might not be good
+choices; thus self-correction is necessary.
+
+The benefits of simple self-correcting heuristics are self-evident.
+Again, with generations as a common frame of reference, this becomes
+attainable. Specifically, pages in the same generation can be
+categorized based on additional factors, and a feedback loop can
+statistically compare the refault percentages across those categories
+and infer which of them are better choices.
+
+Assumptions
+-----------
+The protection of hot pages and the selection of cold pages are based
+on page access channels and patterns. There are two access channels:
+
+* Accesses through page tables
+* Accesses through file descriptors
+
+The protection of the former channel is by design stronger because:
+
+1. The uncertainty in determining the access patterns of the former
+ channel is higher due to the approximation of the accessed bit.
+2. The cost of evicting the former channel is higher due to the TLB
+ flushes required and the likelihood of encountering the dirty bit.
+3. The penalty of underprotecting the former channel is higher because
+ applications usually do not prepare themselves for major page
+ faults like they do for blocked I/O. E.g., GUI applications
+ commonly use dedicated I/O threads to avoid blocking rendering
+ threads.
+
+There are also two access patterns:
+
+* Accesses exhibiting temporal locality
+* Accesses not exhibiting temporal locality
+
+For the reasons listed above, the former channel is assumed to follow
+the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is
+present, and the latter channel is assumed to follow the latter
+pattern unless outlying refaults have been observed.
+
+Workflow overview
+=================
+Evictable pages are divided into multiple generations for each
+``lruvec``. The youngest generation number is stored in
+``lrugen->max_seq`` for both anon and file types as they are aged on
+an equal footing. The oldest generation numbers are stored in
+``lrugen->min_seq[]`` separately for anon and file types as clean file
+pages can be evicted regardless of swap constraints. These three
+variables are monotonically increasing.
+
+Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)``
+bits in order to fit into the gen counter in ``folio->flags``. Each
+truncated generation number is an index to ``lrugen->lists[]``. The
+sliding window technique is used to track at least ``MIN_NR_GENS`` and
+at most ``MAX_NR_GENS`` generations. The gen counter stores a value
+within ``[1, MAX_NR_GENS]`` while a page is on one of
+``lrugen->lists[]``; otherwise it stores zero.
+
+Each generation is divided into multiple tiers. A page accessed ``N``
+times through file descriptors is in tier ``order_base_2(N)``. Unlike
+generations, tiers do not have dedicated ``lrugen->lists[]``. In
+contrast to moving across generations, which requires the LRU lock,
+moving across tiers only involves atomic operations on
+``folio->flags`` and therefore has a negligible cost. A feedback loop
+modeled after the PID controller monitors refaults over all the tiers
+from anon and file types and decides which tiers from which types to
+evict or protect.
+
+There are two conceptually independent procedures: the aging and the
+eviction. They form a closed-loop system, i.e., the page reclaim.
+
+Aging
+-----
+The aging produces young generations. Given an ``lruvec``, it
+increments ``max_seq`` when ``max_seq-min_seq+1`` approaches
+``MIN_NR_GENS``. The aging promotes hot pages to the youngest
+generation when it finds them accessed through page tables; the
+demotion of cold pages happens consequently when it increments
+``max_seq``. The aging uses page table walks and rmap walks to find
+young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list``
+and calls ``walk_page_range()`` with each ``mm_struct`` on this list
+to scan PTEs, and after each iteration, it increments ``max_seq``. For
+the latter, when the eviction walks the rmap and finds a young PTE,
+the aging scans the adjacent PTEs. For both, on finding a young PTE,
+the aging clears the accessed bit and updates the gen counter of the
+page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``.
+
+Eviction
+--------
+The eviction consumes old generations. Given an ``lruvec``, it
+increments ``min_seq`` when ``lrugen->lists[]`` indexed by
+``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
+evict from, it first compares ``min_seq[]`` to select the older type.
+If both types are equally old, it selects the one whose first tier has
+a lower refault percentage. The first tier contains single-use
+unmapped clean pages, which are the best bet. The eviction sorts a
+page according to its gen counter if the aging has found this page
+accessed through page tables and updated its gen counter. It also
+moves a page to the next generation, i.e., ``min_seq+1``, if this page
+was accessed multiple times through file descriptors and the feedback
+loop has detected outlying refaults from the tier this page is in. To
+this end, the feedback loop uses the first tier as the baseline, for
+the reason stated earlier.
+
+Summary
+-------
+The multi-gen LRU can be disassembled into the following parts:
+
+* Generations
+* Rmap walks
+* Page table walks
+* Bloom filters
+* PID controller
+
+The aging and the eviction form a producer-consumer model;
+specifically, the latter drives the former by the sliding window over
+generations. Within the aging, rmap walks drive page table walks by
+inserting hot densely populated page tables to the Bloom filters.
+Within the eviction, the PID controller uses refaults as the feedback
+to select types to evict and tiers to protect.
label patching functionality is available. Following is the kernel's code
size change due to this facility.
-- Without page owner::
-
- text data bss dec hex filename
- 48392 2333 644 51369 c8a9 mm/page_alloc.o
-
-- With page owner::
-
- text data bss dec hex filename
- 48800 2445 644 51889 cab1 mm/page_alloc.o
- 6662 108 29 6799 1a8f mm/page_owner.o
- 1025 8 8 1041 411 mm/page_ext.o
-
-Although, roughly, 8 KB code is added in total, page_alloc.o increase by
-520 bytes and less than half of it is in hotpath. Building the kernel with
-page owner and turning it on if needed would be great option to debug
-kernel memory problem.
+Although enabling page owner increases kernel size by several kilobytes,
+most of this code is outside page allocator and its hot path. Building
+the kernel with page owner and turning it on if needed would be great
+option to debug kernel memory problem.
There is one notice that is caused by implementation detail. page owner
stores information into the memory from struct page extension. This memory
Page allocated via order XXX, ...
PFN XXX ...
// Detailed stack
+ By default, it will do full pfn dump, to start with a given pfn,
+ page_owner supports fseek.
+
+ FILE *fp = fopen("/sys/kernel/debug/page_owner", "r");
+ fseek(fp, pfn_start, SEEK_SET);
The ``page_owner_sort`` tool ignores ``PFN`` rows, puts the remaining rows
in buf, uses regexp to extract the page order value, counts the times
vlan_tci skb_vlan_tag_get(skb)
vlan_avail skb_vlan_tag_present(skb)
vlan_tpid skb->vlan_proto
- rand prandom_u32()
+ rand get_random_u32()
=================================== =================================================
These extensions can also be prefixed with '#'.
available:
* Some SoCs may offer a pin pad/mux/controller capable of configuring a given
- set of pins'strength, delays, and voltage; and it may be a suitable
+ set of pins' strength, delays, and voltage; and it may be a suitable
option to insert the expected 2ns RGMII delay.
* Modifying the PCB design to include a fixed delay (e.g: using a specifically
There are archives of the mailing list on the web in many different
places. Use a search engine to find these archives. For example:
- http://dir.gmane.org/gmane.linux.kernel
+ https://lore.kernel.org/lkml/
It is highly recommended that you search the archives about the topic
you want to bring up, before you post it to the list. A lot of things
boot-image-header
vm-layout
patch-acceptance
+ uabi
features
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+RISC-V Linux User ABI
+=====================
+
+Misaligned accesses are supported in userspace, but they may perform poorly.
| MDEV CORE |
| MODULE |
| mdev.ko |
- | +---------+ | mdev_register_device() +--------------+
+ | +---------+ | mdev_register_parent() +--------------+
| |Physical | +<-----------------------+ |
| | device | | | vfio_ap.ko |<-> matrix
| |interface| +----------------------->+ | device
| MDEV CORE |
| MODULE |
| mdev.ko |
- | +---------+ | mdev_register_device() +--------------+
+ | +---------+ | mdev_register_parent() +--------------+
| |Physical | +<-----------------------+ |
| | device | | | vfio_ccw.ko |<-> subchannel
| |interface| +----------------------->+ | device
latency higher than *30 us* is hit. Finally, it is set to save the trace
buffer if the stop condition is hit::
- [root@alien ~]# rtla timerlat top -s 30 -t 30 -T
+ [root@alien ~]# rtla timerlat top -s 30 -T 30 -t
Timer Latency
0 00:00:59 | IRQ Timer Latency (us) | Thread Timer Latency (us)
CPU COUNT | cur min avg max | cur min avg max
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+================
+CoreSight - Perf
+================
+
+ :Author: Carsten Haitzler <carsten.haitzler@arm.com>
+ :Date: June 29th, 2022
+
+Perf is able to locally access CoreSight trace data and store it to the
+output perf data files. This data can then be later decoded to give the
+instructions that were traced for debugging or profiling purposes. You
+can log such data with a perf record command like::
+
+ perf record -e cs_etm//u testbinary
+
+This would run some test binary (testbinary) until it exits and record
+a perf.data trace file. That file would have AUX sections if CoreSight
+is working correctly. You can dump the content of this file as
+readable text with a command like::
+
+ perf report --stdio --dump -i perf.data
+
+You should find some sections of this file have AUX data blocks like::
+
+ 0x1e78 [0x30]: PERF_RECORD_AUXTRACE size: 0x11dd0 offset: 0 ref: 0x1b614fc1061b0ad1 idx: 0 tid: 531230 cpu: -1
+
+ . ... CoreSight ETM Trace data: size 73168 bytes
+ Idx:0; ID:10; I_ASYNC : Alignment Synchronisation.
+ Idx:12; ID:10; I_TRACE_INFO : Trace Info.; INFO=0x0 { CC.0 }
+ Idx:17; ID:10; I_ADDR_L_64IS0 : Address, Long, 64 bit, IS0.; Addr=0x0000000000000000;
+ Idx:26; ID:10; I_TRACE_ON : Trace On.
+ Idx:27; ID:10; I_ADDR_CTXT_L_64IS0 : Address & Context, Long, 64 bit, IS0.; Addr=0x0000FFFFB6069140; Ctxt: AArch64,EL0, NS;
+ Idx:38; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEEEEEEEEEEEEEE
+ Idx:39; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEEEEEEEEEEEEEE
+ Idx:40; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEEEEEEEEEEEEEE
+ Idx:41; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEN
+ ...
+
+If you see these above, then your system is tracing CoreSight data
+correctly.
+
+To compile perf with CoreSight support in the tools/perf directory do::
+
+ make CORESIGHT=1
+
+This requires OpenCSD to build. You may install distribution packages
+for the support such as libopencsd and libopencsd-dev or download it
+and build yourself. Upstream OpenCSD is located at:
+
+ https://github.com/Linaro/OpenCSD
+
+For complete information on building perf with CoreSight support and
+more extensive usage look at:
+
+ https://github.com/Linaro/OpenCSD/blob/master/HOWTO.md
+
+
+Kernel CoreSight Support
+------------------------
+
+You will also want CoreSight support enabled in your kernel config.
+Ensure it is enabled with::
+
+ CONFIG_CORESIGHT=y
+
+There are various other CoreSight options you probably also want
+enabled like::
+
+ CONFIG_CORESIGHT_LINKS_AND_SINKS=y
+ CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y
+ CONFIG_CORESIGHT_CATU=y
+ CONFIG_CORESIGHT_SINK_TPIU=y
+ CONFIG_CORESIGHT_SINK_ETBV10=y
+ CONFIG_CORESIGHT_SOURCE_ETM4X=y
+ CONFIG_CORESIGHT_CTI=y
+ CONFIG_CORESIGHT_CTI_INTEGRATION_REGS=y
+
+Please refer to the kernel configuration help for more information.
+
+Perf test - Verify kernel and userspace perf CoreSight work
+-----------------------------------------------------------
+
+When you run perf test, it will do a lot of self tests. Some of those
+tests will cover CoreSight (only if enabled and on ARM64). You
+generally would run perf test from the tools/perf directory in the
+kernel tree. Some tests will check some internal perf support like:
+
+ Check Arm CoreSight trace data recording and synthesized samples
+ Check Arm SPE trace data recording and synthesized samples
+
+Some others will actually use perf record and some test binaries that
+are in tests/shell/coresight and will collect traces to ensure a
+minimum level of functionality is met. The scripts that launch these
+tests are in the same directory. These will all look like:
+
+ CoreSight / ASM Pure Loop
+ CoreSight / Memcpy 16k 10 Threads
+ CoreSight / Thread Loop 10 Threads - Check TID
+ etc.
+
+These perf record tests will not run if the tool binaries do not exist
+in tests/shell/coresight/\*/ and will be skipped. If you do not have
+CoreSight support in hardware then either do not build perf with
+CoreSight support or remove these binaries in order to not have these
+tests fail and have them skip instead.
+
+These tests will log historical results in the current working
+directory (e.g. tools/perf) and will be named stats-\*.csv like:
+
+ stats-asm_pure_loop-out.csv
+ stats-memcpy_thread-16k_10.csv
+ ...
+
+These statistic files log some aspects of the AUX data sections in
+the perf data output counting some numbers of certain encodings (a
+good way to know that it's working in a very simple way). One problem
+with CoreSight is that given a large enough amount of data needing to
+be logged, some of it can be lost due to the processor not waking up
+in time to read out all the data from buffers etc.. You will notice
+that the amount of data collected can vary a lot per run of perf test.
+If you wish to see how this changes over time, simply run perf test
+multiple times and all these csv files will have more and more data
+appended to it that you can later examine, graph and otherwise use to
+figure out if things have become worse or better.
+
+This means sometimes these tests fail as they don't capture all the
+data needed. This is about tracking quality and amount of data
+produced over time and to see when changes to the Linux kernel improve
+quality of traces.
+
+Be aware that some of these tests take quite a while to run, specifically
+in processing the perf data file and dumping contents to then examine what
+is inside.
+
+You can change where these csv logs are stored by setting the
+PERF_TEST_CORESIGHT_STATDIR environment variable before running perf
+test like::
+
+ export PERF_TEST_CORESIGHT_STATDIR=/var/tmp
+ perf test
+
+They will also store resulting perf output data in the current
+directory for later inspection like::
+
+ perf-asm_pure_loop-out.data
+ perf-memcpy_thread-16k_10.data
+ ...
+
+You can alter where the perf data files are stored by setting the
+PERF_TEST_CORESIGHT_DATADIR environment variable such as::
+
+ PERF_TEST_CORESIGHT_DATADIR=/var/tmp
+ perf test
+
+You may wish to set these above environment variables if you wish to
+keep the output of tests outside of the current working directory for
+longer term storage and examination.
start::
- trace_fd = open("trace_marker", WR_ONLY);
+ trace_fd = open("trace_marker", O_WRONLY);
Note: Writing into the trace_marker file can also initiate triggers
that are written into /sys/kernel/tracing/events/ftrace/print/trigger
Ci sono diversi archivi della lista di discussione. Usate un qualsiasi motore
di ricerca per trovarli. Per esempio:
- http://dir.gmane.org/gmane.linux.kernel
+ https://lore.kernel.org/lkml/
É caldamente consigliata una ricerca in questi archivi sul tema che volete
sollevare, prima di pubblicarlo sulla lista. Molte cose sono già state
このメーリングリストのアーカイブは web 上の多数の場所に存在します。こ
れらのアーカイブを探すにはサーチエンジンを使いましょう。例えば-
- http://dir.gmane.org/gmane.linux.kernel
+ https://lore.kernel.org/lkml/
リストに投稿する前にすでにその話題がアーカイブに存在するかどうかを検索
することを是非やってください。多数の事がすでに詳細に渡って議論されてお
웹상의 많은 다른 곳에도 메일링 리스트의 아카이브들이 있다.
이러한 아카이브들을 찾으려면 검색 엔진을 사용하라. 예를 들어:
- http://dir.gmane.org/gmane.linux.kernel
+ https://lore.kernel.org/lkml/
여러분이 새로운 문제에 관해 리스트에 올리기 전에 말하고 싶은 주제에 관한
것을 아카이브에서 먼저 찾아보기를 강력히 권장한다. 이미 상세하게 토론된 많은
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+处理器体系结构
+==============
+
+以下文档提供了具体架构实现的编程细节。
+
+.. toctree::
+ :maxdepth: 2
+
+ mips/index
+ arm64/index
+ riscv/index
+ openrisc/index
+ parisc/index
+ loongarch/index
+
+TODOList:
+
+* arm/index
+* ia64/index
+* m68k/index
+* nios2/index
+* powerpc/index
+* s390/index
+* sh/index
+* sparc/index
+* x86/index
+* xtensa/index
.. SPDX-License-Identifier: GPL-2.0
.. include:: ../disclaimer-zh_CN.rst
-:Original: Documentation/Devicetree/changesets.rst
+:Original: Documentation/devicetree/changesets.rst
:翻译:
.. SPDX-License-Identifier: GPL-2.0
.. include:: ../disclaimer-zh_CN.rst
-:Original: Documentation/Devicetree/dynamic-resolution-notes.rst
+:Original: Documentation/devicetree/dynamic-resolution-notes.rst
:翻译:
.. SPDX-License-Identifier: GPL-2.0
.. include:: ../disclaimer-zh_CN.rst
-:Original: Documentation/Devicetree/kernel-api.rst
+:Original: Documentation/devicetree/kernel-api.rst
:翻译:
.. SPDX-License-Identifier: GPL-2.0
.. include:: ../disclaimer-zh_CN.rst
-:Original: Documentation/Devicetree/overlay-notes.rst
+:Original: Documentation/devicetree/overlay-notes.rst
:翻译:
顺便说下，中文文档也需要遵守内核编码风格，风格中中文和英文的主要不同就是中文
的字符标点占用两个英文字符宽度， 所以，当英文要求不要超过每行100个字符时，
中文就不要超过50个字符。另外，也要注意'-'，'=' 等符号与相关标题的对齐。在将
-补丁提交到社区之前，一定要进行必要的checkpatch.pl检查和编译测试。
+补丁提交到社区之前，一定要进行必要的 ``checkpatch.pl`` 检查和编译测试。
-许可证文档
-----------
-
-下面的文档介绍了Linux内核源代码的许可证（GPLv2）、如何在源代码树中正确标记
-单个文件的许可证、以及指向完整许可证文本的链接。
+与Linux 内核社区一起工作
+------------------------
-* Documentation/translations/zh_CN/process/license-rules.rst
-
-用户文档
---------
-
-下面的手册是为内核用户编写的——即那些试图让它在给定系统上以最佳方式工作的
-用户。
+与内核开发社区进行协作并将工作推向上游的基本指南。
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
- admin-guide/index
-
-TODOList:
-
-* kbuild/index
+ process/development-process
+ process/submitting-patches
+ 行为准则 <process/code-of-conduct>
+ maintainer/index
+ 完整开发流程文档 <process/index>
-å\9bºä»¶ç\9b¸å\85³文档
-------------
+å\86\85é\83¨API文档
+-----------
-下列文档描述了内核需要的平台固件相关信息。
+开发人员使用的内核内部交互接口手册。
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
- devicetree/index
+ core-api/index
+ driver-api/index
+ 内核中的锁 <locking/index>
TODOList:
-* firmware-guide/index
-
-应用程序开发人员文档
---------------------
-
-用户空间API手册涵盖了描述应用程序开发人员可见内核接口方面的文档。
+* subsystem-apis
-TODOlist:
+开发工具和流程
+--------------
-* userspace-api/index
-
-内核开发简介
-------------
-
-这些手册包含有关如何开发内核的整体信息。内核社区非常庞大，一年下来有数千名
-开发人员做出贡献。与任何大型社区一样，知道如何完成任务将使得更改合并的过程
-变得更加容易。
+为所有内核开发人员提供有用信息的各种其他手册。
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
- process/index
- dev-tools/index
+ process/license-rules
doc-guide/index
+ dev-tools/index
+ dev-tools/testing-overview
kernel-hacking/index
- maintainer/index
TODOList:
* trace/index
* fault-injection/index
* livepatch/index
-* rust/index
-内核API文档
------------
+面向用户的文档
+--------------
-以ä¸\8bæ\89\8bå\86\8cä»\8eå\86\85æ ¸å¼\80å\8f\91人å\91\98ç\9a\84è§\92度详ç»\86ä»\8bç»\8däº\86ç\89¹å®\9aç\9a\84å\86\85æ ¸å\90ç³»ç»\9fæ\98¯å¦\82ä½\95å·¥ä½\9cç\9a\84ã\80\82è¿\99é\87\8cç\9a\84
-大é\83¨å\88\86ä¿¡æ\81¯é\83½æ\98¯ç\9b´æ\8e¥ä»\8eå\86\85æ ¸æº\90代ç \81è\8e·å\8f\96ç\9a\84ï¼\8cå¹¶æ ¹æ\8d®é\9c\80è¦\81æ·»å\8a è¡¥å\85\85æ\9d\90æ\96\99ï¼\88æ\88\96è\80\85è\87³å°\91æ\98¯å\9c¨
-我们设法添加的时候——可能不是所有的都是有需要的）。
+ä¸\8bå\88\97æ\89\8bå\86\8cé\92\88对
+å¸\8cæ\9c\9bå\86\85æ ¸å\9c¨ç»\99å®\9aç³»ç»\9fä¸\8a以æ\9c\80ä½³æ\96¹å¼\8få·¥ä½\9cç\9a\84*ç\94¨æ\88·*ï¼\8c
+和查找内核用户空间API信息的程序开发人员。
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
- core-api/index
- driver-api/index
- locking/index
- accounting/index
- cpu-freq/index
- iio/index
- infiniband/index
- power/index
- virt/index
- sound/index
- filesystems/index
- scheduler/index
- mm/index
- peci/index
- PCI/index
+ admin-guide/index
+ admin-guide/reporting-issues.rst
TODOList:
-* block/index
-* cdrom/index
-* ide/index
-* fb/index
-* fpga/index
-* hid/index
-* i2c/index
-* isdn/index
-* leds/index
-* netlabel/index
-* networking/index
-* pcmcia/index
-* target/index
-* timers/index
-* spi/index
-* w1/index
-* watchdog/index
-* input/index
-* hwmon/index
-* gpu/index
-* security/index
-* crypto/index
-* bpf/index
-* usb/index
-* scsi/index
-* misc-devices/index
-* mhi/index
-
-体系结构无关文档
-----------------
+* 内核构建系统 <kbuild/index>
+* 用户空间工具 <tools/index>
+* userspace-api/index
-TODOList:
+也可参考独立于内核文档的 `Linux 手册页 <https://www.kernel.org/doc/man-pages/>`_ 。
-* asm-annotations
+固件相关文档
+------------
-特定体系结构文档
-----------------
+下列文档描述了内核需要的平台固件相关信息。
.. toctree::
:maxdepth: 2
- mips/index
- arm64/index
- riscv/index
- openrisc/index
- parisc/index
- loongarch/index
+ devicetree/index
TODOList:
-* arm/index
-* ia64/index
-* m68k/index
-* nios2/index
-* powerpc/index
-* s390/index
-* sh/index
-* sparc/index
-* x86/index
-* xtensa/index
+* firmware-guide/index
+
+体系结构文档
+------------
+
+.. toctree::
+ :maxdepth: 2
+
+ arch
其他文档
--------
TODOList:
* staging/index
-* watch_queue
-目录和表格
+
+索引和表格
----------
* :ref:`genindex`
KSM维护着稳定树中的KSM页的逆映射信息。
当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了
-KSM页的稳定树其中的节点指向了一个rmap_item结构体类型的列表。同时，这个KSM页
+KSM页的稳定树其中的节点指向了一个ksm_rmap_item结构体类型的列表。同时，这个KSM页
的 ``page->mapping`` 指向了该稳定树节点。
如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多
cat /sys/kernel/debug/page_owner > page_owner_full.txt
./page_owner_sort page_owner_full.txt sorted_page_owner.txt
- ``page_owner_full.txt`` 的一般输出情况如下(输出信息无翻译价值)::
+ ``page_owner_full.txt`` 的一般输出情况如下::
Page allocated via order XXX, ...
PFN XXX ...
- // Detailed stack
+ // 栈详情
Page allocated via order XXX, ...
PFN XXX ...
- // Detailed stack
+ // 栈详情
+ 默认情况下，它将以一个给定的pfn开始，做完整的pfn转储，且page_owner支持fseek。
+
+ FILE *fp = fopen("/sys/kernel/debug/page_owner", "r");
+ fseek(fp, pfn_start, SEEK_SET);
``page_owner_sort`` 工具忽略了 ``PFN`` 行，将剩余的行放在buf中，使用regexp提
取页序值，计算buf的次数和页数，最后根据参数进行排序。
网上很多地方都有这个邮件列表的存档(archive)。可以使用搜索引擎来找到这些
存档。比如：
- http://dir.gmane.org/gmane.linux.kernel
+ https://lore.kernel.org/lkml/
在发信之前，我们强烈建议你先在存档中搜索你想要讨论的问题。很多已经被详细
讨论过的问题只在邮件列表的存档中可以找到。
.. _cn_process_index:
+========================
与Linux 内核社区一起工作
========================
網上很多地方都有這個郵件列表的存檔(archive)。可以使用搜尋引擎來找到這些
存檔。比如：
- http://dir.gmane.org/gmane.linux.kernel
+ https://lore.kernel.org/lkml/
在發信之前，我們強烈建議你先在存檔中搜索你想要討論的問題。很多已經被詳細
討論過的問題只在郵件列表的存檔中可以找到。
(0x40000001). Otherwise, a guest may use the paravirtual features
regardless of what has actually been exposed through the CPUID leaf.
-8.29 KVM_CAP_DIRTY_LOG_RING
----------------------------
+8.29 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL
+----------------------------------------------------------
:Architectures: x86
:Parameters: args[0] - size of the dirty log ring
flags of a GFN have the DIRTY bit cleared, meaning that it has harvested
all the dirty GFNs that were available.
+Note that on weakly ordered architectures, userspace accesses to the
+ring buffer (and more specifically the 'flags' field) must be ordered,
+using load-acquire/store-release accessors when available, or any
+other memory barrier that will ensure this ordering.
+
It's not necessary for userspace to harvest the all dirty GFNs at once.
However it must collect the dirty GFNs in sequence, i.e., the userspace
program cannot skip one dirty GFN to collect the one next to it.
machine will switch to ring-buffer dirty page tracking and further
KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail.
+NOTE: KVM_CAP_DIRTY_LOG_RING_ACQ_REL is the only capability that
+should be exposed by weakly ordered architecture, in order to indicate
+the additional memory ordering requirements imposed on userspace when
+reading the state of an entry and mutating it from DIRTY to HARVESTED.
+Architecture with TSO-like ordering (such as x86) are allowed to
+expose both KVM_CAP_DIRTY_LOG_RING and KVM_CAP_DIRTY_LOG_RING_ACQ_REL
+to userspace.
+
8.30 KVM_CAP_XEN_HVM
--------------------
S: Supported
W: http://wiki.analog.com/ADP5588
W: https://ez.analog.com/linux-software-drivers
-F: drivers/gpio/gpio-adp5588.c
+F: Documentation/devicetree/bindings/input/adi,adp5588.yaml
F: drivers/input/keyboard/adp5588-keys.c
ADP8860 BACKLIGHT DRIVER (ADP8860/ADP8861/ADP8863)
M: Shuai Xue <xueshuai@linux.alibaba.com>
S: Supported
F: Documentation/admin-guide/perf/alibaba_pmu.rst
-F: drivers/perf/alibaba_uncore_dwr_pmu.c
+F: drivers/perf/alibaba_uncore_drw_pmu.c
ALIENWARE WMI DRIVER
L: Dell.Client.Kernel@dell.com
F: include/dt-bindings/arm/coresight-cti-dt.h
F: include/linux/coresight*
F: samples/coresight/*
+F: tools/perf/tests/shell/coresight/*
F: tools/perf/arch/arm/util/auxtrace.c
F: tools/perf/arch/arm/util/cs-etm.c
F: tools/perf/arch/arm/util/cs-etm.h
R: Paulo Alcantara <pc@cjr.nz> (DFS, global name space)
R: Ronnie Sahlberg <lsahlber@redhat.com> (directory leases, sparse files)
R: Shyam Prasad N <sprasad@microsoft.com> (multichannel)
+R: Tom Talpey <tom@talpey.com> (RDMA, smbdirect)
L: linux-cifs@vger.kernel.org
L: samba-technical@lists.samba.org (moderated for non-subscribers)
S: Supported
T: git git://linuxtv.org/anttip/media_tree.git
F: drivers/media/common/cypress_firmware*
+CYPRESS CY8C95X0 PINCTRL DRIVER
+M: Patrick Rudolph <patrick.rudolph@9elements.com>
+L: linux-gpio@vger.kernel.org
+S: Maintained
+F: drivers/pinctrl/pinctrl-cy8c95x0.c
+
CYPRESS CY8CTMA140 TOUCHSCREEN DRIVER
M: Linus Walleij <linus.walleij@linaro.org>
L: linux-input@vger.kernel.org
F: Documentation/ia64/
F: arch/ia64/
+IBM Operation Panel Input Driver
+M: Eddie James <eajames@linux.ibm.com>
+L: linux-input@vger.kernel.org
+S: Maintained
+F: Documentation/devicetree/bindings/input/ibm,op-panel.yaml
+F: drivers/input/misc/ibm-panel.c
+
IBM Power 842 compression accelerator
M: Haren Myneni <haren@us.ibm.com>
S: Supported
F: arch/*/mm/kasan_init*
F: include/linux/kasan*.h
F: lib/Kconfig.kasan
-F: lib/test_kasan*.c
F: mm/kasan/
F: scripts/Makefile.kasan
R: Suzuki K Poulose <suzuki.poulose@arm.com>
R: Oliver Upton <oliver.upton@linux.dev>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
-L: kvmarm@lists.cs.columbia.edu (moderated for non-subscribers)
+L: kvmarm@lists.linux.dev
+L: kvmarm@lists.cs.columbia.edu (deprecated, moderated for non-subscribers)
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git
F: arch/arm64/include/asm/kvm*
F: lib/test_kmod.c
F: tools/testing/selftests/kmod/
+KMSAN
+M: Alexander Potapenko <glider@google.com>
+R: Marco Elver <elver@google.com>
+R: Dmitry Vyukov <dvyukov@google.com>
+L: kasan-dev@googlegroups.com
+S: Maintained
+F: Documentation/dev-tools/kmsan.rst
+F: arch/*/include/asm/kmsan.h
+F: arch/*/mm/kmsan_*
+F: include/linux/kmsan*.h
+F: lib/Kconfig.kmsan
+F: mm/kmsan/
+F: scripts/Makefile.kmsan
+
KPROBES
M: Naveen N. Rao <naveen.n.rao@linux.ibm.com>
M: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
S: Maintained
W: http://www.kernel.org/doc/man-pages
+MAPLE TREE
+M: Liam R. Howlett <Liam.Howlett@oracle.com>
+L: linux-mm@kvack.org
+S: Supported
+F: Documentation/core-api/maple_tree.rst
+F: include/linux/maple_tree.h
+F: include/trace/events/maple_tree.h
+F: lib/maple_tree.c
+F: lib/test_maple_tree.c
+F: tools/testing/radix-tree/linux/maple_tree.h
+F: tools/testing/radix-tree/maple.c
+
MARDUK (CREATOR CI40) DEVICE TREE SUPPORT
M: Rahul Bedarkar <rahulbedarkar89@gmail.com>
L: linux-mips@vger.kernel.org
F: Documentation/devicetree/bindings/media/mediatek-jpeg-*.yaml
F: drivers/media/platform/mediatek/jpeg/
+MEDIATEK KEYPAD DRIVER
+M: Mattijs Korpershoek <mkorpershoek@baylibre.com>
+S: Supported
+F: Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml
+F: drivers/input/keyboard/mt6779-keypad.c
+
MEDIATEK MDP DRIVER
M: Minghsiu Tsai <minghsiu.tsai@mediatek.com>
M: Houlong Wei <houlong.wei@mediatek.com>
S: Supported
F: drivers/infiniband/ulp/opa_vnic
-OPEN FIRMWARE AND DEVICE TREE OVERLAYS
-M: Pantelis Antoniou <pantelis.antoniou@konsulko.com>
-M: Frank Rowand <frowand.list@gmail.com>
-L: devicetree@vger.kernel.org
-S: Maintained
-F: Documentation/devicetree/dynamic-resolution-notes.rst
-F: Documentation/devicetree/overlay-notes.rst
-F: drivers/of/overlay.c
-F: drivers/of/resolver.c
-K: of_overlay_notifier_
-
OPEN FIRMWARE AND FLATTENED DEVICE TREE
M: Rob Herring <robh+dt@kernel.org>
M: Frank Rowand <frowand.list@gmail.com>
F: drivers/of/
F: include/linux/of*.h
F: scripts/dtc/
+K: of_overlay_notifier_
+K: of_overlay_fdt_apply
+K: of_overlay_remove
OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS
M: Rob Herring <robh+dt@kernel.org>
L: openrisc@lists.librecores.org
S: Maintained
W: http://openrisc.io
-T: git git://github.com/openrisc/linux.git
+T: git https://github.com/openrisc/linux.git
F: Documentation/devicetree/bindings/openrisc/
F: Documentation/openrisc/
F: arch/openrisc/
M: Kishon Vijay Abraham I <kishon@ti.com>
M: Lorenzo Pieralisi <lpieralisi@kernel.org>
R: Krzysztof Wilczyński <kw@linux.com>
+R: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
L: linux-pci@vger.kernel.org
S: Supported
Q: https://patchwork.kernel.org/project/linux-pci/list/
F: tools/pci/
PCI ENHANCED ERROR HANDLING (EEH) FOR POWERPC
-M: Russell Currey <ruscur@russell.cc>
-M: Oliver O'Halloran <oohall@gmail.com>
+M: Mahesh J Salgaonkar <mahesh@linux.ibm.com>
+R: Oliver O'Halloran <oohall@gmail.com>
L: linuxppc-dev@lists.ozlabs.org
S: Supported
F: Documentation/PCI/pci-error-recovery.rst
F: drivers/pinctrl/sunplus/
F: include/dt-bindings/pinctrl/sppctl*.h
+PINE64 PINEPHONE KEYBOARD DRIVER
+M: Samuel Holland <samuel@sholland.org>
+S: Supported
+F: Documentation/devicetree/bindings/input/pine64,pinephone-keyboard.yaml
+F: drivers/input/keyboard/pinephone-keyboard.c
+
PKTCDVD DRIVER
M: linux-block@vger.kernel.org
S: Orphan
M: Albert Ou <aou@eecs.berkeley.edu>
L: linux-riscv@lists.infradead.org
S: Supported
+Q: https://patchwork.kernel.org/project/linux-riscv/list/
P: Documentation/riscv/patch-acceptance.rst
T: git git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git
F: arch/riscv/
M: Daire McNamara <daire.mcnamara@microchip.com>
L: linux-riscv@lists.infradead.org
S: Supported
-F: Documentation/devicetree/bindings/clock/microchip,mpfs.yaml
+F: Documentation/devicetree/bindings/clock/microchip,mpfs*.yaml
F: Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml
F: Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml
F: Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml
F: Documentation/devicetree/bindings/net/can/microchip,mpfs-can.yaml
F: Documentation/devicetree/bindings/pwm/microchip,corepwm.yaml
+F: Documentation/devicetree/bindings/riscv/microchip.yaml
F: Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-sys-controller.yaml
F: Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
F: Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml
F: drivers/mmc/host/sdhci-esdhc-imx.c
SECURE ENCRYPTING DEVICE (SED) OPAL DRIVER
-M: Jonathan Derrick <jonathan.derrick@intel.com>
-M: Revanth Rajashekar <revanth.rajashekar@intel.com>
+M: Jonathan Derrick <jonathan.derrick@linux.dev>
L: linux-block@vger.kernel.org
S: Supported
F: block/opal_proto.h
SHARED MEMORY COMMUNICATIONS (SMC) SOCKETS
M: Karsten Graul <kgraul@linux.ibm.com>
M: Wenjia Zhang <wenjia@linux.ibm.com>
+M: Jan Karcher <jaka@linux.ibm.com>
L: linux-s390@vger.kernel.org
S: Supported
W: http://www.ibm.com/developerworks/linux/linux390/
L: linux-gpio@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/pinctrl/starfive,jh7100-pinctrl.yaml
-F: drivers/pinctrl/pinctrl-starfive.c
-F: include/dt-bindings/pinctrl/pinctrl-starfive.h
+F: drivers/pinctrl/starfive/
+F: include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h
STARFIVE JH7100 RESET CONTROLLER DRIVER
M: Emil Renner Berthing <kernel@esmil.dk>
L: kvm@vger.kernel.org
S: Maintained
T: git git://github.com/awilliam/linux-vfio.git
+F: Documentation/ABI/testing/sysfs-devices-vfio-dev
F: Documentation/driver-api/vfio.rst
F: drivers/vfio/
F: include/linux/vfio.h
F: include/uapi/linux/virtio_*.h
F: tools/virtio/
+IFCVF VIRTIO DATA PATH ACCELERATOR
+R: Zhu Lingshan <lingshan.zhu@intel.com>
+F: drivers/vdpa/ifcvf/
+
VIRTIO BALLOON
M: "Michael S. Tsirkin" <mst@redhat.com>
M: David Hildenbrand <david@redhat.com>
# SPDX-License-Identifier: GPL-2.0
VERSION = 6
-PATCHLEVEL = 0
+PATCHLEVEL = 1
SUBLEVEL = 0
-EXTRAVERSION =
+EXTRAVERSION = -rc1
NAME = Hurr durr I'ma ninja sloth
# *DOCUMENTATION*
include-$(CONFIG_DEBUG_INFO) += scripts/Makefile.debug
include-$(CONFIG_KASAN) += scripts/Makefile.kasan
include-$(CONFIG_KCSAN) += scripts/Makefile.kcsan
+include-$(CONFIG_KMSAN) += scripts/Makefile.kmsan
include-$(CONFIG_UBSAN) += scripts/Makefile.ubsan
include-$(CONFIG_KCOV) += scripts/Makefile.kcov
include-$(CONFIG_RANDSTRUCT) += scripts/Makefile.randstruct
single-goals := $(addprefix $(build-dir)/, $(single-no-ko))
+KBUILD_MODULES := 1
+
endif
# Preset locale variables to speed up the build process. Limit locale
config HAVE_ARCH_NODE_DEV_GROUP
bool
+config ARCH_HAS_NONLEAF_PMD_YOUNG
+ bool
+ help
+ Architectures that select this option are capable of setting the
+ accessed bit in non-leaf PMD entries when using them as part of linear
+ address translations. Page table walkers that clear the accessed bit
+ may use this capability to reduce their search space.
+
source "kernel/gcov/Kconfig"
source "scripts/gcc-plugins/Kconfig"
CONFIG_NLS_CODEPAGE_437=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_KERNEL=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_ALPHA_LEGACY_START_ADDRESS=y
CONFIG_MATHEMU=y
CONFIG_CRYPTO_HMAC=y
/* Free all resources held by a thread. */
struct task_struct;
-extern void release_thread(struct task_struct *);
-
unsigned long __get_wchan(struct task_struct *p);
#define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc)
#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */
+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */
+
/* compatibility flags */
#define MAP_FILE 0
return (void __iomem *)addr;
}
-unsigned u8
+u8
marvel_ioread8(const void __iomem *xaddr)
{
unsigned long addr = (unsigned long) xaddr;
return addr;
}
-SYSCALL_DEFINE3(osf_readv, unsigned long, fd,
- const struct iovec __user *, vector, unsigned long, count)
-{
- return sys_readv(fd, vector, count);
-}
-
-SYSCALL_DEFINE3(osf_writev, unsigned long, fd,
- const struct iovec __user *, vector, unsigned long, count)
-{
- return sys_writev(fd, vector, count);
-}
-
SYSCALL_DEFINE2(osf_getpriority, int, which, int, who)
{
int prio = sys_getpriority(which, who);
current_thread_info()->pcb.unique = 0;
}
-void
-release_thread(struct task_struct *dead_task)
-{
-}
-
/*
* Copy architecture-specific thread state
*/
boot flags depending on the boot mode, we need some shorthand.
This should do for installation. */
if (strcmp(COMMAND_LINE, "INSTALL") == 0) {
- strlcpy(command_line, "root=/dev/fd0 load_ramdisk=1", sizeof command_line);
+ strscpy(command_line, "root=/dev/fd0 load_ramdisk=1", sizeof(command_line));
} else {
- strlcpy(command_line, COMMAND_LINE, sizeof command_line);
+ strscpy(command_line, COMMAND_LINE, sizeof(command_line));
}
strcpy(boot_command_line, command_line);
*cmdline_p = command_line;
116 common osf_gettimeofday sys_osf_gettimeofday
117 common osf_getrusage sys_osf_getrusage
118 common getsockopt sys_getsockopt
-120 common readv sys_osf_readv
-121 common writev sys_osf_writev
+120 common readv sys_readv
+121 common writev sys_writev
122 common osf_settimeofday sys_osf_settimeofday
123 common fchown sys_fchown
124 common fchmod sys_fchmod
endmenu # "ARC Architecture Configuration"
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
default "12" if ARC_HUGEPAGE_16M
default "11"
CONFIG_CONFIGFS_FS=y
# CONFIG_MISC_FILESYSTEMS is not set
# CONFIG_NETWORK_FILESYSTEMS is not set
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_DEBUG_FS=y
CONFIG_HEADERS_INSTALL=y
#define task_pt_regs(p) \
((struct pt_regs *)(THREAD_SIZE + (void *)task_stack_page(p)) - 1)
-/* Free all resources held by a thread */
-#define release_thread(thread) do { } while (0)
-
/*
* A lot of busy-wait loops in SMP are based off of non-volatile data otherwise
* get optimised away by gcc
Disabling this is usually safe for small single-platform
configurations. If unsure, say y.
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
default "12" if SOC_AM33XX
default "9" if SA1111
CONFIG_SMP=y
CONFIG_ARM_PSCI=y
CONFIG_HIGHMEM=y
-CONFIG_FORCE_MAX_ZONEORDER=14
+CONFIG_ARCH_FORCE_MAX_ORDER=14
CONFIG_CMDLINE="noinitrd console=ttymxc0,115200"
CONFIG_KEXEC=y
CONFIG_CPU_FREQ=y
# CONFIG_THUMB2_AVOID_R_ARM_THM_JUMP11 is not set
# CONFIG_ARM_PATCH_IDIV is not set
CONFIG_HIGHMEM=y
-CONFIG_FORCE_MAX_ZONEORDER=12
+CONFIG_ARCH_FORCE_MAX_ORDER=12
CONFIG_SECCOMP=y
CONFIG_KEXEC=y
CONFIG_EFI=y
CONFIG_MACH_OX820=y
CONFIG_SMP=y
CONFIG_NR_CPUS=16
-CONFIG_FORCE_MAX_ZONEORDER=12
+CONFIG_ARCH_FORCE_MAX_ORDER=12
CONFIG_SECCOMP=y
CONFIG_ARM_APPENDED_DTB=y
CONFIG_ARM_ATAG_DTB_COMPAT=y
CONFIG_MACH_BORZOI=y
CONFIG_PXA_SYSTEMS_CPLDS=y
CONFIG_AEABI=y
-CONFIG_FORCE_MAX_ZONEORDER=9
+CONFIG_ARCH_FORCE_MAX_ORDER=9
CONFIG_CMDLINE="root=/dev/ram0 ro"
CONFIG_KEXEC=y
CONFIG_CPU_FREQ=y
# CONFIG_CACHE_L2X0 is not set
# CONFIG_ARM_PATCH_IDIV is not set
# CONFIG_CPU_SW_DOMAIN_PAN is not set
-CONFIG_FORCE_MAX_ZONEORDER=15
+CONFIG_ARCH_FORCE_MAX_ORDER=15
CONFIG_UACCESS_WITH_MEMCPY=y
# CONFIG_ATAGS is not set
CONFIG_CMDLINE="console=ttyS0,115200 earlyprintk ignore_loglevel"
# CONFIG_VDSO is not set
CONFIG_SMP=y
CONFIG_THUMB2_KERNEL=y
-CONFIG_FORCE_MAX_ZONEORDER=12
+CONFIG_ARCH_FORCE_MAX_ORDER=12
CONFIG_VFP=y
CONFIG_NEON=y
CONFIG_MODULES=y
/* Forward declaration, a strange C thing */
struct task_struct;
-/* Free all resources held by a thread. */
-extern void release_thread(struct task_struct *);
-
unsigned long __get_wchan(struct task_struct *p);
#define task_pt_regs(p) \
thread_notify(THREAD_NOTIFY_FLUSH, thread);
}
-void release_thread(struct task_struct *dead_task)
-{
-}
-
asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
slots = ((last - first) >> PAGE_SHIFT) + 1;
- offset = get_random_int() % slots;
+ offset = prandom_u32_max(slots);
addr = first + (offset << PAGE_SHIFT);
PAGE_SIZE / sizeof(u32));
/* Give the signal return code some randomness */
- offset = 0x200 + (get_random_int() & 0x7fc);
+ offset = 0x200 + (get_random_u16() & 0x7fc);
signal_return_offset = offset;
/* Copy signal return handlers into the page */
#include <linux/clk/at91_pmc.h>
#include <linux/platform_data/atmel.h>
-#include <soc/at91/pm.h>
-
#include <asm/cacheflush.h>
#include <asm/fncpy.h>
#include <asm/system_misc.h>
if (ret)
return ret;
-#ifdef CONFIG_PINCTRL_AT91
- /*
- * FIXME: this is needed to communicate between the pinctrl driver and
- * the PM implementation in the machine. Possibly part of the PM
- * implementation should be moved down into the pinctrl driver and get
- * called as part of the generic suspend/resume path.
- */
- at91_pinctrl_gpio_suspend();
-#endif
-
switch (state) {
case PM_SUSPEND_MEM:
case PM_SUSPEND_STANDBY:
}
error:
-#ifdef CONFIG_PINCTRL_AT91
- at91_pinctrl_gpio_resume();
-#endif
at91_pm_config_quirks(false);
return 0;
}
static u64 __maybe_unused usb_dma_mask = ~(u32)0;
#if IS_ENABLED(CONFIG_PHY_PXA_USB)
-struct resource pxa168_usb_phy_resources[] = {
+static struct resource pxa168_usb_phy_resources[] = {
[0] = {
.start = PXA168_U2O_PHYBASE,
.end = PXA168_U2O_PHYBASE + USB_PHY_RANGE,
#endif /* CONFIG_PHY_PXA_USB */
#if IS_ENABLED(CONFIG_USB_MV_UDC)
-struct resource pxa168_u2o_resources[] = {
+static struct resource pxa168_u2o_resources[] = {
/* regbase */
[0] = {
.start = PXA168_U2O_REGBASE + U2x_CAPREGS_OFFSET,
#endif /* CONFIG_USB_MV_UDC */
#if IS_ENABLED(CONFIG_USB_EHCI_MV_U2O)
-struct resource pxa168_u2oehci_resources[] = {
+static struct resource pxa168_u2oehci_resources[] = {
[0] = {
.start = PXA168_U2O_REGBASE,
.end = PXA168_U2O_REGBASE + USB_REG_RANGE,
#endif
#if IS_ENABLED(CONFIG_USB_MV_OTG)
-struct resource pxa168_u2ootg_resources[] = {
+static struct resource pxa168_u2ootg_resources[] = {
/* regbase */
[0] = {
.start = PXA168_U2O_REGBASE + U2x_CAPREGS_OFFSET,
extern struct pl08x_platform_data pl080_plat_data;
void __init spear_setup_of_timer(void);
-void __init spear3xx_clk_init(void __iomem *misc_base,
- void __iomem *soc_config_base);
void __init spear3xx_map_io(void);
void __init spear3xx_dt_init_irq(void);
-void __init spear6xx_clk_init(void __iomem *misc_base);
void __init spear13xx_map_io(void);
void __init spear13xx_l2x0_init(void);
#include <linux/amba/pl022.h>
#include <linux/amba/pl080.h>
#include <linux/clk.h>
+#include <linux/clk/spear.h>
#include <linux/io.h>
#include <asm/mach/map.h>
#include "pl080.h"
#include <linux/amba/pl08x.h>
#include <linux/clk.h>
+#include <linux/clk/spear.h>
#include <linux/err.h>
#include <linux/of.h>
#include <linux/of_address.h>
* 0xD0000000 0xFD000000
* 0xFC000000 0xFC000000
*/
-struct map_desc spear6xx_io_desc[] __initdata = {
+static struct map_desc spear6xx_io_desc[] __initdata = {
{
.virtual = (unsigned long)VA_SPEAR6XX_ML_CPU_BASE,
.pfn = __phys_to_pfn(SPEAR_ICM3_ML1_2_BASE),
};
/* This will create static memory mapping for selected devices */
-void __init spear6xx_map_io(void)
+static void __init spear6xx_map_io(void)
{
iotable_init(spear6xx_io_desc, ARRAY_SIZE(spear6xx_io_desc));
}
-void __init spear6xx_timer_init(void)
+static void __init spear6xx_timer_init(void)
{
char pclk_name[] = "pll3_clk";
struct clk *gpt_clk, *pclk;
}
/* Add auxdata to pass platform data */
-struct of_dev_auxdata spear6xx_auxdata_lookup[] __initdata = {
+static struct of_dev_auxdata spear6xx_auxdata_lookup[] __initdata = {
OF_DEV_AUXDATA("arm,pl080", SPEAR_ICM3_DMA_BASE, NULL,
&spear6xx_pl080_plat_data),
{}
config ARM64_WORKAROUND_REPEAT_TLBI
bool
+config ARM64_ERRATUM_2441007
+ bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI"
+ default y
+ select ARM64_WORKAROUND_REPEAT_TLBI
+ help
+ This option adds a workaround for ARM Cortex-A55 erratum #2441007.
+
+ Under very rare circumstances, affected Cortex-A55 CPUs
+ may not handle a race between a break-before-make sequence on one
+ CPU, and another CPU accessing the same page. This could allow a
+ store to a page that has been unmapped.
+
+ Work around this by adding the affected CPUs to the list that needs
+ TLB sequences to be done twice.
+
+ If unsure, say Y.
+
config ARM64_ERRATUM_1286807
bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation"
default y
help
Say Y if you want to run Linux in a Virtual Machine on Xen on ARM64.
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int
default "14" if ARM64_64K_PAGES
default "12" if ARM64_16K_PAGES
return 0;
/* Setup the guest ID */
- guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0);
+ guest_id = hv_generate_guest_id(LINUX_VERSION_CODE);
hv_set_vpreg(HV_REGISTER_GUEST_OSID, guest_id);
/* Get the features and hints from Hyper-V */
#define ARM_CPU_IMP_FUJITSU 0x46
#define ARM_CPU_IMP_HISI 0x48
#define ARM_CPU_IMP_APPLE 0x61
+#define ARM_CPU_IMP_AMPERE 0xC0
#define ARM_CPU_PART_AEM_V8 0xD0F
#define ARM_CPU_PART_FOUNDATION 0xD00
#define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028
#define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029
+#define AMPERE_CPU_PART_AMPERE1 0xAC3
+
#define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72)
#define MIDR_APPLE_M1_FIRESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO)
#define MIDR_APPLE_M1_ICESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX)
#define MIDR_APPLE_M1_FIRESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX)
+#define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1)
/* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */
#define MIDR_FUJITSU_ERRATUM_010001 MIDR_FUJITSU_A64FX
*/
struct {
u32 mdscr_el1;
+ bool pstate_ss;
} guest_debug_preserved;
/* vcpu power state */
#define IN_WFIT __vcpu_single_flag(sflags, BIT(3))
/* vcpu system registers loaded on physical CPU */
#define SYSREGS_ON_CPU __vcpu_single_flag(sflags, BIT(4))
+/* Software step state is Active-pending */
+#define DBG_SS_ACTIVE_PENDING __vcpu_single_flag(sflags, BIT(5))
+
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \
* page after fork() + CoW for pfn mappings. We don't always have a
* hardware-managed access flag on arm64.
*/
-static inline bool arch_faults_on_old_pte(void)
-{
- /* The register read below requires a stable CPU to make any sense */
- cant_migrate();
-
- return !cpu_has_hw_af();
-}
-#define arch_faults_on_old_pte arch_faults_on_old_pte
+#define arch_has_hw_pte_young cpu_has_hw_af
/*
* Experimentally, it's cheap to set the access flag in hardware and we
* benefit from prefaulting mappings as 'old' to start with.
*/
-static inline bool arch_wants_old_prefaulted_pte(void)
-{
- return !arch_faults_on_old_pte();
-}
-#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte
+#define arch_wants_old_prefaulted_pte cpu_has_hw_af
static inline bool pud_sect_supported(void)
{
/* Forward declaration, a strange C thing */
struct task_struct;
-/* Free all resources held by a thread. */
-extern void release_thread(struct task_struct *);
-
unsigned long __get_wchan(struct task_struct *p);
void update_sctlr_el1(u64 sctlr);
ERRATA_MIDR_RANGE(MIDR_QCOM_KRYO_4XX_GOLD, 0xc, 0xe, 0xf, 0xe),
},
#endif
+#ifdef CONFIG_ARM64_ERRATUM_2441007
+ {
+ ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
+ },
+#endif
#ifdef CONFIG_ARM64_ERRATUM_2441009
{
/* Cortex-A510 r0p0 -> r1p1. Fixed in r1p2 */
#include <asm/cpufeature.h>
#include <asm/mte.h>
-#define for_each_mte_vma(tsk, vma) \
+#define for_each_mte_vma(vmi, vma) \
if (system_supports_mte()) \
- for (vma = tsk->mm->mmap; vma; vma = vma->vm_next) \
+ for_each_vma(vmi, vma) \
if (vma->vm_flags & VM_MTE)
static unsigned long mte_vma_tag_dump_size(struct vm_area_struct *vma)
{
struct vm_area_struct *vma;
int vma_count = 0;
+ VMA_ITERATOR(vmi, current->mm, 0);
- for_each_mte_vma(current, vma)
+ for_each_mte_vma(vmi, vma)
vma_count++;
return vma_count;
int elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, current->mm, 0);
- for_each_mte_vma(current, vma) {
+ for_each_mte_vma(vmi, vma) {
struct elf_phdr phdr;
phdr.p_type = PT_AARCH64_MEMTAG_MTE;
{
struct vm_area_struct *vma;
size_t data_size = 0;
+ VMA_ITERATOR(vmi, current->mm, 0);
- for_each_mte_vma(current, vma)
+ for_each_mte_vma(vmi, vma)
data_size += mte_vma_tag_dump_size(vma);
return data_size;
int elf_core_write_extra_data(struct coredump_params *cprm)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, current->mm, 0);
- for_each_mte_vma(current, vma) {
+ for_each_mte_vma(vmi, vma) {
if (vma->vm_flags & VM_DONTDUMP)
continue;
if (!pte_is_tagged)
return;
- mte_clear_page_tags(page_address(page));
+ /*
+ * Test PG_mte_tagged again in case it was racing with another
+ * set_pte_at().
+ */
+ if (!test_and_set_bit(PG_mte_tagged, &page->flags))
+ mte_clear_page_tags(page_address(page));
}
void mte_sync_tags(pte_t old_pte, pte_t pte)
/* if PG_mte_tagged is set, tags have already been initialised */
for (i = 0; i < nr_pages; i++, page++) {
- if (!test_and_set_bit(PG_mte_tagged, &page->flags))
+ if (!test_bit(PG_mte_tagged, &page->flags))
mte_sync_page_tags(page, old_pte, check_swap,
pte_is_tagged);
}
flush_tagged_addr_state();
}
-void release_thread(struct task_struct *dead_task)
-{
-}
-
void arch_release_task_struct(struct task_struct *tsk)
{
fpsimd_release_task(tsk);
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
+ sp -= prandom_u32_max(PAGE_SIZE);
return sp & ~0xf;
}
MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
{},
};
+ static const struct midr_range spectre_bhb_k11_list[] = {
+ MIDR_ALL_VERSIONS(MIDR_AMPERE1),
+ {},
+ };
static const struct midr_range spectre_bhb_k8_list[] = {
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
k = 32;
else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list))
k = 24;
+ else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k11_list))
+ k = 11;
else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list))
k = 8;
*
* The resulting 5 bits of entropy is seen in SP[8:4].
*/
- choose_random_kstack_offset(get_random_int() & 0x1FF);
+ choose_random_kstack_offset(get_random_u16() & 0x1FF);
}
static inline bool has_syscall_work(unsigned long flags)
{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
unsigned long size = vma->vm_end - vma->vm_start;
if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm))
if (!arg)
return -EINVAL;
+ if (strcmp(arg, "none") == 0) {
+ kvm_mode = KVM_MODE_NONE;
+ return 0;
+ }
+
+ if (!is_hyp_mode_available()) {
+ pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
+ return 0;
+ }
+
if (strcmp(arg, "protected") == 0) {
if (!is_kernel_in_hyp_mode())
kvm_mode = KVM_MODE_PROTECTED;
return 0;
}
- if (strcmp(arg, "none") == 0) {
- kvm_mode = KVM_MODE_NONE;
- return 0;
- }
-
return -EINVAL;
}
early_param("kvm-arm.mode", early_kvm_mode_cfg);
*
* Guest access to MDSCR_EL1 is trapped by the hypervisor and handled
* after we have restored the preserved value to the main context.
+ *
+ * When single-step is enabled by userspace, we tweak PSTATE.SS on every
+ * guest entry. Preserve PSTATE.SS so we can restore the original value
+ * for the vcpu after the single-step is disabled.
*/
static void save_guest_debug_regs(struct kvm_vcpu *vcpu)
{
trace_kvm_arm_set_dreg32("Saved MDSCR_EL1",
vcpu->arch.guest_debug_preserved.mdscr_el1);
+
+ vcpu->arch.guest_debug_preserved.pstate_ss =
+ (*vcpu_cpsr(vcpu) & DBG_SPSR_SS);
}
static void restore_guest_debug_regs(struct kvm_vcpu *vcpu)
trace_kvm_arm_set_dreg32("Restored MDSCR_EL1",
vcpu_read_sys_reg(vcpu, MDSCR_EL1));
+
+ if (vcpu->arch.guest_debug_preserved.pstate_ss)
+ *vcpu_cpsr(vcpu) |= DBG_SPSR_SS;
+ else
+ *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
}
/**
* debugging the system.
*/
if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
- *vcpu_cpsr(vcpu) |= DBG_SPSR_SS;
+ /*
+ * If the software step state at the last guest exit
+ * was Active-pending, we don't set DBG_SPSR_SS so
+ * that the state is maintained (to not run another
+ * single-step until the pending Software Step
+ * exception is taken).
+ */
+ if (!vcpu_get_flag(vcpu, DBG_SS_ACTIVE_PENDING))
+ *vcpu_cpsr(vcpu) |= DBG_SPSR_SS;
+ else
+ *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
+
mdscr = vcpu_read_sys_reg(vcpu, MDSCR_EL1);
mdscr |= DBG_MDSCR_SS;
vcpu_write_sys_reg(vcpu, mdscr, MDSCR_EL1);
* Restore the guest's debug registers if we were using them.
*/
if (vcpu->guest_debug || kvm_vcpu_os_lock_enabled(vcpu)) {
+ if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
+ if (!(*vcpu_cpsr(vcpu) & DBG_SPSR_SS))
+ /*
+ * Mark the vcpu as ACTIVE_PENDING
+ * until Software Step exception is taken.
+ */
+ vcpu_set_flag(vcpu, DBG_SS_ACTIVE_PENDING);
+ }
+
restore_guest_debug_regs(vcpu);
/*
} else {
/* If not enabled clear all flags */
vcpu->guest_debug = 0;
+ vcpu_clear_flag(vcpu, DBG_SS_ACTIVE_PENDING);
}
out:
run->debug.arch.hsr_high = upper_32_bits(esr);
run->flags = KVM_DEBUG_ARCH_HSR_HIGH_VALID;
- if (ESR_ELx_EC(esr) == ESR_ELx_EC_WATCHPT_LOW)
+ switch (ESR_ELx_EC(esr)) {
+ case ESR_ELx_EC_WATCHPT_LOW:
run->debug.arch.far = vcpu->arch.fault.far_el2;
+ break;
+ case ESR_ELx_EC_SOFTSTP_LOW:
+ vcpu_clear_flag(vcpu, DBG_SS_ACTIVE_PENDING);
+ break;
+ }
return 0;
}
}
}
-/* Restore VGICv3 state on non_VEH systems */
+/* Restore VGICv3 state on non-VHE systems */
static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
{
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
}
/* Read a sanitised cpufeature ID register by sys_reg_desc */
-static u64 read_id_reg(const struct kvm_vcpu *vcpu,
- struct sys_reg_desc const *r, bool raz)
+static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r)
{
u32 id = reg_to_encoding(r);
u64 val;
- if (raz)
+ if (sysreg_visible_as_raz(vcpu, r))
return 0;
val = read_sanitised_ftr_reg(id);
return 0;
}
-/* cpufeature ID register access trap handlers */
-
-static bool __access_id_reg(struct kvm_vcpu *vcpu,
- struct sys_reg_params *p,
- const struct sys_reg_desc *r,
- bool raz)
+static unsigned int aa32_id_visibility(const struct kvm_vcpu *vcpu,
+ const struct sys_reg_desc *r)
{
- if (p->is_write)
- return write_to_read_only(vcpu, p, r);
+ /*
+ * AArch32 ID registers are UNKNOWN if AArch32 isn't implemented at any
+ * EL. Promote to RAZ/WI in order to guarantee consistency between
+ * systems.
+ */
+ if (!kvm_supports_32bit_el0())
+ return REG_RAZ | REG_USER_WI;
- p->regval = read_id_reg(vcpu, r, raz);
- return true;
+ return id_visibility(vcpu, r);
}
+static unsigned int raz_visibility(const struct kvm_vcpu *vcpu,
+ const struct sys_reg_desc *r)
+{
+ return REG_RAZ;
+}
+
+/* cpufeature ID register access trap handlers */
+
static bool access_id_reg(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
- bool raz = sysreg_visible_as_raz(vcpu, r);
-
- return __access_id_reg(vcpu, p, r, raz);
-}
+ if (p->is_write)
+ return write_to_read_only(vcpu, p, r);
-static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
- struct sys_reg_params *p,
- const struct sys_reg_desc *r)
-{
- return __access_id_reg(vcpu, p, r, true);
+ p->regval = read_id_reg(vcpu, r);
+ return true;
}
/* Visibility overrides for SVE-specific control registers */
return -EINVAL;
/* We can only differ with CSV[23], and anything else is an error */
- val ^= read_id_reg(vcpu, rd, false);
- val &= ~((0xFUL << ID_AA64PFR0_EL1_CSV2_SHIFT) |
- (0xFUL << ID_AA64PFR0_EL1_CSV3_SHIFT));
+ val ^= read_id_reg(vcpu, rd);
+ val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) |
+ ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3));
if (val)
return -EINVAL;
* are stored, and for set_id_reg() we don't allow the effective value
* to be changed.
*/
-static int __get_id_reg(const struct kvm_vcpu *vcpu,
- const struct sys_reg_desc *rd, u64 *val,
- bool raz)
-{
- *val = read_id_reg(vcpu, rd, raz);
- return 0;
-}
-
-static int __set_id_reg(const struct kvm_vcpu *vcpu,
- const struct sys_reg_desc *rd, u64 val,
- bool raz)
-{
- /* This is what we mean by invariant: you can't change it. */
- if (val != read_id_reg(vcpu, rd, raz))
- return -EINVAL;
-
- return 0;
-}
-
static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
u64 *val)
{
- bool raz = sysreg_visible_as_raz(vcpu, rd);
-
- return __get_id_reg(vcpu, rd, val, raz);
+ *val = read_id_reg(vcpu, rd);
+ return 0;
}
static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
u64 val)
{
- bool raz = sysreg_visible_as_raz(vcpu, rd);
-
- return __set_id_reg(vcpu, rd, val, raz);
-}
+ /* This is what we mean by invariant: you can't change it. */
+ if (val != read_id_reg(vcpu, rd))
+ return -EINVAL;
-static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
- u64 val)
-{
- return __set_id_reg(vcpu, rd, val, true);
+ return 0;
}
static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
.visibility = id_visibility, \
}
+/* sys_reg_desc initialiser for known cpufeature ID registers */
+#define AA32_ID_SANITISED(name) { \
+ SYS_DESC(SYS_##name), \
+ .access = access_id_reg, \
+ .get_user = get_id_reg, \
+ .set_user = set_id_reg, \
+ .visibility = aa32_id_visibility, \
+}
+
/*
* sys_reg_desc initialiser for architecturally unallocated cpufeature ID
* register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
*/
#define ID_UNALLOCATED(crm, op2) { \
Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
- .access = access_raz_id_reg, \
- .get_user = get_raz_reg, \
- .set_user = set_raz_id_reg, \
+ .access = access_id_reg, \
+ .get_user = get_id_reg, \
+ .set_user = set_id_reg, \
+ .visibility = raz_visibility \
}
/*
*/
#define ID_HIDDEN(name) { \
SYS_DESC(SYS_##name), \
- .access = access_raz_id_reg, \
- .get_user = get_raz_reg, \
- .set_user = set_raz_id_reg, \
+ .access = access_id_reg, \
+ .get_user = get_id_reg, \
+ .set_user = set_id_reg, \
+ .visibility = raz_visibility, \
}
/*
/* AArch64 mappings of the AArch32 ID registers */
/* CRm=1 */
- ID_SANITISED(ID_PFR0_EL1),
- ID_SANITISED(ID_PFR1_EL1),
- ID_SANITISED(ID_DFR0_EL1),
+ AA32_ID_SANITISED(ID_PFR0_EL1),
+ AA32_ID_SANITISED(ID_PFR1_EL1),
+ AA32_ID_SANITISED(ID_DFR0_EL1),
ID_HIDDEN(ID_AFR0_EL1),
- ID_SANITISED(ID_MMFR0_EL1),
- ID_SANITISED(ID_MMFR1_EL1),
- ID_SANITISED(ID_MMFR2_EL1),
- ID_SANITISED(ID_MMFR3_EL1),
+ AA32_ID_SANITISED(ID_MMFR0_EL1),
+ AA32_ID_SANITISED(ID_MMFR1_EL1),
+ AA32_ID_SANITISED(ID_MMFR2_EL1),
+ AA32_ID_SANITISED(ID_MMFR3_EL1),
/* CRm=2 */
- ID_SANITISED(ID_ISAR0_EL1),
- ID_SANITISED(ID_ISAR1_EL1),
- ID_SANITISED(ID_ISAR2_EL1),
- ID_SANITISED(ID_ISAR3_EL1),
- ID_SANITISED(ID_ISAR4_EL1),
- ID_SANITISED(ID_ISAR5_EL1),
- ID_SANITISED(ID_MMFR4_EL1),
- ID_SANITISED(ID_ISAR6_EL1),
+ AA32_ID_SANITISED(ID_ISAR0_EL1),
+ AA32_ID_SANITISED(ID_ISAR1_EL1),
+ AA32_ID_SANITISED(ID_ISAR2_EL1),
+ AA32_ID_SANITISED(ID_ISAR3_EL1),
+ AA32_ID_SANITISED(ID_ISAR4_EL1),
+ AA32_ID_SANITISED(ID_ISAR5_EL1),
+ AA32_ID_SANITISED(ID_MMFR4_EL1),
+ AA32_ID_SANITISED(ID_ISAR6_EL1),
/* CRm=3 */
- ID_SANITISED(MVFR0_EL1),
- ID_SANITISED(MVFR1_EL1),
- ID_SANITISED(MVFR2_EL1),
+ AA32_ID_SANITISED(MVFR0_EL1),
+ AA32_ID_SANITISED(MVFR1_EL1),
+ AA32_ID_SANITISED(MVFR2_EL1),
ID_UNALLOCATED(3,3),
- ID_SANITISED(ID_PFR2_EL1),
+ AA32_ID_SANITISED(ID_PFR2_EL1),
ID_HIDDEN(ID_DFR1_EL1),
- ID_SANITISED(ID_MMFR5_EL1),
+ AA32_ID_SANITISED(ID_MMFR5_EL1),
ID_UNALLOCATED(3,7),
/* AArch64 ID registers */
if (!r)
return -ENOENT;
+ if (sysreg_user_write_ignore(vcpu, r))
+ return 0;
+
if (r->set_user) {
ret = (r->set_user)(vcpu, r, val);
} else {
#define REG_HIDDEN (1 << 0) /* hidden from userspace and guest */
#define REG_RAZ (1 << 1) /* RAZ from userspace and guest */
+#define REG_USER_WI (1 << 2) /* WI from userspace only */
static __printf(2, 3)
inline void print_sys_reg_msg(const struct sys_reg_params *p,
__vcpu_sys_reg(vcpu, r->reg) = r->val;
}
-static inline bool sysreg_hidden(const struct kvm_vcpu *vcpu,
- const struct sys_reg_desc *r)
+static inline unsigned int sysreg_visibility(const struct kvm_vcpu *vcpu,
+ const struct sys_reg_desc *r)
{
if (likely(!r->visibility))
- return false;
+ return 0;
- return r->visibility(vcpu, r) & REG_HIDDEN;
+ return r->visibility(vcpu, r);
+}
+
+static inline bool sysreg_hidden(const struct kvm_vcpu *vcpu,
+ const struct sys_reg_desc *r)
+{
+ return sysreg_visibility(vcpu, r) & REG_HIDDEN;
}
static inline bool sysreg_visible_as_raz(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r)
{
- if (likely(!r->visibility))
- return false;
+ return sysreg_visibility(vcpu, r) & REG_RAZ;
+}
- return r->visibility(vcpu, r) & REG_RAZ;
+static inline bool sysreg_user_write_ignore(const struct kvm_vcpu *vcpu,
+ const struct sys_reg_desc *r)
+{
+ return sysreg_visibility(vcpu, r) & REG_USER_WI;
}
static inline int cmp_sys_reg(const struct sys_reg_desc *i1,
struct its_ite *ite;
for_each_lpi_its(device, ite, its) {
- if (!ite->collection || coll != ite->collection)
+ if (ite->collection != coll)
continue;
update_affinity_ite(kvm, ite);
{
VM_BUG_ON(!is_migration_entry(entry) && !is_hwpoison_entry(entry));
- return page_folio(pfn_to_page(swp_offset(entry)));
+ return page_folio(pfn_to_page(swp_offset_pfn(entry)));
}
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
if (!tags)
return false;
- mte_restore_page_tags(page_address(page), tags);
+ /*
+ * Test PG_mte_tagged again in case it was racing with another
+ * set_pte_at().
+ */
+ if (!test_and_set_bit(PG_mte_tagged, &page->flags))
+ mte_restore_page_tags(page_address(page), tags);
return true;
}
Sysreg SCTLR_EL1 3 0 1 0 0
Field 63 TIDCP
-Field 62 SPINMASK
+Field 62 SPINTMASK
Field 61 NMI
Field 60 EnTP2
Res0 59:58
select KMAP_LOCAL
default y
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
default "11"
/* Forward declaration, a strange C thing */
struct task_struct;
-/* Free all resources held by a thread. */
-static inline void release_thread(struct task_struct *dead_task)
-{
-}
-
/* Prepare to copy thread state - unlazy all lazy status */
#define prepare_to_copy(tsk) do { } while (0)
#define KSTK_EIP(tsk) (pt_elr(task_pt_regs(tsk)))
#define KSTK_ESP(tsk) (pt_psp(task_pt_regs(tsk)))
-/* Free all resources held by a thread; defined in process.c */
-extern void release_thread(struct task_struct *dead_task);
-
-/* Get wait channel for task P. */
extern unsigned long __get_wchan(struct task_struct *p);
/* The following stuff is pretty HEXAGON specific. */
return 0;
}
-/*
- * Release any architecture-specific resources locked by thread
- */
-void release_thread(struct task_struct *dead_task)
-{
-}
-
/*
* Some archs flush debug and FPU info here
*/
Say Y here to enable support for IBM EXA Cyclone time source.
If you're unsure, answer N.
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "MAX_ORDER (11 - 17)" if !HUGETLB_PAGE
range 11 17 if !HUGETLB_PAGE
default "17" if HUGETLB_PAGE
CONFIG_INET=y
# CONFIG_IPV6 is not set
CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=m
CONFIG_ATA=m
CONFIG_NFSD=m
CONFIG_NFSD_V4=y
CONFIG_CIFS=m
-CONFIG_CIFS_STATS=y
CONFIG_CIFS_XATTR=y
CONFIG_CIFS_POSIX=y
CONFIG_NLS_CODEPAGE_437=y
CONFIG_CONNECTOR=y
# CONFIG_PNP_DEBUG_MESSAGES is not set
CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
CONFIG_SGI_XP=m
# CONFIG_HW_RANDOM is not set
CONFIG_RTC_CLASS=y
CONFIG_RTC_DRV_EFI=y
-CONFIG_RAW_DRIVER=m
CONFIG_HPET=y
CONFIG_AGP=m
CONFIG_AGP_I460=m
CONFIG_SYN_COOKIES=y
# CONFIG_IPV6 is not set
CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
CONFIG_ATA=y
-CONFIG_BLK_DEV_IDECD=y
CONFIG_ATA_GENERIC=y
CONFIG_PATA_CMD64X=y
CONFIG_ATA_PIIX=y
# CONFIG_HW_RANDOM is not set
CONFIG_RTC_CLASS=y
CONFIG_RTC_DRV_EFI=y
-CONFIG_RAW_DRIVER=m
CONFIG_HPET=y
CONFIG_AGP=m
CONFIG_AGP_I460=m
CONFIG_SYN_COOKIES=y
# CONFIG_IPV6 is not set
CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
CONFIG_ATA=y
# CONFIG_HW_RANDOM is not set
CONFIG_RTC_CLASS=y
CONFIG_RTC_DRV_EFI=y
-CONFIG_RAW_DRIVER=m
CONFIG_HPET=y
CONFIG_AGP=m
CONFIG_AGP_I460=m
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=y
-CONFIG_CHR_DEV_OSST=y
CONFIG_BLK_DEV_SR=y
CONFIG_CHR_DEV_SG=y
CONFIG_SCSI_CONSTANTS=y
struct mm_struct;
struct task_struct;
-/*
- * Free all resources held by a thread. This is called after the
- * parent of DEAD_TASK has collected the exit status of the task via
- * wait().
- */
-#define release_thread(dead_task)
-
/* Get wait channel for task P. */
extern unsigned long __get_wchan (struct task_struct *p);
#define SECTION_SIZE_BITS (30)
#define MAX_PHYSMEM_BITS (50)
-#ifdef CONFIG_FORCE_MAX_ZONEORDER
-#if ((CONFIG_FORCE_MAX_ZONEORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS)
+#ifdef CONFIG_ARCH_FORCE_MAX_ORDER
+#if ((CONFIG_ARCH_FORCE_MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS)
#undef SECTION_SIZE_BITS
-#define SECTION_SIZE_BITS (CONFIG_FORCE_MAX_ZONEORDER - 1 + PAGE_SHIFT)
+#define SECTION_SIZE_BITS (CONFIG_ARCH_FORCE_MAX_ORDER - 1 + PAGE_SHIFT)
#endif
#endif
p->parent = p->real_parent = p->group_leader = p;
INIT_LIST_HEAD(&p->children);
INIT_LIST_HEAD(&p->sibling);
- strncpy(p->comm, type, sizeof(p->comm)-1);
+ strscpy(p->comm, type, sizeof(p->comm)-1);
}
/* Caller prevents this from being called after init */
ia64_patch_vtop((u64) __start___vtop_patchlist, (u64) __end___vtop_patchlist);
*cmdline_p = __va(ia64_boot_param->command_line);
- strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE);
+ strscpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE);
efi_init();
io_port_init();
force_successful_syscall_return();
return addr;
}
+
+asmlinkage long
+ia64_clock_getres(const clockid_t which_clock, struct __kernel_timespec __user *tp)
+{
+ /*
+ * ia64's clock_gettime() syscall is implemented as a vdso call
+ * fsys_clock_gettime(). Currently it handles only
+ * CLOCK_REALTIME and CLOCK_MONOTONIC. Both are based on
+ * 'ar.itc' counter which gets incremented at a constant
+ * frequency. It's usually 400MHz, ~2.5x times slower than CPU
+ * clock frequency. Which is almost a 1ns hrtimer, but not quite.
+ *
+ * Let's special-case these timers to report correct precision
+ * based on ITC frequency and not HZ frequency for supported
+ * clocks.
+ */
+ switch (which_clock) {
+ case CLOCK_REALTIME:
+ case CLOCK_MONOTONIC:
+ s64 tick_ns = DIV_ROUND_UP(NSEC_PER_SEC, local_cpu_data->itc_freq);
+ struct timespec64 rtn_tp = ns_to_timespec64(tick_ns);
+ return put_timespec64(&rtn_tp, tp);
+ }
+
+ return sys_clock_getres(which_clock, tp);
+}
228 common timer_delete sys_timer_delete
229 common clock_settime sys_clock_settime
230 common clock_gettime sys_clock_gettime
-231 common clock_getres sys_clock_getres
+231 common clock_getres ia64_clock_getres
232 common clock_nanosleep sys_clock_nanosleep
233 common fstatfs64 sys_fstatfs64
234 common statfs64 sys_statfs64
obj-y += kernel/
obj-y += mm/
+obj-y += net/
obj-y += vdso/
# for cleaning
select ARCH_USE_BUILTIN_BSWAP
select ARCH_USE_CMPXCHG_LOCKREF
select ARCH_USE_QUEUED_RWLOCKS
+ select ARCH_USE_QUEUED_SPINLOCKS
select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
select ARCH_WANT_LD_ORPHAN_WARN
select ARCH_WANTS_NO_INSTR
select GENERIC_CPU_AUTOPROBE
select GENERIC_ENTRY
select GENERIC_GETTIMEOFDAY
+ select GENERIC_IOREMAP if !ARCH_IOREMAP
select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
select GENERIC_LIB_CMPDI2
select GENERIC_LIB_LSHRDI3
select GENERIC_LIB_UCMPDI2
+ select GENERIC_LIB_DEVMEM_IS_ALLOWED
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
select HAVE_CONTEXT_TRACKING_USER
select HAVE_DEBUG_STACKOVERFLOW
select HAVE_DMA_CONTIGUOUS
+ select HAVE_EBPF_JIT
select HAVE_EXIT_THREAD
select HAVE_FAST_GUP
select HAVE_GENERIC_VDSO
select HAVE_NMI
select HAVE_PCI
select HAVE_PERF_EVENTS
+ select HAVE_PERF_REGS
+ select HAVE_PERF_USER_STACK_DUMP
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RSEQ
select HAVE_SETUP_PER_CPU_AREA if NUMA
bool
default y
+config GENERIC_BUG
+ def_bool y
+ depends on BUG
+
+config GENERIC_BUG_RELATIVE_POINTERS
+ def_bool y
+ depends on GENERIC_BUG
+
config GENERIC_CALIBRATE_DELAY
def_bool y
bool
default y
-# MACH_LOONGSON32 and MACH_LOONGSON64 are delibrately carried over from the
+# MACH_LOONGSON32 and MACH_LOONGSON64 are deliberately carried over from the
# MIPS Loongson code, to preserve Loongson-specific code paths in drivers that
# are shared between architectures, and specifically expecting the symbols.
config MACH_LOONGSON32
config MACH_LOONGSON64
def_bool 64BIT
+config FIX_EARLYCON_MEM
+ def_bool y
+
config PAGE_SIZE_4KB
bool
bool
default y
+config AS_HAS_EXPLICIT_RELOCS
+ def_bool $(as-instr,x:pcalau12i \$t0$(comma)%pc_hi20(x))
+
menu "Kernel type and options"
source "kernel/Kconfig.hz"
default "6"
depends on NUMA
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
range 14 64 if PAGE_SIZE_64KB
default "14" if PAGE_SIZE_64KB
The page size is not necessarily 4KB. Keep this in mind
when choosing a value for this option.
+config ARCH_IOREMAP
+ bool "Enable LoongArch DMW-based ioremap()"
+ help
+ We use generic TLB-based ioremap() by default since it has page
+ protection support. However, you can enable LoongArch DMW-based
+ ioremap() for better performance.
+
+config KEXEC
+ bool "Kexec system call"
+ select KEXEC_CORE
+ help
+ kexec is a system call that implements the ability to shutdown your
+ current kernel, and to start another kernel. It is like a reboot
+ but it is independent of the system firmware. And like a reboot
+ you can start any kernel with it, not just Linux.
+
+ The name comes from the similarity to the exec system call.
+
+config CRASH_DUMP
+ bool "Build kdump crash kernel"
+ help
+ Generate crash dump after being started by kexec. This should
+ be normally only set in special crash dump kernels which are
+ loaded in the main kernel with kexec-tools into a specially
+ reserved region and then later executed after a crash by
+ kdump/kexec.
+
+ For more details see Documentation/admin-guide/kdump/kdump.rst
+
+config PHYSICAL_START
+ hex "Physical address where the kernel is loaded"
+ default "0x90000000a0000000"
+ depends on CRASH_DUMP
+ help
+ This gives the XKPRANGE address where the kernel is loaded.
+ If you plan to use kernel for capturing the crash dump change
+ this value to start of the reserved region (the "X" value as
+ specified in the "crashkernel=YM@XM" command line boot parameter
+ passed to the panic-ed kernel).
+
config SECCOMP
bool "Enable seccomp to safely compute untrusted bytecode"
depends on PROC_FS
cflags-y += -G0 -pipe -msoft-float
LDFLAGS_vmlinux += -G0 -static -n -nostdlib
+
+# When the assembler supports explicit relocation hint, we must use it.
+# GCC may have -mexplicit-relocs off by default if it was built with an old
+# assembler, so we force it via an option.
+#
+# When the assembler does not supports explicit relocation hint, we can't use
+# it. Disable it if the compiler supports it.
+#
+# If you've seen "unknown reloc hint" message building the kernel and you are
+# now wondering why "-mexplicit-relocs" is not wrapped with cc-option: the
+# combination of a "new" assembler and "old" compiler is not supported. Either
+# upgrade the compiler or downgrade the assembler.
+ifdef CONFIG_AS_HAS_EXPLICIT_RELOCS
+cflags-y += -mexplicit-relocs
+KBUILD_CFLAGS_KERNEL += -mdirect-extern-access
+else
+cflags-y += $(call cc-option,-mno-explicit-relocs)
KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel
KBUILD_CFLAGS_KERNEL += -Wa,-mla-global-with-pcrel
KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs
KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs
+endif
cflags-y += -ffreestanding
cflags-y += $(call cc-option, -mno-check-zero-division)
+ifndef CONFIG_PHYSICAL_START
load-y = 0x9000000000200000
+else
+load-y = $(CONFIG_PHYSICAL_START)
+endif
bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y)
drivers-$(CONFIG_PCI) += arch/loongarch/pci/
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_BPF_SYSCALL=y
+CONFIG_BPF_JIT=y
CONFIG_PREEMPT=y
CONFIG_BSD_PROCESS_ACCT=y
CONFIG_BSD_PROCESS_ACCT_V3=y
CONFIG_HOTPLUG_CPU=y
CONFIG_NR_CPUS=64
CONFIG_NUMA=y
+CONFIG_KEXEC=y
CONFIG_PAGE_SIZE_16KB=y
CONFIG_HZ_250=y
CONFIG_ACPI=y
CONFIG_ACPI_IPMI=m
CONFIG_ACPI_PCI_SLOT=y
CONFIG_ACPI_HOTPLUG_MEMORY=y
+CONFIG_EFI_ZBOOT=y
CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER=y
CONFIG_EFI_CAPSULE_LOADER=m
CONFIG_EFI_TEST=m
CONFIG_MODVERSIONS=y
CONFIG_BLK_DEV_THROTTLING=y
CONFIG_PARTITION_ADVANCED=y
+CONFIG_BSD_DISKLABEL=y
+CONFIG_UNIXWARE_DISKLABEL=y
CONFIG_IOSCHED_BFQ=y
CONFIG_BFQ_GROUP_IOSCHED=y
CONFIG_BINFMT_MISC=m
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
CONFIG_XFRM_USER=y
CONFIG_NET_KEY=y
+CONFIG_XDP_SOCKETS=y
CONFIG_INET=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_PNP_BOOTP=y
CONFIG_IP_PNP_RARP=y
CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
CONFIG_IP_MROUTE=y
CONFIG_INET_ESP=m
CONFIG_INET_UDP_DIAG=y
CONFIG_TCP_CONG_BBR=m
CONFIG_IPV6_ROUTER_PREF=y
CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_INET6_ESP=m
CONFIG_IPV6_MROUTE=y
CONFIG_NETWORK_PHY_TIMESTAMPING=y
CONFIG_NETFILTER=y
CONFIG_NF_CONNTRACK_AMANDA=m
CONFIG_NF_CONNTRACK_FTP=m
CONFIG_NF_CONNTRACK_NETBIOS_NS=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
CONFIG_NF_CONNTRACK_TFTP=m
CONFIG_NF_CT_NETLINK=m
CONFIG_NF_TABLES=m
-CONFIG_NFT_COUNTER=m
CONFIG_NFT_CONNLIMIT=m
CONFIG_NFT_LOG=m
CONFIG_NFT_LIMIT=m
CONFIG_NFT_DUP_IPV4=m
CONFIG_NFT_FIB_IPV4=m
CONFIG_NF_TABLES_ARP=y
-CONFIG_NF_LOG_ARP=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_MATCH_AH=m
CONFIG_IP_NF_MATCH_ECN=m
CONFIG_IP_SCTP=m
CONFIG_RDS=y
CONFIG_L2TP=m
+CONFIG_L2TP_V3=y
+CONFIG_L2TP_IP=m
+CONFIG_L2TP_ETH=m
CONFIG_BRIDGE=m
CONFIG_VLAN_8021Q=m
CONFIG_VLAN_8021Q_GVRP=y
CONFIG_VLAN_8021Q_MVRP=y
+CONFIG_LLC2=m
CONFIG_NET_SCHED=y
CONFIG_NET_SCH_HTB=m
CONFIG_NET_SCH_PRIO=m
CONFIG_VIRTIO_VSOCKETS=m
CONFIG_NETLINK_DIAG=y
CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BPF_STREAM_PARSER=y
CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=m
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=m
+CONFIG_BT_HS=y
CONFIG_BT_HCIBTUSB=m
-# CONFIG_BT_HCIBTUSB_BCM is not set
+CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y
+CONFIG_BT_HCIBTUSB_MTK=y
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_BCSP=y
+CONFIG_BT_HCIUART_ATH3K=y
+CONFIG_BT_HCIUART_INTEL=y
+CONFIG_BT_HCIUART_AG6XX=y
+CONFIG_BT_HCIBCM203X=m
+CONFIG_BT_HCIBPA10X=m
+CONFIG_BT_HCIBFUSB=m
+CONFIG_BT_HCIDTL1=m
+CONFIG_BT_HCIBT3C=m
+CONFIG_BT_HCIBLUECARD=m
+CONFIG_BT_HCIVHCI=m
+CONFIG_BT_MRVL=m
+CONFIG_BT_ATH3K=m
+CONFIG_BT_VIRTIO=m
CONFIG_CFG80211=m
CONFIG_CFG80211_WEXT=y
CONFIG_MAC80211=m
CONFIG_ZRAM=m
CONFIG_ZRAM_DEF_COMP_ZSTD=y
CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=y
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_SIZE=8192
CONFIG_PPP_MPPE=m
CONFIG_PPP_MULTILINK=y
CONFIG_PPPOE=m
+CONFIG_PPTP=m
CONFIG_PPPOL2TP=m
CONFIG_PPP_ASYNC=m
CONFIG_PPP_SYNC_TTY=m
CONFIG_IWLWIFI=m
CONFIG_IWLDVM=m
CONFIG_IWLMVM=m
-CONFIG_IWLWIFI_BCAST_FILTERING=y
CONFIG_HOSTAP=m
CONFIG_MT7601U=m
CONFIG_RT2X00=m
CONFIG_RTL8192CU=m
# CONFIG_RTLWIFI_DEBUG is not set
CONFIG_RTL8XXXU=m
+CONFIG_RTW88=m
+CONFIG_RTW88_8822BE=m
+CONFIG_RTW88_8822CE=m
+CONFIG_RTW88_8723DE=m
+CONFIG_RTW88_8821CE=m
+CONFIG_RTW89=m
+CONFIG_RTW89_8852AE=m
+CONFIG_RTW89_8852CE=m
CONFIG_ZD1211RW=m
CONFIG_USB_NET_RNDIS_WLAN=m
CONFIG_INPUT_MOUSEDEV=y
CONFIG_USB_SERIAL_PL2303=m
CONFIG_USB_SERIAL_OPTION=m
CONFIG_USB_GADGET=y
+CONFIG_TYPEC=m
+CONFIG_TYPEC_TCPM=m
+CONFIG_TYPEC_TCPCI=m
+CONFIG_TYPEC_UCSI=m
+CONFIG_UCSI_ACPI=m
CONFIG_INFINIBAND=m
CONFIG_RTC_CLASS=y
CONFIG_RTC_DRV_EFI=y
CONFIG_COMEDI_NI_PCIMIO=m
CONFIG_STAGING=y
CONFIG_R8188EU=m
-# CONFIG_88EU_AP_MODE is not set
CONFIG_PM_DEVFREQ=y
CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y
CONFIG_DEVFREQ_GOV_PERFORMANCE=y
CONFIG_CRYPTO_CHACHA20POLY1305=m
CONFIG_CRYPTO_HMAC=y
CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_TGR192=m
CONFIG_CRYPTO_WP512=m
CONFIG_CRYPTO_ANUBIS=m
CONFIG_CRYPTO_BLOWFISH=m
CONFIG_CRYPTO_CAST5=m
CONFIG_CRYPTO_CAST6=m
CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
CONFIG_CRYPTO_SEED=m
CONFIG_CRYPTO_SERPENT=m
CONFIG_CRYPTO_TEA=m
# SPDX-License-Identifier: GPL-2.0
generic-y += dma-contiguous.h
generic-y += export.h
+generic-y += mcs_spinlock.h
generic-y += parport.h
generic-y += early_ioremap.h
generic-y += qrwlock.h
-generic-y += qrwlock_types.h
-generic-y += spinlock.h
-generic-y += spinlock_types.h
+generic-y += qspinlock.h
generic-y += rwsem.h
generic-y += segment.h
generic-y += user.h
extern struct loongson_board_info b_info;
extern struct loongson_system_configuration loongson_sysconf;
+static inline bool io_master(int cpu)
+{
+ return test_bit(cpu, &loongson_sysconf.cores_io_master);
+}
+
#endif /* _ASM_BOOTINFO_H */
#ifndef __ASM_BUG_H
#define __ASM_BUG_H
-#include <linux/compiler.h>
+#include <asm/break.h>
+#include <linux/stringify.h>
+
+#ifndef CONFIG_DEBUG_BUGVERBOSE
+#define _BUGVERBOSE_LOCATION(file, line)
+#else
+#define __BUGVERBOSE_LOCATION(file, line) \
+ .pushsection .rodata.str, "aMS", @progbits, 1; \
+ 10002: .string file; \
+ .popsection; \
+ \
+ .long 10002b - .; \
+ .short line;
+#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
+#endif
-#ifdef CONFIG_BUG
+#ifndef CONFIG_GENERIC_BUG
+#define __BUG_ENTRY(flags)
+#else
+#define __BUG_ENTRY(flags) \
+ .pushsection __bug_table, "aw"; \
+ .align 2; \
+ 10000: .long 10001f - .; \
+ _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
+ .short flags; \
+ .popsection; \
+ 10001:
+#endif
-#include <asm/break.h>
+#define ASM_BUG_FLAGS(flags) \
+ __BUG_ENTRY(flags) \
+ break BRK_BUG
-static inline void __noreturn BUG(void)
-{
- __asm__ __volatile__("break %0" : : "i" (BRK_BUG));
- unreachable();
-}
+#define ASM_BUG() ASM_BUG_FLAGS(0)
-#define HAVE_ARCH_BUG
+#define __BUG_FLAGS(flags) \
+ asm_inline volatile (__stringify(ASM_BUG_FLAGS(flags)));
-#endif
+#define __WARN_FLAGS(flags) \
+do { \
+ instrumentation_begin(); \
+ __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \
+ instrumentation_end(); \
+} while (0)
+
+#define BUG() \
+do { \
+ instrumentation_begin(); \
+ __BUG_FLAGS(0); \
+ unreachable(); \
+} while (0)
+
+#define HAVE_ARCH_BUG
#include <asm-generic/bug.h>
#define _ASM_CACHEFLUSH_H
#include <linux/mm.h>
-#include <asm/cpu-features.h>
+#include <asm/cpu-info.h>
#include <asm/cacheops.h>
-extern void local_flush_icache_range(unsigned long start, unsigned long end);
+static inline bool cache_present(struct cache_desc *cdesc)
+{
+ return cdesc->flags & CACHE_PRESENT;
+}
+
+static inline bool cache_private(struct cache_desc *cdesc)
+{
+ return cdesc->flags & CACHE_PRIVATE;
+}
+
+static inline bool cache_inclusive(struct cache_desc *cdesc)
+{
+ return cdesc->flags & CACHE_INCLUSIVE;
+}
+
+static inline unsigned int cpu_last_level_cache_line_size(void)
+{
+ int cache_present = boot_cpu_data.cache_leaves_present;
+
+ return boot_cpu_data.cache_leaves[cache_present - 1].linesz;
+}
+
+asmlinkage void __flush_cache_all(void);
+void local_flush_icache_range(unsigned long start, unsigned long end);
#define flush_icache_range local_flush_icache_range
#define flush_icache_user_range local_flush_icache_range
: \
: "i" (op), "ZC" (*(unsigned char *)(addr)))
-static inline void flush_icache_line_indexed(unsigned long addr)
-{
- cache_op(Index_Invalidate_I, addr);
-}
-
-static inline void flush_dcache_line_indexed(unsigned long addr)
-{
- cache_op(Index_Writeback_Inv_D, addr);
-}
-
-static inline void flush_vcache_line_indexed(unsigned long addr)
-{
- cache_op(Index_Writeback_Inv_V, addr);
-}
-
-static inline void flush_scache_line_indexed(unsigned long addr)
-{
- cache_op(Index_Writeback_Inv_S, addr);
-}
-
-static inline void flush_icache_line(unsigned long addr)
-{
- cache_op(Hit_Invalidate_I, addr);
-}
-
-static inline void flush_dcache_line(unsigned long addr)
-{
- cache_op(Hit_Writeback_Inv_D, addr);
-}
-
-static inline void flush_vcache_line(unsigned long addr)
-{
- cache_op(Hit_Writeback_Inv_V, addr);
-}
-
-static inline void flush_scache_line(unsigned long addr)
+static inline void flush_cache_line(int leaf, unsigned long addr)
{
- cache_op(Hit_Writeback_Inv_S, addr);
+ switch (leaf) {
+ case Cache_LEAF0:
+ cache_op(Index_Writeback_Inv_LEAF0, addr);
+ break;
+ case Cache_LEAF1:
+ cache_op(Index_Writeback_Inv_LEAF1, addr);
+ break;
+ case Cache_LEAF2:
+ cache_op(Index_Writeback_Inv_LEAF2, addr);
+ break;
+ case Cache_LEAF3:
+ cache_op(Index_Writeback_Inv_LEAF3, addr);
+ break;
+ case Cache_LEAF4:
+ cache_op(Index_Writeback_Inv_LEAF4, addr);
+ break;
+ case Cache_LEAF5:
+ cache_op(Index_Writeback_Inv_LEAF5, addr);
+ break;
+ default:
+ break;
+ }
}
#include <asm-generic/cacheflush.h>
#define __ASM_CACHEOPS_H
/*
- * Most cache ops are split into a 2 bit field identifying the cache, and a 3
+ * Most cache ops are split into a 3 bit field identifying the cache, and a 2
* bit field identifying the cache operation.
*/
-#define CacheOp_Cache 0x03
-#define CacheOp_Op 0x1c
+#define CacheOp_Cache 0x07
+#define CacheOp_Op 0x18
-#define Cache_I 0x00
-#define Cache_D 0x01
-#define Cache_V 0x02
-#define Cache_S 0x03
+#define Cache_LEAF0 0x00
+#define Cache_LEAF1 0x01
+#define Cache_LEAF2 0x02
+#define Cache_LEAF3 0x03
+#define Cache_LEAF4 0x04
+#define Cache_LEAF5 0x05
#define Index_Invalidate 0x08
#define Index_Writeback_Inv 0x08
#define Hit_Writeback_Inv 0x10
#define CacheOp_User_Defined 0x18
-#define Index_Invalidate_I (Cache_I | Index_Invalidate)
-#define Index_Writeback_Inv_D (Cache_D | Index_Writeback_Inv)
-#define Index_Writeback_Inv_V (Cache_V | Index_Writeback_Inv)
-#define Index_Writeback_Inv_S (Cache_S | Index_Writeback_Inv)
-#define Hit_Invalidate_I (Cache_I | Hit_Invalidate)
-#define Hit_Writeback_Inv_D (Cache_D | Hit_Writeback_Inv)
-#define Hit_Writeback_Inv_V (Cache_V | Hit_Writeback_Inv)
-#define Hit_Writeback_Inv_S (Cache_S | Hit_Writeback_Inv)
+#define Index_Writeback_Inv_LEAF0 (Cache_LEAF0 | Index_Writeback_Inv)
+#define Index_Writeback_Inv_LEAF1 (Cache_LEAF1 | Index_Writeback_Inv)
+#define Index_Writeback_Inv_LEAF2 (Cache_LEAF2 | Index_Writeback_Inv)
+#define Index_Writeback_Inv_LEAF3 (Cache_LEAF3 | Index_Writeback_Inv)
+#define Index_Writeback_Inv_LEAF4 (Cache_LEAF4 | Index_Writeback_Inv)
+#define Index_Writeback_Inv_LEAF5 (Cache_LEAF5 | Index_Writeback_Inv)
+#define Hit_Writeback_Inv_LEAF0 (Cache_LEAF0 | Hit_Writeback_Inv)
+#define Hit_Writeback_Inv_LEAF1 (Cache_LEAF1 | Hit_Writeback_Inv)
+#define Hit_Writeback_Inv_LEAF2 (Cache_LEAF2 | Hit_Writeback_Inv)
+#define Hit_Writeback_Inv_LEAF3 (Cache_LEAF3 | Hit_Writeback_Inv)
+#define Hit_Writeback_Inv_LEAF4 (Cache_LEAF4 | Hit_Writeback_Inv)
+#define Hit_Writeback_Inv_LEAF5 (Cache_LEAF5 | Hit_Writeback_Inv)
#endif /* __ASM_CACHEOPS_H */
return (old32 & mask) >> shift;
}
-static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
- int size)
+static __always_inline unsigned long
+__xchg(volatile void *ptr, unsigned long x, int size)
{
switch (size) {
case 1:
return (old32 & mask) >> shift;
}
-static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
- unsigned long new, unsigned int size)
+static __always_inline unsigned long
+__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size)
{
switch (size) {
case 1:
#define cpu_has_loongarch32 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_32BIT)
#define cpu_has_loongarch64 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT)
-#define cpu_icache_line_size() cpu_data[0].icache.linesz
-#define cpu_dcache_line_size() cpu_data[0].dcache.linesz
-#define cpu_vcache_line_size() cpu_data[0].vcache.linesz
-#define cpu_scache_line_size() cpu_data[0].scache.linesz
-
#ifdef CONFIG_32BIT
# define cpu_has_64bits (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT)
# define cpu_vabits 31
#include <asm/loongarch.h>
+/* cache_desc->flags */
+enum {
+ CACHE_PRESENT = (1 << 0),
+ CACHE_PRIVATE = (1 << 1), /* core private cache */
+ CACHE_INCLUSIVE = (1 << 2), /* include the inner level caches */
+};
+
/*
* Descriptor for a cache
*/
struct cache_desc {
- unsigned int waysize; /* Bytes per way */
+ unsigned char type;
+ unsigned char level;
unsigned short sets; /* Number of lines per set */
unsigned char ways; /* Number of ways */
unsigned char linesz; /* Size of line in bytes */
- unsigned char waybit; /* Bits to select in a cache set */
unsigned char flags; /* Flags describing cache properties */
};
+#define CACHE_LEVEL_MAX 3
+#define CACHE_LEAVES_MAX 6
+
struct cpuinfo_loongarch {
u64 asid_cache;
unsigned long asid_mask;
int tlbsizemtlb;
int tlbsizestlbsets;
int tlbsizestlbways;
- struct cache_desc icache; /* Primary I-cache */
- struct cache_desc dcache; /* Primary D or combined I/D cache */
- struct cache_desc vcache; /* Victim cache, between pcache and scache */
- struct cache_desc scache; /* Secondary cache */
- struct cache_desc tcache; /* Tertiary/split secondary cache */
+ int cache_leaves_present; /* number of cache_leaves[] elements */
+ struct cache_desc cache_leaves[CACHE_LEAVES_MAX];
int core; /* physical core number in package */
int package;/* physical package number */
int vabits; /* Virtual Address size in bits */
#define R_LARCH_SUB64 56
#define R_LARCH_GNU_VTINHERIT 57
#define R_LARCH_GNU_VTENTRY 58
+#define R_LARCH_B16 64
+#define R_LARCH_B21 65
+#define R_LARCH_B26 66
+#define R_LARCH_ABS_HI20 67
+#define R_LARCH_ABS_LO12 68
+#define R_LARCH_ABS64_LO20 69
+#define R_LARCH_ABS64_HI12 70
+#define R_LARCH_PCALA_HI20 71
+#define R_LARCH_PCALA_LO12 72
+#define R_LARCH_PCALA64_LO20 73
+#define R_LARCH_PCALA64_HI12 74
+#define R_LARCH_GOT_PC_HI20 75
+#define R_LARCH_GOT_PC_LO12 76
+#define R_LARCH_GOT64_PC_LO20 77
+#define R_LARCH_GOT64_PC_HI12 78
+#define R_LARCH_GOT_HI20 79
+#define R_LARCH_GOT_LO12 80
+#define R_LARCH_GOT64_LO20 81
+#define R_LARCH_GOT64_HI12 82
+#define R_LARCH_TLS_LE_HI20 83
+#define R_LARCH_TLS_LE_LO12 84
+#define R_LARCH_TLS_LE64_LO20 85
+#define R_LARCH_TLS_LE64_HI12 86
+#define R_LARCH_TLS_IE_PC_HI20 87
+#define R_LARCH_TLS_IE_PC_LO12 88
+#define R_LARCH_TLS_IE64_PC_LO20 89
+#define R_LARCH_TLS_IE64_PC_HI12 90
+#define R_LARCH_TLS_IE_HI20 91
+#define R_LARCH_TLS_IE_LO12 92
+#define R_LARCH_TLS_IE64_LO20 93
+#define R_LARCH_TLS_IE64_HI12 94
+#define R_LARCH_TLS_LD_PC_HI20 95
+#define R_LARCH_TLS_LD_HI20 96
+#define R_LARCH_TLS_GD_PC_HI20 97
+#define R_LARCH_TLS_GD_HI20 98
+#define R_LARCH_32_PCREL 99
+#define R_LARCH_RELAX 100
#ifndef ELF_ARCH
#define NR_FIX_BTMAPS 64
+enum fixed_addresses {
+ FIX_HOLE,
+ FIX_EARLYCON_MEM_BASE,
+ __end_of_fixed_addresses
+};
+
+#define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT)
+#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
+#define FIXMAP_PAGE_IO PAGE_KERNEL_SUC
+
+extern void __set_fixmap(enum fixed_addresses idx,
+ phys_addr_t phys, pgprot_t flags);
+
+#include <asm-generic/fixmap.h>
+
#endif
#include <linux/types.h>
#include <asm/asm.h>
+#define INSN_BREAK 0x002a0000
+
#define ADDR_IMMMASK_LU52ID 0xFFF0000000000000
#define ADDR_IMMMASK_LU32ID 0x000FFFFF00000000
#define ADDR_IMMMASK_ADDU16ID 0x00000000FFFF0000
#define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN)
+enum reg0i26_op {
+ b_op = 0x14,
+ bl_op = 0x15,
+};
+
enum reg1i20_op {
lu12iw_op = 0x0a,
lu32id_op = 0x0b,
+ pcaddu12i_op = 0x0e,
+ pcaddu18i_op = 0x0f,
};
enum reg1i21_op {
bnez_op = 0x11,
};
+enum reg2_op {
+ revb2h_op = 0x0c,
+ revb4h_op = 0x0d,
+ revb2w_op = 0x0e,
+ revbd_op = 0x0f,
+ revh2w_op = 0x10,
+ revhd_op = 0x11,
+};
+
+enum reg2i5_op {
+ slliw_op = 0x81,
+ srliw_op = 0x89,
+ sraiw_op = 0x91,
+};
+
+enum reg2i6_op {
+ sllid_op = 0x41,
+ srlid_op = 0x45,
+ sraid_op = 0x49,
+};
+
enum reg2i12_op {
addiw_op = 0x0a,
addid_op = 0x0b,
lu52id_op = 0x0c,
+ andi_op = 0x0d,
+ ori_op = 0x0e,
+ xori_op = 0x0f,
ldb_op = 0xa0,
ldh_op = 0xa1,
ldw_op = 0xa2,
sth_op = 0xa5,
stw_op = 0xa6,
std_op = 0xa7,
+ ldbu_op = 0xa8,
+ ldhu_op = 0xa9,
+ ldwu_op = 0xaa,
+};
+
+enum reg2i14_op {
+ llw_op = 0x20,
+ scw_op = 0x21,
+ lld_op = 0x22,
+ scd_op = 0x23,
+ ldptrw_op = 0x24,
+ stptrw_op = 0x25,
+ ldptrd_op = 0x26,
+ stptrd_op = 0x27,
};
enum reg2i16_op {
bgeu_op = 0x1b,
};
+enum reg2bstrd_op {
+ bstrinsd_op = 0x2,
+ bstrpickd_op = 0x3,
+};
+
+enum reg3_op {
+ addw_op = 0x20,
+ addd_op = 0x21,
+ subw_op = 0x22,
+ subd_op = 0x23,
+ nor_op = 0x28,
+ and_op = 0x29,
+ or_op = 0x2a,
+ xor_op = 0x2b,
+ orn_op = 0x2c,
+ andn_op = 0x2d,
+ sllw_op = 0x2e,
+ srlw_op = 0x2f,
+ sraw_op = 0x30,
+ slld_op = 0x31,
+ srld_op = 0x32,
+ srad_op = 0x33,
+ mulw_op = 0x38,
+ mulhw_op = 0x39,
+ mulhwu_op = 0x3a,
+ muld_op = 0x3b,
+ mulhd_op = 0x3c,
+ mulhdu_op = 0x3d,
+ divw_op = 0x40,
+ modw_op = 0x41,
+ divwu_op = 0x42,
+ modwu_op = 0x43,
+ divd_op = 0x44,
+ modd_op = 0x45,
+ divdu_op = 0x46,
+ moddu_op = 0x47,
+ ldxb_op = 0x7000,
+ ldxh_op = 0x7008,
+ ldxw_op = 0x7010,
+ ldxd_op = 0x7018,
+ stxb_op = 0x7020,
+ stxh_op = 0x7028,
+ stxw_op = 0x7030,
+ stxd_op = 0x7038,
+ ldxbu_op = 0x7040,
+ ldxhu_op = 0x7048,
+ ldxwu_op = 0x7050,
+ amswapw_op = 0x70c0,
+ amswapd_op = 0x70c1,
+ amaddw_op = 0x70c2,
+ amaddd_op = 0x70c3,
+ amandw_op = 0x70c4,
+ amandd_op = 0x70c5,
+ amorw_op = 0x70c6,
+ amord_op = 0x70c7,
+ amxorw_op = 0x70c8,
+ amxord_op = 0x70c9,
+};
+
+enum reg3sa2_op {
+ alslw_op = 0x02,
+ alslwu_op = 0x03,
+ alsld_op = 0x16,
+};
+
struct reg0i26_format {
unsigned int immediate_h : 10;
unsigned int immediate_l : 16;
unsigned int opcode : 6;
};
+struct reg2_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int opcode : 22;
+};
+
+struct reg2i5_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int immediate : 5;
+ unsigned int opcode : 17;
+};
+
+struct reg2i6_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int immediate : 6;
+ unsigned int opcode : 16;
+};
+
struct reg2i12_format {
unsigned int rd : 5;
unsigned int rj : 5;
unsigned int opcode : 10;
};
+struct reg2i14_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int immediate : 14;
+ unsigned int opcode : 8;
+};
+
struct reg2i16_format {
unsigned int rd : 5;
unsigned int rj : 5;
unsigned int opcode : 6;
};
+struct reg2bstrd_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int lsbd : 6;
+ unsigned int msbd : 6;
+ unsigned int opcode : 10;
+};
+
+struct reg3_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int rk : 5;
+ unsigned int opcode : 17;
+};
+
+struct reg3sa2_format {
+ unsigned int rd : 5;
+ unsigned int rj : 5;
+ unsigned int rk : 5;
+ unsigned int immediate : 2;
+ unsigned int opcode : 15;
+};
+
union loongarch_instruction {
unsigned int word;
- struct reg0i26_format reg0i26_format;
- struct reg1i20_format reg1i20_format;
- struct reg1i21_format reg1i21_format;
- struct reg2i12_format reg2i12_format;
- struct reg2i16_format reg2i16_format;
+ struct reg0i26_format reg0i26_format;
+ struct reg1i20_format reg1i20_format;
+ struct reg1i21_format reg1i21_format;
+ struct reg2_format reg2_format;
+ struct reg2i5_format reg2i5_format;
+ struct reg2i6_format reg2i6_format;
+ struct reg2i12_format reg2i12_format;
+ struct reg2i14_format reg2i14_format;
+ struct reg2i16_format reg2i16_format;
+ struct reg2bstrd_format reg2bstrd_format;
+ struct reg3_format reg3_format;
+ struct reg3sa2_format reg3sa2_format;
};
#define LOONGARCH_INSN_SIZE sizeof(union loongarch_instruction)
u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest);
+static inline bool signed_imm_check(long val, unsigned int bit)
+{
+ return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1));
+}
+
+static inline bool unsigned_imm_check(unsigned long val, unsigned int bit)
+{
+ return val < (1UL << bit);
+}
+
+#define DEF_EMIT_REG0I26_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ int offset) \
+{ \
+ unsigned int immediate_l, immediate_h; \
+ \
+ immediate_l = offset & 0xffff; \
+ offset >>= 16; \
+ immediate_h = offset & 0x3ff; \
+ \
+ insn->reg0i26_format.opcode = OP; \
+ insn->reg0i26_format.immediate_l = immediate_l; \
+ insn->reg0i26_format.immediate_h = immediate_h; \
+}
+
+DEF_EMIT_REG0I26_FORMAT(b, b_op)
+
+#define DEF_EMIT_REG1I20_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, int imm) \
+{ \
+ insn->reg1i20_format.opcode = OP; \
+ insn->reg1i20_format.immediate = imm; \
+ insn->reg1i20_format.rd = rd; \
+}
+
+DEF_EMIT_REG1I20_FORMAT(lu12iw, lu12iw_op)
+DEF_EMIT_REG1I20_FORMAT(lu32id, lu32id_op)
+DEF_EMIT_REG1I20_FORMAT(pcaddu18i, pcaddu18i_op)
+
+#define DEF_EMIT_REG2_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj) \
+{ \
+ insn->reg2_format.opcode = OP; \
+ insn->reg2_format.rd = rd; \
+ insn->reg2_format.rj = rj; \
+}
+
+DEF_EMIT_REG2_FORMAT(revb2h, revb2h_op)
+DEF_EMIT_REG2_FORMAT(revb2w, revb2w_op)
+DEF_EMIT_REG2_FORMAT(revbd, revbd_op)
+
+#define DEF_EMIT_REG2I5_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ int imm) \
+{ \
+ insn->reg2i5_format.opcode = OP; \
+ insn->reg2i5_format.immediate = imm; \
+ insn->reg2i5_format.rd = rd; \
+ insn->reg2i5_format.rj = rj; \
+}
+
+DEF_EMIT_REG2I5_FORMAT(slliw, slliw_op)
+DEF_EMIT_REG2I5_FORMAT(srliw, srliw_op)
+DEF_EMIT_REG2I5_FORMAT(sraiw, sraiw_op)
+
+#define DEF_EMIT_REG2I6_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ int imm) \
+{ \
+ insn->reg2i6_format.opcode = OP; \
+ insn->reg2i6_format.immediate = imm; \
+ insn->reg2i6_format.rd = rd; \
+ insn->reg2i6_format.rj = rj; \
+}
+
+DEF_EMIT_REG2I6_FORMAT(sllid, sllid_op)
+DEF_EMIT_REG2I6_FORMAT(srlid, srlid_op)
+DEF_EMIT_REG2I6_FORMAT(sraid, sraid_op)
+
+#define DEF_EMIT_REG2I12_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ int imm) \
+{ \
+ insn->reg2i12_format.opcode = OP; \
+ insn->reg2i12_format.immediate = imm; \
+ insn->reg2i12_format.rd = rd; \
+ insn->reg2i12_format.rj = rj; \
+}
+
+DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op)
+DEF_EMIT_REG2I12_FORMAT(addid, addid_op)
+DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op)
+DEF_EMIT_REG2I12_FORMAT(andi, andi_op)
+DEF_EMIT_REG2I12_FORMAT(ori, ori_op)
+DEF_EMIT_REG2I12_FORMAT(xori, xori_op)
+DEF_EMIT_REG2I12_FORMAT(ldbu, ldbu_op)
+DEF_EMIT_REG2I12_FORMAT(ldhu, ldhu_op)
+DEF_EMIT_REG2I12_FORMAT(ldwu, ldwu_op)
+DEF_EMIT_REG2I12_FORMAT(ldd, ldd_op)
+DEF_EMIT_REG2I12_FORMAT(stb, stb_op)
+DEF_EMIT_REG2I12_FORMAT(sth, sth_op)
+DEF_EMIT_REG2I12_FORMAT(stw, stw_op)
+DEF_EMIT_REG2I12_FORMAT(std, std_op)
+
+#define DEF_EMIT_REG2I14_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ int imm) \
+{ \
+ insn->reg2i14_format.opcode = OP; \
+ insn->reg2i14_format.immediate = imm; \
+ insn->reg2i14_format.rd = rd; \
+ insn->reg2i14_format.rj = rj; \
+}
+
+DEF_EMIT_REG2I14_FORMAT(llw, llw_op)
+DEF_EMIT_REG2I14_FORMAT(scw, scw_op)
+DEF_EMIT_REG2I14_FORMAT(lld, lld_op)
+DEF_EMIT_REG2I14_FORMAT(scd, scd_op)
+DEF_EMIT_REG2I14_FORMAT(ldptrw, ldptrw_op)
+DEF_EMIT_REG2I14_FORMAT(stptrw, stptrw_op)
+DEF_EMIT_REG2I14_FORMAT(ldptrd, ldptrd_op)
+DEF_EMIT_REG2I14_FORMAT(stptrd, stptrd_op)
+
+#define DEF_EMIT_REG2I16_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rj, \
+ enum loongarch_gpr rd, \
+ int offset) \
+{ \
+ insn->reg2i16_format.opcode = OP; \
+ insn->reg2i16_format.immediate = offset; \
+ insn->reg2i16_format.rj = rj; \
+ insn->reg2i16_format.rd = rd; \
+}
+
+DEF_EMIT_REG2I16_FORMAT(beq, beq_op)
+DEF_EMIT_REG2I16_FORMAT(bne, bne_op)
+DEF_EMIT_REG2I16_FORMAT(blt, blt_op)
+DEF_EMIT_REG2I16_FORMAT(bge, bge_op)
+DEF_EMIT_REG2I16_FORMAT(bltu, bltu_op)
+DEF_EMIT_REG2I16_FORMAT(bgeu, bgeu_op)
+DEF_EMIT_REG2I16_FORMAT(jirl, jirl_op)
+
+#define DEF_EMIT_REG2BSTRD_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ int msbd, \
+ int lsbd) \
+{ \
+ insn->reg2bstrd_format.opcode = OP; \
+ insn->reg2bstrd_format.msbd = msbd; \
+ insn->reg2bstrd_format.lsbd = lsbd; \
+ insn->reg2bstrd_format.rj = rj; \
+ insn->reg2bstrd_format.rd = rd; \
+}
+
+DEF_EMIT_REG2BSTRD_FORMAT(bstrpickd, bstrpickd_op)
+
+#define DEF_EMIT_REG3_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ enum loongarch_gpr rk) \
+{ \
+ insn->reg3_format.opcode = OP; \
+ insn->reg3_format.rd = rd; \
+ insn->reg3_format.rj = rj; \
+ insn->reg3_format.rk = rk; \
+}
+
+DEF_EMIT_REG3_FORMAT(addd, addd_op)
+DEF_EMIT_REG3_FORMAT(subd, subd_op)
+DEF_EMIT_REG3_FORMAT(muld, muld_op)
+DEF_EMIT_REG3_FORMAT(divdu, divdu_op)
+DEF_EMIT_REG3_FORMAT(moddu, moddu_op)
+DEF_EMIT_REG3_FORMAT(and, and_op)
+DEF_EMIT_REG3_FORMAT(or, or_op)
+DEF_EMIT_REG3_FORMAT(xor, xor_op)
+DEF_EMIT_REG3_FORMAT(sllw, sllw_op)
+DEF_EMIT_REG3_FORMAT(slld, slld_op)
+DEF_EMIT_REG3_FORMAT(srlw, srlw_op)
+DEF_EMIT_REG3_FORMAT(srld, srld_op)
+DEF_EMIT_REG3_FORMAT(sraw, sraw_op)
+DEF_EMIT_REG3_FORMAT(srad, srad_op)
+DEF_EMIT_REG3_FORMAT(ldxbu, ldxbu_op)
+DEF_EMIT_REG3_FORMAT(ldxhu, ldxhu_op)
+DEF_EMIT_REG3_FORMAT(ldxwu, ldxwu_op)
+DEF_EMIT_REG3_FORMAT(ldxd, ldxd_op)
+DEF_EMIT_REG3_FORMAT(stxb, stxb_op)
+DEF_EMIT_REG3_FORMAT(stxh, stxh_op)
+DEF_EMIT_REG3_FORMAT(stxw, stxw_op)
+DEF_EMIT_REG3_FORMAT(stxd, stxd_op)
+DEF_EMIT_REG3_FORMAT(amaddw, amaddw_op)
+DEF_EMIT_REG3_FORMAT(amaddd, amaddd_op)
+DEF_EMIT_REG3_FORMAT(amandw, amandw_op)
+DEF_EMIT_REG3_FORMAT(amandd, amandd_op)
+DEF_EMIT_REG3_FORMAT(amorw, amorw_op)
+DEF_EMIT_REG3_FORMAT(amord, amord_op)
+DEF_EMIT_REG3_FORMAT(amxorw, amxorw_op)
+DEF_EMIT_REG3_FORMAT(amxord, amxord_op)
+DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op)
+DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op)
+
+#define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \
+static inline void emit_##NAME(union loongarch_instruction *insn, \
+ enum loongarch_gpr rd, \
+ enum loongarch_gpr rj, \
+ enum loongarch_gpr rk, \
+ int imm) \
+{ \
+ insn->reg3sa2_format.opcode = OP; \
+ insn->reg3sa2_format.immediate = imm; \
+ insn->reg3sa2_format.rd = rd; \
+ insn->reg3sa2_format.rj = rj; \
+ insn->reg3sa2_format.rk = rk; \
+}
+
+DEF_EMIT_REG3SA2_FORMAT(alsld, alsld_op)
+
#endif /* _ASM_INST_H */
#define early_memremap early_ioremap
#define early_memunmap early_iounmap
+#ifdef CONFIG_ARCH_IOREMAP
+
static inline void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size,
unsigned long prot_val)
{
- if (prot_val == _CACHE_CC)
+ if (prot_val & _CACHE_CC)
return (void __iomem *)(unsigned long)(CACHE_BASE + offset);
else
return (void __iomem *)(unsigned long)(UNCACHE_BASE + offset);
}
-/*
- * ioremap - map bus memory into CPU space
- * @offset: bus address of the memory
- * @size: size of the resource to map
- *
- * ioremap performs a platform specific sequence of operations to
- * make bus memory CPU accessible via the readb/readw/readl/writeb/
- * writew/writel functions and the other mmio helpers. The returned
- * address is not guaranteed to be usable directly as a virtual
- * address.
- */
-#define ioremap(offset, size) \
- ioremap_prot((offset), (size), _CACHE_SUC)
+#define ioremap(offset, size) \
+ ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL_SUC))
-/*
- * ioremap_wc - map bus memory into CPU space
- * @offset: bus address of the memory
- * @size: size of the resource to map
- *
- * ioremap_wc performs a platform specific sequence of operations to
- * make bus memory CPU accessible via the readb/readw/readl/writeb/
- * writew/writel functions and the other mmio helpers. The returned
- * address is not guaranteed to be usable directly as a virtual
- * address.
- *
- * This version of ioremap ensures that the memory is marked uncachable
- * but accelerated by means of write-combining feature. It is specifically
- * useful for PCIe prefetchable windows, which may vastly improve a
- * communications performance. If it was determined on boot stage, what
- * CPU CCA doesn't support WUC, the method shall fall-back to the
- * _CACHE_SUC option (see cpu_probe() method).
- */
-#define ioremap_wc(offset, size) \
- ioremap_prot((offset), (size), _CACHE_WUC)
+#define iounmap(addr) ((void)(addr))
+
+#endif
/*
- * ioremap_cache - map bus memory into CPU space
- * @offset: bus address of the memory
- * @size: size of the resource to map
+ * On LoongArch, ioremap() has two variants, ioremap_wc() and ioremap_cache().
+ * They map bus memory into CPU space, the mapped memory is marked uncachable
+ * (_CACHE_SUC), uncachable but accelerated by write-combine (_CACHE_WUC) and
+ * cachable (_CACHE_CC) respectively for CPU access.
*
- * ioremap_cache performs a platform specific sequence of operations to
- * make bus memory CPU accessible via the readb/readw/readl/writeb/
- * writew/writel functions and the other mmio helpers. The returned
- * address is not guaranteed to be usable directly as a virtual
- * address.
- *
- * This version of ioremap ensures that the memory is marked cachable by
- * the CPU. Also enables full write-combining. Useful for some
- * memory-like regions on I/O busses.
+ * @offset: bus address of the memory
+ * @size: size of the resource to map
*/
-#define ioremap_cache(offset, size) \
- ioremap_prot((offset), (size), _CACHE_CC)
+#define ioremap_wc(offset, size) \
+ ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL_WUC))
-static inline void iounmap(const volatile void __iomem *addr)
-{
-}
+#define ioremap_cache(offset, size) \
+ ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL))
#define mmiowb() asm volatile ("dbar 0" ::: "memory")
#include <asm-generic/io.h>
+#define ARCH_HAS_VALID_PHYS_ADDR_RANGE
+extern int valid_phys_addr_range(phys_addr_t addr, size_t size);
+extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size);
+
#endif /* _ASM_IO_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * kexec.h for kexec
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+
+#ifndef _ASM_KEXEC_H
+#define _ASM_KEXEC_H
+
+#include <asm/stacktrace.h>
+#include <asm/page.h>
+
+/* Maximum physical address we can use pages from */
+#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL)
+/* Maximum address we can reach in physical address mode */
+#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL)
+ /* Maximum address we can use for the control code buffer */
+#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL)
+
+/* Reserve a page for the control code buffer */
+#define KEXEC_CONTROL_PAGE_SIZE PAGE_SIZE
+
+/* The native architecture */
+#define KEXEC_ARCH KEXEC_ARCH_LOONGARCH
+
+static inline void crash_setup_regs(struct pt_regs *newregs,
+ struct pt_regs *oldregs)
+{
+ if (oldregs)
+ memcpy(newregs, oldregs, sizeof(*newregs));
+ else
+ prepare_frametrace(newregs);
+}
+
+#define ARCH_HAS_KIMAGE_ARCH
+
+struct kimage_arch {
+ unsigned long efi_boot;
+ unsigned long cmdline_ptr;
+ unsigned long systable_ptr;
+};
+
+typedef void (*do_kexec_t)(unsigned long efi_boot,
+ unsigned long cmdline_ptr,
+ unsigned long systable_ptr,
+ unsigned long start_addr,
+ unsigned long first_ind_entry);
+
+struct kimage;
+extern const unsigned char relocate_new_kernel[];
+extern const size_t relocate_new_kernel_size;
+extern void kexec_reboot(void);
+
+#ifdef CONFIG_SMP
+extern atomic_t kexec_ready_to_reboot;
+extern const unsigned char kexec_smp_wait[];
+#endif
+
+#endif /* !_ASM_KEXEC_H */
#define CPUCFG16_L3_DINCL BIT(16)
#define LOONGARCH_CPUCFG17 0x11
-#define CPUCFG17_L1I_WAYS_M GENMASK(15, 0)
-#define CPUCFG17_L1I_SETS_M GENMASK(23, 16)
-#define CPUCFG17_L1I_SIZE_M GENMASK(30, 24)
-#define CPUCFG17_L1I_WAYS 0
-#define CPUCFG17_L1I_SETS 16
-#define CPUCFG17_L1I_SIZE 24
-
#define LOONGARCH_CPUCFG18 0x12
-#define CPUCFG18_L1D_WAYS_M GENMASK(15, 0)
-#define CPUCFG18_L1D_SETS_M GENMASK(23, 16)
-#define CPUCFG18_L1D_SIZE_M GENMASK(30, 24)
-#define CPUCFG18_L1D_WAYS 0
-#define CPUCFG18_L1D_SETS 16
-#define CPUCFG18_L1D_SIZE 24
-
#define LOONGARCH_CPUCFG19 0x13
-#define CPUCFG19_L2_WAYS_M GENMASK(15, 0)
-#define CPUCFG19_L2_SETS_M GENMASK(23, 16)
-#define CPUCFG19_L2_SIZE_M GENMASK(30, 24)
-#define CPUCFG19_L2_WAYS 0
-#define CPUCFG19_L2_SETS 16
-#define CPUCFG19_L2_SIZE 24
-
#define LOONGARCH_CPUCFG20 0x14
-#define CPUCFG20_L3_WAYS_M GENMASK(15, 0)
-#define CPUCFG20_L3_SETS_M GENMASK(23, 16)
-#define CPUCFG20_L3_SIZE_M GENMASK(30, 24)
-#define CPUCFG20_L3_WAYS 0
-#define CPUCFG20_L3_SETS 16
-#define CPUCFG20_L3_SIZE 24
+#define CPUCFG_CACHE_WAYS_M GENMASK(15, 0)
+#define CPUCFG_CACHE_SETS_M GENMASK(23, 16)
+#define CPUCFG_CACHE_LSIZE_M GENMASK(30, 24)
+#define CPUCFG_CACHE_WAYS 0
+#define CPUCFG_CACHE_SETS 16
+#define CPUCFG_CACHE_LSIZE 24
#define LOONGARCH_CPUCFG48 0x30
#define CPUCFG48_MCSR_LCK BIT(0)
};
struct mod_arch_specific {
+ struct mod_section got;
struct mod_section plt;
struct mod_section plt_idx;
};
+struct got_entry {
+ Elf_Addr symbol_addr;
+};
+
struct plt_entry {
u32 inst_lu12iw;
u32 inst_lu32id;
};
struct plt_idx_entry {
- unsigned long symbol_addr;
+ Elf_Addr symbol_addr;
};
-Elf_Addr module_emit_plt_entry(struct module *mod, unsigned long val);
+Elf_Addr module_emit_got_entry(struct module *mod, Elf_Addr val);
+Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Addr val);
+
+static inline struct got_entry emit_got_entry(Elf_Addr val)
+{
+ return (struct got_entry) { val };
+}
static inline struct plt_entry emit_plt_entry(unsigned long val)
{
return plt + plt_idx;
}
+static inline struct got_entry *get_got_entry(Elf_Addr val,
+ const struct mod_section *sec)
+{
+ struct got_entry *got = (struct got_entry *)sec->shdr->sh_addr;
+ int i;
+
+ for (i = 0; i < sec->num_entries; i++)
+ if (got[i].symbol_addr == val)
+ return &got[i];
+ return NULL;
+}
+
#endif /* _ASM_MODULE_H */
/* Copyright (C) 2020-2022 Loongson Technology Corporation Limited */
SECTIONS {
. = ALIGN(4);
+ .got : { BYTE(0) }
.plt : { BYTE(0) }
.plt.idx : { BYTE(0) }
}
#include <asm/cmpxchg.h>
#include <asm/loongarch.h>
+/*
+ * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
+ * the loading address of main kernel image, but far from where the modules are
+ * loaded. Tell the compiler this fact when using explicit relocs.
+ */
+#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
+#define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
+#endif
+
/* Use r21 for fast access */
register unsigned long __my_cpu_offset __asm__("$r21");
#ifndef __LOONGARCH_PERF_EVENT_H__
#define __LOONGARCH_PERF_EVENT_H__
-/* Nothing to show here; the file is required by linux/perf_event.h. */
+
+#define perf_arch_bpf_user_pt_regs(regs) (struct user_pt_regs *)regs
+
#endif /* __LOONGARCH_PERF_EVENT_H__ */
_PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC)
#define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
_PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC)
+
#ifndef __ASSEMBLY__
+#define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL_SUC)
+
#define pgprot_noncached pgprot_noncached
static inline pgprot_t pgprot_noncached(pgprot_t _prot)
__update_tlb(vma, address, ptep);
}
+#define __HAVE_ARCH_UPDATE_MMU_TLB
+#define update_mmu_tlb update_mmu_cache
+
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
struct task_struct;
-/* Free all resources held by a thread. */
-#define release_thread(thread) do { } while (0)
-
enum idle_boot_override {IDLE_NO_OVERRIDE = 0, IDLE_HALT, IDLE_NOMWAIT, IDLE_POLL};
extern unsigned long boot_option_idle_override;
extern unsigned long eentry;
extern unsigned long tlbrentry;
+extern void tlb_init(int cpu);
extern void cpu_cache_init(void);
+extern void cache_error_setup(void);
extern void per_cpu_trap_init(int cpu);
extern void set_handler(unsigned long offset, void *addr, unsigned long len);
extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+#ifndef _ASM_SPINLOCK_H
+#define _ASM_SPINLOCK_H
+
+#include <asm/processor.h>
+#include <asm/qspinlock.h>
+#include <asm/qrwlock.h>
+
+#endif /* _ASM_SPINLOCK_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+#ifndef _ASM_SPINLOCK_TYPES_H
+#define _ASM_SPINLOCK_TYPES_H
+
+#include <asm-generic/qspinlock_types.h>
+#include <asm-generic/qrwlock_types.h>
+
+#endif
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__
+#define _UAPI__ASM_BPF_PERF_EVENT_H__
+
+#include <linux/ptrace.h>
+
+typedef struct user_pt_regs bpf_user_pt_regs_t;
+
+#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _ASM_LOONGARCH_PERF_REGS_H
+#define _ASM_LOONGARCH_PERF_REGS_H
+
+enum perf_event_loongarch_regs {
+ PERF_REG_LOONGARCH_PC,
+ PERF_REG_LOONGARCH_R1,
+ PERF_REG_LOONGARCH_R2,
+ PERF_REG_LOONGARCH_R3,
+ PERF_REG_LOONGARCH_R4,
+ PERF_REG_LOONGARCH_R5,
+ PERF_REG_LOONGARCH_R6,
+ PERF_REG_LOONGARCH_R7,
+ PERF_REG_LOONGARCH_R8,
+ PERF_REG_LOONGARCH_R9,
+ PERF_REG_LOONGARCH_R10,
+ PERF_REG_LOONGARCH_R11,
+ PERF_REG_LOONGARCH_R12,
+ PERF_REG_LOONGARCH_R13,
+ PERF_REG_LOONGARCH_R14,
+ PERF_REG_LOONGARCH_R15,
+ PERF_REG_LOONGARCH_R16,
+ PERF_REG_LOONGARCH_R17,
+ PERF_REG_LOONGARCH_R18,
+ PERF_REG_LOONGARCH_R19,
+ PERF_REG_LOONGARCH_R20,
+ PERF_REG_LOONGARCH_R21,
+ PERF_REG_LOONGARCH_R22,
+ PERF_REG_LOONGARCH_R23,
+ PERF_REG_LOONGARCH_R24,
+ PERF_REG_LOONGARCH_R25,
+ PERF_REG_LOONGARCH_R26,
+ PERF_REG_LOONGARCH_R27,
+ PERF_REG_LOONGARCH_R28,
+ PERF_REG_LOONGARCH_R29,
+ PERF_REG_LOONGARCH_R30,
+ PERF_REG_LOONGARCH_R31,
+ PERF_REG_LOONGARCH_MAX,
+};
+#endif /* _ASM_LOONGARCH_PERF_REGS_H */
obj-$(CONFIG_NUMA) += numa.o
+obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o
+
+obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o
+obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
+
obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o
+obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o
+
CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS)
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
*/
#include <linux/cacheinfo.h>
+#include <linux/topology.h>
#include <asm/bootinfo.h>
#include <asm/cpu-info.h>
-/* Populates leaf and increments to next leaf */
-#define populate_cache(cache, leaf, c_level, c_type) \
-do { \
- leaf->type = c_type; \
- leaf->level = c_level; \
- leaf->coherency_line_size = c->cache.linesz; \
- leaf->number_of_sets = c->cache.sets; \
- leaf->ways_of_associativity = c->cache.ways; \
- leaf->size = c->cache.linesz * c->cache.sets * \
- c->cache.ways; \
- if (leaf->level > 2) \
- leaf->size *= nodes_per_package; \
- leaf++; \
-} while (0)
-
int init_cache_level(unsigned int cpu)
{
- struct cpuinfo_loongarch *c = ¤t_cpu_data;
+ int cache_present = current_cpu_data.cache_leaves_present;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
- int levels = 0, leaves = 0;
-
- /*
- * If Dcache is not set, we assume the cache structures
- * are not properly initialized.
- */
- if (c->dcache.waysize)
- levels += 1;
- else
- return -ENOENT;
-
-
- leaves += (c->icache.waysize) ? 2 : 1;
-
- if (c->vcache.waysize) {
- levels++;
- leaves++;
- }
- if (c->scache.waysize) {
- levels++;
- leaves++;
- }
+ this_cpu_ci->num_levels =
+ current_cpu_data.cache_leaves[cache_present - 1].level;
+ this_cpu_ci->num_leaves = cache_present;
- if (c->tcache.waysize) {
- levels++;
- leaves++;
- }
-
- this_cpu_ci->num_levels = levels;
- this_cpu_ci->num_leaves = leaves;
return 0;
}
static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
struct cacheinfo *sib_leaf)
{
- return !((this_leaf->level == 1) || (this_leaf->level == 2));
+ return (!(*(unsigned char *)(this_leaf->priv) & CACHE_PRIVATE)
+ && !(*(unsigned char *)(sib_leaf->priv) & CACHE_PRIVATE));
}
static void cache_cpumap_setup(unsigned int cpu)
{
- struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
- struct cacheinfo *this_leaf, *sib_leaf;
unsigned int index;
+ struct cacheinfo *this_leaf, *sib_leaf;
+ struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
for (index = 0; index < this_cpu_ci->num_leaves; index++) {
unsigned int i;
for_each_online_cpu(i) {
struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i);
- if (i == cpu || !sib_cpu_ci->info_list)
- continue;/* skip if itself or no cacheinfo */
+ if (i == cpu || !sib_cpu_ci->info_list ||
+ (cpu_to_node(i) != cpu_to_node(cpu)))
+ continue;
+
sib_leaf = sib_cpu_ci->info_list + index;
if (cache_leaves_are_shared(this_leaf, sib_leaf)) {
cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map);
int populate_cache_leaves(unsigned int cpu)
{
- int level = 1, nodes_per_package = 1;
- struct cpuinfo_loongarch *c = ¤t_cpu_data;
+ int i, cache_present = current_cpu_data.cache_leaves_present;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
struct cacheinfo *this_leaf = this_cpu_ci->info_list;
-
- if (loongson_sysconf.nr_nodes > 1)
- nodes_per_package = loongson_sysconf.cores_per_package
- / loongson_sysconf.cores_per_node;
-
- if (c->icache.waysize) {
- populate_cache(dcache, this_leaf, level, CACHE_TYPE_DATA);
- populate_cache(icache, this_leaf, level++, CACHE_TYPE_INST);
- } else {
- populate_cache(dcache, this_leaf, level++, CACHE_TYPE_UNIFIED);
+ struct cache_desc *cd, *cdesc = current_cpu_data.cache_leaves;
+
+ for (i = 0; i < cache_present; i++) {
+ cd = cdesc + i;
+
+ this_leaf->type = cd->type;
+ this_leaf->level = cd->level;
+ this_leaf->coherency_line_size = cd->linesz;
+ this_leaf->number_of_sets = cd->sets;
+ this_leaf->ways_of_associativity = cd->ways;
+ this_leaf->size = cd->linesz * cd->sets * cd->ways;
+ this_leaf->priv = &cd->flags;
+ this_leaf++;
}
- if (c->vcache.waysize)
- populate_cache(vcache, this_leaf, level++, CACHE_TYPE_UNIFIED);
-
- if (c->scache.waysize)
- populate_cache(scache, this_leaf, level++, CACHE_TYPE_UNIFIED);
-
- if (c->tcache.waysize)
- populate_cache(tcache, this_leaf, level++, CACHE_TYPE_UNIFIED);
-
cache_cpumap_setup(cpu);
this_cpu_ci->cpu_map_populated = true;
uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]);
uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]);
- __cpu_full_name[cpu] = cpu_full_name;
+ if (!__cpu_full_name[cpu])
+ __cpu_full_name[cpu] = cpu_full_name;
+
*vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR);
*cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/crash_dump.h>
+#include <linux/io.h>
+#include <linux/uio.h>
+
+ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn,
+ size_t csize, unsigned long offset)
+{
+ void *vaddr;
+
+ if (!csize)
+ return 0;
+
+ vaddr = memremap(__pfn_to_phys(pfn), PAGE_SIZE, MEMREMAP_WB);
+ if (!vaddr)
+ return -ENOMEM;
+
+ csize = copy_to_iter(vaddr + offset, csize, iter);
+
+ memunmap(vaddr);
+
+ return csize;
+}
#include <asm/addrspace.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
+#include <asm/bug.h>
#include <asm/regdef.h>
#include <asm/loongarch.h>
#include <asm/stackframe.h>
_head:
.word MZ_MAGIC /* "MZ", MS-DOS header */
- .org 0x3c /* 0x04 ~ 0x3b reserved */
+ .org 0x8
+ .dword kernel_entry /* Kernel entry point */
+ .dword _end - _text /* Kernel image effective size */
+ .quad 0 /* Kernel image load offset from start of RAM */
+ .org 0x3c /* 0x20 ~ 0x3b reserved */
.long pe_header - _head /* Offset to the PE header */
pe_header:
li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0
csrwr t0, LOONGARCH_CSR_EUEN
- la t0, __bss_start # clear .bss
+ la.pcrel t0, __bss_start # clear .bss
st.d zero, t0, 0
- la t1, __bss_stop - LONGSIZE
+ la.pcrel t1, __bss_stop - LONGSIZE
1:
addi.d t0, t0, LONGSIZE
st.d zero, t0, 0
bne t0, t1, 1b
- la t0, fw_arg0
+ la.pcrel t0, fw_arg0
st.d a0, t0, 0 # firmware arguments
- la t0, fw_arg1
+ la.pcrel t0, fw_arg1
st.d a1, t0, 0
- la t0, fw_arg2
+ la.pcrel t0, fw_arg2
st.d a2, t0, 0
/* KSave3 used for percpu base, initialized as 0 */
/* GPR21 used for percpu base (runtime), initialized as 0 */
move u0, zero
- la tp, init_thread_union
+ la.pcrel tp, init_thread_union
/* Set the SP after an empty pt_regs. */
PTR_LI sp, (_THREAD_SIZE - 32 - PT_SIZE)
PTR_ADD sp, sp, tp
PTR_ADDI sp, sp, -4 * SZREG # init stack pointer
bl start_kernel
+ ASM_BUG()
SYM_CODE_END(kernel_entry)
ld.d tp, t0, CPU_BOOT_TINFO
bl start_secondary
+ ASM_BUG()
+
SYM_CODE_END(smpboot_entry)
#endif /* CONFIG_SMP */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * machine_kexec.c for kexec
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+#include <linux/compiler.h>
+#include <linux/cpu.h>
+#include <linux/kexec.h>
+#include <linux/crash_dump.h>
+#include <linux/delay.h>
+#include <linux/irq.h>
+#include <linux/libfdt.h>
+#include <linux/mm.h>
+#include <linux/of_fdt.h>
+#include <linux/reboot.h>
+#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/bootinfo.h>
+#include <asm/cacheflush.h>
+#include <asm/page.h>
+
+/* 0x100000 ~ 0x200000 is safe */
+#define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL)
+#define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL)
+
+static unsigned long reboot_code_buffer;
+static cpumask_t cpus_in_crash = CPU_MASK_NONE;
+
+#ifdef CONFIG_SMP
+static void (*relocated_kexec_smp_wait)(void *);
+atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0);
+#endif
+
+static unsigned long efi_boot;
+static unsigned long cmdline_ptr;
+static unsigned long systable_ptr;
+static unsigned long start_addr;
+static unsigned long first_ind_entry;
+
+static void kexec_image_info(const struct kimage *kimage)
+{
+ unsigned long i;
+
+ pr_debug("kexec kimage info:\n");
+ pr_debug("\ttype: %d\n", kimage->type);
+ pr_debug("\tstart: %lx\n", kimage->start);
+ pr_debug("\thead: %lx\n", kimage->head);
+ pr_debug("\tnr_segments: %lu\n", kimage->nr_segments);
+
+ for (i = 0; i < kimage->nr_segments; i++) {
+ pr_debug("\t segment[%lu]: %016lx - %016lx", i,
+ kimage->segment[i].mem,
+ kimage->segment[i].mem + kimage->segment[i].memsz);
+ pr_debug("\t\t0x%lx bytes, %lu pages\n",
+ (unsigned long)kimage->segment[i].memsz,
+ (unsigned long)kimage->segment[i].memsz / PAGE_SIZE);
+ }
+}
+
+int machine_kexec_prepare(struct kimage *kimage)
+{
+ int i;
+ char *bootloader = "kexec";
+ void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR;
+
+ kexec_image_info(kimage);
+
+ kimage->arch.efi_boot = fw_arg0;
+ kimage->arch.systable_ptr = fw_arg2;
+
+ /* Find the command line */
+ for (i = 0; i < kimage->nr_segments; i++) {
+ if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) {
+ if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE))
+ kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr;
+ break;
+ }
+ }
+
+ if (!kimage->arch.cmdline_ptr) {
+ pr_err("Command line not included in the provided image\n");
+ return -EINVAL;
+ }
+
+ /* kexec/kdump need a safe page to save reboot_code_buffer */
+ kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE);
+
+ reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page);
+ memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size);
+
+#ifdef CONFIG_SMP
+ /* All secondary cpus now may jump to kexec_smp_wait cycle */
+ relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel);
+#endif
+
+ return 0;
+}
+
+void machine_kexec_cleanup(struct kimage *kimage)
+{
+}
+
+void kexec_reboot(void)
+{
+ do_kexec_t do_kexec = NULL;
+
+ /*
+ * We know we were online, and there will be no incoming IPIs at
+ * this point. Mark online again before rebooting so that the crash
+ * analysis tool will see us correctly.
+ */
+ set_cpu_online(smp_processor_id(), true);
+
+ /* Ensure remote CPUs observe that we're online before rebooting. */
+ smp_mb__after_atomic();
+
+ /*
+ * Make sure we get correct instructions written by the
+ * machine_kexec_prepare() CPU.
+ */
+ __asm__ __volatile__ ("\tibar 0\n"::);
+
+#ifdef CONFIG_SMP
+ /* All secondary cpus go to kexec_smp_wait */
+ if (smp_processor_id() > 0) {
+ relocated_kexec_smp_wait(NULL);
+ unreachable();
+ }
+#endif
+
+ do_kexec = (void *)reboot_code_buffer;
+ do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry);
+
+ unreachable();
+}
+
+
+#ifdef CONFIG_SMP
+static void kexec_shutdown_secondary(void *regs)
+{
+ int cpu = smp_processor_id();
+
+ if (!cpu_online(cpu))
+ return;
+
+ /* We won't be sent IPIs any more. */
+ set_cpu_online(cpu, false);
+
+ local_irq_disable();
+ while (!atomic_read(&kexec_ready_to_reboot))
+ cpu_relax();
+
+ kexec_reboot();
+}
+
+static void crash_shutdown_secondary(void *passed_regs)
+{
+ int cpu = smp_processor_id();
+ struct pt_regs *regs = passed_regs;
+
+ /*
+ * If we are passed registers, use those. Otherwise get the
+ * regs from the last interrupt, which should be correct, as
+ * we are in an interrupt. But if the regs are not there,
+ * pull them from the top of the stack. They are probably
+ * wrong, but we need something to keep from crashing again.
+ */
+ if (!regs)
+ regs = get_irq_regs();
+ if (!regs)
+ regs = task_pt_regs(current);
+
+ if (!cpu_online(cpu))
+ return;
+
+ /* We won't be sent IPIs any more. */
+ set_cpu_online(cpu, false);
+
+ local_irq_disable();
+ if (!cpumask_test_cpu(cpu, &cpus_in_crash))
+ crash_save_cpu(regs, cpu);
+ cpumask_set_cpu(cpu, &cpus_in_crash);
+
+ while (!atomic_read(&kexec_ready_to_reboot))
+ cpu_relax();
+
+ kexec_reboot();
+}
+
+void crash_smp_send_stop(void)
+{
+ unsigned int ncpus;
+ unsigned long timeout;
+ static int cpus_stopped;
+
+ /*
+ * This function can be called twice in panic path, but obviously
+ * we should execute this only once.
+ */
+ if (cpus_stopped)
+ return;
+
+ cpus_stopped = 1;
+
+ /* Excluding the panic cpu */
+ ncpus = num_online_cpus() - 1;
+
+ smp_call_function(crash_shutdown_secondary, NULL, 0);
+ smp_wmb();
+
+ /*
+ * The crash CPU sends an IPI and wait for other CPUs to
+ * respond. Delay of at least 10 seconds.
+ */
+ timeout = MSEC_PER_SEC * 10;
+ pr_emerg("Sending IPI to other cpus...\n");
+ while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) {
+ mdelay(1);
+ cpu_relax();
+ }
+}
+#endif /* defined(CONFIG_SMP) */
+
+void machine_shutdown(void)
+{
+ int cpu;
+
+ /* All CPUs go to reboot_code_buffer */
+ for_each_possible_cpu(cpu)
+ if (!cpu_online(cpu))
+ cpu_device_up(get_cpu_device(cpu));
+
+#ifdef CONFIG_SMP
+ smp_call_function(kexec_shutdown_secondary, NULL, 0);
+#endif
+}
+
+void machine_crash_shutdown(struct pt_regs *regs)
+{
+ int crashing_cpu;
+
+ local_irq_disable();
+
+ crashing_cpu = smp_processor_id();
+ crash_save_cpu(regs, crashing_cpu);
+
+#ifdef CONFIG_SMP
+ crash_smp_send_stop();
+#endif
+ cpumask_set_cpu(crashing_cpu, &cpus_in_crash);
+
+ pr_info("Starting crashdump kernel...\n");
+}
+
+void machine_kexec(struct kimage *image)
+{
+ unsigned long entry, *ptr;
+ struct kimage_arch *internal = &image->arch;
+
+ efi_boot = internal->efi_boot;
+ cmdline_ptr = internal->cmdline_ptr;
+ systable_ptr = internal->systable_ptr;
+
+ start_addr = (unsigned long)phys_to_virt(image->start);
+
+ first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ?
+ (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0;
+
+ /*
+ * The generic kexec code builds a page list with physical
+ * addresses. they are directly accessible through XKPRANGE
+ * hence the phys_to_virt() call.
+ */
+ for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE);
+ ptr = (entry & IND_INDIRECTION) ?
+ phys_to_virt(entry & PAGE_MASK) : ptr + 1) {
+ if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION ||
+ *ptr & IND_DESTINATION)
+ *ptr = (unsigned long) phys_to_virt(*ptr);
+ }
+
+ /* Mark offline before disabling local irq. */
+ set_cpu_online(smp_processor_id(), false);
+
+ /* We do not want to be bothered. */
+ local_irq_disable();
+
+ pr_notice("EFI boot flag 0x%lx\n", efi_boot);
+ pr_notice("Command line at 0x%lx\n", cmdline_ptr);
+ pr_notice("System table at 0x%lx\n", systable_ptr);
+ pr_notice("We will call new kernel at 0x%lx\n", start_addr);
+ pr_notice("Bye ...\n");
+
+ /* Make reboot code buffer available to the boot CPU. */
+ flush_cache_all();
+
+#ifdef CONFIG_SMP
+ atomic_set(&kexec_ready_to_reboot, 1);
+#endif
+
+ kexec_reboot();
+}
/* Reserve the kernel text/data/bss */
memblock_reserve(__pa_symbol(&_text),
__pa_symbol(&_end) - __pa_symbol(&_text));
-
- /* Reserve the initrd */
- reserve_initrd_mem();
}
#include <linux/kernel.h>
#include <linux/module.h>
-Elf_Addr module_emit_plt_entry(struct module *mod, unsigned long val)
+Elf_Addr module_emit_got_entry(struct module *mod, Elf_Addr val)
+{
+ struct mod_section *got_sec = &mod->arch.got;
+ int i = got_sec->num_entries;
+ struct got_entry *got = get_got_entry(val, got_sec);
+
+ if (got)
+ return (Elf_Addr)got;
+
+ /* There is no GOT entry for val yet, create a new one. */
+ got = (struct got_entry *)got_sec->shdr->sh_addr;
+ got[i] = emit_got_entry(val);
+
+ got_sec->num_entries++;
+ if (got_sec->num_entries > got_sec->max_entries) {
+ /*
+ * This may happen when the module contains a GOT_HI20 without
+ * a paired GOT_LO12. Such a module is broken, reject it.
+ */
+ pr_err("%s: module contains bad GOT relocation\n", mod->name);
+ return 0;
+ }
+
+ return (Elf_Addr)&got[i];
+}
+
+Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Addr val)
{
int nr;
struct mod_section *plt_sec = &mod->arch.plt;
return false;
}
-static void count_max_entries(Elf_Rela *relas, int num, unsigned int *plts)
+static void count_max_entries(Elf_Rela *relas, int num,
+ unsigned int *plts, unsigned int *gots)
{
unsigned int i, type;
for (i = 0; i < num; i++) {
type = ELF_R_TYPE(relas[i].r_info);
- if (type == R_LARCH_SOP_PUSH_PLT_PCREL) {
+ switch (type) {
+ case R_LARCH_SOP_PUSH_PLT_PCREL:
+ case R_LARCH_B26:
if (!duplicate_rela(relas, i))
(*plts)++;
+ break;
+ case R_LARCH_GOT_PC_HI20:
+ if (!duplicate_rela(relas, i))
+ (*gots)++;
+ break;
+ default:
+ break; /* Do nothing. */
}
}
}
int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
char *secstrings, struct module *mod)
{
- unsigned int i, num_plts = 0;
+ unsigned int i, num_plts = 0, num_gots = 0;
/*
* Find the empty .plt sections.
*/
for (i = 0; i < ehdr->e_shnum; i++) {
- if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
+ if (!strcmp(secstrings + sechdrs[i].sh_name, ".got"))
+ mod->arch.got.shdr = sechdrs + i;
+ else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
mod->arch.plt.shdr = sechdrs + i;
else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt.idx"))
mod->arch.plt_idx.shdr = sechdrs + i;
}
+ if (!mod->arch.got.shdr) {
+ pr_err("%s: module GOT section(s) missing\n", mod->name);
+ return -ENOEXEC;
+ }
if (!mod->arch.plt.shdr) {
pr_err("%s: module PLT section(s) missing\n", mod->name);
return -ENOEXEC;
if (!(dst_sec->sh_flags & SHF_EXECINSTR))
continue;
- count_max_entries(relas, num_rela, &num_plts);
+ count_max_entries(relas, num_rela, &num_plts, &num_gots);
}
+ mod->arch.got.shdr->sh_type = SHT_NOBITS;
+ mod->arch.got.shdr->sh_flags = SHF_ALLOC;
+ mod->arch.got.shdr->sh_addralign = L1_CACHE_BYTES;
+ mod->arch.got.shdr->sh_size = (num_gots + 1) * sizeof(struct got_entry);
+ mod->arch.got.num_entries = 0;
+ mod->arch.got.max_entries = num_gots;
+
mod->arch.plt.shdr->sh_type = SHT_NOBITS;
mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES;
#include <linux/string.h>
#include <linux/kernel.h>
-static inline bool signed_imm_check(long val, unsigned int bit)
-{
- return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1));
-}
-
-static inline bool unsigned_imm_check(unsigned long val, unsigned int bit)
-{
- return val < (1UL << bit);
-}
-
static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
{
if (*rela_stack_top >= RELA_STACK_DEPTH)
}
}
+static int apply_r_larch_b26(struct module *mod, u32 *location, Elf_Addr v,
+ s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
+{
+ ptrdiff_t offset = (void *)v - (void *)location;
+ union loongarch_instruction *insn = (union loongarch_instruction *)location;
+
+ if (offset >= SZ_128M)
+ v = module_emit_plt_entry(mod, v);
+
+ if (offset < -SZ_128M)
+ v = module_emit_plt_entry(mod, v);
+
+ offset = (void *)v - (void *)location;
+
+ if (offset & 3) {
+ pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
+ mod->name, (long long)offset, type);
+ return -ENOEXEC;
+ }
+
+ if (!signed_imm_check(offset, 28)) {
+ pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
+ mod->name, (long long)offset, type);
+ return -ENOEXEC;
+ }
+
+ offset >>= 2;
+ insn->reg0i26_format.immediate_l = offset & 0xffff;
+ insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
+
+ return 0;
+}
+
+static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
+ s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
+{
+ union loongarch_instruction *insn = (union loongarch_instruction *)location;
+ /* Use s32 for a sign-extension deliberately. */
+ s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
+ (void *)((Elf_Addr)location & ~0xfff);
+ Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
+ ptrdiff_t offset_rem = (void *)v - (void *)anchor;
+
+ switch (type) {
+ case R_LARCH_PCALA_LO12:
+ insn->reg2i12_format.immediate = v & 0xfff;
+ break;
+ case R_LARCH_PCALA_HI20:
+ v = offset_hi20 >> 12;
+ insn->reg1i20_format.immediate = v & 0xfffff;
+ break;
+ case R_LARCH_PCALA64_LO20:
+ v = offset_rem >> 32;
+ insn->reg1i20_format.immediate = v & 0xfffff;
+ break;
+ case R_LARCH_PCALA64_HI12:
+ v = offset_rem >> 52;
+ insn->reg2i12_format.immediate = v & 0xfff;
+ break;
+ default:
+ pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int apply_r_larch_got_pc(struct module *mod, u32 *location, Elf_Addr v,
+ s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
+{
+ Elf_Addr got = module_emit_got_entry(mod, v);
+
+ if (!got)
+ return -EINVAL;
+
+ switch (type) {
+ case R_LARCH_GOT_PC_LO12:
+ type = R_LARCH_PCALA_LO12;
+ break;
+ case R_LARCH_GOT_PC_HI20:
+ type = R_LARCH_PCALA_HI20;
+ break;
+ default:
+ pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
+ return -EINVAL;
+ }
+
+ return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
+}
+
/*
* reloc_handlers_rela() - Apply a particular relocation to a module
* @mod: the module to apply the reloc to
/* The handlers for known reloc types */
static reloc_rela_handler reloc_rela_handlers[] = {
- [R_LARCH_NONE ... R_LARCH_SUB64] = apply_r_larch_error,
+ [R_LARCH_NONE ... R_LARCH_RELAX] = apply_r_larch_error,
[R_LARCH_NONE] = apply_r_larch_none,
[R_LARCH_32] = apply_r_larch_32,
[R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop,
[R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
[R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub,
+ [R_LARCH_B26] = apply_r_larch_b26,
+ [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala,
+ [R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12] = apply_r_larch_got_pc,
};
int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux performance counter support for LoongArch.
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ *
+ * Derived from MIPS:
+ * Copyright (C) 2010 MIPS Technologies, Inc.
+ * Copyright (C) 2011 Cavium Networks, Inc.
+ * Author: Deng-Cheng Zhu
+ */
+
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/kernel.h>
+#include <linux/perf_event.h>
+#include <linux/uaccess.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/irq.h>
+#include <asm/irq_regs.h>
+#include <asm/stacktrace.h>
+#include <asm/unwind.h>
+
+/*
+ * Get the return address for a single stackframe and return a pointer to the
+ * next frame tail.
+ */
+static unsigned long
+user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp)
+{
+ unsigned long err;
+ unsigned long __user *user_frame_tail;
+ struct stack_frame buftail;
+
+ user_frame_tail = (unsigned long __user *)(fp - sizeof(struct stack_frame));
+
+ /* Also check accessibility of one struct frame_tail beyond */
+ if (!access_ok(user_frame_tail, sizeof(buftail)))
+ return 0;
+
+ pagefault_disable();
+ err = __copy_from_user_inatomic(&buftail, user_frame_tail, sizeof(buftail));
+ pagefault_enable();
+
+ if (err || (unsigned long)user_frame_tail >= buftail.fp)
+ return 0;
+
+ perf_callchain_store(entry, buftail.ra);
+
+ return buftail.fp;
+}
+
+void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+{
+ unsigned long fp;
+
+ if (perf_guest_state()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+
+ perf_callchain_store(entry, regs->csr_era);
+
+ fp = regs->regs[22];
+
+ while (entry->nr < entry->max_stack && fp && !((unsigned long)fp & 0xf))
+ fp = user_backtrace(entry, fp);
+}
+
+void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+{
+ struct unwind_state state;
+ unsigned long addr;
+
+ for (unwind_start(&state, current, regs);
+ !unwind_done(&state); unwind_next_frame(&state)) {
+ addr = unwind_get_return_address(&state);
+ if (!addr || perf_callchain_store(entry, addr))
+ return;
+ }
+}
+
+#define LOONGARCH_MAX_HWEVENTS 32
+
+struct cpu_hw_events {
+ /* Array of events on this cpu. */
+ struct perf_event *events[LOONGARCH_MAX_HWEVENTS];
+
+ /*
+ * Set the bit (indexed by the counter number) when the counter
+ * is used for an event.
+ */
+ unsigned long used_mask[BITS_TO_LONGS(LOONGARCH_MAX_HWEVENTS)];
+
+ /*
+ * Software copy of the control register for each performance counter.
+ */
+ unsigned int saved_ctrl[LOONGARCH_MAX_HWEVENTS];
+};
+static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
+ .saved_ctrl = {0},
+};
+
+/* The description of LoongArch performance events. */
+struct loongarch_perf_event {
+ unsigned int event_id;
+};
+
+static struct loongarch_perf_event raw_event;
+static DEFINE_MUTEX(raw_event_mutex);
+
+#define C(x) PERF_COUNT_HW_CACHE_##x
+#define HW_OP_UNSUPPORTED 0xffffffff
+#define CACHE_OP_UNSUPPORTED 0xffffffff
+
+#define PERF_MAP_ALL_UNSUPPORTED \
+ [0 ... PERF_COUNT_HW_MAX - 1] = {HW_OP_UNSUPPORTED}
+
+#define PERF_CACHE_MAP_ALL_UNSUPPORTED \
+[0 ... C(MAX) - 1] = { \
+ [0 ... C(OP_MAX) - 1] = { \
+ [0 ... C(RESULT_MAX) - 1] = {CACHE_OP_UNSUPPORTED}, \
+ }, \
+}
+
+struct loongarch_pmu {
+ u64 max_period;
+ u64 valid_count;
+ u64 overflow;
+ const char *name;
+ unsigned int num_counters;
+ u64 (*read_counter)(unsigned int idx);
+ void (*write_counter)(unsigned int idx, u64 val);
+ const struct loongarch_perf_event *(*map_raw_event)(u64 config);
+ const struct loongarch_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
+ const struct loongarch_perf_event (*cache_event_map)
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX];
+};
+
+static struct loongarch_pmu loongarch_pmu;
+
+#define M_PERFCTL_EVENT(event) (event & CSR_PERFCTRL_EVENT)
+
+#define M_PERFCTL_COUNT_EVENT_WHENEVER (CSR_PERFCTRL_PLV0 | \
+ CSR_PERFCTRL_PLV1 | \
+ CSR_PERFCTRL_PLV2 | \
+ CSR_PERFCTRL_PLV3 | \
+ CSR_PERFCTRL_IE)
+
+#define M_PERFCTL_CONFIG_MASK 0x1f0000
+
+static void pause_local_counters(void);
+static void resume_local_counters(void);
+
+static u64 loongarch_pmu_read_counter(unsigned int idx)
+{
+ u64 val = -1;
+
+ switch (idx) {
+ case 0:
+ val = read_csr_perfcntr0();
+ break;
+ case 1:
+ val = read_csr_perfcntr1();
+ break;
+ case 2:
+ val = read_csr_perfcntr2();
+ break;
+ case 3:
+ val = read_csr_perfcntr3();
+ break;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return 0;
+ }
+
+ return val;
+}
+
+static void loongarch_pmu_write_counter(unsigned int idx, u64 val)
+{
+ switch (idx) {
+ case 0:
+ write_csr_perfcntr0(val);
+ return;
+ case 1:
+ write_csr_perfcntr1(val);
+ return;
+ case 2:
+ write_csr_perfcntr2(val);
+ return;
+ case 3:
+ write_csr_perfcntr3(val);
+ return;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return;
+ }
+}
+
+static unsigned int loongarch_pmu_read_control(unsigned int idx)
+{
+ unsigned int val = -1;
+
+ switch (idx) {
+ case 0:
+ val = read_csr_perfctrl0();
+ break;
+ case 1:
+ val = read_csr_perfctrl1();
+ break;
+ case 2:
+ val = read_csr_perfctrl2();
+ break;
+ case 3:
+ val = read_csr_perfctrl3();
+ break;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return 0;
+ }
+
+ return val;
+}
+
+static void loongarch_pmu_write_control(unsigned int idx, unsigned int val)
+{
+ switch (idx) {
+ case 0:
+ write_csr_perfctrl0(val);
+ return;
+ case 1:
+ write_csr_perfctrl1(val);
+ return;
+ case 2:
+ write_csr_perfctrl2(val);
+ return;
+ case 3:
+ write_csr_perfctrl3(val);
+ return;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return;
+ }
+}
+
+static int loongarch_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
+{
+ int i;
+
+ for (i = 0; i < loongarch_pmu.num_counters; i++) {
+ if (!test_and_set_bit(i, cpuc->used_mask))
+ return i;
+ }
+
+ return -EAGAIN;
+}
+
+static void loongarch_pmu_enable_event(struct hw_perf_event *evt, int idx)
+{
+ unsigned int cpu;
+ struct perf_event *event = container_of(evt, struct perf_event, hw);
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters);
+
+ /* Make sure interrupt enabled. */
+ cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |
+ (evt->config_base & M_PERFCTL_CONFIG_MASK) | CSR_PERFCTRL_IE;
+
+ cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id();
+
+ /*
+ * We do not actually let the counter run. Leave it until start().
+ */
+ pr_debug("Enabling perf counter for CPU%d\n", cpu);
+}
+
+static void loongarch_pmu_disable_event(int idx)
+{
+ unsigned long flags;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters);
+
+ local_irq_save(flags);
+ cpuc->saved_ctrl[idx] = loongarch_pmu_read_control(idx) &
+ ~M_PERFCTL_COUNT_EVENT_WHENEVER;
+ loongarch_pmu_write_control(idx, cpuc->saved_ctrl[idx]);
+ local_irq_restore(flags);
+}
+
+static int loongarch_pmu_event_set_period(struct perf_event *event,
+ struct hw_perf_event *hwc,
+ int idx)
+{
+ int ret = 0;
+ u64 left = local64_read(&hwc->period_left);
+ u64 period = hwc->sample_period;
+
+ if (unlikely((left + period) & (1ULL << 63))) {
+ /* left underflowed by more than period. */
+ left = period;
+ local64_set(&hwc->period_left, left);
+ hwc->last_period = period;
+ ret = 1;
+ } else if (unlikely((left + period) <= period)) {
+ /* left underflowed by less than period. */
+ left += period;
+ local64_set(&hwc->period_left, left);
+ hwc->last_period = period;
+ ret = 1;
+ }
+
+ if (left > loongarch_pmu.max_period) {
+ left = loongarch_pmu.max_period;
+ local64_set(&hwc->period_left, left);
+ }
+
+ local64_set(&hwc->prev_count, loongarch_pmu.overflow - left);
+
+ loongarch_pmu.write_counter(idx, loongarch_pmu.overflow - left);
+
+ perf_event_update_userpage(event);
+
+ return ret;
+}
+
+static void loongarch_pmu_event_update(struct perf_event *event,
+ struct hw_perf_event *hwc,
+ int idx)
+{
+ u64 delta;
+ u64 prev_raw_count, new_raw_count;
+
+again:
+ prev_raw_count = local64_read(&hwc->prev_count);
+ new_raw_count = loongarch_pmu.read_counter(idx);
+
+ if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
+ new_raw_count) != prev_raw_count)
+ goto again;
+
+ delta = new_raw_count - prev_raw_count;
+
+ local64_add(delta, &event->count);
+ local64_sub(delta, &hwc->period_left);
+}
+
+static void loongarch_pmu_start(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+
+ hwc->state = 0;
+
+ /* Set the period for the event. */
+ loongarch_pmu_event_set_period(event, hwc, hwc->idx);
+
+ /* Enable the event. */
+ loongarch_pmu_enable_event(hwc, hwc->idx);
+}
+
+static void loongarch_pmu_stop(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (!(hwc->state & PERF_HES_STOPPED)) {
+ /* We are working on a local event. */
+ loongarch_pmu_disable_event(hwc->idx);
+ barrier();
+ loongarch_pmu_event_update(event, hwc, hwc->idx);
+ hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ }
+}
+
+static int loongarch_pmu_add(struct perf_event *event, int flags)
+{
+ int idx, err = 0;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+
+ perf_pmu_disable(event->pmu);
+
+ /* To look for a free counter for this event. */
+ idx = loongarch_pmu_alloc_counter(cpuc, hwc);
+ if (idx < 0) {
+ err = idx;
+ goto out;
+ }
+
+ /*
+ * If there is an event in the counter we are going to use then
+ * make sure it is disabled.
+ */
+ event->hw.idx = idx;
+ loongarch_pmu_disable_event(idx);
+ cpuc->events[idx] = event;
+
+ hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ if (flags & PERF_EF_START)
+ loongarch_pmu_start(event, PERF_EF_RELOAD);
+
+ /* Propagate our changes to the userspace mapping. */
+ perf_event_update_userpage(event);
+
+out:
+ perf_pmu_enable(event->pmu);
+ return err;
+}
+
+static void loongarch_pmu_del(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->idx;
+
+ WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters);
+
+ loongarch_pmu_stop(event, PERF_EF_UPDATE);
+ cpuc->events[idx] = NULL;
+ clear_bit(idx, cpuc->used_mask);
+
+ perf_event_update_userpage(event);
+}
+
+static void loongarch_pmu_read(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ /* Don't read disabled counters! */
+ if (hwc->idx < 0)
+ return;
+
+ loongarch_pmu_event_update(event, hwc, hwc->idx);
+}
+
+static void loongarch_pmu_enable(struct pmu *pmu)
+{
+ resume_local_counters();
+}
+
+static void loongarch_pmu_disable(struct pmu *pmu)
+{
+ pause_local_counters();
+}
+
+static DEFINE_MUTEX(pmu_reserve_mutex);
+static atomic_t active_events = ATOMIC_INIT(0);
+
+static int get_pmc_irq(void)
+{
+ struct irq_domain *d = irq_find_matching_fwnode(cpuintc_handle, DOMAIN_BUS_ANY);
+
+ if (d)
+ return irq_create_mapping(d, EXCCODE_PMC - EXCCODE_INT_START);
+
+ return -EINVAL;
+}
+
+static void reset_counters(void *arg);
+static int __hw_perf_event_init(struct perf_event *event);
+
+static void hw_perf_event_destroy(struct perf_event *event)
+{
+ if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) {
+ on_each_cpu(reset_counters, NULL, 1);
+ free_irq(get_pmc_irq(), &loongarch_pmu);
+ mutex_unlock(&pmu_reserve_mutex);
+ }
+}
+
+static void handle_associated_event(struct cpu_hw_events *cpuc, int idx,
+ struct perf_sample_data *data, struct pt_regs *regs)
+{
+ struct perf_event *event = cpuc->events[idx];
+ struct hw_perf_event *hwc = &event->hw;
+
+ loongarch_pmu_event_update(event, hwc, idx);
+ data->period = event->hw.last_period;
+ if (!loongarch_pmu_event_set_period(event, hwc, idx))
+ return;
+
+ if (perf_event_overflow(event, data, regs))
+ loongarch_pmu_disable_event(idx);
+}
+
+static irqreturn_t pmu_handle_irq(int irq, void *dev)
+{
+ int n;
+ int handled = IRQ_NONE;
+ uint64_t counter;
+ struct pt_regs *regs;
+ struct perf_sample_data data;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ /*
+ * First we pause the local counters, so that when we are locked
+ * here, the counters are all paused. When it gets locked due to
+ * perf_disable(), the timer interrupt handler will be delayed.
+ *
+ * See also loongarch_pmu_start().
+ */
+ pause_local_counters();
+
+ regs = get_irq_regs();
+
+ perf_sample_data_init(&data, 0, 0);
+
+ for (n = 0; n < loongarch_pmu.num_counters; n++) {
+ if (test_bit(n, cpuc->used_mask)) {
+ counter = loongarch_pmu.read_counter(n);
+ if (counter & loongarch_pmu.overflow) {
+ handle_associated_event(cpuc, n, &data, regs);
+ handled = IRQ_HANDLED;
+ }
+ }
+ }
+
+ resume_local_counters();
+
+ /*
+ * Do all the work for the pending perf events. We can do this
+ * in here because the performance counter interrupt is a regular
+ * interrupt, not NMI.
+ */
+ if (handled == IRQ_HANDLED)
+ irq_work_run();
+
+ return handled;
+}
+
+static int loongarch_pmu_event_init(struct perf_event *event)
+{
+ int r, irq;
+ unsigned long flags;
+
+ /* does not support taken branch sampling */
+ if (has_branch_stack(event))
+ return -EOPNOTSUPP;
+
+ switch (event->attr.type) {
+ case PERF_TYPE_RAW:
+ case PERF_TYPE_HARDWARE:
+ case PERF_TYPE_HW_CACHE:
+ break;
+
+ default:
+ /* Init it to avoid false validate_group */
+ event->hw.event_base = 0xffffffff;
+ return -ENOENT;
+ }
+
+ if (event->cpu >= 0 && !cpu_online(event->cpu))
+ return -ENODEV;
+
+ irq = get_pmc_irq();
+ flags = IRQF_PERCPU | IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_SUSPEND | IRQF_SHARED;
+ if (!atomic_inc_not_zero(&active_events)) {
+ mutex_lock(&pmu_reserve_mutex);
+ if (atomic_read(&active_events) == 0) {
+ r = request_irq(irq, pmu_handle_irq, flags, "Perf_PMU", &loongarch_pmu);
+ if (r < 0) {
+ mutex_unlock(&pmu_reserve_mutex);
+ pr_warn("PMU IRQ request failed\n");
+ return -ENODEV;
+ }
+ }
+ atomic_inc(&active_events);
+ mutex_unlock(&pmu_reserve_mutex);
+ }
+
+ return __hw_perf_event_init(event);
+}
+
+static struct pmu pmu = {
+ .pmu_enable = loongarch_pmu_enable,
+ .pmu_disable = loongarch_pmu_disable,
+ .event_init = loongarch_pmu_event_init,
+ .add = loongarch_pmu_add,
+ .del = loongarch_pmu_del,
+ .start = loongarch_pmu_start,
+ .stop = loongarch_pmu_stop,
+ .read = loongarch_pmu_read,
+};
+
+static unsigned int loongarch_pmu_perf_event_encode(const struct loongarch_perf_event *pev)
+{
+ return (pev->event_id & 0xff);
+}
+
+static const struct loongarch_perf_event *loongarch_pmu_map_general_event(int idx)
+{
+ const struct loongarch_perf_event *pev;
+
+ pev = &(*loongarch_pmu.general_event_map)[idx];
+
+ if (pev->event_id == HW_OP_UNSUPPORTED)
+ return ERR_PTR(-ENOENT);
+
+ return pev;
+}
+
+static const struct loongarch_perf_event *loongarch_pmu_map_cache_event(u64 config)
+{
+ unsigned int cache_type, cache_op, cache_result;
+ const struct loongarch_perf_event *pev;
+
+ cache_type = (config >> 0) & 0xff;
+ if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
+ return ERR_PTR(-EINVAL);
+
+ cache_op = (config >> 8) & 0xff;
+ if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
+ return ERR_PTR(-EINVAL);
+
+ cache_result = (config >> 16) & 0xff;
+ if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
+ return ERR_PTR(-EINVAL);
+
+ pev = &((*loongarch_pmu.cache_event_map)
+ [cache_type]
+ [cache_op]
+ [cache_result]);
+
+ if (pev->event_id == CACHE_OP_UNSUPPORTED)
+ return ERR_PTR(-ENOENT);
+
+ return pev;
+}
+
+static int validate_group(struct perf_event *event)
+{
+ struct cpu_hw_events fake_cpuc;
+ struct perf_event *sibling, *leader = event->group_leader;
+
+ memset(&fake_cpuc, 0, sizeof(fake_cpuc));
+
+ if (loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0)
+ return -EINVAL;
+
+ for_each_sibling_event(sibling, leader) {
+ if (loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0)
+ return -EINVAL;
+ }
+
+ if (loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0)
+ return -EINVAL;
+
+ return 0;
+}
+
+static void reset_counters(void *arg)
+{
+ int n;
+ int counters = loongarch_pmu.num_counters;
+
+ for (n = 0; n < counters; n++) {
+ loongarch_pmu_write_control(n, 0);
+ loongarch_pmu.write_counter(n, 0);
+ }
+}
+
+static const struct loongarch_perf_event loongson_event_map[PERF_COUNT_HW_MAX] = {
+ PERF_MAP_ALL_UNSUPPORTED,
+ [PERF_COUNT_HW_CPU_CYCLES] = { 0x00 },
+ [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01 },
+ [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x08 },
+ [PERF_COUNT_HW_CACHE_MISSES] = { 0x09 },
+ [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02 },
+ [PERF_COUNT_HW_BRANCH_MISSES] = { 0x03 },
+};
+
+static const struct loongarch_perf_event loongson_cache_map
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+PERF_CACHE_MAP_ALL_UNSUPPORTED,
+[C(L1D)] = {
+ /*
+ * Like some other architectures (e.g. ARM), the performance
+ * counters don't differentiate between read and write
+ * accesses/misses, so this isn't strictly correct, but it's the
+ * best we can do. Writes and reads get combined.
+ */
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x8 },
+ [C(RESULT_MISS)] = { 0x9 },
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = { 0x8 },
+ [C(RESULT_MISS)] = { 0x9 },
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = { 0xaa },
+ [C(RESULT_MISS)] = { 0xa9 },
+ },
+},
+[C(L1I)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x6 },
+ [C(RESULT_MISS)] = { 0x7 },
+ },
+},
+[C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0xc },
+ [C(RESULT_MISS)] = { 0xd },
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = { 0xc },
+ [C(RESULT_MISS)] = { 0xd },
+ },
+},
+[C(ITLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_MISS)] = { 0x3b },
+ },
+},
+[C(DTLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x4 },
+ [C(RESULT_MISS)] = { 0x3c },
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = { 0x4 },
+ [C(RESULT_MISS)] = { 0x3c },
+ },
+},
+[C(BPU)] = {
+ /* Using the same code for *HW_BRANCH* */
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x02 },
+ [C(RESULT_MISS)] = { 0x03 },
+ },
+},
+};
+
+static int __hw_perf_event_init(struct perf_event *event)
+{
+ int err;
+ struct hw_perf_event *hwc = &event->hw;
+ struct perf_event_attr *attr = &event->attr;
+ const struct loongarch_perf_event *pev;
+
+ /* Returning LoongArch event descriptor for generic perf event. */
+ if (PERF_TYPE_HARDWARE == event->attr.type) {
+ if (event->attr.config >= PERF_COUNT_HW_MAX)
+ return -EINVAL;
+ pev = loongarch_pmu_map_general_event(event->attr.config);
+ } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
+ pev = loongarch_pmu_map_cache_event(event->attr.config);
+ } else if (PERF_TYPE_RAW == event->attr.type) {
+ /* We are working on the global raw event. */
+ mutex_lock(&raw_event_mutex);
+ pev = loongarch_pmu.map_raw_event(event->attr.config);
+ } else {
+ /* The event type is not (yet) supported. */
+ return -EOPNOTSUPP;
+ }
+
+ if (IS_ERR(pev)) {
+ if (PERF_TYPE_RAW == event->attr.type)
+ mutex_unlock(&raw_event_mutex);
+ return PTR_ERR(pev);
+ }
+
+ /*
+ * We allow max flexibility on how each individual counter shared
+ * by the single CPU operates (the mode exclusion and the range).
+ */
+ hwc->config_base = CSR_PERFCTRL_IE;
+
+ hwc->event_base = loongarch_pmu_perf_event_encode(pev);
+ if (PERF_TYPE_RAW == event->attr.type)
+ mutex_unlock(&raw_event_mutex);
+
+ if (!attr->exclude_user) {
+ hwc->config_base |= CSR_PERFCTRL_PLV3;
+ hwc->config_base |= CSR_PERFCTRL_PLV2;
+ }
+ if (!attr->exclude_kernel) {
+ hwc->config_base |= CSR_PERFCTRL_PLV0;
+ }
+ if (!attr->exclude_hv) {
+ hwc->config_base |= CSR_PERFCTRL_PLV1;
+ }
+
+ hwc->config_base &= M_PERFCTL_CONFIG_MASK;
+ /*
+ * The event can belong to another cpu. We do not assign a local
+ * counter for it for now.
+ */
+ hwc->idx = -1;
+ hwc->config = 0;
+
+ if (!hwc->sample_period) {
+ hwc->sample_period = loongarch_pmu.max_period;
+ hwc->last_period = hwc->sample_period;
+ local64_set(&hwc->period_left, hwc->sample_period);
+ }
+
+ err = 0;
+ if (event->group_leader != event)
+ err = validate_group(event);
+
+ event->destroy = hw_perf_event_destroy;
+
+ if (err)
+ event->destroy(event);
+
+ return err;
+}
+
+static void pause_local_counters(void)
+{
+ unsigned long flags;
+ int ctr = loongarch_pmu.num_counters;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ local_irq_save(flags);
+ do {
+ ctr--;
+ cpuc->saved_ctrl[ctr] = loongarch_pmu_read_control(ctr);
+ loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] &
+ ~M_PERFCTL_COUNT_EVENT_WHENEVER);
+ } while (ctr > 0);
+ local_irq_restore(flags);
+}
+
+static void resume_local_counters(void)
+{
+ int ctr = loongarch_pmu.num_counters;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ do {
+ ctr--;
+ loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]);
+ } while (ctr > 0);
+}
+
+static const struct loongarch_perf_event *loongarch_pmu_map_raw_event(u64 config)
+{
+ raw_event.event_id = config & 0xff;
+
+ return &raw_event;
+}
+
+static int __init init_hw_perf_events(void)
+{
+ int counters;
+
+ if (!cpu_has_pmp)
+ return -ENODEV;
+
+ pr_info("Performance counters: ");
+ counters = ((read_cpucfg(LOONGARCH_CPUCFG6) & CPUCFG6_PMNUM) >> 4) + 1;
+
+ loongarch_pmu.num_counters = counters;
+ loongarch_pmu.max_period = (1ULL << 63) - 1;
+ loongarch_pmu.valid_count = (1ULL << 63) - 1;
+ loongarch_pmu.overflow = 1ULL << 63;
+ loongarch_pmu.name = "loongarch/loongson64";
+ loongarch_pmu.read_counter = loongarch_pmu_read_counter;
+ loongarch_pmu.write_counter = loongarch_pmu_write_counter;
+ loongarch_pmu.map_raw_event = loongarch_pmu_map_raw_event;
+ loongarch_pmu.general_event_map = &loongson_event_map;
+ loongarch_pmu.cache_event_map = &loongson_cache_map;
+
+ on_each_cpu(reset_counters, NULL, 1);
+
+ pr_cont("%s PMU enabled, %d %d-bit counters available to each CPU.\n",
+ loongarch_pmu.name, counters, 64);
+
+ perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
+
+ return 0;
+}
+early_initcall(init_hw_perf_events);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ *
+ * Derived from MIPS:
+ * Copyright (C) 2013 Cavium, Inc.
+ */
+
+#include <linux/perf_event.h>
+
+#include <asm/ptrace.h>
+
+#ifdef CONFIG_32BIT
+u64 perf_reg_abi(struct task_struct *tsk)
+{
+ return PERF_SAMPLE_REGS_ABI_32;
+}
+#else /* Must be CONFIG_64BIT */
+u64 perf_reg_abi(struct task_struct *tsk)
+{
+ if (test_tsk_thread_flag(tsk, TIF_32BIT_REGS))
+ return PERF_SAMPLE_REGS_ABI_32;
+ else
+ return PERF_SAMPLE_REGS_ABI_64;
+}
+#endif /* CONFIG_32BIT */
+
+int perf_reg_validate(u64 mask)
+{
+ if (!mask)
+ return -EINVAL;
+ if (mask & ~((1ull << PERF_REG_LOONGARCH_MAX) - 1))
+ return -EINVAL;
+ return 0;
+}
+
+u64 perf_reg_value(struct pt_regs *regs, int idx)
+{
+ if (WARN_ON_ONCE((u32)idx >= PERF_REG_LOONGARCH_MAX))
+ return 0;
+
+ if ((u32)idx == PERF_REG_LOONGARCH_PC)
+ return regs->csr_era;
+
+ return regs->regs[idx];
+}
+
+void perf_get_regs_user(struct perf_regs *regs_user,
+ struct pt_regs *regs)
+{
+ regs_user->regs = task_pt_regs(current);
+ regs_user->abi = perf_reg_abi(current);
+}
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
+ sp -= prandom_u32_max(PAGE_SIZE);
return sp & STACK_ALIGN;
}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * relocate_kernel.S for kexec
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kexec.h>
+
+#include <asm/asm.h>
+#include <asm/asmmacro.h>
+#include <asm/regdef.h>
+#include <asm/loongarch.h>
+#include <asm/stackframe.h>
+#include <asm/addrspace.h>
+
+SYM_CODE_START(relocate_new_kernel)
+ /*
+ * a0: EFI boot flag for the new kernel
+ * a1: Command line pointer for the new kernel
+ * a2: System table pointer for the new kernel
+ * a3: Start address to jump to after relocation
+ * a4: Pointer to the current indirection page entry
+ */
+ move s0, a4
+
+ /*
+ * In case of a kdump/crash kernel, the indirection page is not
+ * populated as the kernel is directly copied to a reserved location
+ */
+ beqz s0, done
+
+process_entry:
+ PTR_L s1, s0, 0
+ PTR_ADDI s0, s0, SZREG
+
+ /* destination page */
+ andi s2, s1, IND_DESTINATION
+ beqz s2, 1f
+ li.w t0, ~0x1
+ and s3, s1, t0 /* store destination addr in s3 */
+ b process_entry
+
+1:
+ /* indirection page, update s0 */
+ andi s2, s1, IND_INDIRECTION
+ beqz s2, 1f
+ li.w t0, ~0x2
+ and s0, s1, t0
+ b process_entry
+
+1:
+ /* done page */
+ andi s2, s1, IND_DONE
+ beqz s2, 1f
+ b done
+
+1:
+ /* source page */
+ andi s2, s1, IND_SOURCE
+ beqz s2, process_entry
+ li.w t0, ~0x8
+ and s1, s1, t0
+ li.w s5, (1 << _PAGE_SHIFT) / SZREG
+
+copy_word:
+ /* copy page word by word */
+ REG_L s4, s1, 0
+ REG_S s4, s3, 0
+ PTR_ADDI s3, s3, SZREG
+ PTR_ADDI s1, s1, SZREG
+ LONG_ADDI s5, s5, -1
+ beqz s5, process_entry
+ b copy_word
+ b process_entry
+
+done:
+ ibar 0
+ dbar 0
+
+ /*
+ * Jump to the new kernel,
+ * make sure the values of a0, a1, a2 and a3 are not changed.
+ */
+ jr a3
+SYM_CODE_END(relocate_new_kernel)
+
+#ifdef CONFIG_SMP
+/*
+ * Other CPUs should wait until code is relocated and
+ * then start at the entry point from LOONGARCH_IOCSR_MBUF0.
+ */
+SYM_CODE_START(kexec_smp_wait)
+1: li.w t0, 0x100 /* wait for init loop */
+2: addi.w t0, t0, -1 /* limit mailbox access */
+ bnez t0, 2b
+ li.w t1, LOONGARCH_IOCSR_MBUF0
+ iocsrrd.w s0, t1 /* check PC as an indicator */
+ beqz s0, 1b
+ iocsrrd.d s0, t1 /* get PC via mailbox */
+
+ li.d t0, CACHE_BASE
+ or s0, s0, t0 /* s0 = TO_CACHE(s0) */
+ jr s0 /* jump to initial PC */
+SYM_CODE_END(kexec_smp_wait)
+#endif
+
+relocate_new_kernel_end:
+
+SYM_DATA_START(relocate_new_kernel_size)
+ PTR relocate_new_kernel_end - relocate_new_kernel
+SYM_DATA_END(relocate_new_kernel_size)
#include <linux/memblock.h>
#include <linux/initrd.h>
#include <linux/ioport.h>
+#include <linux/kexec.h>
+#include <linux/crash_dump.h>
#include <linux/root_dev.h>
#include <linux/console.h>
#include <linux/pfn.h>
}
early_param("mem", early_parse_mem);
+static void __init arch_reserve_vmcore(void)
+{
+#ifdef CONFIG_PROC_VMCORE
+ u64 i;
+ phys_addr_t start, end;
+
+ if (!is_kdump_kernel())
+ return;
+
+ if (!elfcorehdr_size) {
+ for_each_mem_range(i, &start, &end) {
+ if (elfcorehdr_addr >= start && elfcorehdr_addr < end) {
+ /*
+ * Reserve from the elf core header to the end of
+ * the memory segment, that should all be kdump
+ * reserved memory.
+ */
+ elfcorehdr_size = end - elfcorehdr_addr;
+ break;
+ }
+ }
+ }
+
+ if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) {
+ pr_warn("elfcorehdr is overlapped\n");
+ return;
+ }
+
+ memblock_reserve(elfcorehdr_addr, elfcorehdr_size);
+
+ pr_info("Reserving %llu KiB of memory at 0x%llx for elfcorehdr\n",
+ elfcorehdr_size >> 10, elfcorehdr_addr);
+#endif
+}
+
+static void __init arch_parse_crashkernel(void)
+{
+#ifdef CONFIG_KEXEC
+ int ret;
+ unsigned long long start;
+ unsigned long long total_mem;
+ unsigned long long crash_base, crash_size;
+
+ total_mem = memblock_phys_mem_size();
+ ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base);
+ if (ret < 0 || crash_size <= 0)
+ return;
+
+ start = memblock_phys_alloc_range(crash_size, 1, crash_base, crash_base + crash_size);
+ if (start != crash_base) {
+ pr_warn("Invalid memory region reserved for crash kernel\n");
+ return;
+ }
+
+ crashk_res.start = crash_base;
+ crashk_res.end = crash_base + crash_size - 1;
+#endif
+}
+
void __init platform_init(void)
{
+ arch_reserve_vmcore();
+ arch_parse_crashkernel();
+
#ifdef CONFIG_ACPI_TABLE_UPGRADE
acpi_table_upgrade();
#endif
request_resource(res, &data_resource);
request_resource(res, &bss_resource);
}
+
+#ifdef CONFIG_KEXEC
+ if (crashk_res.start < crashk_res.end) {
+ insert_resource(&iomem_resource, &crashk_res);
+ pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n",
+ (unsigned long)((crashk_res.end - crashk_res.start + 1) >> 20),
+ (unsigned long)(crashk_res.start >> 20));
+ }
+#endif
}
static int __init reserve_memblock_reserved_regions(void)
init_environ();
efi_init();
memblock_init();
+ pagetable_init();
parse_early_param();
+ reserve_initrd_mem();
platform_init();
- pagetable_init();
arch_mem_init(cmdline_p);
resource_init();
#ifdef CONFIG_HOTPLUG_CPU
-static bool io_master(int cpu)
-{
- return test_bit(cpu, &loongson_sysconf.cores_io_master);
-}
-
int loongson3_cpu_disable(void)
{
unsigned long flags;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * LoongArch specific sysrq operations.
+ *
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/spinlock.h>
+#include <linux/sysrq.h>
+#include <linux/workqueue.h>
+
+#include <asm/cpu-features.h>
+#include <asm/tlb.h>
+
+/*
+ * Dump TLB entries on all CPUs.
+ */
+
+static DEFINE_SPINLOCK(show_lock);
+
+static void sysrq_tlbdump_single(void *dummy)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&show_lock, flags);
+
+ pr_info("CPU%d:\n", smp_processor_id());
+ dump_tlb_regs();
+ pr_info("\n");
+ dump_tlb_all();
+ pr_info("\n");
+
+ spin_unlock_irqrestore(&show_lock, flags);
+}
+
+#ifdef CONFIG_SMP
+static void sysrq_tlbdump_othercpus(struct work_struct *dummy)
+{
+ smp_call_function(sysrq_tlbdump_single, NULL, 0);
+}
+
+static DECLARE_WORK(sysrq_tlbdump, sysrq_tlbdump_othercpus);
+#endif
+
+static void sysrq_handle_tlbdump(int key)
+{
+ sysrq_tlbdump_single(NULL);
+#ifdef CONFIG_SMP
+ schedule_work(&sysrq_tlbdump);
+#endif
+}
+
+static struct sysrq_key_op sysrq_tlbdump_op = {
+ .handler = sysrq_handle_tlbdump,
+ .help_msg = "show-tlbs(x)",
+ .action_msg = "Show TLB entries",
+ .enable_mask = SYSRQ_ENABLE_DUMP,
+};
+
+static int __init loongarch_sysrq_init(void)
+{
+ return register_sysrq_key('x', &sysrq_tlbdump_op);
+}
+arch_initcall(loongarch_sysrq_init);
#include <linux/node.h>
#include <linux/nodemask.h>
#include <linux/percpu.h>
+#include <asm/bootinfo.h>
static DEFINE_PER_CPU(struct cpu, cpu_devices);
for_each_present_cpu(i) {
struct cpu *c = &per_cpu(cpu_devices, i);
- c->hotpluggable = !!i;
+ c->hotpluggable = !io_master(i);
ret = register_cpu(c, i);
if (ret < 0)
pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret);
#include <linux/entry-common.h>
#include <linux/init.h>
#include <linux/kernel.h>
+#include <linux/kexec.h>
#include <linux/module.h>
#include <linux/extable.h>
#include <linux/mm.h>
oops_exit();
+ if (regs && kexec_should_crash(current))
+ crash_kexec(regs);
+
if (in_interrupt())
panic("Fatal exception in interrupt");
irqentry_exit(regs, state);
}
+#ifdef CONFIG_GENERIC_BUG
+int is_valid_bugaddr(unsigned long addr)
+{
+ return 1;
+}
+#endif /* CONFIG_GENERIC_BUG */
+
+static void bug_handler(struct pt_regs *regs)
+{
+ switch (report_bug(regs->csr_era, regs)) {
+ case BUG_TRAP_TYPE_BUG:
+ case BUG_TRAP_TYPE_NONE:
+ die_if_kernel("Oops - BUG", regs);
+ force_sig(SIGTRAP);
+ break;
+
+ case BUG_TRAP_TYPE_WARN:
+ /* Skip the BUG instruction and continue */
+ regs->csr_era += LOONGARCH_INSN_SIZE;
+ break;
+ }
+}
+
asmlinkage void noinstr do_bp(struct pt_regs *regs)
{
bool user = user_mode(regs);
switch (bcode) {
case BRK_BUG:
- die_if_kernel("Kernel bug detected", regs);
- force_sig(SIGTRAP);
+ bug_handler(regs);
break;
case BRK_DIVZERO:
die_if_kernel("Break instruction in kernel code", regs);
irqentry_exit(regs, state);
}
-extern void tlb_init(int cpu);
-extern void cache_error_setup(void);
-
unsigned long eentry;
unsigned long tlbrentry;
unsigned long base = STACK_TOP;
if (current->flags & PF_RANDOMIZE) {
- base += get_random_int() & (VDSO_RANDOMIZE_SIZE - 1);
+ base += prandom_u32_max(VDSO_RANDOMIZE_SIZE);
base = PAGE_ALIGN(base);
}
EXCEPTION_TABLE(16)
+ .got : ALIGN(16) { *(.got) }
+ .plt : ALIGN(16) { *(.plt) }
+ .got.plt : ALIGN(16) { *(.got.plt) }
+
. = ALIGN(PECOFF_SEGMENT_ALIGN);
__init_begin = .;
__inittext_begin = .;
* Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
* Copyright (C) 2007 MIPS Technologies, Inc.
*/
+#include <linux/cacheinfo.h>
#include <linux/export.h>
-#include <linux/fcntl.h>
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/syscalls.h>
+#include <asm/bootinfo.h>
#include <asm/cacheflush.h>
#include <asm/cpu.h>
#include <asm/cpu-features.h>
-#include <asm/dma.h>
#include <asm/loongarch.h>
+#include <asm/numa.h>
#include <asm/processor.h>
#include <asm/setup.h>
+void cache_error_setup(void)
+{
+ extern char __weak except_vec_cex;
+ set_merr_handler(0x0, &except_vec_cex, 0x80);
+}
+
/*
* LoongArch maintains ICache/DCache coherency by hardware,
* we just need "ibar" to avoid instruction hazard here.
}
EXPORT_SYMBOL(local_flush_icache_range);
-void cache_error_setup(void)
-{
- extern char __weak except_vec_cex;
- set_merr_handler(0x0, &except_vec_cex, 0x80);
-}
-
-static unsigned long icache_size __read_mostly;
-static unsigned long dcache_size __read_mostly;
-static unsigned long vcache_size __read_mostly;
-static unsigned long scache_size __read_mostly;
-
-static char *way_string[] = { NULL, "direct mapped", "2-way",
- "3-way", "4-way", "5-way", "6-way", "7-way", "8-way",
- "9-way", "10-way", "11-way", "12-way",
- "13-way", "14-way", "15-way", "16-way",
-};
-
-static void probe_pcache(void)
+static void flush_cache_leaf(unsigned int leaf)
{
- struct cpuinfo_loongarch *c = ¤t_cpu_data;
- unsigned int lsize, sets, ways;
- unsigned int config;
-
- config = read_cpucfg(LOONGARCH_CPUCFG17);
- lsize = 1 << ((config & CPUCFG17_L1I_SIZE_M) >> CPUCFG17_L1I_SIZE);
- sets = 1 << ((config & CPUCFG17_L1I_SETS_M) >> CPUCFG17_L1I_SETS);
- ways = ((config & CPUCFG17_L1I_WAYS_M) >> CPUCFG17_L1I_WAYS) + 1;
-
- c->icache.linesz = lsize;
- c->icache.sets = sets;
- c->icache.ways = ways;
- icache_size = sets * ways * lsize;
- c->icache.waysize = icache_size / c->icache.ways;
-
- config = read_cpucfg(LOONGARCH_CPUCFG18);
- lsize = 1 << ((config & CPUCFG18_L1D_SIZE_M) >> CPUCFG18_L1D_SIZE);
- sets = 1 << ((config & CPUCFG18_L1D_SETS_M) >> CPUCFG18_L1D_SETS);
- ways = ((config & CPUCFG18_L1D_WAYS_M) >> CPUCFG18_L1D_WAYS) + 1;
-
- c->dcache.linesz = lsize;
- c->dcache.sets = sets;
- c->dcache.ways = ways;
- dcache_size = sets * ways * lsize;
- c->dcache.waysize = dcache_size / c->dcache.ways;
-
- c->options |= LOONGARCH_CPU_PREFETCH;
-
- pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
- icache_size >> 10, way_string[c->icache.ways], "VIPT", c->icache.linesz);
-
- pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
- dcache_size >> 10, way_string[c->dcache.ways], "VIPT", "no aliases", c->dcache.linesz);
+ int i, j, nr_nodes;
+ uint64_t addr = CSR_DMW0_BASE;
+ struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf;
+
+ nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes;
+
+ do {
+ for (i = 0; i < cdesc->sets; i++) {
+ for (j = 0; j < cdesc->ways; j++) {
+ flush_cache_line(leaf, addr);
+ addr++;
+ }
+
+ addr -= cdesc->ways;
+ addr += cdesc->linesz;
+ }
+ addr += (1ULL << NODE_ADDRSPACE_SHIFT);
+ } while (--nr_nodes > 0);
}
-static void probe_vcache(void)
+asmlinkage __visible void __flush_cache_all(void)
{
- struct cpuinfo_loongarch *c = ¤t_cpu_data;
- unsigned int lsize, sets, ways;
- unsigned int config;
-
- config = read_cpucfg(LOONGARCH_CPUCFG19);
- lsize = 1 << ((config & CPUCFG19_L2_SIZE_M) >> CPUCFG19_L2_SIZE);
- sets = 1 << ((config & CPUCFG19_L2_SETS_M) >> CPUCFG19_L2_SETS);
- ways = ((config & CPUCFG19_L2_WAYS_M) >> CPUCFG19_L2_WAYS) + 1;
-
- c->vcache.linesz = lsize;
- c->vcache.sets = sets;
- c->vcache.ways = ways;
- vcache_size = lsize * sets * ways;
- c->vcache.waysize = vcache_size / c->vcache.ways;
-
- pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n",
- vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz);
+ int leaf;
+ struct cache_desc *cdesc = current_cpu_data.cache_leaves;
+ unsigned int cache_present = current_cpu_data.cache_leaves_present;
+
+ leaf = cache_present - 1;
+ if (cache_inclusive(cdesc + leaf)) {
+ flush_cache_leaf(leaf);
+ return;
+ }
+
+ for (leaf = 0; leaf < cache_present; leaf++)
+ flush_cache_leaf(leaf);
}
-static void probe_scache(void)
-{
- struct cpuinfo_loongarch *c = ¤t_cpu_data;
- unsigned int lsize, sets, ways;
- unsigned int config;
-
- config = read_cpucfg(LOONGARCH_CPUCFG20);
- lsize = 1 << ((config & CPUCFG20_L3_SIZE_M) >> CPUCFG20_L3_SIZE);
- sets = 1 << ((config & CPUCFG20_L3_SETS_M) >> CPUCFG20_L3_SETS);
- ways = ((config & CPUCFG20_L3_WAYS_M) >> CPUCFG20_L3_WAYS) + 1;
-
- c->scache.linesz = lsize;
- c->scache.sets = sets;
- c->scache.ways = ways;
- /* 4 cores. scaches are shared */
- scache_size = lsize * sets * ways;
- c->scache.waysize = scache_size / c->scache.ways;
-
- pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
- scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
-}
+#define L1IUPRE (1 << 0)
+#define L1IUUNIFY (1 << 1)
+#define L1DPRE (1 << 2)
+
+#define LXIUPRE (1 << 0)
+#define LXIUUNIFY (1 << 1)
+#define LXIUPRIV (1 << 2)
+#define LXIUINCL (1 << 3)
+#define LXDPRE (1 << 4)
+#define LXDPRIV (1 << 5)
+#define LXDINCL (1 << 6)
+
+#define populate_cache_properties(cfg0, cdesc, level, leaf) \
+do { \
+ unsigned int cfg1; \
+ \
+ cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \
+ if (level == 1) { \
+ cdesc->flags |= CACHE_PRIVATE; \
+ } else { \
+ if (cfg0 & LXIUPRIV) \
+ cdesc->flags |= CACHE_PRIVATE; \
+ if (cfg0 & LXIUINCL) \
+ cdesc->flags |= CACHE_INCLUSIVE; \
+ } \
+ cdesc->level = level; \
+ cdesc->flags |= CACHE_PRESENT; \
+ cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \
+ cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \
+ cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \
+ cdesc++; leaf++; \
+} while (0)
void cpu_cache_init(void)
{
- probe_pcache();
- probe_vcache();
- probe_scache();
-
+ unsigned int leaf = 0, level = 1;
+ unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16);
+ struct cache_desc *cdesc = current_cpu_data.cache_leaves;
+
+ if (config & L1IUPRE) {
+ if (config & L1IUUNIFY)
+ cdesc->type = CACHE_TYPE_UNIFIED;
+ else
+ cdesc->type = CACHE_TYPE_INST;
+ populate_cache_properties(config, cdesc, level, leaf);
+ }
+
+ if (config & L1DPRE) {
+ cdesc->type = CACHE_TYPE_DATA;
+ populate_cache_properties(config, cdesc, level, leaf);
+ }
+
+ config = config >> 3;
+ for (level = 2; level <= CACHE_LEVEL_MAX; level++) {
+ if (!config)
+ break;
+
+ if (config & LXIUPRE) {
+ if (config & LXIUUNIFY)
+ cdesc->type = CACHE_TYPE_UNIFIED;
+ else
+ cdesc->type = CACHE_TYPE_INST;
+ populate_cache_properties(config, cdesc, level, leaf);
+ }
+
+ if (config & LXDPRE) {
+ cdesc->type = CACHE_TYPE_DATA;
+ populate_cache_properties(config, cdesc, level, leaf);
+ }
+
+ config = config >> 7;
+ }
+
+ BUG_ON(leaf > CACHE_LEAVES_MAX);
+
+ current_cpu_data.cache_leaves_present = leaf;
+ current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
shm_align_mask = PAGE_SIZE - 1;
}
#endif
#endif
+static pte_t *fixmap_pte(unsigned long addr)
+{
+ pgd_t *pgd;
+ p4d_t *p4d;
+ pud_t *pud;
+ pmd_t *pmd;
+
+ pgd = pgd_offset_k(addr);
+ p4d = p4d_offset(pgd, addr);
+
+ if (pgd_none(*pgd)) {
+ pud_t *new __maybe_unused;
+
+ new = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
+ pgd_populate(&init_mm, pgd, new);
+#ifndef __PAGETABLE_PUD_FOLDED
+ pud_init((unsigned long)new, (unsigned long)invalid_pmd_table);
+#endif
+ }
+
+ pud = pud_offset(p4d, addr);
+ if (pud_none(*pud)) {
+ pmd_t *new __maybe_unused;
+
+ new = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
+ pud_populate(&init_mm, pud, new);
+#ifndef __PAGETABLE_PMD_FOLDED
+ pmd_init((unsigned long)new, (unsigned long)invalid_pte_table);
+#endif
+ }
+
+ pmd = pmd_offset(pud, addr);
+ if (pmd_none(*pmd)) {
+ pte_t *new __maybe_unused;
+
+ new = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
+ pmd_populate_kernel(&init_mm, pmd, new);
+ }
+
+ return pte_offset_kernel(pmd, addr);
+}
+
+void __init __set_fixmap(enum fixed_addresses idx,
+ phys_addr_t phys, pgprot_t flags)
+{
+ unsigned long addr = __fix_to_virt(idx);
+ pte_t *ptep;
+
+ BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
+
+ ptep = fixmap_pte(addr);
+ if (!pte_none(*ptep)) {
+ pte_ERROR(*ptep);
+ return;
+ }
+
+ if (pgprot_val(flags))
+ set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
+ else {
+ pte_clear(&init_mm, addr, ptep);
+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+ }
+}
+
/*
* Align swapper_pg_dir in to 64K, allows its address to be loaded
* with a single LUI instruction in the TLB handlers. If we used
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
*/
#include <linux/export.h>
+#include <linux/io.h>
+#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/mman.h>
return pfn_valid(PFN_DOWN(PHYSADDR(kaddr)));
}
EXPORT_SYMBOL_GPL(__virt_addr_valid);
+
+/*
+ * You really shouldn't be using read() or write() on /dev/mem. This might go
+ * away in the future.
+ */
+int valid_phys_addr_range(phys_addr_t addr, size_t size)
+{
+ /*
+ * Check whether addr is covered by a memory region without the
+ * MEMBLOCK_NOMAP attribute, and whether that region covers the
+ * entire range. In theory, this could lead to false negatives
+ * if the range is covered by distinct but adjacent memory regions
+ * that only differ in other attributes. However, few of such
+ * attributes have been defined, and it is debatable whether it
+ * follows that /dev/mem read() calls should be able traverse
+ * such boundaries.
+ */
+ return memblock_is_region_memory(addr, size) && memblock_is_map_memory(addr);
+}
+
+/*
+ * Do not allow /dev/mem mappings beyond the supported physical range.
+ */
+int valid_mmap_phys_addr_range(unsigned long pfn, size_t size)
+{
+ return !(((pfn << PAGE_SHIFT) + size) & ~(GENMASK_ULL(cpu_pabits, 0)));
+}
void setup_tlb_handler(int cpu)
{
setup_ptwalker();
- output_pgtable_bits_defines();
+ local_flush_tlb_all();
/* The tlb handlers are generated only once */
if (cpu == 0) {
write_csr_pagesize(PS_DEFAULT_SIZE);
write_csr_stlbpgsize(PS_DEFAULT_SIZE);
write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE);
+
setup_tlb_handler(cpu);
- local_flush_tlb_all();
+ output_pgtable_bits_defines();
}
#include <asm/regdef.h>
#include <asm/stackframe.h>
+#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3)
+#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3)
+#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3)
+#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3)
+
.macro tlb_do_page_fault, write
SYM_FUNC_START(tlb_do_page_fault_\write)
SAVE_ALL
- csrrd a2, LOONGARCH_CSR_BADV
- move a0, sp
- REG_S a2, sp, PT_BVADDR
- li.w a1, \write
- la.abs t0, do_page_fault
- jirl ra, t0, 0
+ csrrd a2, LOONGARCH_CSR_BADV
+ move a0, sp
+ REG_S a2, sp, PT_BVADDR
+ li.w a1, \write
+ la.abs t0, do_page_fault
+ jirl ra, t0, 0
RESTORE_ALL_AND_RET
SYM_FUNC_END(tlb_do_page_fault_\write)
.endm
SYM_FUNC_START(handle_tlb_protect)
BACKUP_T0T1
SAVE_ALL
- move a0, sp
- move a1, zero
- csrrd a2, LOONGARCH_CSR_BADV
- REG_S a2, sp, PT_BVADDR
- la.abs t0, do_page_fault
- jirl ra, t0, 0
+ move a0, sp
+ move a1, zero
+ csrrd a2, LOONGARCH_CSR_BADV
+ REG_S a2, sp, PT_BVADDR
+ la.abs t0, do_page_fault
+ jirl ra, t0, 0
RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_tlb_protect)
SYM_FUNC_START(handle_tlb_load)
- csrwr t0, EXCEPTION_KS0
- csrwr t1, EXCEPTION_KS1
- csrwr ra, EXCEPTION_KS2
+ csrwr t0, EXCEPTION_KS0
+ csrwr t1, EXCEPTION_KS1
+ csrwr ra, EXCEPTION_KS2
/*
* The vmalloc handling is not in the hotpath.
*/
- csrrd t0, LOONGARCH_CSR_BADV
- bltz t0, vmalloc_load
- csrrd t1, LOONGARCH_CSR_PGDL
+ csrrd t0, LOONGARCH_CSR_BADV
+ bltz t0, vmalloc_load
+ csrrd t1, LOONGARCH_CSR_PGDL
vmalloc_done_load:
/* Get PGD offset in bytes */
- srli.d t0, t0, PGDIR_SHIFT
- andi t0, t0, (PTRS_PER_PGD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
+ alsl.d t1, ra, t1, 3
#if CONFIG_PGTABLE_LEVELS > 3
- csrrd t0, LOONGARCH_CSR_BADV
- ld.d t1, t1, 0
- srli.d t0, t0, PUD_SHIFT
- andi t0, t0, (PTRS_PER_PUD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ ld.d t1, t1, 0
+ bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
+ alsl.d t1, ra, t1, 3
#endif
#if CONFIG_PGTABLE_LEVELS > 2
- csrrd t0, LOONGARCH_CSR_BADV
- ld.d t1, t1, 0
- srli.d t0, t0, PMD_SHIFT
- andi t0, t0, (PTRS_PER_PMD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ ld.d t1, t1, 0
+ bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
+ alsl.d t1, ra, t1, 3
#endif
- ld.d ra, t1, 0
+ ld.d ra, t1, 0
/*
* For huge tlb entries, pmde doesn't contain an address but
* instead contains the tlb pte. Check the PAGE_HUGE bit and
* see if we need to jump to huge tlb processing.
*/
- andi t0, ra, _PAGE_HUGE
- bnez t0, tlb_huge_update_load
+ rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1
+ bltz ra, tlb_huge_update_load
- csrrd t0, LOONGARCH_CSR_BADV
- srli.d t0, t0, PAGE_SHIFT
- andi t0, t0, (PTRS_PER_PTE - 1)
- slli.d t0, t0, _PTE_T_LOG2
- add.d t1, ra, t0
+ rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
+ bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
+ alsl.d t1, t0, ra, _PTE_T_LOG2
#ifdef CONFIG_SMP
smp_pgtable_change_load:
-#endif
-#ifdef CONFIG_SMP
- ll.d t0, t1, 0
+ ll.d t0, t1, 0
#else
- ld.d t0, t1, 0
+ ld.d t0, t1, 0
#endif
- tlbsrch
-
- srli.d ra, t0, _PAGE_PRESENT_SHIFT
- andi ra, ra, 1
- beqz ra, nopage_tlb_load
+ andi ra, t0, _PAGE_PRESENT
+ beqz ra, nopage_tlb_load
- ori t0, t0, _PAGE_VALID
+ ori t0, t0, _PAGE_VALID
#ifdef CONFIG_SMP
- sc.d t0, t1, 0
- beqz t0, smp_pgtable_change_load
+ sc.d t0, t1, 0
+ beqz t0, smp_pgtable_change_load
#else
- st.d t0, t1, 0
+ st.d t0, t1, 0
#endif
- ori t1, t1, 8
- xori t1, t1, 8
- ld.d t0, t1, 0
- ld.d t1, t1, 8
- csrwr t0, LOONGARCH_CSR_TLBELO0
- csrwr t1, LOONGARCH_CSR_TLBELO1
+ tlbsrch
+ bstrins.d t1, zero, 3, 3
+ ld.d t0, t1, 0
+ ld.d t1, t1, 8
+ csrwr t0, LOONGARCH_CSR_TLBELO0
+ csrwr t1, LOONGARCH_CSR_TLBELO1
tlbwr
-leave_load:
- csrrd t0, EXCEPTION_KS0
- csrrd t1, EXCEPTION_KS1
- csrrd ra, EXCEPTION_KS2
+
+ csrrd t0, EXCEPTION_KS0
+ csrrd t1, EXCEPTION_KS1
+ csrrd ra, EXCEPTION_KS2
ertn
+
#ifdef CONFIG_64BIT
vmalloc_load:
- la.abs t1, swapper_pg_dir
- b vmalloc_done_load
+ la.abs t1, swapper_pg_dir
+ b vmalloc_done_load
#endif
- /*
- * This is the entry point when build_tlbchange_handler_head
- * spots a huge page.
- */
+ /* This is the entry point of a huge page. */
tlb_huge_update_load:
#ifdef CONFIG_SMP
- ll.d t0, t1, 0
-#else
- ld.d t0, t1, 0
+ ll.d ra, t1, 0
#endif
- srli.d ra, t0, _PAGE_PRESENT_SHIFT
- andi ra, ra, 1
- beqz ra, nopage_tlb_load
- tlbsrch
+ andi t0, ra, _PAGE_PRESENT
+ beqz t0, nopage_tlb_load
- ori t0, t0, _PAGE_VALID
#ifdef CONFIG_SMP
- sc.d t0, t1, 0
- beqz t0, tlb_huge_update_load
- ld.d t0, t1, 0
+ ori t0, ra, _PAGE_VALID
+ sc.d t0, t1, 0
+ beqz t0, tlb_huge_update_load
+ ori t0, ra, _PAGE_VALID
#else
- st.d t0, t1, 0
+ rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
+ ori t0, ra, _PAGE_VALID
+ st.d t0, t1, 0
#endif
+ tlbsrch
addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16)
addi.d ra, t1, 0
csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
tlbwr
- csrxchg zero, t1, LOONGARCH_CSR_TLBIDX
+ csrxchg zero, t1, LOONGARCH_CSR_TLBIDX
/*
* A huge PTE describes an area the size of the
* address space.
*/
/* Huge page: Move Global bit */
- xori t0, t0, _PAGE_HUGE
- lu12i.w t1, _PAGE_HGLOBAL >> 12
- and t1, t0, t1
- srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
- or t0, t0, t1
+ xori t0, t0, _PAGE_HUGE
+ lu12i.w t1, _PAGE_HGLOBAL >> 12
+ and t1, t0, t1
+ srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
+ or t0, t0, t1
- addi.d ra, t0, 0
- csrwr t0, LOONGARCH_CSR_TLBELO0
- addi.d t0, ra, 0
+ move ra, t0
+ csrwr ra, LOONGARCH_CSR_TLBELO0
/* Convert to entrylo1 */
- addi.d t1, zero, 1
- slli.d t1, t1, (HPAGE_SHIFT - 1)
- add.d t0, t0, t1
- csrwr t0, LOONGARCH_CSR_TLBELO1
+ addi.d t1, zero, 1
+ slli.d t1, t1, (HPAGE_SHIFT - 1)
+ add.d t0, t0, t1
+ csrwr t0, LOONGARCH_CSR_TLBELO1
/* Set huge page tlb entry size */
addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
+ csrrd t0, EXCEPTION_KS0
+ csrrd t1, EXCEPTION_KS1
+ csrrd ra, EXCEPTION_KS2
+ ertn
+
nopage_tlb_load:
- dbar 0
- csrrd ra, EXCEPTION_KS2
- la.abs t0, tlb_do_page_fault_0
- jr t0
+ dbar 0
+ csrrd ra, EXCEPTION_KS2
+ la.abs t0, tlb_do_page_fault_0
+ jr t0
SYM_FUNC_END(handle_tlb_load)
SYM_FUNC_START(handle_tlb_store)
- csrwr t0, EXCEPTION_KS0
- csrwr t1, EXCEPTION_KS1
- csrwr ra, EXCEPTION_KS2
+ csrwr t0, EXCEPTION_KS0
+ csrwr t1, EXCEPTION_KS1
+ csrwr ra, EXCEPTION_KS2
/*
* The vmalloc handling is not in the hotpath.
*/
- csrrd t0, LOONGARCH_CSR_BADV
- bltz t0, vmalloc_store
- csrrd t1, LOONGARCH_CSR_PGDL
+ csrrd t0, LOONGARCH_CSR_BADV
+ bltz t0, vmalloc_store
+ csrrd t1, LOONGARCH_CSR_PGDL
vmalloc_done_store:
/* Get PGD offset in bytes */
- srli.d t0, t0, PGDIR_SHIFT
- andi t0, t0, (PTRS_PER_PGD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
-
+ bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
+ alsl.d t1, ra, t1, 3
#if CONFIG_PGTABLE_LEVELS > 3
- csrrd t0, LOONGARCH_CSR_BADV
- ld.d t1, t1, 0
- srli.d t0, t0, PUD_SHIFT
- andi t0, t0, (PTRS_PER_PUD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ ld.d t1, t1, 0
+ bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
+ alsl.d t1, ra, t1, 3
#endif
#if CONFIG_PGTABLE_LEVELS > 2
- csrrd t0, LOONGARCH_CSR_BADV
- ld.d t1, t1, 0
- srli.d t0, t0, PMD_SHIFT
- andi t0, t0, (PTRS_PER_PMD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ ld.d t1, t1, 0
+ bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
+ alsl.d t1, ra, t1, 3
#endif
- ld.d ra, t1, 0
+ ld.d ra, t1, 0
/*
* For huge tlb entries, pmde doesn't contain an address but
* instead contains the tlb pte. Check the PAGE_HUGE bit and
* see if we need to jump to huge tlb processing.
*/
- andi t0, ra, _PAGE_HUGE
- bnez t0, tlb_huge_update_store
+ rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1
+ bltz ra, tlb_huge_update_store
- csrrd t0, LOONGARCH_CSR_BADV
- srli.d t0, t0, PAGE_SHIFT
- andi t0, t0, (PTRS_PER_PTE - 1)
- slli.d t0, t0, _PTE_T_LOG2
- add.d t1, ra, t0
+ rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
+ bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
+ alsl.d t1, t0, ra, _PTE_T_LOG2
#ifdef CONFIG_SMP
smp_pgtable_change_store:
-#endif
-#ifdef CONFIG_SMP
- ll.d t0, t1, 0
+ ll.d t0, t1, 0
#else
- ld.d t0, t1, 0
+ ld.d t0, t1, 0
#endif
- tlbsrch
-
- srli.d ra, t0, _PAGE_PRESENT_SHIFT
- andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
- xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
- bnez ra, nopage_tlb_store
+ andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE
+ xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE
+ bnez ra, nopage_tlb_store
- ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#ifdef CONFIG_SMP
- sc.d t0, t1, 0
- beqz t0, smp_pgtable_change_store
+ sc.d t0, t1, 0
+ beqz t0, smp_pgtable_change_store
#else
- st.d t0, t1, 0
+ st.d t0, t1, 0
#endif
-
- ori t1, t1, 8
- xori t1, t1, 8
- ld.d t0, t1, 0
- ld.d t1, t1, 8
- csrwr t0, LOONGARCH_CSR_TLBELO0
- csrwr t1, LOONGARCH_CSR_TLBELO1
+ tlbsrch
+ bstrins.d t1, zero, 3, 3
+ ld.d t0, t1, 0
+ ld.d t1, t1, 8
+ csrwr t0, LOONGARCH_CSR_TLBELO0
+ csrwr t1, LOONGARCH_CSR_TLBELO1
tlbwr
-leave_store:
- csrrd t0, EXCEPTION_KS0
- csrrd t1, EXCEPTION_KS1
- csrrd ra, EXCEPTION_KS2
+
+ csrrd t0, EXCEPTION_KS0
+ csrrd t1, EXCEPTION_KS1
+ csrrd ra, EXCEPTION_KS2
ertn
+
#ifdef CONFIG_64BIT
vmalloc_store:
- la.abs t1, swapper_pg_dir
- b vmalloc_done_store
+ la.abs t1, swapper_pg_dir
+ b vmalloc_done_store
#endif
- /*
- * This is the entry point when build_tlbchange_handler_head
- * spots a huge page.
- */
+ /* This is the entry point of a huge page. */
tlb_huge_update_store:
#ifdef CONFIG_SMP
- ll.d t0, t1, 0
-#else
- ld.d t0, t1, 0
+ ll.d ra, t1, 0
#endif
- srli.d ra, t0, _PAGE_PRESENT_SHIFT
- andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
- xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
- bnez ra, nopage_tlb_store
-
- tlbsrch
- ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE
+ xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE
+ bnez t0, nopage_tlb_store
#ifdef CONFIG_SMP
- sc.d t0, t1, 0
- beqz t0, tlb_huge_update_store
- ld.d t0, t1, 0
+ ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ sc.d t0, t1, 0
+ beqz t0, tlb_huge_update_store
+ ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
- st.d t0, t1, 0
+ rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
+ ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ st.d t0, t1, 0
#endif
+ tlbsrch
addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16)
addi.d ra, t1, 0
csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
tlbwr
- csrxchg zero, t1, LOONGARCH_CSR_TLBIDX
+ csrxchg zero, t1, LOONGARCH_CSR_TLBIDX
/*
* A huge PTE describes an area the size of the
* configured huge page size. This is twice the
* address space.
*/
/* Huge page: Move Global bit */
- xori t0, t0, _PAGE_HUGE
- lu12i.w t1, _PAGE_HGLOBAL >> 12
- and t1, t0, t1
- srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
- or t0, t0, t1
+ xori t0, t0, _PAGE_HUGE
+ lu12i.w t1, _PAGE_HGLOBAL >> 12
+ and t1, t0, t1
+ srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
+ or t0, t0, t1
- addi.d ra, t0, 0
- csrwr t0, LOONGARCH_CSR_TLBELO0
- addi.d t0, ra, 0
+ move ra, t0
+ csrwr ra, LOONGARCH_CSR_TLBELO0
/* Convert to entrylo1 */
- addi.d t1, zero, 1
- slli.d t1, t1, (HPAGE_SHIFT - 1)
- add.d t0, t0, t1
- csrwr t0, LOONGARCH_CSR_TLBELO1
+ addi.d t1, zero, 1
+ slli.d t1, t1, (HPAGE_SHIFT - 1)
+ add.d t0, t0, t1
+ csrwr t0, LOONGARCH_CSR_TLBELO1
/* Set huge page tlb entry size */
addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
+ csrrd t0, EXCEPTION_KS0
+ csrrd t1, EXCEPTION_KS1
+ csrrd ra, EXCEPTION_KS2
+ ertn
+
nopage_tlb_store:
- dbar 0
- csrrd ra, EXCEPTION_KS2
- la.abs t0, tlb_do_page_fault_1
- jr t0
+ dbar 0
+ csrrd ra, EXCEPTION_KS2
+ la.abs t0, tlb_do_page_fault_1
+ jr t0
SYM_FUNC_END(handle_tlb_store)
SYM_FUNC_START(handle_tlb_modify)
- csrwr t0, EXCEPTION_KS0
- csrwr t1, EXCEPTION_KS1
- csrwr ra, EXCEPTION_KS2
+ csrwr t0, EXCEPTION_KS0
+ csrwr t1, EXCEPTION_KS1
+ csrwr ra, EXCEPTION_KS2
/*
* The vmalloc handling is not in the hotpath.
*/
- csrrd t0, LOONGARCH_CSR_BADV
- bltz t0, vmalloc_modify
- csrrd t1, LOONGARCH_CSR_PGDL
+ csrrd t0, LOONGARCH_CSR_BADV
+ bltz t0, vmalloc_modify
+ csrrd t1, LOONGARCH_CSR_PGDL
vmalloc_done_modify:
/* Get PGD offset in bytes */
- srli.d t0, t0, PGDIR_SHIFT
- andi t0, t0, (PTRS_PER_PGD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
+ alsl.d t1, ra, t1, 3
#if CONFIG_PGTABLE_LEVELS > 3
- csrrd t0, LOONGARCH_CSR_BADV
- ld.d t1, t1, 0
- srli.d t0, t0, PUD_SHIFT
- andi t0, t0, (PTRS_PER_PUD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ ld.d t1, t1, 0
+ bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
+ alsl.d t1, ra, t1, 3
#endif
#if CONFIG_PGTABLE_LEVELS > 2
- csrrd t0, LOONGARCH_CSR_BADV
- ld.d t1, t1, 0
- srli.d t0, t0, PMD_SHIFT
- andi t0, t0, (PTRS_PER_PMD - 1)
- slli.d t0, t0, 3
- add.d t1, t1, t0
+ ld.d t1, t1, 0
+ bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
+ alsl.d t1, ra, t1, 3
#endif
- ld.d ra, t1, 0
+ ld.d ra, t1, 0
/*
* For huge tlb entries, pmde doesn't contain an address but
* instead contains the tlb pte. Check the PAGE_HUGE bit and
* see if we need to jump to huge tlb processing.
*/
- andi t0, ra, _PAGE_HUGE
- bnez t0, tlb_huge_update_modify
+ rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1
+ bltz ra, tlb_huge_update_modify
- csrrd t0, LOONGARCH_CSR_BADV
- srli.d t0, t0, PAGE_SHIFT
- andi t0, t0, (PTRS_PER_PTE - 1)
- slli.d t0, t0, _PTE_T_LOG2
- add.d t1, ra, t0
+ rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
+ bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
+ alsl.d t1, t0, ra, _PTE_T_LOG2
#ifdef CONFIG_SMP
smp_pgtable_change_modify:
-#endif
-#ifdef CONFIG_SMP
- ll.d t0, t1, 0
+ ll.d t0, t1, 0
#else
- ld.d t0, t1, 0
+ ld.d t0, t1, 0
#endif
- tlbsrch
-
- srli.d ra, t0, _PAGE_WRITE_SHIFT
- andi ra, ra, 1
- beqz ra, nopage_tlb_modify
+ andi ra, t0, _PAGE_WRITE
+ beqz ra, nopage_tlb_modify
- ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#ifdef CONFIG_SMP
- sc.d t0, t1, 0
- beqz t0, smp_pgtable_change_modify
+ sc.d t0, t1, 0
+ beqz t0, smp_pgtable_change_modify
#else
- st.d t0, t1, 0
+ st.d t0, t1, 0
#endif
- ori t1, t1, 8
- xori t1, t1, 8
- ld.d t0, t1, 0
- ld.d t1, t1, 8
- csrwr t0, LOONGARCH_CSR_TLBELO0
- csrwr t1, LOONGARCH_CSR_TLBELO1
+ tlbsrch
+ bstrins.d t1, zero, 3, 3
+ ld.d t0, t1, 0
+ ld.d t1, t1, 8
+ csrwr t0, LOONGARCH_CSR_TLBELO0
+ csrwr t1, LOONGARCH_CSR_TLBELO1
tlbwr
-leave_modify:
- csrrd t0, EXCEPTION_KS0
- csrrd t1, EXCEPTION_KS1
- csrrd ra, EXCEPTION_KS2
+
+ csrrd t0, EXCEPTION_KS0
+ csrrd t1, EXCEPTION_KS1
+ csrrd ra, EXCEPTION_KS2
ertn
+
#ifdef CONFIG_64BIT
vmalloc_modify:
- la.abs t1, swapper_pg_dir
- b vmalloc_done_modify
+ la.abs t1, swapper_pg_dir
+ b vmalloc_done_modify
#endif
- /*
- * This is the entry point when
- * build_tlbchange_handler_head spots a huge page.
- */
+ /* This is the entry point of a huge page. */
tlb_huge_update_modify:
#ifdef CONFIG_SMP
- ll.d t0, t1, 0
-#else
- ld.d t0, t1, 0
+ ll.d ra, t1, 0
#endif
-
- srli.d ra, t0, _PAGE_WRITE_SHIFT
- andi ra, ra, 1
- beqz ra, nopage_tlb_modify
-
- tlbsrch
- ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ andi t0, ra, _PAGE_WRITE
+ beqz t0, nopage_tlb_modify
#ifdef CONFIG_SMP
- sc.d t0, t1, 0
- beqz t0, tlb_huge_update_modify
- ld.d t0, t1, 0
+ ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ sc.d t0, t1, 0
+ beqz t0, tlb_huge_update_modify
+ ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
- st.d t0, t1, 0
+ rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
+ ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
+ st.d t0, t1, 0
#endif
/*
* A huge PTE describes an area the size of the
* address space.
*/
/* Huge page: Move Global bit */
- xori t0, t0, _PAGE_HUGE
- lu12i.w t1, _PAGE_HGLOBAL >> 12
- and t1, t0, t1
- srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
- or t0, t0, t1
+ xori t0, t0, _PAGE_HUGE
+ lu12i.w t1, _PAGE_HGLOBAL >> 12
+ and t1, t0, t1
+ srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
+ or t0, t0, t1
- addi.d ra, t0, 0
- csrwr t0, LOONGARCH_CSR_TLBELO0
- addi.d t0, ra, 0
+ move ra, t0
+ csrwr ra, LOONGARCH_CSR_TLBELO0
/* Convert to entrylo1 */
- addi.d t1, zero, 1
- slli.d t1, t1, (HPAGE_SHIFT - 1)
- add.d t0, t0, t1
- csrwr t0, LOONGARCH_CSR_TLBELO1
+ addi.d t1, zero, 1
+ slli.d t1, t1, (HPAGE_SHIFT - 1)
+ add.d t0, t0, t1
+ csrwr t0, LOONGARCH_CSR_TLBELO1
/* Set huge page tlb entry size */
addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
+ csrrd t0, EXCEPTION_KS0
+ csrrd t1, EXCEPTION_KS1
+ csrrd ra, EXCEPTION_KS2
+ ertn
+
nopage_tlb_modify:
- dbar 0
- csrrd ra, EXCEPTION_KS2
- la.abs t0, tlb_do_page_fault_1
- jr t0
+ dbar 0
+ csrrd ra, EXCEPTION_KS2
+ la.abs t0, tlb_do_page_fault_1
+ jr t0
SYM_FUNC_END(handle_tlb_modify)
SYM_FUNC_START(handle_tlb_refill)
- csrwr t0, LOONGARCH_CSR_TLBRSAVE
- csrrd t0, LOONGARCH_CSR_PGD
- lddir t0, t0, 3
+ csrwr t0, LOONGARCH_CSR_TLBRSAVE
+ csrrd t0, LOONGARCH_CSR_PGD
+ lddir t0, t0, 3
#if CONFIG_PGTABLE_LEVELS > 3
- lddir t0, t0, 2
+ lddir t0, t0, 2
#endif
#if CONFIG_PGTABLE_LEVELS > 2
- lddir t0, t0, 1
+ lddir t0, t0, 1
#endif
- ldpte t0, 0
- ldpte t0, 1
+ ldpte t0, 0
+ ldpte t0, 1
tlbfill
- csrrd t0, LOONGARCH_CSR_TLBRSAVE
+ csrrd t0, LOONGARCH_CSR_TLBRSAVE
ertn
SYM_FUNC_END(handle_tlb_refill)
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for arch/loongarch/net
+#
+# Copyright (C) 2022 Loongson Technology Corporation Limited
+#
+obj-$(CONFIG_BPF_JIT) += bpf_jit.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * BPF JIT compiler for LoongArch
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+#include "bpf_jit.h"
+
+#define REG_TCC LOONGARCH_GPR_A6
+#define TCC_SAVED LOONGARCH_GPR_S5
+
+#define SAVE_RA BIT(0)
+#define SAVE_TCC BIT(1)
+
+static const int regmap[] = {
+ /* return value from in-kernel function, and exit value for eBPF program */
+ [BPF_REG_0] = LOONGARCH_GPR_A5,
+ /* arguments from eBPF program to in-kernel function */
+ [BPF_REG_1] = LOONGARCH_GPR_A0,
+ [BPF_REG_2] = LOONGARCH_GPR_A1,
+ [BPF_REG_3] = LOONGARCH_GPR_A2,
+ [BPF_REG_4] = LOONGARCH_GPR_A3,
+ [BPF_REG_5] = LOONGARCH_GPR_A4,
+ /* callee saved registers that in-kernel function will preserve */
+ [BPF_REG_6] = LOONGARCH_GPR_S0,
+ [BPF_REG_7] = LOONGARCH_GPR_S1,
+ [BPF_REG_8] = LOONGARCH_GPR_S2,
+ [BPF_REG_9] = LOONGARCH_GPR_S3,
+ /* read-only frame pointer to access stack */
+ [BPF_REG_FP] = LOONGARCH_GPR_S4,
+ /* temporary register for blinding constants */
+ [BPF_REG_AX] = LOONGARCH_GPR_T0,
+};
+
+static void mark_call(struct jit_ctx *ctx)
+{
+ ctx->flags |= SAVE_RA;
+}
+
+static void mark_tail_call(struct jit_ctx *ctx)
+{
+ ctx->flags |= SAVE_TCC;
+}
+
+static bool seen_call(struct jit_ctx *ctx)
+{
+ return (ctx->flags & SAVE_RA);
+}
+
+static bool seen_tail_call(struct jit_ctx *ctx)
+{
+ return (ctx->flags & SAVE_TCC);
+}
+
+static u8 tail_call_reg(struct jit_ctx *ctx)
+{
+ if (seen_call(ctx))
+ return TCC_SAVED;
+
+ return REG_TCC;
+}
+
+/*
+ * eBPF prog stack layout:
+ *
+ * high
+ * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
+ * | $ra |
+ * +-------------------------+
+ * | $fp |
+ * +-------------------------+
+ * | $s0 |
+ * +-------------------------+
+ * | $s1 |
+ * +-------------------------+
+ * | $s2 |
+ * +-------------------------+
+ * | $s3 |
+ * +-------------------------+
+ * | $s4 |
+ * +-------------------------+
+ * | $s5 |
+ * +-------------------------+ <--BPF_REG_FP
+ * | prog->aux->stack_depth |
+ * | (optional) |
+ * current $sp -------------> +-------------------------+
+ * low
+ */
+static void build_prologue(struct jit_ctx *ctx)
+{
+ int stack_adjust = 0, store_offset, bpf_stack_adjust;
+
+ bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
+
+ /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */
+ stack_adjust += sizeof(long) * 8;
+
+ stack_adjust = round_up(stack_adjust, 16);
+ stack_adjust += bpf_stack_adjust;
+
+ /*
+ * First instruction initializes the tail call count (TCC).
+ * On tail call we skip this instruction, and the TCC is
+ * passed in REG_TCC from the caller.
+ */
+ emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
+
+ emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
+
+ store_offset = stack_adjust - sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
+
+ store_offset -= sizeof(long);
+ emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
+
+ emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
+
+ if (bpf_stack_adjust)
+ emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
+
+ /*
+ * Program contains calls and tail calls, so REG_TCC need
+ * to be saved across calls.
+ */
+ if (seen_tail_call(ctx) && seen_call(ctx))
+ move_reg(ctx, TCC_SAVED, REG_TCC);
+
+ ctx->stack_size = stack_adjust;
+}
+
+static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
+{
+ int stack_adjust = ctx->stack_size;
+ int load_offset;
+
+ load_offset = stack_adjust - sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
+
+ load_offset -= sizeof(long);
+ emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
+
+ emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
+
+ if (!is_tail_call) {
+ /* Set return value */
+ move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]);
+ /* Return to the caller */
+ emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0);
+ } else {
+ /*
+ * Call the next bpf prog and skip the first instruction
+ * of TCC initialization.
+ */
+ emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1);
+ }
+}
+
+static void build_epilogue(struct jit_ctx *ctx)
+{
+ __build_epilogue(ctx, false);
+}
+
+bool bpf_jit_supports_kfunc_call(void)
+{
+ return true;
+}
+
+/* initialized on the first pass of build_body() */
+static int out_offset = -1;
+static int emit_bpf_tail_call(struct jit_ctx *ctx)
+{
+ int off;
+ u8 tcc = tail_call_reg(ctx);
+ u8 a1 = LOONGARCH_GPR_A1;
+ u8 a2 = LOONGARCH_GPR_A2;
+ u8 t1 = LOONGARCH_GPR_T1;
+ u8 t2 = LOONGARCH_GPR_T2;
+ u8 t3 = LOONGARCH_GPR_T3;
+ const int idx0 = ctx->idx;
+
+#define cur_offset (ctx->idx - idx0)
+#define jmp_offset (out_offset - (cur_offset))
+
+ /*
+ * a0: &ctx
+ * a1: &array
+ * a2: index
+ *
+ * if (index >= array->map.max_entries)
+ * goto out;
+ */
+ off = offsetof(struct bpf_array, map.max_entries);
+ emit_insn(ctx, ldwu, t1, a1, off);
+ /* bgeu $a2, $t1, jmp_offset */
+ if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
+ goto toofar;
+
+ /*
+ * if (--TCC < 0)
+ * goto out;
+ */
+ emit_insn(ctx, addid, REG_TCC, tcc, -1);
+ if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
+ goto toofar;
+
+ /*
+ * prog = array->ptrs[index];
+ * if (!prog)
+ * goto out;
+ */
+ emit_insn(ctx, alsld, t2, a2, a1, 2);
+ off = offsetof(struct bpf_array, ptrs);
+ emit_insn(ctx, ldd, t2, t2, off);
+ /* beq $t2, $zero, jmp_offset */
+ if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
+ goto toofar;
+
+ /* goto *(prog->bpf_func + 4); */
+ off = offsetof(struct bpf_prog, bpf_func);
+ emit_insn(ctx, ldd, t3, t2, off);
+ __build_epilogue(ctx, true);
+
+ /* out: */
+ if (out_offset == -1)
+ out_offset = cur_offset;
+ if (cur_offset != out_offset) {
+ pr_err_once("tail_call out_offset = %d, expected %d!\n",
+ cur_offset, out_offset);
+ return -1;
+ }
+
+ return 0;
+
+toofar:
+ pr_info_once("tail_call: jump too far\n");
+ return -1;
+#undef cur_offset
+#undef jmp_offset
+}
+
+static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
+{
+ const u8 t1 = LOONGARCH_GPR_T1;
+ const u8 t2 = LOONGARCH_GPR_T2;
+ const u8 t3 = LOONGARCH_GPR_T3;
+ const u8 src = regmap[insn->src_reg];
+ const u8 dst = regmap[insn->dst_reg];
+ const s16 off = insn->off;
+ const s32 imm = insn->imm;
+ const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
+
+ move_imm(ctx, t1, off, false);
+ emit_insn(ctx, addd, t1, dst, t1);
+ move_reg(ctx, t3, src);
+
+ switch (imm) {
+ /* lock *(size *)(dst + off) <op>= src */
+ case BPF_ADD:
+ if (isdw)
+ emit_insn(ctx, amaddd, t2, t1, src);
+ else
+ emit_insn(ctx, amaddw, t2, t1, src);
+ break;
+ case BPF_AND:
+ if (isdw)
+ emit_insn(ctx, amandd, t2, t1, src);
+ else
+ emit_insn(ctx, amandw, t2, t1, src);
+ break;
+ case BPF_OR:
+ if (isdw)
+ emit_insn(ctx, amord, t2, t1, src);
+ else
+ emit_insn(ctx, amorw, t2, t1, src);
+ break;
+ case BPF_XOR:
+ if (isdw)
+ emit_insn(ctx, amxord, t2, t1, src);
+ else
+ emit_insn(ctx, amxorw, t2, t1, src);
+ break;
+ /* src = atomic_fetch_<op>(dst + off, src) */
+ case BPF_ADD | BPF_FETCH:
+ if (isdw) {
+ emit_insn(ctx, amaddd, src, t1, t3);
+ } else {
+ emit_insn(ctx, amaddw, src, t1, t3);
+ emit_zext_32(ctx, src, true);
+ }
+ break;
+ case BPF_AND | BPF_FETCH:
+ if (isdw) {
+ emit_insn(ctx, amandd, src, t1, t3);
+ } else {
+ emit_insn(ctx, amandw, src, t1, t3);
+ emit_zext_32(ctx, src, true);
+ }
+ break;
+ case BPF_OR | BPF_FETCH:
+ if (isdw) {
+ emit_insn(ctx, amord, src, t1, t3);
+ } else {
+ emit_insn(ctx, amorw, src, t1, t3);
+ emit_zext_32(ctx, src, true);
+ }
+ break;
+ case BPF_XOR | BPF_FETCH:
+ if (isdw) {
+ emit_insn(ctx, amxord, src, t1, t3);
+ } else {
+ emit_insn(ctx, amxorw, src, t1, t3);
+ emit_zext_32(ctx, src, true);
+ }
+ break;
+ /* src = atomic_xchg(dst + off, src); */
+ case BPF_XCHG:
+ if (isdw) {
+ emit_insn(ctx, amswapd, src, t1, t3);
+ } else {
+ emit_insn(ctx, amswapw, src, t1, t3);
+ emit_zext_32(ctx, src, true);
+ }
+ break;
+ /* r0 = atomic_cmpxchg(dst + off, r0, src); */
+ case BPF_CMPXCHG:
+ u8 r0 = regmap[BPF_REG_0];
+
+ move_reg(ctx, t2, r0);
+ if (isdw) {
+ emit_insn(ctx, lld, r0, t1, 0);
+ emit_insn(ctx, bne, t2, r0, 4);
+ move_reg(ctx, t3, src);
+ emit_insn(ctx, scd, t3, t1, 0);
+ emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
+ } else {
+ emit_insn(ctx, llw, r0, t1, 0);
+ emit_zext_32(ctx, t2, true);
+ emit_zext_32(ctx, r0, true);
+ emit_insn(ctx, bne, t2, r0, 4);
+ move_reg(ctx, t3, src);
+ emit_insn(ctx, scw, t3, t1, 0);
+ emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
+ emit_zext_32(ctx, r0, true);
+ }
+ break;
+ }
+}
+
+static bool is_signed_bpf_cond(u8 cond)
+{
+ return cond == BPF_JSGT || cond == BPF_JSLT ||
+ cond == BPF_JSGE || cond == BPF_JSLE;
+}
+
+static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
+{
+ const bool is32 = BPF_CLASS(insn->code) == BPF_ALU ||
+ BPF_CLASS(insn->code) == BPF_JMP32;
+ const u8 code = insn->code;
+ const u8 cond = BPF_OP(code);
+ const u8 t1 = LOONGARCH_GPR_T1;
+ const u8 t2 = LOONGARCH_GPR_T2;
+ const u8 src = regmap[insn->src_reg];
+ const u8 dst = regmap[insn->dst_reg];
+ const s16 off = insn->off;
+ const s32 imm = insn->imm;
+ int jmp_offset;
+ int i = insn - ctx->prog->insnsi;
+
+ switch (code) {
+ /* dst = src */
+ case BPF_ALU | BPF_MOV | BPF_X:
+ case BPF_ALU64 | BPF_MOV | BPF_X:
+ move_reg(ctx, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = imm */
+ case BPF_ALU | BPF_MOV | BPF_K:
+ case BPF_ALU64 | BPF_MOV | BPF_K:
+ move_imm(ctx, dst, imm, is32);
+ break;
+
+ /* dst = dst + src */
+ case BPF_ALU | BPF_ADD | BPF_X:
+ case BPF_ALU64 | BPF_ADD | BPF_X:
+ emit_insn(ctx, addd, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst + imm */
+ case BPF_ALU | BPF_ADD | BPF_K:
+ case BPF_ALU64 | BPF_ADD | BPF_K:
+ if (is_signed_imm12(imm)) {
+ emit_insn(ctx, addid, dst, dst, imm);
+ } else {
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, addd, dst, dst, t1);
+ }
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst - src */
+ case BPF_ALU | BPF_SUB | BPF_X:
+ case BPF_ALU64 | BPF_SUB | BPF_X:
+ emit_insn(ctx, subd, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst - imm */
+ case BPF_ALU | BPF_SUB | BPF_K:
+ case BPF_ALU64 | BPF_SUB | BPF_K:
+ if (is_signed_imm12(-imm)) {
+ emit_insn(ctx, addid, dst, dst, -imm);
+ } else {
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, subd, dst, dst, t1);
+ }
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst * src */
+ case BPF_ALU | BPF_MUL | BPF_X:
+ case BPF_ALU64 | BPF_MUL | BPF_X:
+ emit_insn(ctx, muld, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst * imm */
+ case BPF_ALU | BPF_MUL | BPF_K:
+ case BPF_ALU64 | BPF_MUL | BPF_K:
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, muld, dst, dst, t1);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst / src */
+ case BPF_ALU | BPF_DIV | BPF_X:
+ case BPF_ALU64 | BPF_DIV | BPF_X:
+ emit_zext_32(ctx, dst, is32);
+ move_reg(ctx, t1, src);
+ emit_zext_32(ctx, t1, is32);
+ emit_insn(ctx, divdu, dst, dst, t1);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst / imm */
+ case BPF_ALU | BPF_DIV | BPF_K:
+ case BPF_ALU64 | BPF_DIV | BPF_K:
+ move_imm(ctx, t1, imm, is32);
+ emit_zext_32(ctx, dst, is32);
+ emit_insn(ctx, divdu, dst, dst, t1);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst % src */
+ case BPF_ALU | BPF_MOD | BPF_X:
+ case BPF_ALU64 | BPF_MOD | BPF_X:
+ emit_zext_32(ctx, dst, is32);
+ move_reg(ctx, t1, src);
+ emit_zext_32(ctx, t1, is32);
+ emit_insn(ctx, moddu, dst, dst, t1);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst % imm */
+ case BPF_ALU | BPF_MOD | BPF_K:
+ case BPF_ALU64 | BPF_MOD | BPF_K:
+ move_imm(ctx, t1, imm, is32);
+ emit_zext_32(ctx, dst, is32);
+ emit_insn(ctx, moddu, dst, dst, t1);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = -dst */
+ case BPF_ALU | BPF_NEG:
+ case BPF_ALU64 | BPF_NEG:
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst & src */
+ case BPF_ALU | BPF_AND | BPF_X:
+ case BPF_ALU64 | BPF_AND | BPF_X:
+ emit_insn(ctx, and, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst & imm */
+ case BPF_ALU | BPF_AND | BPF_K:
+ case BPF_ALU64 | BPF_AND | BPF_K:
+ if (is_unsigned_imm12(imm)) {
+ emit_insn(ctx, andi, dst, dst, imm);
+ } else {
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, and, dst, dst, t1);
+ }
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst | src */
+ case BPF_ALU | BPF_OR | BPF_X:
+ case BPF_ALU64 | BPF_OR | BPF_X:
+ emit_insn(ctx, or, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst | imm */
+ case BPF_ALU | BPF_OR | BPF_K:
+ case BPF_ALU64 | BPF_OR | BPF_K:
+ if (is_unsigned_imm12(imm)) {
+ emit_insn(ctx, ori, dst, dst, imm);
+ } else {
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, or, dst, dst, t1);
+ }
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst ^ src */
+ case BPF_ALU | BPF_XOR | BPF_X:
+ case BPF_ALU64 | BPF_XOR | BPF_X:
+ emit_insn(ctx, xor, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst ^ imm */
+ case BPF_ALU | BPF_XOR | BPF_K:
+ case BPF_ALU64 | BPF_XOR | BPF_K:
+ if (is_unsigned_imm12(imm)) {
+ emit_insn(ctx, xori, dst, dst, imm);
+ } else {
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, xor, dst, dst, t1);
+ }
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ /* dst = dst << src (logical) */
+ case BPF_ALU | BPF_LSH | BPF_X:
+ emit_insn(ctx, sllw, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ case BPF_ALU64 | BPF_LSH | BPF_X:
+ emit_insn(ctx, slld, dst, dst, src);
+ break;
+
+ /* dst = dst << imm (logical) */
+ case BPF_ALU | BPF_LSH | BPF_K:
+ emit_insn(ctx, slliw, dst, dst, imm);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ case BPF_ALU64 | BPF_LSH | BPF_K:
+ emit_insn(ctx, sllid, dst, dst, imm);
+ break;
+
+ /* dst = dst >> src (logical) */
+ case BPF_ALU | BPF_RSH | BPF_X:
+ emit_insn(ctx, srlw, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ case BPF_ALU64 | BPF_RSH | BPF_X:
+ emit_insn(ctx, srld, dst, dst, src);
+ break;
+
+ /* dst = dst >> imm (logical) */
+ case BPF_ALU | BPF_RSH | BPF_K:
+ emit_insn(ctx, srliw, dst, dst, imm);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ case BPF_ALU64 | BPF_RSH | BPF_K:
+ emit_insn(ctx, srlid, dst, dst, imm);
+ break;
+
+ /* dst = dst >> src (arithmetic) */
+ case BPF_ALU | BPF_ARSH | BPF_X:
+ emit_insn(ctx, sraw, dst, dst, src);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ case BPF_ALU64 | BPF_ARSH | BPF_X:
+ emit_insn(ctx, srad, dst, dst, src);
+ break;
+
+ /* dst = dst >> imm (arithmetic) */
+ case BPF_ALU | BPF_ARSH | BPF_K:
+ emit_insn(ctx, sraiw, dst, dst, imm);
+ emit_zext_32(ctx, dst, is32);
+ break;
+
+ case BPF_ALU64 | BPF_ARSH | BPF_K:
+ emit_insn(ctx, sraid, dst, dst, imm);
+ break;
+
+ /* dst = BSWAP##imm(dst) */
+ case BPF_ALU | BPF_END | BPF_FROM_LE:
+ switch (imm) {
+ case 16:
+ /* zero-extend 16 bits into 64 bits */
+ emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
+ break;
+ case 32:
+ /* zero-extend 32 bits into 64 bits */
+ emit_zext_32(ctx, dst, is32);
+ break;
+ case 64:
+ /* do nothing */
+ break;
+ }
+ break;
+
+ case BPF_ALU | BPF_END | BPF_FROM_BE:
+ switch (imm) {
+ case 16:
+ emit_insn(ctx, revb2h, dst, dst);
+ /* zero-extend 16 bits into 64 bits */
+ emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
+ break;
+ case 32:
+ emit_insn(ctx, revb2w, dst, dst);
+ /* zero-extend 32 bits into 64 bits */
+ emit_zext_32(ctx, dst, is32);
+ break;
+ case 64:
+ emit_insn(ctx, revbd, dst, dst);
+ break;
+ }
+ break;
+
+ /* PC += off if dst cond src */
+ case BPF_JMP | BPF_JEQ | BPF_X:
+ case BPF_JMP | BPF_JNE | BPF_X:
+ case BPF_JMP | BPF_JGT | BPF_X:
+ case BPF_JMP | BPF_JGE | BPF_X:
+ case BPF_JMP | BPF_JLT | BPF_X:
+ case BPF_JMP | BPF_JLE | BPF_X:
+ case BPF_JMP | BPF_JSGT | BPF_X:
+ case BPF_JMP | BPF_JSGE | BPF_X:
+ case BPF_JMP | BPF_JSLT | BPF_X:
+ case BPF_JMP | BPF_JSLE | BPF_X:
+ case BPF_JMP32 | BPF_JEQ | BPF_X:
+ case BPF_JMP32 | BPF_JNE | BPF_X:
+ case BPF_JMP32 | BPF_JGT | BPF_X:
+ case BPF_JMP32 | BPF_JGE | BPF_X:
+ case BPF_JMP32 | BPF_JLT | BPF_X:
+ case BPF_JMP32 | BPF_JLE | BPF_X:
+ case BPF_JMP32 | BPF_JSGT | BPF_X:
+ case BPF_JMP32 | BPF_JSGE | BPF_X:
+ case BPF_JMP32 | BPF_JSLT | BPF_X:
+ case BPF_JMP32 | BPF_JSLE | BPF_X:
+ jmp_offset = bpf2la_offset(i, off, ctx);
+ move_reg(ctx, t1, dst);
+ move_reg(ctx, t2, src);
+ if (is_signed_bpf_cond(BPF_OP(code))) {
+ emit_sext_32(ctx, t1, is32);
+ emit_sext_32(ctx, t2, is32);
+ } else {
+ emit_zext_32(ctx, t1, is32);
+ emit_zext_32(ctx, t2, is32);
+ }
+ if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
+ goto toofar;
+ break;
+
+ /* PC += off if dst cond imm */
+ case BPF_JMP | BPF_JEQ | BPF_K:
+ case BPF_JMP | BPF_JNE | BPF_K:
+ case BPF_JMP | BPF_JGT | BPF_K:
+ case BPF_JMP | BPF_JGE | BPF_K:
+ case BPF_JMP | BPF_JLT | BPF_K:
+ case BPF_JMP | BPF_JLE | BPF_K:
+ case BPF_JMP | BPF_JSGT | BPF_K:
+ case BPF_JMP | BPF_JSGE | BPF_K:
+ case BPF_JMP | BPF_JSLT | BPF_K:
+ case BPF_JMP | BPF_JSLE | BPF_K:
+ case BPF_JMP32 | BPF_JEQ | BPF_K:
+ case BPF_JMP32 | BPF_JNE | BPF_K:
+ case BPF_JMP32 | BPF_JGT | BPF_K:
+ case BPF_JMP32 | BPF_JGE | BPF_K:
+ case BPF_JMP32 | BPF_JLT | BPF_K:
+ case BPF_JMP32 | BPF_JLE | BPF_K:
+ case BPF_JMP32 | BPF_JSGT | BPF_K:
+ case BPF_JMP32 | BPF_JSGE | BPF_K:
+ case BPF_JMP32 | BPF_JSLT | BPF_K:
+ case BPF_JMP32 | BPF_JSLE | BPF_K:
+ u8 t7 = -1;
+ jmp_offset = bpf2la_offset(i, off, ctx);
+ if (imm) {
+ move_imm(ctx, t1, imm, false);
+ t7 = t1;
+ } else {
+ /* If imm is 0, simply use zero register. */
+ t7 = LOONGARCH_GPR_ZERO;
+ }
+ move_reg(ctx, t2, dst);
+ if (is_signed_bpf_cond(BPF_OP(code))) {
+ emit_sext_32(ctx, t7, is32);
+ emit_sext_32(ctx, t2, is32);
+ } else {
+ emit_zext_32(ctx, t7, is32);
+ emit_zext_32(ctx, t2, is32);
+ }
+ if (emit_cond_jmp(ctx, cond, t2, t7, jmp_offset) < 0)
+ goto toofar;
+ break;
+
+ /* PC += off if dst & src */
+ case BPF_JMP | BPF_JSET | BPF_X:
+ case BPF_JMP32 | BPF_JSET | BPF_X:
+ jmp_offset = bpf2la_offset(i, off, ctx);
+ emit_insn(ctx, and, t1, dst, src);
+ emit_zext_32(ctx, t1, is32);
+ if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
+ goto toofar;
+ break;
+
+ /* PC += off if dst & imm */
+ case BPF_JMP | BPF_JSET | BPF_K:
+ case BPF_JMP32 | BPF_JSET | BPF_K:
+ jmp_offset = bpf2la_offset(i, off, ctx);
+ move_imm(ctx, t1, imm, is32);
+ emit_insn(ctx, and, t1, dst, t1);
+ emit_zext_32(ctx, t1, is32);
+ if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
+ goto toofar;
+ break;
+
+ /* PC += off */
+ case BPF_JMP | BPF_JA:
+ jmp_offset = bpf2la_offset(i, off, ctx);
+ if (emit_uncond_jmp(ctx, jmp_offset) < 0)
+ goto toofar;
+ break;
+
+ /* function call */
+ case BPF_JMP | BPF_CALL:
+ int ret;
+ u64 func_addr;
+ bool func_addr_fixed;
+
+ mark_call(ctx);
+ ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
+ &func_addr, &func_addr_fixed);
+ if (ret < 0)
+ return ret;
+
+ move_imm(ctx, t1, func_addr, is32);
+ emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0);
+ move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
+ break;
+
+ /* tail call */
+ case BPF_JMP | BPF_TAIL_CALL:
+ mark_tail_call(ctx);
+ if (emit_bpf_tail_call(ctx) < 0)
+ return -EINVAL;
+ break;
+
+ /* function return */
+ case BPF_JMP | BPF_EXIT:
+ emit_sext_32(ctx, regmap[BPF_REG_0], true);
+
+ if (i == ctx->prog->len - 1)
+ break;
+
+ jmp_offset = epilogue_offset(ctx);
+ if (emit_uncond_jmp(ctx, jmp_offset) < 0)
+ goto toofar;
+ break;
+
+ /* dst = imm64 */
+ case BPF_LD | BPF_IMM | BPF_DW:
+ u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
+
+ move_imm(ctx, dst, imm64, is32);
+ return 1;
+
+ /* dst = *(size *)(src + off) */
+ case BPF_LDX | BPF_MEM | BPF_B:
+ case BPF_LDX | BPF_MEM | BPF_H:
+ case BPF_LDX | BPF_MEM | BPF_W:
+ case BPF_LDX | BPF_MEM | BPF_DW:
+ switch (BPF_SIZE(code)) {
+ case BPF_B:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, ldbu, dst, src, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, ldxbu, dst, src, t1);
+ }
+ break;
+ case BPF_H:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, ldhu, dst, src, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, ldxhu, dst, src, t1);
+ }
+ break;
+ case BPF_W:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, ldwu, dst, src, off);
+ } else if (is_signed_imm14(off)) {
+ emit_insn(ctx, ldptrw, dst, src, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, ldxwu, dst, src, t1);
+ }
+ break;
+ case BPF_DW:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, ldd, dst, src, off);
+ } else if (is_signed_imm14(off)) {
+ emit_insn(ctx, ldptrd, dst, src, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, ldxd, dst, src, t1);
+ }
+ break;
+ }
+ break;
+
+ /* *(size *)(dst + off) = imm */
+ case BPF_ST | BPF_MEM | BPF_B:
+ case BPF_ST | BPF_MEM | BPF_H:
+ case BPF_ST | BPF_MEM | BPF_W:
+ case BPF_ST | BPF_MEM | BPF_DW:
+ switch (BPF_SIZE(code)) {
+ case BPF_B:
+ move_imm(ctx, t1, imm, is32);
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, stb, t1, dst, off);
+ } else {
+ move_imm(ctx, t2, off, is32);
+ emit_insn(ctx, stxb, t1, dst, t2);
+ }
+ break;
+ case BPF_H:
+ move_imm(ctx, t1, imm, is32);
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, sth, t1, dst, off);
+ } else {
+ move_imm(ctx, t2, off, is32);
+ emit_insn(ctx, stxh, t1, dst, t2);
+ }
+ break;
+ case BPF_W:
+ move_imm(ctx, t1, imm, is32);
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, stw, t1, dst, off);
+ } else if (is_signed_imm14(off)) {
+ emit_insn(ctx, stptrw, t1, dst, off);
+ } else {
+ move_imm(ctx, t2, off, is32);
+ emit_insn(ctx, stxw, t1, dst, t2);
+ }
+ break;
+ case BPF_DW:
+ move_imm(ctx, t1, imm, is32);
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, std, t1, dst, off);
+ } else if (is_signed_imm14(off)) {
+ emit_insn(ctx, stptrd, t1, dst, off);
+ } else {
+ move_imm(ctx, t2, off, is32);
+ emit_insn(ctx, stxd, t1, dst, t2);
+ }
+ break;
+ }
+ break;
+
+ /* *(size *)(dst + off) = src */
+ case BPF_STX | BPF_MEM | BPF_B:
+ case BPF_STX | BPF_MEM | BPF_H:
+ case BPF_STX | BPF_MEM | BPF_W:
+ case BPF_STX | BPF_MEM | BPF_DW:
+ switch (BPF_SIZE(code)) {
+ case BPF_B:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, stb, src, dst, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, stxb, src, dst, t1);
+ }
+ break;
+ case BPF_H:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, sth, src, dst, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, stxh, src, dst, t1);
+ }
+ break;
+ case BPF_W:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, stw, src, dst, off);
+ } else if (is_signed_imm14(off)) {
+ emit_insn(ctx, stptrw, src, dst, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, stxw, src, dst, t1);
+ }
+ break;
+ case BPF_DW:
+ if (is_signed_imm12(off)) {
+ emit_insn(ctx, std, src, dst, off);
+ } else if (is_signed_imm14(off)) {
+ emit_insn(ctx, stptrd, src, dst, off);
+ } else {
+ move_imm(ctx, t1, off, is32);
+ emit_insn(ctx, stxd, src, dst, t1);
+ }
+ break;
+ }
+ break;
+
+ case BPF_STX | BPF_ATOMIC | BPF_W:
+ case BPF_STX | BPF_ATOMIC | BPF_DW:
+ emit_atomic(insn, ctx);
+ break;
+
+ default:
+ pr_err("bpf_jit: unknown opcode %02x\n", code);
+ return -EINVAL;
+ }
+
+ return 0;
+
+toofar:
+ pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
+ return -E2BIG;
+}
+
+static int build_body(struct jit_ctx *ctx, bool extra_pass)
+{
+ int i;
+ const struct bpf_prog *prog = ctx->prog;
+
+ for (i = 0; i < prog->len; i++) {
+ const struct bpf_insn *insn = &prog->insnsi[i];
+ int ret;
+
+ if (ctx->image == NULL)
+ ctx->offset[i] = ctx->idx;
+
+ ret = build_insn(insn, ctx, extra_pass);
+ if (ret > 0) {
+ i++;
+ if (ctx->image == NULL)
+ ctx->offset[i] = ctx->idx;
+ continue;
+ }
+ if (ret)
+ return ret;
+ }
+
+ if (ctx->image == NULL)
+ ctx->offset[i] = ctx->idx;
+
+ return 0;
+}
+
+/* Fill space with break instructions */
+static void jit_fill_hole(void *area, unsigned int size)
+{
+ u32 *ptr;
+
+ /* We are guaranteed to have aligned memory */
+ for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
+ *ptr++ = INSN_BREAK;
+}
+
+static int validate_code(struct jit_ctx *ctx)
+{
+ int i;
+ union loongarch_instruction insn;
+
+ for (i = 0; i < ctx->idx; i++) {
+ insn = ctx->image[i];
+ /* Check INSN_BREAK */
+ if (insn.word == INSN_BREAK)
+ return -1;
+ }
+
+ return 0;
+}
+
+struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+{
+ bool tmp_blinded = false, extra_pass = false;
+ u8 *image_ptr;
+ int image_size;
+ struct jit_ctx ctx;
+ struct jit_data *jit_data;
+ struct bpf_binary_header *header;
+ struct bpf_prog *tmp, *orig_prog = prog;
+
+ /*
+ * If BPF JIT was not enabled then we must fall back to
+ * the interpreter.
+ */
+ if (!prog->jit_requested)
+ return orig_prog;
+
+ tmp = bpf_jit_blind_constants(prog);
+ /*
+ * If blinding was requested and we failed during blinding,
+ * we must fall back to the interpreter. Otherwise, we save
+ * the new JITed code.
+ */
+ if (IS_ERR(tmp))
+ return orig_prog;
+
+ if (tmp != prog) {
+ tmp_blinded = true;
+ prog = tmp;
+ }
+
+ jit_data = prog->aux->jit_data;
+ if (!jit_data) {
+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+ if (!jit_data) {
+ prog = orig_prog;
+ goto out;
+ }
+ prog->aux->jit_data = jit_data;
+ }
+ if (jit_data->ctx.offset) {
+ ctx = jit_data->ctx;
+ image_ptr = jit_data->image;
+ header = jit_data->header;
+ extra_pass = true;
+ image_size = sizeof(u32) * ctx.idx;
+ goto skip_init_ctx;
+ }
+
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.prog = prog;
+
+ ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
+ if (ctx.offset == NULL) {
+ prog = orig_prog;
+ goto out_offset;
+ }
+
+ /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
+ build_prologue(&ctx);
+ if (build_body(&ctx, extra_pass)) {
+ prog = orig_prog;
+ goto out_offset;
+ }
+ ctx.epilogue_offset = ctx.idx;
+ build_epilogue(&ctx);
+
+ /* Now we know the actual image size.
+ * As each LoongArch instruction is of length 32bit,
+ * we are translating number of JITed intructions into
+ * the size required to store these JITed code.
+ */
+ image_size = sizeof(u32) * ctx.idx;
+ /* Now we know the size of the structure to make */
+ header = bpf_jit_binary_alloc(image_size, &image_ptr,
+ sizeof(u32), jit_fill_hole);
+ if (header == NULL) {
+ prog = orig_prog;
+ goto out_offset;
+ }
+
+ /* 2. Now, the actual pass to generate final JIT code */
+ ctx.image = (union loongarch_instruction *)image_ptr;
+
+skip_init_ctx:
+ ctx.idx = 0;
+
+ build_prologue(&ctx);
+ if (build_body(&ctx, extra_pass)) {
+ bpf_jit_binary_free(header);
+ prog = orig_prog;
+ goto out_offset;
+ }
+ build_epilogue(&ctx);
+
+ /* 3. Extra pass to validate JITed code */
+ if (validate_code(&ctx)) {
+ bpf_jit_binary_free(header);
+ prog = orig_prog;
+ goto out_offset;
+ }
+
+ /* And we're done */
+ if (bpf_jit_enable > 1)
+ bpf_jit_dump(prog->len, image_size, 2, ctx.image);
+
+ /* Update the icache */
+ flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
+
+ if (!prog->is_func || extra_pass) {
+ if (extra_pass && ctx.idx != jit_data->ctx.idx) {
+ pr_err_once("multi-func JIT bug %d != %d\n",
+ ctx.idx, jit_data->ctx.idx);
+ bpf_jit_binary_free(header);
+ prog->bpf_func = NULL;
+ prog->jited = 0;
+ prog->jited_len = 0;
+ goto out_offset;
+ }
+ bpf_jit_binary_lock_ro(header);
+ } else {
+ jit_data->ctx = ctx;
+ jit_data->image = image_ptr;
+ jit_data->header = header;
+ }
+ prog->jited = 1;
+ prog->jited_len = image_size;
+ prog->bpf_func = (void *)ctx.image;
+
+ if (!prog->is_func || extra_pass) {
+ int i;
+
+ /* offset[prog->len] is the size of program */
+ for (i = 0; i <= prog->len; i++)
+ ctx.offset[i] *= LOONGARCH_INSN_SIZE;
+ bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
+
+out_offset:
+ kvfree(ctx.offset);
+ kfree(jit_data);
+ prog->aux->jit_data = NULL;
+ }
+
+out:
+ if (tmp_blinded)
+ bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
+
+ out_offset = -1;
+
+ return prog;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * BPF JIT compiler for LoongArch
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <asm/cacheflush.h>
+#include <asm/inst.h>
+
+struct jit_ctx {
+ const struct bpf_prog *prog;
+ unsigned int idx;
+ unsigned int flags;
+ unsigned int epilogue_offset;
+ u32 *offset;
+ union loongarch_instruction *image;
+ u32 stack_size;
+};
+
+struct jit_data {
+ struct bpf_binary_header *header;
+ u8 *image;
+ struct jit_ctx ctx;
+};
+
+#define emit_insn(ctx, func, ...) \
+do { \
+ if (ctx->image != NULL) { \
+ union loongarch_instruction *insn = &ctx->image[ctx->idx]; \
+ emit_##func(insn, ##__VA_ARGS__); \
+ } \
+ ctx->idx++; \
+} while (0)
+
+#define is_signed_imm12(val) signed_imm_check(val, 12)
+#define is_signed_imm14(val) signed_imm_check(val, 14)
+#define is_signed_imm16(val) signed_imm_check(val, 16)
+#define is_signed_imm26(val) signed_imm_check(val, 26)
+#define is_signed_imm32(val) signed_imm_check(val, 32)
+#define is_signed_imm52(val) signed_imm_check(val, 52)
+#define is_unsigned_imm12(val) unsigned_imm_check(val, 12)
+
+static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx)
+{
+ /* BPF JMP offset is relative to the next instruction */
+ bpf_insn++;
+ /*
+ * Whereas LoongArch branch instructions encode the offset
+ * from the branch itself, so we must subtract 1 from the
+ * instruction offset.
+ */
+ return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1));
+}
+
+static inline int epilogue_offset(const struct jit_ctx *ctx)
+{
+ int from = ctx->idx;
+ int to = ctx->epilogue_offset;
+
+ return (to - from);
+}
+
+/* Zero-extend 32 bits into 64 bits */
+static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
+{
+ if (!is32)
+ return;
+
+ emit_insn(ctx, lu32id, reg, 0);
+}
+
+/* Signed-extend 32 bits into 64 bits */
+static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
+{
+ if (!is32)
+ return;
+
+ emit_insn(ctx, addiw, reg, reg, 0);
+}
+
+static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm, bool is32)
+{
+ long imm_11_0, imm_31_12, imm_51_32, imm_63_52, imm_51_0, imm_51_31;
+
+ /* or rd, $zero, $zero */
+ if (imm == 0) {
+ emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO);
+ return;
+ }
+
+ /* addiw rd, $zero, imm_11_0 */
+ if (is_signed_imm12(imm)) {
+ emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm);
+ goto zext;
+ }
+
+ /* ori rd, $zero, imm_11_0 */
+ if (is_unsigned_imm12(imm)) {
+ emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm);
+ goto zext;
+ }
+
+ /* lu52id rd, $zero, imm_63_52 */
+ imm_63_52 = (imm >> 52) & 0xfff;
+ imm_51_0 = imm & 0xfffffffffffff;
+ if (imm_63_52 != 0 && imm_51_0 == 0) {
+ emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, imm_63_52);
+ return;
+ }
+
+ /* lu12iw rd, imm_31_12 */
+ imm_31_12 = (imm >> 12) & 0xfffff;
+ emit_insn(ctx, lu12iw, rd, imm_31_12);
+
+ /* ori rd, rd, imm_11_0 */
+ imm_11_0 = imm & 0xfff;
+ if (imm_11_0 != 0)
+ emit_insn(ctx, ori, rd, rd, imm_11_0);
+
+ if (!is_signed_imm32(imm)) {
+ if (imm_51_0 != 0) {
+ /*
+ * If bit[51:31] is all 0 or all 1,
+ * it means bit[51:32] is sign extended by lu12iw,
+ * no need to call lu32id to do a new filled operation.
+ */
+ imm_51_31 = (imm >> 31) & 0x1fffff;
+ if (imm_51_31 != 0 || imm_51_31 != 0x1fffff) {
+ /* lu32id rd, imm_51_32 */
+ imm_51_32 = (imm >> 32) & 0xfffff;
+ emit_insn(ctx, lu32id, rd, imm_51_32);
+ }
+ }
+
+ /* lu52id rd, rd, imm_63_52 */
+ if (!is_signed_imm52(imm))
+ emit_insn(ctx, lu52id, rd, rd, imm_63_52);
+ }
+
+zext:
+ emit_zext_32(ctx, rd, is32);
+}
+
+static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd,
+ enum loongarch_gpr rj)
+{
+ emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO);
+}
+
+static inline int invert_jmp_cond(u8 cond)
+{
+ switch (cond) {
+ case BPF_JEQ:
+ return BPF_JNE;
+ case BPF_JNE:
+ case BPF_JSET:
+ return BPF_JEQ;
+ case BPF_JGT:
+ return BPF_JLE;
+ case BPF_JGE:
+ return BPF_JLT;
+ case BPF_JLT:
+ return BPF_JGE;
+ case BPF_JLE:
+ return BPF_JGT;
+ case BPF_JSGT:
+ return BPF_JSLE;
+ case BPF_JSGE:
+ return BPF_JSLT;
+ case BPF_JSLT:
+ return BPF_JSGE;
+ case BPF_JSLE:
+ return BPF_JSGT;
+ }
+ return -1;
+}
+
+static inline void cond_jmp_offset(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
+ enum loongarch_gpr rd, int jmp_offset)
+{
+ switch (cond) {
+ case BPF_JEQ:
+ /* PC += jmp_offset if rj == rd */
+ emit_insn(ctx, beq, rj, rd, jmp_offset);
+ return;
+ case BPF_JNE:
+ case BPF_JSET:
+ /* PC += jmp_offset if rj != rd */
+ emit_insn(ctx, bne, rj, rd, jmp_offset);
+ return;
+ case BPF_JGT:
+ /* PC += jmp_offset if rj > rd (unsigned) */
+ emit_insn(ctx, bltu, rd, rj, jmp_offset);
+ return;
+ case BPF_JLT:
+ /* PC += jmp_offset if rj < rd (unsigned) */
+ emit_insn(ctx, bltu, rj, rd, jmp_offset);
+ return;
+ case BPF_JGE:
+ /* PC += jmp_offset if rj >= rd (unsigned) */
+ emit_insn(ctx, bgeu, rj, rd, jmp_offset);
+ return;
+ case BPF_JLE:
+ /* PC += jmp_offset if rj <= rd (unsigned) */
+ emit_insn(ctx, bgeu, rd, rj, jmp_offset);
+ return;
+ case BPF_JSGT:
+ /* PC += jmp_offset if rj > rd (signed) */
+ emit_insn(ctx, blt, rd, rj, jmp_offset);
+ return;
+ case BPF_JSLT:
+ /* PC += jmp_offset if rj < rd (signed) */
+ emit_insn(ctx, blt, rj, rd, jmp_offset);
+ return;
+ case BPF_JSGE:
+ /* PC += jmp_offset if rj >= rd (signed) */
+ emit_insn(ctx, bge, rj, rd, jmp_offset);
+ return;
+ case BPF_JSLE:
+ /* PC += jmp_offset if rj <= rd (signed) */
+ emit_insn(ctx, bge, rd, rj, jmp_offset);
+ return;
+ }
+}
+
+static inline void cond_jmp_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
+ enum loongarch_gpr rd, int jmp_offset)
+{
+ cond = invert_jmp_cond(cond);
+ cond_jmp_offset(ctx, cond, rj, rd, 2);
+ emit_insn(ctx, b, jmp_offset);
+}
+
+static inline void uncond_jmp_offs26(struct jit_ctx *ctx, int jmp_offset)
+{
+ emit_insn(ctx, b, jmp_offset);
+}
+
+static inline int emit_cond_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
+ enum loongarch_gpr rd, int jmp_offset)
+{
+ /*
+ * A large PC-relative jump offset may overflow the immediate field of
+ * the native conditional branch instruction, triggering a conversion
+ * to use an absolute jump instead, this jump sequence is particularly
+ * nasty. For now, use cond_jmp_offs26() directly to keep it simple.
+ * In the future, maybe we can add support for far branching, the branch
+ * relaxation requires more than two passes to converge, the code seems
+ * too complex to understand, not quite sure whether it is necessary and
+ * worth the extra pain. Anyway, just leave it as it is to enhance code
+ * readability now.
+ */
+ if (is_signed_imm26(jmp_offset)) {
+ cond_jmp_offs26(ctx, cond, rj, rd, jmp_offset);
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static inline int emit_uncond_jmp(struct jit_ctx *ctx, int jmp_offset)
+{
+ if (is_signed_imm26(jmp_offset)) {
+ uncond_jmp_offs26(ctx, jmp_offset);
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
+ enum loongarch_gpr rd, int jmp_offset)
+{
+ if (is_signed_imm16(jmp_offset)) {
+ cond_jmp_offset(ctx, cond, rj, rd, jmp_offset);
+ return 0;
+ }
+
+ return -EINVAL;
+}
return 0;
}
+/*
+ * Create a PCI config space window
+ * - reserve mem region
+ * - alloc struct pci_config_window with space for all mappings
+ * - ioremap the config space
+ */
+static struct pci_config_window *arch_pci_ecam_create(struct device *dev,
+ struct resource *cfgres, struct resource *busr, const struct pci_ecam_ops *ops)
+{
+ int bsz, bus_range, err;
+ struct resource *conflict;
+ struct pci_config_window *cfg;
+
+ if (busr->start > busr->end)
+ return ERR_PTR(-EINVAL);
+
+ cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+ if (!cfg)
+ return ERR_PTR(-ENOMEM);
+
+ cfg->parent = dev;
+ cfg->ops = ops;
+ cfg->busr.start = busr->start;
+ cfg->busr.end = busr->end;
+ cfg->busr.flags = IORESOURCE_BUS;
+ bus_range = resource_size(cfgres) >> ops->bus_shift;
+
+ bsz = 1 << ops->bus_shift;
+
+ cfg->res.start = cfgres->start;
+ cfg->res.end = cfgres->end;
+ cfg->res.flags = IORESOURCE_MEM | IORESOURCE_BUSY;
+ cfg->res.name = "PCI ECAM";
+
+ conflict = request_resource_conflict(&iomem_resource, &cfg->res);
+ if (conflict) {
+ err = -EBUSY;
+ dev_err(dev, "can't claim ECAM area %pR: address conflict with %s %pR\n",
+ &cfg->res, conflict->name, conflict);
+ goto err_exit;
+ }
+
+ cfg->win = pci_remap_cfgspace(cfgres->start, bus_range * bsz);
+ if (!cfg->win)
+ goto err_exit_iomap;
+
+ if (ops->init) {
+ err = ops->init(cfg);
+ if (err)
+ goto err_exit;
+ }
+ dev_info(dev, "ECAM at %pR for %pR\n", &cfg->res, &cfg->busr);
+
+ return cfg;
+
+err_exit_iomap:
+ err = -ENOMEM;
+ dev_err(dev, "ECAM ioremap failed\n");
+err_exit:
+ pci_ecam_free(cfg);
+ return ERR_PTR(err);
+}
+
/*
* Lookup the bus range for the domain in MCFG, and set up config space
* mapping.
bus_shift = ecam_ops->bus_shift ? : 20;
- cfgres.start = root->mcfg_addr + (bus_res->start << bus_shift);
- cfgres.end = cfgres.start + (resource_size(bus_res) << bus_shift) - 1;
- cfgres.flags = IORESOURCE_MEM;
+ if (bus_shift == 20)
+ cfg = pci_ecam_create(dev, &cfgres, bus_res, ecam_ops);
+ else {
+ cfgres.start = root->mcfg_addr + (bus_res->start << bus_shift);
+ cfgres.end = cfgres.start + (resource_size(bus_res) << bus_shift) - 1;
+ cfgres.end |= BIT(28) + (((PCI_CFG_SPACE_EXP_SIZE - 1) & 0xf00) << 16);
+ cfgres.flags = IORESOURCE_MEM;
+ cfg = arch_pci_ecam_create(dev, &cfgres, bus_res, ecam_ops);
+ }
- cfg = pci_ecam_create(dev, &cfgres, bus_res, ecam_ops);
if (IS_ERR(cfg)) {
dev_err(dev, "%04x:%pR error %ld mapping ECAM\n", seg, bus_res, PTR_ERR(cfg));
return NULL;
#include <linux/types.h>
#include <linux/pci.h>
#include <linux/vgaarb.h>
+#include <asm/cacheflush.h>
#include <asm/loongson.h>
#define PCI_DEVICE_ID_LOONGSON_HOST 0x7a00
unsigned int lsize;
/*
- * Set PCI cacheline size to that of the highest level in the
+ * Set PCI cacheline size to that of the last level in the
* cache hierarchy.
*/
- lsize = cpu_dcache_line_size();
- lsize = cpu_vcache_line_size() ? : lsize;
- lsize = cpu_scache_line_size() ? : lsize;
+ lsize = cpu_last_level_cache_line_size();
BUG_ON(!lsize);
#include <asm/io.h>
#include <asm/machdep.h>
-#if defined(CONFIG_M68328)
-#include <asm/MC68328.h>
-#elif defined(CONFIG_M68EZ328)
+#if defined(CONFIG_M68EZ328)
#include <asm/MC68EZ328.h>
#elif defined(CONFIG_M68VZ328)
#include <asm/MC68VZ328.h>
+#else
+#include <asm/MC68328.h>
#endif
/* assembler routines */
select GENERIC_CSUM
select CPU_NO_EFFICIENT_FFS
select HAVE_ARCH_HASH
+ select LEGACY_TIMER_TICK
help
The Freescale (was Motorola) 68000 CPU is the first generation of
the well known M68K family of processors. The CPU core as well as
config M68328
bool
depends on !MMU
- select LEGACY_TIMER_TICK
select M68000
help
Motorola 68328 processor support.
config M68EZ328
bool
depends on !MMU
- select LEGACY_TIMER_TICK
select M68000
help
Motorola 68EX328 processor support.
config M68VZ328
bool
depends on !MMU
- select LEGACY_TIMER_TICK
select M68000
help
Motorola 68VZ328 processor support.
order" to save memory that could be wasted for unused memory map.
Say N if not sure.
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order" if ADVANCED
depends on !SINGLE_MEMORY_CHUNK
default "11"
CONFIG_UNIX=y
CONFIG_INET=y
CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_IPV6 is not set
# CONFIG_WIRELESS is not set
# CONFIG_UEVENT_HELPER is not set
CONFIG_ROMFS_BACKED_BY_BOTH=y
# CONFIG_NETWORK_FILESYSTEMS is not set
CONFIG_PRINTK_TIME=y
-# CONFIG_ENABLE_MUST_CHECK is not set
# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
CONFIG_PANIC_ON_OOPS=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_DIAG is not set
# CONFIG_IPV6 is not set
# CONFIG_FW_LOADER is not set
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_DIAG is not set
# CONFIG_IPV6 is not set
# CONFIG_FW_LOADER is not set
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_DIAG is not set
# CONFIG_IPV6 is not set
# CONFIG_FW_LOADER is not set
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_DIAG is not set
# CONFIG_IPV6 is not set
# CONFIG_FW_LOADER is not set
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_DIAG is not set
# CONFIG_IPV6 is not set
# CONFIG_FW_LOADER is not set
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_DIAG is not set
# CONFIG_IPV6 is not set
# CONFIG_FW_LOADER is not set
/* Forward declaration, a strange C thing */
struct task_struct;
-/* Free all resources held by a thread. */
-static inline void release_thread(struct task_struct *dead_task)
-{
-}
-
unsigned long __get_wchan(struct task_struct *p);
void show_registers(struct pt_regs *regs);
CONFIG_CIFS_STATS2=y
CONFIG_ENCRYPTED_KEYS=y
CONFIG_DMA_CMA=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_KGDB=y
CONFIG_KGDB_TESTS=y
CONFIG_KGDB_KDB=y
.pgdir = swapper_pg_dir, \
}
-/* Free all resources held by a thread. */
-static inline void release_thread(struct task_struct *dead_task)
-{
-}
-
unsigned long __get_wchan(struct task_struct *p);
/* The size allocated for kernel stacks. This _must_ be a power of two! */
endchoice
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
range 14 64 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB
default "14" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB
CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
CONFIG_CRC32_SARWATE=y
CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_INFO_REDUCED=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_DEBUG_FS=y
CONFIG_CRYPTO_SHA256_OCTEON=m
CONFIG_CRYPTO_SHA512_OCTEON=m
CONFIG_CRYPTO_DES=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_FS=y
CONFIG_MAGIC_SYSRQ=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_DMA_CMA=y
CONFIG_CMA_SIZE_MBYTES=32
CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_FS=y
CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
CONFIG_CONSOLE_LOGLEVEL_QUIET=15
CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_FS=y
CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15
CONFIG_CONSOLE_LOGLEVEL_QUIET=15
CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_FS=y
CONFIG_LOG_BUF_SHIFT=16
CONFIG_CGROUPS=y
CONFIG_MEMCG=y
-CONFIG_MEMCG_SWAP=y
CONFIG_BLK_CGROUP=y
CONFIG_CGROUP_SCHED=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
CONFIG_MEMCG=y
-CONFIG_MEMCG_SWAP=y
CONFIG_BLK_CGROUP=y
CONFIG_CFS_BANDWIDTH=y
CONFIG_RT_GROUP_SCHED=y
# CONFIG_XZ_DEC_ARMTHUMB is not set
# CONFIG_XZ_DEC_SPARC is not set
CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_INFO_REDUCED=y
CONFIG_DEBUG_FS=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_CRC16=y
CONFIG_XZ_DEC=y
CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_DEBUG_FS=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_FONTS=y
CONFIG_FONT_SUN8x16=y
CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_READABLE_ASM=y
CONFIG_KGDB=y
CONFIG_CRC16=y
CONFIG_XZ_DEC=y
CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_STRIP_ASM_SYMS=y
CONFIG_DEBUG_FS=y
CONFIG_MAGIC_SYSRQ=y
struct task_struct;
-/* Free all resources held by a thread. */
-#define release_thread(thread) do { } while(0)
-
/*
* Do necessary setup to start up a newly executed thread.
*/
#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */
+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */
+
/* compatibility flags */
#define MAP_FILE 0
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
+ sp -= prandom_u32_max(PAGE_SIZE);
return sp & ALMASK;
}
}
if (current->flags & PF_RANDOMIZE) {
- base += get_random_int() & (VDSO_RANDOMIZE_SIZE - 1);
+ base += prandom_u32_max(VDSO_RANDOMIZE_SIZE);
base = PAGE_ALIGN(base);
}
source "kernel/Kconfig.hz"
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
range 9 20
default "11"
CONFIG_NFS_V3_ACL=y
CONFIG_ROOT_NFS=y
CONFIG_SUNRPC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_NFS_V3_ACL=y
CONFIG_ROOT_NFS=y
CONFIG_SUNRPC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
struct task_struct;
-/* Free all resources held by a thread. */
-static inline void release_thread(struct task_struct *dead_task)
-{
-}
-
extern unsigned long __get_wchan(struct task_struct *p);
#define task_pt_regs(p) \
void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp);
-void release_thread(struct task_struct *);
unsigned long __get_wchan(struct task_struct *p);
#define cpu_relax() barrier()
* We need to iterate through the pages, clearing the dcache for
* them and setting the cache-inhibit bit.
*/
- mmap_read_lock(&init_mm);
- error = walk_page_range(&init_mm, va, va + size, &set_nocache_walk_ops,
- NULL);
- mmap_read_unlock(&init_mm);
+ mmap_write_lock(&init_mm);
+ error = walk_page_range_novma(&init_mm, va, va + size,
+ &set_nocache_walk_ops, NULL, NULL);
+ mmap_write_unlock(&init_mm);
if (error)
return ERR_PTR(error);
{
unsigned long va = (unsigned long)cpu_addr;
- mmap_read_lock(&init_mm);
+ mmap_write_lock(&init_mm);
/* walk_page_range shouldn't be able to fail here */
- WARN_ON(walk_page_range(&init_mm, va, va + size,
- &clear_nocache_walk_ops, NULL));
- mmap_read_unlock(&init_mm);
+ WARN_ON(walk_page_range_novma(&init_mm, va, va + size,
+ &clear_nocache_walk_ops, NULL, NULL));
+ mmap_write_unlock(&init_mm);
}
void arch_sync_dma_for_device(phys_addr_t addr, size_t size,
show_registers(regs);
}
-void release_thread(struct task_struct *dead_task)
-{
-}
-
/*
* Copy the thread-specific (arch specific) info from the current
* process to the new one p
struct alt_instr {
s32 orig_offset; /* offset to original instructions */
- s32 len; /* end of original instructions */
- u32 cond; /* see ALT_COND_XXX */
+ s16 len; /* end of original instructions */
+ u16 cond; /* see ALT_COND_XXX */
u32 replacement; /* replacement instruction or code */
-};
+} __packed;
void set_kernel_text_rw(int enable_read_write);
void apply_alternatives_all(void);
/* Alternative SMP implementation. */
#define ALTERNATIVE(cond, replacement) "!0:" \
".section .altinstructions, \"aw\" !" \
- ".word (0b-4-.), 1, " __stringify(cond) "," \
- __stringify(replacement) " !" \
+ ".word (0b-4-.) !" \
+ ".hword 1, " __stringify(cond) " !" \
+ ".word " __stringify(replacement) " !" \
".previous"
#else
/* to replace one single instructions by a new instruction */
#define ALTERNATIVE(from, to, cond, replacement)\
.section .altinstructions, "aw" ! \
- .word (from - .), (to - from)/4 ! \
- .word cond, replacement ! \
+ .word (from - .) ! \
+ .hword (to - from)/4, cond ! \
+ .word replacement ! \
.previous
/* to replace multiple instructions by new code */
#define ALTERNATIVE_CODE(from, num_instructions, cond, new_instr_ptr)\
.section .altinstructions, "aw" ! \
- .word (from - .), -num_instructions ! \
- .word cond, (new_instr_ptr - .) ! \
+ .word (from - .) ! \
+ .hword -num_instructions, cond ! \
+ .word (new_instr_ptr - .) ! \
.previous
#endif /* __ASSEMBLY__ */
#define PDC_TYPE_SYSTEM_MAP 1 /* 32-bit, but supports PDC_SYSTEM_MAP */
#define PDC_TYPE_SNAKE 2 /* Doesn't support SYSTEM_MAP */
-void pdc_console_init(void); /* in pdc_console.c */
-void pdc_console_restart(void);
-
void setup_pdc(void); /* in inventory.c */
/* wrapper-functions from pdc.c */
#define _PAGE_PRESENT_BIT 22 /* (0x200) Software: translation valid */
#define _PAGE_HPAGE_BIT 21 /* (0x400) Software: Huge Page */
#define _PAGE_USER_BIT 20 /* (0x800) Software: User accessible page */
+#ifdef CONFIG_HUGETLB_PAGE
+#define _PAGE_SPECIAL_BIT _PAGE_DMB_BIT /* DMB feature is currently unused */
+#else
+#define _PAGE_SPECIAL_BIT _PAGE_HPAGE_BIT /* use unused HUGE PAGE bit */
+#endif
/* N.B. The bits are defined in terms of a 32 bit word above, so the */
/* following macro is ok for both 32 and 64 bit. */
#define _PAGE_PRESENT (1 << xlate_pabit(_PAGE_PRESENT_BIT))
#define _PAGE_HUGE (1 << xlate_pabit(_PAGE_HPAGE_BIT))
#define _PAGE_USER (1 << xlate_pabit(_PAGE_USER_BIT))
-#define _PAGE_SPECIAL (_PAGE_DMB)
+#define _PAGE_SPECIAL (1 << xlate_pabit(_PAGE_SPECIAL_BIT))
#define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED)
#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_SPECIAL)
struct mm_struct;
-/* Free all resources held by a thread. */
-extern void release_thread(struct task_struct *);
-
extern unsigned long __get_wchan(struct task_struct *p);
#define KSTK_EIP(tsk) ((tsk)->thread.regs.iaoq[0])
#define MADV_WIPEONFORK 71 /* Zero memory on fork, child only */
#define MADV_KEEPONFORK 72 /* Undo MADV_WIPEONFORK */
+#define MADV_COLLAPSE 73 /* Synchronous hugepage collapse */
+
#define MADV_HWPOISON 100 /* poison a page for testing */
#define MADV_SOFT_OFFLINE 101 /* soft offline page for testing */
struct alt_instr *entry;
int index = 0, applied = 0;
int num_cpus = num_online_cpus();
- u32 cond_check;
+ u16 cond_check;
cond_check = ALT_COND_ALWAYS |
((num_cpus == 1) ? ALT_COND_NO_SMP : 0) |
for (entry = start; entry < end; entry++, index++) {
- u32 *from, cond, replacement;
- s32 len;
+ u32 *from, replacement;
+ u16 cond;
+ s16 len;
from = (u32 *)((ulong)&entry->orig_offset + entry->orig_offset);
len = entry->len;
{
struct vm_area_struct *vma;
unsigned long usize = 0;
+ VMA_ITERATOR(vmi, mm, 0);
- for (vma = mm->mmap; vma && usize < parisc_cache_flush_threshold; vma = vma->vm_next)
+ for_each_vma(vmi, vma) {
+ if (usize >= parisc_cache_flush_threshold)
+ break;
usize += vma->vm_end - vma->vm_start;
+ }
return usize;
}
void flush_cache_mm(struct mm_struct *mm)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
/*
* Flushing the whole cache on each cpu takes forever on
}
/* Flush mm */
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ for_each_vma(vmi, vma)
flush_cache_pages(vma, vma->vm_start, vma->vm_end);
}
* Finally, _PAGE_READ goes in the top bit of PL1 (so we
* trigger an access rights trap in user space if the user
* tries to read an unreadable page */
+#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT
+ /* need to drop DMB bit, as it's used as SPECIAL flag */
+ depi 0,_PAGE_SPECIAL_BIT,1,\pte
+#endif
depd \pte,8,7,\prot
/* PAGE_USER indicates the page can be read with user privileges,
* makes the tlb entry for the differently formatted pa11
* insertion instructions */
.macro make_insert_tlb_11 spc,pte,prot
+#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT
+ /* need to drop DMB bit, as it's used as SPECIAL flag */
+ depi 0,_PAGE_SPECIAL_BIT,1,\pte
+#endif
zdep \spc,30,15,\prot
dep \pte,8,7,\prot
extru,= \pte,_PAGE_NO_CACHE_BIT,1,%r0
// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * PDC Console support - ie use firmware to dump text via boot console
+ * PDC early console support - use PDC firmware to dump text via boot console
*
- * Copyright (C) 1999-2003 Matthew Wilcox <willy at parisc-linux.org>
- * Copyright (C) 2000 Martin K Petersen <mkp at mkp.net>
- * Copyright (C) 2000 John Marvin <jsm at parisc-linux.org>
- * Copyright (C) 2000-2003 Paul Bame <bame at parisc-linux.org>
- * Copyright (C) 2000 Philipp Rumpf <prumpf with tux.org>
- * Copyright (C) 2000 Michael Ang <mang with subcarrier.org>
- * Copyright (C) 2000 Grant Grundler <grundler with parisc-linux.org>
- * Copyright (C) 2001-2002 Ryan Bradetich <rbrad at parisc-linux.org>
- * Copyright (C) 2001 Helge Deller <deller at parisc-linux.org>
- * Copyright (C) 2001 Thomas Bogendoerfer <tsbogend at parisc-linux.org>
- * Copyright (C) 2002 Randolph Chung <tausq with parisc-linux.org>
- * Copyright (C) 2010 Guy Martin <gmsoft at tuxicoman.be>
+ * Copyright (C) 2001-2022 Helge Deller <deller@gmx.de>
*/
-/*
- * The PDC console is a simple console, which can be used for debugging
- * boot related problems on HP PA-RISC machines. It is also useful when no
- * other console works.
- *
- * This code uses the ROM (=PDC) based functions to read and write characters
- * from and to PDC's boot path.
- */
-
-/* Define EARLY_BOOTUP_DEBUG to debug kernel related boot problems.
- * On production kernels EARLY_BOOTUP_DEBUG should be undefined. */
-#define EARLY_BOOTUP_DEBUG
-
-
-#include <linux/kernel.h>
#include <linux/console.h>
-#include <linux/string.h>
#include <linux/init.h>
-#include <linux/major.h>
-#include <linux/tty.h>
+#include <linux/serial_core.h>
+#include <linux/kgdb.h>
#include <asm/page.h> /* for PAGE0 */
#include <asm/pdc.h> /* for iodc_call() proto and friends */
static DEFINE_SPINLOCK(pdc_console_lock);
-static struct console pdc_cons;
static void pdc_console_write(struct console *co, const char *s, unsigned count)
{
spin_unlock_irqrestore(&pdc_console_lock, flags);
}
-int pdc_console_poll_key(struct console *co)
+#ifdef CONFIG_KGDB
+static int kgdb_pdc_read_char(void)
{
int c;
unsigned long flags;
c = pdc_iodc_getc();
spin_unlock_irqrestore(&pdc_console_lock, flags);
- return c;
-}
-
-static int pdc_console_setup(struct console *co, char *options)
-{
- return 0;
-}
-
-#if defined(CONFIG_PDC_CONSOLE)
-#include <linux/vt_kern.h>
-#include <linux/tty_flip.h>
-
-#define PDC_CONS_POLL_DELAY (30 * HZ / 1000)
-
-static void pdc_console_poll(struct timer_list *unused);
-static DEFINE_TIMER(pdc_console_timer, pdc_console_poll);
-static struct tty_port tty_port;
-
-static int pdc_console_tty_open(struct tty_struct *tty, struct file *filp)
-{
- tty_port_tty_set(&tty_port, tty);
- mod_timer(&pdc_console_timer, jiffies + PDC_CONS_POLL_DELAY);
-
- return 0;
+ return (c <= 0) ? NO_POLL_CHAR : c;
}
-static void pdc_console_tty_close(struct tty_struct *tty, struct file *filp)
+static void kgdb_pdc_write_char(u8 chr)
{
- if (tty->count == 1) {
- del_timer_sync(&pdc_console_timer);
- tty_port_tty_set(&tty_port, NULL);
- }
+ if (PAGE0->mem_cons.cl_class != CL_DUPLEX)
+ pdc_console_write(NULL, &chr, 1);
}
-static int pdc_console_tty_write(struct tty_struct *tty, const unsigned char *buf, int count)
-{
- pdc_console_write(NULL, buf, count);
- return count;
-}
-
-static unsigned int pdc_console_tty_write_room(struct tty_struct *tty)
-{
- return 32768; /* no limit, no buffer used */
-}
-
-static const struct tty_operations pdc_console_tty_ops = {
- .open = pdc_console_tty_open,
- .close = pdc_console_tty_close,
- .write = pdc_console_tty_write,
- .write_room = pdc_console_tty_write_room,
+static struct kgdb_io kgdb_pdc_io_ops = {
+ .name = "kgdb_pdc",
+ .read_char = kgdb_pdc_read_char,
+ .write_char = kgdb_pdc_write_char,
};
-
-static void pdc_console_poll(struct timer_list *unused)
-{
- int data, count = 0;
-
- while (1) {
- data = pdc_console_poll_key(NULL);
- if (data == -1)
- break;
- tty_insert_flip_char(&tty_port, data & 0xFF, TTY_NORMAL);
- count ++;
- }
-
- if (count)
- tty_flip_buffer_push(&tty_port);
-
- if (pdc_cons.flags & CON_ENABLED)
- mod_timer(&pdc_console_timer, jiffies + PDC_CONS_POLL_DELAY);
-}
-
-static struct tty_driver *pdc_console_tty_driver;
-
-static int __init pdc_console_tty_driver_init(void)
-{
- struct tty_driver *driver;
- int err;
-
- /* Check if the console driver is still registered.
- * It is unregistered if the pdc console was not selected as the
- * primary console. */
-
- struct console *tmp;
-
- console_lock();
- for_each_console(tmp)
- if (tmp == &pdc_cons)
- break;
- console_unlock();
-
- if (!tmp) {
- printk(KERN_INFO "PDC console driver not registered anymore, not creating %s\n", pdc_cons.name);
- return -ENODEV;
- }
-
- printk(KERN_INFO "The PDC console driver is still registered, removing CON_BOOT flag\n");
- pdc_cons.flags &= ~CON_BOOT;
-
- driver = tty_alloc_driver(1, TTY_DRIVER_REAL_RAW |
- TTY_DRIVER_RESET_TERMIOS);
- if (IS_ERR(driver))
- return PTR_ERR(driver);
-
- tty_port_init(&tty_port);
-
- driver->driver_name = "pdc_cons";
- driver->name = "ttyB";
- driver->major = MUX_MAJOR;
- driver->minor_start = 0;
- driver->type = TTY_DRIVER_TYPE_SYSTEM;
- driver->init_termios = tty_std_termios;
- tty_set_operations(driver, &pdc_console_tty_ops);
- tty_port_link_device(&tty_port, driver, 0);
-
- err = tty_register_driver(driver);
- if (err) {
- printk(KERN_ERR "Unable to register the PDC console TTY driver\n");
- tty_port_destroy(&tty_port);
- tty_driver_kref_put(driver);
- return err;
- }
-
- pdc_console_tty_driver = driver;
-
- return 0;
-}
-device_initcall(pdc_console_tty_driver_init);
-
-static struct tty_driver * pdc_console_device (struct console *c, int *index)
-{
- *index = c->index;
- return pdc_console_tty_driver;
-}
-#else
-#define pdc_console_device NULL
#endif
-static struct console pdc_cons = {
- .name = "ttyB",
- .write = pdc_console_write,
- .device = pdc_console_device,
- .setup = pdc_console_setup,
- .flags = CON_BOOT | CON_PRINTBUFFER,
- .index = -1,
-};
-
-static int pdc_console_initialized;
-
-static void pdc_console_init_force(void)
+static int __init pdc_earlycon_setup(struct earlycon_device *device,
+ const char *opt)
{
- if (pdc_console_initialized)
- return;
- ++pdc_console_initialized;
-
+ struct console *earlycon_console;
+
/* If the console is duplex then copy the COUT parameters to CIN. */
if (PAGE0->mem_cons.cl_class == CL_DUPLEX)
memcpy(&PAGE0->mem_kbd, &PAGE0->mem_cons, sizeof(PAGE0->mem_cons));
- /* register the pdc console */
- register_console(&pdc_cons);
-}
+ earlycon_console = device->con;
+ earlycon_console->write = pdc_console_write;
+ device->port.iotype = UPIO_MEM32BE;
-void __init pdc_console_init(void)
-{
-#if defined(EARLY_BOOTUP_DEBUG) || defined(CONFIG_PDC_CONSOLE)
- pdc_console_init_force();
+#ifdef CONFIG_KGDB
+ kgdb_register_io_module(&kgdb_pdc_io_ops);
#endif
-#ifdef EARLY_BOOTUP_DEBUG
- printk(KERN_INFO "Initialized PDC Console for debugging.\n");
-#endif
-}
-
-
-/*
- * Used for emergencies. Currently only used if an HPMC occurs. If an
- * HPMC occurs, it is possible that the current console may not be
- * properly initialised after the PDC IO reset. This routine unregisters
- * all of the current consoles, reinitializes the pdc console and
- * registers it.
- */
-
-void pdc_console_restart(void)
-{
- struct console *console;
-
- if (pdc_console_initialized)
- return;
- /* If we've already seen the output, don't bother to print it again */
- if (console_drivers != NULL)
- pdc_cons.flags &= ~CON_PRINTBUFFER;
-
- while ((console = console_drivers) != NULL)
- unregister_console(console_drivers);
-
- /* force registering the pdc console */
- pdc_console_init_force();
+ return 0;
}
+
+EARLYCON_DECLARE(pdc, pdc_earlycon_setup);
*/
}
-void release_thread(struct task_struct *dead_task)
-{
-}
-
/*
* Idle thread support
*
static inline unsigned long brk_rnd(void)
{
- return (get_random_int() & BRK_RND_MASK) << PAGE_SHIFT;
+ return (get_random_u32() & BRK_RND_MASK) << PAGE_SHIFT;
}
unsigned long arch_randomize_brk(struct mm_struct *mm)
strlcat(p, "tty0", COMMAND_LINE_SIZE);
}
+ /* default to use early console */
+ if (!strstr(p, "earlycon"))
+ strlcat(p, " earlycon=pdc", COMMAND_LINE_SIZE);
+
#ifdef CONFIG_BLK_DEV_INITRD
if (boot_args[2] != 0) /* did palo pass us a ramdisk? */
{
if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE)
panic("KERNEL_INITIAL_ORDER too small!");
- pdc_console_init();
-
#ifdef CONFIG_64BIT
if(parisc_narrow_firmware) {
printk(KERN_INFO "Kernel is using PDC in 32-bit mode.\n");
unsigned long rnd = 0;
if (current->flags & PF_RANDOMIZE)
- rnd = get_random_int() & MMAP_RND_MASK;
+ rnd = get_random_u32() & MMAP_RND_MASK;
return rnd << PAGE_SHIFT;
}
unsigned long arch_mmap_rnd(void)
{
- return (get_random_int() & MMAP_RND_MASK) << PAGE_SHIFT;
+ return (get_random_u32() & MMAP_RND_MASK) << PAGE_SHIFT;
}
static unsigned long mmap_legacy_base(void)
/* unlock the pdc lock if necessary */
pdc_emergency_unlock();
- /* maybe the kernel hasn't booted very far yet and hasn't been able
- * to initialize the serial or STI console. In that case we should
- * re-enable the pdc console, so that the user will be able to
- * identify the problem. */
- if (!console_drivers)
- pdc_console_restart();
-
if (err)
printk(KERN_CRIT "%s (pid %d): %s (code %ld)\n",
current->comm, task_pid_nr(current), str, err);
/* unlock the pdc lock if necessary */
pdc_emergency_unlock();
- /* restart pdc console if necessary */
- if (!console_drivers)
- pdc_console_restart();
-
/* Not all paths will gutter the processor... */
switch(code){
unsigned long fault_space = 0;
int si_code;
- if (code == 1)
- pdc_console_restart(); /* switch back to pdc if HPMC */
- else if (!irqs_disabled_flags(regs->gr[0]))
+ if (!irqs_disabled_flags(regs->gr[0]))
local_irq_enable();
/* Security check:
map_base = mm->mmap_base;
if (current->flags & PF_RANDOMIZE)
- map_base -= (get_random_int() & 0x1f) * PAGE_SIZE;
+ map_base -= prandom_u32_max(0x20) * PAGE_SIZE;
vdso_text_start = get_unmapped_area(NULL, map_base, vdso_text_len, 0, 0);
in that case. If PIN_TLB is selected, it must be aligned to 8M as
8M pages will be pinned.
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
range 8 9 if PPC64 && PPC_64K_PAGES
default "9" if PPC64 && PPC_64K_PAGES
# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
CONFIG_BINFMT_MISC=m
CONFIG_MATH_EMULATION=y
-CONFIG_FORCE_MAX_ZONEORDER=17
+CONFIG_ARCH_FORCE_MAX_ORDER=17
CONFIG_PCI=y
CONFIG_PCIEPORTBUS=y
CONFIG_PCI_MSI=y
CONFIG_FONT_8x16=y
CONFIG_FONT_8x8=y
CONFIG_FONTS=y
-CONFIG_FORCE_MAX_ZONEORDER=13
+CONFIG_ARCH_FORCE_MAX_ORDER=13
CONFIG_FRAMEBUFFER_CONSOLE=y
CONFIG_FRAME_WARN=1024
CONFIG_FTL=y
CONFIG_NUMA_BALANCING=y
CONFIG_CGROUPS=y
CONFIG_MEMCG=y
-CONFIG_MEMCG_SWAP=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CPUSETS=y
CONFIG_NUMA_BALANCING=y
CONFIG_CGROUPS=y
CONFIG_MEMCG=y
-CONFIG_MEMCG_SWAP=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CPUSETS=y
if (len <= offset)
continue;
- prandom_bytes(data, len);
+ get_random_bytes(data, len);
len -= offset;
crypto_shash_update(crct10dif_shash, data+offset, len);
struct task_struct;
void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp);
-void release_thread(struct task_struct *);
#define TS_FPR(i) fp_state.fpr[i][TS_FPROFFSET]
#define TS_CKFPR(i) ckfp_state.fpr[i][TS_FPROFFSET]
* responsible for combining parameter pairs.
*/
+#ifdef CONFIG_PPC32
+long sys_ppc_pread64(unsigned int fd,
+ char __user *ubuf, compat_size_t count,
+ u32 reg6, u32 pos1, u32 pos2);
+long sys_ppc_pwrite64(unsigned int fd,
+ const char __user *ubuf, compat_size_t count,
+ u32 reg6, u32 pos1, u32 pos2);
+long sys_ppc_readahead(int fd, u32 r4,
+ u32 offset1, u32 offset2, u32 count);
+long sys_ppc_truncate64(const char __user *path, u32 reg4,
+ unsigned long len1, unsigned long len2);
+long sys_ppc_ftruncate64(unsigned int fd, u32 reg4,
+ unsigned long len1, unsigned long len2);
+long sys_ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2,
+ size_t len, int advice);
+#endif
#ifdef CONFIG_COMPAT
long compat_sys_mmap2(unsigned long addr, size_t len,
unsigned long prot, unsigned long flags,
obj-y += ptrace/
obj-$(CONFIG_PPC64) += setup_64.o irq_64.o\
paca.o nvram_64.o note.o
+obj-$(CONFIG_PPC32) += sys_ppc32.o
obj-$(CONFIG_COMPAT) += sys_ppc32.o signal_32.o
obj-$(CONFIG_VDSO32) += vdso32_wrapper.o
obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o
obj-$(CONFIG_40x) += head_40x.o
obj-$(CONFIG_44x) += head_44x.o
obj-$(CONFIG_PPC_8xx) += head_8xx.o
-obj-$(CONFIG_FSL_BOOKE) += head_85xx.o
+obj-$(CONFIG_PPC_85xx) += head_85xx.o
extra-y += vmlinux.lds
obj-$(CONFIG_RELOCATABLE) += reloc_$(BITS).o
beq .Lfast_kernel_interrupt_return_\srr\() // EE already disabled
lbz r11,PACAIRQHAPPENED(r13)
andi. r10,r11,PACA_IRQ_MUST_HARD_MASK
- beq 1f // No HARD_MASK pending
+ beq .Lfast_kernel_interrupt_return_\srr\() // No HARD_MASK pending
/* Must clear MSR_EE from _MSR */
#ifdef CONFIG_PPC_BOOK3S
b .Lfast_kernel_interrupt_return_\srr\()
.Linterrupt_return_\srr\()_soft_enabled:
+ /*
+ * In the soft-enabled case, need to double-check that we have no
+ * pending interrupts that might have come in before we reached the
+ * restart section of code, and restart the exit so those can be
+ * handled.
+ *
+ * If there are none, it is be possible that the interrupt still
+ * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the
+ * interrupted context. This clear will not clobber a new pending
+ * interrupt coming in, because we're in the restart section, so
+ * such would return to the restart location.
+ */
#ifdef CONFIG_PPC_BOOK3S
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- interrupt_return_\srr\()_kernel_restart
#endif
-1:
li r11,0
stb r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS
#endif /* CONFIG_PPC64 */
-void
-release_thread(struct task_struct *t)
-{
-}
-
/*
* this gets called so that we can store coprocessor state into memory and
* copy the current task into the new thread.
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
+ sp -= prandom_u32_max(PAGE_SIZE);
return sp & ~0xf;
}
// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * sys_ppc32.c: Conversion between 32bit and 64bit native syscalls.
+ * sys_ppc32.c: 32-bit system calls with complex calling conventions.
*
* Copyright (C) 2001 IBM
* Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
* Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
*
- * These routines maintain argument size conversion between 32bit and 64bit
- * environment.
+ * 32-bit system calls with 64-bit arguments pass those in register pairs.
+ * This must be specially dealt with on 64-bit kernels. The compat_arg_u64_dual
+ * in generic compat syscalls is not always usable because the register
+ * pairing is constrained depending on preceding arguments.
+ *
+ * An analogous problem exists on 32-bit kernels with ARCH_HAS_SYSCALL_WRAPPER,
+ * the defined system call functions take the pt_regs as an argument, and there
+ * is a mapping macro which maps registers to arguments
+ * (SC_POWERPC_REGS_TO_ARGS) which also does not deal with these 64-bit
+ * arguments.
+ *
+ * This file contains these system calls.
*/
#include <linux/kernel.h>
#include <asm/syscalls.h>
#include <asm/switch_to.h>
-COMPAT_SYSCALL_DEFINE6(ppc_pread64,
+#ifdef CONFIG_PPC32
+#define PPC32_SYSCALL_DEFINE4 SYSCALL_DEFINE4
+#define PPC32_SYSCALL_DEFINE5 SYSCALL_DEFINE5
+#define PPC32_SYSCALL_DEFINE6 SYSCALL_DEFINE6
+#else
+#define PPC32_SYSCALL_DEFINE4 COMPAT_SYSCALL_DEFINE4
+#define PPC32_SYSCALL_DEFINE5 COMPAT_SYSCALL_DEFINE5
+#define PPC32_SYSCALL_DEFINE6 COMPAT_SYSCALL_DEFINE6
+#endif
+
+PPC32_SYSCALL_DEFINE6(ppc_pread64,
unsigned int, fd,
char __user *, ubuf, compat_size_t, count,
u32, reg6, u32, pos1, u32, pos2)
return ksys_pread64(fd, ubuf, count, merge_64(pos1, pos2));
}
-COMPAT_SYSCALL_DEFINE6(ppc_pwrite64,
+PPC32_SYSCALL_DEFINE6(ppc_pwrite64,
unsigned int, fd,
const char __user *, ubuf, compat_size_t, count,
u32, reg6, u32, pos1, u32, pos2)
return ksys_pwrite64(fd, ubuf, count, merge_64(pos1, pos2));
}
-COMPAT_SYSCALL_DEFINE5(ppc_readahead,
+PPC32_SYSCALL_DEFINE5(ppc_readahead,
int, fd, u32, r4,
u32, offset1, u32, offset2, u32, count)
{
return ksys_readahead(fd, merge_64(offset1, offset2), count);
}
-COMPAT_SYSCALL_DEFINE4(ppc_truncate64,
+PPC32_SYSCALL_DEFINE4(ppc_truncate64,
const char __user *, path, u32, reg4,
unsigned long, len1, unsigned long, len2)
{
return ksys_truncate(path, merge_64(len1, len2));
}
-COMPAT_SYSCALL_DEFINE4(ppc_ftruncate64,
+PPC32_SYSCALL_DEFINE4(ppc_ftruncate64,
unsigned int, fd, u32, reg4,
unsigned long, len1, unsigned long, len2)
{
return ksys_ftruncate(fd, merge_64(len1, len2));
}
-COMPAT_SYSCALL_DEFINE6(ppc32_fadvise64,
+PPC32_SYSCALL_DEFINE6(ppc32_fadvise64,
int, fd, u32, unused, u32, offset1, u32, offset2,
size_t, len, int, advice)
{
176 64 rt_sigtimedwait sys_rt_sigtimedwait
177 nospu rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo
178 nospu rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend
-179 common pread64 sys_pread64 compat_sys_ppc_pread64
-180 common pwrite64 sys_pwrite64 compat_sys_ppc_pwrite64
+179 32 pread64 sys_ppc_pread64 compat_sys_ppc_pread64
+179 64 pread64 sys_pread64
+180 32 pwrite64 sys_ppc_pwrite64 compat_sys_ppc_pwrite64
+180 64 pwrite64 sys_pwrite64
181 common chown sys_chown
182 common getcwd sys_getcwd
183 common capget sys_capget
188 common putpmsg sys_ni_syscall
189 nospu vfork sys_vfork
190 common ugetrlimit sys_getrlimit compat_sys_getrlimit
-191 common readahead sys_readahead compat_sys_ppc_readahead
+191 32 readahead sys_ppc_readahead compat_sys_ppc_readahead
+191 64 readahead sys_readahead
192 32 mmap2 sys_mmap2 compat_sys_mmap2
-193 32 truncate64 sys_truncate64 compat_sys_ppc_truncate64
-194 32 ftruncate64 sys_ftruncate64 compat_sys_ppc_ftruncate64
+193 32 truncate64 sys_ppc_truncate64 compat_sys_ppc_truncate64
+194 32 ftruncate64 sys_ppc_ftruncate64 compat_sys_ppc_ftruncate64
195 32 stat64 sys_stat64
196 32 lstat64 sys_lstat64
197 32 fstat64 sys_fstat64
230 common io_submit sys_io_submit compat_sys_io_submit
231 common io_cancel sys_io_cancel
232 nospu set_tid_address sys_set_tid_address
-233 common fadvise64 sys_fadvise64 compat_sys_ppc32_fadvise64
+233 32 fadvise64 sys_ppc32_fadvise64 compat_sys_ppc32_fadvise64
+233 64 fadvise64 sys_fadvise64
234 nospu exit_group sys_exit_group
235 nospu lookup_dcookie sys_lookup_dcookie compat_sys_lookup_dcookie
236 common epoll_create sys_epoll_create
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
{
struct mm_struct *mm = task->mm;
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
mmap_read_lock(mm);
-
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
unsigned long size = vma->vm_end - vma->vm_start;
if (vma_is_special_mapping(vma, &vvar_spec))
zap_page_range(vma, vma->vm_start, size);
}
-
mmap_read_unlock(mm);
+
return 0;
}
static int __kvmppc_svm_page_out(struct vm_area_struct *vma,
unsigned long start,
unsigned long end, unsigned long page_shift,
- struct kvm *kvm, unsigned long gpa)
+ struct kvm *kvm, unsigned long gpa, struct page *fault_page)
{
unsigned long src_pfn, dst_pfn = 0;
- struct migrate_vma mig;
+ struct migrate_vma mig = { 0 };
struct page *dpage, *spage;
struct kvmppc_uvmem_page_pvt *pvt;
unsigned long pfn;
mig.dst = &dst_pfn;
mig.pgmap_owner = &kvmppc_uvmem_pgmap;
mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
+ mig.fault_page = fault_page;
/* The requested page is already paged-out, nothing to do */
if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL))
static inline int kvmppc_svm_page_out(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
unsigned long page_shift,
- struct kvm *kvm, unsigned long gpa)
+ struct kvm *kvm, unsigned long gpa,
+ struct page *fault_page)
{
int ret;
mutex_lock(&kvm->arch.uvmem_lock);
- ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa);
+ ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa,
+ fault_page);
mutex_unlock(&kvm->arch.uvmem_lock);
return ret;
pvt->remove_gfn = true;
if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE,
- PAGE_SHIFT, kvm, pvt->gpa))
+ PAGE_SHIFT, kvm, pvt->gpa, NULL))
pr_err("Can't page out gpa:0x%lx addr:0x%lx\n",
pvt->gpa, addr);
} else {
dpage = pfn_to_page(uvmem_pfn);
dpage->zone_device_data = pvt;
- lock_page(dpage);
+ zone_device_page_init(dpage);
return dpage;
out_clear:
spin_lock(&kvmppc_uvmem_bitmap_lock);
bool pagein)
{
unsigned long src_pfn, dst_pfn = 0;
- struct migrate_vma mig;
+ struct migrate_vma mig = { 0 };
struct page *spage;
unsigned long pfn;
struct page *dpage;
if (kvmppc_svm_page_out(vmf->vma, vmf->address,
vmf->address + PAGE_SIZE, PAGE_SHIFT,
- pvt->kvm, pvt->gpa))
+ pvt->kvm, pvt->gpa, vmf->page))
return VM_FAULT_SIGBUS;
else
return 0;
if (!vma || vma->vm_start > start || vma->vm_end < end)
goto out;
- if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa))
+ if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, NULL))
ret = H_SUCCESS;
out:
mmap_read_unlock(kvm->mm);
void hash__flush_tlb_mm(struct mm_struct *mm)
{
struct vm_area_struct *mp;
+ VMA_ITERATOR(vmi, mm, 0);
/*
- * It is safe to go down the mm's list of vmas when called
- * from dup_mmap, holding mmap_lock. It would also be safe from
- * unmap_region or exit_mmap, but not from vmtruncate on SMP -
- * but it seems dup_mmap is the only SMP case which gets here.
+ * It is safe to iterate the vmas when called from dup_mmap,
+ * holding mmap_lock. It would also be safe from unmap_region
+ * or exit_mmap, but not from vmtruncate on SMP - but it seems
+ * dup_mmap is the only SMP case which gets here.
*/
- for (mp = mm->mmap; mp != NULL; mp = mp->vm_next)
+ for_each_vma(vmi, mp)
hash__flush_range(mp->vm_mm, mp->vm_start, mp->vm_end);
}
EXPORT_SYMBOL(hash__flush_tlb_mm);
unsigned long len)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, addr);
/*
* We don't try too hard, we just mark all the vma in that range
* VM_NOHUGEPAGE and split them.
*/
- vma = find_vma(mm, addr);
- /*
- * If the range is in unmapped range, just return
- */
- if (vma && ((addr + len) <= vma->vm_start))
- return;
-
- while (vma) {
- if (vma->vm_start >= (addr + len))
- break;
+ for_each_vma_range(vmi, vma, addr + len) {
vma->vm_flags |= VM_NOHUGEPAGE;
walk_page_vma(vma, &subpage_walk_ops, NULL);
- vma = vma->vm_next;
}
}
#else
setup.o iommu.o event_sources.o ras.o \
firmware.o power.o dlpar.o mobility.o rng.o \
pci.o pci_dlpar.o eeh_pseries.o msi.o \
- papr_platform_attributes.o
+ papr_platform_attributes.o dtl.o
obj-$(CONFIG_SMP) += smp.o
obj-$(CONFIG_KEXEC_CORE) += kexec.o
obj-$(CONFIG_PSERIES_ENERGY) += pseries_energy.o
obj-$(CONFIG_HVCS) += hvcserver.o
obj-$(CONFIG_HCALL_STATS) += hvCall_inst.o
obj-$(CONFIG_CMM) += cmm.o
-obj-$(CONFIG_DTL) += dtl.o
obj-$(CONFIG_IO_EVENT_IRQ) += io_event_irq.o
obj-$(CONFIG_LPARCFG) += lparcfg.o
obj-$(CONFIG_IBMVIO) += vio.o
#include <asm/plpar_wrappers.h>
#include <asm/machdep.h>
+#ifdef CONFIG_DTL
struct dtl {
struct dtl_entry *buf;
int cpu;
static atomic_t dtl_count;
-/*
- * Scan the dispatch trace log and count up the stolen time.
- * Should be called with interrupts disabled.
- */
-static notrace u64 scan_dispatch_log(u64 stop_tb)
-{
- u64 i = local_paca->dtl_ridx;
- struct dtl_entry *dtl = local_paca->dtl_curr;
- struct dtl_entry *dtl_end = local_paca->dispatch_log_end;
- struct lppaca *vpa = local_paca->lppaca_ptr;
- u64 tb_delta;
- u64 stolen = 0;
- u64 dtb;
-
- if (!dtl)
- return 0;
-
- if (i == be64_to_cpu(vpa->dtl_idx))
- return 0;
- while (i < be64_to_cpu(vpa->dtl_idx)) {
- dtb = be64_to_cpu(dtl->timebase);
- tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) +
- be32_to_cpu(dtl->ready_to_enqueue_time);
- barrier();
- if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {
- /* buffer has overflowed */
- i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;
- dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);
- continue;
- }
- if (dtb > stop_tb)
- break;
- if (dtl_consumer)
- dtl_consumer(dtl, i);
- stolen += tb_delta;
- ++i;
- ++dtl;
- if (dtl == dtl_end)
- dtl = local_paca->dispatch_log;
- }
- local_paca->dtl_ridx = i;
- local_paca->dtl_curr = dtl;
- return stolen;
-}
-
-/*
- * Accumulate stolen time by scanning the dispatch trace log.
- * Called on entry from user mode.
- */
-void notrace pseries_accumulate_stolen_time(void)
-{
- u64 sst, ust;
- struct cpu_accounting_data *acct = &local_paca->accounting;
-
- sst = scan_dispatch_log(acct->starttime_user);
- ust = scan_dispatch_log(acct->starttime);
- acct->stime -= sst;
- acct->utime -= ust;
- acct->steal_time += ust + sst;
-}
-
-u64 pseries_calculate_stolen_time(u64 stop_tb)
-{
- if (!firmware_has_feature(FW_FEATURE_SPLPAR))
- return 0;
-
- if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx))
- return scan_dispatch_log(stop_tb);
-
- return 0;
-}
-
/*
* The cpu accounting code controls the DTL ring buffer, and we get
* given entries as they are processed.
return 0;
}
machine_arch_initcall(pseries, dtl_init);
+#endif /* CONFIG_DTL */
+
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
+/*
+ * Scan the dispatch trace log and count up the stolen time.
+ * Should be called with interrupts disabled.
+ */
+static notrace u64 scan_dispatch_log(u64 stop_tb)
+{
+ u64 i = local_paca->dtl_ridx;
+ struct dtl_entry *dtl = local_paca->dtl_curr;
+ struct dtl_entry *dtl_end = local_paca->dispatch_log_end;
+ struct lppaca *vpa = local_paca->lppaca_ptr;
+ u64 tb_delta;
+ u64 stolen = 0;
+ u64 dtb;
+
+ if (!dtl)
+ return 0;
+
+ if (i == be64_to_cpu(vpa->dtl_idx))
+ return 0;
+ while (i < be64_to_cpu(vpa->dtl_idx)) {
+ dtb = be64_to_cpu(dtl->timebase);
+ tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) +
+ be32_to_cpu(dtl->ready_to_enqueue_time);
+ barrier();
+ if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {
+ /* buffer has overflowed */
+ i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;
+ dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);
+ continue;
+ }
+ if (dtb > stop_tb)
+ break;
+#ifdef CONFIG_DTL
+ if (dtl_consumer)
+ dtl_consumer(dtl, i);
+#endif
+ stolen += tb_delta;
+ ++i;
+ ++dtl;
+ if (dtl == dtl_end)
+ dtl = local_paca->dispatch_log;
+ }
+ local_paca->dtl_ridx = i;
+ local_paca->dtl_curr = dtl;
+ return stolen;
+}
+
+/*
+ * Accumulate stolen time by scanning the dispatch trace log.
+ * Called on entry from user mode.
+ */
+void notrace pseries_accumulate_stolen_time(void)
+{
+ u64 sst, ust;
+ struct cpu_accounting_data *acct = &local_paca->accounting;
+
+ sst = scan_dispatch_log(acct->starttime_user);
+ ust = scan_dispatch_log(acct->starttime);
+ acct->stime -= sst;
+ acct->utime -= ust;
+ acct->steal_time += ust + sst;
+}
+
+u64 pseries_calculate_stolen_time(u64 stop_tb)
+{
+ if (!firmware_has_feature(FW_FEATURE_SPLPAR))
+ return 0;
+
+ if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx))
+ return scan_dispatch_log(stop_tb);
+
+ return 0;
+}
+
+#endif
select GENERIC_SMP_IDLE_THREAD
select GENERIC_TIME_VSYSCALL if MMU && 64BIT
select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO
+ select HARDIRQS_SW_RESEND
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL
select HAVE_ARCH_JUMP_LABEL_RELATIVE if !XIP_KERNEL
select HAVE_PERF_EVENTS
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
+ select HAVE_POSIX_CPU_TIMERS_TASK_WORK
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_FUNCTION_ARG_ACCESS_API
select HAVE_STACKPROTECTOR
select ARCH_HAS_SETUP_DMA_OPS
select DMA_DIRECT_REMAP
+config AS_HAS_INSN
+ def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero)
+
source "arch/riscv/Kconfig.socs"
source "arch/riscv/Kconfig.erratas"
endif
ifeq ($(CONFIG_LD_IS_LLD),y)
+ifeq ($(shell test $(CONFIG_LLD_VERSION) -lt 150000; echo $$?),0)
KBUILD_CFLAGS += -mno-relax
KBUILD_AFLAGS += -mno-relax
ifndef CONFIG_AS_IS_LLVM
KBUILD_AFLAGS += -Wa,-mno-relax
endif
endif
+endif
# ISA string setting
riscv-march-$(CONFIG_ARCH_RV32I) := rv32ima
# SPDX-License-Identifier: GPL-2.0
dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-icicle-kit.dtb
+dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb
dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb
+dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-sev-kit.dtb
obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y))
/* Copyright (c) 2020-2021 Microchip Technology Inc */
/ {
- compatible = "microchip,mpfs-icicle-reference-rtlv2203", "microchip,mpfs";
+ compatible = "microchip,mpfs-icicle-reference-rtlv2210", "microchip,mpfs-icicle-kit",
+ "microchip,mpfs";
- core_pwm0: pwm@41000000 {
+ core_pwm0: pwm@40000000 {
compatible = "microchip,corepwm-rtl-v4";
- reg = <0x0 0x41000000 0x0 0xF0>;
+ reg = <0x0 0x40000000 0x0 0xF0>;
microchip,sync-update-mask = /bits/ 32 <0>;
#pwm-cells = <2>;
clocks = <&fabric_clk3>;
status = "disabled";
};
- i2c2: i2c@44000000 {
+ i2c2: i2c@40000200 {
compatible = "microchip,corei2c-rtl-v7";
- reg = <0x0 0x44000000 0x0 0x1000>;
+ reg = <0x0 0x40000200 0x0 0x100>;
#address-cells = <1>;
#size-cells = <0>;
clocks = <&fabric_clk3>;
fabric_clk3: fabric-clk3 {
compatible = "fixed-clock";
#clock-cells = <0>;
- clock-frequency = <62500000>;
+ clock-frequency = <50000000>;
};
fabric_clk1: fabric-clk1 {
#clock-cells = <0>;
clock-frequency = <125000000>;
};
+
+ pcie: pcie@3000000000 {
+ compatible = "microchip,pcie-host-1.0";
+ #address-cells = <0x3>;
+ #interrupt-cells = <0x1>;
+ #size-cells = <0x2>;
+ device_type = "pci";
+ reg = <0x30 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
+ reg-names = "cfg", "apb";
+ bus-range = <0x0 0x7f>;
+ interrupt-parent = <&plic>;
+ interrupts = <119>;
+ interrupt-map = <0 0 0 1 &pcie_intc 0>,
+ <0 0 0 2 &pcie_intc 1>,
+ <0 0 0 3 &pcie_intc 2>,
+ <0 0 0 4 &pcie_intc 3>;
+ interrupt-map-mask = <0 0 0 7>;
+ clocks = <&fabric_clk1>, <&fabric_clk3>;
+ clock-names = "fic1", "fic3";
+ ranges = <0x3000000 0x0 0x8000000 0x30 0x8000000 0x0 0x80000000>;
+ dma-ranges = <0x02000000 0x0 0x00000000 0x0 0x00000000 0x1 0x00000000>;
+ msi-parent = <&pcie>;
+ msi-controller;
+ status = "disabled";
+ pcie_intc: interrupt-controller {
+ #address-cells = <0>;
+ #interrupt-cells = <1>;
+ interrupt-controller;
+ };
+ };
};
/ {
model = "Microchip PolarFire-SoC Icicle Kit";
- compatible = "microchip,mpfs-icicle-kit", "microchip,mpfs";
+ compatible = "microchip,mpfs-icicle-reference-rtlv2210", "microchip,mpfs-icicle-kit",
+ "microchip,mpfs";
aliases {
ethernet0 = &mac1;
ddrc_cache_lo: memory@80000000 {
device_type = "memory";
- reg = <0x0 0x80000000 0x0 0x2e000000>;
+ reg = <0x0 0x80000000 0x0 0x40000000>;
status = "okay";
};
ddrc_cache_hi: memory@1000000000 {
device_type = "memory";
- reg = <0x10 0x0 0x0 0x40000000>;
+ reg = <0x10 0x40000000 0x0 0x40000000>;
status = "okay";
};
+
+ reserved-memory {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ hss_payload: region@BFC00000 {
+ reg = <0x0 0xBFC00000 0x0 0x400000>;
+ no-map;
+ };
+ };
};
&core_pwm0 {
--- /dev/null
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2022 Microchip Technology Inc */
+
+/ {
+ fabric_clk3: fabric-clk3 {
+ compatible = "fixed-clock";
+ #clock-cells = <0>;
+ clock-frequency = <62500000>;
+ };
+
+ fabric_clk1: fabric-clk1 {
+ compatible = "fixed-clock";
+ #clock-cells = <0>;
+ clock-frequency = <125000000>;
+ };
+
+ pcie: pcie@2000000000 {
+ compatible = "microchip,pcie-host-1.0";
+ #address-cells = <0x3>;
+ #interrupt-cells = <0x1>;
+ #size-cells = <0x2>;
+ device_type = "pci";
+ reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
+ reg-names = "cfg", "apb";
+ bus-range = <0x0 0x7f>;
+ interrupt-parent = <&plic>;
+ interrupts = <119>;
+ interrupt-map = <0 0 0 1 &pcie_intc 0>,
+ <0 0 0 2 &pcie_intc 1>,
+ <0 0 0 3 &pcie_intc 2>,
+ <0 0 0 4 &pcie_intc 3>;
+ interrupt-map-mask = <0 0 0 7>;
+ clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>;
+ clock-names = "fic0", "fic1", "fic3";
+ ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>;
+ msi-parent = <&pcie>;
+ msi-controller;
+ status = "disabled";
+ pcie_intc: interrupt-controller {
+ #address-cells = <0>;
+ #interrupt-cells = <1>;
+ interrupt-controller;
+ };
+ };
+};
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Original all-in-one devicetree:
+ * Copyright (C) 2021-2022 - Wolfgang Grandegger <wg@aries-embedded.de>
+ * Rewritten to use includes:
+ * Copyright (C) 2022 - Conor Dooley <conor.dooley@microchip.com>
+ */
+/dts-v1/;
+
+#include "mpfs.dtsi"
+#include "mpfs-m100pfs-fabric.dtsi"
+
+/* Clock frequency (in Hz) of the rtcclk */
+#define MTIMER_FREQ 1000000
+
+/ {
+ model = "Aries Embedded M100PFEVPS";
+ compatible = "aries,m100pfsevp", "microchip,mpfs";
+
+ aliases {
+ ethernet0 = &mac0;
+ ethernet1 = &mac1;
+ serial0 = &mmuart0;
+ serial1 = &mmuart1;
+ serial2 = &mmuart2;
+ serial3 = &mmuart3;
+ serial4 = &mmuart4;
+ gpio0 = &gpio0;
+ gpio1 = &gpio2;
+ };
+
+ chosen {
+ stdout-path = "serial1:115200n8";
+ };
+
+ cpus {
+ timebase-frequency = <MTIMER_FREQ>;
+ };
+
+ ddrc_cache_lo: memory@80000000 {
+ device_type = "memory";
+ reg = <0x0 0x80000000 0x0 0x40000000>;
+ };
+ ddrc_cache_hi: memory@1040000000 {
+ device_type = "memory";
+ reg = <0x10 0x40000000 0x0 0x40000000>;
+ };
+};
+
+&can0 {
+ status = "okay";
+};
+
+&i2c0 {
+ status = "okay";
+};
+
+&i2c1 {
+ status = "okay";
+};
+
+&gpio0 {
+ interrupts = <13>, <14>, <15>, <16>,
+ <17>, <18>, <19>, <20>,
+ <21>, <22>, <23>, <24>,
+ <25>, <26>;
+ ngpios = <14>;
+ status = "okay";
+
+ pmic-irq-hog {
+ gpio-hog;
+ gpios = <13 0>;
+ input;
+ };
+
+ /* Set to low for eMMC, high for SD-card */
+ mmc-sel-hog {
+ gpio-hog;
+ gpios = <12 0>;
+ output-high;
+ };
+};
+
+&gpio2 {
+ interrupts = <13>, <14>, <15>, <16>,
+ <17>, <18>, <19>, <20>,
+ <21>, <22>, <23>, <24>,
+ <25>, <26>, <27>, <28>,
+ <29>, <30>, <31>, <32>,
+ <33>, <34>, <35>, <36>,
+ <37>, <38>, <39>, <40>,
+ <41>, <42>, <43>, <44>;
+ status = "okay";
+};
+
+&mac0 {
+ status = "okay";
+ phy-mode = "gmii";
+ phy-handle = <&phy0>;
+ phy0: ethernet-phy@0 {
+ reg = <0>;
+ };
+};
+
+&mac1 {
+ status = "okay";
+ phy-mode = "gmii";
+ phy-handle = <&phy1>;
+ phy1: ethernet-phy@0 {
+ reg = <0>;
+ };
+};
+
+&mbox {
+ status = "okay";
+};
+
+&mmc {
+ max-frequency = <50000000>;
+ bus-width = <4>;
+ cap-mmc-highspeed;
+ cap-sd-highspeed;
+ no-1-8-v;
+ sd-uhs-sdr12;
+ sd-uhs-sdr25;
+ sd-uhs-sdr50;
+ sd-uhs-sdr104;
+ disable-wp;
+ status = "okay";
+};
+
+&mmuart1 {
+ status = "okay";
+};
+
+&mmuart2 {
+ status = "okay";
+};
+
+&mmuart3 {
+ status = "okay";
+};
+
+&mmuart4 {
+ status = "okay";
+};
+
+&pcie {
+ status = "okay";
+};
+
+&qspi {
+ status = "okay";
+};
+
+&refclk {
+ clock-frequency = <125000000>;
+};
+
+&rtc {
+ status = "okay";
+};
+
+&spi0 {
+ status = "okay";
+};
+
+&spi1 {
+ status = "okay";
+};
+
+&syscontroller {
+ status = "okay";
+};
+
+&usb {
+ status = "okay";
+ dr_mode = "host";
+};
#clock-cells = <0>;
clock-frequency = <125000000>;
};
+
+ pcie: pcie@2000000000 {
+ compatible = "microchip,pcie-host-1.0";
+ #address-cells = <0x3>;
+ #interrupt-cells = <0x1>;
+ #size-cells = <0x2>;
+ device_type = "pci";
+ reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
+ reg-names = "cfg", "apb";
+ bus-range = <0x0 0x7f>;
+ interrupt-parent = <&plic>;
+ interrupts = <119>;
+ interrupt-map = <0 0 0 1 &pcie_intc 0>,
+ <0 0 0 2 &pcie_intc 1>,
+ <0 0 0 3 &pcie_intc 2>,
+ <0 0 0 4 &pcie_intc 3>;
+ interrupt-map-mask = <0 0 0 7>;
+ clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>;
+ clock-names = "fic0", "fic1", "fic3";
+ ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>;
+ msi-parent = <&pcie>;
+ msi-controller;
+ status = "disabled";
+ pcie_intc: interrupt-controller {
+ #address-cells = <0>;
+ #interrupt-cells = <1>;
+ interrupt-controller;
+ };
+ };
};
--- /dev/null
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2022 Microchip Technology Inc */
+
+/ {
+ fabric_clk3: fabric-clk3 {
+ compatible = "fixed-clock";
+ #clock-cells = <0>;
+ clock-frequency = <0>;
+ };
+
+ fabric_clk1: fabric-clk1 {
+ compatible = "fixed-clock";
+ #clock-cells = <0>;
+ clock-frequency = <125000000>;
+ };
+
+ pcie: pcie@2000000000 {
+ compatible = "microchip,pcie-host-1.0";
+ #address-cells = <0x3>;
+ #interrupt-cells = <0x1>;
+ #size-cells = <0x2>;
+ device_type = "pci";
+ reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
+ reg-names = "cfg", "apb";
+ bus-range = <0x0 0x7f>;
+ interrupt-parent = <&plic>;
+ interrupts = <119>;
+ interrupt-map = <0 0 0 1 &pcie_intc 0>,
+ <0 0 0 2 &pcie_intc 1>,
+ <0 0 0 3 &pcie_intc 2>,
+ <0 0 0 4 &pcie_intc 3>;
+ interrupt-map-mask = <0 0 0 7>;
+ clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>;
+ clock-names = "fic0", "fic1", "fic3";
+ ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>;
+ msi-parent = <&pcie>;
+ msi-controller;
+ status = "disabled";
+ pcie_intc: interrupt-controller {
+ #address-cells = <0>;
+ #interrupt-cells = <1>;
+ interrupt-controller;
+ };
+ };
+};
--- /dev/null
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2022 Microchip Technology Inc */
+
+/dts-v1/;
+
+#include "mpfs.dtsi"
+#include "mpfs-sev-kit-fabric.dtsi"
+
+/* Clock frequency (in Hz) of the rtcclk */
+#define MTIMER_FREQ 1000000
+
+/ {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ model = "Microchip PolarFire-SoC SEV Kit";
+ compatible = "microchip,mpfs-sev-kit", "microchip,mpfs";
+
+ aliases {
+ ethernet0 = &mac1;
+ serial0 = &mmuart0;
+ serial1 = &mmuart1;
+ serial2 = &mmuart2;
+ serial3 = &mmuart3;
+ serial4 = &mmuart4;
+ };
+
+ chosen {
+ stdout-path = "serial1:115200n8";
+ };
+
+ cpus {
+ timebase-frequency = <MTIMER_FREQ>;
+ };
+
+ reserved-memory {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ ranges;
+
+ fabricbuf0ddrc: buffer@80000000 {
+ compatible = "shared-dma-pool";
+ reg = <0x0 0x80000000 0x0 0x2000000>;
+ };
+
+ fabricbuf1ddrnc: buffer@c4000000 {
+ compatible = "shared-dma-pool";
+ reg = <0x0 0xc4000000 0x0 0x4000000>;
+ };
+
+ fabricbuf2ddrncwcb: buffer@d4000000 {
+ compatible = "shared-dma-pool";
+ reg = <0x0 0xd4000000 0x0 0x4000000>;
+ };
+ };
+
+ ddrc_cache: memory@1000000000 {
+ device_type = "memory";
+ reg = <0x10 0x0 0x0 0x76000000>;
+ };
+};
+
+&i2c0 {
+ status = "okay";
+};
+
+&gpio2 {
+ interrupts = <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>,
+ <53>, <53>, <53>, <53>;
+ status = "okay";
+};
+
+&mac0 {
+ status = "okay";
+ phy-mode = "sgmii";
+ phy-handle = <&phy0>;
+ phy1: ethernet-phy@9 {
+ reg = <9>;
+ };
+ phy0: ethernet-phy@8 {
+ reg = <8>;
+ };
+};
+
+&mac1 {
+ status = "okay";
+ phy-mode = "sgmii";
+ phy-handle = <&phy1>;
+};
+
+&mbox {
+ status = "okay";
+};
+
+&mmc {
+ status = "okay";
+ bus-width = <4>;
+ disable-wp;
+ cap-sd-highspeed;
+ cap-mmc-highspeed;
+ mmc-ddr-1_8v;
+ mmc-hs200-1_8v;
+ sd-uhs-sdr12;
+ sd-uhs-sdr25;
+ sd-uhs-sdr50;
+ sd-uhs-sdr104;
+};
+
+&mmuart1 {
+ status = "okay";
+};
+
+&mmuart2 {
+ status = "okay";
+};
+
+&mmuart3 {
+ status = "okay";
+};
+
+&mmuart4 {
+ status = "okay";
+};
+
+&refclk {
+ clock-frequency = <125000000>;
+};
+
+&rtc {
+ status = "okay";
+};
+
+&syscontroller {
+ status = "okay";
+};
+
+&usb {
+ status = "okay";
+ dr_mode = "otg";
+};
};
qspi: spi@21000000 {
- compatible = "microchip,mpfs-qspi";
+ compatible = "microchip,mpfs-qspi", "microchip,coreqspi-rtl-v2";
#address-cells = <1>;
#size-cells = <0>;
reg = <0x0 0x21000000 0x0 0x1000>;
status = "disabled";
};
- pcie: pcie@2000000000 {
- compatible = "microchip,pcie-host-1.0";
- #address-cells = <0x3>;
- #interrupt-cells = <0x1>;
- #size-cells = <0x2>;
- device_type = "pci";
- reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>;
- reg-names = "cfg", "apb";
- bus-range = <0x0 0x7f>;
- interrupt-parent = <&plic>;
- interrupts = <119>;
- interrupt-map = <0 0 0 1 &pcie_intc 0>,
- <0 0 0 2 &pcie_intc 1>,
- <0 0 0 3 &pcie_intc 2>,
- <0 0 0 4 &pcie_intc 3>;
- interrupt-map-mask = <0 0 0 7>;
- clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>;
- clock-names = "fic0", "fic1", "fic3";
- ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>;
- msi-parent = <&pcie>;
- msi-controller;
- status = "disabled";
- pcie_intc: interrupt-controller {
- #address-cells = <0>;
- #interrupt-cells = <1>;
- interrupt-controller;
- };
- };
-
mbox: mailbox@37020000 {
compatible = "microchip,mpfs-mailbox";
reg = <0x0 0x37020000 0x0 0x1000>, <0x0 0x2000318C 0x0 0x40>;
#include "jh7100.dtsi"
#include <dt-bindings/gpio/gpio.h>
#include <dt-bindings/leds/common.h>
-#include <dt-bindings/pinctrl/pinctrl-starfive.h>
+#include <dt-bindings/pinctrl/pinctrl-starfive-jh7100.h>
/ {
model = "BeagleV Starlight Beta";
static bool errata_probe_pbmt(unsigned int stage,
unsigned long arch_id, unsigned long impid)
{
+ if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PBMT))
+ return false;
+
if (arch_id != 0 || impid != 0)
return false;
static bool errata_probe_cmo(unsigned int stage,
unsigned long arch_id, unsigned long impid)
{
-#ifdef CONFIG_ERRATA_THEAD_CMO
+ if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO))
+ return false;
+
if (arch_id != 0 || impid != 0)
return false;
riscv_cbom_block_size = L1_CACHE_BYTES;
riscv_noncoherent_supported();
return true;
-#else
- return false;
-#endif
}
static u32 thead_errata_probe(unsigned int stage,
u32 cpu_req_errata = 0;
if (errata_probe_pbmt(stage, archid, impid))
- cpu_req_errata |= (1U << ERRATA_THEAD_PBMT);
+ cpu_req_errata |= BIT(ERRATA_THEAD_PBMT);
if (errata_probe_cmo(stage, archid, impid))
- cpu_req_errata |= (1U << ERRATA_THEAD_CMO);
+ cpu_req_errata |= BIT(ERRATA_THEAD_CMO);
return cpu_req_errata;
}
#ifdef CONFIG_RISCV_DMA_NONCOHERENT
void riscv_noncoherent_supported(void);
+#else
+static inline void riscv_noncoherent_supported(void) {}
#endif
/*
get_cache_size(2, CACHE_TYPE_UNIFIED)); \
NEW_AUX_ENT(AT_L2_CACHEGEOMETRY, \
get_cache_geometry(2, CACHE_TYPE_UNIFIED)); \
+ NEW_AUX_ENT(AT_L3_CACHESIZE, \
+ get_cache_size(3, CACHE_TYPE_UNIFIED)); \
+ NEW_AUX_ENT(AT_L3_CACHEGEOMETRY, \
+ get_cache_geometry(3, CACHE_TYPE_UNIFIED)); \
} while (0)
#define ARCH_HAS_SETUP_ADDITIONAL_PAGES
struct linux_binprm;
#define __ASM_GPR_NUM_H
#ifdef __ASSEMBLY__
+
+ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+ .equ .L__gpr_num_x\num, \num
+ .endr
+
.equ .L__gpr_num_zero, 0
.equ .L__gpr_num_ra, 1
.equ .L__gpr_num_sp, 2
#else /* __ASSEMBLY__ */
#define __DEFINE_ASM_GPR_NUMS \
+" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \
+" .equ .L__gpr_num_x\\num, \\num\n" \
+" .endr\n" \
" .equ .L__gpr_num_zero, 0\n" \
" .equ .L__gpr_num_ra, 1\n" \
" .equ .L__gpr_num_sp, 2\n" \
RISCV_ISA_EXT_ZICBOM,
RISCV_ISA_EXT_ZIHINTPAUSE,
RISCV_ISA_EXT_SSTC,
+ RISCV_ISA_EXT_SVINVAL,
RISCV_ISA_EXT_ID_MAX = RISCV_ISA_EXT_MAX,
};
enum riscv_isa_ext_key {
RISCV_ISA_EXT_KEY_FPU, /* For 'F' and 'D' */
RISCV_ISA_EXT_KEY_ZIHINTPAUSE,
+ RISCV_ISA_EXT_KEY_SVINVAL,
RISCV_ISA_EXT_KEY_MAX,
};
return RISCV_ISA_EXT_KEY_FPU;
case RISCV_ISA_EXT_ZIHINTPAUSE:
return RISCV_ISA_EXT_KEY_ZIHINTPAUSE;
+ case RISCV_ISA_EXT_SVINVAL:
+ return RISCV_ISA_EXT_KEY_SVINVAL;
default:
return -EINVAL;
}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef __ASM_INSN_DEF_H
+#define __ASM_INSN_DEF_H
+
+#include <asm/asm.h>
+
+#define INSN_R_FUNC7_SHIFT 25
+#define INSN_R_RS2_SHIFT 20
+#define INSN_R_RS1_SHIFT 15
+#define INSN_R_FUNC3_SHIFT 12
+#define INSN_R_RD_SHIFT 7
+#define INSN_R_OPCODE_SHIFT 0
+
+#ifdef __ASSEMBLY__
+
+#ifdef CONFIG_AS_HAS_INSN
+
+ .macro insn_r, opcode, func3, func7, rd, rs1, rs2
+ .insn r \opcode, \func3, \func7, \rd, \rs1, \rs2
+ .endm
+
+#else
+
+#include <asm/gpr-num.h>
+
+ .macro insn_r, opcode, func3, func7, rd, rs1, rs2
+ .4byte ((\opcode << INSN_R_OPCODE_SHIFT) | \
+ (\func3 << INSN_R_FUNC3_SHIFT) | \
+ (\func7 << INSN_R_FUNC7_SHIFT) | \
+ (.L__gpr_num_\rd << INSN_R_RD_SHIFT) | \
+ (.L__gpr_num_\rs1 << INSN_R_RS1_SHIFT) | \
+ (.L__gpr_num_\rs2 << INSN_R_RS2_SHIFT))
+ .endm
+
+#endif
+
+#define __INSN_R(...) insn_r __VA_ARGS__
+
+#else /* ! __ASSEMBLY__ */
+
+#ifdef CONFIG_AS_HAS_INSN
+
+#define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \
+ ".insn r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n"
+
+#else
+
+#include <linux/stringify.h>
+#include <asm/gpr-num.h>
+
+#define DEFINE_INSN_R \
+ __DEFINE_ASM_GPR_NUMS \
+" .macro insn_r, opcode, func3, func7, rd, rs1, rs2\n" \
+" .4byte ((\\opcode << " __stringify(INSN_R_OPCODE_SHIFT) ") |" \
+" (\\func3 << " __stringify(INSN_R_FUNC3_SHIFT) ") |" \
+" (\\func7 << " __stringify(INSN_R_FUNC7_SHIFT) ") |" \
+" (.L__gpr_num_\\rd << " __stringify(INSN_R_RD_SHIFT) ") |" \
+" (.L__gpr_num_\\rs1 << " __stringify(INSN_R_RS1_SHIFT) ") |" \
+" (.L__gpr_num_\\rs2 << " __stringify(INSN_R_RS2_SHIFT) "))\n" \
+" .endm\n"
+
+#define UNDEFINE_INSN_R \
+" .purgem insn_r\n"
+
+#define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \
+ DEFINE_INSN_R \
+ "insn_r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n" \
+ UNDEFINE_INSN_R
+
+#endif
+
+#endif /* ! __ASSEMBLY__ */
+
+#define INSN_R(opcode, func3, func7, rd, rs1, rs2) \
+ __INSN_R(RV_##opcode, RV_##func3, RV_##func7, \
+ RV_##rd, RV_##rs1, RV_##rs2)
+
+#define RV_OPCODE(v) __ASM_STR(v)
+#define RV_FUNC3(v) __ASM_STR(v)
+#define RV_FUNC7(v) __ASM_STR(v)
+#define RV_RD(v) __ASM_STR(v)
+#define RV_RS1(v) __ASM_STR(v)
+#define RV_RS2(v) __ASM_STR(v)
+#define __RV_REG(v) __ASM_STR(x ## v)
+#define RV___RD(v) __RV_REG(v)
+#define RV___RS1(v) __RV_REG(v)
+#define RV___RS2(v) __RV_REG(v)
+
+#define RV_OPCODE_SYSTEM RV_OPCODE(115)
+
+#define HFENCE_VVMA(vaddr, asid) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(17), \
+ __RD(0), RS1(vaddr), RS2(asid))
+
+#define HFENCE_GVMA(gaddr, vmid) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(49), \
+ __RD(0), RS1(gaddr), RS2(vmid))
+
+#define HLVX_HU(dest, addr) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(50), \
+ RD(dest), RS1(addr), __RS2(3))
+
+#define HLV_W(dest, addr) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(52), \
+ RD(dest), RS1(addr), __RS2(0))
+
+#ifdef CONFIG_64BIT
+#define HLV_D(dest, addr) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(54), \
+ RD(dest), RS1(addr), __RS2(0))
+#else
+#define HLV_D(dest, addr) \
+ __ASM_STR(.error "hlv.d requires 64-bit support")
+#endif
+
+#define SINVAL_VMA(vaddr, asid) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(11), \
+ __RD(0), RS1(vaddr), RS2(asid))
+
+#define SFENCE_W_INVAL() \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(12), \
+ __RD(0), __RS1(0), __RS2(0))
+
+#define SFENCE_INVAL_IR() \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(12), \
+ __RD(0), __RS1(0), __RS2(1))
+
+#define HINVAL_VVMA(vaddr, asid) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(19), \
+ __RD(0), RS1(vaddr), RS2(asid))
+
+#define HINVAL_GVMA(gaddr, vmid) \
+ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(51), \
+ __RD(0), RS1(gaddr), RS2(vmid))
+
+#endif /* __ASM_INSN_DEF_H */
__io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr))
__io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr))
__io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr))
-#define insb(addr, buffer, count) __insb((void __iomem *)(long)addr, buffer, count)
-#define insw(addr, buffer, count) __insw((void __iomem *)(long)addr, buffer, count)
-#define insl(addr, buffer, count) __insl((void __iomem *)(long)addr, buffer, count)
+#define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count)
+#define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count)
+#define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count)
__io_writes_outs(writes, u8, b, __io_bw(), __io_aw())
__io_writes_outs(writes, u16, w, __io_bw(), __io_aw())
__io_writes_outs(outs, u8, b, __io_pbw(), __io_paw())
__io_writes_outs(outs, u16, w, __io_pbw(), __io_paw())
__io_writes_outs(outs, u32, l, __io_pbw(), __io_paw())
-#define outsb(addr, buffer, count) __outsb((void __iomem *)(long)addr, buffer, count)
-#define outsw(addr, buffer, count) __outsw((void __iomem *)(long)addr, buffer, count)
-#define outsl(addr, buffer, count) __outsl((void __iomem *)(long)addr, buffer, count)
+#define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count)
+#define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count)
+#define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count)
#ifdef CONFIG_64BIT
__io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr))
#define readsq(addr, buffer, count) __readsq(addr, buffer, count)
__io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr))
-#define insq(addr, buffer, count) __insq((void __iomem *)addr, buffer, count)
+#define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count)
__io_writes_outs(writes, u64, q, __io_bw(), __io_aw())
#define writesq(addr, buffer, count) __writesq(addr, buffer, count)
__io_writes_outs(outs, u64, q, __io_pbr(), __io_paw())
-#define outsq(addr, buffer, count) __outsq((void __iomem *)addr, buffer, count)
+#define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count)
#endif
#include <asm-generic/io.h>
u64 mmio_exit_kernel;
u64 csr_exit_user;
u64 csr_exit_kernel;
+ u64 signal_exits;
u64 exits;
};
#define KVM_SBI_IMPID 3
-#define KVM_SBI_VERSION_MAJOR 0
-#define KVM_SBI_VERSION_MINOR 3
+#define KVM_SBI_VERSION_MAJOR 1
+#define KVM_SBI_VERSION_MINOR 0
struct kvm_vcpu_sbi_extension {
unsigned long extid_start;
atomic_long_t id;
#endif
void *vdso;
- void *vdso_info;
#ifdef CONFIG_SMP
/* A local icache flush is needed before user execution can resume. */
cpumask_t icache_stale_mask;
extern void start_thread(struct pt_regs *regs,
unsigned long pc, unsigned long sp);
-/* Free all resources held by a thread. */
-static inline void release_thread(struct task_struct *dead_task)
-{
-}
-
extern unsigned long __get_wchan(struct task_struct *p);
#define AT_L1D_CACHEGEOMETRY 43
#define AT_L2_CACHESIZE 44
#define AT_L2_CACHEGEOMETRY 45
+#define AT_L3_CACHESIZE 46
+#define AT_L3_CACHEGEOMETRY 47
/* entries in ARCH_DLINFO */
-#define AT_VECTOR_SIZE_ARCH 7
+#define AT_VECTOR_SIZE_ARCH 9
#endif /* _UAPI_ASM_RISCV_AUXVEC_H */
/* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
struct kvm_riscv_config {
unsigned long isa;
+ unsigned long zicbom_block_size;
};
/* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
KVM_RISCV_ISA_EXT_M,
KVM_RISCV_ISA_EXT_SVPBMT,
KVM_RISCV_ISA_EXT_SSTC,
+ KVM_RISCV_ISA_EXT_SVINVAL,
+ KVM_RISCV_ISA_EXT_ZIHINTPAUSE,
+ KVM_RISCV_ISA_EXT_ZICBOM,
KVM_RISCV_ISA_EXT_MAX,
};
* Copyright (C) 2012 Regents of the University of California
*/
+#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/seq_file.h>
#include <linux/of.h>
+#include <asm/csr.h>
#include <asm/hwcap.h>
+#include <asm/sbi.h>
#include <asm/smp.h>
#include <asm/pgtable.h>
}
#ifdef CONFIG_PROC_FS
+
+struct riscv_cpuinfo {
+ unsigned long mvendorid;
+ unsigned long marchid;
+ unsigned long mimpid;
+};
+static DEFINE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo);
+
+static int riscv_cpuinfo_starting(unsigned int cpu)
+{
+ struct riscv_cpuinfo *ci = this_cpu_ptr(&riscv_cpuinfo);
+
+#if IS_ENABLED(CONFIG_RISCV_SBI)
+ ci->mvendorid = sbi_spec_is_0_1() ? 0 : sbi_get_mvendorid();
+ ci->marchid = sbi_spec_is_0_1() ? 0 : sbi_get_marchid();
+ ci->mimpid = sbi_spec_is_0_1() ? 0 : sbi_get_mimpid();
+#elif IS_ENABLED(CONFIG_RISCV_M_MODE)
+ ci->mvendorid = csr_read(CSR_MVENDORID);
+ ci->marchid = csr_read(CSR_MARCHID);
+ ci->mimpid = csr_read(CSR_MIMPID);
+#else
+ ci->mvendorid = 0;
+ ci->marchid = 0;
+ ci->mimpid = 0;
+#endif
+
+ return 0;
+}
+
+static int __init riscv_cpuinfo_init(void)
+{
+ int ret;
+
+ ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/cpuinfo:starting",
+ riscv_cpuinfo_starting, NULL);
+ if (ret < 0) {
+ pr_err("cpuinfo: failed to register hotplug callbacks.\n");
+ return ret;
+ }
+
+ return 0;
+}
+device_initcall(riscv_cpuinfo_init);
+
#define __RISCV_ISA_EXT_DATA(UPROP, EXTID) \
{ \
.uprop = #UPROP, \
static struct riscv_isa_ext_data isa_ext_arr[] = {
__RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
__RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
+ __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
__RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
__RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM),
__RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
{
unsigned long cpu_id = (unsigned long)v - 1;
struct device_node *node = of_get_cpu_node(cpu_id, NULL);
+ struct riscv_cpuinfo *ci = per_cpu_ptr(&riscv_cpuinfo, cpu_id);
const char *compat, *isa;
seq_printf(m, "processor\t: %lu\n", cpu_id);
if (!of_property_read_string(node, "compatible", &compat)
&& strcmp(compat, "riscv"))
seq_printf(m, "uarch\t\t: %s\n", compat);
+ seq_printf(m, "mvendorid\t: 0x%lx\n", ci->mvendorid);
+ seq_printf(m, "marchid\t\t: 0x%lx\n", ci->marchid);
+ seq_printf(m, "mimpid\t\t: 0x%lx\n", ci->mimpid);
seq_puts(m, "\n");
of_node_put(node);
SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM);
SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE);
SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC);
+ SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL);
}
#undef SET_ISA_EXT_MAP
}
#ifdef CONFIG_RISCV_ALTERNATIVE
static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage)
{
-#ifdef CONFIG_RISCV_ISA_SVPBMT
- switch (stage) {
- case RISCV_ALTERNATIVES_EARLY_BOOT:
+ if (!IS_ENABLED(CONFIG_RISCV_ISA_SVPBMT))
+ return false;
+
+ if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
return false;
- default:
- return riscv_isa_extension_available(NULL, SVPBMT);
- }
-#endif
- return false;
+ return riscv_isa_extension_available(NULL, SVPBMT);
}
static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage)
{
-#ifdef CONFIG_RISCV_ISA_ZICBOM
- switch (stage) {
- case RISCV_ALTERNATIVES_EARLY_BOOT:
+ if (!IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM))
+ return false;
+
+ if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
+ return false;
+
+ if (!riscv_isa_extension_available(NULL, ZICBOM))
return false;
- default:
- if (riscv_isa_extension_available(NULL, ZICBOM)) {
- riscv_noncoherent_supported();
- return true;
- } else {
- return false;
- }
- }
-#endif
- return false;
+ riscv_noncoherent_supported();
+ return true;
}
/*
u32 cpu_req_feature = 0;
if (cpufeature_probe_svpbmt(stage))
- cpu_req_feature |= (1U << CPUFEATURE_SVPBMT);
+ cpu_req_feature |= BIT(CPUFEATURE_SVPBMT);
if (cpufeature_probe_zicbom(stage))
- cpu_req_feature |= (1U << CPUFEATURE_ZICBOM);
+ cpu_req_feature |= BIT(CPUFEATURE_ZICBOM);
return cpu_req_feature;
}
pr_info("Machine model: %s\n", name);
dump_stack_set_arch_desc("%s (DT)", name);
}
- return;
+ } else {
+ pr_err("No DTB passed to the kernel\n");
}
- pr_err("No DTB passed to the kernel\n");
#ifdef CONFIG_CMDLINE_FORCE
strscpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE);
pr_info("Forcing kernel command line to: %s\n", boot_command_line);
if (unlikely(offset & (~PAGE_MASK >> page_shift_offset)))
return -EINVAL;
- if (unlikely((prot & PROT_WRITE) && !(prot & PROT_READ)))
- return -EINVAL;
-
return ksys_mmap_pgoff(addr, len, prot, flags, fd,
offset >> (PAGE_SHIFT - page_shift_offset));
}
{
static int die_counter;
int ret;
+ long cause;
oops_enter();
pr_emerg("%s [#%d]\n", str, ++die_counter);
print_modules();
- show_regs(regs);
+ if (regs)
+ show_regs(regs);
- ret = notify_die(DIE_OOPS, str, regs, 0, regs->cause, SIGSEGV);
+ cause = regs ? regs->cause : -1;
+ ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
- if (regs && kexec_should_crash(current))
+ if (kexec_should_crash(current))
crash_kexec(regs);
bust_spinlocks(0);
struct vm_special_mapping *cm;
};
+static struct __vdso_info vdso_info;
+#ifdef CONFIG_COMPAT
+static struct __vdso_info compat_vdso_info;
+#endif
+
static int vdso_mremap(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma)
{
{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
- struct __vdso_info *vdso_info = mm->context.vdso_info;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
unsigned long size = vma->vm_end - vma->vm_start;
- if (vma_is_special_mapping(vma, vdso_info->dm))
+ if (vma_is_special_mapping(vma, vdso_info.dm))
zap_page_range(vma, vma->vm_start, size);
+#ifdef CONFIG_COMPAT
+ if (vma_is_special_mapping(vma, compat_vdso_info.dm))
+ zap_page_range(vma, vma->vm_start, size);
+#endif
}
mmap_read_unlock(mm);
vdso_base += VVAR_SIZE;
mm->context.vdso = (void *)vdso_base;
- mm->context.vdso_info = (void *)vdso_info;
ret =
_install_special_mapping(mm, vdso_base, vdso_text_len,
select PREEMPT_NOTIFIERS
select KVM_MMIO
select KVM_GENERIC_DIRTYLOG_READ_PROTECT
+ select KVM_XFER_TO_GUEST_WORK
select HAVE_KVM_VCPU_ASYNC_IOCTL
select HAVE_KVM_EVENTFD
select SRCU
{
}
-static int riscv_kvm_init(void)
+static int __init riscv_kvm_init(void)
{
return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
}
#include <linux/kvm_host.h>
#include <asm/cacheflush.h>
#include <asm/csr.h>
+#include <asm/hwcap.h>
+#include <asm/insn-def.h>
-/*
- * Instruction encoding of hfence.gvma is:
- * HFENCE.GVMA rs1, rs2
- * HFENCE.GVMA zero, rs2
- * HFENCE.GVMA rs1
- * HFENCE.GVMA
- *
- * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2
- * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2
- * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1
- * rs1==zero and rs2==zero ==> HFENCE.GVMA
- *
- * Instruction encoding of HFENCE.GVMA is:
- * 0110001 rs2(5) rs1(5) 000 00000 1110011
- */
+#define has_svinval() \
+ static_branch_unlikely(&riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_SVINVAL])
void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
gpa_t gpa, gpa_t gpsz,
return;
}
- for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) {
- /*
- * rs1 = a0 (GPA >> 2)
- * rs2 = a1 (VMID)
- * HFENCE.GVMA a0, a1
- * 0110001 01011 01010 000 00000 1110011
- */
- asm volatile ("srli a0, %0, 2\n"
- "add a1, %1, zero\n"
- ".word 0x62b50073\n"
- :: "r" (pos), "r" (vmid)
- : "a0", "a1", "memory");
+ if (has_svinval()) {
+ asm volatile (SFENCE_W_INVAL() ::: "memory");
+ for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
+ asm volatile (HINVAL_GVMA(%0, %1)
+ : : "r" (pos >> 2), "r" (vmid) : "memory");
+ asm volatile (SFENCE_INVAL_IR() ::: "memory");
+ } else {
+ for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
+ asm volatile (HFENCE_GVMA(%0, %1)
+ : : "r" (pos >> 2), "r" (vmid) : "memory");
}
}
void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)
{
- /*
- * rs1 = zero
- * rs2 = a0 (VMID)
- * HFENCE.GVMA zero, a0
- * 0110001 01010 00000 000 00000 1110011
- */
- asm volatile ("add a0, %0, zero\n"
- ".word 0x62a00073\n"
- :: "r" (vmid) : "a0", "memory");
+ asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory");
}
void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
return;
}
- for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) {
- /*
- * rs1 = a0 (GPA >> 2)
- * rs2 = zero
- * HFENCE.GVMA a0
- * 0110001 00000 01010 000 00000 1110011
- */
- asm volatile ("srli a0, %0, 2\n"
- ".word 0x62050073\n"
- :: "r" (pos) : "a0", "memory");
+ if (has_svinval()) {
+ asm volatile (SFENCE_W_INVAL() ::: "memory");
+ for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
+ asm volatile(HINVAL_GVMA(%0, zero)
+ : : "r" (pos >> 2) : "memory");
+ asm volatile (SFENCE_INVAL_IR() ::: "memory");
+ } else {
+ for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))
+ asm volatile(HFENCE_GVMA(%0, zero)
+ : : "r" (pos >> 2) : "memory");
}
}
void kvm_riscv_local_hfence_gvma_all(void)
{
- /*
- * rs1 = zero
- * rs2 = zero
- * HFENCE.GVMA
- * 0110001 00000 00000 000 00000 1110011
- */
- asm volatile (".word 0x62000073" ::: "memory");
+ asm volatile(HFENCE_GVMA(zero, zero) : : : "memory");
}
-/*
- * Instruction encoding of hfence.gvma is:
- * HFENCE.VVMA rs1, rs2
- * HFENCE.VVMA zero, rs2
- * HFENCE.VVMA rs1
- * HFENCE.VVMA
- *
- * rs1!=zero and rs2!=zero ==> HFENCE.VVMA rs1, rs2
- * rs1==zero and rs2!=zero ==> HFENCE.VVMA zero, rs2
- * rs1!=zero and rs2==zero ==> HFENCE.VVMA rs1
- * rs1==zero and rs2==zero ==> HFENCE.VVMA
- *
- * Instruction encoding of HFENCE.VVMA is:
- * 0010001 rs2(5) rs1(5) 000 00000 1110011
- */
-
void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
unsigned long asid,
unsigned long gva,
hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
- for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) {
- /*
- * rs1 = a0 (GVA)
- * rs2 = a1 (ASID)
- * HFENCE.VVMA a0, a1
- * 0010001 01011 01010 000 00000 1110011
- */
- asm volatile ("add a0, %0, zero\n"
- "add a1, %1, zero\n"
- ".word 0x22b50073\n"
- :: "r" (pos), "r" (asid)
- : "a0", "a1", "memory");
+ if (has_svinval()) {
+ asm volatile (SFENCE_W_INVAL() ::: "memory");
+ for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
+ asm volatile(HINVAL_VVMA(%0, %1)
+ : : "r" (pos), "r" (asid) : "memory");
+ asm volatile (SFENCE_INVAL_IR() ::: "memory");
+ } else {
+ for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
+ asm volatile(HFENCE_VVMA(%0, %1)
+ : : "r" (pos), "r" (asid) : "memory");
}
csr_write(CSR_HGATP, hgatp);
hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
- /*
- * rs1 = zero
- * rs2 = a0 (ASID)
- * HFENCE.VVMA zero, a0
- * 0010001 01010 00000 000 00000 1110011
- */
- asm volatile ("add a0, %0, zero\n"
- ".word 0x22a00073\n"
- :: "r" (asid) : "a0", "memory");
+ asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory");
csr_write(CSR_HGATP, hgatp);
}
hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
- for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) {
- /*
- * rs1 = a0 (GVA)
- * rs2 = zero
- * HFENCE.VVMA a0
- * 0010001 00000 01010 000 00000 1110011
- */
- asm volatile ("add a0, %0, zero\n"
- ".word 0x22050073\n"
- :: "r" (pos) : "a0", "memory");
+ if (has_svinval()) {
+ asm volatile (SFENCE_W_INVAL() ::: "memory");
+ for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
+ asm volatile(HINVAL_VVMA(%0, zero)
+ : : "r" (pos) : "memory");
+ asm volatile (SFENCE_INVAL_IR() ::: "memory");
+ } else {
+ for (pos = gva; pos < (gva + gvsz); pos += BIT(order))
+ asm volatile(HFENCE_VVMA(%0, zero)
+ : : "r" (pos) : "memory");
}
csr_write(CSR_HGATP, hgatp);
hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
- /*
- * rs1 = zero
- * rs2 = zero
- * HFENCE.VVMA
- * 0010001 00000 00000 000 00000 1110011
- */
- asm volatile (".word 0x22000073" ::: "memory");
+ asm volatile(HFENCE_VVMA(zero, zero) : : : "memory");
csr_write(CSR_HGATP, hgatp);
}
*/
#include <linux/bitops.h>
+#include <linux/entry-kvm.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kdebug.h>
#include <linux/fs.h>
#include <linux/kvm_host.h>
#include <asm/csr.h>
+#include <asm/cacheflush.h>
#include <asm/hwcap.h>
const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
STATS_DESC_COUNTER(VCPU, csr_exit_user),
STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
+ STATS_DESC_COUNTER(VCPU, signal_exits),
STATS_DESC_COUNTER(VCPU, exits)
};
#define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0)
+#define KVM_ISA_EXT_ARR(ext) [KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext
+
/* Mapping between KVM ISA Extension ID & Host ISA extension ID */
static const unsigned long kvm_isa_ext_arr[] = {
- RISCV_ISA_EXT_a,
- RISCV_ISA_EXT_c,
- RISCV_ISA_EXT_d,
- RISCV_ISA_EXT_f,
- RISCV_ISA_EXT_h,
- RISCV_ISA_EXT_i,
- RISCV_ISA_EXT_m,
- RISCV_ISA_EXT_SVPBMT,
- RISCV_ISA_EXT_SSTC,
+ [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a,
+ [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c,
+ [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d,
+ [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f,
+ [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h,
+ [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i,
+ [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
+
+ KVM_ISA_EXT_ARR(SSTC),
+ KVM_ISA_EXT_ARR(SVINVAL),
+ KVM_ISA_EXT_ARR(SVPBMT),
+ KVM_ISA_EXT_ARR(ZIHINTPAUSE),
+ KVM_ISA_EXT_ARR(ZICBOM),
};
static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext)
case KVM_RISCV_ISA_EXT_I:
case KVM_RISCV_ISA_EXT_M:
case KVM_RISCV_ISA_EXT_SSTC:
+ case KVM_RISCV_ISA_EXT_SVINVAL:
+ case KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
return false;
default:
break;
case KVM_REG_RISCV_CONFIG_REG(isa):
reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK;
break;
+ case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
+ if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM))
+ return -EINVAL;
+ reg_val = riscv_cbom_block_size;
+ break;
default:
return -EINVAL;
}
return -EOPNOTSUPP;
}
break;
+ case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
+ return -EOPNOTSUPP;
default:
return -EINVAL;
}
{
u64 henvcfg = 0;
- if (__riscv_isa_extension_available(isa, RISCV_ISA_EXT_SVPBMT))
+ if (riscv_isa_extension_available(isa, SVPBMT))
henvcfg |= ENVCFG_PBMTE;
- if (__riscv_isa_extension_available(isa, RISCV_ISA_EXT_SSTC))
+ if (riscv_isa_extension_available(isa, SSTC))
henvcfg |= ENVCFG_STCE;
+
+ if (riscv_isa_extension_available(isa, ZICBOM))
+ henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE);
+
csr_write(CSR_HENVCFG, henvcfg);
#ifdef CONFIG_32BIT
csr_write(CSR_HENVCFGH, henvcfg >> 32);
run->exit_reason = KVM_EXIT_UNKNOWN;
while (ret > 0) {
/* Check conditions before entering the guest */
- cond_resched();
+ ret = xfer_to_guest_mode_handle_work(vcpu);
+ if (!ret)
+ ret = 1;
kvm_riscv_gstage_vmid_update(vcpu);
local_irq_disable();
- /*
- * Exit if we have a signal pending so that we can deliver
- * the signal to user space.
- */
- if (signal_pending(current)) {
- ret = -EINTR;
- run->exit_reason = KVM_EXIT_INTR;
- }
-
/*
* Ensure we set mode to IN_GUEST_MODE after we disable
* interrupts and before the final VCPU requests check.
if (ret <= 0 ||
kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
- kvm_request_pending(vcpu)) {
+ kvm_request_pending(vcpu) ||
+ xfer_to_guest_mode_work_pending()) {
vcpu->mode = OUTSIDE_GUEST_MODE;
local_irq_enable();
kvm_vcpu_srcu_read_lock(vcpu);
#include <linux/kvm_host.h>
#include <asm/csr.h>
+#include <asm/insn-def.h>
static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
struct kvm_cpu_trap *trap)
{
register unsigned long taddr asm("a0") = (unsigned long)trap;
register unsigned long ttmp asm("a1");
- register unsigned long val asm("t0");
- register unsigned long tmp asm("t1");
- register unsigned long addr asm("t2") = guest_addr;
- unsigned long flags;
- unsigned long old_stvec, old_hstatus;
+ unsigned long flags, val, tmp, old_stvec, old_hstatus;
local_irq_save(flags);
".option push\n"
".option norvc\n"
"add %[ttmp], %[taddr], 0\n"
- /*
- * HLVX.HU %[val], (%[addr])
- * HLVX.HU t0, (t2)
- * 0110010 00011 00111 100 00101 1110011
- */
- ".word 0x6433c2f3\n"
+ HLVX_HU(%[val], %[addr])
"andi %[tmp], %[val], 3\n"
"addi %[tmp], %[tmp], -3\n"
"bne %[tmp], zero, 2f\n"
"addi %[addr], %[addr], 2\n"
- /*
- * HLVX.HU %[tmp], (%[addr])
- * HLVX.HU t1, (t2)
- * 0110010 00011 00111 100 00110 1110011
- */
- ".word 0x6433c373\n"
+ HLVX_HU(%[tmp], %[addr])
"sll %[tmp], %[tmp], 16\n"
"add %[val], %[val], %[tmp]\n"
"2:\n"
".option pop"
: [val] "=&r" (val), [tmp] "=&r" (tmp),
[taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp),
- [addr] "+&r" (addr) : : "memory");
+ [addr] "+&r" (guest_addr) : : "memory");
if (trap->scause == EXC_LOAD_PAGE_FAULT)
trap->scause = EXC_INST_PAGE_FAULT;
".option norvc\n"
"add %[ttmp], %[taddr], 0\n"
#ifdef CONFIG_64BIT
- /*
- * HLV.D %[val], (%[addr])
- * HLV.D t0, (t2)
- * 0110110 00000 00111 100 00101 1110011
- */
- ".word 0x6c03c2f3\n"
+ HLV_D(%[val], %[addr])
#else
- /*
- * HLV.W %[val], (%[addr])
- * HLV.W t0, (t2)
- * 0110100 00000 00111 100 00101 1110011
- */
- ".word 0x6803c2f3\n"
+ HLV_W(%[val], %[addr])
#endif
".option pop"
: [val] "=&r" (val),
[taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp)
- : [addr] "r" (addr) : "memory");
+ : [addr] "r" (guest_addr) : "memory");
}
csr_write(CSR_STVEC, old_stvec);
#include <asm/cacheflush.h>
unsigned int riscv_cbom_block_size;
+EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
+
static bool noncoherent_supported;
void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
}
break;
case EXC_LOAD_PAGE_FAULT:
- if (!(vma->vm_flags & VM_READ)) {
+ /* Write implies read */
+ if (!(vma->vm_flags & (VM_READ | VM_WRITE))) {
return true;
}
break;
void show_registers(struct pt_regs *regs);
void show_cacheinfo(struct seq_file *m);
-/* Free all resources held by a thread. */
-static inline void release_thread(struct task_struct *tsk) { }
-
/* Free guarded storage control block */
void guarded_storage_release(struct task_struct *tsk);
void gs_load_bc_cb(struct pt_regs *regs);
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() & ~PAGE_MASK;
+ sp -= prandom_u32_max(PAGE_SIZE);
return sp & ~0xf;
}
static inline unsigned long brk_rnd(void)
{
- return (get_random_int() & BRK_RND_MASK) << PAGE_SHIFT;
+ return (get_random_u16() & BRK_RND_MASK) << PAGE_SHIFT;
}
unsigned long arch_randomize_brk(struct mm_struct *mm)
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
{
struct mm_struct *mm = task->mm;
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
unsigned long size = vma->vm_end - vma->vm_start;
if (!vma_is_special_mapping(vma, &vvar_mapping))
end -= len;
if (end > start) {
- offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1);
+ offset = prandom_u32_max(((end - start) >> PAGE_SHIFT) + 1);
addr = start + (offset << PAGE_SHIFT);
} else {
addr = start;
might_fault();
if (!should_fail_usercopy()) {
- instrument_copy_from_user(to, from, n);
+ instrument_copy_from_user_before(to, from, n);
res = raw_copy_from_user_key(to, from, n, key);
+ instrument_copy_from_user_after(to, from, n, res);
}
if (unlikely(res))
memset(to + (n - res), 0, res);
static inline void thp_split_mm(struct mm_struct *mm)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
- for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
vma->vm_flags &= ~VM_HUGEPAGE;
vma->vm_flags |= VM_NOHUGEPAGE;
walk_page_vma(vma, &thp_split_walk_ops, NULL);
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
int ret;
+ VMA_ITERATOR(vmi, mm, 0);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
MADV_UNMERGEABLE, &vma->vm_flags);
if (ret)
return pud_large(pud);
}
-struct page *
-follow_huge_pud(struct mm_struct *mm, unsigned long address,
- pud_t *pud, int flags)
-{
- if (flags & FOLL_GET)
- return NULL;
-
- return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
-}
-
bool __init arch_hugetlb_valid_size(unsigned long size)
{
if (MACHINE_HAS_EDAT1 && size == PMD_SIZE)
unsigned long arch_mmap_rnd(void)
{
- return (get_random_int() & MMAP_RND_MASK) << PAGE_SHIFT;
+ return (get_random_u32() & MMAP_RND_MASK) << PAGE_SHIFT;
}
static unsigned long mmap_base_legacy(unsigned long rnd)
CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_PREEMPT is not set
# CONFIG_DEBUG_BUGVERBOSE is not set
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
# CONFIG_FTRACE is not set
# CONFIG_CRYPTO_ANSI_CPRNG is not set
# CONFIG_CRYPTO_HW is not set
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_SHIRQ=y
CONFIG_DETECT_HUNG_TASK=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_VM=y
CONFIG_DWARF_UNWINDER=y
# CONFIG_CRYPTO_ANSI_CPRNG is not set
CONFIG_MODULE_UNLOAD=y
# CONFIG_BLK_DEV_BSG is not set
CONFIG_CPU_SUBTYPE_SH7724=y
-CONFIG_FORCE_MAX_ZONEORDER=12
+CONFIG_ARCH_FORCE_MAX_ORDER=12
CONFIG_MEMORY_SIZE=0x10000000
CONFIG_FLATMEM_MANUAL=y
CONFIG_SH_ECOVEC=y
CONFIG_DETECT_HUNG_TASK=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_TIMER_STATS=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_CRYPTO=y
CONFIG_CRYPTO_MD5=y
CONFIG_CRYPTO_DES=y
CONFIG_DEBUG_KERNEL=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_DEBUG_KOBJECT=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_FRAME_POINTER=y
CONFIG_CRC_CCITT=m
CONFIG_CRC16=m
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_DEBUG_SPINLOCK_SLEEP=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_SG=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DETECT_HUNG_TASK=y
# CONFIG_DEBUG_PREEMPT is not set
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_CRYPTO_ECB=m
CONFIG_CRYPTO_PCBC=m
CONFIG_CRYPTO_HMAC=y
# CONFIG_DEBUG_PREEMPT is not set
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_DEBUG_LOCKING_API_SELFTESTS=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_SH_STANDARD_BIOS=y
CONFIG_DEBUG_STACK_USAGE=y
CONFIG_4KSTACKS=y
CONFIG_DEBUG_OBJECTS=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_SPINLOCK_SLEEP=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_VM=y
CONFIG_DEBUG_LIST=y
CONFIG_DEBUG_SG=y
CONFIG_DETECT_HUNG_TASK=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_TIMER_STATS=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_SH_STANDARD_BIOS=y
CONFIG_CRYPTO_MD5=y
CONFIG_CRYPTO_DES=y
# CONFIG_PROC_PID_CPUSET is not set
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_MEMCG=y
-CONFIG_CGROUP_MEMCG_SWAP=y
CONFIG_CGROUP_SCHED=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_BLK_CGROUP=y
CONFIG_NFS_FS=y
CONFIG_ROOT_NFS=y
CONFIG_DEBUG_KERNEL=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_FRAME_POINTER=y
CONFIG_CRYPTO_ECB=m
CONFIG_CRYPTO_PCBC=m
CONFIG_NLS_CODEPAGE_932=y
CONFIG_NLS_ISO8859_1=y
CONFIG_DEBUG_KERNEL=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_FRAME_POINTER=y
# CONFIG_CRYPTO_ANSI_CPRNG is not set
CONFIG_CRC_CCITT=y
CONFIG_DEBUG_KERNEL=y
# CONFIG_DETECT_SOFTLOCKUP is not set
# CONFIG_SCHED_DEBUG is not set
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_FRAME_POINTER=y
CONFIG_SH_STANDARD_BIOS=y
CONFIG_CRYPTO_NULL=y
CONFIG_DEBUG_KERNEL=y
# CONFIG_SCHED_DEBUG is not set
# CONFIG_DEBUG_BUGVERBOSE is not set
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
# CONFIG_FTRACE is not set
# CONFIG_CRYPTO_ANSI_CPRNG is not set
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_SPINLOCK_SLEEP=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_LATENCYTOP=y
# CONFIG_FTRACE is not set
CONFIG_CRYPTO_HMAC=y
# CONFIG_PROC_PID_CPUSET is not set
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_MEMCG=y
-CONFIG_CGROUP_MEMCG_SWAP=y
CONFIG_CGROUP_SCHED=y
CONFIG_RT_GROUP_SCHED=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_DEBUG_FS=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DETECT_HUNG_TASK=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_FRAME_POINTER=y
# CONFIG_FTRACE is not set
# CONFIG_DUMP_CODE is not set
extern void start_thread(struct pt_regs *regs, unsigned long new_pc, unsigned long new_sp);
-/* Free all resources held by a thread. */
-extern void release_thread(struct task_struct *);
-
/*
* FPU lazy state save handling.
*/
#endif
}
-void release_thread(struct task_struct *dead_task)
-{
- /* do nothing */
-}
-
asmlinkage void ret_from_fork(void);
asmlinkage void ret_from_kernel_thread(void);
default "0x80000000" if MMU
default "0x00000000"
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
range 9 64 if PAGE_SIZE_16KB
default "9" if PAGE_SIZE_16KB
config ARCH_SPARSEMEM_DEFAULT
def_bool y if SPARC64
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
default "13"
help
: "memory");
}
-/* Free all resources held by a thread. */
-#define release_thread(tsk) do { } while(0)
-
unsigned long __get_wchan(struct task_struct *);
#define task_pt_regs(tsk) ((tsk)->thread.kregs)
regs->tstate &= ~TSTATE_PEF; \
} while (0)
-/* Free all resources held by a thread. */
-#define release_thread(tsk) do { } while (0)
-
unsigned long __get_wchan(struct task_struct *task);
#define task_pt_regs(tsk) (task_thread_info(tsk)->kregs)
unsigned int offset;
/* This loses some more bits than a modulo, but is cheaper */
- offset = get_random_int() & (PTRS_PER_PTE - 1);
+ offset = prandom_u32_max(PTRS_PER_PTE);
return start + (offset << PAGE_SHIFT);
}
CONFIG_PROC_KCORE=y
CONFIG_TMPFS=y
CONFIG_NLS=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_DEBUG_KERNEL=y
CONFIG_PROC_KCORE=y
CONFIG_TMPFS=y
CONFIG_NLS=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_FRAME_WARN=1024
CONFIG_DEBUG_KERNEL=y
int len);
extern int console_open_chan(struct line *line, struct console *co);
extern void deactivate_chan(struct chan *chan, int irq);
-extern void reactivate_chan(struct chan *chan, int irq);
extern void chan_enable_winch(struct chan *chan, struct tty_port *port);
extern int enable_chan(struct line *line);
extern void close_chan(struct line *line);
};
static DEFINE_MUTEX(plug_mem_mutex);
-static unsigned long long unplugged_pages_count = 0;
+static unsigned long long unplugged_pages_count;
static LIST_HEAD(unplugged_pages);
static int unplug_index = UNPLUGGED_PER_PAGE;
mconsole_notify(notify_socket, MCONSOLE_PANIC, message,
strlen(message) + 1);
- return 0;
+ return NOTIFY_DONE;
}
static struct notifier_block panic_exit_notifier = {
- .notifier_call = notify_panic,
- .next = NULL,
- .priority = 1
+ .notifier_call = notify_panic,
+ .priority = INT_MAX, /* run as soon as possible */
};
static int add_notifier(void)
return 0;
}
-static void mmapper_exit(void)
+static void __exit mmapper_exit(void)
{
misc_deregister(&mmapper_dev);
}
static void uml_net_get_drvinfo(struct net_device *dev,
struct ethtool_drvinfo *info)
{
- strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+ strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
}
static const struct ethtool_ops uml_net_ethtool_ops = {
/* Changed by ssl_init and referenced by ssl_exit, which are both serialized
* by being an initcall and exitcall, respectively.
*/
-static int ssl_init_done = 0;
+static int ssl_init_done;
static void ssl_console_write(struct console *c, const char *string,
unsigned len)
}
/* Set in an initcall, checked in an exitcall */
-static int con_init_done = 0;
+static int con_init_done;
static int con_install(struct tty_driver *driver, struct tty_struct *tty)
{
int kernel_fd = -1;
/* Only changed by the io thread. XXX: currently unused. */
-static int io_count = 0;
+static int io_count;
int io_thread(void *arg)
{
static void vector_net_get_drvinfo(struct net_device *dev,
struct ethtool_drvinfo *info)
{
- strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+ strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
}
static int vector_net_load_bpf_flash(struct net_device *dev,
return um_pci_fwnode;
}
-static int um_pci_init(void)
+static int __init um_pci_init(void)
{
int err, i;
}
module_init(um_pci_init);
-static void um_pci_exit(void)
+static void __exit um_pci_exit(void)
{
unregister_virtio_driver(&um_pci_virtio_driver);
irq_domain_remove(um_pci_msi_domain);
u8 extra_payload[512];
} msg;
int rc;
+ irqreturn_t irq_rc = IRQ_NONE;
- rc = vhost_user_recv_req(vu_dev, &msg.msg,
- sizeof(msg.msg.payload) +
- sizeof(msg.extra_payload));
-
- vu_dev->recv_rc = rc;
- if (rc)
- return IRQ_NONE;
-
- switch (msg.msg.header.request) {
- case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG:
- vu_dev->config_changed_irq = true;
- response = 0;
- break;
- case VHOST_USER_SLAVE_VRING_CALL:
- virtio_device_for_each_vq((&vu_dev->vdev), vq) {
- if (vq->index == msg.msg.payload.vring_state.index) {
- response = 0;
- vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index);
- break;
+ while (1) {
+ rc = vhost_user_recv_req(vu_dev, &msg.msg,
+ sizeof(msg.msg.payload) +
+ sizeof(msg.extra_payload));
+ if (rc)
+ break;
+
+ switch (msg.msg.header.request) {
+ case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG:
+ vu_dev->config_changed_irq = true;
+ response = 0;
+ break;
+ case VHOST_USER_SLAVE_VRING_CALL:
+ virtio_device_for_each_vq((&vu_dev->vdev), vq) {
+ if (vq->index == msg.msg.payload.vring_state.index) {
+ response = 0;
+ vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index);
+ break;
+ }
}
+ break;
+ case VHOST_USER_SLAVE_IOTLB_MSG:
+ /* not supported - VIRTIO_F_ACCESS_PLATFORM */
+ case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
+ /* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */
+ default:
+ vu_err(vu_dev, "unexpected slave request %d\n",
+ msg.msg.header.request);
}
- break;
- case VHOST_USER_SLAVE_IOTLB_MSG:
- /* not supported - VIRTIO_F_ACCESS_PLATFORM */
- case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
- /* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */
- default:
- vu_err(vu_dev, "unexpected slave request %d\n",
- msg.msg.header.request);
- }
-
- if (ev && !vu_dev->suspended)
- time_travel_add_irq_event(ev);
- if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY)
- vhost_user_reply(vu_dev, &msg.msg, response);
+ if (ev && !vu_dev->suspended)
+ time_travel_add_irq_event(ev);
- return IRQ_HANDLED;
+ if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY)
+ vhost_user_reply(vu_dev, &msg.msg, response);
+ irq_rc = IRQ_HANDLED;
+ };
+ /* mask EAGAIN as we try non-blocking read until socket is empty */
+ vu_dev->recv_rc = (rc == -EAGAIN) ? 0 : rc;
+ return irq_rc;
}
static irqreturn_t vu_req_interrupt(int irq, void *data)
.request = { 0 } \
}
-static inline void release_thread(struct task_struct *task)
-{
-}
-
/*
* User space process size: 3GB (default).
*/
);
/*
- * This list is constructed in parse_iomem and addresses filled in in
+ * This list is constructed in parse_iomem and addresses filled in
* setup_iomem, both of which run during early boot. Afterwards, it's
* unchanged.
*/
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() % 8192;
+ sp -= prandom_u32_max(8192);
return sp & ~0xf;
}
#endif
void flush_tlb_mm(struct mm_struct *mm)
{
- struct vm_area_struct *vma = mm->mmap;
+ struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
- while (vma != NULL) {
+ for_each_vma(vmi, vma)
fix_range(mm, vma->vm_start, vma->vm_end, 0);
- vma = vma->vm_next;
- }
}
void force_flush_all(void)
{
struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma = mm->mmap;
+ struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
- while (vma != NULL) {
+ for_each_vma(vmi, vma)
fix_range(mm, vma->vm_start, vma->vm_end, 1);
- vma = vma->vm_next;
- }
}
static void *c_start(struct seq_file *m, loff_t *pos)
{
- return *pos < NR_CPUS ? cpu_data + *pos : NULL;
+ return *pos < nr_cpu_ids ? cpu_data + *pos : NULL;
}
static void *c_next(struct seq_file *m, void *v, loff_t *pos)
static int have_console __initdata;
/* Set in uml_mem_setup and modified in linux_main */
-long long physmem_size = 32 * 1024 * 1024;
+long long physmem_size = 64 * 1024 * 1024;
EXPORT_SYMBOL(physmem_size);
static const char *usage_string =
bust_spinlocks(0);
uml_exitcode = 1;
os_dump_core();
- return 0;
+
+ return NOTIFY_DONE;
}
static struct notifier_block panic_exit_notifier = {
- .notifier_call = panic_exit,
- .next = NULL,
- .priority = 0
+ .notifier_call = panic_exit,
+ .priority = INT_MAX - 1, /* run as 2nd notifier, won't return */
};
void uml_finishsetup(void)
read_initrd();
paging_init();
- strlcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
+ strscpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
*cmdline_p = command_line;
setup_hostinfo(host_info, sizeof host_info);
#include <os.h>
/* Changed by set_umid_arg */
-static int umid_inited = 0;
+static int umid_inited;
static int __init set_umid_arg(char *name, int *add)
{
select ARCH_HAS_PMEM_API if X86_64
select ARCH_HAS_PTE_DEVMAP if X86_64
select ARCH_HAS_PTE_SPECIAL
+ select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2
select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64
select ARCH_HAS_COPY_MC if X86_64
select ARCH_HAS_SET_MEMORY
select CLKEVT_I8253
select CLOCKSOURCE_VALIDATE_LAST_CYCLE
select CLOCKSOURCE_WATCHDOG
- select DCACHE_WORD_ACCESS
+ # Word-size accesses may read uninitialized data past the trailing \0
+ # in strings and cause false KMSAN reports.
+ select DCACHE_WORD_ACCESS if !KMSAN
select DYNAMIC_SIGFRAME
select EDAC_ATOMIC_SCRUB
select EDAC_SUPPORT
select HAVE_ARCH_KASAN if X86_64
select HAVE_ARCH_KASAN_VMALLOC if X86_64
select HAVE_ARCH_KFENCE
+ select HAVE_ARCH_KMSAN if X86_64
select HAVE_ARCH_KGDB
select HAVE_ARCH_MMAP_RND_BITS if MMU
select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT
def_bool y
depends on ISA_DMA_API
+config GENERIC_CSUM
+ bool
+ default y if KMSAN || KASAN
+
config GENERIC_BUG
def_bool y
depends on BUG
# Sanitizer runtimes are unavailable and cannot be linked for early boot code.
KASAN_SANITIZE := n
KCSAN_SANITIZE := n
+KMSAN_SANITIZE := n
OBJECT_FILES_NON_STANDARD := y
# Kernel does not boot with kcov instrumentation here.
# Sanitizer runtimes are unavailable and cannot be linked for early boot code.
KASAN_SANITIZE := n
KCSAN_SANITIZE := n
+KMSAN_SANITIZE := n
OBJECT_FILES_NON_STANDARD := y
# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
CONFIG_CGROUP_DEBUG=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_KALLSYMS_ALL=y
-# CONFIG_COMPAT_BRK is not set
CONFIG_PROFILING=y
CONFIG_SMP=y
CONFIG_HYPERVISOR_GUEST=y
CONFIG_HZ_1000=y
CONFIG_KEXEC=y
CONFIG_CRASH_DUMP=y
+# CONFIG_RETHUNK is not set
CONFIG_HIBERNATION=y
CONFIG_PM_DEBUG=y
CONFIG_PM_TRACE_RTC=y
CONFIG_BLK_CGROUP_IOCOST=y
CONFIG_BLK_CGROUP_IOPRIO=y
CONFIG_BINFMT_MISC=y
+# CONFIG_COMPAT_BRK is not set
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_SECURITY_SELINUX_BOOTPARAM=y
CONFIG_SECURITY_SELINUX_DISABLE=y
CONFIG_PRINTK_TIME=y
+CONFIG_DEBUG_KERNEL=y
CONFIG_FRAME_WARN=1024
CONFIG_MAGIC_SYSRQ=y
-CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_WX=y
CONFIG_DEBUG_STACK_USAGE=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_SCHEDSTATS=y
CONFIG_CGROUP_DEBUG=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_KALLSYMS_ALL=y
-# CONFIG_COMPAT_BRK is not set
CONFIG_PROFILING=y
CONFIG_SMP=y
CONFIG_HYPERVISOR_GUEST=y
CONFIG_BLK_CGROUP_IOCOST=y
CONFIG_BLK_CGROUP_IOPRIO=y
CONFIG_BINFMT_MISC=y
+# CONFIG_COMPAT_BRK is not set
CONFIG_NET=y
CONFIG_PACKET=y
CONFIG_UNIX=y
CONFIG_SECURITY_SELINUX_BOOTPARAM=y
CONFIG_SECURITY_SELINUX_DISABLE=y
CONFIG_PRINTK_TIME=y
-CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_KERNEL=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DEBUG_WX=y
CONFIG_DEBUG_STACK_USAGE=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_SCHEDSTATS=y
# Sanitizer runtimes are unavailable and cannot be linked here.
KASAN_SANITIZE := n
+KMSAN_SANITIZE_vclock_gettime.o := n
+KMSAN_SANITIZE_vgetcpu.o := n
+
UBSAN_SANITIZE := n
KCSAN_SANITIZE := n
OBJECT_FILES_NON_STANDARD := y
{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
-
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
unsigned long size = vma->vm_end - vma->vm_start;
if (vma_is_special_mapping(vma, &vvar_mapping))
zap_page_range(vma, vma->vm_start, size);
}
-
mmap_read_unlock(mm);
+
return 0;
}
#else
end -= len;
if (end > start) {
- offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1);
+ offset = prandom_u32_max(((end - start) >> PAGE_SHIFT) + 1);
addr = start + (offset << PAGE_SHIFT);
} else {
addr = start;
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_write_lock(mm);
/*
* We could search vma near context.vdso, but it's a slowpath,
* so let's explicitly check all VMAs to be completely sure.
*/
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (vma_is_special_mapping(vma, &vdso_mapping) ||
vma_is_special_mapping(vma, &vvar_mapping)) {
mmap_write_unlock(mm);
* 1. Register the guest ID
* 2. Enable the hypercall and register the hypercall page
*/
- guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0);
+ guest_id = hv_generate_guest_id(LINUX_VERSION_CODE);
wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
/* Hyper-V requires to write guest os id via ghcb in SNP IVM. */
wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
pg = vmalloc_to_page(hv_hypercall_pg);
- dst = kmap(pg);
+ dst = kmap_local_page(pg);
src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE,
MEMREMAP_WB);
BUG_ON(!(src && dst));
memcpy(dst, src, HV_HYP_PAGE_SIZE);
memunmap(src);
- kunmap(pg);
+ kunmap_local(dst);
} else {
hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
/* SPDX-License-Identifier: GPL-2.0 */
-#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1
-#define HAVE_CSUM_COPY_USER
-#define _HAVE_ARCH_CSUM_AND_COPY
-#ifdef CONFIG_X86_32
-# include <asm/checksum_32.h>
+#ifdef CONFIG_GENERIC_CSUM
+# include <asm-generic/checksum.h>
#else
-# include <asm/checksum_64.h>
+# define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1
+# define HAVE_CSUM_COPY_USER
+# define _HAVE_ARCH_CSUM_AND_COPY
+# ifdef CONFIG_X86_32
+# include <asm/checksum_32.h>
+# else
+# include <asm/checksum_64.h>
+# endif
#endif
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * x86 KMSAN support.
+ *
+ * Copyright (C) 2022, Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ */
+
+#ifndef _ASM_X86_KMSAN_H
+#define _ASM_X86_KMSAN_H
+
+#ifndef MODULE
+
+#include <asm/cpu_entry_area.h>
+#include <asm/processor.h>
+#include <linux/mmzone.h>
+
+DECLARE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_shadow);
+DECLARE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_origin);
+
+/*
+ * Functions below are declared in the header to make sure they are inlined.
+ * They all are called from kmsan_get_metadata() for every memory access in
+ * the kernel, so speed is important here.
+ */
+
+/*
+ * Compute metadata addresses for the CPU entry area on x86.
+ */
+static inline void *arch_kmsan_get_meta_or_null(void *addr, bool is_origin)
+{
+ unsigned long addr64 = (unsigned long)addr;
+ char *metadata_array;
+ unsigned long off;
+ int cpu;
+
+ if ((addr64 < CPU_ENTRY_AREA_BASE) ||
+ (addr64 >= (CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE)))
+ return NULL;
+ cpu = (addr64 - CPU_ENTRY_AREA_BASE) / CPU_ENTRY_AREA_SIZE;
+ off = addr64 - (unsigned long)get_cpu_entry_area(cpu);
+ if ((off < 0) || (off >= CPU_ENTRY_AREA_SIZE))
+ return NULL;
+ metadata_array = is_origin ? cpu_entry_area_origin :
+ cpu_entry_area_shadow;
+ return &per_cpu(metadata_array[off], cpu);
+}
+
+/*
+ * Taken from arch/x86/mm/physaddr.h to avoid using an instrumented version.
+ */
+static inline bool kmsan_phys_addr_valid(unsigned long addr)
+{
+ if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
+ return !(addr >> boot_cpu_data.x86_phys_bits);
+ else
+ return true;
+}
+
+/*
+ * Taken from arch/x86/mm/physaddr.c to avoid using an instrumented version.
+ */
+static inline bool kmsan_virt_addr_valid(void *addr)
+{
+ unsigned long x = (unsigned long)addr;
+ unsigned long y = x - __START_KERNEL_map;
+
+ /* use the carry flag to determine if x was < __START_KERNEL_map */
+ if (unlikely(x > y)) {
+ x = y + phys_base;
+
+ if (y >= KERNEL_IMAGE_SIZE)
+ return false;
+ } else {
+ x = y + (__START_KERNEL_map - PAGE_OFFSET);
+
+ /* carry flag will be set if starting x was >= PAGE_OFFSET */
+ if ((x > y) || !kmsan_phys_addr_valid(x))
+ return false;
+ }
+
+ return pfn_valid(x >> PAGE_SHIFT);
+}
+
+#endif /* !MODULE */
+
+#endif /* _ASM_X86_KMSAN_H */
bool tdp_mmu_enabled;
/*
- * List of struct kvm_mmu_pages being used as roots.
- * All struct kvm_mmu_pages in the list should have
+ * List of kvm_mmu_page structs being used as roots.
+ * All kvm_mmu_page structs in the list should have
* tdp_mmu_page set.
*
* For reads, this list is protected by:
struct list_head tdp_mmu_roots;
/*
- * List of struct kvmp_mmu_pages not being used as roots.
- * All struct kvm_mmu_pages in the list should have
+ * List of kvm_mmu_page structs not being used as roots.
+ * All kvm_mmu_page structs in the list should have
* tdp_mmu_page set and a tdp_mmu_root_count of 0.
*/
struct list_head tdp_mmu_pages;
* is held in read mode:
* - tdp_mmu_roots (above)
* - tdp_mmu_pages (above)
- * - the link field of struct kvm_mmu_pages used by the TDP MMU
+ * - the link field of kvm_mmu_page structs used by the TDP MMU
* - lpage_disallowed_mmu_pages
- * - the lpage_disallowed_link field of struct kvm_mmu_pages used
+ * - the lpage_disallowed_link field of kvm_mmu_page structs used
* by the TDP MMU
* It is acceptable, but not necessary, to acquire this lock when
* the thread holds the MMU lock in write mode.
#include <asm/cpufeatures.h>
#include <asm/alternative.h>
+#include <linux/kmsan-checks.h>
+
/* duplicated to the one in bootmem.h */
extern unsigned long max_pfn;
extern unsigned long phys_base;
static inline void clear_page(void *page)
{
+ /*
+ * Clean up KMSAN metadata for the page being cleared. The assembly call
+ * below clobbers @page, so we perform unpoisoning before it.
+ */
+ kmsan_unpoison_memory(page, PAGE_SIZE);
alternative_call_2(clear_page_orig,
clear_page_rep, X86_FEATURE_REP_GOOD,
clear_page_erms, X86_FEATURE_ERMS,
/* We always extract/encode the offset by shifting it all the way up, and then down again */
#define SWP_OFFSET_SHIFT (SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS)
-#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5)
-#define __swp_type(x) (((x).val) & 0x1f)
-#define __swp_offset(x) ((x).val >> 5)
-#define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << 5})
+#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)
+#define __swp_type(x) (((x).val) & ((1UL << SWP_TYPE_BITS) - 1))
+#define __swp_offset(x) ((x).val >> SWP_TYPE_BITS)
+#define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << SWP_TYPE_BITS})
/*
* Normally, __swp_entry() converts from arch-independent swp_entry_t to
static inline int pmd_bad(pmd_t pmd)
{
- return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
+ return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) !=
+ (_KERNPG_TABLE & ~_PAGE_ACCESSED);
}
static inline unsigned long pages_to_mb(unsigned long npg)
return boot_cpu_has_bug(X86_BUG_L1TF);
}
-#define arch_faults_on_old_pte arch_faults_on_old_pte
-static inline bool arch_faults_on_old_pte(void)
+#define arch_has_hw_pte_young arch_has_hw_pte_young
+static inline bool arch_has_hw_pte_young(void)
{
- return false;
+ return true;
}
#ifdef CONFIG_PAGE_TABLE_CHECK
# define VMEMMAP_START __VMEMMAP_BASE_L4
#endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
-#define VMALLOC_END (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1)
+/*
+ * End of the region for which vmalloc page tables are pre-allocated.
+ * For non-KMSAN builds, this is the same as VMALLOC_END.
+ * For KMSAN builds, VMALLOC_START..VMEMORY_END is 4 times bigger than
+ * VMALLOC_START..VMALLOC_END (see below).
+ */
+#define VMEMORY_END (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1)
+
+#ifndef CONFIG_KMSAN
+#define VMALLOC_END VMEMORY_END
+#else
+/*
+ * In KMSAN builds vmalloc area is four times smaller, and the remaining 3/4
+ * are used to keep the metadata for virtual pages. The memory formerly
+ * belonging to vmalloc area is now laid out as follows:
+ *
+ * 1st quarter: VMALLOC_START to VMALLOC_END - new vmalloc area
+ * 2nd quarter: KMSAN_VMALLOC_SHADOW_START to
+ * VMALLOC_END+KMSAN_VMALLOC_SHADOW_OFFSET - vmalloc area shadow
+ * 3rd quarter: KMSAN_VMALLOC_ORIGIN_START to
+ * VMALLOC_END+KMSAN_VMALLOC_ORIGIN_OFFSET - vmalloc area origins
+ * 4th quarter: KMSAN_MODULES_SHADOW_START to KMSAN_MODULES_ORIGIN_START
+ * - shadow for modules,
+ * KMSAN_MODULES_ORIGIN_START to
+ * KMSAN_MODULES_ORIGIN_START + MODULES_LEN - origins for modules.
+ */
+#define VMALLOC_QUARTER_SIZE ((VMALLOC_SIZE_TB << 40) >> 2)
+#define VMALLOC_END (VMALLOC_START + VMALLOC_QUARTER_SIZE - 1)
+
+/*
+ * vmalloc metadata addresses are calculated by adding shadow/origin offsets
+ * to vmalloc address.
+ */
+#define KMSAN_VMALLOC_SHADOW_OFFSET VMALLOC_QUARTER_SIZE
+#define KMSAN_VMALLOC_ORIGIN_OFFSET (VMALLOC_QUARTER_SIZE << 1)
+
+#define KMSAN_VMALLOC_SHADOW_START (VMALLOC_START + KMSAN_VMALLOC_SHADOW_OFFSET)
+#define KMSAN_VMALLOC_ORIGIN_START (VMALLOC_START + KMSAN_VMALLOC_ORIGIN_OFFSET)
+
+/*
+ * The shadow/origin for modules are placed one by one in the last 1/4 of
+ * vmalloc space.
+ */
+#define KMSAN_MODULES_SHADOW_START (VMALLOC_END + KMSAN_VMALLOC_ORIGIN_OFFSET + 1)
+#define KMSAN_MODULES_ORIGIN_START (KMSAN_MODULES_SHADOW_START + MODULES_LEN)
+#endif /* CONFIG_KMSAN */
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
/* The module sections ends with the start of the fixmap */
#endif /* CONFIG_PARAVIRT_XXL */
-/* Free all resources held by a thread. */
-extern void release_thread(struct task_struct *);
-
unsigned long __get_wchan(struct task_struct *p);
/*
#ifndef _ASM_X86_SPARSEMEM_H
#define _ASM_X86_SPARSEMEM_H
+#include <linux/types.h>
+
#ifdef CONFIG_SPARSEMEM
/*
* generic non-linear memory support:
function. */
#define __HAVE_ARCH_MEMCPY 1
+#if defined(__SANITIZE_MEMORY__)
+#undef memcpy
+void *__msan_memcpy(void *dst, const void *src, size_t size);
+#define memcpy __msan_memcpy
+#else
extern void *memcpy(void *to, const void *from, size_t len);
+#endif
extern void *__memcpy(void *to, const void *from, size_t len);
#define __HAVE_ARCH_MEMSET
+#if defined(__SANITIZE_MEMORY__)
+extern void *__msan_memset(void *s, int c, size_t n);
+#undef memset
+#define memset __msan_memset
+#else
void *memset(void *s, int c, size_t n);
+#endif
void *__memset(void *s, int c, size_t n);
#define __HAVE_ARCH_MEMSET16
}
#define __HAVE_ARCH_MEMMOVE
+#if defined(__SANITIZE_MEMORY__)
+#undef memmove
+void *__msan_memmove(void *dest, const void *src, size_t len);
+#define memmove __msan_memmove
+#else
void *memmove(void *dest, const void *src, size_t count);
+#endif
void *__memmove(void *dest, const void *src, size_t count);
int memcmp(const void *cs, const void *ct, size_t count);
char *strcat(char *dest, const char *src);
int strcmp(const char *cs, const char *ct);
-#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
-
+#if (defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__))
/*
* For files that not instrumented (e.g. mm/slub.c) we
* should use not instrumented version of mem* functions.
#undef memcpy
#define memcpy(dst, src, len) __memcpy(dst, src, len)
+#undef memmove
#define memmove(dst, src, len) __memmove(dst, src, len)
+#undef memset
#define memset(s, c, n) __memset(s, c, n)
#ifndef __NO_FORTIFY
* User space memory access functions
*/
#include <linux/compiler.h>
+#include <linux/instrumented.h>
#include <linux/kasan-checks.h>
#include <linux/string.h>
#include <asm/asm.h>
: "=a" (__ret_gu), "=r" (__val_gu), \
ASM_CALL_CONSTRAINT \
: "0" (ptr), "i" (sizeof(*(ptr)))); \
+ instrument_get_user(__val_gu); \
(x) = (__force __typeof__(*(ptr))) __val_gu; \
__builtin_expect(__ret_gu, 0); \
})
int __ret_pu; \
void __user *__ptr_pu; \
register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX); \
- __chk_user_ptr(ptr); \
- __ptr_pu = (ptr); \
- __val_pu = (x); \
+ __typeof__(*(ptr)) __x = (x); /* eval x once */ \
+ __typeof__(ptr) __ptr = (ptr); /* eval ptr once */ \
+ __chk_user_ptr(__ptr); \
+ __ptr_pu = __ptr; \
+ __val_pu = __x; \
asm volatile("call __" #fn "_%P[size]" \
: "=c" (__ret_pu), \
ASM_CALL_CONSTRAINT \
"r" (__val_pu), \
[size] "i" (sizeof(*(ptr))) \
:"ebx"); \
+ instrument_put_user(__x, __ptr, sizeof(*(ptr))); \
__builtin_expect(__ret_pu, 0); \
})
#define __put_user_size(x, ptr, size, label) \
do { \
+ __typeof__(*(ptr)) __x = (x); /* eval x once */ \
__chk_user_ptr(ptr); \
switch (size) { \
case 1: \
- __put_user_goto(x, ptr, "b", "iq", label); \
+ __put_user_goto(__x, ptr, "b", "iq", label); \
break; \
case 2: \
- __put_user_goto(x, ptr, "w", "ir", label); \
+ __put_user_goto(__x, ptr, "w", "ir", label); \
break; \
case 4: \
- __put_user_goto(x, ptr, "l", "ir", label); \
+ __put_user_goto(__x, ptr, "l", "ir", label); \
break; \
case 8: \
- __put_user_goto_u64(x, ptr, label); \
+ __put_user_goto_u64(__x, ptr, label); \
break; \
default: \
__put_user_bad(); \
} \
+ instrument_put_user(__x, ptr, size); \
} while (0)
#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
default: \
(x) = __get_user_bad(); \
} \
+ instrument_get_user(x); \
} while (0)
#define __get_user_asm(x, addr, itype, ltype, label) \
GUEST_LDTR_AR_BYTES = 0x00004820,
GUEST_TR_AR_BYTES = 0x00004822,
GUEST_INTERRUPTIBILITY_INFO = 0x00004824,
- GUEST_ACTIVITY_STATE = 0X00004826,
+ GUEST_ACTIVITY_STATE = 0x00004826,
GUEST_SYSENTER_CS = 0x0000482A,
VMX_PREEMPTION_TIMER_VALUE = 0x0000482E,
HOST_IA32_SYSENTER_CS = 0x00004c00,
# With some compiler versions the generated code results in boot hangs, caused
# by several compilation units. To be safe, disable all instrumentation.
KCSAN_SANITIZE := n
+KMSAN_SANITIZE_head$(BITS).o := n
+KMSAN_SANITIZE_nmi.o := n
# If instrumentation of this dir is enabled, boot hangs during first second.
# Probably could be more selective here, but note that files related to irqs,
# If these files are instrumented, boot hangs during the first second.
KCOV_INSTRUMENT_common.o := n
KCOV_INSTRUMENT_perf_event.o := n
+KMSAN_SANITIZE_common.o := n
# As above, instrumenting secondary CPU boot code causes boot hangs.
KCSAN_SANITIZE_common.o := n
va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
/* A random value per boot for bit slice [12:upper_bit) */
- va_align.bits = get_random_int() & va_align.mask;
+ va_align.bits = get_random_u32() & va_align.mask;
}
if (cpu_has(c, X86_FEATURE_MWAITX))
}
}
+/*
+ * This function reads pointers from the stack and dereferences them. The
+ * pointers may not have their KMSAN shadow set up properly, which may result
+ * in false positive reports. Disable instrumentation to avoid those.
+ */
+__no_kmsan_checks
static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
unsigned long *stack, const char *log_lvl)
{
*/
if (module_load_offset == 0)
module_load_offset =
- (get_random_int() % 1024 + 1) * PAGE_SIZE;
+ (prandom_u32_max(1024) + 1) * PAGE_SIZE;
mutex_unlock(&module_kaslr_mutex);
}
return module_load_offset;
unsigned long arch_align_stack(unsigned long sp)
{
if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
- sp -= get_random_int() % 8192;
+ sp -= prandom_u32_max(8192);
return sp & ~0xf;
}
* Kprobes not supported here. Set the probe on schedule instead.
* Function graph tracer not supported too.
*/
+__no_kmsan_checks
__visible __notrace_funcgraph struct task_struct *
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
{
static pgd_t *tboot_pg_dir;
static struct mm_struct tboot_mm = {
- .mm_rb = RB_ROOT,
+ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, tboot_mm.mmap_lock),
.pgd = swapper_pg_dir,
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),
}
#endif
+/*
+ * While walking the stack, KMSAN may stomp on stale locals from other
+ * functions that were marked as uninitialized upon function exit, and
+ * now hold the call frame information for the current function (e.g. the frame
+ * pointer). Because KMSAN does not specifically mark call frames as
+ * initialized, false positive reports are possible. To prevent such reports,
+ * we mark the functions scanning the stack (here and below) with
+ * __no_kmsan_checks.
+ */
+__no_kmsan_checks
static bool update_stack_state(struct unwind_state *state,
unsigned long *next_bp)
{
return true;
}
+__no_kmsan_checks
bool unwind_next_frame(struct unwind_state *state)
{
struct pt_regs *regs;
select HAVE_KVM_IRQCHIP
select HAVE_KVM_PFNCACHE
select HAVE_KVM_IRQFD
- select HAVE_KVM_DIRTY_RING
+ select HAVE_KVM_DIRTY_RING_TSO
+ select HAVE_KVM_DIRTY_RING_ACQ_REL
select IRQ_BYPASS_MANAGER
select HAVE_KVM_IRQ_BYPASS
select HAVE_KVM_IRQ_ROUTING
return;
if (pmc->perf_event && pmc->perf_event->attr.precise_ip) {
- /* Indicate PEBS overflow PMI to guest. */
- skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
- (unsigned long *)&pmu->global_status);
+ if (!in_pmi) {
+ /*
+ * TODO: KVM is currently _choosing_ to not generate records
+ * for emulated instructions, avoiding BUFFER_OVF PMI when
+ * there are no records. Strictly speaking, it should be done
+ * as well in the right context to improve sampling accuracy.
+ */
+ skip_pmi = true;
+ } else {
+ /* Indicate PEBS overflow PMI to guest. */
+ skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
+ (unsigned long *)&pmu->global_status);
+ }
} else {
__set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
}
get_sample_period(pmc, pmc->counter)))
return false;
- if (!test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) &&
- pmc->perf_event->attr.precise_ip)
+ if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) !=
+ (!!pmc->perf_event->attr.precise_ip))
return false;
/* reuse perf_event to serve as pmc_reprogram_counter() does*/
PMU_TYPE_EVNTSEL,
};
-enum index {
- INDEX_ZERO = 0,
- INDEX_ONE,
- INDEX_TWO,
- INDEX_THREE,
- INDEX_FOUR,
- INDEX_FIVE,
- INDEX_ERROR,
-};
-
-static unsigned int get_msr_base(struct kvm_pmu *pmu, enum pmu_type type)
+static struct kvm_pmc *amd_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx)
{
- struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
+ unsigned int num_counters = pmu->nr_arch_gp_counters;
- if (guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) {
- if (type == PMU_TYPE_COUNTER)
- return MSR_F15H_PERF_CTR;
- else
- return MSR_F15H_PERF_CTL;
- } else {
- if (type == PMU_TYPE_COUNTER)
- return MSR_K7_PERFCTR0;
- else
- return MSR_K7_EVNTSEL0;
- }
-}
+ if (pmc_idx >= num_counters)
+ return NULL;
-static enum index msr_to_index(u32 msr)
-{
- switch (msr) {
- case MSR_F15H_PERF_CTL0:
- case MSR_F15H_PERF_CTR0:
- case MSR_K7_EVNTSEL0:
- case MSR_K7_PERFCTR0:
- return INDEX_ZERO;
- case MSR_F15H_PERF_CTL1:
- case MSR_F15H_PERF_CTR1:
- case MSR_K7_EVNTSEL1:
- case MSR_K7_PERFCTR1:
- return INDEX_ONE;
- case MSR_F15H_PERF_CTL2:
- case MSR_F15H_PERF_CTR2:
- case MSR_K7_EVNTSEL2:
- case MSR_K7_PERFCTR2:
- return INDEX_TWO;
- case MSR_F15H_PERF_CTL3:
- case MSR_F15H_PERF_CTR3:
- case MSR_K7_EVNTSEL3:
- case MSR_K7_PERFCTR3:
- return INDEX_THREE;
- case MSR_F15H_PERF_CTL4:
- case MSR_F15H_PERF_CTR4:
- return INDEX_FOUR;
- case MSR_F15H_PERF_CTL5:
- case MSR_F15H_PERF_CTR5:
- return INDEX_FIVE;
- default:
- return INDEX_ERROR;
- }
+ return &pmu->gp_counters[array_index_nospec(pmc_idx, num_counters)];
}
static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr,
enum pmu_type type)
{
struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
+ unsigned int idx;
if (!vcpu->kvm->arch.enable_pmu)
return NULL;
switch (msr) {
- case MSR_F15H_PERF_CTL0:
- case MSR_F15H_PERF_CTL1:
- case MSR_F15H_PERF_CTL2:
- case MSR_F15H_PERF_CTL3:
- case MSR_F15H_PERF_CTL4:
- case MSR_F15H_PERF_CTL5:
+ case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
if (!guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE))
return NULL;
- fallthrough;
+ /*
+ * Each PMU counter has a pair of CTL and CTR MSRs. CTLn
+ * MSRs (accessed via EVNTSEL) are even, CTRn MSRs are odd.
+ */
+ idx = (unsigned int)((msr - MSR_F15H_PERF_CTL0) / 2);
+ if (!(msr & 0x1) != (type == PMU_TYPE_EVNTSEL))
+ return NULL;
+ break;
case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
if (type != PMU_TYPE_EVNTSEL)
return NULL;
+ idx = msr - MSR_K7_EVNTSEL0;
break;
- case MSR_F15H_PERF_CTR0:
- case MSR_F15H_PERF_CTR1:
- case MSR_F15H_PERF_CTR2:
- case MSR_F15H_PERF_CTR3:
- case MSR_F15H_PERF_CTR4:
- case MSR_F15H_PERF_CTR5:
- if (!guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE))
- return NULL;
- fallthrough;
case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
if (type != PMU_TYPE_COUNTER)
return NULL;
+ idx = msr - MSR_K7_PERFCTR0;
break;
default:
return NULL;
}
- return &pmu->gp_counters[msr_to_index(msr)];
+ return amd_pmc_idx_to_pmc(pmu, idx);
}
static bool amd_hw_event_available(struct kvm_pmc *pmc)
return true;
}
-static struct kvm_pmc *amd_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx)
-{
- unsigned int base = get_msr_base(pmu, PMU_TYPE_COUNTER);
- struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
-
- if (guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) {
- /*
- * The idx is contiguous. The MSRs are not. The counter MSRs
- * are interleaved with the event select MSRs.
- */
- pmc_idx *= 2;
- }
-
- return get_gp_pmc_amd(pmu, base + pmc_idx, PMU_TYPE_COUNTER);
-}
-
static bool amd_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
{
struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
static struct kvm_pmc *amd_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu,
unsigned int idx, u64 *mask)
{
- struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
- struct kvm_pmc *counters;
-
- idx &= ~(3u << 30);
- if (idx >= pmu->nr_arch_gp_counters)
- return NULL;
- counters = pmu->gp_counters;
-
- return &counters[idx];
+ return amd_pmc_idx_to_pmc(vcpu_to_pmu(vcpu), idx & ~(3u << 30));
}
static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
}
}
-/* function is called when global control register has been updated. */
-static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data)
+static void reprogram_counters(struct kvm_pmu *pmu, u64 diff)
{
int bit;
- u64 diff = pmu->global_ctrl ^ data;
struct kvm_pmc *pmc;
- pmu->global_ctrl = data;
-
for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX) {
pmc = intel_pmc_idx_to_pmc(pmu, bit);
if (pmc)
struct kvm_pmc *pmc;
u32 msr = msr_info->index;
u64 data = msr_info->data;
- u64 reserved_bits;
+ u64 reserved_bits, diff;
switch (msr) {
case MSR_CORE_PERF_FIXED_CTR_CTRL:
if (pmu->global_ctrl == data)
return 0;
if (kvm_valid_perf_global_ctrl(pmu, data)) {
- global_ctrl_changed(pmu, data);
+ diff = pmu->global_ctrl ^ data;
+ pmu->global_ctrl = data;
+ reprogram_counters(pmu, diff);
return 0;
}
break;
if (pmu->pebs_enable == data)
return 0;
if (!(data & pmu->pebs_enable_mask)) {
+ diff = pmu->pebs_enable ^ data;
pmu->pebs_enable = data;
+ reprogram_counters(pmu, diff);
return 0;
}
break;
void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu)
{
struct kvm_pmc *pmc = NULL;
- int bit;
+ int bit, hw_idx;
for_each_set_bit(bit, (unsigned long *)&pmu->global_ctrl,
X86_PMC_IDX_MAX) {
pmc = intel_pmc_idx_to_pmc(pmu, bit);
if (!pmc || !pmc_speculative_in_use(pmc) ||
- !intel_pmc_is_enabled(pmc))
+ !intel_pmc_is_enabled(pmc) || !pmc->perf_event)
continue;
- if (pmc->perf_event && pmc->idx != pmc->perf_event->hw.idx) {
- pmu->host_cross_mapped_mask |=
- BIT_ULL(pmc->perf_event->hw.idx);
- }
+ /*
+ * A negative index indicates the event isn't mapped to a
+ * physical counter in the host, e.g. due to contention.
+ */
+ hw_idx = pmc->perf_event->hw.idx;
+ if (hw_idx != pmc->idx && hw_idx > -1)
+ pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx);
}
}
endif
else
obj-y += iomap_copy_64.o
+ifneq ($(CONFIG_GENERIC_CSUM),y)
lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o
+endif
lib-y += clear_page_64.o copy_page_64.o
lib-y += memmove_64.o memset_64.o
lib-y += copy_user_64.o
#include <linux/string.h>
#include <linux/module.h>
#include <linux/io.h>
+#include <linux/kmsan-checks.h>
#define movs(type,to,from) \
asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory")
n-=2;
}
rep_movs(to, (const void *)from, n);
+ /* KMSAN must treat values read from devices as initialized. */
+ kmsan_unpoison_memory(to, n);
}
static void string_memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
if (unlikely(!n))
return;
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(from, n);
/* Align any unaligned destination IO */
if (unlikely(1 & (unsigned long)to)) {
movs("b", to, from);
# Disable KCSAN entirely, because otherwise we get warnings that some functions
# reference __initdata sections.
KCSAN_SANITIZE := n
+# Avoid recursion by not calling KMSAN hooks for CEA code.
+KMSAN_SANITIZE_cpu_entry_area.o := n
ifdef CONFIG_FUNCTION_TRACER
CFLAGS_REMOVE_mem_encrypt.o = -pg
KASAN_SANITIZE_kasan_init_$(BITS).o := n
obj-$(CONFIG_KASAN) += kasan_init_$(BITS).o
+KMSAN_SANITIZE_kmsan_shadow.o := n
+obj-$(CONFIG_KMSAN) += kmsan_shadow.o
+
obj-$(CONFIG_MMIOTRACE) += mmiotrace.o
mmiotrace-y := kmmio.o pf_in.o mmio-mod.o
obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o
}
NOKPROBE_SYMBOL(vmalloc_fault);
-void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
+static void __arch_sync_kernel_mappings(unsigned long start, unsigned long end)
{
unsigned long addr;
}
}
+void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
+{
+ __arch_sync_kernel_mappings(start, end);
+#ifdef CONFIG_KMSAN
+ /*
+ * KMSAN maintains two additional metadata page mappings for the
+ * [VMALLOC_START, VMALLOC_END) range. These mappings start at
+ * KMSAN_VMALLOC_SHADOW_START and KMSAN_VMALLOC_ORIGIN_START and
+ * have to be synced together with the vmalloc memory mapping.
+ */
+ if (start >= VMALLOC_START && end < VMALLOC_END) {
+ __arch_sync_kernel_mappings(
+ start - VMALLOC_START + KMSAN_VMALLOC_SHADOW_START,
+ end - VMALLOC_START + KMSAN_VMALLOC_SHADOW_START);
+ __arch_sync_kernel_mappings(
+ start - VMALLOC_START + KMSAN_VMALLOC_ORIGIN_START,
+ end - VMALLOC_START + KMSAN_VMALLOC_ORIGIN_START);
+ }
+#endif
+}
+
static bool low_pfn(unsigned long pfn)
{
return pfn < max_low_pfn;
}
#ifdef CONFIG_SWAP
-unsigned long max_swapfile_size(void)
+unsigned long arch_max_swapfile_size(void)
{
unsigned long pages;
DEFINE_ENTRY(pmd, pmd, init)
DEFINE_ENTRY(pte, pte, init)
+static inline pgprot_t prot_sethuge(pgprot_t prot)
+{
+ WARN_ON_ONCE(pgprot_val(prot) & _PAGE_PAT);
+
+ return __pgprot(pgprot_val(prot) | _PAGE_PSE);
+}
/*
* NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
if (page_size_mask & (1<<PG_LEVEL_2M)) {
pages++;
spin_lock(&init_mm.page_table_lock);
- set_pte_init((pte_t *)pmd,
- pfn_pte((paddr & PMD_MASK) >> PAGE_SHIFT,
- __pgprot(pgprot_val(prot) | _PAGE_PSE)),
+ set_pmd_init(pmd,
+ pfn_pmd(paddr >> PAGE_SHIFT, prot_sethuge(prot)),
init);
spin_unlock(&init_mm.page_table_lock);
paddr_last = paddr_next;
if (page_size_mask & (1<<PG_LEVEL_1G)) {
pages++;
spin_lock(&init_mm.page_table_lock);
-
- prot = __pgprot(pgprot_val(prot) | _PAGE_PSE);
-
- set_pte_init((pte_t *)pud,
- pfn_pte((paddr & PUD_MASK) >> PAGE_SHIFT,
- prot),
+ set_pud_init(pud,
+ pfn_pud(paddr >> PAGE_SHIFT, prot_sethuge(prot)),
init);
spin_unlock(&init_mm.page_table_lock);
paddr_last = paddr_next;
unsigned long addr;
const char *lvl;
- for (addr = VMALLOC_START; addr <= VMALLOC_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
+ for (addr = VMALLOC_START; addr <= VMEMORY_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud;
#include <linux/cc_platform.h>
#include <linux/efi.h>
#include <linux/pgtable.h>
+#include <linux/kmsan.h>
#include <asm/set_memory.h>
#include <asm/e820/api.h>
return;
}
+ kmsan_iounmap_page_range((unsigned long)addr,
+ (unsigned long)addr + get_vm_area_size(p));
memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p));
/* Finally remove it */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * x86-specific bits of KMSAN shadow implementation.
+ *
+ * Copyright (C) 2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ */
+
+#include <asm/cpu_entry_area.h>
+#include <linux/percpu-defs.h>
+
+/*
+ * Addresses within the CPU entry area (including e.g. exception stacks) do not
+ * have struct page entries corresponding to them, so they need separate
+ * handling.
+ * arch_kmsan_get_meta_or_null() (declared in the header) maps the addresses in
+ * CPU entry area to addresses in cpu_entry_area_shadow/cpu_entry_area_origin.
+ */
+DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_shadow);
+DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_origin);
failed += print_split(&sa);
for (i = 0; i < NTEST; i++) {
- unsigned long pfn = prandom_u32() % max_pfn_mapped;
+ unsigned long pfn = prandom_u32_max(max_pfn_mapped);
addr[i] = (unsigned long)__va(pfn << PAGE_SHIFT);
- len[i] = prandom_u32() % NPAGES;
+ len[i] = prandom_u32_max(NPAGES);
len[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1);
if (len[i] == 0)
return __pgprot(pgprot_val(prot) & ~forbidden);
}
+/*
+ * Validate strict W^X semantics.
+ */
+static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long start,
+ unsigned long pfn, unsigned long npg)
+{
+ unsigned long end;
+
+ /*
+ * 32-bit has some unfixable W+X issues, like EFI code
+ * and writeable data being in the same page. Disable
+ * detection and enforcement there.
+ */
+ if (IS_ENABLED(CONFIG_X86_32))
+ return new;
+
+ /* Only verify when NX is supported: */
+ if (!(__supported_pte_mask & _PAGE_NX))
+ return new;
+
+ if (!((pgprot_val(old) ^ pgprot_val(new)) & (_PAGE_RW | _PAGE_NX)))
+ return new;
+
+ if ((pgprot_val(new) & (_PAGE_RW | _PAGE_NX)) != _PAGE_RW)
+ return new;
+
+ end = start + npg * PAGE_SIZE - 1;
+ WARN_ONCE(1, "CPA detected W^X violation: %016llx -> %016llx range: 0x%016lx - 0x%016lx PFN %lx\n",
+ (unsigned long long)pgprot_val(old),
+ (unsigned long long)pgprot_val(new),
+ start, end, pfn);
+
+ /*
+ * For now, allow all permission change attempts by returning the
+ * attempted permissions. This can 'return old' to actively
+ * refuse the permission change at a later time.
+ */
+ return new;
+}
+
/*
* Lookup the page table entry for a virtual address in a specific pgd.
* Return a pointer to the entry and the level of the mapping.
new_prot = static_protections(req_prot, lpaddr, old_pfn, numpages,
psize, CPA_DETECT);
+ new_prot = verify_rwx(old_prot, new_prot, lpaddr, old_pfn, numpages);
+
/*
* If there is a conflict, split the large page.
*
if (level == PG_LEVEL_4K) {
pte_t new_pte;
+ pgprot_t old_prot = pte_pgprot(old_pte);
pgprot_t new_prot = pte_pgprot(old_pte);
unsigned long pfn = pte_pfn(old_pte);
new_prot = static_protections(new_prot, address, pfn, 1, 0,
CPA_PROTECT);
+ new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1);
+
new_prot = pgprot_clear_protnone_bits(new_prot);
/*
return rc;
}
-static int set_memory_present(unsigned long *addr, int numpages)
+static int set_memory_p(unsigned long *addr, int numpages)
{
return change_page_attr_set(addr, numpages, __pgprot(_PAGE_PRESENT), 0);
}
{
unsigned long addr = (unsigned long) pfn_to_kaddr(pfn);
- return set_memory_present(&addr, 1);
+ return set_memory_p(&addr, 1);
}
EXPORT_SYMBOL_GPL(clear_mce_nospec);
#endif /* CONFIG_X86_64 */
return ret;
}
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pmd_t *pmdp)
{
return ret;
}
+#endif
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pudp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long addr, pud_t *pudp)
{
# Sanitizer runtimes are unavailable and cannot be linked here.
KASAN_SANITIZE := n
KCSAN_SANITIZE := n
+KMSAN_SANITIZE := n
OBJECT_FILES_NON_STANDARD := y
# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
select X86_X2APIC if XEN_PVH && X86_64
help
Support running as a Xen Dom0 guest.
+
+config XEN_PV_MSR_SAFE
+ bool "Always use safe MSR accesses in PV guests"
+ default y
+ depends on XEN_PV
+ help
+ Use safe (not faulting) MSR access functions even if the MSR access
+ should not fault anyway.
+ The default can be changed by using the "xen_msr_safe" boot parameter.
return;
if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT))
- virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc);
+ virtio_set_mem_acc_cb(xen_virtio_restricted_mem_acc);
init_hvm_pv_info();
*/
static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc);
+static __read_mostly bool xen_msr_safe = IS_ENABLED(CONFIG_XEN_PV_MSR_SAFE);
+
+static int __init parse_xen_msr_safe(char *str)
+{
+ if (str)
+ return strtobool(str, &xen_msr_safe);
+ return -EINVAL;
+}
+early_param("xen_msr_safe", parse_xen_msr_safe);
+
static void __init xen_pv_init_platform(void)
{
/* PV guests can't operate virtio devices without grants. */
if (IS_ENABLED(CONFIG_XEN_VIRTIO))
- virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc);
+ virtio_set_mem_acc_cb(xen_virtio_restricted_mem_acc);
populate_extra_pte(fix_to_virt(FIX_PARAVIRT_BOOTMAP));
native_write_cr4(cr4);
}
-static u64 xen_read_msr_safe(unsigned int msr, int *err)
+static u64 xen_do_read_msr(unsigned int msr, int *err)
{
- u64 val;
+ u64 val = 0; /* Avoid uninitialized value for safe variant. */
if (pmu_msr_read(msr, &val, err))
return val;
- val = native_read_msr_safe(msr, err);
+ if (err)
+ val = native_read_msr_safe(msr, err);
+ else
+ val = native_read_msr(msr);
+
switch (msr) {
case MSR_IA32_APICBASE:
val &= ~X2APIC_ENABLE;
return val;
}
-static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
+static void set_seg(unsigned int which, unsigned int low, unsigned int high,
+ int *err)
{
- int ret;
- unsigned int which;
- u64 base;
+ u64 base = ((u64)high << 32) | low;
- ret = 0;
+ if (HYPERVISOR_set_segment_base(which, base) == 0)
+ return;
+ if (err)
+ *err = -EIO;
+ else
+ WARN(1, "Xen set_segment_base(%u, %llx) failed\n", which, base);
+}
+
+/*
+ * Support write_msr_safe() and write_msr() semantics.
+ * With err == NULL write_msr() semantics are selected.
+ * Supplying an err pointer requires err to be pre-initialized with 0.
+ */
+static void xen_do_write_msr(unsigned int msr, unsigned int low,
+ unsigned int high, int *err)
+{
switch (msr) {
- case MSR_FS_BASE: which = SEGBASE_FS; goto set;
- case MSR_KERNEL_GS_BASE: which = SEGBASE_GS_USER; goto set;
- case MSR_GS_BASE: which = SEGBASE_GS_KERNEL; goto set;
-
- set:
- base = ((u64)high << 32) | low;
- if (HYPERVISOR_set_segment_base(which, base) != 0)
- ret = -EIO;
+ case MSR_FS_BASE:
+ set_seg(SEGBASE_FS, low, high, err);
+ break;
+
+ case MSR_KERNEL_GS_BASE:
+ set_seg(SEGBASE_GS_USER, low, high, err);
+ break;
+
+ case MSR_GS_BASE:
+ set_seg(SEGBASE_GS_KERNEL, low, high, err);
break;
case MSR_STAR:
break;
default:
- if (!pmu_msr_write(msr, low, high, &ret))
- ret = native_write_msr_safe(msr, low, high);
+ if (!pmu_msr_write(msr, low, high, err)) {
+ if (err)
+ *err = native_write_msr_safe(msr, low, high);
+ else
+ native_write_msr(msr, low, high);
+ }
}
+}
- return ret;
+static u64 xen_read_msr_safe(unsigned int msr, int *err)
+{
+ return xen_do_read_msr(msr, err);
+}
+
+static int xen_write_msr_safe(unsigned int msr, unsigned int low,
+ unsigned int high)
+{
+ int err = 0;
+
+ xen_do_write_msr(msr, low, high, &err);
+
+ return err;
}
static u64 xen_read_msr(unsigned int msr)
{
- /*
- * This will silently swallow a #GP from RDMSR. It may be worth
- * changing that.
- */
int err;
- return xen_read_msr_safe(msr, &err);
+ return xen_do_read_msr(msr, xen_msr_safe ? &err : NULL);
}
static void xen_write_msr(unsigned int msr, unsigned low, unsigned high)
{
- /*
- * This will silently swallow a #GP from WRMSR. It may be worth
- * changing that.
- */
- xen_write_msr_safe(msr, low, high);
+ int err;
+
+ xen_do_write_msr(msr, low, high, xen_msr_safe ? &err : NULL);
}
/* This is called once we have the cpu_possible_mask */
static inline bool is_amd_pmu_msr(unsigned int msr)
{
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
+ boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
+ return false;
+
if ((msr >= MSR_F15H_PERF_CTL &&
msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
(msr >= MSR_K7_EVNTSEL0 &&
return false;
}
-static int is_intel_pmu_msr(u32 msr_index, int *type, int *index)
+static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index)
{
u32 msr_index_pmc;
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
+ boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR &&
+ boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
+ return false;
+
switch (msr_index) {
case MSR_CORE_PERF_FIXED_CTR_CTRL:
case MSR_IA32_DS_AREA:
return false;
}
+static bool pmu_msr_chk_emulated(unsigned int msr, uint64_t *val, bool is_read,
+ bool *emul)
+{
+ int type, index;
+
+ if (is_amd_pmu_msr(msr))
+ *emul = xen_amd_pmu_emulate(msr, val, is_read);
+ else if (is_intel_pmu_msr(msr, &type, &index))
+ *emul = xen_intel_pmu_emulate(msr, val, type, index, is_read);
+ else
+ return false;
+
+ return true;
+}
+
bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
{
- if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
- if (is_amd_pmu_msr(msr)) {
- if (!xen_amd_pmu_emulate(msr, val, 1))
- *val = native_read_msr_safe(msr, err);
- return true;
- }
- } else {
- int type, index;
+ bool emulated;
- if (is_intel_pmu_msr(msr, &type, &index)) {
- if (!xen_intel_pmu_emulate(msr, val, type, index, 1))
- *val = native_read_msr_safe(msr, err);
- return true;
- }
+ if (!pmu_msr_chk_emulated(msr, val, true, &emulated))
+ return false;
+
+ if (!emulated) {
+ *val = err ? native_read_msr_safe(msr, err)
+ : native_read_msr(msr);
}
- return false;
+ return true;
}
bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
{
uint64_t val = ((uint64_t)high << 32) | low;
+ bool emulated;
- if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
- if (is_amd_pmu_msr(msr)) {
- if (!xen_amd_pmu_emulate(msr, &val, 0))
- *err = native_write_msr_safe(msr, low, high);
- return true;
- }
- } else {
- int type, index;
+ if (!pmu_msr_chk_emulated(msr, &val, false, &emulated))
+ return false;
- if (is_intel_pmu_msr(msr, &type, &index)) {
- if (!xen_intel_pmu_emulate(msr, &val, type, index, 0))
- *err = native_write_msr_safe(msr, low, high);
- return true;
- }
+ if (!emulated) {
+ if (err)
+ *err = native_write_msr_safe(msr, low, high);
+ else
+ native_write_msr(msr, low, high);
}
- return false;
+ return true;
}
static unsigned long long xen_amd_read_pmc(int counter)
If unsure, say Y.
-config FORCE_MAX_ZONEORDER
+config ARCH_FORCE_MAX_ORDER
int "Maximum zone order"
default "11"
help
CONFIG_NLS_ISO8859_1=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_LOCKUP_DETECTOR=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_NLS_ISO8859_1=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_LOCKUP_DETECTOR=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_NLS_ISO8859_1=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_LOCKUP_DETECTOR=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_NLS_ISO8859_1=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
# CONFIG_FRAME_POINTER is not set
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_VM=y
CONFIG_NLS_ISO8859_1=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DEBUG_VM=y
CONFIG_LOCKUP_DETECTOR=y
CONFIG_FONTS=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_MAGIC_SYSRQ=y
# CONFIG_SCHED_DEBUG is not set
CONFIG_SCHEDSTATS=y
CONFIG_CRYPTO_ANSI_CPRNG=y
CONFIG_PRINTK_TIME=y
CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_DETECT_HUNG_TASK=y
# CONFIG_SCHED_DEBUG is not set
#define elf_check_arch(x) ( ( (x)->e_machine == EM_XTENSA ) || \
( (x)->e_machine == EM_XTENSA_OLD ) )
+#define ELFOSABI_XTENSA_FDPIC 65
+#define elf_check_fdpic(x) ((x)->e_ident[EI_OSABI] == ELFOSABI_XTENSA_FDPIC)
+#define ELF_FDPIC_CORE_EFLAGS 0
+
/*
* These are used to set parameters in the core dumps.
*/
*/
#define ELF_PLAT_INIT(_r, load_addr) \
- do { _r->areg[0]=0; /*_r->areg[1]=0;*/ _r->areg[2]=0; _r->areg[3]=0; \
- _r->areg[4]=0; _r->areg[5]=0; _r->areg[6]=0; _r->areg[7]=0; \
- _r->areg[8]=0; _r->areg[9]=0; _r->areg[10]=0; _r->areg[11]=0; \
- _r->areg[12]=0; _r->areg[13]=0; _r->areg[14]=0; _r->areg[15]=0; \
+ do { \
+ (_r)->areg[0] = 0; /*(_r)->areg[1] = 0;*/ \
+ (_r)->areg[2] = 0; (_r)->areg[3] = 0; \
+ (_r)->areg[4] = 0; (_r)->areg[5] = 0; \
+ (_r)->areg[6] = 0; (_r)->areg[7] = 0; \
+ (_r)->areg[8] = 0; (_r)->areg[9] = 0; \
+ (_r)->areg[10] = 0; (_r)->areg[11] = 0; \
+ (_r)->areg[12] = 0; (_r)->areg[13] = 0; \
+ (_r)->areg[14] = 0; (_r)->areg[15] = 0; \
+ } while (0)
+
+#define ELF_FDPIC_PLAT_INIT(_r, _exec_map_addr, _interp_map_addr, dynamic_addr) \
+ do { \
+ (_r)->areg[4] = _exec_map_addr; \
+ (_r)->areg[5] = _interp_map_addr; \
+ (_r)->areg[6] = dynamic_addr; \
} while (0)
typedef struct {
#define start_thread(regs, new_pc, new_sp) \
do { \
unsigned long syscall = (regs)->syscall; \
+ unsigned long current_aregs[16]; \
+ memcpy(current_aregs, (regs)->areg, sizeof(current_aregs)); \
memset((regs), 0, sizeof(*(regs))); \
(regs)->pc = (new_pc); \
(regs)->ps = USER_PS_VALUE; \
+ memcpy((regs)->areg, current_aregs, sizeof(current_aregs)); \
(regs)->areg[1] = (new_sp); \
(regs)->areg[0] = 0; \
(regs)->wmask = 1; \
struct task_struct;
struct mm_struct;
-/* Free all resources held by a thread. */
-#define release_thread(thread) do { } while(0)
-
extern unsigned long __get_wchan(struct task_struct *p);
#define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc)
#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */
+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */
+
/* compatibility flags */
#define MAP_FILE 0
#define PTRACE_SETXTREGS 19
#define PTRACE_GETHBPREGS 20
#define PTRACE_SETHBPREGS 21
+#define PTRACE_GETFDPIC 22
+
+#define PTRACE_GETFDPIC_EXEC 0
+#define PTRACE_GETFDPIC_INTERP 1
#ifndef __ASSEMBLY__
unsigned long len, unsigned long pgoff, unsigned long flags)
{
struct vm_area_struct *vmm;
+ struct vma_iterator vmi;
if (flags & MAP_FIXED) {
/* We do not accept a shared mapping if it would violate
else
addr = PAGE_ALIGN(addr);
- for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) {
- /* At this point: (!vmm || addr < vmm->vm_end). */
- if (TASK_SIZE - len < addr)
- return -ENOMEM;
- if (!vmm || addr + len <= vm_start_gap(vmm))
- return addr;
+ vma_iter_init(&vmi, current->mm, addr);
+ for_each_vma(vmi, vmm) {
+ /* At this point: (addr < vmm->vm_end). */
+ if (addr + len <= vm_start_gap(vmm))
+ break;
+
addr = vmm->vm_end;
if (flags & MAP_SHARED)
addr = COLOUR_ALIGN(addr, pgoff);
}
+
+ if (TASK_SIZE - len < addr)
+ return -ENOMEM;
+
+ return addr;
}
#endif
* be reused by calling bio_uninit() before calling bio_init() again.
*
* Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this
- * function are not backed by a mempool can can fail. Do not use this function
+ * function are not backed by a mempool can fail. Do not use this function
* for allocations in the file system I/O path.
*
* Returns: Pointer to new bio on success, NULL on failure.
*same_page = ((vec_end_addr & PAGE_MASK) == page_addr);
if (*same_page)
return true;
+ else if (IS_ENABLED(CONFIG_KMSAN))
+ return false;
return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE);
}
if (blk_crypto_fallback_inited)
return 0;
- prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
+ get_random_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
err = bioset_init(&crypto_bio_split, 64, 0, 0);
if (err)
rwb->last_comp = rwb->last_issue = jiffies;
rwb->win_nsec = RWB_WINDOW_NSEC;
rwb->enable_state = WBT_STATE_ON_DEFAULT;
- rwb->wc = 1;
+ rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags);
rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
rwb->min_lat_nsec = wbt_default_latency_nsec(q);
wbt_queue_depth_changed(&rwb->rqos);
- wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags));
/*
* Assign rwb and add the stats callback.
phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset;
phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset;
+ /*
+ * Merging adjacent physical pages may not work correctly under KMSAN
+ * if their metadata pages aren't adjacent. Just disable merging.
+ */
+ if (IS_ENABLED(CONFIG_KMSAN))
+ return false;
+
if (addr1 + vec1->bv_len != addr2)
return false;
if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
*/
dev_set_uevent_suppress(ddev, 0);
disk_uevent(disk, KOBJ_ADD);
+ } else {
+ /*
+ * Even if the block_device for a hidden gendisk is not
+ * registered, it needs to have a valid bd_dev so that the
+ * freeing of the dynamic major works.
+ */
+ disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor);
}
disk_update_readahead(disk);
config CRYPTO_HASH_INFO
bool
+if !KMSAN # avoid false positives from assembly
if ARM
source "arch/arm/crypto/Kconfig"
endif
if X86
source "arch/x86/crypto/Kconfig"
endif
+endif
source "drivers/crypto/Kconfig"
source "crypto/asymmetric_keys/Kconfig"
int i;
for (i = 0; i < disks; i++) {
- prandom_bytes(page_address(data[i]), PAGE_SIZE);
+ get_random_bytes(page_address(data[i]), PAGE_SIZE);
dataptrs[i] = data[i];
dataoffs[i] = 0;
}
/* Generate a random length in range [0, max_len], but prefer smaller values */
static unsigned int generate_random_length(unsigned int max_len)
{
- unsigned int len = prandom_u32() % (max_len + 1);
+ unsigned int len = prandom_u32_max(max_len + 1);
- switch (prandom_u32() % 4) {
+ switch (prandom_u32_max(4)) {
case 0:
return len % 64;
case 1:
{
size_t bitpos;
- bitpos = prandom_u32() % (size * 8);
+ bitpos = prandom_u32_max(size * 8);
buf[bitpos / 8] ^= 1 << (bitpos % 8);
}
/* Flip a random byte in the given nonempty data buffer */
static void flip_random_byte(u8 *buf, size_t size)
{
- buf[prandom_u32() % size] ^= 0xff;
+ buf[prandom_u32_max(size)] ^= 0xff;
}
/* Sometimes make some random changes to the given nonempty data buffer */
size_t i;
/* Sometimes flip some bits */
- if (prandom_u32() % 4 == 0) {
- num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size * 8);
+ if (prandom_u32_max(4) == 0) {
+ num_flips = min_t(size_t, 1 << prandom_u32_max(8), size * 8);
for (i = 0; i < num_flips; i++)
flip_random_bit(buf, size);
}
/* Sometimes flip some bytes */
- if (prandom_u32() % 4 == 0) {
- num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size);
+ if (prandom_u32_max(4) == 0) {
+ num_flips = min_t(size_t, 1 << prandom_u32_max(8), size);
for (i = 0; i < num_flips; i++)
flip_random_byte(buf, size);
}
if (count == 0)
return;
- switch (prandom_u32() % 8) { /* Choose a generation strategy */
+ switch (prandom_u32_max(8)) { /* Choose a generation strategy */
case 0:
case 1:
/* All the same byte, plus optional mutations */
- switch (prandom_u32() % 4) {
+ switch (prandom_u32_max(4)) {
case 0:
b = 0x00;
break;
b = 0xff;
break;
default:
- b = (u8)prandom_u32();
+ b = get_random_u8();
break;
}
memset(buf, b, count);
break;
case 2:
/* Ascending or descending bytes, plus optional mutations */
- increment = (u8)prandom_u32();
- b = (u8)prandom_u32();
+ increment = get_random_u8();
+ b = get_random_u8();
for (i = 0; i < count; i++, b += increment)
buf[i] = b;
mutate_buffer(buf, count);
default:
/* Fully random bytes */
for (i = 0; i < count; i++)
- buf[i] = (u8)prandom_u32();
+ buf[i] = get_random_u8();
}
}
unsigned int this_len;
const char *flushtype_str;
- if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0)
+ if (div == &divs[max_divs - 1] || prandom_u32_max(2) == 0)
this_len = remaining;
else
- this_len = 1 + (prandom_u32() % remaining);
+ this_len = 1 + prandom_u32_max(remaining);
div->proportion_of_total = this_len;
- if (prandom_u32() % 4 == 0)
- div->offset = (PAGE_SIZE - 128) + (prandom_u32() % 128);
- else if (prandom_u32() % 2 == 0)
- div->offset = prandom_u32() % 32;
+ if (prandom_u32_max(4) == 0)
+ div->offset = (PAGE_SIZE - 128) + prandom_u32_max(128);
+ else if (prandom_u32_max(2) == 0)
+ div->offset = prandom_u32_max(32);
else
- div->offset = prandom_u32() % PAGE_SIZE;
- if (prandom_u32() % 8 == 0)
+ div->offset = prandom_u32_max(PAGE_SIZE);
+ if (prandom_u32_max(8) == 0)
div->offset_relative_to_alignmask = true;
div->flush_type = FLUSH_TYPE_NONE;
if (gen_flushes) {
- switch (prandom_u32() % 4) {
+ switch (prandom_u32_max(4)) {
case 0:
div->flush_type = FLUSH_TYPE_REIMPORT;
break;
if (div->flush_type != FLUSH_TYPE_NONE &&
!(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
- prandom_u32() % 2 == 0)
+ prandom_u32_max(2) == 0)
div->nosimd = true;
switch (div->flush_type) {
p += scnprintf(p, end - p, "random:");
- switch (prandom_u32() % 4) {
+ switch (prandom_u32_max(4)) {
case 0:
case 1:
cfg->inplace_mode = OUT_OF_PLACE;
break;
}
- if (prandom_u32() % 2 == 0) {
+ if (prandom_u32_max(2) == 0) {
cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
p += scnprintf(p, end - p, " may_sleep");
}
- switch (prandom_u32() % 4) {
+ switch (prandom_u32_max(4)) {
case 0:
cfg->finalization_type = FINALIZATION_TYPE_FINAL;
p += scnprintf(p, end - p, " use_final");
}
if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
- prandom_u32() % 2 == 0) {
+ prandom_u32_max(2) == 0) {
cfg->nosimd = true;
p += scnprintf(p, end - p, " nosimd");
}
cfg->req_flags);
p += scnprintf(p, end - p, "]");
- if (cfg->inplace_mode == OUT_OF_PLACE && prandom_u32() % 2 == 0) {
+ if (cfg->inplace_mode == OUT_OF_PLACE && prandom_u32_max(2) == 0) {
p += scnprintf(p, end - p, " dst_divs=[");
p = generate_random_sgl_divisions(cfg->dst_divs,
ARRAY_SIZE(cfg->dst_divs),
p += scnprintf(p, end - p, "]");
}
- if (prandom_u32() % 2 == 0) {
- cfg->iv_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
+ if (prandom_u32_max(2) == 0) {
+ cfg->iv_offset = 1 + prandom_u32_max(MAX_ALGAPI_ALIGNMASK);
p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
}
- if (prandom_u32() % 2 == 0) {
- cfg->key_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
+ if (prandom_u32_max(2) == 0) {
+ cfg->key_offset = 1 + prandom_u32_max(MAX_ALGAPI_ALIGNMASK);
p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);
}
vec->ksize = 0;
if (maxkeysize) {
vec->ksize = maxkeysize;
- if (prandom_u32() % 4 == 0)
- vec->ksize = 1 + (prandom_u32() % maxkeysize);
+ if (prandom_u32_max(4) == 0)
+ vec->ksize = 1 + prandom_u32_max(maxkeysize);
generate_random_bytes((u8 *)vec->key, vec->ksize);
vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
const unsigned int authsize = vec->clen - vec->plen;
- if (prandom_u32() % 2 == 0 && vec->alen > aad_tail_size) {
+ if (prandom_u32_max(2) == 0 && vec->alen > aad_tail_size) {
/* Mutate the AAD */
flip_random_bit((u8 *)vec->assoc, vec->alen - aad_tail_size);
- if (prandom_u32() % 2 == 0)
+ if (prandom_u32_max(2) == 0)
return;
}
- if (prandom_u32() % 2 == 0) {
+ if (prandom_u32_max(2) == 0) {
/* Mutate auth tag (assuming it's at the end of ciphertext) */
flip_random_bit((u8 *)vec->ctext + vec->plen, authsize);
} else {
const unsigned int ivsize = crypto_aead_ivsize(tfm);
const unsigned int authsize = vec->clen - vec->plen;
const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
- (prefer_inauthentic || prandom_u32() % 4 == 0);
+ (prefer_inauthentic || prandom_u32_max(4) == 0);
/* Generate the AAD. */
generate_random_bytes((u8 *)vec->assoc, vec->alen);
/* Avoid implementation-defined behavior. */
memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
- if (inauthentic && prandom_u32() % 2 == 0) {
+ if (inauthentic && prandom_u32_max(2) == 0) {
/* Generate a random ciphertext. */
generate_random_bytes((u8 *)vec->ctext, vec->clen);
} else {
/* Key: length in [0, maxkeysize], but usually choose maxkeysize */
vec->klen = maxkeysize;
- if (prandom_u32() % 4 == 0)
- vec->klen = prandom_u32() % (maxkeysize + 1);
+ if (prandom_u32_max(4) == 0)
+ vec->klen = prandom_u32_max(maxkeysize + 1);
generate_random_bytes((u8 *)vec->key, vec->klen);
vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
/* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
authsize = maxauthsize;
- if (prandom_u32() % 4 == 0)
- authsize = prandom_u32() % (maxauthsize + 1);
+ if (prandom_u32_max(4) == 0)
+ authsize = prandom_u32_max(maxauthsize + 1);
if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
authsize = MIN_COLLISION_FREE_AUTHSIZE;
if (WARN_ON(authsize > maxdatasize))
/* AAD, plaintext, and ciphertext lengths */
total_len = generate_random_length(maxdatasize);
- if (prandom_u32() % 4 == 0)
+ if (prandom_u32_max(4) == 0)
vec->alen = 0;
else
vec->alen = generate_random_length(total_len);
/* Key: length in [0, maxkeysize], but usually choose maxkeysize */
vec->klen = maxkeysize;
- if (prandom_u32() % 4 == 0)
- vec->klen = prandom_u32() % (maxkeysize + 1);
+ if (prandom_u32_max(4) == 0)
+ vec->klen = prandom_u32_max(maxkeysize + 1);
generate_random_bytes((u8 *)vec->key, vec->klen);
vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
#include <linux/ratelimit.h>
#include <linux/edac.h>
#include <linux/ras.h>
+#include <acpi/ghes.h>
#include <asm/cpu.h>
#include <asm/mce.h>
int cpu = mce->extcpu;
struct acpi_hest_generic_status *estatus, *tmp;
struct acpi_hest_generic_data *gdata;
- const guid_t *fru_id = &guid_null;
- char *fru_text = "";
+ const guid_t *fru_id;
+ char *fru_text;
guid_t *sec_type;
static u32 err_seq;
/* log event via trace */
err_seq++;
- gdata = (struct acpi_hest_generic_data *)(tmp + 1);
- if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID)
- fru_id = (guid_t *)gdata->fru_id;
- if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT)
- fru_text = gdata->fru_text;
- sec_type = (guid_t *)gdata->section_type;
- if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) {
- struct cper_sec_mem_err *mem = (void *)(gdata + 1);
- if (gdata->error_data_length >= sizeof(*mem))
- trace_extlog_mem_event(mem, err_seq, fru_id, fru_text,
- (u8)gdata->error_severity);
+ apei_estatus_for_each_section(tmp, gdata) {
+ if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID)
+ fru_id = (guid_t *)gdata->fru_id;
+ else
+ fru_id = &guid_null;
+ if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT)
+ fru_text = gdata->fru_text;
+ else
+ fru_text = "";
+ sec_type = (guid_t *)gdata->section_type;
+ if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) {
+ struct cper_sec_mem_err *mem = (void *)(gdata + 1);
+
+ if (gdata->error_data_length >= sizeof(*mem))
+ trace_extlog_mem_event(mem, err_seq, fru_id, fru_text,
+ (u8)gdata->error_severity);
+ }
}
out:
list_for_each_entry(pn, &adev->physical_node_list, node) {
if (dev_is_pci(pn->dev)) {
+ get_device(pn->dev);
pci_dev = to_pci_dev(pn->dev);
break;
}
{ }
};
+static const struct dmi_system_id lenovo_82ra[] = {
+ {
+ .ident = "LENOVO IdeaPad Flex 5 16ALC7",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "82RA"),
+ },
+ },
+ { }
+};
+
struct irq_override_cmp {
const struct dmi_system_id *system;
unsigned char irq;
unsigned char triggering;
unsigned char polarity;
unsigned char shareable;
+ bool override;
};
-static const struct irq_override_cmp skip_override_table[] = {
- { medion_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 },
- { asus_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 },
+static const struct irq_override_cmp override_table[] = {
+ { medion_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, false },
+ { asus_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, false },
+ { lenovo_82ra, 6, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, true },
+ { lenovo_82ra, 10, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, true },
};
static bool acpi_dev_irq_override(u32 gsi, u8 triggering, u8 polarity,
{
int i;
+ for (i = 0; i < ARRAY_SIZE(override_table); i++) {
+ const struct irq_override_cmp *entry = &override_table[i];
+
+ if (dmi_check_system(entry->system) &&
+ entry->irq == gsi &&
+ entry->triggering == triggering &&
+ entry->polarity == polarity &&
+ entry->shareable == shareable)
+ return entry->override;
+ }
+
#ifdef CONFIG_X86
/*
* IRQ override isn't needed on modern AMD Zen systems and
return false;
#endif
- for (i = 0; i < ARRAY_SIZE(skip_override_table); i++) {
- const struct irq_override_cmp *entry = &skip_override_table[i];
-
- if (dmi_check_system(entry->system) &&
- entry->irq == gsi &&
- entry->triggering == triggering &&
- entry->polarity == polarity &&
- entry->shareable == shareable)
- return false;
- }
-
return true;
}
u8 pol = p ? ACPI_ACTIVE_LOW : ACPI_ACTIVE_HIGH;
if (triggering != trig || polarity != pol) {
- pr_warn("ACPI: IRQ %d override to %s, %s\n", gsi,
- t ? "level" : "edge", p ? "low" : "high");
+ pr_warn("ACPI: IRQ %d override to %s%s, %s%s\n", gsi,
+ t ? "level" : "edge",
+ trig == triggering ? "" : "(!)",
+ p ? "low" : "high",
+ pol == polarity ? "" : "(!)");
triggering = trig;
polarity = pol;
}
goto out;
}
+ *map = r;
+
list_for_each_entry(rentry, &list, node) {
if (rentry->res->start >= rentry->res->end) {
- kfree(r);
+ kfree(*map);
+ *map = NULL;
ret = -EINVAL;
dev_dbg(dma_dev, "Invalid DMA regions configuration\n");
goto out;
r->offset = rentry->offset;
r++;
}
-
- *map = r;
}
out:
acpi_dev_free_resource_list(&list);
}
}
-/* return true if the memory block is offlined, otherwise, return false */
-bool is_memblock_offlined(struct memory_block *mem)
-{
- return mem->state == MEM_OFFLINE;
-}
-
static struct attribute *memory_root_attrs[] = {
#ifdef CONFIG_ARCH_MEMORY_PROBE
&dev_attr_probe.attr,
#include <linux/pm_runtime.h>
#include <linux/swap.h>
#include <linux/slab.h>
+#include <linux/hugetlb.h>
static struct bus_type node_subsys = {
.name = "node",
NULL
};
-#ifdef CONFIG_HUGETLBFS
-/*
- * hugetlbfs per node attributes registration interface:
- * When/if hugetlb[fs] subsystem initializes [sometime after this module],
- * it will register its per node attributes for all online nodes with
- * memory. It will also call register_hugetlbfs_with_node(), below, to
- * register its attribute registration functions with this node driver.
- * Once these hooks have been initialized, the node driver will call into
- * the hugetlb module to [un]register attributes for hot-plugged nodes.
- */
-static node_registration_func_t __hugetlb_register_node;
-static node_registration_func_t __hugetlb_unregister_node;
-
-static inline bool hugetlb_register_node(struct node *node)
-{
- if (__hugetlb_register_node &&
- node_state(node->dev.id, N_MEMORY)) {
- __hugetlb_register_node(node);
- return true;
- }
- return false;
-}
-
-static inline void hugetlb_unregister_node(struct node *node)
-{
- if (__hugetlb_unregister_node)
- __hugetlb_unregister_node(node);
-}
-
-void register_hugetlbfs_with_node(node_registration_func_t doregister,
- node_registration_func_t unregister)
-{
- __hugetlb_register_node = doregister;
- __hugetlb_unregister_node = unregister;
-}
-#else
-static inline void hugetlb_register_node(struct node *node) {}
-
-static inline void hugetlb_unregister_node(struct node *node) {}
-#endif
-
static void node_device_release(struct device *dev)
{
- struct node *node = to_node(dev);
-
-#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_HUGETLBFS)
- /*
- * We schedule the work only when a memory section is
- * onlined/offlined on this node. When we come here,
- * all the memory on this node has been offlined,
- * so we won't enqueue new work to this work.
- *
- * The work is using node->node_work, so we should
- * flush work before freeing the memory.
- */
- flush_work(&node->node_work);
-#endif
- kfree(node);
+ kfree(to_node(dev));
}
/*
node->dev.groups = node_dev_groups;
error = device_register(&node->dev);
- if (error)
+ if (error) {
put_device(&node->dev);
- else {
+ } else {
hugetlb_register_node(node);
-
compaction_register_node(node);
}
+
return error;
}
*/
void unregister_node(struct node *node)
{
+ hugetlb_unregister_node(node);
compaction_unregister_node(node);
- hugetlb_unregister_node(node); /* no-op, if memoryless node */
node_remove_accesses(node);
node_remove_caches(node);
device_unregister(&node->dev);
(void *)&nid, func);
return;
}
-
-#ifdef CONFIG_HUGETLBFS
-/*
- * Handle per node hstate attribute [un]registration on transistions
- * to/from memoryless state.
- */
-static void node_hugetlb_work(struct work_struct *work)
-{
- struct node *node = container_of(work, struct node, node_work);
-
- /*
- * We only get here when a node transitions to/from memoryless state.
- * We can detect which transition occurred by examining whether the
- * node has memory now. hugetlb_register_node() already check this
- * so we try to register the attributes. If that fails, then the
- * node has transitioned to memoryless, try to unregister the
- * attributes.
- */
- if (!hugetlb_register_node(node))
- hugetlb_unregister_node(node);
-}
-
-static void init_node_hugetlb_work(int nid)
-{
- INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work);
-}
-
-static int node_memory_callback(struct notifier_block *self,
- unsigned long action, void *arg)
-{
- struct memory_notify *mnb = arg;
- int nid = mnb->status_change_nid;
-
- switch (action) {
- case MEM_ONLINE:
- case MEM_OFFLINE:
- /*
- * offload per node hstate [un]registration to a work thread
- * when transitioning to/from memoryless state.
- */
- if (nid != NUMA_NO_NODE)
- schedule_work(&node_devices[nid]->node_work);
- break;
-
- case MEM_GOING_ONLINE:
- case MEM_GOING_OFFLINE:
- case MEM_CANCEL_ONLINE:
- case MEM_CANCEL_OFFLINE:
- default:
- break;
- }
-
- return NOTIFY_OK;
-}
-#endif /* CONFIG_HUGETLBFS */
#endif /* CONFIG_MEMORY_HOTPLUG */
-#if !defined(CONFIG_MEMORY_HOTPLUG) || !defined(CONFIG_HUGETLBFS)
-static inline int node_memory_callback(struct notifier_block *self,
- unsigned long action, void *arg)
-{
- return NOTIFY_OK;
-}
-
-static void init_node_hugetlb_work(int nid) { }
-
-#endif
-
int __register_one_node(int nid)
{
int error;
}
INIT_LIST_HEAD(&node_devices[nid]->access_list);
- /* initialize work queue for memory hot plug */
- init_node_hugetlb_work(nid);
node_init_caches(nid);
return error;
NULL,
};
-#define NODE_CALLBACK_PRI 2 /* lower than SLAB */
void __init node_dev_init(void)
{
- static struct notifier_block node_memory_callback_nb = {
- .notifier_call = node_memory_callback,
- .priority = NODE_CALLBACK_PRI,
- };
int ret, i;
BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
if (ret)
panic("%s() failed to register subsystem: %d\n", __func__, ret);
- register_hotmemory_notifier(&node_memory_callback_nb);
-
/*
* Create all node devices, which will properly link the node
* to applicable memory block devices and already created cpu devices.
return domain;
}
+EXPORT_SYMBOL_GPL(platform_msi_create_irq_domain);
static int platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec,
irq_write_msi_msg_t write_msi_msg)
/* Always-on domains must be powered on at initialization. */
if ((genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd)) &&
- !genpd_status_on(genpd))
+ !genpd_status_on(genpd)) {
+ pr_err("always-on PM domain %s is not on\n", genpd->name);
return -EINVAL;
+ }
/* Multiple states but no governor doesn't make sense. */
if (!gov && genpd->state_count > 1)
return IRQ_NONE;
for_each_set_bit(gpio, &irqs, gc->ngpio)
- generic_handle_irq(irq_find_mapping(gc->irq.domain, gpio));
+ generic_handle_domain_irq_safe(gc->irq.domain, gpio);
bcma_chipco_gpio_polarity(cc, irqs, val & irqs);
return IRQ_HANDLED;
timeo = connect_int * HZ;
/* 28.5% random jitter */
- timeo += (prandom_u32() & 1) ? timeo / 7 : -timeo / 7;
+ timeo += prandom_u32_max(2) ? timeo / 7 : -timeo / 7;
err = wait_for_completion_interruptible_timeout(&ad->door_bell, timeo);
if (err <= 0)
drbd_warn(connection, "Error receiving initial packet\n");
sock_release(s);
randomize:
- if (prandom_u32() & 1)
+ if (prandom_u32_max(2))
goto retry;
}
}
return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);
}
-static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap)
+static int virtblk_setup_discard_write_zeroes_erase(struct request *req, bool unmap)
{
unsigned short segments = blk_rq_nr_discard_segments(req);
unsigned short n = 0;
type = VIRTIO_BLK_T_WRITE_ZEROES;
unmap = !(req->cmd_flags & REQ_NOUNMAP);
break;
+ case REQ_OP_SECURE_ERASE:
+ type = VIRTIO_BLK_T_SECURE_ERASE;
+ break;
case REQ_OP_DRV_IN:
type = VIRTIO_BLK_T_GET_ID;
break;
vbr->out_hdr.type = cpu_to_virtio32(vdev, type);
vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req));
- if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) {
- if (virtblk_setup_discard_write_zeroes(req, unmap))
+ if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES ||
+ type == VIRTIO_BLK_T_SECURE_ERASE) {
+ if (virtblk_setup_discard_write_zeroes_erase(req, unmap))
return BLK_STS_RESOURCE;
}
int err, index;
u32 v, blk_size, max_size, sg_elems, opt_io_size;
+ u32 max_discard_segs = 0;
+ u32 discard_granularity = 0;
u16 min_io_size;
u8 physical_block_exp, alignment_offset;
unsigned int queue_depth;
if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) {
virtio_cread(vdev, struct virtio_blk_config,
- discard_sector_alignment, &v);
- if (v)
- q->limits.discard_granularity = v << SECTOR_SHIFT;
- else
- q->limits.discard_granularity = blk_size;
+ discard_sector_alignment, &discard_granularity);
virtio_cread(vdev, struct virtio_blk_config,
max_discard_sectors, &v);
blk_queue_max_discard_sectors(q, v ? v : UINT_MAX);
virtio_cread(vdev, struct virtio_blk_config, max_discard_seg,
- &v);
-
- /*
- * max_discard_seg == 0 is out of spec but we always
- * handled it.
- */
- if (!v)
- v = sg_elems;
- blk_queue_max_discard_segments(q,
- min(v, MAX_DISCARD_SEGMENTS));
+ &max_discard_segs);
}
if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) {
blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX);
}
+ /* The discard and secure erase limits are combined since the Linux
+ * block layer uses the same limit for both commands.
+ *
+ * If both VIRTIO_BLK_F_SECURE_ERASE and VIRTIO_BLK_F_DISCARD features
+ * are negotiated, we will use the minimum between the limits.
+ *
+ * discard sector alignment is set to the minimum between discard_sector_alignment
+ * and secure_erase_sector_alignment.
+ *
+ * max discard sectors is set to the minimum between max_discard_seg and
+ * max_secure_erase_seg.
+ */
+ if (virtio_has_feature(vdev, VIRTIO_BLK_F_SECURE_ERASE)) {
+
+ virtio_cread(vdev, struct virtio_blk_config,
+ secure_erase_sector_alignment, &v);
+
+ /* secure_erase_sector_alignment should not be zero, the device should set a
+ * valid number of sectors.
+ */
+ if (!v) {
+ dev_err(&vdev->dev,
+ "virtio_blk: secure_erase_sector_alignment can't be 0\n");
+ err = -EINVAL;
+ goto out_cleanup_disk;
+ }
+
+ discard_granularity = min_not_zero(discard_granularity, v);
+
+ virtio_cread(vdev, struct virtio_blk_config,
+ max_secure_erase_sectors, &v);
+
+ /* max_secure_erase_sectors should not be zero, the device should set a
+ * valid number of sectors.
+ */
+ if (!v) {
+ dev_err(&vdev->dev,
+ "virtio_blk: max_secure_erase_sectors can't be 0\n");
+ err = -EINVAL;
+ goto out_cleanup_disk;
+ }
+
+ blk_queue_max_secure_erase_sectors(q, v);
+
+ virtio_cread(vdev, struct virtio_blk_config,
+ max_secure_erase_seg, &v);
+
+ /* max_secure_erase_seg should not be zero, the device should set a
+ * valid number of segments
+ */
+ if (!v) {
+ dev_err(&vdev->dev,
+ "virtio_blk: max_secure_erase_seg can't be 0\n");
+ err = -EINVAL;
+ goto out_cleanup_disk;
+ }
+
+ max_discard_segs = min_not_zero(max_discard_segs, v);
+ }
+
+ if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD) ||
+ virtio_has_feature(vdev, VIRTIO_BLK_F_SECURE_ERASE)) {
+ /* max_discard_seg and discard_granularity will be 0 only
+ * if max_discard_seg and discard_sector_alignment fields in the virtio
+ * config are 0 and VIRTIO_BLK_F_SECURE_ERASE feature is not negotiated.
+ * In this case, we use default values.
+ */
+ if (!max_discard_segs)
+ max_discard_segs = sg_elems;
+
+ blk_queue_max_discard_segments(q,
+ min(max_discard_segs, MAX_DISCARD_SEGMENTS));
+
+ if (discard_granularity)
+ q->limits.discard_granularity = discard_granularity << SECTOR_SHIFT;
+ else
+ q->limits.discard_granularity = blk_size;
+ }
+
virtblk_update_capacity(vblk, false);
virtio_device_ready(vdev);
VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
+ VIRTIO_BLK_F_SECURE_ERASE,
}
;
static unsigned int features[] = {
VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
+ VIRTIO_BLK_F_SECURE_ERASE,
};
static struct virtio_driver virtio_blk = {
static size_t huge_class_size;
static const struct block_device_operations zram_devops;
-#ifdef CONFIG_ZRAM_WRITEBACK
-static const struct block_device_operations zram_wb_devops;
-#endif
static void zram_free_page(struct zram *zram, size_t index);
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
if (!sysfs_streq(buf, "all")) {
/*
- * If it did not parse as 'all' try to treat it as an integer when
- * we have memory tracking enabled.
+ * If it did not parse as 'all' try to treat it as an integer
+ * when we have memory tracking enabled.
*/
u64 age_sec;
if (!init_done(zram))
goto out_unlock;
- /* A cutoff_time of 0 marks everything as idle, this is the "all" behavior */
+ /*
+ * A cutoff_time of 0 marks everything as idle, this is the
+ * "all" behavior.
+ */
mark_idle(zram, cutoff_time);
rv = len;
zram->backing_dev = backing_dev;
zram->bitmap = bitmap;
zram->nr_pages = nr_pages;
- /*
- * With writeback feature, zram does asynchronous IO so it's no longer
- * synchronous device so let's remove synchronous io flag. Othewise,
- * upper layer(e.g., swap) could wait IO completion rather than
- * (submit and return), which will cause system sluggish.
- * Furthermore, when the IO function returns(e.g., swap_readpage),
- * upper layer expects IO was done so it could deallocate the page
- * freely but in fact, IO is going on so finally could cause
- * use-after-free when the IO is really done.
- */
- zram->disk->fops = &zram_wb_devops;
up_write(&zram->init_lock);
pr_info("setup backing device %s\n", file_name);
struct bio_vec bvec;
zram_slot_unlock(zram, index);
+ /* A null bio means rw_page was used, we must fallback to bio */
+ if (!bio)
+ return -EOPNOTSUPP;
bvec.bv_page = page;
bvec.bv_len = PAGE_SIZE;
handle = zs_malloc(zram->mem_pool, comp_len,
GFP_NOIO | __GFP_HIGHMEM |
__GFP_MOVABLE);
- if (!IS_ERR((void *)handle))
+ if (IS_ERR((void *)handle))
+ return PTR_ERR((void *)handle);
+
+ if (comp_len != PAGE_SIZE)
goto compress_again;
- return PTR_ERR((void *)handle);
+ /*
+ * If the page is not compressible, you need to acquire the
+ * lock and execute the code below. The zcomp_stream_get()
+ * call is needed to disable the cpu hotplug and grab the
+ * zstrm buffer back. It is necessary that the dereferencing
+ * of the zstrm variable below occurs correctly.
+ */
+ zstrm = zcomp_stream_get(zram->comp);
}
alloced_pages = zs_get_total_pages(zram->mem_pool);
static void zram_reset_device(struct zram *zram)
{
- struct zcomp *comp;
- u64 disksize;
-
down_write(&zram->init_lock);
zram->limit_pages = 0;
return;
}
- comp = zram->comp;
- disksize = zram->disksize;
- zram->disksize = 0;
-
set_capacity_and_notify(zram->disk, 0);
part_stat_set_all(zram->disk->part0, 0);
/* I/O operation under all of CPU are done so let's free */
- zram_meta_free(zram, disksize);
+ zram_meta_free(zram, zram->disksize);
+ zram->disksize = 0;
memset(&zram->stats, 0, sizeof(zram->stats));
- zcomp_destroy(comp);
+ zcomp_destroy(zram->comp);
+ zram->comp = NULL;
reset_bdev(zram);
up_write(&zram->init_lock);
.owner = THIS_MODULE
};
-#ifdef CONFIG_ZRAM_WRITEBACK
-static const struct block_device_operations zram_wb_devops = {
- .open = zram_open,
- .submit_bio = zram_submit_bio,
- .swap_slot_free_notify = zram_slot_free_notify,
- .owner = THIS_MODULE
-};
-#endif
-
static DEVICE_ATTR_WO(compact);
static DEVICE_ATTR_RW(disksize);
static DEVICE_ATTR_RO(initstate);
{
int ret;
+ BUILD_BUG_ON(__NR_ZRAM_PAGEFLAGS > BITS_PER_LONG);
+
ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare",
zcomp_cpu_up_prepare, zcomp_cpu_dead);
if (ret < 0)
/*
- * The lower ZRAM_FLAG_SHIFT bits of table.flags is for
- * object size (excluding header), the higher bits is for
- * zram_pageflags.
+ * ZRAM is mainly used for memory efficiency so we want to keep memory
+ * footprint small and thus squeeze size and zram pageflags into a flags
+ * member. The lower ZRAM_FLAG_SHIFT bits is for object size (excluding
+ * header), which cannot be larger than PAGE_SIZE (requiring PAGE_SHIFT
+ * bits), the higher bits are for zram_pageflags.
*
- * zram is mainly used for memory efficiency so we want to keep memory
- * footprint small so we can squeeze size and flags into a field.
- * The lower ZRAM_FLAG_SHIFT bits is for object size (excluding header),
- * the higher bits is for zram_pageflags.
+ * We use BUILD_BUG_ON() to make sure that zram pageflags don't overflow.
*/
-#define ZRAM_FLAG_SHIFT 24
+#define ZRAM_FLAG_SHIFT (PAGE_SHIFT + 1)
/* Flags for zram pages (table[page_no].flags) */
enum zram_pageflags {
provides the access of KCS IO space for BMC side.
config NPCM7XX_KCS_IPMI_BMC
- depends on ARCH_NPCM7XX || COMPILE_TEST
+ depends on ARCH_NPCM || COMPILE_TEST
select IPMI_KCS_BMC
select REGMAP_MMIO
- tristate "NPCM7xx KCS IPMI BMC driver"
+ tristate "NPCM KCS IPMI BMC driver"
help
Provides a driver for the KCS (Keyboard Controller Style) IPMI
- interface found on Nuvoton NPCM7xx SOCs.
+ interface found on Nuvoton NPCM SOCs.
The driver implements the BMC side of the KCS contorller, it
provides the access of KCS IO space for BMC side.
{
if ((msg->data[0] >> 2) & 1) {
/*
- * It's a response being sent, we needto return a
- * response response. Fake a send msg command
+ * It's a response being sent, we need to return a
+ * response to the response. Fake a send msg command
* response with channel 0. This will always be ipmb
* direct.
*/
/* We don't fetch events here. */
}
-static void ipmi_ipmb_remove(struct i2c_client *client)
+static void ipmi_ipmb_cleanup(struct ipmi_ipmb_dev *iidev)
{
- struct ipmi_ipmb_dev *iidev = i2c_get_clientdata(client);
-
if (iidev->slave) {
i2c_slave_unregister(iidev->slave);
if (iidev->slave != iidev->client)
iidev->slave = NULL;
iidev->client = NULL;
ipmi_ipmb_stop_thread(iidev);
+}
+
+static void ipmi_ipmb_remove(struct i2c_client *client)
+{
+ struct ipmi_ipmb_dev *iidev = i2c_get_clientdata(client);
+ ipmi_ipmb_cleanup(iidev);
ipmi_unregister_smi(iidev->intf);
}
out_err:
if (slave && slave != client)
i2c_unregister_device(slave);
- ipmi_ipmb_remove(client);
+ ipmi_ipmb_cleanup(iidev);
return rv;
}
kfree(intf);
}
-struct watcher_entry {
- int intf_num;
- struct ipmi_smi *intf;
- struct list_head link;
-};
-
int ipmi_smi_watcher_register(struct ipmi_smi_watcher *watcher)
{
struct ipmi_smi *intf;
/*
* The message starts at byte 4 which follows the
- * the Channel Byte in the "GET MESSAGE" command
+ * Channel Byte in the "GET MESSAGE" command
*/
recv_msg->msg.data_len = msg->rsp_size - 4;
memcpy(recv_msg->msg_data, &msg->rsp[4],
.id_table = ssif_plat_ids
};
-static int init_ipmi_ssif(void)
+static int __init init_ipmi_ssif(void)
{
int i;
int rv;
}
module_init(init_ipmi_ssif);
-static void cleanup_ipmi_ssif(void)
+static void __exit cleanup_ipmi_ssif(void)
{
if (!initialized)
return;
}
/*
- * AST_usrGuide_KCS.pdf
- * 2. Background:
- * we note D for Data, and C for Cmd/Status, default rules are
- * A. KCS1 / KCS2 ( D / C:X / X+4 )
- * D / C : CA0h / CA4h
- * D / C : CA8h / CACh
- * B. KCS3 ( D / C:XX2h / XX3h )
- * D / C : CA2h / CA3h
- * D / C : CB2h / CB3h
- * C. KCS4
- * D / C : CA4h / CA5h
+ * We note D for Data, and C for Cmd/Status, default rules are
+ *
+ * 1. Only the D address is given:
+ * A. KCS1/KCS2 (D/C: X/X+4)
+ * D/C: CA0h/CA4h
+ * D/C: CA8h/CACh
+ * B. KCS3 (D/C: XX2/XX3h)
+ * D/C: CA2h/CA3h
+ * C. KCS4 (D/C: X/X+1)
+ * D/C: CA4h/CA5h
+ *
+ * 2. Both the D/C addresses are given:
+ * A. KCS1/KCS2/KCS4 (D/C: X/Y)
+ * D/C: CA0h/CA1h
+ * D/C: CA8h/CA9h
+ * D/C: CA4h/CA5h
+ * B. KCS3 (D/C: XX2/XX3h)
+ * D/C: CA2h/CA3h
*/
static int aspeed_kcs_set_address(struct kcs_bmc_device *kcs_bmc, u32 addrs[2], int nr_addrs)
{
.ops = &kcs_bmc_ipmi_driver_ops,
};
-static int kcs_bmc_ipmi_init(void)
+static int __init kcs_bmc_ipmi_init(void)
{
kcs_bmc_register_driver(&kcs_bmc_ipmi_driver);
}
module_init(kcs_bmc_ipmi_init);
-static void kcs_bmc_ipmi_exit(void)
+static void __exit kcs_bmc_ipmi_exit(void)
{
kcs_bmc_unregister_driver(&kcs_bmc_ipmi_driver);
}
.ops = &kcs_bmc_serio_driver_ops,
};
-static int kcs_bmc_serio_init(void)
+static int __init kcs_bmc_serio_init(void)
{
kcs_bmc_register_driver(&kcs_bmc_serio_driver);
}
module_init(kcs_bmc_serio_init);
-static void kcs_bmc_serio_exit(void)
+static void __exit kcs_bmc_serio_exit(void)
{
kcs_bmc_unregister_driver(&kcs_bmc_serio_driver);
}
* Returns whether or not the input pool has been seeded and thus guaranteed
* to supply cryptographically secure random numbers. This applies to: the
* /dev/urandom device, the get_random_bytes function, and the get_random_{u8,
- * u16,u32,u64,int,long} family of functions.
+ * u16,u32,u64,long} family of functions.
*
* Returns: true if the input pool has been seeded.
* false if the input pool has not been seeded.
* u16 get_random_u16()
* u32 get_random_u32()
* u64 get_random_u64()
- * unsigned int get_random_int()
* unsigned long get_random_long()
*
* These interfaces will return the requested number of random bytes
* into the given buffer or as a return value. This is equivalent to
- * a read from /dev/urandom. The u8, u16, u32, u64, int, and long
- * family of functions may be higher performance for one-off random
- * integers, because they do a bit of buffering and do not invoke
- * reseeding until the buffer is emptied.
+ * a read from /dev/urandom. The u8, u16, u32, u64, long family of
+ * functions may be higher performance for one-off random integers,
+ * because they do a bit of buffering and do not invoke reseeding
+ * until the buffer is emptied.
*
*********************************************************************/
{
struct clk_generated *gck = to_clk_generated(hw);
struct clk_hw *parent = NULL;
- struct clk_rate_request req_parent = *req;
long best_rate = -EINVAL;
unsigned long min_rate, parent_rate;
int best_diff = -1;
goto end;
for (div = 1; div < GENERATED_MAX_DIV + 2; div++) {
- req_parent.rate = req->rate * div;
+ struct clk_rate_request req_parent;
+
+ clk_hw_forward_rate_request(hw, req, parent, &req_parent, req->rate * div);
if (__clk_determine_rate(parent, &req_parent))
continue;
clk_generated_best_diff(req, parent, req_parent.rate, div,
struct clk_rate_request *req)
{
struct clk_master *master = to_clk_master(hw);
- struct clk_rate_request req_parent = *req;
struct clk_hw *parent;
long best_rate = LONG_MIN, best_diff = LONG_MIN;
unsigned long parent_rate;
goto end;
for (div = 0; div < MASTER_PRES_MAX + 1; div++) {
+ struct clk_rate_request req_parent;
+ unsigned long req_rate;
+
if (div == MASTER_PRES_MAX)
- req_parent.rate = req->rate * 3;
+ req_rate = req->rate * 3;
else
- req_parent.rate = req->rate << div;
+ req_rate = req->rate << div;
+ clk_hw_forward_rate_request(hw, req, parent, &req_parent, req_rate);
if (__clk_determine_rate(parent, &req_parent))
continue;
{
struct clk_sam9x5_peripheral *periph = to_clk_sam9x5_peripheral(hw);
struct clk_hw *parent = clk_hw_get_parent(hw);
- struct clk_rate_request req_parent = *req;
unsigned long parent_rate = clk_hw_get_rate(parent);
unsigned long tmp_rate;
long best_rate = LONG_MIN;
goto end;
for (shift = 0; shift <= PERIPHERAL_MAX_SHIFT; shift++) {
- req_parent.rate = req->rate << shift;
+ struct clk_rate_request req_parent;
+ clk_hw_forward_rate_request(hw, req, parent, &req_parent, req->rate << shift);
if (__clk_determine_rate(parent, &req_parent))
continue;
req->best_parent_hw = NULL;
if (clk_hw_get_flags(hw) & CLK_SET_RATE_NO_REPARENT) {
- struct clk_rate_request tmp_req = *req;
+ struct clk_rate_request tmp_req;
parent = clk_hw_get_parent(mux_hw);
+ clk_hw_forward_rate_request(hw, req, parent, &tmp_req, req->rate);
ret = clk_composite_determine_rate_for_parent(rate_hw,
&tmp_req,
parent,
}
for (i = 0; i < clk_hw_get_num_parents(mux_hw); i++) {
- struct clk_rate_request tmp_req = *req;
+ struct clk_rate_request tmp_req;
parent = clk_hw_get_parent_by_index(mux_hw, i);
if (!parent)
continue;
+ clk_hw_forward_rate_request(hw, req, parent, &tmp_req, req->rate);
ret = clk_composite_determine_rate_for_parent(rate_hw,
&tmp_req,
parent,
const struct clk_div_table *table,
u8 width, unsigned long flags)
{
- struct clk_rate_request req = {
- .rate = rate,
- .best_parent_rate = *prate,
- .best_parent_hw = parent,
- };
+ struct clk_rate_request req;
int ret;
+ clk_hw_init_rate_request(hw, &req, rate);
+ req.best_parent_rate = *prate;
+ req.best_parent_hw = parent;
+
ret = divider_determine_rate(hw, &req, table, width, flags);
if (ret)
return ret;
const struct clk_div_table *table, u8 width,
unsigned long flags, unsigned int val)
{
- struct clk_rate_request req = {
- .rate = rate,
- .best_parent_rate = *prate,
- .best_parent_hw = parent,
- };
+ struct clk_rate_request req;
int ret;
+ clk_hw_init_rate_request(hw, &req, rate);
+ req.best_parent_rate = *prate;
+ req.best_parent_hw = parent;
+
ret = divider_ro_determine_rate(hw, &req, table, width, flags, val);
if (ret)
return ret;
return now <= rate && now > best;
}
+static void clk_core_init_rate_req(struct clk_core * const core,
+ struct clk_rate_request *req,
+ unsigned long rate);
+
+static int clk_core_round_rate_nolock(struct clk_core *core,
+ struct clk_rate_request *req);
+
+static bool clk_core_has_parent(struct clk_core *core, const struct clk_core *parent)
+{
+ struct clk_core *tmp;
+ unsigned int i;
+
+ /* Optimize for the case where the parent is already the parent. */
+ if (core->parent == parent)
+ return true;
+
+ for (i = 0; i < core->num_parents; i++) {
+ tmp = clk_core_get_parent_by_index(core, i);
+ if (!tmp)
+ continue;
+
+ if (tmp == parent)
+ return true;
+ }
+
+ return false;
+}
+
+static void
+clk_core_forward_rate_req(struct clk_core *core,
+ const struct clk_rate_request *old_req,
+ struct clk_core *parent,
+ struct clk_rate_request *req,
+ unsigned long parent_rate)
+{
+ if (WARN_ON(!clk_core_has_parent(core, parent)))
+ return;
+
+ clk_core_init_rate_req(parent, req, parent_rate);
+
+ if (req->min_rate < old_req->min_rate)
+ req->min_rate = old_req->min_rate;
+
+ if (req->max_rate > old_req->max_rate)
+ req->max_rate = old_req->max_rate;
+}
+
int clk_mux_determine_rate_flags(struct clk_hw *hw,
struct clk_rate_request *req,
unsigned long flags)
struct clk_core *core = hw->core, *parent, *best_parent = NULL;
int i, num_parents, ret;
unsigned long best = 0;
- struct clk_rate_request parent_req = *req;
/* if NO_REPARENT flag set, pass through to current parent */
if (core->flags & CLK_SET_RATE_NO_REPARENT) {
parent = core->parent;
if (core->flags & CLK_SET_RATE_PARENT) {
- ret = __clk_determine_rate(parent ? parent->hw : NULL,
- &parent_req);
+ struct clk_rate_request parent_req;
+
+ if (!parent) {
+ req->rate = 0;
+ return 0;
+ }
+
+ clk_core_forward_rate_req(core, req, parent, &parent_req, req->rate);
+ ret = clk_core_round_rate_nolock(parent, &parent_req);
if (ret)
return ret;
/* find the parent that can provide the fastest rate <= rate */
num_parents = core->num_parents;
for (i = 0; i < num_parents; i++) {
+ unsigned long parent_rate;
+
parent = clk_core_get_parent_by_index(core, i);
if (!parent)
continue;
if (core->flags & CLK_SET_RATE_PARENT) {
- parent_req = *req;
- ret = __clk_determine_rate(parent->hw, &parent_req);
+ struct clk_rate_request parent_req;
+
+ clk_core_forward_rate_req(core, req, parent, &parent_req, req->rate);
+ ret = clk_core_round_rate_nolock(parent, &parent_req);
if (ret)
continue;
+
+ parent_rate = parent_req.rate;
} else {
- parent_req.rate = clk_core_get_rate_nolock(parent);
+ parent_rate = clk_core_get_rate_nolock(parent);
}
- if (mux_is_better_rate(req->rate, parent_req.rate,
+ if (mux_is_better_rate(req->rate, parent_rate,
best, flags)) {
best_parent = parent;
- best = parent_req.rate;
+ best = parent_rate;
}
}
*max_rate = min(*max_rate, clk_user->max_rate);
}
+/*
+ * clk_hw_get_rate_range() - returns the clock rate range for a hw clk
+ * @hw: the hw clk we want to get the range from
+ * @min_rate: pointer to the variable that will hold the minimum
+ * @max_rate: pointer to the variable that will hold the maximum
+ *
+ * Fills the @min_rate and @max_rate variables with the minimum and
+ * maximum that clock can reach.
+ */
+void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate,
+ unsigned long *max_rate)
+{
+ clk_core_get_boundaries(hw->core, min_rate, max_rate);
+}
+EXPORT_SYMBOL_GPL(clk_hw_get_rate_range);
+
static bool clk_core_check_boundaries(struct clk_core *core,
unsigned long min_rate,
unsigned long max_rate)
if (!core)
return 0;
- req->rate = clamp(req->rate, req->min_rate, req->max_rate);
+ /*
+ * Some clock providers hand-craft their clk_rate_requests and
+ * might not fill min_rate and max_rate.
+ *
+ * If it's the case, clamping the rate is equivalent to setting
+ * the rate to 0 which is bad. Skip the clamping but complain so
+ * that it gets fixed, hopefully.
+ */
+ if (!req->min_rate && !req->max_rate)
+ pr_warn("%s: %s: clk_rate_request has initialized min or max rate.\n",
+ __func__, core->name);
+ else
+ req->rate = clamp(req->rate, req->min_rate, req->max_rate);
/*
* At this point, core protection will be disabled
}
static void clk_core_init_rate_req(struct clk_core * const core,
- struct clk_rate_request *req)
+ struct clk_rate_request *req,
+ unsigned long rate)
{
struct clk_core *parent;
if (WARN_ON(!core || !req))
return;
+ memset(req, 0, sizeof(*req));
+
+ req->rate = rate;
+ clk_core_get_boundaries(core, &req->min_rate, &req->max_rate);
+
parent = core->parent;
if (parent) {
req->best_parent_hw = parent->hw;
}
}
+/**
+ * clk_hw_init_rate_request - Initializes a clk_rate_request
+ * @hw: the clk for which we want to submit a rate request
+ * @req: the clk_rate_request structure we want to initialise
+ * @rate: the rate which is to be requested
+ *
+ * Initializes a clk_rate_request structure to submit to
+ * __clk_determine_rate() or similar functions.
+ */
+void clk_hw_init_rate_request(const struct clk_hw *hw,
+ struct clk_rate_request *req,
+ unsigned long rate)
+{
+ if (WARN_ON(!hw || !req))
+ return;
+
+ clk_core_init_rate_req(hw->core, req, rate);
+}
+EXPORT_SYMBOL_GPL(clk_hw_init_rate_request);
+
+/**
+ * clk_hw_forward_rate_request - Forwards a clk_rate_request to a clock's parent
+ * @hw: the original clock that got the rate request
+ * @old_req: the original clk_rate_request structure we want to forward
+ * @parent: the clk we want to forward @old_req to
+ * @req: the clk_rate_request structure we want to initialise
+ * @parent_rate: The rate which is to be requested to @parent
+ *
+ * Initializes a clk_rate_request structure to submit to a clock parent
+ * in __clk_determine_rate() or similar functions.
+ */
+void clk_hw_forward_rate_request(const struct clk_hw *hw,
+ const struct clk_rate_request *old_req,
+ const struct clk_hw *parent,
+ struct clk_rate_request *req,
+ unsigned long parent_rate)
+{
+ if (WARN_ON(!hw || !old_req || !parent || !req))
+ return;
+
+ clk_core_forward_rate_req(hw->core, old_req,
+ parent->core, req,
+ parent_rate);
+}
+
static bool clk_core_can_round(struct clk_core * const core)
{
return core->ops->determine_rate || core->ops->round_rate;
static int clk_core_round_rate_nolock(struct clk_core *core,
struct clk_rate_request *req)
{
+ int ret;
+
lockdep_assert_held(&prepare_lock);
if (!core) {
return 0;
}
- clk_core_init_rate_req(core, req);
-
if (clk_core_can_round(core))
return clk_core_determine_round_nolock(core, req);
- else if (core->flags & CLK_SET_RATE_PARENT)
- return clk_core_round_rate_nolock(core->parent, req);
+
+ if (core->flags & CLK_SET_RATE_PARENT) {
+ struct clk_rate_request parent_req;
+
+ clk_core_forward_rate_req(core, req, core->parent, &parent_req, req->rate);
+ ret = clk_core_round_rate_nolock(core->parent, &parent_req);
+ if (ret)
+ return ret;
+
+ req->best_parent_rate = parent_req.rate;
+ req->rate = parent_req.rate;
+
+ return 0;
+ }
req->rate = core->rate;
return 0;
int ret;
struct clk_rate_request req;
- clk_core_get_boundaries(hw->core, &req.min_rate, &req.max_rate);
- req.rate = rate;
+ clk_core_init_rate_req(hw->core, &req, rate);
ret = clk_core_round_rate_nolock(hw->core, &req);
if (ret)
if (clk->exclusive_count)
clk_core_rate_unprotect(clk->core);
- clk_core_get_boundaries(clk->core, &req.min_rate, &req.max_rate);
- req.rate = rate;
+ clk_core_init_rate_req(clk->core, &req, rate);
ret = clk_core_round_rate_nolock(clk->core, &req);
/**
* __clk_recalc_rates
* @core: first clk in the subtree
+ * @update_req: Whether req_rate should be updated with the new rate
* @msg: notification type (see include/linux/clk.h)
*
* Walks the subtree of clks starting with clk and recalculates rates as it
* clk_recalc_rates also propagates the POST_RATE_CHANGE notification,
* if necessary.
*/
-static void __clk_recalc_rates(struct clk_core *core, unsigned long msg)
+static void __clk_recalc_rates(struct clk_core *core, bool update_req,
+ unsigned long msg)
{
unsigned long old_rate;
unsigned long parent_rate = 0;
parent_rate = core->parent->rate;
core->rate = clk_recalc(core, parent_rate);
+ if (update_req)
+ core->req_rate = core->rate;
/*
* ignore NOTIFY_STOP and NOTIFY_BAD return values for POST_RATE_CHANGE
__clk_notify(core, msg, old_rate, core->rate);
hlist_for_each_entry(child, &core->children, child_node)
- __clk_recalc_rates(child, msg);
+ __clk_recalc_rates(child, update_req, msg);
}
static unsigned long clk_core_get_rate_recalc(struct clk_core *core)
{
if (core && (core->flags & CLK_GET_RATE_NOCACHE))
- __clk_recalc_rates(core, 0);
+ __clk_recalc_rates(core, false, 0);
return clk_core_get_rate_nolock(core);
}
* @clk: the clk whose rate is being returned
*
* Simply returns the cached rate of the clk, unless CLK_GET_RATE_NOCACHE flag
- * is set, which means a recalc_rate will be issued.
- * If clk is NULL then returns 0.
+ * is set, which means a recalc_rate will be issued. Can be called regardless of
+ * the clock enabledness. If clk is NULL, or if an error occurred, then returns
+ * 0.
*/
unsigned long clk_get_rate(struct clk *clk)
{
flags = clk_enable_lock();
clk_reparent(core, old_parent);
clk_enable_unlock(flags);
+
__clk_set_parent_after(core, old_parent, parent);
return ret;
if (clk_core_can_round(core)) {
struct clk_rate_request req;
- req.rate = rate;
- req.min_rate = min_rate;
- req.max_rate = max_rate;
-
- clk_core_init_rate_req(core, &req);
+ clk_core_init_rate_req(core, &req, rate);
ret = clk_core_determine_round_nolock(core, &req);
if (ret < 0)
if (cnt < 0)
return cnt;
- clk_core_get_boundaries(core, &req.min_rate, &req.max_rate);
- req.rate = req_rate;
+ clk_core_init_rate_req(core, &req, req_rate);
ret = clk_core_round_rate_nolock(core, &req);
}
EXPORT_SYMBOL_GPL(clk_set_rate_exclusive);
-/**
- * clk_set_rate_range - set a rate range for a clock source
- * @clk: clock source
- * @min: desired minimum clock rate in Hz, inclusive
- * @max: desired maximum clock rate in Hz, inclusive
- *
- * Returns success (0) or negative errno.
- */
-int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max)
+static int clk_set_rate_range_nolock(struct clk *clk,
+ unsigned long min,
+ unsigned long max)
{
int ret = 0;
unsigned long old_min, old_max, rate;
+ lockdep_assert_held(&prepare_lock);
+
if (!clk)
return 0;
return -EINVAL;
}
- clk_prepare_lock();
-
if (clk->exclusive_count)
clk_core_rate_unprotect(clk->core);
goto out;
}
+ rate = clk->core->req_rate;
+ if (clk->core->flags & CLK_GET_RATE_NOCACHE)
+ rate = clk_core_get_rate_recalc(clk->core);
+
/*
* Since the boundaries have been changed, let's give the
* opportunity to the provider to adjust the clock rate based on
* - the determine_rate() callback does not really check for
* this corner case when determining the rate
*/
- rate = clamp(clk->core->req_rate, min, max);
+ rate = clamp(rate, min, max);
ret = clk_core_set_rate_nolock(clk->core, rate);
if (ret) {
/* rollback the changes */
if (clk->exclusive_count)
clk_core_rate_protect(clk->core);
+ return ret;
+}
+
+/**
+ * clk_set_rate_range - set a rate range for a clock source
+ * @clk: clock source
+ * @min: desired minimum clock rate in Hz, inclusive
+ * @max: desired maximum clock rate in Hz, inclusive
+ *
+ * Return: 0 for success or negative errno on failure.
+ */
+int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max)
+{
+ int ret;
+
+ if (!clk)
+ return 0;
+
+ clk_prepare_lock();
+
+ ret = clk_set_rate_range_nolock(clk, min, max);
+
clk_prepare_unlock();
return ret;
{
clk_reparent(core, new_parent);
__clk_recalc_accuracies(core);
- __clk_recalc_rates(core, POST_RATE_CHANGE);
+ __clk_recalc_rates(core, true, POST_RATE_CHANGE);
}
void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent)
*
* Returns true if @parent is a possible parent for @clk, false otherwise.
*/
-bool clk_has_parent(struct clk *clk, struct clk *parent)
+bool clk_has_parent(const struct clk *clk, const struct clk *parent)
{
- struct clk_core *core, *parent_core;
- int i;
-
/* NULL clocks should be nops, so return success if either is NULL. */
if (!clk || !parent)
return true;
- core = clk->core;
- parent_core = parent->core;
-
- /* Optimize for the case where the parent is already the parent. */
- if (core->parent == parent_core)
- return true;
-
- for (i = 0; i < core->num_parents; i++)
- if (!strcmp(core->parents[i].name, parent_core->name))
- return true;
-
- return false;
+ return clk_core_has_parent(clk->core, parent->core);
}
EXPORT_SYMBOL_GPL(clk_has_parent);
/* propagate rate an accuracy recalculation accordingly */
if (ret) {
- __clk_recalc_rates(core, ABORT_RATE_CHANGE);
+ __clk_recalc_rates(core, true, ABORT_RATE_CHANGE);
} else {
- __clk_recalc_rates(core, POST_RATE_CHANGE);
+ __clk_recalc_rates(core, true, POST_RATE_CHANGE);
__clk_recalc_accuracies(core);
}
__clk_set_parent_before(orphan, parent);
__clk_set_parent_after(orphan, parent, NULL);
__clk_recalc_accuracies(orphan);
- __clk_recalc_rates(orphan, 0);
+ __clk_recalc_rates(orphan, true, 0);
/*
* __clk_init_parent() will set the initial req_rate to
}
hlist_del(&clk->clks_node);
- if (clk->min_rate > clk->core->req_rate ||
- clk->max_rate < clk->core->req_rate)
- clk_core_set_rate_nolock(clk->core, clk->core->req_rate);
+
+ /* If we had any boundaries on that clock, let's drop them. */
+ if (clk->min_rate > 0 || clk->max_rate < ULONG_MAX)
+ clk_set_rate_range_nolock(clk, 0, ULONG_MAX);
owner = clk->core->owner;
kref_put(&clk->core->ref, __clk_release);
.get_parent = clk_dummy_single_get_parent,
};
+struct clk_multiple_parent_ctx {
+ struct clk_dummy_context parents_ctx[2];
+ struct clk_hw hw;
+ u8 current_parent;
+};
+
+static int clk_multiple_parents_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+ struct clk_multiple_parent_ctx *ctx =
+ container_of(hw, struct clk_multiple_parent_ctx, hw);
+
+ if (index >= clk_hw_get_num_parents(hw))
+ return -EINVAL;
+
+ ctx->current_parent = index;
+
+ return 0;
+}
+
+static u8 clk_multiple_parents_mux_get_parent(struct clk_hw *hw)
+{
+ struct clk_multiple_parent_ctx *ctx =
+ container_of(hw, struct clk_multiple_parent_ctx, hw);
+
+ return ctx->current_parent;
+}
+
+static const struct clk_ops clk_multiple_parents_mux_ops = {
+ .get_parent = clk_multiple_parents_mux_get_parent,
+ .set_parent = clk_multiple_parents_mux_set_parent,
+ .determine_rate = __clk_mux_determine_rate_closest,
+};
+
static int clk_test_init_with_ops(struct kunit *test, const struct clk_ops *ops)
{
struct clk_dummy_context *ctx;
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, ctx->rate);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rounded_rate, set_rate;
rounded_rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1);
set_rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, set_rate, 0);
KUNIT_EXPECT_EQ(test, rounded_rate, set_rate);
+
+ clk_put(clk);
}
static struct kunit_case clk_test_cases[] = {
{}
};
+/*
+ * Test suite for a basic rate clock, without any parent.
+ *
+ * These tests exercise the rate API with simple scenarios
+ */
static struct kunit_suite clk_test_suite = {
.name = "clk-test",
.init = clk_test_init,
.test_cases = clk_test_cases,
};
-struct clk_single_parent_ctx {
- struct clk_dummy_context parent_ctx;
- struct clk_hw hw;
+static int clk_uncached_test_init(struct kunit *test)
+{
+ struct clk_dummy_context *ctx;
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+
+ ctx->rate = DUMMY_CLOCK_INIT_RATE;
+ ctx->hw.init = CLK_HW_INIT_NO_PARENT("test-clk",
+ &clk_dummy_rate_ops,
+ CLK_GET_RATE_NOCACHE);
+
+ ret = clk_hw_register(NULL, &ctx->hw);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+/*
+ * Test that for an uncached clock, the clock framework doesn't cache
+ * the rate and clk_get_rate() will return the underlying clock rate
+ * even if it changed.
+ */
+static void clk_test_uncached_get_rate(struct kunit *test)
+{
+ struct clk_dummy_context *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate;
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_INIT_RATE);
+
+ /* We change the rate behind the clock framework's back */
+ ctx->rate = DUMMY_CLOCK_RATE_1;
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for an uncached clock, clk_set_rate_range() will work
+ * properly if the rate hasn't changed.
+ */
+static void clk_test_uncached_set_range(struct kunit *test)
+{
+ struct clk_dummy_context *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate;
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(clk,
+ DUMMY_CLOCK_RATE_1,
+ DUMMY_CLOCK_RATE_2),
+ 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for an uncached clock, clk_set_rate_range() will work
+ * properly if the rate has changed in hardware.
+ *
+ * In this case, it means that if the rate wasn't initially in the range
+ * we're trying to set, but got changed at some point into the range
+ * without the kernel knowing about it, its rate shouldn't be affected.
+ */
+static void clk_test_uncached_updated_rate_set_range(struct kunit *test)
+{
+ struct clk_dummy_context *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate;
+
+ /* We change the rate behind the clock framework's back */
+ ctx->rate = DUMMY_CLOCK_RATE_1 + 1000;
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(clk,
+ DUMMY_CLOCK_RATE_1,
+ DUMMY_CLOCK_RATE_2),
+ 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1 + 1000);
+
+ clk_put(clk);
+}
+
+static struct kunit_case clk_uncached_test_cases[] = {
+ KUNIT_CASE(clk_test_uncached_get_rate),
+ KUNIT_CASE(clk_test_uncached_set_range),
+ KUNIT_CASE(clk_test_uncached_updated_rate_set_range),
+ {}
};
-static int clk_orphan_transparent_single_parent_mux_test_init(struct kunit *test)
+/*
+ * Test suite for a basic, uncached, rate clock, without any parent.
+ *
+ * These tests exercise the rate API with simple scenarios
+ */
+static struct kunit_suite clk_uncached_test_suite = {
+ .name = "clk-uncached-test",
+ .init = clk_uncached_test_init,
+ .exit = clk_test_exit,
+ .test_cases = clk_uncached_test_cases,
+};
+
+static int
+clk_multiple_parents_mux_test_init(struct kunit *test)
{
- struct clk_single_parent_ctx *ctx;
- struct clk_init_data init = { };
- const char * const parents[] = { "orphan_parent" };
+ struct clk_multiple_parent_ctx *ctx;
+ const char *parents[2] = { "parent-0", "parent-1"};
int ret;
ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
return -ENOMEM;
test->priv = ctx;
- init.name = "test_orphan_dummy_parent";
- init.ops = &clk_dummy_single_parent_ops;
- init.parent_names = parents;
- init.num_parents = ARRAY_SIZE(parents);
- init.flags = CLK_SET_RATE_PARENT;
- ctx->hw.init = &init;
+ ctx->parents_ctx[0].hw.init = CLK_HW_INIT_NO_PARENT("parent-0",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->parents_ctx[0].rate = DUMMY_CLOCK_RATE_1;
+ ret = clk_hw_register(NULL, &ctx->parents_ctx[0].hw);
+ if (ret)
+ return ret;
+
+ ctx->parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("parent-1",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->parents_ctx[1].rate = DUMMY_CLOCK_RATE_2;
+ ret = clk_hw_register(NULL, &ctx->parents_ctx[1].hw);
+ if (ret)
+ return ret;
+ ctx->current_parent = 0;
+ ctx->hw.init = CLK_HW_INIT_PARENTS("test-mux", parents,
+ &clk_multiple_parents_mux_ops,
+ CLK_SET_RATE_PARENT);
ret = clk_hw_register(NULL, &ctx->hw);
if (ret)
return ret;
- memset(&init, 0, sizeof(init));
- init.name = "orphan_parent";
- init.ops = &clk_dummy_rate_ops;
- ctx->parent_ctx.hw.init = &init;
- ctx->parent_ctx.rate = DUMMY_CLOCK_INIT_RATE;
+ return 0;
+}
- ret = clk_hw_register(NULL, &ctx->parent_ctx.hw);
+static void
+clk_multiple_parents_mux_test_exit(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+
+ clk_hw_unregister(&ctx->hw);
+ clk_hw_unregister(&ctx->parents_ctx[0].hw);
+ clk_hw_unregister(&ctx->parents_ctx[1].hw);
+}
+
+/*
+ * Test that for a clock with multiple parents, clk_get_parent()
+ * actually returns the current one.
+ */
+static void
+clk_test_multiple_parents_mux_get_parent(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent = clk_hw_get_clk(&ctx->parents_ctx[0].hw, NULL);
+
+ KUNIT_EXPECT_TRUE(test, clk_is_match(clk_get_parent(clk), parent));
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock with a multiple parents, clk_has_parent()
+ * actually reports all of them as parents.
+ */
+static void
+clk_test_multiple_parents_mux_has_parent(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[0].hw, NULL);
+ KUNIT_EXPECT_TRUE(test, clk_has_parent(clk, parent));
+ clk_put(parent);
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_EXPECT_TRUE(test, clk_has_parent(clk, parent));
+ clk_put(parent);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock with a multiple parents, if we set a range on
+ * that clock and the parent is changed, its rate after the reparenting
+ * is still within the range we asked for.
+ *
+ * FIXME: clk_set_parent() only does the reparenting but doesn't
+ * reevaluate whether the new clock rate is within its boundaries or
+ * not.
+ */
+static void
+clk_test_multiple_parents_mux_set_range_set_parent_get_rate(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent1, *parent2;
+ unsigned long rate;
+ int ret;
+
+ kunit_skip(test, "This needs to be fixed in the core.");
+
+ parent1 = clk_hw_get_clk(&ctx->parents_ctx[0].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent1);
+ KUNIT_ASSERT_TRUE(test, clk_is_match(clk_get_parent(clk), parent1));
+
+ parent2 = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent2);
+
+ ret = clk_set_rate(parent1, DUMMY_CLOCK_RATE_1);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate(parent2, DUMMY_CLOCK_RATE_2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(clk,
+ DUMMY_CLOCK_RATE_1 - 1000,
+ DUMMY_CLOCK_RATE_1 + 1000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_parent(clk, parent2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 - 1000);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_1 + 1000);
+
+ clk_put(parent2);
+ clk_put(parent1);
+ clk_put(clk);
+}
+
+static struct kunit_case clk_multiple_parents_mux_test_cases[] = {
+ KUNIT_CASE(clk_test_multiple_parents_mux_get_parent),
+ KUNIT_CASE(clk_test_multiple_parents_mux_has_parent),
+ KUNIT_CASE(clk_test_multiple_parents_mux_set_range_set_parent_get_rate),
+ {}
+};
+
+/*
+ * Test suite for a basic mux clock with two parents, with
+ * CLK_SET_RATE_PARENT on the child.
+ *
+ * These tests exercise the consumer API and check that the state of the
+ * child and parents are sane and consistent.
+ */
+static struct kunit_suite
+clk_multiple_parents_mux_test_suite = {
+ .name = "clk-multiple-parents-mux-test",
+ .init = clk_multiple_parents_mux_test_init,
+ .exit = clk_multiple_parents_mux_test_exit,
+ .test_cases = clk_multiple_parents_mux_test_cases,
+};
+
+static int
+clk_orphan_transparent_multiple_parent_mux_test_init(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx;
+ const char *parents[2] = { "missing-parent", "proper-parent"};
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+
+ ctx->parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("proper-parent",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->parents_ctx[1].rate = DUMMY_CLOCK_INIT_RATE;
+ ret = clk_hw_register(NULL, &ctx->parents_ctx[1].hw);
+ if (ret)
+ return ret;
+
+ ctx->hw.init = CLK_HW_INIT_PARENTS("test-orphan-mux", parents,
+ &clk_multiple_parents_mux_ops,
+ CLK_SET_RATE_PARENT);
+ ret = clk_hw_register(NULL, &ctx->hw);
if (ret)
return ret;
return 0;
}
-static void clk_orphan_transparent_single_parent_mux_test_exit(struct kunit *test)
+static void
+clk_orphan_transparent_multiple_parent_mux_test_exit(struct kunit *test)
{
- struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_multiple_parent_ctx *ctx = test->priv;
clk_hw_unregister(&ctx->hw);
- clk_hw_unregister(&ctx->parent_ctx.hw);
+ clk_hw_unregister(&ctx->parents_ctx[1].hw);
}
/*
- * Test that a mux-only clock, with an initial rate within a range,
- * will still have the same rate after the range has been enforced.
+ * Test that, for a mux whose current parent hasn't been registered yet and is
+ * thus orphan, clk_get_parent() will return NULL.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_get_parent(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+
+ KUNIT_EXPECT_PTR_EQ(test, clk_get_parent(clk), NULL);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux whose current parent hasn't been registered yet,
+ * calling clk_set_parent() to a valid parent will properly update the
+ * mux parent and its orphan status.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_parent(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent, *new_parent;
+ int ret;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ new_parent = clk_get_parent(clk);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+ KUNIT_EXPECT_TRUE(test, clk_is_match(parent, new_parent));
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux that started orphan but got switched to a valid
+ * parent, calling clk_drop_range() on the mux won't affect the parent
+ * rate.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_parent_drop_range(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long parent_rate, new_parent_rate;
+ int ret;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ parent_rate = clk_get_rate(parent);
+ KUNIT_ASSERT_GT(test, parent_rate, 0);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_drop_range(clk);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ new_parent_rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, new_parent_rate, 0);
+ KUNIT_EXPECT_EQ(test, parent_rate, new_parent_rate);
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux that started orphan but got switched to a valid
+ * parent, the rate of the mux and its new parent are consistent.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_parent_get_rate(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long parent_rate, rate;
+ int ret;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ parent_rate = clk_get_rate(parent);
+ KUNIT_ASSERT_GT(test, parent_rate, 0);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, parent_rate, rate);
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux that started orphan but got switched to a valid
+ * parent, calling clk_put() on the mux won't affect the parent rate.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_parent_put(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk *clk, *parent;
+ unsigned long parent_rate, new_parent_rate;
+ int ret;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ clk = clk_hw_get_clk(&ctx->hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, clk);
+
+ parent_rate = clk_get_rate(parent);
+ KUNIT_ASSERT_GT(test, parent_rate, 0);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ clk_put(clk);
+
+ new_parent_rate = clk_get_rate(parent);
+ KUNIT_ASSERT_GT(test, new_parent_rate, 0);
+ KUNIT_EXPECT_EQ(test, parent_rate, new_parent_rate);
+
+ clk_put(parent);
+}
+
+/*
+ * Test that, for a mux that started orphan but got switched to a valid
+ * parent, calling clk_set_rate_range() will affect the parent state if
+ * its rate is out of range.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_modified(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long rate;
+ int ret;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux that started orphan but got switched to a valid
+ * parent, calling clk_set_rate_range() won't affect the parent state if
+ * its rate is within range.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_untouched(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long parent_rate, new_parent_rate;
+ int ret;
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ parent_rate = clk_get_rate(parent);
+ KUNIT_ASSERT_GT(test, parent_rate, 0);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(clk,
+ DUMMY_CLOCK_INIT_RATE - 1000,
+ DUMMY_CLOCK_INIT_RATE + 1000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ new_parent_rate = clk_get_rate(parent);
+ KUNIT_ASSERT_GT(test, new_parent_rate, 0);
+ KUNIT_EXPECT_EQ(test, parent_rate, new_parent_rate);
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux whose current parent hasn't been registered yet,
+ * calling clk_set_rate_range() will succeed, and will be taken into
+ * account when rounding a rate.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_range_round_rate(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate;
+ int ret;
+
+ ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a mux that started orphan, was assigned and rate and
+ * then got switched to a valid parent, its rate is eventually within
+ * range.
+ *
+ * FIXME: Even though we update the rate as part of clk_set_parent(), we
+ * don't evaluate whether that new rate is within range and needs to be
+ * adjusted.
+ */
+static void
+clk_test_orphan_transparent_multiple_parent_mux_set_range_set_parent_get_rate(struct kunit *test)
+{
+ struct clk_multiple_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long rate;
+ int ret;
+
+ kunit_skip(test, "This needs to be fixed in the core.");
+
+ clk_hw_set_rate_range(hw, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2);
+
+ parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent);
+
+ ret = clk_set_parent(clk, parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+static struct kunit_case clk_orphan_transparent_multiple_parent_mux_test_cases[] = {
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_get_parent),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_drop_range),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_get_rate),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_put),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_modified),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_untouched),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_range_round_rate),
+ KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_range_set_parent_get_rate),
+ {}
+};
+
+/*
+ * Test suite for a basic mux clock with two parents. The default parent
+ * isn't registered, only the second parent is. By default, the clock
+ * will thus be orphan.
+ *
+ * These tests exercise the behaviour of the consumer API when dealing
+ * with an orphan clock, and how we deal with the transition to a valid
+ * parent.
+ */
+static struct kunit_suite clk_orphan_transparent_multiple_parent_mux_test_suite = {
+ .name = "clk-orphan-transparent-multiple-parent-mux-test",
+ .init = clk_orphan_transparent_multiple_parent_mux_test_init,
+ .exit = clk_orphan_transparent_multiple_parent_mux_test_exit,
+ .test_cases = clk_orphan_transparent_multiple_parent_mux_test_cases,
+};
+
+struct clk_single_parent_ctx {
+ struct clk_dummy_context parent_ctx;
+ struct clk_hw hw;
+};
+
+static int clk_single_parent_mux_test_init(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx;
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+
+ ctx->parent_ctx.rate = DUMMY_CLOCK_INIT_RATE;
+ ctx->parent_ctx.hw.init =
+ CLK_HW_INIT_NO_PARENT("parent-clk",
+ &clk_dummy_rate_ops,
+ 0);
+
+ ret = clk_hw_register(NULL, &ctx->parent_ctx.hw);
+ if (ret)
+ return ret;
+
+ ctx->hw.init = CLK_HW_INIT("test-clk", "parent-clk",
+ &clk_dummy_single_parent_ops,
+ CLK_SET_RATE_PARENT);
+
+ ret = clk_hw_register(NULL, &ctx->hw);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static void
+clk_single_parent_mux_test_exit(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+
+ clk_hw_unregister(&ctx->hw);
+ clk_hw_unregister(&ctx->parent_ctx.hw);
+}
+
+/*
+ * Test that for a clock with a single parent, clk_get_parent() actually
+ * returns the parent.
+ */
+static void
+clk_test_single_parent_mux_get_parent(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent = clk_hw_get_clk(&ctx->parent_ctx.hw, NULL);
+
+ KUNIT_EXPECT_TRUE(test, clk_is_match(clk_get_parent(clk), parent));
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock with a single parent, clk_has_parent() actually
+ * reports it as a parent.
+ */
+static void
+clk_test_single_parent_mux_has_parent(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent = clk_hw_get_clk(&ctx->parent_ctx.hw, NULL);
+
+ KUNIT_EXPECT_TRUE(test, clk_has_parent(clk, parent));
+
+ clk_put(parent);
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock that can't modify its rate and with a single
+ * parent, if we set disjoints range on the parent and then the child,
+ * the second will return an error.
+ *
+ * FIXME: clk_set_rate_range() only considers the current clock when
+ * evaluating whether ranges are disjoints and not the upstream clocks
+ * ranges.
+ */
+static void
+clk_test_single_parent_mux_set_range_disjoint_child_last(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ int ret;
+
+ kunit_skip(test, "This needs to be fixed in the core.");
+
+ parent = clk_get_parent(clk);
+ KUNIT_ASSERT_PTR_NE(test, parent, NULL);
+
+ ret = clk_set_rate_range(parent, 1000, 2000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(clk, 3000, 4000);
+ KUNIT_EXPECT_LT(test, ret, 0);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock that can't modify its rate and with a single
+ * parent, if we set disjoints range on the child and then the parent,
+ * the second will return an error.
+ *
+ * FIXME: clk_set_rate_range() only considers the current clock when
+ * evaluating whether ranges are disjoints and not the downstream clocks
+ * ranges.
+ */
+static void
+clk_test_single_parent_mux_set_range_disjoint_parent_last(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ int ret;
+
+ kunit_skip(test, "This needs to be fixed in the core.");
+
+ parent = clk_get_parent(clk);
+ KUNIT_ASSERT_PTR_NE(test, parent, NULL);
+
+ ret = clk_set_rate_range(clk, 1000, 2000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(parent, 3000, 4000);
+ KUNIT_EXPECT_LT(test, ret, 0);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock that can't modify its rate and with a single
+ * parent, if we set a range on the parent and then call
+ * clk_round_rate(), the boundaries of the parent are taken into
+ * account.
+ */
+static void
+clk_test_single_parent_mux_set_range_round_rate_parent_only(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long rate;
+ int ret;
+
+ parent = clk_get_parent(clk);
+ KUNIT_ASSERT_PTR_NE(test, parent, NULL);
+
+ ret = clk_set_rate_range(parent, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock that can't modify its rate and with a single
+ * parent, if we set a range on the parent and a more restrictive one on
+ * the child, and then call clk_round_rate(), the boundaries of the
+ * two clocks are taken into account.
+ */
+static void
+clk_test_single_parent_mux_set_range_round_rate_child_smaller(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long rate;
+ int ret;
+
+ parent = clk_get_parent(clk);
+ KUNIT_ASSERT_PTR_NE(test, parent, NULL);
+
+ ret = clk_set_rate_range(parent, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1 + 1000, DUMMY_CLOCK_RATE_2 - 1000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000);
+
+ rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_2 + 1000);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that for a clock that can't modify its rate and with a single
+ * parent, if we set a range on the child and a more restrictive one on
+ * the parent, and then call clk_round_rate(), the boundaries of the
+ * two clocks are taken into account.
+ */
+static void
+clk_test_single_parent_mux_set_range_round_rate_parent_smaller(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *parent;
+ unsigned long rate;
+ int ret;
+
+ parent = clk_get_parent(clk);
+ KUNIT_ASSERT_PTR_NE(test, parent, NULL);
+
+ ret = clk_set_rate_range(parent, DUMMY_CLOCK_RATE_1 + 1000, DUMMY_CLOCK_RATE_2 - 1000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000);
+
+ rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_2 + 1000);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000);
+ KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000);
+
+ clk_put(clk);
+}
+
+static struct kunit_case clk_single_parent_mux_test_cases[] = {
+ KUNIT_CASE(clk_test_single_parent_mux_get_parent),
+ KUNIT_CASE(clk_test_single_parent_mux_has_parent),
+ KUNIT_CASE(clk_test_single_parent_mux_set_range_disjoint_child_last),
+ KUNIT_CASE(clk_test_single_parent_mux_set_range_disjoint_parent_last),
+ KUNIT_CASE(clk_test_single_parent_mux_set_range_round_rate_child_smaller),
+ KUNIT_CASE(clk_test_single_parent_mux_set_range_round_rate_parent_only),
+ KUNIT_CASE(clk_test_single_parent_mux_set_range_round_rate_parent_smaller),
+ {}
+};
+
+/*
+ * Test suite for a basic mux clock with one parent, with
+ * CLK_SET_RATE_PARENT on the child.
+ *
+ * These tests exercise the consumer API and check that the state of the
+ * child and parent are sane and consistent.
+ */
+static struct kunit_suite
+clk_single_parent_mux_test_suite = {
+ .name = "clk-single-parent-mux-test",
+ .init = clk_single_parent_mux_test_init,
+ .exit = clk_single_parent_mux_test_exit,
+ .test_cases = clk_single_parent_mux_test_cases,
+};
+
+static int clk_orphan_transparent_single_parent_mux_test_init(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx;
+ struct clk_init_data init = { };
+ const char * const parents[] = { "orphan_parent" };
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+
+ init.name = "test_orphan_dummy_parent";
+ init.ops = &clk_dummy_single_parent_ops;
+ init.parent_names = parents;
+ init.num_parents = ARRAY_SIZE(parents);
+ init.flags = CLK_SET_RATE_PARENT;
+ ctx->hw.init = &init;
+
+ ret = clk_hw_register(NULL, &ctx->hw);
+ if (ret)
+ return ret;
+
+ memset(&init, 0, sizeof(init));
+ init.name = "orphan_parent";
+ init.ops = &clk_dummy_rate_ops;
+ ctx->parent_ctx.hw.init = &init;
+ ctx->parent_ctx.rate = DUMMY_CLOCK_INIT_RATE;
+
+ ret = clk_hw_register(NULL, &ctx->parent_ctx.hw);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+/*
+ * Test that a mux-only clock, with an initial rate within a range,
+ * will still have the same rate after the range has been enforced.
+ *
+ * See:
+ * https://lore.kernel.org/linux-clk/7720158d-10a7-a17b-73a4-a8615c9c6d5c@collabora.com/
+ */
+static void clk_test_orphan_transparent_parent_mux_set_range(struct kunit *test)
+{
+ struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate, new_rate;
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(clk,
+ ctx->parent_ctx.rate - 1000,
+ ctx->parent_ctx.rate + 1000),
+ 0);
+
+ new_rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, new_rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, new_rate);
+
+ clk_put(clk);
+}
+
+static struct kunit_case clk_orphan_transparent_single_parent_mux_test_cases[] = {
+ KUNIT_CASE(clk_test_orphan_transparent_parent_mux_set_range),
+ {}
+};
+
+/*
+ * Test suite for a basic mux clock with one parent. The parent is
+ * registered after its child. The clock will thus be an orphan when
+ * registered, but will no longer be when the tests run.
+ *
+ * These tests make sure a clock that used to be orphan has a sane,
+ * consistent, behaviour.
+ */
+static struct kunit_suite clk_orphan_transparent_single_parent_test_suite = {
+ .name = "clk-orphan-transparent-single-parent-test",
+ .init = clk_orphan_transparent_single_parent_mux_test_init,
+ .exit = clk_single_parent_mux_test_exit,
+ .test_cases = clk_orphan_transparent_single_parent_mux_test_cases,
+};
+
+struct clk_single_parent_two_lvl_ctx {
+ struct clk_dummy_context parent_parent_ctx;
+ struct clk_dummy_context parent_ctx;
+ struct clk_hw hw;
+};
+
+static int
+clk_orphan_two_level_root_last_test_init(struct kunit *test)
+{
+ struct clk_single_parent_two_lvl_ctx *ctx;
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+
+ ctx->parent_ctx.hw.init =
+ CLK_HW_INIT("intermediate-parent",
+ "root-parent",
+ &clk_dummy_single_parent_ops,
+ CLK_SET_RATE_PARENT);
+ ret = clk_hw_register(NULL, &ctx->parent_ctx.hw);
+ if (ret)
+ return ret;
+
+ ctx->hw.init =
+ CLK_HW_INIT("test-clk", "intermediate-parent",
+ &clk_dummy_single_parent_ops,
+ CLK_SET_RATE_PARENT);
+ ret = clk_hw_register(NULL, &ctx->hw);
+ if (ret)
+ return ret;
+
+ ctx->parent_parent_ctx.rate = DUMMY_CLOCK_INIT_RATE;
+ ctx->parent_parent_ctx.hw.init =
+ CLK_HW_INIT_NO_PARENT("root-parent",
+ &clk_dummy_rate_ops,
+ 0);
+ ret = clk_hw_register(NULL, &ctx->parent_parent_ctx.hw);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static void
+clk_orphan_two_level_root_last_test_exit(struct kunit *test)
+{
+ struct clk_single_parent_two_lvl_ctx *ctx = test->priv;
+
+ clk_hw_unregister(&ctx->hw);
+ clk_hw_unregister(&ctx->parent_ctx.hw);
+ clk_hw_unregister(&ctx->parent_parent_ctx.hw);
+}
+
+/*
+ * Test that, for a clock whose parent used to be orphan, clk_get_rate()
+ * will return the proper rate.
+ */
+static void
+clk_orphan_two_level_root_last_test_get_rate(struct kunit *test)
+{
+ struct clk_single_parent_two_lvl_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate;
+
+ rate = clk_get_rate(clk);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_INIT_RATE);
+
+ clk_put(clk);
+}
+
+/*
+ * Test that, for a clock whose parent used to be orphan,
+ * clk_set_rate_range() won't affect its rate if it is already within
+ * range.
+ *
+ * See (for Exynos 4210):
+ * https://lore.kernel.org/linux-clk/366a0232-bb4a-c357-6aa8-636e398e05eb@samsung.com/
*/
-static void clk_test_orphan_transparent_parent_mux_set_range(struct kunit *test)
+static void
+clk_orphan_two_level_root_last_test_set_range(struct kunit *test)
{
- struct clk_single_parent_ctx *ctx = test->priv;
+ struct clk_single_parent_two_lvl_ctx *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
- unsigned long rate, new_rate;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ unsigned long rate;
+ int ret;
+
+ ret = clk_set_rate_range(clk,
+ DUMMY_CLOCK_INIT_RATE - 1000,
+ DUMMY_CLOCK_INIT_RATE + 1000);
+ KUNIT_ASSERT_EQ(test, ret, 0);
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_INIT_RATE);
- KUNIT_ASSERT_EQ(test,
- clk_set_rate_range(clk,
- ctx->parent_ctx.rate - 1000,
- ctx->parent_ctx.rate + 1000),
- 0);
-
- new_rate = clk_get_rate(clk);
- KUNIT_ASSERT_GT(test, new_rate, 0);
- KUNIT_EXPECT_EQ(test, rate, new_rate);
+ clk_put(clk);
}
-static struct kunit_case clk_orphan_transparent_single_parent_mux_test_cases[] = {
- KUNIT_CASE(clk_test_orphan_transparent_parent_mux_set_range),
+static struct kunit_case
+clk_orphan_two_level_root_last_test_cases[] = {
+ KUNIT_CASE(clk_orphan_two_level_root_last_test_get_rate),
+ KUNIT_CASE(clk_orphan_two_level_root_last_test_set_range),
{}
};
-static struct kunit_suite clk_orphan_transparent_single_parent_test_suite = {
- .name = "clk-orphan-transparent-single-parent-test",
- .init = clk_orphan_transparent_single_parent_mux_test_init,
- .exit = clk_orphan_transparent_single_parent_mux_test_exit,
- .test_cases = clk_orphan_transparent_single_parent_mux_test_cases,
+/*
+ * Test suite for a basic, transparent, clock with a parent that is also
+ * such a clock. The parent's parent is registered last, while the
+ * parent and its child are registered in that order. The intermediate
+ * and leaf clocks will thus be orphan when registered, but the leaf
+ * clock itself will always have its parent and will never be
+ * reparented. Indeed, it's only orphan because its parent is.
+ *
+ * These tests exercise the behaviour of the consumer API when dealing
+ * with an orphan clock, and how we deal with the transition to a valid
+ * parent.
+ */
+static struct kunit_suite
+clk_orphan_two_level_root_last_test_suite = {
+ .name = "clk-orphan-two-level-root-last-test",
+ .init = clk_orphan_two_level_root_last_test_init,
+ .exit = clk_orphan_two_level_root_last_test_exit,
+ .test_cases = clk_orphan_two_level_root_last_test_cases,
};
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
KUNIT_EXPECT_LT(test,
clk_set_rate_range(clk,
DUMMY_CLOCK_RATE_1 + 1000,
DUMMY_CLOCK_RATE_1),
0);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
long rate;
KUNIT_ASSERT_EQ(test,
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
long rounded;
KUNIT_ASSERT_EQ(test,
0);
KUNIT_EXPECT_EQ(test, rounded, clk_get_rate(clk));
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
long rate;
KUNIT_ASSERT_EQ(test,
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1);
KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
long rounded;
KUNIT_ASSERT_EQ(test,
0);
KUNIT_EXPECT_EQ(test, rounded, clk_get_rate(clk));
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
static struct kunit_case clk_range_test_cases[] = {
{}
};
+/*
+ * Test suite for a basic rate clock, without any parent.
+ *
+ * These tests exercise the rate range API: clk_set_rate_range(),
+ * clk_set_min_rate(), clk_set_max_rate(), clk_drop_range().
+ */
static struct kunit_suite clk_range_test_suite = {
.name = "clk-range-test",
.init = clk_test_init,
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
struct clk *user1, *user2;
unsigned long rate;
clk_put(user2);
clk_put(user1);
+ clk_put(clk);
+}
+
+/*
+ * Test that if we have several subsequent calls to
+ * clk_set_rate_range(), across multiple users, the core will reevaluate
+ * whether a new rate is needed, including when a user drop its clock.
+ *
+ * With clk_dummy_maximize_rate_ops, this means that the rate will
+ * trail along the maximum as it evolves.
+ */
+static void clk_range_test_multiple_set_range_rate_put_maximized(struct kunit *test)
+{
+ struct clk_dummy_context *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *user1, *user2;
+ unsigned long rate;
+
+ user1 = clk_hw_get_clk(hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user1);
+
+ user2 = clk_hw_get_clk(hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user2);
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate(clk, DUMMY_CLOCK_RATE_2 + 1000),
+ 0);
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(user1,
+ 0,
+ DUMMY_CLOCK_RATE_2),
+ 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2);
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(user2,
+ 0,
+ DUMMY_CLOCK_RATE_1),
+ 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_put(user2);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(user1);
+ clk_put(clk);
}
static struct kunit_case clk_range_maximize_test_cases[] = {
KUNIT_CASE(clk_range_test_set_range_rate_maximized),
KUNIT_CASE(clk_range_test_multiple_set_range_rate_maximized),
+ KUNIT_CASE(clk_range_test_multiple_set_range_rate_put_maximized),
{}
};
+/*
+ * Test suite for a basic rate clock, without any parent.
+ *
+ * These tests exercise the rate range API: clk_set_rate_range(),
+ * clk_set_min_rate(), clk_set_max_rate(), clk_drop_range(), with a
+ * driver that will always try to run at the highest possible rate.
+ */
static struct kunit_suite clk_range_maximize_test_suite = {
.name = "clk-range-maximize-test",
.init = clk_maximize_test_init,
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
unsigned long rate;
KUNIT_ASSERT_EQ(test,
rate = clk_get_rate(clk);
KUNIT_ASSERT_GT(test, rate, 0);
KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_put(clk);
}
/*
{
struct clk_dummy_context *ctx = test->priv;
struct clk_hw *hw = &ctx->hw;
- struct clk *clk = hw->clk;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
struct clk *user1, *user2;
unsigned long rate;
clk_put(user2);
clk_put(user1);
+ clk_put(clk);
+}
+
+/*
+ * Test that if we have several subsequent calls to
+ * clk_set_rate_range(), across multiple users, the core will reevaluate
+ * whether a new rate is needed, including when a user drop its clock.
+ *
+ * With clk_dummy_minimize_rate_ops, this means that the rate will
+ * trail along the minimum as it evolves.
+ */
+static void clk_range_test_multiple_set_range_rate_put_minimized(struct kunit *test)
+{
+ struct clk_dummy_context *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *user1, *user2;
+ unsigned long rate;
+
+ user1 = clk_hw_get_clk(hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user1);
+
+ user2 = clk_hw_get_clk(hw, NULL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user2);
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(user1,
+ DUMMY_CLOCK_RATE_1,
+ ULONG_MAX),
+ 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ KUNIT_ASSERT_EQ(test,
+ clk_set_rate_range(user2,
+ DUMMY_CLOCK_RATE_2,
+ ULONG_MAX),
+ 0);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(user2);
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_GT(test, rate, 0);
+ KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_put(user1);
+ clk_put(clk);
}
static struct kunit_case clk_range_minimize_test_cases[] = {
KUNIT_CASE(clk_range_test_set_range_rate_minimized),
KUNIT_CASE(clk_range_test_multiple_set_range_rate_minimized),
+ KUNIT_CASE(clk_range_test_multiple_set_range_rate_put_minimized),
{}
};
+/*
+ * Test suite for a basic rate clock, without any parent.
+ *
+ * These tests exercise the rate range API: clk_set_rate_range(),
+ * clk_set_min_rate(), clk_set_max_rate(), clk_drop_range(), with a
+ * driver that will always try to run at the lowest possible rate.
+ */
static struct kunit_suite clk_range_minimize_test_suite = {
.name = "clk-range-minimize-test",
.init = clk_minimize_test_init,
.test_cases = clk_range_minimize_test_cases,
};
+struct clk_leaf_mux_ctx {
+ struct clk_multiple_parent_ctx mux_ctx;
+ struct clk_hw hw;
+};
+
+static int
+clk_leaf_mux_set_rate_parent_test_init(struct kunit *test)
+{
+ struct clk_leaf_mux_ctx *ctx;
+ const char *top_parents[2] = { "parent-0", "parent-1" };
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+
+ ctx->mux_ctx.parents_ctx[0].hw.init = CLK_HW_INIT_NO_PARENT("parent-0",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->mux_ctx.parents_ctx[0].rate = DUMMY_CLOCK_RATE_1;
+ ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[0].hw);
+ if (ret)
+ return ret;
+
+ ctx->mux_ctx.parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("parent-1",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->mux_ctx.parents_ctx[1].rate = DUMMY_CLOCK_RATE_2;
+ ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[1].hw);
+ if (ret)
+ return ret;
+
+ ctx->mux_ctx.current_parent = 0;
+ ctx->mux_ctx.hw.init = CLK_HW_INIT_PARENTS("test-mux", top_parents,
+ &clk_multiple_parents_mux_ops,
+ 0);
+ ret = clk_hw_register(NULL, &ctx->mux_ctx.hw);
+ if (ret)
+ return ret;
+
+ ctx->hw.init = CLK_HW_INIT_HW("test-clock", &ctx->mux_ctx.hw,
+ &clk_dummy_single_parent_ops,
+ CLK_SET_RATE_PARENT);
+ ret = clk_hw_register(NULL, &ctx->hw);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static void clk_leaf_mux_set_rate_parent_test_exit(struct kunit *test)
+{
+ struct clk_leaf_mux_ctx *ctx = test->priv;
+
+ clk_hw_unregister(&ctx->hw);
+ clk_hw_unregister(&ctx->mux_ctx.hw);
+ clk_hw_unregister(&ctx->mux_ctx.parents_ctx[0].hw);
+ clk_hw_unregister(&ctx->mux_ctx.parents_ctx[1].hw);
+}
+
+/*
+ * Test that, for a clock that will forward any rate request to its
+ * parent, the rate request structure returned by __clk_determine_rate
+ * is sane and will be what we expect.
+ */
+static void clk_leaf_mux_set_rate_parent_determine_rate(struct kunit *test)
+{
+ struct clk_leaf_mux_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk_rate_request req;
+ unsigned long rate;
+ int ret;
+
+ rate = clk_get_rate(clk);
+ KUNIT_ASSERT_EQ(test, rate, DUMMY_CLOCK_RATE_1);
+
+ clk_hw_init_rate_request(hw, &req, DUMMY_CLOCK_RATE_2);
+
+ ret = __clk_determine_rate(hw, &req);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ KUNIT_EXPECT_EQ(test, req.rate, DUMMY_CLOCK_RATE_2);
+ KUNIT_EXPECT_EQ(test, req.best_parent_rate, DUMMY_CLOCK_RATE_2);
+ KUNIT_EXPECT_PTR_EQ(test, req.best_parent_hw, &ctx->mux_ctx.hw);
+
+ clk_put(clk);
+}
+
+static struct kunit_case clk_leaf_mux_set_rate_parent_test_cases[] = {
+ KUNIT_CASE(clk_leaf_mux_set_rate_parent_determine_rate),
+ {}
+};
+
+/*
+ * Test suite for a clock whose parent is a mux with multiple parents.
+ * The leaf clock has CLK_SET_RATE_PARENT, and will forward rate
+ * requests to the mux, which will then select which parent is the best
+ * fit for a given rate.
+ *
+ * These tests exercise the behaviour of muxes, and the proper selection
+ * of parents.
+ */
+static struct kunit_suite clk_leaf_mux_set_rate_parent_test_suite = {
+ .name = "clk-leaf-mux-set-rate-parent",
+ .init = clk_leaf_mux_set_rate_parent_test_init,
+ .exit = clk_leaf_mux_set_rate_parent_test_exit,
+ .test_cases = clk_leaf_mux_set_rate_parent_test_cases,
+};
+
+struct clk_mux_notifier_rate_change {
+ bool done;
+ unsigned long old_rate;
+ unsigned long new_rate;
+ wait_queue_head_t wq;
+};
+
+struct clk_mux_notifier_ctx {
+ struct clk_multiple_parent_ctx mux_ctx;
+ struct clk *clk;
+ struct notifier_block clk_nb;
+ struct clk_mux_notifier_rate_change pre_rate_change;
+ struct clk_mux_notifier_rate_change post_rate_change;
+};
+
+#define NOTIFIER_TIMEOUT_MS 100
+
+static int clk_mux_notifier_callback(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct clk_notifier_data *clk_data = data;
+ struct clk_mux_notifier_ctx *ctx = container_of(nb,
+ struct clk_mux_notifier_ctx,
+ clk_nb);
+
+ if (action & PRE_RATE_CHANGE) {
+ ctx->pre_rate_change.old_rate = clk_data->old_rate;
+ ctx->pre_rate_change.new_rate = clk_data->new_rate;
+ ctx->pre_rate_change.done = true;
+ wake_up_interruptible(&ctx->pre_rate_change.wq);
+ }
+
+ if (action & POST_RATE_CHANGE) {
+ ctx->post_rate_change.old_rate = clk_data->old_rate;
+ ctx->post_rate_change.new_rate = clk_data->new_rate;
+ ctx->post_rate_change.done = true;
+ wake_up_interruptible(&ctx->post_rate_change.wq);
+ }
+
+ return 0;
+}
+
+static int clk_mux_notifier_test_init(struct kunit *test)
+{
+ struct clk_mux_notifier_ctx *ctx;
+ const char *top_parents[2] = { "parent-0", "parent-1" };
+ int ret;
+
+ ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ test->priv = ctx;
+ ctx->clk_nb.notifier_call = clk_mux_notifier_callback;
+ init_waitqueue_head(&ctx->pre_rate_change.wq);
+ init_waitqueue_head(&ctx->post_rate_change.wq);
+
+ ctx->mux_ctx.parents_ctx[0].hw.init = CLK_HW_INIT_NO_PARENT("parent-0",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->mux_ctx.parents_ctx[0].rate = DUMMY_CLOCK_RATE_1;
+ ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[0].hw);
+ if (ret)
+ return ret;
+
+ ctx->mux_ctx.parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("parent-1",
+ &clk_dummy_rate_ops,
+ 0);
+ ctx->mux_ctx.parents_ctx[1].rate = DUMMY_CLOCK_RATE_2;
+ ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[1].hw);
+ if (ret)
+ return ret;
+
+ ctx->mux_ctx.current_parent = 0;
+ ctx->mux_ctx.hw.init = CLK_HW_INIT_PARENTS("test-mux", top_parents,
+ &clk_multiple_parents_mux_ops,
+ 0);
+ ret = clk_hw_register(NULL, &ctx->mux_ctx.hw);
+ if (ret)
+ return ret;
+
+ ctx->clk = clk_hw_get_clk(&ctx->mux_ctx.hw, NULL);
+ ret = clk_notifier_register(ctx->clk, &ctx->clk_nb);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static void clk_mux_notifier_test_exit(struct kunit *test)
+{
+ struct clk_mux_notifier_ctx *ctx = test->priv;
+ struct clk *clk = ctx->clk;
+
+ clk_notifier_unregister(clk, &ctx->clk_nb);
+ clk_put(clk);
+
+ clk_hw_unregister(&ctx->mux_ctx.hw);
+ clk_hw_unregister(&ctx->mux_ctx.parents_ctx[0].hw);
+ clk_hw_unregister(&ctx->mux_ctx.parents_ctx[1].hw);
+}
+
+/*
+ * Test that if the we have a notifier registered on a mux, the core
+ * will notify us when we switch to another parent, and with the proper
+ * old and new rates.
+ */
+static void clk_mux_notifier_set_parent_test(struct kunit *test)
+{
+ struct clk_mux_notifier_ctx *ctx = test->priv;
+ struct clk_hw *hw = &ctx->mux_ctx.hw;
+ struct clk *clk = clk_hw_get_clk(hw, NULL);
+ struct clk *new_parent = clk_hw_get_clk(&ctx->mux_ctx.parents_ctx[1].hw, NULL);
+ int ret;
+
+ ret = clk_set_parent(clk, new_parent);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ ret = wait_event_interruptible_timeout(ctx->pre_rate_change.wq,
+ ctx->pre_rate_change.done,
+ msecs_to_jiffies(NOTIFIER_TIMEOUT_MS));
+ KUNIT_ASSERT_GT(test, ret, 0);
+
+ KUNIT_EXPECT_EQ(test, ctx->pre_rate_change.old_rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_EQ(test, ctx->pre_rate_change.new_rate, DUMMY_CLOCK_RATE_2);
+
+ ret = wait_event_interruptible_timeout(ctx->post_rate_change.wq,
+ ctx->post_rate_change.done,
+ msecs_to_jiffies(NOTIFIER_TIMEOUT_MS));
+ KUNIT_ASSERT_GT(test, ret, 0);
+
+ KUNIT_EXPECT_EQ(test, ctx->post_rate_change.old_rate, DUMMY_CLOCK_RATE_1);
+ KUNIT_EXPECT_EQ(test, ctx->post_rate_change.new_rate, DUMMY_CLOCK_RATE_2);
+
+ clk_put(new_parent);
+ clk_put(clk);
+}
+
+static struct kunit_case clk_mux_notifier_test_cases[] = {
+ KUNIT_CASE(clk_mux_notifier_set_parent_test),
+ {}
+};
+
+/*
+ * Test suite for a mux with multiple parents, and a notifier registered
+ * on the mux.
+ *
+ * These tests exercise the behaviour of notifiers.
+ */
+static struct kunit_suite clk_mux_notifier_test_suite = {
+ .name = "clk-mux-notifier",
+ .init = clk_mux_notifier_test_init,
+ .exit = clk_mux_notifier_test_exit,
+ .test_cases = clk_mux_notifier_test_cases,
+};
+
kunit_test_suites(
+ &clk_leaf_mux_set_rate_parent_test_suite,
&clk_test_suite,
+ &clk_multiple_parents_mux_test_suite,
+ &clk_mux_notifier_test_suite,
+ &clk_orphan_transparent_multiple_parent_mux_test_suite,
&clk_orphan_transparent_single_parent_test_suite,
+ &clk_orphan_two_level_root_last_test_suite,
&clk_range_test_suite,
&clk_range_maximize_test_suite,
- &clk_range_minimize_test_suite
+ &clk_range_minimize_test_suite,
+ &clk_single_parent_mux_test_suite,
+ &clk_uncached_test_suite
);
MODULE_LICENSE("GPL v2");
return 0;
}
+static int mtk_clk_mux_determine_rate(struct clk_hw *hw,
+ struct clk_rate_request *req)
+{
+ struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+
+ return clk_mux_determine_rate_flags(hw, req, mux->data->flags);
+}
+
const struct clk_ops mtk_mux_clr_set_upd_ops = {
.get_parent = mtk_clk_mux_get_parent,
.set_parent = mtk_clk_mux_set_parent_setclr_lock,
+ .determine_rate = mtk_clk_mux_determine_rate,
};
EXPORT_SYMBOL_GPL(mtk_mux_clr_set_upd_ops);
.is_enabled = mtk_clk_mux_is_enabled,
.get_parent = mtk_clk_mux_get_parent,
.set_parent = mtk_clk_mux_set_parent_setclr_lock,
+ .determine_rate = mtk_clk_mux_determine_rate,
};
EXPORT_SYMBOL_GPL(mtk_mux_gate_clr_set_upd_ops);
req->best_parent_hw = p2;
}
+ clk_hw_get_rate_range(req->best_parent_hw,
+ &parent_req.min_rate, &parent_req.max_rate);
+
+ if (req->min_rate > parent_req.min_rate)
+ parent_req.min_rate = req->min_rate;
+
+ if (req->max_rate < parent_req.max_rate)
+ parent_req.max_rate = req->max_rate;
+
ret = __clk_determine_rate(req->best_parent_hw, &parent_req);
if (ret)
return ret;
static int gcc_msm8660_probe(struct platform_device *pdev)
{
- int ret;
- struct device *dev = &pdev->dev;
-
- ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000);
- if (ret)
- return ret;
-
- ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000);
- if (ret)
- return ret;
-
return qcom_cc_probe(pdev, &gcc_msm8660_desc);
}
#include <linux/clk.h>
#include <linux/clkdev.h>
+#include <linux/clk/spear.h>
#include <linux/err.h>
#include <linux/io.h>
#include <linux/of_platform.h>
*/
#include <linux/clkdev.h>
+#include <linux/clk/spear.h>
#include <linux/io.h>
#include <linux/spinlock_types.h>
#include "clk.h"
{ TEGRA114_CLK_I2S3_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 },
{ TEGRA114_CLK_I2S4_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 },
{ TEGRA114_CLK_VIMCLK_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 },
+ { TEGRA114_CLK_PWM, TEGRA114_CLK_PLL_P, 408000000, 0 },
/* must be the last entry */
{ TEGRA114_CLK_CLK_MAX, TEGRA114_CLK_CLK_MAX, 0, 0 },
};
{ TEGRA124_CLK_I2S3_SYNC, TEGRA124_CLK_CLK_MAX, 24576000, 0 },
{ TEGRA124_CLK_I2S4_SYNC, TEGRA124_CLK_CLK_MAX, 24576000, 0 },
{ TEGRA124_CLK_VIMCLK_SYNC, TEGRA124_CLK_CLK_MAX, 24576000, 0 },
+ { TEGRA124_CLK_PWM, TEGRA124_CLK_PLL_P, 408000000, 0 },
/* must be the last entry */
{ TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0 },
};
{ TEGRA20_CLK_GR2D, TEGRA20_CLK_PLL_C, 300000000, 0 },
{ TEGRA20_CLK_GR3D, TEGRA20_CLK_PLL_C, 300000000, 0 },
{ TEGRA20_CLK_VDE, TEGRA20_CLK_PLL_C, 300000000, 0 },
+ { TEGRA20_CLK_PWM, TEGRA20_CLK_PLL_P, 48000000, 0 },
/* must be the last entry */
{ TEGRA20_CLK_CLK_MAX, TEGRA20_CLK_CLK_MAX, 0, 0 },
};
{ TEGRA210_CLK_VIMCLK_SYNC, TEGRA210_CLK_CLK_MAX, 24576000, 0 },
{ TEGRA210_CLK_HDA, TEGRA210_CLK_PLL_P, 51000000, 0 },
{ TEGRA210_CLK_HDA2CODEC_2X, TEGRA210_CLK_PLL_P, 48000000, 0 },
+ { TEGRA210_CLK_PWM, TEGRA210_CLK_PLL_P, 48000000, 0 },
/* This MUST be the last entry. */
{ TEGRA210_CLK_CLK_MAX, TEGRA210_CLK_CLK_MAX, 0, 0 },
};
{ TEGRA30_CLK_VIMCLK_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 },
{ TEGRA30_CLK_HDA, TEGRA30_CLK_PLL_P, 102000000, 0 },
{ TEGRA30_CLK_HDA2CODEC_2X, TEGRA30_CLK_PLL_P, 48000000, 0 },
+ { TEGRA30_CLK_PWM, TEGRA30_CLK_PLL_P, 48000000, 0 },
/* must be the last entry */
{ TEGRA30_CLK_CLK_MAX, TEGRA30_CLK_CLK_MAX, 0, 0 },
};
rc = platform_device_add_data(pdev, &info, sizeof(info));
if (rc < 0) {
pr_err("hmem memregion_info allocation failure for %pr\n", &res);
- goto out_pdev;
+ goto out_resource;
}
rc = platform_device_add_resources(pdev, &res, 1);
return;
out_resource:
- put_device(&pdev->dev);
+ platform_device_put(pdev);
out_pdev:
memregion_free(id);
}
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mman.h>
+#include <linux/memory-tiers.h>
#include "dax-private.h"
#include "bus.h"
+/*
+ * Default abstract distance assigned to the NUMA node onlined
+ * by DAX/kmem if the low level platform driver didn't initialize
+ * one for this NUMA node.
+ */
+#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5)
+
/* Memory resource name used for add_memory_driver_managed(). */
static const char *kmem_name;
/* Set if any memory will remain added when the driver will be unloaded. */
struct resource *res[];
};
+static struct memory_dev_type *dax_slowmem_type;
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
{
struct device *dev = &dev_dax->dev;
return -EINVAL;
}
+ init_node_memory_type(numa_node, dax_slowmem_type);
+
+ rc = -ENOMEM;
data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL);
if (!data)
- return -ENOMEM;
+ goto err_dax_kmem_data;
- rc = -ENOMEM;
data->res_name = kstrdup(dev_name(dev), GFP_KERNEL);
if (!data->res_name)
goto err_res_name;
kfree(data->res_name);
err_res_name:
kfree(data);
+err_dax_kmem_data:
+ clear_node_memory_type(numa_node, dax_slowmem_type);
return rc;
}
static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
{
int i, success = 0;
+ int node = dev_dax->target_node;
struct device *dev = &dev_dax->dev;
struct dax_kmem_data *data = dev_get_drvdata(dev);
kfree(data->res_name);
kfree(data);
dev_set_drvdata(dev, NULL);
+ /*
+ * Clear the memtype association on successful unplug.
+ * If not, we have memory blocks left which can be
+ * offlined/onlined later. We need to keep memory_dev_type
+ * for that. This implies this reference will be around
+ * till next reboot.
+ */
+ clear_node_memory_type(node, dax_slowmem_type);
}
}
#else
if (!kmem_name)
return -ENOMEM;
+ dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE);
+ if (IS_ERR(dax_slowmem_type)) {
+ rc = PTR_ERR(dax_slowmem_type);
+ goto err_dax_slowmem_type;
+ }
+
rc = dax_driver_register(&device_dax_kmem_driver);
if (rc)
- kfree_const(kmem_name);
+ goto error_dax_driver;
+
+ return rc;
+
+error_dax_driver:
+ destroy_memory_type(dax_slowmem_type);
+err_dax_slowmem_type:
+ kfree_const(kmem_name);
return rc;
}
dax_driver_unregister(&device_dax_kmem_driver);
if (!any_hotremove_failed)
kfree_const(kmem_name);
+ destroy_memory_type(dax_slowmem_type);
}
MODULE_AUTHOR("Intel Corporation");
{
struct dax_device *dax_dev = to_dax_dev(inode);
if (inode->i_rdev)
- ida_simple_remove(&dax_minor_ida, iminor(inode));
+ ida_free(&dax_minor_ida, iminor(inode));
kmem_cache_free(dax_cache, dax_dev);
}
if (WARN_ON_ONCE(ops && !ops->zero_page_range))
return ERR_PTR(-EINVAL);
- minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL);
+ minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL);
if (minor < 0)
return ERR_PTR(-ENOMEM);
return dax_dev;
err_dev:
- ida_simple_remove(&dax_minor_ida, minor);
+ ida_free(&dax_minor_ida, minor);
return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL_GPL(alloc_dax);
{
unsigned long buf;
- prandom_bytes(&buf, sizeof(buf));
+ get_random_bytes(&buf, sizeof(buf));
return buf;
}
config EDAC_SIFIVE
bool "Sifive platform EDAC driver"
- depends on EDAC=y && SIFIVE_L2
+ depends on EDAC=y && SIFIVE_CCACHE
help
Support for error detection and correction on the SiFive SoCs.
/*
* SiFive Platform EDAC Driver
*
- * Copyright (C) 2018-2019 SiFive, Inc.
+ * Copyright (C) 2018-2022 SiFive, Inc.
*
* This driver is partially based on octeon_edac-pc.c
*
#include <linux/edac.h>
#include <linux/platform_device.h>
#include "edac_module.h"
-#include <soc/sifive/sifive_l2_cache.h>
+#include <soc/sifive/sifive_ccache.h>
#define DRVNAME "sifive_edac"
p = container_of(this, struct sifive_edac_priv, notifier);
- if (event == SIFIVE_L2_ERR_TYPE_UE)
+ if (event == SIFIVE_CCACHE_ERR_TYPE_UE)
edac_device_handle_ue(p->dci, 0, 0, msg);
- else if (event == SIFIVE_L2_ERR_TYPE_CE)
+ else if (event == SIFIVE_CCACHE_ERR_TYPE_CE)
edac_device_handle_ce(p->dci, 0, 0, msg);
return NOTIFY_OK;
goto err;
}
- register_sifive_l2_error_notifier(&p->notifier);
+ register_sifive_ccache_error_notifier(&p->notifier);
return 0;
{
struct sifive_edac_priv *p = platform_get_drvdata(pdev);
- unregister_sifive_l2_error_notifier(&p->notifier);
+ unregister_sifive_ccache_error_notifier(&p->notifier);
edac_device_del_device(&pdev->dev);
edac_device_free_ctl_info(p->dci);
{
u32 smbios_ver;
+ /*
+ * The size of this structure is 31 bytes, but we also accept value
+ * 30 due to a mistake in SMBIOS specification version 2.1.
+ */
if (memcmp(buf, "_SM_", 4) == 0 &&
- buf[5] < 32 && dmi_checksum(buf, buf[5])) {
+ buf[5] >= 30 && buf[5] <= 32 &&
+ dmi_checksum(buf, buf[5])) {
smbios_ver = get_unaligned_be16(buf + 6);
smbios_entry_point_size = buf[5];
memcpy(smbios_entry_point, buf, smbios_entry_point_size);
static int __init dmi_smbios3_present(const u8 *buf)
{
if (memcmp(buf, "_SM3_", 5) == 0 &&
- buf[6] < 32 && dmi_checksum(buf, buf[6])) {
+ buf[6] >= 24 && buf[6] <= 32 &&
+ dmi_checksum(buf, buf[6])) {
dmi_ver = get_unaligned_be24(buf + 7);
dmi_num = 0; /* No longer specified */
dmi_len = get_unaligned_le32(buf + 12);
static unsigned long __initdata initrd = EFI_INVALID_TABLE_ADDR;
struct mm_struct efi_mm = {
- .mm_rb = RB_ROOT,
+ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, efi_mm.mmap_lock),
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),
.write_protect_seq = SEQCNT_ZERO(efi_mm.write_protect_seq),
# Sanitizer runtimes are unavailable and cannot be linked here.
KASAN_SANITIZE := n
KCSAN_SANITIZE := n
+KMSAN_SANITIZE := n
UBSAN_SANITIZE := n
OBJECT_FILES_NON_STANDARD := y
menu "I2C GPIO expanders"
depends on I2C
-config GPIO_ADP5588
- tristate "ADP5588 I2C GPIO expander"
- help
- This option enables support for 18 GPIOs found
- on Analog Devices ADP5588 GPIO Expanders.
-
-config GPIO_ADP5588_IRQ
- bool "Interrupt controller support for ADP5588"
- depends on GPIO_ADP5588=y
- select GPIOLIB_IRQCHIP
- help
- Say yes here to enable the adp5588 to be used as an interrupt
- controller. It requires the driver to be built in the kernel.
-
config GPIO_ADNP
tristate "Avionic Design N-bit GPIO expander"
depends on OF_GPIO
obj-$(CONFIG_GPIO_74XX_MMIO) += gpio-74xx-mmio.o
obj-$(CONFIG_GPIO_ADNP) += gpio-adnp.o
obj-$(CONFIG_GPIO_ADP5520) += gpio-adp5520.o
-obj-$(CONFIG_GPIO_ADP5588) += gpio-adp5588.o
obj-$(CONFIG_GPIO_AGGREGATOR) += gpio-aggregator.o
obj-$(CONFIG_GPIO_ALTERA_A10SR) += gpio-altera-a10sr.o
obj-$(CONFIG_GPIO_ALTERA) += gpio-altera.o
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * GPIO Chip driver for Analog Devices
- * ADP5588/ADP5587 I/O Expander and QWERTY Keypad Controller
- *
- * Copyright 2009-2010 Analog Devices Inc.
- */
-
-#include <linux/gpio/driver.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/kernel.h>
-#include <linux/mod_devicetable.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-
-#include <linux/platform_data/adp5588.h>
-
-/*
- * Early pre 4.0 Silicon required to delay readout by at least 25ms,
- * since the Event Counter Register updated 25ms after the interrupt
- * asserted.
- */
-#define WA_DELAYED_READOUT_REVID(rev) ((rev) < 4)
-
-struct adp5588_gpio {
- struct i2c_client *client;
- struct gpio_chip gpio_chip;
- struct mutex lock; /* protect cached dir, dat_out */
- /* protect serialized access to the interrupt controller bus */
- struct mutex irq_lock;
- uint8_t dat_out[3];
- uint8_t dir[3];
- uint8_t int_lvl_low[3];
- uint8_t int_lvl_high[3];
- uint8_t int_en[3];
- uint8_t irq_mask[3];
- uint8_t int_input_en[3];
-};
-
-static int adp5588_gpio_read(struct i2c_client *client, u8 reg)
-{
- int ret = i2c_smbus_read_byte_data(client, reg);
-
- if (ret < 0)
- dev_err(&client->dev, "Read Error\n");
-
- return ret;
-}
-
-static int adp5588_gpio_write(struct i2c_client *client, u8 reg, u8 val)
-{
- int ret = i2c_smbus_write_byte_data(client, reg, val);
-
- if (ret < 0)
- dev_err(&client->dev, "Write Error\n");
-
- return ret;
-}
-
-static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned off)
-{
- struct adp5588_gpio *dev = gpiochip_get_data(chip);
- unsigned bank = ADP5588_BANK(off);
- unsigned bit = ADP5588_BIT(off);
- int val;
-
- mutex_lock(&dev->lock);
-
- if (dev->dir[bank] & bit)
- val = dev->dat_out[bank];
- else
- val = adp5588_gpio_read(dev->client, GPIO_DAT_STAT1 + bank);
-
- mutex_unlock(&dev->lock);
-
- return !!(val & bit);
-}
-
-static void adp5588_gpio_set_value(struct gpio_chip *chip,
- unsigned off, int val)
-{
- unsigned bank, bit;
- struct adp5588_gpio *dev = gpiochip_get_data(chip);
-
- bank = ADP5588_BANK(off);
- bit = ADP5588_BIT(off);
-
- mutex_lock(&dev->lock);
- if (val)
- dev->dat_out[bank] |= bit;
- else
- dev->dat_out[bank] &= ~bit;
-
- adp5588_gpio_write(dev->client, GPIO_DAT_OUT1 + bank,
- dev->dat_out[bank]);
- mutex_unlock(&dev->lock);
-}
-
-static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned off)
-{
- int ret;
- unsigned bank;
- struct adp5588_gpio *dev = gpiochip_get_data(chip);
-
- bank = ADP5588_BANK(off);
-
- mutex_lock(&dev->lock);
- dev->dir[bank] &= ~ADP5588_BIT(off);
- ret = adp5588_gpio_write(dev->client, GPIO_DIR1 + bank, dev->dir[bank]);
- mutex_unlock(&dev->lock);
-
- return ret;
-}
-
-static int adp5588_gpio_direction_output(struct gpio_chip *chip,
- unsigned off, int val)
-{
- int ret;
- unsigned bank, bit;
- struct adp5588_gpio *dev = gpiochip_get_data(chip);
-
- bank = ADP5588_BANK(off);
- bit = ADP5588_BIT(off);
-
- mutex_lock(&dev->lock);
- dev->dir[bank] |= bit;
-
- if (val)
- dev->dat_out[bank] |= bit;
- else
- dev->dat_out[bank] &= ~bit;
-
- ret = adp5588_gpio_write(dev->client, GPIO_DAT_OUT1 + bank,
- dev->dat_out[bank]);
- ret |= adp5588_gpio_write(dev->client, GPIO_DIR1 + bank,
- dev->dir[bank]);
- mutex_unlock(&dev->lock);
-
- return ret;
-}
-
-#ifdef CONFIG_GPIO_ADP5588_IRQ
-
-static void adp5588_irq_bus_lock(struct irq_data *d)
-{
- struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
- struct adp5588_gpio *dev = gpiochip_get_data(gc);
-
- mutex_lock(&dev->irq_lock);
-}
-
- /*
- * genirq core code can issue chip->mask/unmask from atomic context.
- * This doesn't work for slow busses where an access needs to sleep.
- * bus_sync_unlock() is therefore called outside the atomic context,
- * syncs the current irq mask state with the slow external controller
- * and unlocks the bus.
- */
-
-static void adp5588_irq_bus_sync_unlock(struct irq_data *d)
-{
- struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
- struct adp5588_gpio *dev = gpiochip_get_data(gc);
- int i;
-
- for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) {
- if (dev->int_input_en[i]) {
- mutex_lock(&dev->lock);
- dev->dir[i] &= ~dev->int_input_en[i];
- dev->int_input_en[i] = 0;
- adp5588_gpio_write(dev->client, GPIO_DIR1 + i,
- dev->dir[i]);
- mutex_unlock(&dev->lock);
- }
-
- if (dev->int_en[i] ^ dev->irq_mask[i]) {
- dev->int_en[i] = dev->irq_mask[i];
- adp5588_gpio_write(dev->client, GPI_EM1 + i,
- dev->int_en[i]);
- }
- }
-
- mutex_unlock(&dev->irq_lock);
-}
-
-static void adp5588_irq_mask(struct irq_data *d)
-{
- struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
- struct adp5588_gpio *dev = gpiochip_get_data(gc);
-
- dev->irq_mask[ADP5588_BANK(d->hwirq)] &= ~ADP5588_BIT(d->hwirq);
-}
-
-static void adp5588_irq_unmask(struct irq_data *d)
-{
- struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
- struct adp5588_gpio *dev = gpiochip_get_data(gc);
-
- dev->irq_mask[ADP5588_BANK(d->hwirq)] |= ADP5588_BIT(d->hwirq);
-}
-
-static int adp5588_irq_set_type(struct irq_data *d, unsigned int type)
-{
- struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
- struct adp5588_gpio *dev = gpiochip_get_data(gc);
- uint16_t gpio = d->hwirq;
- unsigned bank, bit;
-
- bank = ADP5588_BANK(gpio);
- bit = ADP5588_BIT(gpio);
-
- dev->int_lvl_low[bank] &= ~bit;
- dev->int_lvl_high[bank] &= ~bit;
-
- if (type & IRQ_TYPE_EDGE_BOTH || type & IRQ_TYPE_LEVEL_HIGH)
- dev->int_lvl_high[bank] |= bit;
-
- if (type & IRQ_TYPE_EDGE_BOTH || type & IRQ_TYPE_LEVEL_LOW)
- dev->int_lvl_low[bank] |= bit;
-
- dev->int_input_en[bank] |= bit;
-
- return 0;
-}
-
-static struct irq_chip adp5588_irq_chip = {
- .name = "adp5588",
- .irq_mask = adp5588_irq_mask,
- .irq_unmask = adp5588_irq_unmask,
- .irq_bus_lock = adp5588_irq_bus_lock,
- .irq_bus_sync_unlock = adp5588_irq_bus_sync_unlock,
- .irq_set_type = adp5588_irq_set_type,
-};
-
-static irqreturn_t adp5588_irq_handler(int irq, void *devid)
-{
- struct adp5588_gpio *dev = devid;
- int status = adp5588_gpio_read(dev->client, INT_STAT);
-
- if (status & ADP5588_KE_INT) {
- int ev_cnt = adp5588_gpio_read(dev->client, KEY_LCK_EC_STAT);
-
- if (ev_cnt > 0) {
- int i;
-
- for (i = 0; i < (ev_cnt & ADP5588_KEC); i++) {
- int key = adp5588_gpio_read(dev->client,
- Key_EVENTA + i);
- /* GPIN events begin at 97,
- * bit 7 indicates logic level
- */
- int gpio = (key & 0x7f) - 97;
- int lvl = key & (1 << 7);
- int bank = ADP5588_BANK(gpio);
- int bit = ADP5588_BIT(gpio);
-
- if ((lvl && dev->int_lvl_high[bank] & bit) ||
- (!lvl && dev->int_lvl_low[bank] & bit))
- handle_nested_irq(irq_find_mapping(
- dev->gpio_chip.irq.domain, gpio));
- }
- }
- }
-
- adp5588_gpio_write(dev->client, INT_STAT, status); /* Status is W1C */
-
- return IRQ_HANDLED;
-}
-
-
-static int adp5588_irq_init_hw(struct gpio_chip *gc)
-{
- struct adp5588_gpio *dev = gpiochip_get_data(gc);
- /* Enable IRQs after registering chip */
- adp5588_gpio_write(dev->client, CFG,
- ADP5588_AUTO_INC | ADP5588_INT_CFG | ADP5588_KE_IEN);
-
- return 0;
-}
-
-static int adp5588_irq_setup(struct adp5588_gpio *dev)
-{
- struct i2c_client *client = dev->client;
- int ret;
- struct adp5588_gpio_platform_data *pdata =
- dev_get_platdata(&client->dev);
- struct gpio_irq_chip *girq;
-
- adp5588_gpio_write(client, CFG, ADP5588_AUTO_INC);
- adp5588_gpio_write(client, INT_STAT, -1); /* status is W1C */
-
- mutex_init(&dev->irq_lock);
-
- ret = devm_request_threaded_irq(&client->dev, client->irq,
- NULL, adp5588_irq_handler, IRQF_ONESHOT
- | IRQF_TRIGGER_FALLING | IRQF_SHARED,
- dev_name(&client->dev), dev);
- if (ret) {
- dev_err(&client->dev, "failed to request irq %d\n",
- client->irq);
- return ret;
- }
-
- /* This will be registered in the call to devm_gpiochip_add_data() */
- girq = &dev->gpio_chip.irq;
- girq->chip = &adp5588_irq_chip;
- /* This will let us handle the parent IRQ in the driver */
- girq->parent_handler = NULL;
- girq->num_parents = 0;
- girq->parents = NULL;
- girq->first = pdata ? pdata->irq_base : 0;
- girq->default_type = IRQ_TYPE_NONE;
- girq->handler = handle_simple_irq;
- girq->init_hw = adp5588_irq_init_hw;
- girq->threaded = true;
-
- return 0;
-}
-
-#else
-static int adp5588_irq_setup(struct adp5588_gpio *dev)
-{
- struct i2c_client *client = dev->client;
- dev_warn(&client->dev, "interrupt support not compiled in\n");
-
- return 0;
-}
-
-#endif /* CONFIG_GPIO_ADP5588_IRQ */
-
-static int adp5588_gpio_probe(struct i2c_client *client)
-{
- struct adp5588_gpio_platform_data *pdata =
- dev_get_platdata(&client->dev);
- struct adp5588_gpio *dev;
- struct gpio_chip *gc;
- int ret, i, revid;
- unsigned int pullup_dis_mask = 0;
-
- if (!i2c_check_functionality(client->adapter,
- I2C_FUNC_SMBUS_BYTE_DATA)) {
- dev_err(&client->dev, "SMBUS Byte Data not Supported\n");
- return -EIO;
- }
-
- dev = devm_kzalloc(&client->dev, sizeof(*dev), GFP_KERNEL);
- if (!dev)
- return -ENOMEM;
-
- dev->client = client;
-
- gc = &dev->gpio_chip;
- gc->direction_input = adp5588_gpio_direction_input;
- gc->direction_output = adp5588_gpio_direction_output;
- gc->get = adp5588_gpio_get_value;
- gc->set = adp5588_gpio_set_value;
- gc->can_sleep = true;
- gc->base = -1;
- gc->parent = &client->dev;
-
- if (pdata) {
- gc->base = pdata->gpio_start;
- gc->names = pdata->names;
- pullup_dis_mask = pdata->pullup_dis_mask;
- }
-
- gc->ngpio = ADP5588_MAXGPIO;
- gc->label = client->name;
- gc->owner = THIS_MODULE;
-
- mutex_init(&dev->lock);
-
- ret = adp5588_gpio_read(dev->client, DEV_ID);
- if (ret < 0)
- return ret;
-
- revid = ret & ADP5588_DEVICE_ID_MASK;
-
- for (i = 0, ret = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) {
- dev->dat_out[i] = adp5588_gpio_read(client, GPIO_DAT_OUT1 + i);
- dev->dir[i] = adp5588_gpio_read(client, GPIO_DIR1 + i);
- ret |= adp5588_gpio_write(client, KP_GPIO1 + i, 0);
- ret |= adp5588_gpio_write(client, GPIO_PULL1 + i,
- (pullup_dis_mask >> (8 * i)) & 0xFF);
- ret |= adp5588_gpio_write(client, GPIO_INT_EN1 + i, 0);
- if (ret)
- return ret;
- }
-
- if (client->irq) {
- if (WA_DELAYED_READOUT_REVID(revid)) {
- dev_warn(&client->dev, "GPIO int not supported\n");
- } else {
- ret = adp5588_irq_setup(dev);
- if (ret)
- return ret;
- }
- }
-
- ret = devm_gpiochip_add_data(&client->dev, &dev->gpio_chip, dev);
- if (ret)
- return ret;
-
- i2c_set_clientdata(client, dev);
-
- return 0;
-}
-
-static void adp5588_gpio_remove(struct i2c_client *client)
-{
- struct adp5588_gpio *dev = i2c_get_clientdata(client);
-
- if (dev->client->irq)
- free_irq(dev->client->irq, dev);
-}
-
-static const struct i2c_device_id adp5588_gpio_id[] = {
- { "adp5588-gpio" },
- {}
-};
-MODULE_DEVICE_TABLE(i2c, adp5588_gpio_id);
-
-static const struct of_device_id adp5588_gpio_of_id[] = {
- { .compatible = "adi,adp5588-gpio" },
- {}
-};
-MODULE_DEVICE_TABLE(of, adp5588_gpio_of_id);
-
-static struct i2c_driver adp5588_gpio_driver = {
- .driver = {
- .name = "adp5588-gpio",
- .of_match_table = adp5588_gpio_of_id,
- },
- .probe_new = adp5588_gpio_probe,
- .remove = adp5588_gpio_remove,
- .id_table = adp5588_gpio_id,
-};
-
-module_i2c_driver(adp5588_gpio_driver);
-
-MODULE_AUTHOR("Michael Hennerich <michael.hennerich@analog.com>");
-MODULE_DESCRIPTION("GPIO ADP5588 Driver");
-MODULE_LICENSE("GPL");
pending = readl(gs->gpio_io + YU_GPIO_CAUSE_OR_CAUSE_EVTEN0);
writel(pending, gs->gpio_io + YU_GPIO_CAUSE_OR_CLRCAUSE);
- for_each_set_bit(level, &pending, gc->ngpio) {
- int gpio_irq = irq_find_mapping(gc->irq.domain, level);
- generic_handle_irq(gpio_irq);
- }
+ for_each_set_bit(level, &pending, gc->ngpio)
+ generic_handle_domain_irq_safe(gc->irq.domain, level);
return IRQ_RETVAL(pending);
}
#include <linux/of_address.h>
#include <linux/of_device.h>
#include <linux/of_irq.h>
+#include <linux/pinctrl/consumer.h>
#include <linux/pinctrl/pinconf-generic.h>
#include <linux/regmap.h>
unsigned long flags;
u32 data = input ? 0 : 1;
+
+ if (input)
+ pinctrl_gpio_direction_input(bank->pin_base + offset);
+ else
+ pinctrl_gpio_direction_output(bank->pin_base + offset);
+
raw_spin_lock_irqsave(&bank->slock, flags);
rockchip_gpio_writel_bit(bank, offset, data, bank->gpio_regs->port_ddr);
raw_spin_unlock_irqrestore(&bank->slock, flags);
return;
adev->kfd.dev = kgd2kfd_probe(adev, vf);
-
- if (adev->kfd.dev)
- amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
}
/**
adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev,
adev_to_drm(adev), &gpu_resources);
+ amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size;
+
INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work);
}
}
if (adev->kfd.dev) {
kgd2kfd_device_exit(adev->kfd.dev);
adev->kfd.dev = NULL;
+ amdgpu_amdkfd_total_mem_size -= adev->gmc.real_vram_size;
}
}
#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <drm/drm_crtc_helper.h>
-#include <drm/drm_damage_helper.h>
-#include <drm/drm_drv.h>
#include <drm/drm_edid.h>
#include <drm/drm_gem_framebuffer_helper.h>
#include <drm/drm_fb_helper.h>
.create_handle = drm_gem_fb_create_handle,
};
-static const struct drm_framebuffer_funcs amdgpu_fb_funcs_atomic = {
- .destroy = drm_gem_fb_destroy,
- .create_handle = drm_gem_fb_create_handle,
- .dirty = drm_atomic_helper_dirtyfb,
-};
-
uint32_t amdgpu_display_supported_domains(struct amdgpu_device *adev,
uint64_t bo_flags)
{
if (ret)
goto err;
- if (drm_drv_uses_atomic_modeset(dev))
- ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs_atomic);
- else
- ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs);
+ ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs);
+
if (ret)
goto err;
* num of amdgpu_vm_pt entries.
*/
BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
- bp->destroy = &amdgpu_bo_vm_destroy;
r = amdgpu_bo_create(adev, bp, &bo_ptr);
if (r)
return r;
*vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
INIT_LIST_HEAD(&(*vmbo_ptr)->shadow_list);
+ /* Set destroy callback to amdgpu_bo_vm_destroy after vmbo->shadow_list
+ * is initialized.
+ */
+ bo_ptr->tbo.destroy = &amdgpu_bo_vm_destroy;
return r;
}
err_data.err_addr =
kcalloc(adev->umc.max_ras_err_cnt_per_query,
sizeof(struct eeprom_table_record), GFP_KERNEL);
- if(!err_data.err_addr) {
- dev_warn(adev->dev, "Failed to alloc memory for "
- "umc error address record in mca notifier!\n");
+ if (!err_data.err_addr) {
+ dev_warn(adev->dev,
+ "Failed to alloc memory for umc error record in mca notifier!\n");
return NOTIFY_DONE;
}
if (adev->umc.ras &&
adev->umc.ras->convert_ras_error_address)
adev->umc.ras->convert_ras_error_address(adev,
- &err_data, 0, ch_inst, umc_inst, m->addr);
+ &err_data, m->addr, ch_inst, umc_inst);
if (amdgpu_bad_page_threshold != 0) {
amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
adev->sdma.instance[instance].fw->data;
version_major = le16_to_cpu(header->header_version_major);
- if ((duplicate && instance) || (!duplicate && version_major > 1))
- return -EINVAL;
+ if ((duplicate && instance) || (!duplicate && version_major > 1)) {
+ err = -EINVAL;
+ goto out;
+ }
err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
if (err)
ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
break;
default:
- return -EINVAL;
+ err = -EINVAL;
}
}
}
return err;
}
+
+void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev)
+{
+ struct amdgpu_ring *sdma;
+ int i;
+
+ for (i = 0; i < adev->sdma.num_instances; i++) {
+ if (adev->sdma.has_page_queue) {
+ sdma = &adev->sdma.instance[i].page;
+ if (adev->mman.buffer_funcs_ring == sdma) {
+ amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ break;
+ }
+ }
+ sdma = &adev->sdma.instance[i].ring;
+ if (adev->mman.buffer_funcs_ring == sdma) {
+ amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ break;
+ }
+ }
+}
char *fw_name, u32 instance, bool duplicate);
void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
bool duplicate);
+void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev);
+
#endif
static bool amdgpu_mem_visible(struct amdgpu_device *adev,
struct ttm_resource *mem)
{
- uint64_t mem_size = (u64)mem->num_pages << PAGE_SHIFT;
+ u64 mem_size = (u64)mem->num_pages << PAGE_SHIFT;
struct amdgpu_res_cursor cursor;
+ u64 end;
if (mem->mem_type == TTM_PL_SYSTEM ||
mem->mem_type == TTM_PL_TT)
return false;
amdgpu_res_first(mem, 0, mem_size, &cursor);
+ end = cursor.start + cursor.size;
+ while (cursor.remaining) {
+ amdgpu_res_next(&cursor, cursor.size);
- /* ttm_resource_ioremap only supports contiguous memory */
- if (cursor.size != mem_size)
- return false;
+ /* ttm_resource_ioremap only supports contiguous memory */
+ if (end != cursor.start)
+ return false;
+
+ end = cursor.start + cursor.size;
+ }
- return cursor.start + cursor.size <= adev->gmc.visible_vram_size;
+ return end <= adev->gmc.visible_vram_size;
}
/*
#define __AMDGPU_UMC_H__
#include "amdgpu_ras.h"
-#define UMC_INVALID_ADDR 0x1ULL
-
/*
* (addr / 256) * 4096, the higher 26 bits in ErrorAddr
* is the index of 4KB block
void (*err_cnt_init)(struct amdgpu_device *adev);
bool (*query_ras_poison_mode)(struct amdgpu_device *adev);
void (*convert_ras_error_address)(struct amdgpu_device *adev,
- struct ras_err_data *err_data,
- uint32_t umc_reg_offset, uint32_t ch_inst,
- uint32_t umc_inst, uint64_t mca_addr);
+ struct ras_err_data *err_data, uint64_t err_addr,
+ uint32_t ch_inst, uint32_t umc_inst);
void (*ecc_info_query_ras_error_count)(struct amdgpu_device *adev,
void *ras_error_status);
void (*ecc_info_query_ras_error_address)(struct amdgpu_device *adev,
*/
static void cik_sdma_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl;
int i;
- if ((adev->mman.buffer_funcs_ring == sdma0) ||
- (adev->mman.buffer_funcs_ring == sdma1))
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
*/
static void sdma_v2_4_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl, ib_cntl;
int i;
- if ((adev->mman.buffer_funcs_ring == sdma0) ||
- (adev->mman.buffer_funcs_ring == sdma1))
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
*/
static void sdma_v3_0_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl, ib_cntl;
int i;
- if ((adev->mman.buffer_funcs_ring == sdma0) ||
- (adev->mman.buffer_funcs_ring == sdma1))
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
*/
static void sdma_v4_0_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
u32 rb_cntl, ib_cntl;
- int i, unset = 0;
-
- for (i = 0; i < adev->sdma.num_instances; i++) {
- sdma[i] = &adev->sdma.instance[i].ring;
+ int i;
- if ((adev->mman.buffer_funcs_ring == sdma[i]) && unset != 1) {
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
- unset = 1;
- }
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
+ for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL);
rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);
*/
static void sdma_v4_0_page_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
u32 rb_cntl, ib_cntl;
int i;
- bool unset = false;
-
- for (i = 0; i < adev->sdma.num_instances; i++) {
- sdma[i] = &adev->sdma.instance[i].page;
- if ((adev->mman.buffer_funcs_ring == sdma[i]) &&
- (!unset)) {
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
- unset = true;
- }
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
+ for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL);
rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,
RB_ENABLE, 0);
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
int i;
- if (amdgpu_sriov_vf(adev))
+ if (amdgpu_sriov_vf(adev)) {
+ /* disable the scheduler for SDMA */
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
return 0;
+ }
for (i = 0; i < adev->sdma.num_instances; i++) {
amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
*/
static void sdma_v5_0_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl, ib_cntl;
int i;
- if ((adev->mman.buffer_funcs_ring == sdma0) ||
- (adev->mman.buffer_funcs_ring == sdma1))
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- if (amdgpu_sriov_vf(adev))
+ if (amdgpu_sriov_vf(adev)) {
+ /* disable the scheduler for SDMA */
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
return 0;
+ }
sdma_v5_0_ctx_switch_enable(adev, false);
sdma_v5_0_enable(adev, false);
*/
static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
- struct amdgpu_ring *sdma2 = &adev->sdma.instance[2].ring;
- struct amdgpu_ring *sdma3 = &adev->sdma.instance[3].ring;
u32 rb_cntl, ib_cntl;
int i;
- if ((adev->mman.buffer_funcs_ring == sdma0) ||
- (adev->mman.buffer_funcs_ring == sdma1) ||
- (adev->mman.buffer_funcs_ring == sdma2) ||
- (adev->mman.buffer_funcs_ring == sdma3))
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- if (amdgpu_sriov_vf(adev))
+ if (amdgpu_sriov_vf(adev)) {
+ /* disable the scheduler for SDMA */
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
return 0;
+ }
sdma_v5_2_ctx_switch_enable(adev, false);
sdma_v5_2_enable(adev, false);
*/
static void sdma_v6_0_gfx_stop(struct amdgpu_device *adev)
{
- struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;
- struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;
u32 rb_cntl, ib_cntl;
int i;
- if ((adev->mman.buffer_funcs_ring == sdma0) ||
- (adev->mman.buffer_funcs_ring == sdma1))
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
for (i = 0; i < adev->sdma.num_instances; i++) {
rb_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 0);
WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
}
-
- sdma0->sched.ready = false;
- sdma1->sched.ready = false;
}
/**
m->sdmax_rlcx_rb_cntl =
order_base_2(prop->queue_size / 4) << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
- 4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+ 4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
+ 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT;
m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- if (amdgpu_sriov_vf(adev))
+ if (amdgpu_sriov_vf(adev)) {
+ /* disable the scheduler for SDMA */
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
return 0;
+ }
sdma_v6_0_ctx_switch_enable(adev, false);
sdma_v6_0_enable(adev, false);
u32 rb_cntl;
unsigned i;
+ amdgpu_sdma_unset_buffer_funcs_helper(adev);
+
for (i = 0; i < adev->sdma.num_instances; i++) {
ring = &adev->sdma.instance[i].ring;
/* dma0 */
rb_cntl = RREG32(DMA_RB_CNTL + sdma_offsets[i]);
rb_cntl &= ~DMA_RB_ENABLE;
WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl);
-
- if (adev->mman.buffer_funcs_ring == ring)
- amdgpu_ttm_set_buffer_funcs_status(adev, false);
}
}
AMD_CG_SUPPORT_JPEG_MGCG;
adev->pg_flags =
AMD_PG_SUPPORT_GFX_PG |
+ AMD_PG_SUPPORT_VCN |
AMD_PG_SUPPORT_VCN_DPG |
AMD_PG_SUPPORT_JPEG;
adev->external_rev_id = adev->rev_id + 0x1;
return;
}
- /* calculate error address if ue/ce error is detected */
+ /* calculate error address if ue error is detected */
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
- (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
- REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
+ REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
/* the lowest lsb bits should be ignored */
ADDR_OF_256B_BLOCK(channel_index) |
OFFSET_IN_256B_BLOCK(err_addr);
- /* we only save ue error information currently, ce is skipped */
- if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
- == 1)
- amdgpu_umc_fill_error_record(err_data, err_addr,
+ amdgpu_umc_fill_error_record(err_data, err_addr,
retired_page, channel_index, umc_inst);
}
}
}
+static void umc_v6_7_convert_error_address(struct amdgpu_device *adev,
+ struct ras_err_data *err_data, uint64_t err_addr,
+ uint32_t ch_inst, uint32_t umc_inst)
+{
+ uint32_t channel_index;
+ uint64_t soc_pa, retired_page, column;
+
+ channel_index =
+ adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
+ /* translate umc channel address to soc pa, 3 parts are included */
+ soc_pa = ADDR_OF_8KB_BLOCK(err_addr) |
+ ADDR_OF_256B_BLOCK(channel_index) |
+ OFFSET_IN_256B_BLOCK(err_addr);
+
+ /* The umc channel bits are not original values, they are hashed */
+ SET_CHANNEL_HASH(channel_index, soc_pa);
+
+ /* clear [C4 C3 C2] in soc physical address */
+ soc_pa &= ~(0x7ULL << UMC_V6_7_PA_C2_BIT);
+
+ /* loop for all possibilities of [C4 C3 C2] */
+ for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) {
+ retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT);
+ dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page);
+ amdgpu_umc_fill_error_record(err_data, err_addr,
+ retired_page, channel_index, umc_inst);
+
+ /* shift R14 bit */
+ retired_page ^= (0x1ULL << UMC_V6_7_PA_R14_BIT);
+ dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page);
+ amdgpu_umc_fill_error_record(err_data, err_addr,
+ retired_page, channel_index, umc_inst);
+ }
+}
+
static void umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev,
struct ras_err_data *err_data,
uint32_t ch_inst,
uint32_t umc_inst)
{
- uint64_t mc_umc_status, err_addr, soc_pa, retired_page, column;
- uint32_t channel_index;
+ uint64_t mc_umc_status, err_addr;
uint32_t eccinfo_table_idx;
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
eccinfo_table_idx = umc_inst * adev->umc.channel_inst_num + ch_inst;
- channel_index =
- adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
-
mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status;
if (mc_umc_status == 0)
if (!err_data->err_addr)
return;
- /* calculate error address if ue/ce error is detected */
+ /* calculate error address if ue error is detected */
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
- (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
- REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
+ REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
err_addr = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_addr;
err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
- /* translate umc channel address to soc pa, 3 parts are included */
- soc_pa = ADDR_OF_8KB_BLOCK(err_addr) |
- ADDR_OF_256B_BLOCK(channel_index) |
- OFFSET_IN_256B_BLOCK(err_addr);
-
- /* The umc channel bits are not original values, they are hashed */
- SET_CHANNEL_HASH(channel_index, soc_pa);
-
- /* clear [C4 C3 C2] in soc physical address */
- soc_pa &= ~(0x7ULL << UMC_V6_7_PA_C2_BIT);
-
- /* we only save ue error information currently, ce is skipped */
- if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
- == 1) {
- /* loop for all possibilities of [C4 C3 C2] */
- for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) {
- retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT);
- dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page);
- amdgpu_umc_fill_error_record(err_data, err_addr,
- retired_page, channel_index, umc_inst);
-
- /* shift R14 bit */
- retired_page ^= (0x1ULL << UMC_V6_7_PA_R14_BIT);
- dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page);
- amdgpu_umc_fill_error_record(err_data, err_addr,
- retired_page, channel_index, umc_inst);
- }
- }
+ umc_v6_7_convert_error_address(adev, err_data, err_addr,
+ ch_inst, umc_inst);
}
}
static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
struct ras_err_data *err_data,
uint32_t umc_reg_offset, uint32_t ch_inst,
- uint32_t umc_inst, uint64_t mca_addr)
+ uint32_t umc_inst)
{
uint32_t mc_umc_status_addr;
- uint32_t channel_index;
- uint64_t mc_umc_status = 0, mc_umc_addrt0;
- uint64_t err_addr, soc_pa, retired_page, column;
+ uint64_t mc_umc_status = 0, mc_umc_addrt0, err_addr;
- if (mca_addr == UMC_INVALID_ADDR) {
- mc_umc_status_addr =
- SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
- mc_umc_addrt0 =
- SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
+ mc_umc_status_addr =
+ SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
+ mc_umc_addrt0 =
+ SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
- mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
+ mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
- if (mc_umc_status == 0)
- return;
+ if (mc_umc_status == 0)
+ return;
- if (!err_data->err_addr) {
- /* clear umc status */
- WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
- return;
- }
+ if (!err_data->err_addr) {
+ /* clear umc status */
+ WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
+ return;
}
- channel_index =
- adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
-
- /* calculate error address if ue/ce error is detected */
- if ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
- (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
- REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) ||
- mca_addr != UMC_INVALID_ADDR) {
- if (mca_addr == UMC_INVALID_ADDR) {
- err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
- err_addr =
- REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
- } else {
- err_addr = mca_addr;
- }
+ /* calculate error address if ue error is detected */
+ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
+ REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
+ err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
+ err_addr =
+ REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
- /* translate umc channel address to soc pa, 3 parts are included */
- soc_pa = ADDR_OF_8KB_BLOCK(err_addr) |
- ADDR_OF_256B_BLOCK(channel_index) |
- OFFSET_IN_256B_BLOCK(err_addr);
-
- /* The umc channel bits are not original values, they are hashed */
- SET_CHANNEL_HASH(channel_index, soc_pa);
-
- /* clear [C4 C3 C2] in soc physical address */
- soc_pa &= ~(0x7ULL << UMC_V6_7_PA_C2_BIT);
-
- /* we only save ue error information currently, ce is skipped */
- if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
- == 1 ||
- mca_addr != UMC_INVALID_ADDR) {
- /* loop for all possibilities of [C4 C3 C2] */
- for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) {
- retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT);
- dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page);
- amdgpu_umc_fill_error_record(err_data, err_addr,
- retired_page, channel_index, umc_inst);
-
- /* shift R14 bit */
- retired_page ^= (0x1ULL << UMC_V6_7_PA_R14_BIT);
- dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page);
- amdgpu_umc_fill_error_record(err_data, err_addr,
- retired_page, channel_index, umc_inst);
- }
- }
+ umc_v6_7_convert_error_address(adev, err_data, err_addr,
+ ch_inst, umc_inst);
}
/* clear umc status */
- if (mca_addr == UMC_INVALID_ADDR)
- WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
+ WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
}
static void umc_v6_7_query_ras_error_address(struct amdgpu_device *adev,
umc_v6_7_query_error_address(adev,
err_data,
umc_reg_offset, ch_inst,
- umc_inst, UMC_INVALID_ADDR);
+ umc_inst);
}
}
.query_ras_poison_mode = umc_v6_7_query_ras_poison_mode,
.ecc_info_query_ras_error_count = umc_v6_7_ecc_info_query_ras_error_count,
.ecc_info_query_ras_error_address = umc_v6_7_ecc_info_query_ras_error_address,
- .convert_ras_error_address = umc_v6_7_query_error_address,
+ .convert_ras_error_address = umc_v6_7_convert_error_address,
};
{
uint64_t mc_umc_status_addr;
uint64_t mc_umc_status, err_addr;
- uint32_t channel_index;
+ uint64_t mc_umc_addrt0, na_err_addr_base;
+ uint64_t na_err_addr, retired_page_addr;
+ uint32_t channel_index, addr_lsb, col = 0;
+ int ret = 0;
mc_umc_status_addr =
SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
umc_inst * adev->umc.channel_inst_num +
ch_inst];
- /* calculate error address if ue/ce error is detected */
+ /* calculate error address if ue error is detected */
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, AddrV) == 1 &&
- (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
- REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
- uint32_t addr_lsb;
- uint64_t mc_umc_addrt0;
+ REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
mc_umc_addrt0 = SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
/* the lowest lsb bits should be ignored */
addr_lsb = REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, AddrLsb);
-
err_addr &= ~((0x1ULL << addr_lsb) - 1);
-
- /* we only save ue error information currently, ce is skipped */
- if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
- uint64_t na_err_addr_base = err_addr & ~(0x3ULL << UMC_V8_10_NA_C5_BIT);
- uint64_t na_err_addr, retired_page_addr;
- uint32_t col = 0;
- int ret = 0;
-
- /* loop for all possibilities of [C6 C5] in normal address. */
- for (col = 0; col < UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM; col++) {
- na_err_addr = na_err_addr_base | (col << UMC_V8_10_NA_C5_BIT);
-
- /* Mapping normal error address to retired soc physical address. */
- ret = umc_v8_10_swizzle_mode_na_to_pa(adev, channel_index,
- na_err_addr, &retired_page_addr);
- if (ret) {
- dev_err(adev->dev, "Failed to map pa from umc na.\n");
- break;
- }
- dev_info(adev->dev, "Error Address(PA): 0x%llx\n",
- retired_page_addr);
- amdgpu_umc_fill_error_record(err_data, na_err_addr,
- retired_page_addr, channel_index, umc_inst);
+ na_err_addr_base = err_addr & ~(0x3ULL << UMC_V8_10_NA_C5_BIT);
+
+ /* loop for all possibilities of [C6 C5] in normal address. */
+ for (col = 0; col < UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM; col++) {
+ na_err_addr = na_err_addr_base | (col << UMC_V8_10_NA_C5_BIT);
+
+ /* Mapping normal error address to retired soc physical address. */
+ ret = umc_v8_10_swizzle_mode_na_to_pa(adev, channel_index,
+ na_err_addr, &retired_page_addr);
+ if (ret) {
+ dev_err(adev->dev, "Failed to map pa from umc na.\n");
+ break;
}
+ dev_info(adev->dev, "Error Address(PA): 0x%llx\n",
+ retired_page_addr);
+ amdgpu_umc_fill_error_record(err_data, na_err_addr,
+ retired_page_addr, channel_index, umc_inst);
}
}
}
}
+static uint32_t umc_v8_10_query_ras_poison_mode_per_channel(
+ struct amdgpu_device *adev,
+ uint32_t umc_reg_offset)
+{
+ uint32_t ecc_ctrl_addr, ecc_ctrl;
+
+ ecc_ctrl_addr =
+ SOC15_REG_OFFSET(UMC, 0, regUMCCH0_0_GeccCtrl);
+ ecc_ctrl = RREG32_PCIE((ecc_ctrl_addr +
+ umc_reg_offset) * 4);
+
+ return REG_GET_FIELD(ecc_ctrl, UMCCH0_0_GeccCtrl, UCFatalEn);
+}
+
+static bool umc_v8_10_query_ras_poison_mode(struct amdgpu_device *adev)
+{
+ uint32_t umc_reg_offset = 0;
+
+ /* Enabling fatal error in umc node0 instance0 channel0 will be
+ * considered as fatal error mode
+ */
+ umc_reg_offset = get_umc_v8_10_reg_offset(adev, 0, 0, 0);
+ return !umc_v8_10_query_ras_poison_mode_per_channel(adev, umc_reg_offset);
+}
+
const struct amdgpu_ras_block_hw_ops umc_v8_10_ras_hw_ops = {
.query_ras_error_count = umc_v8_10_query_ras_error_count,
.query_ras_error_address = umc_v8_10_query_ras_error_address,
.hw_ops = &umc_v8_10_ras_hw_ops,
},
.err_cnt_init = umc_v8_10_err_cnt_init,
+ .query_ras_poison_mode = umc_v8_10_query_ras_poison_mode,
};
}
}
+static void umc_v8_7_convert_error_address(struct amdgpu_device *adev,
+ struct ras_err_data *err_data, uint64_t err_addr,
+ uint32_t ch_inst, uint32_t umc_inst)
+{
+ uint64_t retired_page;
+ uint32_t channel_index;
+
+ channel_index =
+ adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
+
+ /* translate umc channel address to soc pa, 3 parts are included */
+ retired_page = ADDR_OF_4KB_BLOCK(err_addr) |
+ ADDR_OF_256B_BLOCK(channel_index) |
+ OFFSET_IN_256B_BLOCK(err_addr);
+
+ amdgpu_umc_fill_error_record(err_data, err_addr,
+ retired_page, channel_index, umc_inst);
+}
+
static void umc_v8_7_ecc_info_query_error_address(struct amdgpu_device *adev,
struct ras_err_data *err_data,
uint32_t ch_inst,
uint32_t umc_inst)
{
- uint64_t mc_umc_status, err_addr, retired_page;
- uint32_t channel_index;
+ uint64_t mc_umc_status, err_addr;
uint32_t eccinfo_table_idx;
struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
eccinfo_table_idx = umc_inst * adev->umc.channel_inst_num + ch_inst;
- channel_index =
- adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
-
mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status;
if (mc_umc_status == 0)
if (!err_data->err_addr)
return;
- /* calculate error address if ue/ce error is detected */
+ /* calculate error address if ue error is detected */
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
- (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
- REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
+ REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
err_addr = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_addr;
err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
- /* translate umc channel address to soc pa, 3 parts are included */
- retired_page = ADDR_OF_4KB_BLOCK(err_addr) |
- ADDR_OF_256B_BLOCK(channel_index) |
- OFFSET_IN_256B_BLOCK(err_addr);
-
- /* we only save ue error information currently, ce is skipped */
- if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
- == 1)
- amdgpu_umc_fill_error_record(err_data, err_addr,
- retired_page, channel_index, umc_inst);
+ umc_v8_7_convert_error_address(adev, err_data, err_addr,
+ ch_inst, umc_inst);
}
}
uint32_t umc_inst)
{
uint32_t lsb, mc_umc_status_addr;
- uint64_t mc_umc_status, err_addr, retired_page, mc_umc_addrt0;
- uint32_t channel_index = adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
+ uint64_t mc_umc_status, err_addr, mc_umc_addrt0;
mc_umc_status_addr =
SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_STATUST0);
mc_umc_addrt0 =
SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_ADDRT0);
-
mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
if (mc_umc_status == 0)
return;
}
- /* calculate error address if ue/ce error is detected */
+ /* calculate error address if ue error is detected */
if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
- (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
- REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
+ REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) {
err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
/* the lowest lsb bits should be ignored */
err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
err_addr &= ~((0x1ULL << lsb) - 1);
- /* translate umc channel address to soc pa, 3 parts are included */
- retired_page = ADDR_OF_4KB_BLOCK(err_addr) |
- ADDR_OF_256B_BLOCK(channel_index) |
- OFFSET_IN_256B_BLOCK(err_addr);
-
- /* we only save ue error information currently, ce is skipped */
- if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
- == 1)
- amdgpu_umc_fill_error_record(err_data, err_addr,
- retired_page, channel_index, umc_inst);
+ umc_v8_7_convert_error_address(adev, err_data, err_addr,
+ ch_inst, umc_inst);
}
/* clear umc status */
page = pfn_to_page(pfn);
svm_range_bo_ref(prange->svm_bo);
page->zone_device_data = prange->svm_bo;
- lock_page(page);
+ zone_device_page_init(page);
}
static void
uint64_t npages = (end - start) >> PAGE_SHIFT;
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
- struct migrate_vma migrate;
+ struct migrate_vma migrate = { 0 };
unsigned long cpages = 0;
dma_addr_t *scratch;
void *buf;
static long
svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
struct vm_area_struct *vma, uint64_t start, uint64_t end,
- uint32_t trigger)
+ uint32_t trigger, struct page *fault_page)
{
struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
uint64_t npages = (end - start) >> PAGE_SHIFT;
unsigned long cpages = 0;
struct kfd_process_device *pdd;
struct dma_fence *mfence = NULL;
- struct migrate_vma migrate;
+ struct migrate_vma migrate = { 0 };
dma_addr_t *scratch;
void *buf;
int r = -ENOMEM;
migrate.src = buf;
migrate.dst = migrate.src + npages;
+ migrate.fault_page = fault_page;
scratch = (dma_addr_t *)(migrate.dst + npages);
kfd_smi_event_migration_start(adev->kfd.dev, p->lead_thread->pid,
* 0 - OK, otherwise error code
*/
int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
- uint32_t trigger)
+ uint32_t trigger, struct page *fault_page)
{
struct amdgpu_device *adev;
struct vm_area_struct *vma;
}
next = min(vma->vm_end, end);
- r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next, trigger);
+ r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next, trigger,
+ fault_page);
if (r < 0) {
pr_debug("failed %ld to migrate prange %p\n", r, prange);
break;
pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
do {
- r = svm_migrate_vram_to_ram(prange, mm, trigger);
+ r = svm_migrate_vram_to_ram(prange, mm, trigger, NULL);
if (r)
return r;
} while (prange->actual_loc && --retries);
}
r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm,
- KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU);
+ KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU,
+ vmf->page);
if (r)
pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",
r, prange->svms, prange, prange->start, prange->last);
int svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
struct mm_struct *mm, uint32_t trigger);
int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
- uint32_t trigger);
+ uint32_t trigger, struct page *fault_page);
unsigned long
svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);
<< SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
q->vmid << SDMA0_QUEUE0_RB_CNTL__RB_VMID__SHIFT |
1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
- 6 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+ 6 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
+ 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT;
m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
*/
if (prange->actual_loc)
r = svm_migrate_vram_to_ram(prange, mm,
- KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU);
+ KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU,
+ NULL);
else
r = 0;
}
} else {
r = svm_migrate_vram_to_ram(prange, mm,
- KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU);
+ KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU,
+ NULL);
}
if (r) {
pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n",
return 0;
if (!best_loc) {
- r = svm_migrate_vram_to_ram(prange, mm, KFD_MIGRATE_TRIGGER_PREFETCH);
+ r = svm_migrate_vram_to_ram(prange, mm,
+ KFD_MIGRATE_TRIGGER_PREFETCH, NULL);
*migrated = !r;
return r;
}
mutex_lock(&prange->migrate_mutex);
do {
r = svm_migrate_vram_to_ram(prange, mm,
- KFD_MIGRATE_TRIGGER_TTM_EVICTION);
+ KFD_MIGRATE_TRIGGER_TTM_EVICTION, NULL);
} while (!r && prange->actual_loc && --retries);
if (!r && prange->actual_loc)
hw_params.fb[i] = &fb_info->fb[i];
switch (adev->ip_versions[DCE_HWIP][0]) {
- case IP_VERSION(3, 1, 3): /* Only for this asic hw internal rev B0 */
+ case IP_VERSION(3, 1, 3):
+ case IP_VERSION(3, 1, 4):
hw_params.dpia_supported = true;
hw_params.disable_dpia = adev->dm.dc->debug.dpia_debug.bits.disable_dpia;
break;
* We also need vupdate irq for the actual core vblank handling
* at end of vblank.
*/
- dm_set_vupdate_irq(new_state->base.crtc, true);
- drm_crtc_vblank_get(new_state->base.crtc);
+ WARN_ON(dm_set_vupdate_irq(new_state->base.crtc, true) != 0);
+ WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0);
DRM_DEBUG_DRIVER("%s: crtc=%u VRR off->on: Get vblank ref\n",
__func__, new_state->base.crtc->base.id);
} else if (old_vrr_active && !new_vrr_active) {
/* Transition VRR active -> inactive:
* Allow vblank irq disable again for fixed refresh rate.
*/
- dm_set_vupdate_irq(new_state->base.crtc, false);
+ WARN_ON(dm_set_vupdate_irq(new_state->base.crtc, false) != 0);
drm_crtc_vblank_put(new_state->base.crtc);
DRM_DEBUG_DRIVER("%s: crtc=%u VRR on->off: Drop vblank ref\n",
__func__, new_state->base.crtc->base.id);
mutex_unlock(&dm->dc_lock);
}
- /* Count number of newly disabled CRTCs for dropping PM refs later. */
- for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
- new_crtc_state, i) {
- if (old_crtc_state->active && !new_crtc_state->active)
- crtc_disable_count++;
-
- dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
- dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
-
- /* For freesync config update on crtc state and params for irq */
- update_stream_irq_parameters(dm, dm_new_crtc_state);
-
- /* Handle vrr on->off / off->on transitions */
- amdgpu_dm_handle_vrr_transition(dm_old_crtc_state,
- dm_new_crtc_state);
- }
-
/**
* Enable interrupts for CRTCs that are newly enabled or went through
* a modeset. It was intentionally deferred until after the front end
for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
#ifdef CONFIG_DEBUG_FS
- bool configure_crc = false;
enum amdgpu_dm_pipe_crc_source cur_crc_src;
#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
- struct crc_rd_work *crc_rd_wrk = dm->crc_rd_wrk;
+ struct crc_rd_work *crc_rd_wrk;
+#endif
+#endif
+ /* Count number of newly disabled CRTCs for dropping PM refs later. */
+ if (old_crtc_state->active && !new_crtc_state->active)
+ crtc_disable_count++;
+
+ dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+ dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
+
+ /* For freesync config update on crtc state and params for irq */
+ update_stream_irq_parameters(dm, dm_new_crtc_state);
+
+#ifdef CONFIG_DEBUG_FS
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+ crc_rd_wrk = dm->crc_rd_wrk;
#endif
spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
cur_crc_src = acrtc->dm_irq_params.crc_src;
spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
#endif
- dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
if (new_crtc_state->active &&
(!old_crtc_state->active ||
dc_stream_retain(dm_new_crtc_state->stream);
acrtc->dm_irq_params.stream = dm_new_crtc_state->stream;
manage_dm_interrupts(adev, acrtc, true);
+ }
+ /* Handle vrr on->off / off->on transitions */
+ amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, dm_new_crtc_state);
#ifdef CONFIG_DEBUG_FS
+ if (new_crtc_state->active &&
+ (!old_crtc_state->active ||
+ drm_atomic_crtc_needs_modeset(new_crtc_state))) {
/**
* Frontend may have changed so reapply the CRC capture
* settings for the stream.
*/
- dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
-
if (amdgpu_dm_is_valid_crc_source(cur_crc_src)) {
- configure_crc = true;
#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
if (amdgpu_dm_crc_window_is_activated(crtc)) {
spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
}
#endif
- }
-
- if (configure_crc)
if (amdgpu_dm_crtc_configure_crc_source(
crtc, dm_new_crtc_state, cur_crc_src))
DRM_DEBUG_DRIVER("Failed to configure crc source");
-#endif
+ }
}
+#endif
}
for_each_new_crtc_in_state(state, crtc, new_crtc_state, j)
}
}
}
- if (!pre_validate_dsc(state, &dm_state, vars)) {
- ret = -EINVAL;
- goto fail;
- }
}
#endif
for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
}
}
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+ if (dc_resource_is_dsc_encoding_supported(dc)) {
+ if (!pre_validate_dsc(state, &dm_state, vars)) {
+ ret = -EINVAL;
+ goto fail;
+ }
+ }
+#endif
+
/* Run this here since we want to validate the streams we created */
ret = drm_atomic_helper_check_planes(dev, state);
if (ret) {
*/
void amdgpu_dm_set_psr_caps(struct dc_link *link)
{
- if (!(link->connector_signal & SIGNAL_TYPE_EDP))
+ if (!(link->connector_signal & SIGNAL_TYPE_EDP)) {
+ link->psr_settings.psr_feature_enabled = false;
return;
+ }
- if (link->type == dc_connection_none)
+ if (link->type == dc_connection_none) {
+ link->psr_settings.psr_feature_enabled = false;
return;
+ }
if (link->dpcd_caps.psr_info.psr_version == 0) {
link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
#define LAST_RECORD_TYPE 0xff
#define SMU9_SYSPLL0_ID 0
-struct i2c_id_config_access {
- uint8_t bfI2C_LineMux:4;
- uint8_t bfHW_EngineID:3;
- uint8_t bfHW_Capable:1;
- uint8_t ucAccess;
-};
-
static enum bp_result get_gpio_i2c_info(struct bios_parser *bp,
struct atom_i2c_record *record,
struct graphics_object_i2c_info *info);
} else if (dispclk_wdivider == 127 && current_dispclk_wdivider != 127) {
REG_UPDATE(DENTIST_DISPCLK_CNTL,
DENTIST_DISPCLK_WDIVIDER, 126);
- REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 100);
+ REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000);
for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
struct dccg *dccg = clk_mgr->base.ctx->dc->res_pool->dccg;
REG_UPDATE(DENTIST_DISPCLK_CNTL,
DENTIST_DISPCLK_WDIVIDER, dispclk_wdivider);
- REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 1000);
+ REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000);
REG_UPDATE(DENTIST_DISPCLK_CNTL,
DENTIST_DPPCLK_WDIVIDER, dppclk_wdivider);
REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_CHG_DONE, 1, 5, 100);
if (!clk_mgr->smu_present)
return;
- if (!clk_mgr->base.ctx->dc->debug.enable_z9_disable_interface &&
- (support == DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY))
- support = DCN_ZSTATE_SUPPORT_DISALLOW;
-
-
// Arg[15:0] = 8/9/0 for Z8/Z9/disallow -> existing bits
// Arg[16] = Disallow Z9 -> new bit
switch (support) {
case DCN_ZSTATE_SUPPORT_ALLOW:
msg_id = VBIOSSMC_MSG_AllowZstatesEntry;
- param = 9;
+ param = (1 << 10) | (1 << 9) | (1 << 8);
break;
case DCN_ZSTATE_SUPPORT_DISALLOW:
msg_id = VBIOSSMC_MSG_AllowZstatesEntry;
- param = 8;
+ param = 0;
break;
case DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY:
msg_id = VBIOSSMC_MSG_AllowZstatesEntry;
- param = 0x00010008;
+ param = (1 << 10);
break;
default: //DCN_ZSTATE_SUPPORT_UNKNOWN
{
struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
unsigned int num_levels;
- unsigned int num_dcfclk_levels, num_dtbclk_levels, num_dispclk_levels;
+ struct clk_limit_num_entries *num_entries_per_clk = &clk_mgr_base->bw_params->clk_table.num_entries_per_clk;
memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks));
clk_mgr_base->clks.p_state_change_support = true;
/* DCFCLK */
dcn32_init_single_clock(clk_mgr, PPCLK_DCFCLK,
&clk_mgr_base->bw_params->clk_table.entries[0].dcfclk_mhz,
- &num_levels);
- num_dcfclk_levels = num_levels;
+ &num_entries_per_clk->num_dcfclk_levels);
/* SOCCLK */
dcn32_init_single_clock(clk_mgr, PPCLK_SOCCLK,
&clk_mgr_base->bw_params->clk_table.entries[0].socclk_mhz,
- &num_levels);
+ &num_entries_per_clk->num_socclk_levels);
+
/* DTBCLK */
if (!clk_mgr->base.ctx->dc->debug.disable_dtb_ref_clk_switch)
dcn32_init_single_clock(clk_mgr, PPCLK_DTBCLK,
&clk_mgr_base->bw_params->clk_table.entries[0].dtbclk_mhz,
- &num_levels);
- num_dtbclk_levels = num_levels;
+ &num_entries_per_clk->num_dtbclk_levels);
/* DISPCLK */
dcn32_init_single_clock(clk_mgr, PPCLK_DISPCLK,
&clk_mgr_base->bw_params->clk_table.entries[0].dispclk_mhz,
- &num_levels);
- num_dispclk_levels = num_levels;
+ &num_entries_per_clk->num_dispclk_levels);
+ num_levels = num_entries_per_clk->num_dispclk_levels;
- if (num_dcfclk_levels && num_dtbclk_levels && num_dispclk_levels)
+ if (num_entries_per_clk->num_dcfclk_levels &&
+ num_entries_per_clk->num_dtbclk_levels &&
+ num_entries_per_clk->num_dispclk_levels)
clk_mgr->dpm_present = true;
if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) {
if (enter_display_off == safe_to_lower)
dcn30_smu_set_num_of_displays(clk_mgr, display_count);
+ clk_mgr_base->clks.fclk_prev_p_state_change_support = clk_mgr_base->clks.fclk_p_state_change_support;
+
+ total_plane_count = clk_mgr_helper_get_active_plane_cnt(dc, context);
+ fclk_p_state_change_support = new_clocks->fclk_p_state_change_support || (total_plane_count == 0);
+
+ if (should_update_pstate_support(safe_to_lower, fclk_p_state_change_support, clk_mgr_base->clks.fclk_p_state_change_support)) {
+ clk_mgr_base->clks.fclk_p_state_change_support = fclk_p_state_change_support;
+
+ /* To enable FCLK P-state switching, send FCLK_PSTATE_SUPPORTED message to PMFW */
+ if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && clk_mgr_base->clks.fclk_p_state_change_support) {
+ /* Handle the code for sending a message to PMFW that FCLK P-state change is supported */
+ dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_SUPPORTED);
+ }
+ }
+
if (dc->debug.force_min_dcfclk_mhz > 0)
new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
clk_mgr_base->clks.socclk_khz = new_clocks->socclk_khz;
clk_mgr_base->clks.prev_p_state_change_support = clk_mgr_base->clks.p_state_change_support;
- clk_mgr_base->clks.fclk_prev_p_state_change_support = clk_mgr_base->clks.fclk_p_state_change_support;
clk_mgr_base->clks.prev_num_ways = clk_mgr_base->clks.num_ways;
if (clk_mgr_base->clks.num_ways != new_clocks->num_ways &&
dcn32_smu_send_cab_for_uclk_message(clk_mgr, clk_mgr_base->clks.num_ways);
}
- total_plane_count = clk_mgr_helper_get_active_plane_cnt(dc, context);
+
p_state_change_support = new_clocks->p_state_change_support || (total_plane_count == 0);
- fclk_p_state_change_support = new_clocks->fclk_p_state_change_support || (total_plane_count == 0);
if (should_update_pstate_support(safe_to_lower, p_state_change_support, clk_mgr_base->clks.p_state_change_support)) {
clk_mgr_base->clks.p_state_change_support = p_state_change_support;
/* to disable P-State switching, set UCLK min = max */
if (!clk_mgr_base->clks.p_state_change_support)
dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
- clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
+ clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels - 1].memclk_mhz);
}
- if (should_update_pstate_support(safe_to_lower, fclk_p_state_change_support, clk_mgr_base->clks.fclk_p_state_change_support) &&
- clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21) {
- clk_mgr_base->clks.fclk_p_state_change_support = fclk_p_state_change_support;
+ /* Always update saved value, even if new value not set due to P-State switching unsupported. Also check safe_to_lower for FCLK */
+ if (safe_to_lower && (clk_mgr_base->clks.fclk_p_state_change_support != clk_mgr_base->clks.fclk_prev_p_state_change_support)) {
+ update_fclk = true;
+ }
- /* To disable FCLK P-state switching, send FCLK_PSTATE_NOTSUPPORTED message to PMFW */
- if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && !clk_mgr_base->clks.fclk_p_state_change_support) {
- /* Handle code for sending a message to PMFW that FCLK P-state change is not supported */
- dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_NOTSUPPORTED);
- }
+ if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && !clk_mgr_base->clks.fclk_p_state_change_support && update_fclk) {
+ /* Handle code for sending a message to PMFW that FCLK P-state change is not supported */
+ dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_NOTSUPPORTED);
}
/* Always update saved value, even if new value not set due to P-State switching unsupported */
update_uclk = true;
}
- /* Always update saved value, even if new value not set due to P-State switching unsupported. Also check safe_to_lower for FCLK */
- if (safe_to_lower && (clk_mgr_base->clks.fclk_p_state_change_support != clk_mgr_base->clks.fclk_prev_p_state_change_support)) {
- update_fclk = true;
- }
-
/* set UCLK to requested value if P-State switching is supported, or to re-enable P-State switching */
if (clk_mgr_base->clks.p_state_change_support &&
(update_uclk || !clk_mgr_base->clks.prev_p_state_change_support))
dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
- if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && clk_mgr_base->clks.fclk_p_state_change_support && update_fclk) {
- /* Handle the code for sending a message to PMFW that FCLK P-state change is supported */
- dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_SUPPORTED);
- }
-
if (clk_mgr_base->clks.num_ways != new_clocks->num_ways &&
clk_mgr_base->clks.num_ways > new_clocks->num_ways) {
clk_mgr_base->clks.num_ways = new_clocks->num_ways;
khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
else
dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
- clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
+ clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels - 1].memclk_mhz);
} else {
dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz);
return;
dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK,
- clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
+ clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels - 1].memclk_mhz);
}
/* Get current memclk states, update bounding box */
static void dcn32_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base)
{
struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+ struct clk_limit_num_entries *num_entries_per_clk = &clk_mgr_base->bw_params->clk_table.num_entries_per_clk;
unsigned int num_levels;
if (!clk_mgr->smu_present)
return;
- /* Refresh memclk states */
+ /* Refresh memclk and fclk states */
dcn32_init_single_clock(clk_mgr, PPCLK_UCLK,
&clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz,
- &num_levels);
+ &num_entries_per_clk->num_memclk_levels);
+
+ dcn32_init_single_clock(clk_mgr, PPCLK_FCLK,
+ &clk_mgr_base->bw_params->clk_table.entries[0].fclk_mhz,
+ &num_entries_per_clk->num_fclk_levels);
+
+ if (num_entries_per_clk->num_memclk_levels >= num_entries_per_clk->num_fclk_levels) {
+ num_levels = num_entries_per_clk->num_memclk_levels;
+ } else {
+ num_levels = num_entries_per_clk->num_fclk_levels;
+ }
+
clk_mgr_base->bw_params->clk_table.num_entries = num_levels ? num_levels : 1;
if (clk_mgr->dpm_present && !num_levels)
int i, k, l;
struct dc_stream_state *dc_streams[MAX_STREAMS] = {0};
struct dc_state *old_state;
+ bool subvp_prev_use = false;
dc_z10_restore(dc);
dc_allow_idle_optimizations(dc, false);
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+ /* Check old context for SubVP */
+ subvp_prev_use |= (old_pipe->stream && old_pipe->stream->mall_stream_config.type == SUBVP_PHANTOM);
+ if (subvp_prev_use)
+ break;
+ }
+
for (i = 0; i < context->stream_count; i++)
dc_streams[i] = context->streams[i];
dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe);
}
+ if (dc->hwss.subvp_pipe_control_lock)
+ dc->hwss.subvp_pipe_control_lock(dc, context, true, true, NULL, subvp_prev_use);
+
result = dc->hwss.apply_ctx_to_hw(dc, context);
if (result != DC_OK) {
dc->hwss.interdependent_update_lock(dc, context, false);
dc->hwss.post_unlock_program_front_end(dc, context);
}
+
+ if (dc->hwss.commit_subvp_config)
+ dc->hwss.commit_subvp_config(dc, context);
+ if (dc->hwss.subvp_pipe_control_lock)
+ dc->hwss.subvp_pipe_control_lock(dc, context, false, true, NULL, subvp_prev_use);
+
for (i = 0; i < context->stream_count; i++) {
const struct dc_link *link = context->streams[i]->link;
dc_resource_state_copy_construct(
dc->current_state, context);
+ /* For each full update, remove all existing phantom pipes first.
+ * Ensures that we have enough pipes for newly added MPO planes
+ */
+ if (dc->res_pool->funcs->remove_phantom_pipes)
+ dc->res_pool->funcs->remove_phantom_pipes(dc, context);
+
/*remove old surfaces from context */
if (!dc_rem_all_planes_for_stream(dc, stream, context)) {
/* Since phantom pipe programming is moved to post_unlock_program_front_end,
* move the SubVP lock to after the phantom pipes have been setup
*/
- if (dc->hwss.subvp_pipe_control_lock)
- dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
+ if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
+ if (dc->hwss.subvp_pipe_control_lock)
+ dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
+ } else {
+ if (dc->hwss.subvp_pipe_control_lock)
+ dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
+ }
+
return;
}
if (update_type != UPDATE_TYPE_FAST)
dc->hwss.post_unlock_program_front_end(dc, context);
+ if (update_type != UPDATE_TYPE_FAST)
+ if (dc->hwss.commit_subvp_config)
+ dc->hwss.commit_subvp_config(dc, context);
if (update_type != UPDATE_TYPE_FAST)
if (dc->hwss.commit_subvp_config)
struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream);
bool force_minimal_pipe_splitting = false;
+ uint32_t i;
*is_plane_addition = false;
}
}
+ /* For SubVP pipe split case when adding MPO video
+ * we need to add a minimal transition. In this case
+ * there will be 2 streams (1 main stream, 1 phantom
+ * stream).
+ */
+ if (cur_stream_status &&
+ dc->current_state->stream_count == 2 &&
+ stream->mall_stream_config.type == SUBVP_MAIN) {
+ bool is_pipe_split = false;
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream &&
+ (dc->current_state->res_ctx.pipe_ctx[i].bottom_pipe ||
+ dc->current_state->res_ctx.pipe_ctx[i].next_odm_pipe)) {
+ is_pipe_split = true;
+ break;
+ }
+ }
+
+ /* determine if minimal transition is required due to SubVP*/
+ if (surface_count > 0 && is_pipe_split) {
+ if (cur_stream_status->plane_count > surface_count) {
+ force_minimal_pipe_splitting = true;
+ } else if (cur_stream_status->plane_count < surface_count) {
+ force_minimal_pipe_splitting = true;
+ *is_plane_addition = true;
+ }
+ }
+ }
+
return force_minimal_pipe_splitting;
}
struct dc_state *transition_context = dc_create_state(dc);
enum pipe_split_policy tmp_mpc_policy;
bool temp_dynamic_odm_policy;
+ bool temp_subvp_policy;
enum dc_status ret = DC_ERROR_UNEXPECTED;
unsigned int i, j;
temp_dynamic_odm_policy = dc->debug.enable_single_display_2to1_odm_policy;
dc->debug.enable_single_display_2to1_odm_policy = false;
+ temp_subvp_policy = dc->debug.force_disable_subvp;
+ dc->debug.force_disable_subvp = true;
+
dc_resource_state_copy_construct(transition_base_context, transition_context);
//commit minimal state
dc->debug.pipe_split_policy = tmp_mpc_policy;
dc->debug.enable_single_display_2to1_odm_policy = temp_dynamic_odm_policy;
+ dc->debug.force_disable_subvp = temp_subvp_policy;
if (ret != DC_OK) {
/*this should never happen*/
return DC_OK;
}
+/**
+ *****************************************************************************
+ * Function: dc_process_dmub_dpia_hpd_int_enable
+ *
+ * @brief
+ * Submits dpia hpd int enable command to dmub via inbox message
+ *
+ * @param
+ * [in] dc: dc structure
+ * [in] hpd_int_enable: 1 for hpd int enable, 0 to disable
+ *
+ * @return
+ * None
+ *****************************************************************************
+ */
+void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc,
+ uint32_t hpd_int_enable)
+{
+ union dmub_rb_cmd cmd = {0};
+ struct dc_dmub_srv *dmub_srv = dc->ctx->dmub_srv;
+
+ cmd.dpia_hpd_int_enable.header.type = DMUB_CMD__DPIA_HPD_INT_ENABLE;
+ cmd.dpia_hpd_int_enable.enable = hpd_int_enable;
+
+ dc_dmub_srv_cmd_queue(dmub_srv, &cmd);
+ dc_dmub_srv_cmd_execute(dmub_srv);
+ dc_dmub_srv_wait_idle(dmub_srv);
+
+ DC_LOG_DEBUG("%s: hpd_int_enable(%d)\n", __func__, hpd_int_enable);
+}
+
/**
* dc_disable_accelerated_mode - disable accelerated mode
* @dc: dc structure
}
if (link->connector_signal == SIGNAL_TYPE_EDP) {
- // Init dc_panel_config
+ /* Init dc_panel_config by HW config */
+ if (dc_ctx->dc->res_pool->funcs->get_panel_config_defaults)
+ dc_ctx->dc->res_pool->funcs->get_panel_config_defaults(&link->panel_config);
+ /* Pickup base DM settings */
dm_helpers_init_panel_settings(dc_ctx, &link->panel_config, sink);
// Override dc_panel_config if system has specific settings
dm_helpers_override_panel_settings(dc_ctx, &link->panel_config);
if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
return false;
- if (allow_active && link->type == dc_connection_none) {
+ if ((allow_active != NULL) && (*allow_active == true) && (link->type == dc_connection_none)) {
// Don't enter PSR if panel is not connected
return false;
}
case FAMILY_YELLOW_CARP:
case AMDGPU_FAMILY_GC_10_3_6:
case AMDGPU_FAMILY_GC_11_0_1:
- if(!dc->debug.disable_z10)
- psr_context->psr_level.bits.SKIP_CRTC_DISABLE = false;
+ if (dc->debug.disable_z10)
+ psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true;
break;
default:
psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true;
return status;
}
+static enum dc_status dpcd_128b_132b_set_lane_settings(
+ struct dc_link *link,
+ const struct link_training_settings *link_training_setting)
+{
+ enum dc_status status = core_link_write_dpcd(link,
+ DP_TRAINING_LANE0_SET,
+ (uint8_t *)(link_training_setting->dpcd_lane_settings),
+ sizeof(link_training_setting->dpcd_lane_settings));
+
+ DC_LOG_HW_LINK_TRAINING("%s:\n 0x%X TX_FFE_PRESET_VALUE = %x\n",
+ __func__,
+ DP_TRAINING_LANE0_SET,
+ link_training_setting->dpcd_lane_settings[0].tx_ffe.PRESET_VALUE);
+ return status;
+}
+
+
enum dc_status dpcd_set_lane_settings(
struct dc_link *link,
const struct link_training_settings *link_training_setting,
link_training_setting->link_settings.lane_count);
if (is_repeater(link_training_setting, offset)) {
- if (dp_get_link_encoding_format(&link_training_setting->link_settings) ==
- DP_128b_132b_ENCODING)
- DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n"
- " 0x%X TX_FFE_PRESET_VALUE = %x\n",
- __func__,
- offset,
- lane0_set_address,
- link_training_setting->dpcd_lane_settings[0].tx_ffe.PRESET_VALUE);
- else if (dp_get_link_encoding_format(&link_training_setting->link_settings) ==
- DP_8b_10b_ENCODING)
DC_LOG_HW_LINK_TRAINING("%s\n LTTPR Repeater ID: %d\n"
" 0x%X VS set = %x PE set = %x max VS Reached = %x max PE Reached = %x\n",
__func__,
link_training_setting->dpcd_lane_settings[0].bits.MAX_PRE_EMPHASIS_REACHED);
} else {
- if (dp_get_link_encoding_format(&link_training_setting->link_settings) ==
- DP_128b_132b_ENCODING)
- DC_LOG_HW_LINK_TRAINING("%s:\n 0x%X TX_FFE_PRESET_VALUE = %x\n",
- __func__,
- lane0_set_address,
- link_training_setting->dpcd_lane_settings[0].tx_ffe.PRESET_VALUE);
- else if (dp_get_link_encoding_format(&link_training_setting->link_settings) ==
- DP_8b_10b_ENCODING)
DC_LOG_HW_LINK_TRAINING("%s\n 0x%X VS set = %x PE set = %x max VS Reached = %x max PE Reached = %x\n",
__func__,
lane0_set_address,
result = DP_128b_132b_LT_FAILED;
} else {
dp_set_hw_lane_settings(link, link_res, lt_settings, DPRX);
- dpcd_set_lane_settings(link, lt_settings, DPRX);
+ dpcd_128b_132b_set_lane_settings(link, lt_settings);
}
loop_count++;
}
(dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) == 0)) {
ASSERT(0);
link->dpcd_caps.lttpr_caps.phy_repeater_cnt = 0x80;
+ DC_LOG_DC("lttpr_caps forced phy_repeater_cnt = %d\n", link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
}
/* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */
if (is_lttpr_present)
CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: ");
+ DC_LOG_DC("is_lttpr_present = %d\n", is_lttpr_present);
return is_lttpr_present;
}
} else if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_NON_LTTPR) {
*override = LTTPR_MODE_NON_LTTPR;
}
+ DC_LOG_DC("lttpr_mode_override chose LTTPR_MODE = %d\n", (uint8_t)(*override));
}
enum lttpr_mode dp_decide_8b_10b_lttpr_mode(struct dc_link *link)
return LTTPR_MODE_NON_LTTPR;
if (vbios_lttpr_aware) {
- if (vbios_lttpr_force_non_transparent)
+ if (vbios_lttpr_force_non_transparent) {
+ DC_LOG_DC("chose LTTPR_MODE_NON_TRANSPARENT due to VBIOS DCE_INFO_CAPS_LTTPR_SUPPORT_ENABLE set to 1.\n");
return LTTPR_MODE_NON_TRANSPARENT;
- else
+ } else {
+ DC_LOG_DC("chose LTTPR_MODE_NON_TRANSPARENT by default due to VBIOS not set DCE_INFO_CAPS_LTTPR_SUPPORT_ENABLE set to 1.\n");
return LTTPR_MODE_TRANSPARENT;
+ }
}
if (link->dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A &&
- link->dc->caps.extended_aux_timeout_support)
+ link->dc->caps.extended_aux_timeout_support) {
+ DC_LOG_DC("chose LTTPR_MODE_NON_TRANSPARENT by default and dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A set to 1.\n");
return LTTPR_MODE_NON_TRANSPARENT;
+ }
+ DC_LOG_DC("chose LTTPR_MODE_NON_LTTPR.\n");
return LTTPR_MODE_NON_LTTPR;
}
enum lttpr_mode dp_decide_128b_132b_lttpr_mode(struct dc_link *link)
{
- return dp_is_lttpr_present(link) ? LTTPR_MODE_NON_TRANSPARENT : LTTPR_MODE_NON_LTTPR;
+ enum lttpr_mode mode = LTTPR_MODE_NON_LTTPR;
+
+ if (dp_is_lttpr_present(link))
+ mode = LTTPR_MODE_NON_TRANSPARENT;
+
+ DC_LOG_DC("128b_132b chose LTTPR_MODE %d.\n", mode);
+ return mode;
}
static bool get_usbc_cable_id(struct dc_link *link, union dp_cable_id *cable_id)
cmd.cable_id.data.input.phy_inst = resource_transmitter_to_phy_idx(
link->dc, link->link_enc->transmitter);
if (dc_dmub_srv_cmd_with_reply_data(link->ctx->dmub_srv, &cmd) &&
- cmd.cable_id.header.ret_status == 1)
+ cmd.cable_id.header.ret_status == 1) {
cable_id->raw = cmd.cable_id.data.output_raw;
-
+ DC_LOG_DC("usbc_cable_id = %d.\n", cable_id->raw);
+ }
return cmd.cable_id.header.ret_status == 1;
}
lttpr_present = dp_is_lttpr_present(link) ||
(!vbios_lttpr_interop || !link->dc->caps.extended_aux_timeout_support);
+ DC_LOG_DC("lttpr_present = %d.\n", lttpr_present ? 1 : 0);
/* Issue an AUX read to test DPRX responsiveness. If LTTPR is supported the first read is expected to
* be to determine LTTPR capabilities. Otherwise trying to read power state should be an innocuous AUX read.
* Per VESA eDP spec, "The DPCD revision for eDP v1.4 is 13h"
*/
if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_13 &&
- (link->dc->debug.optimize_edp_link_rate ||
+ (link->panel_config.ilr.optimize_edp_link_rate ||
link->reported_link_cap.link_rate == LINK_RATE_UNKNOWN)) {
// Read DPCD 00010h - 0001Fh 16 bytes at one shot
core_link_read_dpcd(link, DP_SUPPORTED_LINK_RATES,
ASSERT(link || crtc_timing); // invalid input
if (link->dpcd_caps.edp_supported_link_rates_count == 0 ||
- !link->dc->debug.optimize_edp_link_rate)
+ !link->panel_config.ilr.optimize_edp_link_rate)
return false;
for (i = 0; i < stream_status->plane_count; i++) {
if (stream_status->plane_states[i] == plane_state) {
-
dc_plane_state_release(stream_status->plane_states[i]);
break;
}
(stream->timing.h_sync_width % 2 == 0);
}
return divisible;
+}
+
+bool dc_resource_acquire_secondary_pipe_for_mpc_odm(
+ const struct dc *dc,
+ struct dc_state *state,
+ struct pipe_ctx *pri_pipe,
+ struct pipe_ctx *sec_pipe,
+ bool odm)
+{
+ int pipe_idx = sec_pipe->pipe_idx;
+ struct pipe_ctx *sec_top, *sec_bottom, *sec_next, *sec_prev;
+ const struct resource_pool *pool = dc->res_pool;
+
+ sec_top = sec_pipe->top_pipe;
+ sec_bottom = sec_pipe->bottom_pipe;
+ sec_next = sec_pipe->next_odm_pipe;
+ sec_prev = sec_pipe->prev_odm_pipe;
+
+ *sec_pipe = *pri_pipe;
+
+ sec_pipe->top_pipe = sec_top;
+ sec_pipe->bottom_pipe = sec_bottom;
+ sec_pipe->next_odm_pipe = sec_next;
+ sec_pipe->prev_odm_pipe = sec_prev;
+
+ sec_pipe->pipe_idx = pipe_idx;
+ sec_pipe->plane_res.mi = pool->mis[pipe_idx];
+ sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
+ sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
+ sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
+ sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
+ sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
+ sec_pipe->stream_res.dsc = NULL;
+ if (odm) {
+ if (!sec_pipe->top_pipe)
+ sec_pipe->stream_res.opp = pool->opps[pipe_idx];
+ else
+ sec_pipe->stream_res.opp = sec_pipe->top_pipe->stream_res.opp;
+ if (sec_pipe->stream->timing.flags.DSC == 1) {
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+ dcn20_acquire_dsc(dc, &state->res_ctx, &sec_pipe->stream_res.dsc, pipe_idx);
+#endif
+ ASSERT(sec_pipe->stream_res.dsc);
+ if (sec_pipe->stream_res.dsc == NULL)
+ return false;
+ }
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+ dcn20_build_mapped_resource(dc, state, sec_pipe->stream);
+#endif
+ }
+
+ return true;
}
\ No newline at end of file
}
dc->hwss.set_cursor_attribute(pipe_ctx);
+
+ dc_send_update_cursor_info_to_dmu(pipe_ctx, i);
if (dc->hwss.set_cursor_sdr_white_level)
dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
}
}
dc->hwss.set_cursor_position(pipe_ctx);
+
+ dc_send_update_cursor_info_to_dmu(pipe_ctx, i);
}
if (pipe_to_program)
}
/* remove writeback info for disabled writeback pipes from stream */
- for (i = 0, j = 0; i < stream->num_wb_info && j < MAX_DWB_PIPES; i++) {
+ for (i = 0, j = 0; i < stream->num_wb_info; i++) {
if (stream->writeback_info[i].wb_enabled) {
- if (i != j)
+ if (j < i)
/* trim the array */
stream->writeback_info[j] = stream->writeback_info[i];
j++;
struct set_config_cmd_payload;
struct dmub_notification;
-#define DC_VER "3.2.205"
+#define DC_VER "3.2.207"
#define MAX_SURFACES 3
#define MAX_PLANES 6
/* Enable dmub aux for legacy ddc */
bool enable_dmub_aux_for_legacy_ddc;
bool disable_fams;
- bool optimize_edp_link_rate; /* eDP ILR */
/* FEC/PSR1 sequence enable delay in 100us */
uint8_t fec_enable_delay_in100us;
bool enable_driver_sequence_debug;
enum dc_irq_source irq_source;
struct kref refcount;
struct tg_color visual_confirm_color;
+
+ bool is_statically_allocated;
};
struct dc_plane_info {
uint8_t mst_alloc_slots,
uint8_t *mst_slots_in_use);
+void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc,
+ uint32_t hpd_int_enable);
+
/*******************************************************************************
* DSC Interfaces
******************************************************************************/
#include "dc_hw_types.h"
#include "core_types.h"
#include "../basics/conversion.h"
+#include "cursor_reg_cache.h"
#define CTX dc_dmub_srv->ctx
#define DC_LOGGER CTX->logger
// Store the original watermark value for this SubVP config so we can lower it when the
// MCLK switch starts
wm_val_refclk = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns *
- dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 / 1000;
+ (dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000) / 1000;
cmd.fw_assisted_mclk_switch_v2.config_data.watermark_a_cache = wm_val_refclk < 0xFFFF ? wm_val_refclk : 0xFFFF;
}
diag_data.is_cw0_enabled,
diag_data.is_cw6_enabled);
}
+
+static bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx)
+{
+ if (pipe_ctx->plane_state != NULL) {
+ if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
+ return false;
+ }
+
+ if ((pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1 ||
+ pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_1) &&
+ pipe_ctx->stream->ctx->dce_version >= DCN_VERSION_3_1)
+ return true;
+
+ return false;
+}
+
+static void dc_build_cursor_update_payload0(
+ struct pipe_ctx *pipe_ctx, uint8_t p_idx,
+ struct dmub_cmd_update_cursor_payload0 *payload)
+{
+ struct hubp *hubp = pipe_ctx->plane_res.hubp;
+ unsigned int panel_inst = 0;
+
+ if (!dc_get_edp_link_panel_inst(hubp->ctx->dc,
+ pipe_ctx->stream->link, &panel_inst))
+ return;
+
+ /* Payload: Cursor Rect is built from position & attribute
+ * x & y are obtained from postion
+ */
+ payload->cursor_rect.x = hubp->cur_rect.x;
+ payload->cursor_rect.y = hubp->cur_rect.y;
+ /* w & h are obtained from attribute */
+ payload->cursor_rect.width = hubp->cur_rect.w;
+ payload->cursor_rect.height = hubp->cur_rect.h;
+
+ payload->enable = hubp->pos.cur_ctl.bits.cur_enable;
+ payload->pipe_idx = p_idx;
+ payload->cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1;
+ payload->panel_inst = panel_inst;
+}
+
+static void dc_send_cmd_to_dmu(struct dc_dmub_srv *dmub_srv,
+ union dmub_rb_cmd *cmd)
+{
+ dc_dmub_srv_cmd_queue(dmub_srv, cmd);
+ dc_dmub_srv_cmd_execute(dmub_srv);
+ dc_dmub_srv_wait_idle(dmub_srv);
+}
+
+static void dc_build_cursor_position_update_payload0(
+ struct dmub_cmd_update_cursor_payload0 *pl, const uint8_t p_idx,
+ const struct hubp *hubp, const struct dpp *dpp)
+{
+ /* Hubp */
+ pl->position_cfg.pHubp.cur_ctl.raw = hubp->pos.cur_ctl.raw;
+ pl->position_cfg.pHubp.position.raw = hubp->pos.position.raw;
+ pl->position_cfg.pHubp.hot_spot.raw = hubp->pos.hot_spot.raw;
+ pl->position_cfg.pHubp.dst_offset.raw = hubp->pos.dst_offset.raw;
+
+ /* dpp */
+ pl->position_cfg.pDpp.cur0_ctl.raw = dpp->pos.cur0_ctl.raw;
+ pl->position_cfg.pipe_idx = p_idx;
+}
+
+static void dc_build_cursor_attribute_update_payload1(
+ struct dmub_cursor_attributes_cfg *pl_A, const uint8_t p_idx,
+ const struct hubp *hubp, const struct dpp *dpp)
+{
+ /* Hubp */
+ pl_A->aHubp.SURFACE_ADDR_HIGH = hubp->att.SURFACE_ADDR_HIGH;
+ pl_A->aHubp.SURFACE_ADDR = hubp->att.SURFACE_ADDR;
+ pl_A->aHubp.cur_ctl.raw = hubp->att.cur_ctl.raw;
+ pl_A->aHubp.size.raw = hubp->att.size.raw;
+ pl_A->aHubp.settings.raw = hubp->att.settings.raw;
+
+ /* dpp */
+ pl_A->aDpp.cur0_ctl.raw = dpp->att.cur0_ctl.raw;
+}
+
+/**
+ * ***************************************************************************************
+ * dc_send_update_cursor_info_to_dmu: Populate the DMCUB Cursor update info command
+ *
+ * This function would store the cursor related information and pass it into dmub
+ *
+ * @param [in] pCtx: pipe context
+ * @param [in] pipe_idx: pipe index
+ *
+ * @return: void
+ *
+ * ***************************************************************************************
+ */
+
+void dc_send_update_cursor_info_to_dmu(
+ struct pipe_ctx *pCtx, uint8_t pipe_idx)
+{
+ union dmub_rb_cmd cmd = { 0 };
+ union dmub_cmd_update_cursor_info_data *update_cursor_info =
+ &cmd.update_cursor_info.update_cursor_info_data;
+
+ if (!dc_dmub_should_update_cursor_data(pCtx))
+ return;
+ /*
+ * Since we use multi_cmd_pending for dmub command, the 2nd command is
+ * only assigned to store cursor attributes info.
+ * 1st command can view as 2 parts, 1st is for PSR/Replay data, the other
+ * is to store cursor position info.
+ *
+ * Command heaer type must be the same type if using multi_cmd_pending.
+ * Besides, while process 2nd command in DMU, the sub type is useless.
+ * So it's meanless to pass the sub type header with different type.
+ */
+
+ {
+ /* Build Payload#0 Header */
+ cmd.update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO;
+ cmd.update_cursor_info.header.payload_bytes =
+ sizeof(cmd.update_cursor_info.update_cursor_info_data);
+ cmd.update_cursor_info.header.multi_cmd_pending = 1; /* To combine multi dmu cmd, 1st cmd */
+
+ /* Prepare Payload */
+ dc_build_cursor_update_payload0(pCtx, pipe_idx, &update_cursor_info->payload0);
+
+ dc_build_cursor_position_update_payload0(&update_cursor_info->payload0, pipe_idx,
+ pCtx->plane_res.hubp, pCtx->plane_res.dpp);
+ /* Send update_curosr_info to queue */
+ dc_dmub_srv_cmd_queue(pCtx->stream->ctx->dmub_srv, &cmd);
+ }
+ {
+ /* Build Payload#1 Header */
+ memset(update_cursor_info, 0, sizeof(union dmub_cmd_update_cursor_info_data));
+ cmd.update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO;
+ cmd.update_cursor_info.header.payload_bytes = sizeof(struct cursor_attributes_cfg);
+ cmd.update_cursor_info.header.multi_cmd_pending = 0; /* Indicate it's the last command. */
+
+ dc_build_cursor_attribute_update_payload1(
+ &cmd.update_cursor_info.update_cursor_info_data.payload1.attribute_cfg,
+ pipe_idx, pCtx->plane_res.hubp, pCtx->plane_res.dpp);
+
+ /* Combine 2nd cmds update_curosr_info to DMU */
+ dc_send_cmd_to_dmu(pCtx->stream->ctx->dmub_srv, &cmd);
+ }
+}
void dc_dmub_setup_subvp_dmub_command(struct dc *dc, struct dc_state *context, bool enable);
void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv);
+void dc_send_update_cursor_info_to_dmu(struct pipe_ctx *pCtx, uint8_t pipe_idx);
#endif /* _DMUB_DC_SRV_H_ */
bool disable_dsc_edp;
unsigned int force_dsc_edp_policy;
} dsc;
+ /* eDP ILR */
+ struct ilr {
+ bool optimize_edp_link_rate; /* eDP ILR */
+ } ilr;
};
/*
* A link contains one or more sinks and their connected status.
case AUX_RET_ERROR_ENGINE_ACQUIRE:
case AUX_RET_ERROR_UNKNOWN:
default:
- DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
- LOG_FLAG_I2cAux_DceAux,
- "dce_aux_transfer_with_retries: Failure: operation_result=%d",
- (int)operation_result);
goto fail;
}
}
fail:
DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
LOG_FLAG_Error_I2cAux,
- "dce_aux_transfer_with_retries: FAILURE");
+ "%s: Failure: operation_result=%d",
+ __func__,
+ (int)operation_result);
if (!payload_reply)
payload->reply = NULL;
- DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
- WPP_BIT_FLAG_DC_ERROR,
- "AUX transaction failed. Result: %d",
- operation_result);
-
return false;
}
REG_UPDATE(CURSOR0_CONTROL,
CUR0_ENABLE, cur_en);
+ dpp_base->pos.cur0_ctl.bits.cur0_enable = cur_en;
}
void dpp1_cnv_set_optional_cursor_attributes(
DC_SYNC_INFO("Setting up OTG reset trigger\n");
for (i = 1; i < group_size; i++) {
+ if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM)
+ continue;
+
opp = grouped_pipes[i]->stream_res.opp;
tg = grouped_pipes[i]->stream_res.tg;
tg->funcs->get_otg_active_size(tg, &width, &height);
for (i = 0; i < group_size; i++) {
if (grouped_pipes[i]->stream == NULL)
continue;
+
+ if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM)
+ continue;
+
grouped_pipes[i]->stream->vblank_synchronized = false;
}
- for (i = 1; i < group_size; i++)
+ for (i = 1; i < group_size; i++) {
+ if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM)
+ continue;
+
grouped_pipes[i]->stream_res.tg->funcs->enable_reset_trigger(
grouped_pipes[i]->stream_res.tg,
grouped_pipes[0]->stream_res.tg->inst);
+ }
DC_SYNC_INFO("Waiting for trigger\n");
* synchronized. Look at last pipe programmed to reset.
*/
- wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[1]->stream_res.tg);
- for (i = 1; i < group_size; i++)
+ if (grouped_pipes[1]->stream && grouped_pipes[1]->stream->mall_stream_config.type != SUBVP_PHANTOM)
+ wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[1]->stream_res.tg);
+
+ for (i = 1; i < group_size; i++) {
+ if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM)
+ continue;
+
grouped_pipes[i]->stream_res.tg->funcs->disable_reset_trigger(
grouped_pipes[i]->stream_res.tg);
+ }
for (i = 1; i < group_size; i++) {
+ if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM)
+ continue;
+
opp = grouped_pipes[i]->stream_res.opp;
tg = grouped_pipes[i]->stream_res.tg;
tg->funcs->get_otg_active_size(tg, &width, &height);
{
struct dce_hwseq *hws = dc->hwseq;
struct hubbub *hubbub = dc->res_pool->hubbub;
+ int min_fclk_khz, min_dcfclk_khz, socclk_khz;
if (dc->debug.sanity_checks)
hws->funcs.verify_allow_pstate_change_high(dc);
if (dc->debug.pplib_wm_report_mode == WM_REPORT_OVERRIDE) {
DC_FP_START();
- dcn_bw_notify_pplib_of_wm_ranges(dc);
+ dcn_get_soc_clks(
+ dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz);
DC_FP_END();
+ dcn_bw_notify_pplib_of_wm_ranges(
+ dc, min_fclk_khz, min_dcfclk_khz, socclk_khz);
}
if (dc->debug.sanity_checks)
{
struct dce_hwseq *hws = dc->hwseq;
struct hubbub *hubbub = dc->res_pool->hubbub;
+ int min_fclk_khz, min_dcfclk_khz, socclk_khz;
if (dc->debug.sanity_checks)
hws->funcs.verify_allow_pstate_change_high(dc);
if (dc->debug.pplib_wm_report_mode == WM_REPORT_OVERRIDE) {
DC_FP_START();
- dcn_bw_notify_pplib_of_wm_ranges(dc);
+ dcn_get_soc_clks(
+ dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz);
DC_FP_END();
+ dcn_bw_notify_pplib_of_wm_ranges(
+ dc, min_fclk_khz, min_dcfclk_khz, socclk_khz);
}
if (dc->debug.sanity_checks)
return false;
}
-static bool dcn10_dmub_should_update_cursor_data(
- struct pipe_ctx *pipe_ctx,
- struct dc_debug_options *debug)
-{
- if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
- return false;
-
- if (dcn10_can_pipe_disable_cursor(pipe_ctx))
- return false;
-
- if ((pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1 || pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_1)
- && pipe_ctx->stream->ctx->dce_version >= DCN_VERSION_3_1)
- return true;
-
- return false;
-}
-
-static void dcn10_dmub_update_cursor_data(
- struct pipe_ctx *pipe_ctx,
- struct hubp *hubp,
- const struct dc_cursor_mi_param *param,
- const struct dc_cursor_position *cur_pos,
- const struct dc_cursor_attributes *cur_attr)
-{
- union dmub_rb_cmd cmd;
- struct dmub_cmd_update_cursor_info_data *update_cursor_info;
- const struct dc_cursor_position *pos;
- const struct dc_cursor_attributes *attr;
- int src_x_offset = 0;
- int src_y_offset = 0;
- int x_hotspot = 0;
- int cursor_height = 0;
- int cursor_width = 0;
- uint32_t cur_en = 0;
- unsigned int panel_inst = 0;
-
- struct dc_debug_options *debug = &hubp->ctx->dc->debug;
-
- if (!dcn10_dmub_should_update_cursor_data(pipe_ctx, debug))
- return;
- /**
- * if cur_pos == NULL means the caller is from cursor_set_attribute
- * then driver use previous cursor position data
- * if cur_attr == NULL means the caller is from cursor_set_position
- * then driver use previous cursor attribute
- * if cur_pos or cur_attr is not NULL then update it
- */
- if (cur_pos != NULL)
- pos = cur_pos;
- else
- pos = &hubp->curs_pos;
-
- if (cur_attr != NULL)
- attr = cur_attr;
- else
- attr = &hubp->curs_attr;
-
- if (!dc_get_edp_link_panel_inst(hubp->ctx->dc, pipe_ctx->stream->link, &panel_inst))
- return;
-
- src_x_offset = pos->x - pos->x_hotspot - param->viewport.x;
- src_y_offset = pos->y - pos->y_hotspot - param->viewport.y;
- x_hotspot = pos->x_hotspot;
- cursor_height = (int)attr->height;
- cursor_width = (int)attr->width;
- cur_en = pos->enable ? 1:0;
-
- // Rotated cursor width/height and hotspots tweaks for offset calculation
- if (param->rotation == ROTATION_ANGLE_90 || param->rotation == ROTATION_ANGLE_270) {
- swap(cursor_height, cursor_width);
- if (param->rotation == ROTATION_ANGLE_90) {
- src_x_offset = pos->x - pos->y_hotspot - param->viewport.x;
- src_y_offset = pos->y - pos->x_hotspot - param->viewport.y;
- }
- } else if (param->rotation == ROTATION_ANGLE_180) {
- src_x_offset = pos->x - param->viewport.x;
- src_y_offset = pos->y - param->viewport.y;
- }
-
- if (param->mirror) {
- x_hotspot = param->viewport.width - x_hotspot;
- src_x_offset = param->viewport.x + param->viewport.width - src_x_offset;
- }
-
- if (src_x_offset >= (int)param->viewport.width)
- cur_en = 0; /* not visible beyond right edge*/
-
- if (src_x_offset + cursor_width <= 0)
- cur_en = 0; /* not visible beyond left edge*/
-
- if (src_y_offset >= (int)param->viewport.height)
- cur_en = 0; /* not visible beyond bottom edge*/
-
- if (src_y_offset + cursor_height <= 0)
- cur_en = 0; /* not visible beyond top edge*/
-
- // Cursor bitmaps have different hotspot values
- // There's a possibility that the above logic returns a negative value, so we clamp them to 0
- if (src_x_offset < 0)
- src_x_offset = 0;
- if (src_y_offset < 0)
- src_y_offset = 0;
-
- memset(&cmd, 0x0, sizeof(cmd));
- cmd.update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO;
- cmd.update_cursor_info.header.payload_bytes =
- sizeof(cmd.update_cursor_info.update_cursor_info_data);
- update_cursor_info = &cmd.update_cursor_info.update_cursor_info_data;
- update_cursor_info->cursor_rect.x = src_x_offset + param->viewport.x;
- update_cursor_info->cursor_rect.y = src_y_offset + param->viewport.y;
- update_cursor_info->cursor_rect.width = attr->width;
- update_cursor_info->cursor_rect.height = attr->height;
- update_cursor_info->enable = cur_en;
- update_cursor_info->pipe_idx = pipe_ctx->pipe_idx;
- update_cursor_info->cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1;
- update_cursor_info->panel_inst = panel_inst;
- dc_dmub_srv_cmd_queue(pipe_ctx->stream->ctx->dmub_srv, &cmd);
- dc_dmub_srv_cmd_execute(pipe_ctx->stream->ctx->dmub_srv);
- dc_dmub_srv_wait_idle(pipe_ctx->stream->ctx->dmub_srv);
-}
-
void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
{
struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position;
pipe_ctx->plane_res.scl_data.viewport.height - pos_cpy.y;
}
- dcn10_dmub_update_cursor_data(pipe_ctx, hubp, ¶m, &pos_cpy, NULL);
hubp->funcs->set_cursor_position(hubp, &pos_cpy, ¶m);
dpp->funcs->set_cursor_position(dpp, &pos_cpy, ¶m, hubp->curs_attr.width, hubp->curs_attr.height);
}
void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
{
struct dc_cursor_attributes *attributes = &pipe_ctx->stream->cursor_attributes;
- struct dc_cursor_mi_param param = { 0 };
-
- /**
- * If enter PSR without cursor attribute update
- * the cursor attribute of dmub_restore_plane
- * are initial value. call dmub to exit PSR and
- * restore plane then update cursor attribute to
- * avoid override with initial value
- */
- if (pipe_ctx->plane_state != NULL) {
- param.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
- param.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz;
- param.viewport = pipe_ctx->plane_res.scl_data.viewport;
- param.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz;
- param.v_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.vert;
- param.rotation = pipe_ctx->plane_state->rotation;
- param.mirror = pipe_ctx->plane_state->horizontal_mirror;
- dcn10_dmub_update_cursor_data(pipe_ctx, pipe_ctx->plane_res.hubp, ¶m, NULL, attributes);
- }
pipe_ctx->plane_res.hubp->funcs->set_cursor_attributes(
pipe_ctx->plane_res.hubp, attributes);
uint32_t *start_line,
uint32_t *end_line)
{
- const struct dc_crtc_timing *dc_crtc_timing = &pipe_ctx->stream->timing;
- int vline_int_offset_from_vupdate =
- pipe_ctx->stream->periodic_interrupt.lines_offset;
- int vupdate_offset_from_vsync = dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx);
- int start_position;
-
- if (vline_int_offset_from_vupdate > 0)
- vline_int_offset_from_vupdate--;
- else if (vline_int_offset_from_vupdate < 0)
- vline_int_offset_from_vupdate++;
-
- start_position = vline_int_offset_from_vupdate + vupdate_offset_from_vsync;
+ const struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
+ int vupdate_pos = dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx);
- if (start_position >= 0)
- *start_line = start_position;
+ if (vupdate_pos >= 0)
+ *start_line = vupdate_pos - ((vupdate_pos / timing->v_total) * timing->v_total);
else
- *start_line = dc_crtc_timing->v_total + start_position - 1;
-
- *end_line = *start_line + 2;
-
- if (*end_line >= dc_crtc_timing->v_total)
- *end_line = 2;
+ *start_line = vupdate_pos + ((-vupdate_pos / timing->v_total) + 1) * timing->v_total - 1;
+ *end_line = (*start_line + 2) % timing->v_total;
}
static void dcn10_cal_vline_position(
uint32_t *start_line,
uint32_t *end_line)
{
- switch (pipe_ctx->stream->periodic_interrupt.ref_point) {
- case START_V_UPDATE:
- dcn10_calc_vupdate_position(
- dc,
- pipe_ctx,
- start_line,
- end_line);
- break;
- case START_V_SYNC:
+ const struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
+ int vline_pos = pipe_ctx->stream->periodic_interrupt.lines_offset;
+
+ if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_UPDATE) {
+ if (vline_pos > 0)
+ vline_pos--;
+ else if (vline_pos < 0)
+ vline_pos++;
+
+ vline_pos += dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx);
+ if (vline_pos >= 0)
+ *start_line = vline_pos - ((vline_pos / timing->v_total) * timing->v_total);
+ else
+ *start_line = vline_pos + ((-vline_pos / timing->v_total) + 1) * timing->v_total - 1;
+ *end_line = (*start_line + 2) % timing->v_total;
+ } else if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_SYNC) {
// vsync is line 0 so start_line is just the requested line offset
- *start_line = pipe_ctx->stream->periodic_interrupt.lines_offset;
- *end_line = *start_line + 2;
- break;
- default:
+ *start_line = vline_pos;
+ *end_line = (*start_line + 2) % timing->v_total;
+ } else
ASSERT(0);
- break;
- }
}
void dcn10_setup_periodic_interrupt(
/* In case of V_TOTAL_CONTROL is on, make sure OTG_V_TOTAL_MAX and
* OTG_V_TOTAL_MIN are equal to V_TOTAL.
*/
- REG_SET(OTG_V_TOTAL_MAX, 0,
- OTG_V_TOTAL_MAX, v_total);
- REG_SET(OTG_V_TOTAL_MIN, 0,
- OTG_V_TOTAL_MIN, v_total);
+ optc->funcs->set_vtotal_min_max(optc, v_total, v_total);
/* v_sync_start = 0, v_sync_end = v_sync_width */
v_sync_end = patched_crtc_timing.v_sync_width;
void optc1_lock(struct timing_generator *optc)
{
struct optc *optc1 = DCN10TG_FROM_TG(optc);
- uint32_t regval = 0;
-
- regval = REG_READ(OTG_CONTROL);
-
- /* otg is not running, do not need to be locked */
- if ((regval & 0x1) == 0x0)
- return;
REG_SET(OTG_GLOBAL_CONTROL0, 0,
OTG_MASTER_UPDATE_LOCK_SEL, optc->inst);
OTG_MASTER_UPDATE_LOCK, 1);
/* Should be fast, status does not update on maximus */
- if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS) {
-
+ if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
REG_WAIT(OTG_MASTER_UPDATE_LOCK,
UPDATE_LOCK_STATUS, 1,
1, 10);
- }
}
void optc1_unlock(struct timing_generator *optc)
OTG_MASTER_UPDATE_LOCK, 0);
}
-bool optc1_is_locked(struct timing_generator *optc)
-{
- struct optc *optc1 = DCN10TG_FROM_TG(optc);
- uint32_t locked;
-
- REG_GET(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, &locked);
-
- return (locked == 1);
-}
-
void optc1_get_position(struct timing_generator *optc,
struct crtc_position *position)
{
}
- REG_SET(OTG_V_TOTAL_MAX, 0,
- OTG_V_TOTAL_MAX, params->vertical_total_max - 1);
-
- REG_SET(OTG_V_TOTAL_MIN, 0,
- OTG_V_TOTAL_MIN, params->vertical_total_min - 1);
+ optc->funcs->set_vtotal_min_max(optc, params->vertical_total_min - 1, params->vertical_total_max - 1);
REG_UPDATE_5(OTG_V_TOTAL_CONTROL,
OTG_V_TOTAL_MIN_SEL, 1,
OTG_V_TOTAL_MAX_SEL, 0,
OTG_FORCE_LOCK_ON_EVENT, 0);
- REG_SET(OTG_V_TOTAL_MIN, 0,
- OTG_V_TOTAL_MIN, 0);
-
- REG_SET(OTG_V_TOTAL_MAX, 0,
- OTG_V_TOTAL_MAX, 0);
+ optc->funcs->set_vtotal_min_max(optc, 0, 0);
}
}
.enable_crtc_reset = optc1_enable_crtc_reset,
.disable_reset_trigger = optc1_disable_reset_trigger,
.lock = optc1_lock,
- .is_locked = optc1_is_locked,
.unlock = optc1_unlock,
.enable_optc_clock = optc1_enable_optc_clock,
.set_drr = optc1_set_drr,
.get_last_used_drr_vtotal = NULL,
+ .set_vtotal_min_max = optc1_set_vtotal_min_max,
.set_static_screen_control = optc1_set_static_screen_control,
.set_test_pattern = optc1_set_test_pattern,
.program_stereo = optc1_program_stereo,
bool enable_blanking);
bool optc1_is_blanked(struct timing_generator *optc);
-bool optc1_is_locked(struct timing_generator *optc);
void optc1_program_blank_color(
struct timing_generator *optc,
}
}
+static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks)
+{
+ int i;
+
+ if (clks->num_levels == 0)
+ return false;
+
+ for (i = 0; i < clks->num_levels; i++)
+ /* Ensure that the result is sane */
+ if (clks->data[i].clocks_in_khz == 0)
+ return false;
+
+ return true;
+}
+
static bool dcn10_resource_construct(
uint8_t num_virtual_links,
struct dc *dc,
int j;
struct dc_context *ctx = dc->ctx;
uint32_t pipe_fuses = read_pipe_fuses(ctx);
+ struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0};
+ int min_fclk_khz, min_dcfclk_khz, socclk_khz;
+ bool res;
ctx->dc_bios->regs = &bios_regs;
&& pool->base.pp_smu->rv_funcs.set_pme_wa_enable != NULL)
dc->debug.az_endpoint_mute_only = false;
- DC_FP_START();
- if (!dc->debug.disable_pplib_clock_request)
- dcn_bw_update_from_pplib(dc);
+
+ if (!dc->debug.disable_pplib_clock_request) {
+ /*
+ * TODO: This is not the proper way to obtain
+ * fabric_and_dram_bandwidth, should be min(fclk, memclk).
+ */
+ res = dm_pp_get_clock_levels_by_type_with_voltage(
+ ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks);
+
+ DC_FP_START();
+
+ if (res)
+ res = verify_clock_values(&fclks);
+
+ if (res)
+ dcn_bw_update_from_pplib_fclks(dc, &fclks);
+ else
+ BREAK_TO_DEBUGGER();
+
+ DC_FP_END();
+
+ res = dm_pp_get_clock_levels_by_type_with_voltage(
+ ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks);
+
+ DC_FP_START();
+
+ if (res)
+ res = verify_clock_values(&dcfclks);
+
+ if (res)
+ dcn_bw_update_from_pplib_dcfclks(dc, &dcfclks);
+ else
+ BREAK_TO_DEBUGGER();
+
+ DC_FP_END();
+ }
+
dcn_bw_sync_calcs_and_dml(dc);
if (!dc->debug.disable_pplib_wm_range) {
dc->res_pool = &pool->base;
- dcn_bw_notify_pplib_of_wm_ranges(dc);
+ DC_FP_START();
+ dcn_get_soc_clks(
+ dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz);
+ DC_FP_END();
+ dcn_bw_notify_pplib_of_wm_ranges(
+ dc, min_fclk_khz, min_dcfclk_khz, socclk_khz);
}
- DC_FP_END();
{
struct irq_service_init_data init_data;
CURSOR0_DST_Y_OFFSET, 0,
/* used to shift the cursor chunk request deadline */
CURSOR0_CHUNK_HDL_ADJUST, 3);
+
+ hubp->att.SURFACE_ADDR_HIGH = attr->address.high_part;
+ hubp->att.SURFACE_ADDR = attr->address.low_part;
+ hubp->att.size.bits.width = attr->width;
+ hubp->att.size.bits.height = attr->height;
+ hubp->att.cur_ctl.bits.mode = attr->color_format;
+ hubp->att.cur_ctl.bits.pitch = hw_pitch;
+ hubp->att.cur_ctl.bits.line_per_chunk = lpc;
+ hubp->att.cur_ctl.bits.cur_2x_magnify = attr->attribute_flags.bits.ENABLE_MAGNIFICATION;
+ hubp->att.settings.bits.dst_y_offset = 0;
+ hubp->att.settings.bits.chunk_hdl_adjust = 3;
}
void hubp2_dmdata_set_attributes(
REG_SET(CURSOR_DST_OFFSET, 0,
CURSOR_DST_X_OFFSET, dst_x_offset);
/* TODO Handle surface pixel formats other than 4:4:4 */
+ /* Cursor Position Register Config */
+ hubp->pos.cur_ctl.bits.cur_enable = cur_en;
+ hubp->pos.position.bits.x_pos = pos->x;
+ hubp->pos.position.bits.y_pos = pos->y;
+ hubp->pos.hot_spot.bits.x_hot = x_hotspot;
+ hubp->pos.hot_spot.bits.y_hot = y_hotspot;
+ hubp->pos.dst_offset.bits.dst_x_offset = dst_x_offset;
+ /* Cursor Rectangle Cache
+ * Cursor bitmaps have different hotspot values
+ * There's a possibility that the above logic returns a negative value,
+ * so we clamp them to 0
+ */
+ if (src_x_offset < 0)
+ src_x_offset = 0;
+ if (src_y_offset < 0)
+ src_y_offset = 0;
+ /* Save necessary cursor info x, y position. w, h is saved in attribute func. */
+ hubp->cur_rect.x = src_x_offset + param->viewport.x;
+ hubp->cur_rect.y = src_y_offset + param->viewport.y;
}
void hubp2_clk_cntl(struct hubp *hubp, bool enable)
}
}
- for (i = 0; i < dc->res_pool->pipe_count; i++) {
- struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
- struct pipe_ctx *mpcc_pipe;
-
- if (pipe->vtp_locked) {
- dc->hwseq->funcs.wait_for_blank_complete(pipe->stream_res.opp);
- pipe->plane_res.hubp->funcs->set_blank(pipe->plane_res.hubp, true);
- pipe->vtp_locked = false;
-
- for (mpcc_pipe = pipe->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe)
- mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, true);
-
- for (i = 0; i < dc->res_pool->pipe_count; i++)
- if (context->res_ctx.pipe_ctx[i].update_flags.bits.disable)
- dc->hwss.disable_plane(dc, &dc->current_state->res_ctx.pipe_ctx[i]);
- }
- }
-
for (i = 0; i < dc->res_pool->pipe_count; i++) {
struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
context->bw_ctx.bw.dcn.clk.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+ /* increase compbuf size */
+ if (hubbub->funcs->program_compbuf_size)
+ hubbub->funcs->program_compbuf_size(hubbub, context->bw_ctx.bw.dcn.compbuf_size_kb, true);
+
dc->clk_mgr->funcs->update_clocks(
dc->clk_mgr,
context,
pipe_ctx->dlg_regs.optimized_min_dst_y_next_start);
}
}
- /* increase compbuf size */
- if (hubbub->funcs->program_compbuf_size)
- hubbub->funcs->program_compbuf_size(hubbub, context->bw_ctx.bw.dcn.compbuf_size_kb, true);
}
bool dcn20_update_bandwidth(
.enable_optc_clock = optc1_enable_optc_clock,
.set_drr = optc1_set_drr,
.get_last_used_drr_vtotal = optc2_get_last_used_drr_vtotal,
+ .set_vtotal_min_max = optc1_set_vtotal_min_max,
.set_static_screen_control = optc1_set_static_screen_control,
.program_stereo = optc1_program_stereo,
.is_stereo_left_eye = optc1_is_stereo_left_eye,
void dcn21_dchvm_init(struct hubbub *hubbub)
{
struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub);
- uint32_t riommu_active, prefetch_done;
+ uint32_t riommu_active;
int i;
- REG_GET(DCHVM_RIOMMU_STAT0, HOSTVM_PREFETCH_DONE, &prefetch_done);
-
- if (prefetch_done) {
- hubbub->riommu_active = true;
- return;
- }
//Init DCHVM block
REG_UPDATE(DCHVM_CTRL0, HOSTVM_INIT_REQ, 1);
.usbc_combo_phy_reset_wa = true,
.dmub_command_table = true,
.use_max_lb = true,
- .optimize_edp_link_rate = true
};
static const struct dc_debug_options debug_defaults_diags = {
.use_max_lb = true
};
+static const struct dc_panel_config panel_config_defaults = {
+ .ilr = {
+ .optimize_edp_link_rate = true,
+ },
+};
+
enum dcn20_clk_src_array_id {
DCN20_CLK_SRC_PLL0,
DCN20_CLK_SRC_PLL1,
return &panel_cntl->base;
}
+static void dcn21_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+ *panel_config = panel_config_defaults;
+}
+
#define CTX ctx
#define REG(reg_name) \
.set_mcif_arb_params = dcn20_set_mcif_arb_params,
.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
.update_bw_bounding_box = dcn21_update_bw_bounding_box,
+ .get_panel_config_defaults = dcn21_get_panel_config_defaults,
};
static bool dcn21_resource_construct(
REG_UPDATE(CURSOR0_COLOR1,
CUR0_COLOR1, 0xFFFFFFFF);
}
+
+ dpp_base->att.cur0_ctl.bits.expansion_mode = 0;
+ dpp_base->att.cur0_ctl.bits.cur0_rom_en = cur_rom_en;
+ dpp_base->att.cur0_ctl.bits.mode = color_format;
}
.enable_crtc_reset = optc1_enable_crtc_reset,
.disable_reset_trigger = optc1_disable_reset_trigger,
.lock = optc3_lock,
- .is_locked = optc1_is_locked,
.unlock = optc1_unlock,
.lock_doublebuffer_enable = optc3_lock_doublebuffer_enable,
.lock_doublebuffer_disable = optc3_lock_doublebuffer_disable,
.enable_optc_clock = optc1_enable_optc_clock,
.set_drr = optc1_set_drr,
.get_last_used_drr_vtotal = optc2_get_last_used_drr_vtotal,
+ .set_vtotal_min_max = optc3_set_vtotal_min_max,
.set_static_screen_control = optc1_set_static_screen_control,
.program_stereo = optc1_program_stereo,
.is_stereo_left_eye = optc1_is_stereo_left_eye,
optc1->min_h_sync_width = 4;
optc1->min_v_sync_width = 1;
}
-
if (!pipes)
return false;
+ context->bw_ctx.dml.vba.maxMpcComb = 0;
+ context->bw_ctx.dml.vba.VoltageLevel = 0;
+ context->bw_ctx.dml.vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_vactive;
dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
if (repopulate_pipes)
pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+ context->bw_ctx.dml.vba.VoltageLevel = vlevel;
*vlevel_out = vlevel;
*pipe_cnt_out = pipe_cnt;
vmid->masks = &vmid_masks;
}
- hubbub3->num_vmid = res_cap_dcn301.num_vmid;
+ hubbub3->num_vmid = res_cap_dcn301.num_vmid;
return &hubbub3->base;
}
uint32_t h_back_porch;
uint32_t h_width;
uint32_t v_height;
- unsigned long long v_freq;
+ uint64_t v_freq;
uint8_t misc0 = 0;
uint8_t misc1 = 0;
uint8_t hsp;
v_height = hw_crtc_timing.v_border_top + hw_crtc_timing.v_addressable + hw_crtc_timing.v_border_bottom;
hsp = hw_crtc_timing.flags.HSYNC_POSITIVE_POLARITY ? 0 : 0x80;
vsp = hw_crtc_timing.flags.VSYNC_POSITIVE_POLARITY ? 0 : 0x80;
- v_freq = hw_crtc_timing.pix_clk_100hz * 100;
+ v_freq = (uint64_t)hw_crtc_timing.pix_clk_100hz * 100;
/* MSA Packet Mapping to 32-bit Link Symbols - DP2 spec, section 2.7.4.1
*
{
struct dcn31_hpo_dp_stream_encoder *enc3 = DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(enc);
uint32_t dmdata_packet_enabled = 0;
- bool sdp_stream_enable = false;
- if (info_frame->vsc.valid) {
+ if (info_frame->vsc.valid)
enc->vpg->funcs->update_generic_info_packet(
enc->vpg,
0, /* packetIndex */
&info_frame->vsc,
true);
- sdp_stream_enable = true;
- }
- if (info_frame->spd.valid) {
+
+ if (info_frame->spd.valid)
enc->vpg->funcs->update_generic_info_packet(
enc->vpg,
2, /* packetIndex */
&info_frame->spd,
true);
- sdp_stream_enable = true;
- }
- if (info_frame->hdrsmd.valid) {
+
+ if (info_frame->hdrsmd.valid)
enc->vpg->funcs->update_generic_info_packet(
enc->vpg,
3, /* packetIndex */
&info_frame->hdrsmd,
true);
- sdp_stream_enable = true;
- }
+
/* enable/disable transmission of packet(s).
* If enabled, packet transmission begins on the next frame
*/
// Setup manual flow control for EOF via TRIG_A
optc->funcs->setup_manual_trigger(optc);
-
} else {
REG_UPDATE_4(OTG_V_TOTAL_CONTROL,
OTG_SET_V_TOTAL_MIN_MASK, 0,
.enable_crtc_reset = optc1_enable_crtc_reset,
.disable_reset_trigger = optc1_disable_reset_trigger,
.lock = optc3_lock,
- .is_locked = optc1_is_locked,
.unlock = optc1_unlock,
.lock_doublebuffer_enable = optc3_lock_doublebuffer_enable,
.lock_doublebuffer_disable = optc3_lock_doublebuffer_disable,
}
},
.disable_z10 = true,
- .optimize_edp_link_rate = true,
.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
- .dml_hostvm_override = DML_HOSTVM_NO_OVERRIDE,
+ .dml_hostvm_override = DML_HOSTVM_OVERRIDE_FALSE,
};
static const struct dc_debug_options debug_defaults_diags = {
.use_max_lb = true
};
+static const struct dc_panel_config panel_config_defaults = {
+ .ilr = {
+ .optimize_edp_link_rate = true,
+ },
+};
+
static void dcn31_dpp_destroy(struct dpp **dpp)
{
kfree(TO_DCN20_DPP(*dpp));
return out;
}
+static void dcn31_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+ *panel_config = panel_config_defaults;
+}
+
static struct dc_cap_funcs cap_funcs = {
.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
};
.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
.update_bw_bounding_box = dcn31_update_bw_bounding_box,
.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+ .get_panel_config_defaults = dcn31_get_panel_config_defaults,
};
static struct clock_source *dcn30_clock_source_create(
return two_pix;
}
-void enc314_stream_encoder_dp_blank(
+static void enc314_stream_encoder_dp_blank(
struct dc_link *link,
struct stream_encoder *enc)
{
};
static const struct dc_debug_options debug_defaults_drv = {
- .disable_z10 = true, /*hw not support it*/
+ .disable_z10 = false,
+ .enable_z9_disable_interface = true,
.disable_dmcu = true,
.force_abm_enable = false,
.timing_trace = false,
.afmt = true,
}
},
- .optimize_edp_link_rate = true,
.seamless_boot_odm_combine = true
};
.use_max_lb = true
};
+static const struct dc_panel_config panel_config_defaults = {
+ .ilr = {
+ .optimize_edp_link_rate = true,
+ },
+};
+
static void dcn31_dpp_destroy(struct dpp **dpp)
{
kfree(TO_DCN20_DPP(*dpp));
DC_FP_END();
}
+static void dcn314_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+ *panel_config = panel_config_defaults;
+}
+
static struct resource_funcs dcn314_res_pool_funcs = {
.destroy = dcn314_destroy_resource_pool,
.link_enc_create = dcn31_link_encoder_create,
.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
.update_bw_bounding_box = dcn314_update_bw_bounding_box,
.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+ .get_panel_config_defaults = dcn314_get_panel_config_defaults,
};
static struct clock_source *dcn30_clock_source_create(
.afmt = true,
}
},
- .optimize_edp_link_rate = true,
.psr_power_use_phy_fsm = 0,
};
.use_max_lb = true
};
+static const struct dc_panel_config panel_config_defaults = {
+ .ilr = {
+ .optimize_edp_link_rate = true,
+ },
+};
+
static void dcn31_dpp_destroy(struct dpp **dpp)
{
kfree(TO_DCN20_DPP(*dpp));
return pipe_cnt;
}
+static void dcn315_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+ *panel_config = panel_config_defaults;
+}
+
static struct dc_cap_funcs cap_funcs = {
.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
};
.panel_cntl_create = dcn31_panel_cntl_create,
.validate_bandwidth = dcn31_validate_bandwidth,
.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
- .update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
+ .update_soc_for_wm_a = dcn315_update_soc_for_wm_a,
.populate_dml_pipes = dcn315_populate_dml_pipes_from_context,
.acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer,
.add_stream_to_ctx = dcn30_add_stream_to_ctx,
.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
.update_bw_bounding_box = dcn315_update_bw_bounding_box,
.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+ .get_panel_config_defaults = dcn315_get_panel_config_defaults,
};
static bool dcn315_resource_construct(
.afmt = true,
}
},
- .optimize_edp_link_rate = true,
};
static const struct dc_debug_options debug_defaults_diags = {
.use_max_lb = true
};
+static const struct dc_panel_config panel_config_defaults = {
+ .ilr = {
+ .optimize_edp_link_rate = true,
+ },
+};
+
static void dcn31_dpp_destroy(struct dpp **dpp)
{
kfree(TO_DCN20_DPP(*dpp));
return pipe_cnt;
}
+static void dcn316_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+ *panel_config = panel_config_defaults;
+}
+
static struct dc_cap_funcs cap_funcs = {
.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
};
.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
.update_bw_bounding_box = dcn316_update_bw_bounding_box,
.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+ .get_panel_config_defaults = dcn316_get_panel_config_defaults,
};
static bool dcn316_resource_construct(
}
-void enc32_set_dig_output_mode(struct link_encoder *enc, uint8_t pix_per_container)
-{
- struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
- REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, pix_per_container);
-}
-
static const struct link_encoder_funcs dcn32_link_enc_funcs = {
.read_state = link_enc2_read_state,
.validate_output_with_stream =
.is_in_alt_mode = dcn32_link_encoder_is_in_alt_mode,
.get_max_link_cap = dcn32_link_encoder_get_max_link_cap,
.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
- .set_dig_output_mode = enc32_set_dig_output_mode,
};
void dcn32_link_encoder_construct(
const struct dc_link_settings *link_settings,
enum clock_source_id clock_source);
-void enc32_set_dig_output_mode(
- struct link_encoder *enc,
- uint8_t pix_per_container);
-
#endif /* __DC_LINK_ENCODER__DCN32_H__ */
return two_pix;
}
+static bool is_h_timing_divisible_by_2(const struct dc_crtc_timing *timing)
+{
+ /* math borrowed from function of same name in inc/resource
+ * checks if h_timing is divisible by 2
+ */
+
+ bool divisible = false;
+ uint16_t h_blank_start = 0;
+ uint16_t h_blank_end = 0;
+
+ if (timing) {
+ h_blank_start = timing->h_total - timing->h_front_porch;
+ h_blank_end = h_blank_start - timing->h_addressable;
+
+ /* HTOTAL, Hblank start/end, and Hsync start/end all must be
+ * divisible by 2 in order for the horizontal timing params
+ * to be considered divisible by 2. Hsync start is always 0.
+ */
+ divisible = (timing->h_total % 2 == 0) &&
+ (h_blank_start % 2 == 0) &&
+ (h_blank_end % 2 == 0) &&
+ (timing->h_sync_width % 2 == 0);
+ }
+ return divisible;
+}
+
+static bool is_dp_dig_pixel_rate_div_policy(struct dc *dc, const struct dc_crtc_timing *timing)
+{
+ /* should be functionally the same as dcn32_is_dp_dig_pixel_rate_div_policy for DP encoders*/
+ return is_h_timing_divisible_by_2(timing) &&
+ dc->debug.enable_dp_dig_pixel_rate_div_policy;
+}
+
static void enc32_stream_encoder_dp_unblank(
struct dc_link *link,
struct stream_encoder *enc,
/* YCbCr 4:2:0 : Computed VID_M will be 2X the input rate */
if (is_two_pixels_per_containter(¶m->timing) || param->opp_cnt > 1
- || dc->debug.enable_dp_dig_pixel_rate_div_policy) {
+ || is_dp_dig_pixel_rate_div_policy(dc, ¶m->timing)) {
/*this logic should be the same in get_pixel_clock_parameters() */
n_multiply = 1;
}
{
struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
- REG_UPDATE(DP_DSC_CNTL, DP_DSC_MODE, dsc_mode);
+ REG_UPDATE(DP_DSC_CNTL, DP_DSC_MODE, dsc_mode == OPTC_DSC_DISABLED ? 0 : 1);
}
/* this function read dsc related register fields to be logged later in dcn10_log_hw_state
}
}
-static void enc32_stream_encoder_reset_fifo(struct stream_encoder *enc)
-{
- struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
- uint32_t fifo_enabled;
-
- REG_GET(DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, &fifo_enabled);
-
- if (fifo_enabled == 0) {
- /* reset DIG resync FIFO */
- REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1);
- /* TODO: fix timeout when wait for DIG_FIFO_RESET_DONE */
- //REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 1, 1, 100);
- udelay(1);
- REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 0);
- REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 0, 1, 100);
- }
-}
-
static void enc32_set_dig_input_mode(struct stream_encoder *enc, unsigned int pix_per_container)
{
struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
enc3_stream_encoder_update_dp_info_packets,
.stop_dp_info_packets =
enc1_stream_encoder_stop_dp_info_packets,
- .reset_fifo =
- enc32_stream_encoder_reset_fifo,
.dp_blank =
enc1_stream_encoder_dp_blank,
.dp_unblank =
SRI(DP_MSE_RATE_UPDATE, DP, id), \
SRI(DP_PIXEL_FORMAT, DP, id), \
SRI(DP_SEC_CNTL, DP, id), \
+ SRI(DP_SEC_CNTL1, DP, id), \
SRI(DP_SEC_CNTL2, DP, id), \
+ SRI(DP_SEC_CNTL5, DP, id), \
SRI(DP_SEC_CNTL6, DP, id), \
SRI(DP_STEER_FIFO, DP, id), \
SRI(DP_VID_M, DP, id), \
SRI(DIG_FIFO_CTRL0, DIG, id)
-#define SE_COMMON_MASK_SH_LIST_DCN32_BASE(mask_sh)\
+#define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\
SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_ENCODING, mask_sh),\
SE_SF(DP0_DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH, mask_sh),\
SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_PER_CYCLE_PROCESSING_MODE, mask_sh),\
SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_CONT, mask_sh),\
SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_SEND, mask_sh),\
SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_NULL_SEND, mask_sh),\
+ SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, mask_sh),\
SE_SF(DIG0_HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_SEND, mask_sh),\
SE_SF(DIG0_HDMI_INFOFRAME_CONTROL1, HDMI_AUDIO_INFO_LINE, mask_sh),\
SE_SF(DIG0_HDMI_GC, HDMI_GC_AVMUTE, mask_sh),\
SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, mask_sh),\
SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, mask_sh)
-#if defined(CONFIG_DRM_AMD_DC_HDCP)
-#define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\
- SE_COMMON_MASK_SH_LIST_DCN32_BASE(mask_sh),\
- SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, mask_sh)
-#else
-#define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\
- SE_COMMON_MASK_SH_LIST_DCN32_BASE(mask_sh)
-#endif
-
void dcn32_dio_stream_encoder_construct(
struct dcn10_stream_encoder *enc1,
struct dc_context *ctx,
SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG, TP_PRBS_SEL1, mask_sh),\
SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG, TP_PRBS_SEL2, mask_sh),\
SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG, TP_PRBS_SEL3, mask_sh),\
+ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE, TP_SQ_PULSE_WIDTH, mask_sh),\
SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0, SAT_STREAM_SOURCE, mask_sh),\
SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0, SAT_SLOT_COUNT, mask_sh),\
SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0, STREAM_VC_RATE_X, mask_sh),\
.program_watermarks = hubbub32_program_watermarks,
.allow_self_refresh_control = hubbub1_allow_self_refresh_control,
.is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled,
+ .verify_allow_pstate_change_high = hubbub1_verify_allow_pstate_change_high,
.force_wm_propagate_to_pipes = hubbub32_force_wm_propagate_to_pipes,
.force_pstate_change_control = hubbub3_force_pstate_change_control,
.init_watermarks = hubbub32_init_watermarks,
uint32_t reg_val;
struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+ /* For phantom pipe enable, disable GSL */
+ REG_UPDATE(DCSURF_FLIP_CONTROL2, SURFACE_GSL_ENABLE, 0);
REG_UPDATE(DCHUBP_CNTL, HUBP_BLANK_EN, 1);
reg_val = REG_READ(DCHUBP_CNTL);
if (reg_val) {
.hubp_init = hubp3_init,
.set_unbounded_requesting = hubp31_set_unbounded_requesting,
.hubp_soft_reset = hubp31_soft_reset,
+ .hubp_set_flip_int = hubp1_set_flip_int,
.hubp_in_blank = hubp1_in_blank,
.hubp_update_force_pstate_disallow = hubp32_update_force_pstate_disallow,
.phantom_hubp_post_enable = hubp32_phantom_hubp_post_enable,
.hubp_update_mall_sel = hubp32_update_mall_sel,
- .hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering,
- .hubp_set_flip_int = hubp1_set_flip_int
+ .hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering
};
bool hubp32_construct(
*/
static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *ctx)
{
- uint8_t i;
- int j;
+ int i, j;
struct dc_stream_state *stream = NULL;
struct dc_plane_state *plane = NULL;
uint32_t cursor_size = 0;
params = &dpp_base->degamma_params;
}
- result = dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
+ dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
- if (result &&
- pipe_ctx->stream_res.opp &&
+ if (pipe_ctx->stream_res.opp &&
pipe_ctx->stream_res.opp->ctx &&
hws->funcs.set_mcm_luts)
result = hws->funcs.set_mcm_luts(pipe_ctx, plane_state);
dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv->dmub);
dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
}
+
+ /* Enable support for ODM and windowed MPO if policy flag is set */
+ if (dc->debug.enable_single_display_2to1_odm_policy)
+ dc->config.enable_windowed_mpo_odm = true;
}
static int calc_mpc_flow_ctrl_cnt(const struct dc_stream_state *stream,
true);
}
- // Don't program pixel clock after link is already enabled
-/* if (false == pipe_ctx->clock_source->funcs->program_pix_clk(
- pipe_ctx->clock_source,
- &pipe_ctx->stream_res.pix_clk_params,
- &pipe_ctx->pll_settings)) {
- BREAK_TO_DEBUGGER();
- }*/
+ if (pipe_ctx->stream_res.dsc) {
+ struct pipe_ctx *current_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
- if (pipe_ctx->stream_res.dsc)
update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC);
+
+ /* Check if no longer using pipe for ODM, then need to disconnect DSC for that pipe */
+ if (!pipe_ctx->next_odm_pipe && current_pipe_ctx->next_odm_pipe &&
+ current_pipe_ctx->next_odm_pipe->stream_res.dsc) {
+ struct display_stream_compressor *dsc = current_pipe_ctx->next_odm_pipe->stream_res.dsc;
+ /* disconnect DSC block from stream */
+ dsc->funcs->dsc_disconnect(dsc);
+ }
+ }
}
unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsigned int *k1_div, unsigned int *k2_div)
{
struct dc_stream_state *stream = pipe_ctx->stream;
unsigned int odm_combine_factor = 0;
- struct dc *dc = pipe_ctx->stream->ctx->dc;
bool two_pix_per_container = false;
// For phantom pipes, use the same programming as the main pipes
} else {
*k1_div = PIXEL_RATE_DIV_BY_1;
*k2_div = PIXEL_RATE_DIV_BY_4;
- if ((odm_combine_factor == 2) || dc->debug.enable_dp_dig_pixel_rate_div_policy)
+ if ((odm_combine_factor == 2) || dcn32_is_dp_dig_pixel_rate_div_policy(pipe_ctx))
*k2_div = PIXEL_RATE_DIV_BY_2;
}
}
struct dc_link *link = stream->link;
struct dce_hwseq *hws = link->dc->hwseq;
struct pipe_ctx *odm_pipe;
- struct dc *dc = pipe_ctx->stream->ctx->dc;
uint32_t pix_per_cycle = 1;
params.opp_cnt = 1;
pipe_ctx->stream_res.tg->inst);
} else if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
if (optc2_is_two_pixels_per_containter(&stream->timing) || params.opp_cnt > 1
- || dc->debug.enable_dp_dig_pixel_rate_div_policy) {
+ || dcn32_is_dp_dig_pixel_rate_div_policy(pipe_ctx)) {
params.timing.pix_clk_100hz /= 2;
pix_per_cycle = 2;
}
{
struct dc *dc = pipe_ctx->stream->ctx->dc;
+ if (!is_h_timing_divisible_by_2(pipe_ctx->stream))
+ return false;
+
if (dc_is_dp_signal(pipe_ctx->stream->signal) && !is_dp_128b_132b_signal(pipe_ctx) &&
dc->debug.enable_dp_dig_pixel_rate_div_policy)
return true;
break;
}
- return pwr_status == 0 ? true : false;
+ return pwr_status == 0;
}
void dcn32_update_dsc_pg(struct dc *dc,
/* CRTC disabled, so disable clock. */
REG_WAIT(OTG_CLOCK_CONTROL,
OTG_BUSY, 0,
- 1, 100000);
+ 1, 150000);
return true;
}
phantom_plane->clip_rect.y = 0;
phantom_plane->clip_rect.height = phantom_stream->timing.v_addressable;
+ phantom_plane->is_phantom = true;
+
dc_add_plane_to_context(dc, phantom_stream, phantom_plane, context);
curr_pipe = curr_pipe->bottom_pipe;
pipe->stream->mall_stream_config.type = SUBVP_NONE;
pipe->stream->mall_stream_config.paired_stream = NULL;
}
+
+ if (pipe->plane_state) {
+ pipe->plane_state->is_phantom = false;
+ }
}
return removed_pipe;
}
int vlevel = 0;
int pipe_cnt = 0;
display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+ struct mall_temp_config mall_temp_config;
+
+ /* To handle Freesync properly, setting FreeSync DML parameters
+ * to its default state for the first stage of validation
+ */
+ context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false;
+ context->bw_ctx.dml.soc.dram_clock_change_requirement_final = true;
+
DC_LOGGER_INIT(dc->ctx->logger);
+ /* For fast validation, there are situations where a shallow copy of
+ * of the dc->current_state is created for the validation. In this case
+ * we want to save and restore the mall config because we always
+ * teardown subvp at the beginning of validation (and don't attempt
+ * to add it back if it's fast validation). If we don't restore the
+ * subvp config in cases of fast validation + shallow copy of the
+ * dc->current_state, the dc->current_state will have a partially
+ * removed subvp state when we did not intend to remove it.
+ */
+ if (fast_validate) {
+ memset(&mall_temp_config, 0, sizeof(mall_temp_config));
+ dcn32_save_mall_state(dc, context, &mall_temp_config);
+ }
+
BW_VAL_TRACE_COUNT();
DC_FP_START();
out = dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate);
DC_FP_END();
+ if (fast_validate)
+ dcn32_restore_mall_state(dc, context, &mall_temp_config);
+
if (pipe_cnt == 0)
goto validate_out;
extern struct _vcs_dpi_ip_params_st dcn3_2_ip;
extern struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc;
+/* Temp struct used to save and restore MALL config
+ * during validation.
+ *
+ * TODO: Move MALL config into dc_state instead of stream struct
+ * to avoid needing to save/restore.
+ */
+struct mall_temp_config {
+ struct mall_stream_config mall_stream_config[MAX_PIPES];
+ bool is_phantom_plane[MAX_PIPES];
+};
+
struct dcn32_resource_pool {
struct resource_pool base;
};
bool dcn32_mpo_in_use(struct dc_state *context);
+bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context);
+
struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
struct dc_state *state,
const struct resource_pool *pool,
void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
display_e2e_pipe_params_st *pipes);
+
+void dcn32_save_mall_state(struct dc *dc,
+ struct dc_state *context,
+ struct mall_temp_config *temp_config);
+
+void dcn32_restore_mall_state(struct dc *dc,
+ struct dc_state *context,
+ struct mall_temp_config *temp_config);
+
/* definitions for run time init of reg offsets */
/* CLK SRC */
return false;
}
+
+bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context)
+{
+ uint32_t i;
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+ if (!pipe->stream)
+ continue;
+
+ if (pipe->plane_state && pipe->plane_state->rotation != ROTATION_ANGLE_0)
+ return true;
+ }
+ return false;
+}
+
/**
* *******************************************************************************************
* dcn32_determine_det_override: Determine DET allocation for each pipe
} else
dcn32_determine_det_override(dc, context, pipes);
}
+
+/**
+ * *******************************************************************************************
+ * dcn32_save_mall_state: Save MALL (SubVP) state for fast validation cases
+ *
+ * This function saves the MALL (SubVP) case for fast validation cases. For fast validation,
+ * there are situations where a shallow copy of the dc->current_state is created for the
+ * validation. In this case we want to save and restore the mall config because we always
+ * teardown subvp at the beginning of validation (and don't attempt to add it back if it's
+ * fast validation). If we don't restore the subvp config in cases of fast validation +
+ * shallow copy of the dc->current_state, the dc->current_state will have a partially
+ * removed subvp state when we did not intend to remove it.
+ *
+ * NOTE: This function ONLY works if the streams are not moved to a different pipe in the
+ * validation. We don't expect this to happen in fast_validation=1 cases.
+ *
+ * @param [in]: dc: Current DC state
+ * @param [in]: context: New DC state to be programmed
+ * @param [out]: temp_config: struct used to cache the existing MALL state
+ *
+ * @return: void
+ *
+ * *******************************************************************************************
+ */
+void dcn32_save_mall_state(struct dc *dc,
+ struct dc_state *context,
+ struct mall_temp_config *temp_config)
+{
+ uint32_t i;
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+ if (pipe->stream)
+ temp_config->mall_stream_config[i] = pipe->stream->mall_stream_config;
+
+ if (pipe->plane_state)
+ temp_config->is_phantom_plane[i] = pipe->plane_state->is_phantom;
+ }
+}
+
+/**
+ * *******************************************************************************************
+ * dcn32_restore_mall_state: Restore MALL (SubVP) state for fast validation cases
+ *
+ * Restore the MALL state based on the previously saved state from dcn32_save_mall_state
+ *
+ * @param [in]: dc: Current DC state
+ * @param [in/out]: context: New DC state to be programmed, restore MALL state into here
+ * @param [in]: temp_config: struct that has the cached MALL state
+ *
+ * @return: void
+ *
+ * *******************************************************************************************
+ */
+void dcn32_restore_mall_state(struct dc *dc,
+ struct dc_state *context,
+ struct mall_temp_config *temp_config)
+{
+ uint32_t i;
+
+ for (i = 0; i < dc->res_pool->pipe_count; i++) {
+ struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+ if (pipe->stream)
+ pipe->stream->mall_stream_config = temp_config->mall_stream_config[i];
+
+ if (pipe->plane_state)
+ pipe->plane_state->is_phantom = temp_config->is_phantom_plane[i];
+ }
+}
.is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode,
.get_max_link_cap = dcn20_link_encoder_get_max_link_cap,
.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
- .set_dig_output_mode = enc32_set_dig_output_mode,
};
void dcn321_link_encoder_construct(
#include "dcn20/dcn20_vmid.h"
#define DC_LOGGER_INIT(logger)
-#define fixed16_to_double(x) (((double)x) / ((double) (1 << 16)))
-#define fixed16_to_double_to_cpu(x) fixed16_to_double(le32_to_cpu(x))
enum dcn321_clk_src_array_id {
DCN321_CLK_SRC_PLL0,
.validate_bandwidth = dcn32_validate_bandwidth,
.calculate_wm_and_dlg = dcn32_calculate_wm_and_dlg,
.populate_dml_pipes = dcn32_populate_dml_pipes_from_context,
- .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer,
+ .acquire_idle_pipe_for_head_pipe_in_layer = dcn32_acquire_idle_pipe_for_head_pipe_in_layer,
.add_stream_to_ctx = dcn30_add_stream_to_ctx,
.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
#undef REG_STRUCT
#define REG_STRUCT dccg_regs
- dccg_regs_init();
+ dccg_regs_init();
ctx->dc_bios->regs = &bios_regs;
return dcf_clk;
}
-static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks)
+void dcn_bw_update_from_pplib_fclks(
+ struct dc *dc,
+ struct dm_pp_clock_levels_with_voltage *fclks)
{
- int i;
-
- if (clks->num_levels == 0)
- return false;
-
- for (i = 0; i < clks->num_levels; i++)
- /* Ensure that the result is sane */
- if (clks->data[i].clocks_in_khz == 0)
- return false;
+ unsigned vmin0p65_idx, vmid0p72_idx, vnom0p8_idx, vmax0p9_idx;
- return true;
+ ASSERT(fclks->num_levels);
+
+ vmin0p65_idx = 0;
+ vmid0p72_idx = fclks->num_levels -
+ (fclks->num_levels > 2 ? 3 : (fclks->num_levels > 1 ? 2 : 1));
+ vnom0p8_idx = fclks->num_levels - (fclks->num_levels > 1 ? 2 : 1);
+ vmax0p9_idx = fclks->num_levels - 1;
+
+ dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 =
+ 32 * (fclks->data[vmin0p65_idx].clocks_in_khz / 1000.0) / 1000.0;
+ dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 =
+ dc->dcn_soc->number_of_channels *
+ (fclks->data[vmid0p72_idx].clocks_in_khz / 1000.0)
+ * ddr4_dram_factor_single_Channel / 1000.0;
+ dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 =
+ dc->dcn_soc->number_of_channels *
+ (fclks->data[vnom0p8_idx].clocks_in_khz / 1000.0)
+ * ddr4_dram_factor_single_Channel / 1000.0;
+ dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 =
+ dc->dcn_soc->number_of_channels *
+ (fclks->data[vmax0p9_idx].clocks_in_khz / 1000.0)
+ * ddr4_dram_factor_single_Channel / 1000.0;
}
-void dcn_bw_update_from_pplib(struct dc *dc)
+void dcn_bw_update_from_pplib_dcfclks(
+ struct dc *dc,
+ struct dm_pp_clock_levels_with_voltage *dcfclks)
{
- struct dc_context *ctx = dc->ctx;
- struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0};
- bool res;
- unsigned vmin0p65_idx, vmid0p72_idx, vnom0p8_idx, vmax0p9_idx;
-
- /* TODO: This is not the proper way to obtain fabric_and_dram_bandwidth, should be min(fclk, memclk) */
- res = dm_pp_get_clock_levels_by_type_with_voltage(
- ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks);
-
- if (res)
- res = verify_clock_values(&fclks);
-
- if (res) {
- ASSERT(fclks.num_levels);
-
- vmin0p65_idx = 0;
- vmid0p72_idx = fclks.num_levels -
- (fclks.num_levels > 2 ? 3 : (fclks.num_levels > 1 ? 2 : 1));
- vnom0p8_idx = fclks.num_levels - (fclks.num_levels > 1 ? 2 : 1);
- vmax0p9_idx = fclks.num_levels - 1;
-
- dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 =
- 32 * (fclks.data[vmin0p65_idx].clocks_in_khz / 1000.0) / 1000.0;
- dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 =
- dc->dcn_soc->number_of_channels *
- (fclks.data[vmid0p72_idx].clocks_in_khz / 1000.0)
- * ddr4_dram_factor_single_Channel / 1000.0;
- dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 =
- dc->dcn_soc->number_of_channels *
- (fclks.data[vnom0p8_idx].clocks_in_khz / 1000.0)
- * ddr4_dram_factor_single_Channel / 1000.0;
- dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 =
- dc->dcn_soc->number_of_channels *
- (fclks.data[vmax0p9_idx].clocks_in_khz / 1000.0)
- * ddr4_dram_factor_single_Channel / 1000.0;
- } else
- BREAK_TO_DEBUGGER();
-
- res = dm_pp_get_clock_levels_by_type_with_voltage(
- ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks);
-
- if (res)
- res = verify_clock_values(&dcfclks);
+ if (dcfclks->num_levels >= 3) {
+ dc->dcn_soc->dcfclkv_min0p65 = dcfclks->data[0].clocks_in_khz / 1000.0;
+ dc->dcn_soc->dcfclkv_mid0p72 = dcfclks->data[dcfclks->num_levels - 3].clocks_in_khz / 1000.0;
+ dc->dcn_soc->dcfclkv_nom0p8 = dcfclks->data[dcfclks->num_levels - 2].clocks_in_khz / 1000.0;
+ dc->dcn_soc->dcfclkv_max0p9 = dcfclks->data[dcfclks->num_levels - 1].clocks_in_khz / 1000.0;
+ }
+}
- if (res && dcfclks.num_levels >= 3) {
- dc->dcn_soc->dcfclkv_min0p65 = dcfclks.data[0].clocks_in_khz / 1000.0;
- dc->dcn_soc->dcfclkv_mid0p72 = dcfclks.data[dcfclks.num_levels - 3].clocks_in_khz / 1000.0;
- dc->dcn_soc->dcfclkv_nom0p8 = dcfclks.data[dcfclks.num_levels - 2].clocks_in_khz / 1000.0;
- dc->dcn_soc->dcfclkv_max0p9 = dcfclks.data[dcfclks.num_levels - 1].clocks_in_khz / 1000.0;
- } else
- BREAK_TO_DEBUGGER();
+void dcn_get_soc_clks(
+ struct dc *dc,
+ int *min_fclk_khz,
+ int *min_dcfclk_khz,
+ int *socclk_khz)
+{
+ *min_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000 / 32;
+ *min_dcfclk_khz = dc->dcn_soc->dcfclkv_min0p65 * 1000;
+ *socclk_khz = dc->dcn_soc->socclk * 1000;
}
-void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc)
+void dcn_bw_notify_pplib_of_wm_ranges(
+ struct dc *dc,
+ int min_fclk_khz,
+ int min_dcfclk_khz,
+ int socclk_khz)
{
struct pp_smu_funcs_rv *pp = NULL;
struct pp_smu_wm_range_sets ranges = {0};
- int min_fclk_khz, min_dcfclk_khz, socclk_khz;
const int overdrive = 5000000; /* 5 GHz to cover Overdrive */
if (dc->res_pool->pp_smu)
if (!pp || !pp->set_wm_ranges)
return;
- min_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000 / 32;
- min_dcfclk_khz = dc->dcn_soc->dcfclkv_min0p65 * 1000;
- socclk_khz = dc->dcn_soc->socclk * 1000;
-
/* Now notify PPLib/SMU about which Watermarks sets they should select
* depending on DPM state they are in. And update BW MGR GFX Engine and
* Memory clock member variables for Watermarks calculations for each
.urgent_latency_adjustment_fabric_clock_component_us = 0,
.urgent_latency_adjustment_fabric_clock_reference_mhz = 0,
.num_chans = 4,
+ .dummy_pstate_latency_us = 10.0
};
struct _vcs_dpi_ip_params_st dcn3_16_ip = {
}
}
+void dcn315_update_soc_for_wm_a(struct dc *dc, struct dc_state *context)
+{
+ dc_assert_fp_enabled();
+
+ if (dc->clk_mgr->bw_params->wm_table.entries[WM_A].valid) {
+ /* For 315 pstate change is only supported if possible in vactive */
+ if (context->bw_ctx.dml.vba.DRAMClockChangeSupport[context->bw_ctx.dml.vba.VoltageLevel][context->bw_ctx.dml.vba.maxMpcComb] != dm_dram_clock_change_vactive)
+ context->bw_ctx.dml.soc.dram_clock_change_latency_us = context->bw_ctx.dml.soc.dummy_pstate_latency_us;
+ else
+ context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.entries[WM_A].pstate_latency_us;
+ context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us =
+ dc->clk_mgr->bw_params->wm_table.entries[WM_A].sr_enter_plus_exit_time_us;
+ context->bw_ctx.dml.soc.sr_exit_time_us =
+ dc->clk_mgr->bw_params->wm_table.entries[WM_A].sr_exit_time_us;
+ }
+}
+
void dcn31_calculate_wm_and_dlg_fp(
struct dc *dc, struct dc_state *context,
display_e2e_pipe_params_st *pipes,
int pipe_cnt,
int vlevel)
{
- int i, pipe_idx;
+ int i, pipe_idx, active_dpp_count = 0;
double dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
dc_assert_fp_enabled();
pipes[0].clks_cfg.dcfclk_mhz = dcfclk;
pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
-#if 0 // TODO
- /* Set B:
- * TODO
- */
- if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].valid) {
- if (vlevel == 0) {
- pipes[0].clks_cfg.voltage = 1;
- pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dcfclk_mhz;
- }
- context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.pstate_latency_us;
- context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_enter_plus_exit_time_us;
- context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_exit_time_us;
- }
- context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-
- pipes[0].clks_cfg.voltage = vlevel;
- pipes[0].clks_cfg.dcfclk_mhz = dcfclk;
-
- /* Set C:
- * TODO
- */
- if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) {
- context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.pstate_latency_us;
- context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_enter_plus_exit_time_us;
- context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us;
- }
- context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-
- /* Set D:
- * TODO
- */
- if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].valid) {
- context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us;
- context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us;
- context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us;
- }
- context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- context->bw_ctx.bw.dcn.watermarks.d.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-#endif
-
/* Set A:
* All clocks min required
*
context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
- /* TODO: remove: */
context->bw_ctx.bw.dcn.watermarks.b = context->bw_ctx.bw.dcn.watermarks.a;
context->bw_ctx.bw.dcn.watermarks.c = context->bw_ctx.bw.dcn.watermarks.a;
context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a;
- /* end remove*/
for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
if (!context->res_ctx.pipe_ctx[i].stream)
continue;
+ if (context->res_ctx.pipe_ctx[i].plane_state)
+ active_dpp_count++;
+
pipes[pipe_idx].clks_cfg.dispclk_mhz = get_dispclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt);
pipes[pipe_idx].clks_cfg.dppclk_mhz = get_dppclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
}
dcn20_calculate_dlg_params(dc, context, pipes, pipe_cnt, vlevel);
+ /* For 31x apu pstate change is only supported if possible in vactive or if there are no active dpps */
+ context->bw_ctx.bw.dcn.clk.p_state_change_support =
+ context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] == dm_dram_clock_change_vactive || !active_dpp_count;
}
void dcn31_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
}
if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
- dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN31);
+ dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN315);
else
dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN31_FPGA);
}
int pipe_cnt);
void dcn31_update_soc_for_wm_a(struct dc *dc, struct dc_state *context);
+void dcn315_update_soc_for_wm_a(struct dc *dc, struct dc_state *context);
void dcn31_calculate_wm_and_dlg_fp(
struct dc *dc, struct dc_state *context,
#define BPP_BLENDED_PIPE 0xffffffff
#define DCN31_MAX_DSC_IMAGE_WIDTH 5184
#define DCN31_MAX_FMT_420_BUFFER_WIDTH 4096
+#define DCN3_15_MIN_COMPBUF_SIZE_KB 128
+#define DCN3_15_MAX_DET_SIZE 384
// For DML-C changes that hasn't been propagated to VBA yet
//#define __DML_VBA_ALLOW_DELTA__
&v->VReadyOffsetPix[k]);
}
+static void PatchDETBufferSizeInKByte(unsigned int NumberOfActivePlanes, int NoOfDPPThisState[], unsigned int config_return_buffer_size_in_kbytes, unsigned int *DETBufferSizeInKByte)
+{
+ int i, total_pipes = 0;
+ for (i = 0; i < NumberOfActivePlanes; i++)
+ total_pipes += NoOfDPPThisState[i];
+ *DETBufferSizeInKByte = ((config_return_buffer_size_in_kbytes - DCN3_15_MIN_COMPBUF_SIZE_KB) / 64 / total_pipes) * 64;
+ if (*DETBufferSizeInKByte > DCN3_15_MAX_DET_SIZE)
+ *DETBufferSizeInKByte = DCN3_15_MAX_DET_SIZE;
+}
+
+
void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib)
{
struct vba_vars_st *v = &mode_lib->vba;
v->ODMCombineEnableThisState[k] = v->ODMCombineEnablePerState[i][k];
}
+ if (v->NumberOfActivePlanes > 1 && mode_lib->project == DML_PROJECT_DCN315)
+ PatchDETBufferSizeInKByte(v->NumberOfActivePlanes, v->NoOfDPPThisState, v->ip.config_return_buffer_size_in_kbytes, &v->DETBufferSizeInKByte[0]);
CalculateSwathAndDETConfiguration(
false,
v->NumberOfActivePlanes,
clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF;
}
-/**
+/*
* Finds dummy_latency_index when MCLK switching using firmware based
* vblank stretch is enabled. This function will iterate through the
* table of dummy pstate latencies until the lowest value that allows
/**
* dcn32_helper_populate_phantom_dlg_params - Get DLG params for phantom pipes
* and populate pipe_ctx with those params.
- *
- * This function must be called AFTER the phantom pipes are added to context
- * and run through DML (so that the DLG params for the phantom pipes can be
- * populated), and BEFORE we program the timing for the phantom pipes.
- *
* @dc: [in] current dc state
* @context: [in] new dc state
* @pipes: [in] DML pipe params array
* @pipe_cnt: [in] DML pipe count
+ *
+ * This function must be called AFTER the phantom pipes are added to context
+ * and run through DML (so that the DLG params for the phantom pipes can be
+ * populated), and BEFORE we program the timing for the phantom pipes.
*/
void dcn32_helper_populate_phantom_dlg_params(struct dc *dc,
struct dc_state *context,
}
/**
- * *******************************************************************************************
- * dcn32_predict_pipe_split: Predict if pipe split will occur for a given DML pipe
+ * dcn32_predict_pipe_split - Predict if pipe split will occur for a given DML pipe
+ * @context: [in] New DC state to be programmed
+ * @pipe_e2e: [in] DML pipe end to end context
*
* This function takes in a DML pipe (pipe_e2e) and predicts if pipe split is required (both
* ODM and MPC). For pipe split, ODM combine is determined by the ODM mode, and MPC combine is
* - MPC combine is only chosen if there is no ODM combine requirements / policy in place, and
* MPC is required
*
- * @param [in]: context: New DC state to be programmed
- * @param [in]: pipe_e2e: DML pipe end to end context
- *
- * @return: Number of splits expected (1 for 2:1 split, 3 for 4:1 split, 0 for no splits).
- *
- * *******************************************************************************************
+ * Return: Number of splits expected (1 for 2:1 split, 3 for 4:1 split, 0 for no splits).
*/
uint8_t dcn32_predict_pipe_split(struct dc_state *context,
display_e2e_pipe_params_st *pipe_e2e)
}
/**
- * dcn32_set_phantom_stream_timing: Set timing params for the phantom stream
+ * dcn32_set_phantom_stream_timing - Set timing params for the phantom stream
+ * @dc: current dc state
+ * @context: new dc state
+ * @ref_pipe: Main pipe for the phantom stream
+ * @phantom_stream: target phantom stream state
+ * @pipes: DML pipe params
+ * @pipe_cnt: number of DML pipes
+ * @dc_pipe_idx: DC pipe index for the main pipe (i.e. ref_pipe)
*
* Set timing params of the phantom stream based on calculated output from DML.
* This function first gets the DML pipe index using the DC pipe index, then
* that separately.
*
* - Set phantom backporch = vstartup of main pipe
- *
- * @dc: current dc state
- * @context: new dc state
- * @ref_pipe: Main pipe for the phantom stream
- * @pipes: DML pipe params
- * @pipe_cnt: number of DML pipes
- * @dc_pipe_idx: DC pipe index for the main pipe (i.e. ref_pipe)
*/
void dcn32_set_phantom_stream_timing(struct dc *dc,
struct dc_state *context,
}
/**
- * dcn32_get_num_free_pipes: Calculate number of free pipes
+ * dcn32_get_num_free_pipes - Calculate number of free pipes
+ * @dc: current dc state
+ * @context: new dc state
*
* This function assumes that a "used" pipe is a pipe that has
* both a stream and a plane assigned to it.
*
- * @dc: current dc state
- * @context: new dc state
- *
- * Return:
- * Number of free pipes available in the context
+ * Return: Number of free pipes available in the context
*/
static unsigned int dcn32_get_num_free_pipes(struct dc *dc, struct dc_state *context)
{
}
/**
- * dcn32_assign_subvp_pipe: Function to decide which pipe will use Sub-VP.
+ * dcn32_assign_subvp_pipe - Function to decide which pipe will use Sub-VP.
+ * @dc: current dc state
+ * @context: new dc state
+ * @index: [out] dc pipe index for the pipe chosen to have phantom pipes assigned
*
* We enter this function if we are Sub-VP capable (i.e. enough pipes available)
* and regular P-State switching (i.e. VACTIVE/VBLANK) is not supported, or if
* for determining which should be the SubVP pipe (need a way to determine if a pipe / plane doesn't
* support MCLK switching naturally [i.e. ACTIVE or VBLANK]).
*
- * @param dc: current dc state
- * @param context: new dc state
- * @param index: [out] dc pipe index for the pipe chosen to have phantom pipes assigned
- *
- * Return:
- * True if a valid pipe assignment was found for Sub-VP. Otherwise false.
+ * Return: True if a valid pipe assignment was found for Sub-VP. Otherwise false.
*/
static bool dcn32_assign_subvp_pipe(struct dc *dc,
struct dc_state *context,
}
/**
- * dcn32_enough_pipes_for_subvp: Function to check if there are "enough" pipes for SubVP.
+ * dcn32_enough_pipes_for_subvp - Function to check if there are "enough" pipes for SubVP.
+ * @dc: current dc state
+ * @context: new dc state
*
* This function returns true if there are enough free pipes
* to create the required phantom pipes for any given stream
* pipe which can be used as the phantom pipe for the non pipe
* split pipe.
*
- * @dc: current dc state
- * @context: new dc state
- *
* Return:
* True if there are enough free pipes to assign phantom pipes to at least one
* stream that does not already have phantom pipes assigned. Otherwise false.
}
/**
- * subvp_subvp_schedulable: Determine if SubVP + SubVP config is schedulable
+ * subvp_subvp_schedulable - Determine if SubVP + SubVP config is schedulable
+ * @dc: current dc state
+ * @context: new dc state
*
* High level algorithm:
* 1. Find longest microschedule length (in us) between the two SubVP pipes
* pipes still allows for the maximum microschedule to fit in the active
* region for both pipes.
*
- * @dc: current dc state
- * @context: new dc state
- *
- * Return:
- * bool - True if the SubVP + SubVP config is schedulable, false otherwise
+ * Return: True if the SubVP + SubVP config is schedulable, false otherwise
*/
static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context)
{
}
/**
- * subvp_drr_schedulable: Determine if SubVP + DRR config is schedulable
+ * subvp_drr_schedulable - Determine if SubVP + DRR config is schedulable
+ * @dc: current dc state
+ * @context: new dc state
+ * @drr_pipe: DRR pipe_ctx for the SubVP + DRR config
*
* High level algorithm:
* 1. Get timing for SubVP pipe, phantom pipe, and DRR pipe
* 3.If (SubVP Active - Prefetch > Stretched DRR frame + max(MALL region, Stretched DRR frame))
* then report the configuration as supported
*
- * @dc: current dc state
- * @context: new dc state
- * @drr_pipe: DRR pipe_ctx for the SubVP + DRR config
- *
- * Return:
- * bool - True if the SubVP + DRR config is schedulable, false otherwise
+ * Return: True if the SubVP + DRR config is schedulable, false otherwise
*/
static bool subvp_drr_schedulable(struct dc *dc, struct dc_state *context, struct pipe_ctx *drr_pipe)
{
/**
- * subvp_vblank_schedulable: Determine if SubVP + VBLANK config is schedulable
+ * subvp_vblank_schedulable - Determine if SubVP + VBLANK config is schedulable
+ * @dc: current dc state
+ * @context: new dc state
*
* High level algorithm:
* 1. Get timing for SubVP pipe, phantom pipe, and VBLANK pipe
* then report the configuration as supported
* 3. If the VBLANK display is DRR, then take the DRR static schedulability path
*
- * @dc: current dc state
- * @context: new dc state
- *
- * Return:
- * bool - True if the SubVP + VBLANK/DRR config is schedulable, false otherwise
+ * Return: True if the SubVP + VBLANK/DRR config is schedulable, false otherwise
*/
static bool subvp_vblank_schedulable(struct dc *dc, struct dc_state *context)
{
}
/**
- * subvp_validate_static_schedulability: Check which SubVP case is calculated and handle
- * static analysis based on the case.
+ * subvp_validate_static_schedulability - Check which SubVP case is calculated
+ * and handle static analysis based on the case.
+ * @dc: current dc state
+ * @context: new dc state
+ * @vlevel: Voltage level calculated by DML
*
* Three cases:
* 1. SubVP + SubVP
* 2. SubVP + VBLANK (DRR checked internally)
* 3. SubVP + VACTIVE (currently unsupported)
*
- * @dc: current dc state
- * @context: new dc state
- * @vlevel: Voltage level calculated by DML
- *
- * Return:
- * bool - True if statically schedulable, false otherwise
+ * Return: True if statically schedulable, false otherwise
*/
static bool subvp_validate_static_schedulability(struct dc *dc,
struct dc_state *context,
* 5. (Config doesn't support MCLK in VACTIVE/VBLANK || dc->debug.force_subvp_mclk_switch)
*/
if (!dc->debug.force_disable_subvp && dcn32_all_pipes_have_stream_and_plane(dc, context) &&
- !dcn32_mpo_in_use(context) && (*vlevel == context->bw_ctx.dml.soc.num_states ||
+ !dcn32_mpo_in_use(context) && !dcn32_any_surfaces_rotated(dc, context) &&
+ (*vlevel == context->bw_ctx.dml.soc.num_states ||
vba->DRAMClockChangeSupport[*vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported ||
dc->debug.force_subvp_mclk_switch)) {
/*MPC split rules will handle this case*/
pipe->bottom_pipe->top_pipe = NULL;
} else {
+ /* when merging an ODM pipes, the bottom MPC pipe must now point to
+ * the previous ODM pipe and its associated stream assets
+ */
if (pipe->prev_odm_pipe->bottom_pipe) {
/* 3 plane MPO*/
pipe->bottom_pipe->top_pipe = pipe->prev_odm_pipe->bottom_pipe;
pipe->bottom_pipe->top_pipe = pipe->prev_odm_pipe;
pipe->prev_odm_pipe->bottom_pipe = pipe->bottom_pipe;
}
+
+ memcpy(&pipe->bottom_pipe->stream_res, &pipe->bottom_pipe->top_pipe->stream_res, sizeof(struct stream_resource));
}
}
int i, pipe_idx, vlevel_temp = 0;
double dcfclk = dcn3_2_soc.clock_limits[0].dcfclk_mhz;
double dcfclk_from_validation = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+ double dcfclk_from_fw_based_mclk_switching = dcfclk_from_validation;
bool pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] !=
dm_dram_clock_change_unsupported;
unsigned int dummy_latency_index = 0;
dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us;
dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, false);
maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb;
- dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+ dcfclk_from_fw_based_mclk_switching = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] !=
dm_dram_clock_change_unsupported;
}
pipes[0].clks_cfg.dcfclk_mhz = dcfclk_from_validation;
pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
+ if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) {
+ pipes[0].clks_cfg.dcfclk_mhz = dcfclk_from_fw_based_mclk_switching;
+ }
+
if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) {
min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed;
min_dram_speed_mts_margin = 160;
return 0;
}
-/**
+/*
* dcn32_update_bw_bounding_box
*
* This would override some dcn3_2 ip_or_soc initial parameters hardcoded from
mode_lib->vba.FCLKChangeLatency, v->UrgentLatency,
mode_lib->vba.SREnterPlusExitTime);
+ memset(&v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe, 0, sizeof(DmlPipe));
+
v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.Dppclk = mode_lib->vba.DPPCLK[k];
v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.Dispclk = mode_lib->vba.DISPCLK;
v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.PixelClock = mode_lib->vba.PixelClock[k];
for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) {
if (!(mode_lib->vba.DSCInputBitPerComponent[k] == 12.0
|| mode_lib->vba.DSCInputBitPerComponent[k] == 10.0
- || mode_lib->vba.DSCInputBitPerComponent[k] == 8.0
- || mode_lib->vba.DSCInputBitPerComponent[k] >
- mode_lib->vba.MaximumDSCBitsPerComponent)) {
+ || mode_lib->vba.DSCInputBitPerComponent[k] == 8.0)
+ || mode_lib->vba.DSCInputBitPerComponent[k] > mode_lib->vba.MaximumDSCBitsPerComponent) {
mode_lib->vba.NonsupportedDSCInputBPC = true;
}
}
if (mode_lib->vba.OutputMultistreamId[k] == k && mode_lib->vba.ForcedOutputLinkBPP[k] == 0)
mode_lib->vba.BPPForMultistreamNotIndicated = true;
for (j = 0; j < mode_lib->vba.NumberOfActiveSurfaces; ++j) {
- if (mode_lib->vba.OutputMultistreamId[k] == j && mode_lib->vba.OutputMultistreamEn[k]
+ if (mode_lib->vba.OutputMultistreamId[k] == j
&& mode_lib->vba.ForcedOutputLinkBPP[k] == 0)
mode_lib->vba.BPPForMultistreamNotIndicated = true;
}
}
if ((mode_lib->vba.Output[k] == dm_edp || mode_lib->vba.Output[k] == dm_hdmi)) {
- if (mode_lib->vba.OutputMultistreamId[k] == k && mode_lib->vba.OutputMultistreamEn[k])
+ if (mode_lib->vba.OutputMultistreamEn[k] == true && mode_lib->vba.OutputMultistreamId[k] == k)
mode_lib->vba.MultistreamWithHDMIOreDP = true;
-
for (j = 0; j < mode_lib->vba.NumberOfActiveSurfaces; ++j) {
if (mode_lib->vba.OutputMultistreamEn[k] == true && mode_lib->vba.OutputMultistreamId[k] == j)
mode_lib->vba.MultistreamWithHDMIOreDP = true;
mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k]);
}
- m = 0;
-
for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) {
for (m = 0; m <= mode_lib->vba.NumberOfActiveSurfaces - 1; m++) {
for (j = 0; j <= mode_lib->vba.NumberOfActiveSurfaces - 1; j++) {
}
}
- m = 0;
-
//Calculate Return BW
for (i = 0; i < (int) v->soc.num_states; ++i) {
for (j = 0; j <= 1; ++j) {
mode_lib->vba.ModeIsSupported = mode_lib->vba.ModeSupport[i][0] == true
|| mode_lib->vba.ModeSupport[i][1] == true;
- if (mode_lib->vba.ModeSupport[i][0] == true) {
+ if (mode_lib->vba.ModeSupport[i][0] == true)
MaximumMPCCombine = 0;
- } else {
+ else
MaximumMPCCombine = 1;
- }
}
}
break;
case DML_PROJECT_DCN31:
case DML_PROJECT_DCN31_FPGA:
+ case DML_PROJECT_DCN315:
lib->funcs = dml31_funcs;
break;
case DML_PROJECT_DCN314:
DML_PROJECT_DCN21,
DML_PROJECT_DCN30,
DML_PROJECT_DCN31,
+ DML_PROJECT_DCN315,
DML_PROJECT_DCN31_FPGA,
DML_PROJECT_DCN314,
DML_PROJECT_DCN32,
#include "panel_cntl.h"
#define MAX_CLOCK_SOURCES 7
+#define MAX_SVP_PHANTOM_STREAMS 2
+#define MAX_SVP_PHANTOM_PLANES 2
void enable_surface_flip_reporting(struct dc_plane_state *plane_state,
uint32_t controller_id);
unsigned int index);
bool (*remove_phantom_pipes)(struct dc *dc, struct dc_state *context);
+ void (*get_panel_config_defaults)(struct dc_panel_config *panel_config);
};
struct audio_support{
union pipe_update_flags update_flags;
struct dwbc *dwbc;
struct mcif_wb *mcif_wb;
- bool vtp_locked;
};
/* Data used for dynamic link encoder assignment.
struct dcn_watermark_set watermarks;
struct dcn_bw_writeback bw_writeback;
int compbuf_size_kb;
+ unsigned int legacy_svp_drr_stream_index;
+ bool legacy_svp_drr_stream_index_valid;
};
union bw_output {
const struct dc *dc,
struct dc_clocks *clocks);
-void dcn_bw_update_from_pplib(struct dc *dc);
-void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc);
+void dcn_get_soc_clks(
+ struct dc *dc,
+ int *min_fclk_khz,
+ int *min_dcfclk_khz,
+ int *socclk_khz);
+
+void dcn_bw_update_from_pplib_fclks(
+ struct dc *dc,
+ struct dm_pp_clock_levels_with_voltage *fclks);
+void dcn_bw_update_from_pplib_dcfclks(
+ struct dc *dc,
+ struct dm_pp_clock_levels_with_voltage *dcfclks);
+void dcn_bw_notify_pplib_of_wm_ranges(
+ struct dc *dc,
+ int min_fclk_khz,
+ int min_dcfclk_khz,
+ int socclk_khz);
void dcn_bw_sync_calcs_and_dml(struct dc *dc);
enum source_macro_tile_size swizzle_mode_to_macro_tile_size(enum swizzle_mode_values sw_mode);
unsigned int wck_ratio;
};
+struct clk_limit_num_entries {
+ unsigned int num_dcfclk_levels;
+ unsigned int num_fclk_levels;
+ unsigned int num_memclk_levels;
+ unsigned int num_socclk_levels;
+ unsigned int num_dtbclk_levels;
+ unsigned int num_dispclk_levels;
+ unsigned int num_dppclk_levels;
+ unsigned int num_phyclk_levels;
+ unsigned int num_phyclk_d18_levels;
+};
+
/* This table is contiguous */
struct clk_limit_table {
struct clk_limit_table_entry entries[MAX_NUM_DPM_LVL];
- unsigned int num_entries;
+ struct clk_limit_num_entries num_entries_per_clk;
+ unsigned int num_entries; /* highest populated dpm level for back compatibility */
};
struct wm_range_table_entry {
--- /dev/null
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2022 Advanced Micro Devices, Inc. All rights reserved. */
+
+#ifndef __DAL_CURSOR_CACHE_H__
+#define __DAL_CURSOR_CACHE_H__
+
+union reg_cursor_control_cfg {
+ struct {
+ uint32_t cur_enable: 1;
+ uint32_t reser0: 3;
+ uint32_t cur_2x_magnify: 1;
+ uint32_t reser1: 3;
+ uint32_t mode: 3;
+ uint32_t reser2: 5;
+ uint32_t pitch: 2;
+ uint32_t reser3: 6;
+ uint32_t line_per_chunk: 5;
+ uint32_t reser4: 3;
+ } bits;
+ uint32_t raw;
+};
+struct cursor_position_cache_hubp {
+ union reg_cursor_control_cfg cur_ctl;
+ union reg_position_cfg {
+ struct {
+ uint32_t x_pos: 16;
+ uint32_t y_pos: 16;
+ } bits;
+ uint32_t raw;
+ } position;
+ union reg_hot_spot_cfg {
+ struct {
+ uint32_t x_hot: 16;
+ uint32_t y_hot: 16;
+ } bits;
+ uint32_t raw;
+ } hot_spot;
+ union reg_dst_offset_cfg {
+ struct {
+ uint32_t dst_x_offset: 13;
+ uint32_t reserved: 19;
+ } bits;
+ uint32_t raw;
+ } dst_offset;
+};
+
+struct cursor_attribute_cache_hubp {
+ uint32_t SURFACE_ADDR_HIGH;
+ uint32_t SURFACE_ADDR;
+ union reg_cursor_control_cfg cur_ctl;
+ union reg_cursor_size_cfg {
+ struct {
+ uint32_t width: 16;
+ uint32_t height: 16;
+ } bits;
+ uint32_t raw;
+ } size;
+ union reg_cursor_settings_cfg {
+ struct {
+ uint32_t dst_y_offset: 8;
+ uint32_t chunk_hdl_adjust: 2;
+ uint32_t reserved: 22;
+ } bits;
+ uint32_t raw;
+ } settings;
+};
+
+struct cursor_rect {
+ uint32_t x;
+ uint32_t y;
+ uint32_t w;
+ uint32_t h;
+};
+
+union reg_cur0_control_cfg {
+ struct {
+ uint32_t cur0_enable: 1;
+ uint32_t expansion_mode: 1;
+ uint32_t reser0: 1;
+ uint32_t cur0_rom_en: 1;
+ uint32_t mode: 3;
+ uint32_t reserved: 25;
+ } bits;
+ uint32_t raw;
+};
+struct cursor_position_cache_dpp {
+ union reg_cur0_control_cfg cur0_ctl;
+};
+
+struct cursor_attribute_cache_dpp {
+ union reg_cur0_control_cfg cur0_ctl;
+};
+
+struct cursor_attributes_cfg {
+ struct cursor_attribute_cache_hubp aHubp;
+ struct cursor_attribute_cache_dpp aDpp;
+};
+
+#endif
#define __DAL_DPP_H__
#include "transform.h"
+#include "cursor_reg_cache.h"
union defer_reg_writes {
struct {
struct pwl_params shaper_params;
bool cm_bypass_mode;
+
+ struct cursor_position_cache_dpp pos;
+ struct cursor_attribute_cache_dpp att;
};
struct dpp_input_csc_matrix {
#define __DAL_HUBP_H__
#include "mem_input.h"
+#include "cursor_reg_cache.h"
#define OPP_ID_INVALID 0xf
#define MAX_TTU 0xffffff
struct dc_cursor_attributes curs_attr;
struct dc_cursor_position curs_pos;
bool power_gated;
+
+ struct cursor_position_cache_hubp pos;
+ struct cursor_attribute_cache_hubp att;
+ struct cursor_rect cur_rect;
};
struct surface_flip_registers {
void (*set_blank)(struct timing_generator *tg,
bool enable_blanking);
bool (*is_blanked)(struct timing_generator *tg);
- bool (*is_locked)(struct timing_generator *tg);
void (*set_overscan_blank_color) (struct timing_generator *tg, const struct tg_color *color);
void (*set_blank_color)(struct timing_generator *tg, const struct tg_color *color);
void (*set_colors)(struct timing_generator *tg,
bool is_h_timing_divisible_by_2(struct dc_stream_state *stream);
+bool dc_resource_acquire_secondary_pipe_for_mpc_odm(
+ const struct dc *dc,
+ struct dc_state *state,
+ struct pipe_ctx *pri_pipe,
+ struct pipe_ctx *sec_pipe,
+ bool odm);
#endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_RESOURCE_H_ */
enum phyd32clk_clock_source phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link);
dto_params.otg_inst = tg->inst;
- dto_params.pixclk_khz = pipe_ctx->stream->phy_pix_clk;
+ dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
dto_params.num_odm_segments = get_odm_segment_count(pipe_ctx);
dto_params.timing = &pipe_ctx->stream->timing;
dto_params.ref_dtbclk_khz = dc->clk_mgr->funcs->get_dtb_ref_clk_frequency(dc->clk_mgr);
{
}
-void virtual_disable_link_output(struct dc_link *link,
+static void virtual_disable_link_output(struct dc_link *link,
const struct link_resource *link_res,
enum signal_type signal)
{
bool disable_dpia;
bool usb4_cm_version;
bool fw_in_system_memory;
+ bool dpia_hpd_int_enable_supported;
};
/**
uint32_t diag_env: 1; /* 1 if diagnostic environment */
uint32_t gpint_scratch8: 1; /* 1 if GPINT is in scratch8*/
uint32_t usb4_cm_version: 1; /**< 1 CM support */
+ uint32_t dpia_hpd_int_enable_supported: 1; /* 1 if dpia hpd int enable supported */
- uint32_t reserved : 17; /**< reserved */
+ uint32_t reserved : 16; /**< reserved */
} bits; /**< boot bits */
uint32_t all; /**< 32-bit access to bits */
};
/**
* Command type used for all VBIOS interface commands.
*/
+
+ /**
+ * Command type used to set DPIA HPD interrupt state
+ */
+ DMUB_CMD__DPIA_HPD_INT_ENABLE = 86,
+
DMUB_CMD__VBIOS = 128,
};
DMUB_CMD__DPIA_MST_ALLOC_SLOTS = 2,
};
-enum dmub_cmd_header_sub_type {
- DMUB_CMD__SUB_TYPE_GENERAL = 0,
- DMUB_CMD__SUB_TYPE_CURSOR_POSITION = 1
-};
-
#pragma pack(push, 1)
/**
struct dmub_cmd_mst_alloc_slots_control_data mst_slots_control; /* mst slots control */
};
+/**
+ * DMUB command structure for DPIA HPD int enable control.
+ */
+struct dmub_rb_cmd_dpia_hpd_int_enable {
+ struct dmub_cmd_header header; /* header */
+ uint32_t enable; /* dpia hpd interrupt enable */
+};
+
/**
* struct dmub_rb_cmd_dpphy_init - DPPHY init.
*/
/**
* Data passed from driver to FW in a DMUB_CMD__UPDATE_CURSOR_INFO command.
*/
-struct dmub_cmd_update_cursor_info_data {
+union dmub_reg_cursor_control_cfg {
+ struct {
+ uint32_t cur_enable: 1;
+ uint32_t reser0: 3;
+ uint32_t cur_2x_magnify: 1;
+ uint32_t reser1: 3;
+ uint32_t mode: 3;
+ uint32_t reser2: 5;
+ uint32_t pitch: 2;
+ uint32_t reser3: 6;
+ uint32_t line_per_chunk: 5;
+ uint32_t reser4: 3;
+ } bits;
+ uint32_t raw;
+};
+struct dmub_cursor_position_cache_hubp {
+ union dmub_reg_cursor_control_cfg cur_ctl;
+ union dmub_reg_position_cfg {
+ struct {
+ uint32_t cur_x_pos: 16;
+ uint32_t cur_y_pos: 16;
+ } bits;
+ uint32_t raw;
+ } position;
+ union dmub_reg_hot_spot_cfg {
+ struct {
+ uint32_t hot_x: 16;
+ uint32_t hot_y: 16;
+ } bits;
+ uint32_t raw;
+ } hot_spot;
+ union dmub_reg_dst_offset_cfg {
+ struct {
+ uint32_t dst_x_offset: 13;
+ uint32_t reserved: 19;
+ } bits;
+ uint32_t raw;
+ } dst_offset;
+};
+
+union dmub_reg_cur0_control_cfg {
+ struct {
+ uint32_t cur0_enable: 1;
+ uint32_t expansion_mode: 1;
+ uint32_t reser0: 1;
+ uint32_t cur0_rom_en: 1;
+ uint32_t mode: 3;
+ uint32_t reserved: 25;
+ } bits;
+ uint32_t raw;
+};
+struct dmub_cursor_position_cache_dpp {
+ union dmub_reg_cur0_control_cfg cur0_ctl;
+};
+struct dmub_cursor_position_cfg {
+ struct dmub_cursor_position_cache_hubp pHubp;
+ struct dmub_cursor_position_cache_dpp pDpp;
+ uint8_t pipe_idx;
+ /*
+ * Padding is required. To be 4 Bytes Aligned.
+ */
+ uint8_t padding[3];
+};
+
+struct dmub_cursor_attribute_cache_hubp {
+ uint32_t SURFACE_ADDR_HIGH;
+ uint32_t SURFACE_ADDR;
+ union dmub_reg_cursor_control_cfg cur_ctl;
+ union dmub_reg_cursor_size_cfg {
+ struct {
+ uint32_t width: 16;
+ uint32_t height: 16;
+ } bits;
+ uint32_t raw;
+ } size;
+ union dmub_reg_cursor_settings_cfg {
+ struct {
+ uint32_t dst_y_offset: 8;
+ uint32_t chunk_hdl_adjust: 2;
+ uint32_t reserved: 22;
+ } bits;
+ uint32_t raw;
+ } settings;
+};
+struct dmub_cursor_attribute_cache_dpp {
+ union dmub_reg_cur0_control_cfg cur0_ctl;
+};
+struct dmub_cursor_attributes_cfg {
+ struct dmub_cursor_attribute_cache_hubp aHubp;
+ struct dmub_cursor_attribute_cache_dpp aDpp;
+};
+
+struct dmub_cmd_update_cursor_payload0 {
/**
* Cursor dirty rects.
*/
* Currently the support is only for 0 or 1
*/
uint8_t panel_inst;
+ /**
+ * Cursor Position Register.
+ * Registers contains Hubp & Dpp modules
+ */
+ struct dmub_cursor_position_cfg position_cfg;
+};
+
+struct dmub_cmd_update_cursor_payload1 {
+ struct dmub_cursor_attributes_cfg attribute_cfg;
+};
+
+union dmub_cmd_update_cursor_info_data {
+ struct dmub_cmd_update_cursor_payload0 payload0;
+ struct dmub_cmd_update_cursor_payload1 payload1;
};
/**
* Definition of a DMUB_CMD__UPDATE_CURSOR_INFO command.
/**
* Data passed from driver to FW in a DMUB_CMD__UPDATE_CURSOR_INFO command.
*/
- struct dmub_cmd_update_cursor_info_data update_cursor_info_data;
+ union dmub_cmd_update_cursor_info_data update_cursor_info_data;
};
/**
struct dmub_optc_state {
uint32_t v_total_max;
uint32_t v_total_min;
- uint32_t v_total_mid;
- uint32_t v_total_mid_frame_num;
uint32_t tg_inst;
- uint32_t enable_manual_trigger;
- uint32_t clear_force_vsync;
};
struct dmub_rb_cmd_drr_update {
* Definition of a DMUB_CMD__QUERY_HPD_STATE command.
*/
struct dmub_rb_cmd_query_hpd_state query_hpd;
+ /**
+ * Definition of a DMUB_CMD__DPIA_HPD_INT_ENABLE command.
+ */
+ struct dmub_rb_cmd_dpia_hpd_int_enable dpia_hpd_int_enable;
};
/**
boot_options.bits.dpia_supported = params->dpia_supported;
boot_options.bits.enable_dpia = params->disable_dpia ? 0 : 1;
boot_options.bits.usb4_cm_version = params->usb4_cm_version;
+ boot_options.bits.dpia_hpd_int_enable_supported = params->dpia_hpd_int_enable_supported;
boot_options.bits.power_optimization = params->power_optimization;
boot_options.bits.sel_mux_phy_c_d_phy_f_g = (dmub->asic == DMUB_ASIC_DCN31B) ? 1 : 0;
struct pwl_float_data_ex *rgb = rgb_regamma;
const struct hw_x_point *coord_x = coordinates_x;
- build_coefficients(&coeff, true);
+ build_coefficients(&coeff, TRANSFER_FUNCTION_SRGB);
i = 0;
while (i != hw_points_num + 1) {
#define regMCA_UMC_UMC0_MCUMC_STATUST0_BASE_IDX 2
#define regMCA_UMC_UMC0_MCUMC_ADDRT0 0x03c4
#define regMCA_UMC_UMC0_MCUMC_ADDRT0_BASE_IDX 2
+#define regUMCCH0_0_GeccCtrl 0x0053
+#define regUMCCH0_0_GeccCtrl_BASE_IDX 2
#endif
#define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr__SHIFT 0x0
#define MCA_UMC_UMC0_MCUMC_ADDRT0__Reserved__SHIFT 0x38
#define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr_MASK 0x00FFFFFFFFFFFFFFL
+//UMCCH0_0_GeccCtrl
+#define UMCCH0_0_GeccCtrl__UCFatalEn__SHIFT 0xd
+#define UMCCH0_0_GeccCtrl__UCFatalEn_MASK 0x00002000L
#endif
static void kv_dpm_disable(struct amdgpu_device *adev)
{
struct kv_power_info *pi = kv_get_pi(adev);
+ int err;
amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
AMDGPU_THERMAL_IRQ_LOW_TO_HIGH);
amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
AMDGPU_THERMAL_IRQ_HIGH_TO_LOW);
- amdgpu_kv_smc_bapm_enable(adev, false);
+ err = amdgpu_kv_smc_bapm_enable(adev, false);
+ if (err)
+ DRM_ERROR("amdgpu_kv_smc_bapm_enable failed\n");
if (adev->asic_type == CHIP_MULLINS)
kv_enable_nb_dpm(adev, false);
return -EINVAL);
PP_ASSERT_WITH_CODE(
- (smu7_power_state->performance_level_count <=
+ (smu7_power_state->performance_level_count <
hwmgr->platform_descriptor.hardwareActivityPerformanceLevels),
"Performance levels exceeds Driver limit!",
return -EINVAL);
return -1);
PP_ASSERT_WITH_CODE(
- (vega10_ps->performance_level_count <=
+ (vega10_ps->performance_level_count <
hwmgr->platform_descriptor.
hardwareActivityPerformanceLevels),
"Performance levels exceeds Driver limit!",
if (ret)
drm_warn(dev, "Failed to update vram location.\n");
- hv->dirt_needed = true;
-
ret = hyperv_mode_config_init(hv);
if (ret)
goto err_free_mmio;
VM_PKT_DATA_INBAND, 0);
if (ret)
- drm_err(&hv->dev, "Unable to send packet via vmbus\n");
+ drm_err_ratelimited(&hv->dev, "Unable to send packet via vmbus; error %d\n", ret);
return ret;
}
pipe_config->hw.adjusted_mode.flags |= flags;
if ((tmp & SDVO_COLOR_FORMAT_MASK) == HDMI_COLOR_FORMAT_12bpc)
- dotclock = pipe_config->port_clock * 2 / 3;
+ dotclock = DIV_ROUND_CLOSEST(pipe_config->port_clock * 2, 3);
else
dotclock = pipe_config->port_clock;
drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
}
+static int max_dotclock(struct drm_i915_private *i915)
+{
+ int max_dotclock = i915->max_dotclk_freq;
+
+ /* icl+ might use bigjoiner */
+ if (DISPLAY_VER(i915) >= 11)
+ max_dotclock *= 2;
+
+ return max_dotclock;
+}
+
static enum drm_mode_status
intel_mode_valid(struct drm_device *dev,
const struct drm_display_mode *mode)
DRM_MODE_FLAG_CLKDIV2))
return MODE_BAD;
+ /*
+ * Reject clearly excessive dotclocks early to
+ * avoid having to worry about huge integers later.
+ */
+ if (mode->clock > max_dotclock(dev_priv))
+ return MODE_CLOCK_HIGH;
+
/* Transcoder timing limits */
if (DISPLAY_VER(dev_priv) >= 11) {
hdisplay_max = 16384;
struct drm_device *dev = fb->dev;
struct drm_i915_private *dev_priv = to_i915(dev);
struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+ struct i915_gem_ww_ctx ww;
struct i915_vma *vma;
u32 alignment;
int ret;
+ /*
+ * We are not syncing against the binding (and potential migrations)
+ * below, so this vm must never be async.
+ */
+ GEM_WARN_ON(vm->bind_async_flags);
+
if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
return ERR_PTR(-EINVAL);
atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
- ret = i915_gem_object_lock_interruptible(obj, NULL);
- if (!ret) {
+ for_i915_gem_ww(&ww, ret, true) {
+ ret = i915_gem_object_lock(obj, &ww);
+ if (ret)
+ continue;
+
+ if (HAS_LMEM(dev_priv)) {
+ unsigned int flags = obj->flags;
+
+ /*
+ * For this type of buffer we need to able to read from the CPU
+ * the clear color value found in the buffer, hence we need to
+ * ensure it is always in the mappable part of lmem, if this is
+ * a small-bar device.
+ */
+ if (intel_fb_rc_ccs_cc_plane(fb) >= 0)
+ flags &= ~I915_BO_ALLOC_GPU_ONLY;
+ ret = __i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0,
+ flags);
+ if (ret)
+ continue;
+ }
+
ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
- i915_gem_object_unlock(obj);
- }
- if (ret) {
- vma = ERR_PTR(ret);
- goto err;
- }
+ if (ret)
+ continue;
- vma = i915_vma_instance(obj, vm, view);
- if (IS_ERR(vma))
- goto err;
+ vma = i915_vma_instance(obj, vm, view);
+ if (IS_ERR(vma)) {
+ ret = PTR_ERR(vma);
+ continue;
+ }
- if (i915_vma_misplaced(vma, 0, alignment, 0)) {
- ret = i915_vma_unbind_unlocked(vma);
- if (ret) {
- vma = ERR_PTR(ret);
- goto err;
+ if (i915_vma_misplaced(vma, 0, alignment, 0)) {
+ ret = i915_vma_unbind(vma);
+ if (ret)
+ continue;
}
- }
- ret = i915_vma_pin(vma, 0, alignment, PIN_GLOBAL);
+ ret = i915_vma_pin_ww(vma, &ww, 0, alignment, PIN_GLOBAL);
+ if (ret)
+ continue;
+ }
if (ret) {
vma = ERR_PTR(ret);
goto err;
}
}
+static u32 psr_irq_psr_error_bit_get(struct intel_dp *intel_dp)
+{
+ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+
+ return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_ERROR :
+ EDP_PSR_ERROR(intel_dp->psr.transcoder);
+}
+
+static u32 psr_irq_post_exit_bit_get(struct intel_dp *intel_dp)
+{
+ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+
+ return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_POST_EXIT :
+ EDP_PSR_POST_EXIT(intel_dp->psr.transcoder);
+}
+
+static u32 psr_irq_pre_entry_bit_get(struct intel_dp *intel_dp)
+{
+ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+
+ return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_PRE_ENTRY :
+ EDP_PSR_PRE_ENTRY(intel_dp->psr.transcoder);
+}
+
+static u32 psr_irq_mask_get(struct intel_dp *intel_dp)
+{
+ struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+
+ return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_MASK :
+ EDP_PSR_MASK(intel_dp->psr.transcoder);
+}
+
static void psr_irq_control(struct intel_dp *intel_dp)
{
struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
- enum transcoder trans_shift;
i915_reg_t imr_reg;
u32 mask, val;
- /*
- * gen12+ has registers relative to transcoder and one per transcoder
- * using the same bit definition: handle it as TRANSCODER_EDP to force
- * 0 shift in bit definition
- */
- if (DISPLAY_VER(dev_priv) >= 12) {
- trans_shift = 0;
+ if (DISPLAY_VER(dev_priv) >= 12)
imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);
- } else {
- trans_shift = intel_dp->psr.transcoder;
+ else
imr_reg = EDP_PSR_IMR;
- }
- mask = EDP_PSR_ERROR(trans_shift);
+ mask = psr_irq_psr_error_bit_get(intel_dp);
if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ)
- mask |= EDP_PSR_POST_EXIT(trans_shift) |
- EDP_PSR_PRE_ENTRY(trans_shift);
+ mask |= psr_irq_post_exit_bit_get(intel_dp) |
+ psr_irq_pre_entry_bit_get(intel_dp);
- /* Warning: it is masking/setting reserved bits too */
val = intel_de_read(dev_priv, imr_reg);
- val &= ~EDP_PSR_TRANS_MASK(trans_shift);
+ val &= ~psr_irq_mask_get(intel_dp);
val |= ~mask;
intel_de_write(dev_priv, imr_reg, val);
}
enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
ktime_t time_ns = ktime_get();
- enum transcoder trans_shift;
i915_reg_t imr_reg;
- if (DISPLAY_VER(dev_priv) >= 12) {
- trans_shift = 0;
+ if (DISPLAY_VER(dev_priv) >= 12)
imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);
- } else {
- trans_shift = intel_dp->psr.transcoder;
+ else
imr_reg = EDP_PSR_IMR;
- }
- if (psr_iir & EDP_PSR_PRE_ENTRY(trans_shift)) {
+ if (psr_iir & psr_irq_pre_entry_bit_get(intel_dp)) {
intel_dp->psr.last_entry_attempt = time_ns;
drm_dbg_kms(&dev_priv->drm,
"[transcoder %s] PSR entry attempt in 2 vblanks\n",
transcoder_name(cpu_transcoder));
}
- if (psr_iir & EDP_PSR_POST_EXIT(trans_shift)) {
+ if (psr_iir & psr_irq_post_exit_bit_get(intel_dp)) {
intel_dp->psr.last_exit = time_ns;
drm_dbg_kms(&dev_priv->drm,
"[transcoder %s] PSR exit completed\n",
}
}
- if (psr_iir & EDP_PSR_ERROR(trans_shift)) {
+ if (psr_iir & psr_irq_psr_error_bit_get(intel_dp)) {
u32 val;
drm_warn(&dev_priv->drm, "[transcoder %s] PSR aux error\n",
* or unset irq_aux_error.
*/
val = intel_de_read(dev_priv, imr_reg);
- val |= EDP_PSR_ERROR(trans_shift);
+ val |= psr_irq_psr_error_bit_get(intel_dp);
intel_de_write(dev_priv, imr_reg, val);
schedule_work(&intel_dp->psr.work);
* first time that PSR HW tries to activate so lets keep PSR disabled
* to avoid any rendering problems.
*/
- if (DISPLAY_VER(dev_priv) >= 12) {
+ if (DISPLAY_VER(dev_priv) >= 12)
val = intel_de_read(dev_priv,
TRANS_PSR_IIR(intel_dp->psr.transcoder));
- val &= EDP_PSR_ERROR(0);
- } else {
+ else
val = intel_de_read(dev_priv, EDP_PSR_IIR);
- val &= EDP_PSR_ERROR(intel_dp->psr.transcoder);
- }
+ val &= psr_irq_psr_error_bit_get(intel_dp);
if (val) {
intel_dp->psr.sink_not_reliable = true;
drm_dbg_kms(&dev_priv->drm,
modifier == I915_FORMAT_MOD_4_TILED ||
modifier == I915_FORMAT_MOD_Yf_TILED ||
modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
- modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+ modifier == I915_FORMAT_MOD_Yf_TILED_CCS ||
+ modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
+ modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS ||
+ modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC ||
+ modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS ||
+ modifier == I915_FORMAT_MOD_4_TILED_DG2_MC_CCS ||
+ modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC;
wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
- modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+ modifier == I915_FORMAT_MOD_Yf_TILED_CCS ||
+ modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
+ modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS ||
+ modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC ||
+ modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS ||
+ modifier == I915_FORMAT_MOD_4_TILED_DG2_MC_CCS ||
+ modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC;
wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
wp->width = width;
*/
for_each_gem_engine(ce, engines, it) {
struct intel_engine_cs *engine;
- bool skip = false;
- if (exit)
- skip = intel_context_set_exiting(ce);
- else if (!persistent)
- skip = intel_context_exit_nonpersistent(ce, NULL);
-
- if (skip)
+ if ((exit || !persistent) && intel_context_revoke(ce))
continue; /* Already marked. */
/*
/* Check whether the file_priv has already selected one ring. */
if ((int)file_priv->bsd_engine < 0)
file_priv->bsd_engine =
- get_random_int() % num_vcs_engines(dev_priv);
+ prandom_u32_max(num_vcs_engines(dev_priv));
return file_priv->bsd_engine;
}
int i915_gem_object_migrate(struct drm_i915_gem_object *obj,
struct i915_gem_ww_ctx *ww,
enum intel_region_id id)
+{
+ return __i915_gem_object_migrate(obj, ww, id, obj->flags);
+}
+
+/**
+ * __i915_gem_object_migrate - Migrate an object to the desired region id, with
+ * control of the extra flags
+ * @obj: The object to migrate.
+ * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may
+ * not be successful in evicting other objects to make room for this object.
+ * @id: The region id to migrate to.
+ * @flags: The object flags. Normally just obj->flags.
+ *
+ * Attempt to migrate the object to the desired memory region. The
+ * object backend must support migration and the object may not be
+ * pinned, (explicitly pinned pages or pinned vmas). The object must
+ * be locked.
+ * On successful completion, the object will have pages pointing to
+ * memory in the new region, but an async migration task may not have
+ * completed yet, and to accomplish that, i915_gem_object_wait_migration()
+ * must be called.
+ *
+ * Note: the @ww parameter is not used yet, but included to make sure
+ * callers put some effort into obtaining a valid ww ctx if one is
+ * available.
+ *
+ * Return: 0 on success. Negative error code on failure. In particular may
+ * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance
+ * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and
+ * -EBUSY if the object is pinned.
+ */
+int __i915_gem_object_migrate(struct drm_i915_gem_object *obj,
+ struct i915_gem_ww_ctx *ww,
+ enum intel_region_id id,
+ unsigned int flags)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
struct intel_memory_region *mr;
return 0;
}
- return obj->ops->migrate(obj, mr);
+ return obj->ops->migrate(obj, mr, flags);
}
/**
int i915_gem_object_migrate(struct drm_i915_gem_object *obj,
struct i915_gem_ww_ctx *ww,
enum intel_region_id id);
+int __i915_gem_object_migrate(struct drm_i915_gem_object *obj,
+ struct i915_gem_ww_ctx *ww,
+ enum intel_region_id id,
+ unsigned int flags);
bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj,
enum intel_region_id id);
* pinning or for as long as the object lock is held.
*/
int (*migrate)(struct drm_i915_gem_object *obj,
- struct intel_memory_region *mr);
+ struct intel_memory_region *mr,
+ unsigned int flags);
void (*release)(struct drm_i915_gem_object *obj);
}
static int i915_ttm_migrate(struct drm_i915_gem_object *obj,
- struct intel_memory_region *mr)
+ struct intel_memory_region *mr,
+ unsigned int flags)
{
- return __i915_ttm_migrate(obj, mr, obj->flags);
+ return __i915_ttm_migrate(obj, mr, flags);
}
static void i915_ttm_put_pages(struct drm_i915_gem_object *obj,
static int
probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len)
{
- const unsigned long end = addr + len;
+ VMA_ITERATOR(vmi, mm, addr);
struct vm_area_struct *vma;
- int ret = -EFAULT;
mmap_read_lock(mm);
- for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
+ for_each_vma_range(vmi, vma, addr + len) {
/* Check for holes, note that we also update the addr below */
if (vma->vm_start > addr)
break;
if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
break;
- if (vma->vm_end >= end) {
- ret = 0;
- break;
- }
-
addr = vma->vm_end;
}
mmap_read_unlock(mm);
- return ret;
+ if (vma)
+ return -EFAULT;
+ return 0;
}
/*
return ret;
}
-bool intel_context_exit_nonpersistent(struct intel_context *ce,
- struct i915_request *rq)
+bool intel_context_revoke(struct intel_context *ce)
{
bool ret = intel_context_set_exiting(ce);
if (ce->ops->revoke)
- ce->ops->revoke(ce, rq, ce->engine->props.preempt_timeout_ms);
+ ce->ops->revoke(ce, NULL, ce->engine->props.preempt_timeout_ms);
return ret;
}
return test_and_set_bit(CONTEXT_EXITING, &ce->flags);
}
-bool intel_context_exit_nonpersistent(struct intel_context *ce,
- struct i915_request *rq);
+bool intel_context_revoke(struct intel_context *ce);
static inline bool
intel_context_force_single_submission(const struct intel_context *ce)
atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
GEM_BUG_ON(!was_bound);
- if (!retained_ptes)
+ if (!retained_ptes) {
+ /*
+ * Clear the bound flags of the vma resource to allow
+ * ptes to be repopulated.
+ */
+ vma->resource->bound_flags = 0;
vma->ops->bind_vma(vm, NULL, vma->resource,
obj ? obj->cache_level : 0,
was_bound);
+ }
if (obj) { /* only used during resume => exclusive access */
write_domain_objs |= fetch_and_zero(&obj->write_domain);
obj->read_domains |= I915_GEM_DOMAIN_GTT;
MOCS_ENTRY(15, \
LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
L3_3_WB), \
+ /* Bypass LLC - Uncached (EHL+) */ \
+ MOCS_ENTRY(16, \
+ LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \
+ L3_1_UC), \
+ /* Bypass LLC - L3 (Read-Only) (EHL+) */ \
+ MOCS_ENTRY(17, \
+ LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \
+ L3_3_WB), \
/* Self-Snoop - L3 + LLC */ \
MOCS_ENTRY(18, \
LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \
* Corner case where requests were sitting in the priority list or a
* request resubmitted after the context was banned.
*/
- if (unlikely(intel_context_is_banned(ce))) {
+ if (unlikely(!intel_context_is_schedulable(ce))) {
i915_request_put(i915_request_mark_eio(rq));
intel_engine_signal_breadcrumbs(ce->engine);
return 0;
struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
- int ret = 0;
+ int ret;
- if (likely(!intel_context_is_banned(ce))) {
- ret = __guc_wq_item_append(rq);
+ if (unlikely(!intel_context_is_schedulable(ce)))
+ return 0;
- if (unlikely(ret == -EBUSY)) {
- guc->stalled_request = rq;
- guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
- }
+ ret = __guc_wq_item_append(rq);
+ if (unlikely(ret == -EBUSY)) {
+ guc->stalled_request = rq;
+ guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
}
return ret;
* submitting all the requests generated in parallel.
*/
return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) ||
- intel_context_is_banned(ce);
+ !intel_context_is_schedulable(ce);
}
static int guc_dequeue_one_context(struct intel_guc *guc)
struct intel_context *ce = request_to_scheduling_context(last);
if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&
- !intel_context_is_banned(ce))) {
+ intel_context_is_schedulable(ce))) {
ret = try_context_registration(ce, false);
if (unlikely(ret == -EPIPE)) {
goto deadlk;
{
struct intel_engine_cs *engine = __context_to_physical_engine(ce);
- if (intel_context_is_banned(ce))
+ if (!intel_context_is_schedulable(ce))
return;
GEM_BUG_ON(!intel_context_is_pinned(ce));
{
trace_intel_context_reset(ce);
- if (likely(!intel_context_is_banned(ce))) {
+ if (likely(intel_context_is_schedulable(ce))) {
capture_error_state(guc, ce);
guc_context_replay(ce);
} else {
drm_info(&guc_to_gt(guc)->i915->drm,
- "Ignoring context reset notification of banned context 0x%04X on %s",
+ "Ignoring context reset notification of exiting context 0x%04X on %s",
ce->guc_id.id, ce->engine->name);
}
}
}
static int alloc_resource(struct intel_vgpu *vgpu,
- struct intel_vgpu_creation_params *param)
+ const struct intel_vgpu_config *conf)
{
struct intel_gvt *gvt = vgpu->gvt;
unsigned long request, avail, max, taken;
const char *item;
- if (!param->low_gm_sz || !param->high_gm_sz || !param->fence_sz) {
+ if (!conf->low_mm || !conf->high_mm || !conf->fence) {
gvt_vgpu_err("Invalid vGPU creation params\n");
return -EINVAL;
}
max = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
taken = gvt->gm.vgpu_allocated_low_gm_size;
avail = max - taken;
- request = MB_TO_BYTES(param->low_gm_sz);
+ request = conf->low_mm;
if (request > avail)
goto no_enough_resource;
max = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
taken = gvt->gm.vgpu_allocated_high_gm_size;
avail = max - taken;
- request = MB_TO_BYTES(param->high_gm_sz);
+ request = conf->high_mm;
if (request > avail)
goto no_enough_resource;
max = gvt_fence_sz(gvt) - HOST_FENCE;
taken = gvt->fence.vgpu_allocated_fence_num;
avail = max - taken;
- request = param->fence_sz;
+ request = conf->fence;
if (request > avail)
goto no_enough_resource;
vgpu_fence_sz(vgpu) = request;
- gvt->gm.vgpu_allocated_low_gm_size += MB_TO_BYTES(param->low_gm_sz);
- gvt->gm.vgpu_allocated_high_gm_size += MB_TO_BYTES(param->high_gm_sz);
- gvt->fence.vgpu_allocated_fence_num += param->fence_sz;
+ gvt->gm.vgpu_allocated_low_gm_size += conf->low_mm;
+ gvt->gm.vgpu_allocated_high_gm_size += conf->high_mm;
+ gvt->fence.vgpu_allocated_fence_num += conf->fence;
return 0;
no_enough_resource:
*
*/
int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
- struct intel_vgpu_creation_params *param)
+ const struct intel_vgpu_config *conf)
{
int ret;
- ret = alloc_resource(vgpu, param);
+ ret = alloc_resource(vgpu, conf);
if (ret)
return ret;
#include <uapi/linux/pci_regs.h>
#include <linux/kvm_host.h>
#include <linux/vfio.h>
+#include <linux/mdev.h>
#include "i915_drv.h"
#include "intel_gvt.h"
#define KVMGT_DEBUGFS_FILENAME "kvmgt_nr_cache_entries"
struct intel_vgpu {
+ struct vfio_device vfio_device;
struct intel_gvt *gvt;
struct mutex vgpu_lock;
int id;
u32 scan_nonprivbb;
- struct vfio_device vfio_device;
struct vfio_region *region;
int num_regions;
struct eventfd_ctx *intx_trigger;
bool firmware_loaded;
};
-#define NR_MAX_INTEL_VGPU_TYPES 20
-struct intel_vgpu_type {
- char name[16];
- unsigned int avail_instance;
- unsigned int low_gm_size;
- unsigned int high_gm_size;
+struct intel_vgpu_config {
+ unsigned int low_mm;
+ unsigned int high_mm;
unsigned int fence;
+
+ /*
+ * A vGPU with a weight of 8 will get twice as much GPU as a vGPU with
+ * a weight of 4 on a contended host, different vGPU type has different
+ * weight set. Legal weights range from 1 to 16.
+ */
unsigned int weight;
- enum intel_vgpu_edid resolution;
+ enum intel_vgpu_edid edid;
+ const char *name;
+};
+
+struct intel_vgpu_type {
+ struct mdev_type type;
+ char name[16];
+ const struct intel_vgpu_config *conf;
};
struct intel_gvt {
struct intel_gvt_workload_scheduler scheduler;
struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES];
DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
+ struct mdev_parent parent;
+ struct mdev_type **mdev_types;
struct intel_vgpu_type *types;
unsigned int num_types;
struct intel_vgpu *idle_vgpu;
/* ring context size i.e. the first 0x50 dwords*/
#define RING_CTX_SIZE 320
-struct intel_vgpu_creation_params {
- __u64 low_gm_sz; /* in MB */
- __u64 high_gm_sz; /* in MB */
- __u64 fence_sz;
- __u64 resolution;
- __s32 primary;
- __u64 vgpu_id;
-
- __u32 weight;
-};
-
int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
- struct intel_vgpu_creation_params *param);
+ const struct intel_vgpu_config *conf);
void intel_vgpu_reset_resource(struct intel_vgpu *vgpu);
void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt);
void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu);
-struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
- struct intel_vgpu_type *type);
+int intel_gvt_create_vgpu(struct intel_vgpu *vgpu,
+ const struct intel_vgpu_config *conf);
void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
void intel_gvt_release_vgpu(struct intel_vgpu *vgpu);
void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
*/
#include <linux/init.h>
-#include <linux/device.h>
#include <linux/mm.h>
#include <linux/kthread.h>
#include <linux/sched/mm.h>
#include <linux/rbtree.h>
#include <linux/spinlock.h>
#include <linux/eventfd.h>
-#include <linux/uuid.h>
#include <linux/mdev.h>
#include <linux/debugfs.h>
struct kvm_memory_slot *slot,
struct kvm_page_track_notifier_node *node);
-static ssize_t available_instances_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr,
- char *buf)
+static ssize_t intel_vgpu_show_description(struct mdev_type *mtype, char *buf)
{
- struct intel_vgpu_type *type;
- unsigned int num = 0;
- struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt;
-
- type = &gvt->types[mtype_get_type_group_id(mtype)];
- if (!type)
- num = 0;
- else
- num = type->avail_instance;
-
- return sprintf(buf, "%u\n", num);
-}
-
-static ssize_t device_api_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
-}
-
-static ssize_t description_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- struct intel_vgpu_type *type;
- struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt;
-
- type = &gvt->types[mtype_get_type_group_id(mtype)];
- if (!type)
- return 0;
+ struct intel_vgpu_type *type =
+ container_of(mtype, struct intel_vgpu_type, type);
return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
"fence: %d\nresolution: %s\n"
"weight: %d\n",
- BYTES_TO_MB(type->low_gm_size),
- BYTES_TO_MB(type->high_gm_size),
- type->fence, vgpu_edid_str(type->resolution),
- type->weight);
-}
-
-static ssize_t name_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- struct intel_vgpu_type *type;
- struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt;
-
- type = &gvt->types[mtype_get_type_group_id(mtype)];
- if (!type)
- return 0;
-
- return sprintf(buf, "%s\n", type->name);
-}
-
-static MDEV_TYPE_ATTR_RO(available_instances);
-static MDEV_TYPE_ATTR_RO(device_api);
-static MDEV_TYPE_ATTR_RO(description);
-static MDEV_TYPE_ATTR_RO(name);
-
-static struct attribute *gvt_type_attrs[] = {
- &mdev_type_attr_available_instances.attr,
- &mdev_type_attr_device_api.attr,
- &mdev_type_attr_description.attr,
- &mdev_type_attr_name.attr,
- NULL,
-};
-
-static struct attribute_group *gvt_vgpu_type_groups[] = {
- [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
-};
-
-static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
-{
- int i, j;
- struct intel_vgpu_type *type;
- struct attribute_group *group;
-
- for (i = 0; i < gvt->num_types; i++) {
- type = &gvt->types[i];
-
- group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
- if (!group)
- goto unwind;
-
- group->name = type->name;
- group->attrs = gvt_type_attrs;
- gvt_vgpu_type_groups[i] = group;
- }
-
- return 0;
-
-unwind:
- for (j = 0; j < i; j++) {
- group = gvt_vgpu_type_groups[j];
- kfree(group);
- }
-
- return -ENOMEM;
-}
-
-static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
-{
- int i;
- struct attribute_group *group;
-
- for (i = 0; i < gvt->num_types; i++) {
- group = gvt_vgpu_type_groups[i];
- gvt_vgpu_type_groups[i] = NULL;
- kfree(group);
- }
+ BYTES_TO_MB(type->conf->low_mm),
+ BYTES_TO_MB(type->conf->high_mm),
+ type->conf->fence, vgpu_edid_str(type->conf->edid),
+ type->conf->weight);
}
static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
NULL,
};
+static int intel_vgpu_init_dev(struct vfio_device *vfio_dev)
+{
+ struct mdev_device *mdev = to_mdev_device(vfio_dev->dev);
+ struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
+ struct intel_vgpu_type *type =
+ container_of(mdev->type, struct intel_vgpu_type, type);
+
+ vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;
+ return intel_gvt_create_vgpu(vgpu, type->conf);
+}
+
+static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
+{
+ struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
+
+ intel_gvt_destroy_vgpu(vgpu);
+ vfio_free_device(vfio_dev);
+}
+
static const struct vfio_device_ops intel_vgpu_dev_ops = {
+ .init = intel_vgpu_init_dev,
+ .release = intel_vgpu_release_dev,
.open_device = intel_vgpu_open_device,
.close_device = intel_vgpu_close_device,
.read = intel_vgpu_read,
static int intel_vgpu_probe(struct mdev_device *mdev)
{
- struct device *pdev = mdev_parent_dev(mdev);
- struct intel_gvt *gvt = kdev_to_i915(pdev)->gvt;
- struct intel_vgpu_type *type;
struct intel_vgpu *vgpu;
int ret;
- type = &gvt->types[mdev_get_type_group_id(mdev)];
- if (!type)
- return -EINVAL;
-
- vgpu = intel_gvt_create_vgpu(gvt, type);
+ vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev,
+ &intel_vgpu_dev_ops);
if (IS_ERR(vgpu)) {
gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu));
return PTR_ERR(vgpu);
}
- vfio_init_group_dev(&vgpu->vfio_device, &mdev->dev,
- &intel_vgpu_dev_ops);
-
dev_set_drvdata(&mdev->dev, vgpu);
ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device);
- if (ret) {
- intel_gvt_destroy_vgpu(vgpu);
- return ret;
- }
+ if (ret)
+ goto out_put_vdev;
gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
dev_name(mdev_dev(mdev)));
return 0;
+
+out_put_vdev:
+ vfio_put_device(&vgpu->vfio_device);
+ return ret;
}
static void intel_vgpu_remove(struct mdev_device *mdev)
if (WARN_ON_ONCE(vgpu->attached))
return;
- intel_gvt_destroy_vgpu(vgpu);
+
+ vfio_unregister_group_dev(&vgpu->vfio_device);
+ vfio_put_device(&vgpu->vfio_device);
+}
+
+static unsigned int intel_vgpu_get_available(struct mdev_type *mtype)
+{
+ struct intel_vgpu_type *type =
+ container_of(mtype, struct intel_vgpu_type, type);
+ struct intel_gvt *gvt = kdev_to_i915(mtype->parent->dev)->gvt;
+ unsigned int low_gm_avail, high_gm_avail, fence_avail;
+
+ mutex_lock(&gvt->lock);
+ low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE -
+ gvt->gm.vgpu_allocated_low_gm_size;
+ high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE -
+ gvt->gm.vgpu_allocated_high_gm_size;
+ fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
+ gvt->fence.vgpu_allocated_fence_num;
+ mutex_unlock(&gvt->lock);
+
+ return min3(low_gm_avail / type->conf->low_mm,
+ high_gm_avail / type->conf->high_mm,
+ fence_avail / type->conf->fence);
}
static struct mdev_driver intel_vgpu_mdev_driver = {
+ .device_api = VFIO_DEVICE_API_PCI_STRING,
.driver = {
.name = "intel_vgpu_mdev",
.owner = THIS_MODULE,
.dev_groups = intel_vgpu_groups,
},
- .probe = intel_vgpu_probe,
- .remove = intel_vgpu_remove,
- .supported_type_groups = gvt_vgpu_type_groups,
+ .probe = intel_vgpu_probe,
+ .remove = intel_vgpu_remove,
+ .get_available = intel_vgpu_get_available,
+ .show_description = intel_vgpu_show_description,
};
int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn)
if (drm_WARN_ON(&i915->drm, !gvt))
return;
- mdev_unregister_device(i915->drm.dev);
- intel_gvt_cleanup_vgpu_type_groups(gvt);
+ mdev_unregister_parent(&gvt->parent);
intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
intel_gvt_clean_vgpu_types(gvt);
intel_gvt_debugfs_init(gvt);
- ret = intel_gvt_init_vgpu_type_groups(gvt);
+ ret = mdev_register_parent(&gvt->parent, i915->drm.dev,
+ &intel_vgpu_mdev_driver,
+ gvt->mdev_types, gvt->num_types);
if (ret)
goto out_destroy_idle_vgpu;
- ret = mdev_register_device(i915->drm.dev, &intel_vgpu_mdev_driver);
- if (ret)
- goto out_cleanup_vgpu_type_groups;
-
gvt_dbg_core("gvt device initialization is done\n");
return 0;
-out_cleanup_vgpu_type_groups:
- intel_gvt_cleanup_vgpu_type_groups(gvt);
out_destroy_idle_vgpu:
intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
intel_gvt_debugfs_clean(gvt);
drm_WARN_ON(&i915->drm, sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
}
+/*
+ * vGPU type name is defined as GVTg_Vx_y which contains the physical GPU
+ * generation type (e.g V4 as BDW server, V5 as SKL server).
+ *
+ * Depening on the physical SKU resource, we might see vGPU types like
+ * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create different types of
+ * vGPU on same physical GPU depending on available resource. Each vGPU
+ * type will have a different number of avail_instance to indicate how
+ * many vGPU instance can be created for this type.
+ */
#define VGPU_MAX_WEIGHT 16
#define VGPU_WEIGHT(vgpu_num) \
(VGPU_MAX_WEIGHT / (vgpu_num))
-static const struct {
- unsigned int low_mm;
- unsigned int high_mm;
- unsigned int fence;
-
- /* A vGPU with a weight of 8 will get twice as much GPU as a vGPU
- * with a weight of 4 on a contended host, different vGPU type has
- * different weight set. Legal weights range from 1 to 16.
- */
- unsigned int weight;
- enum intel_vgpu_edid edid;
- const char *name;
-} vgpu_types[] = {
-/* Fixed vGPU type table */
+static const struct intel_vgpu_config intel_vgpu_configs[] = {
{ MB_TO_BYTES(64), MB_TO_BYTES(384), 4, VGPU_WEIGHT(8), GVT_EDID_1024_768, "8" },
{ MB_TO_BYTES(128), MB_TO_BYTES(512), 4, VGPU_WEIGHT(4), GVT_EDID_1920_1200, "4" },
{ MB_TO_BYTES(256), MB_TO_BYTES(1024), 4, VGPU_WEIGHT(2), GVT_EDID_1920_1200, "2" },
*/
int intel_gvt_init_vgpu_types(struct intel_gvt *gvt)
{
- unsigned int num_types;
- unsigned int i, low_avail, high_avail;
- unsigned int min_low;
-
- /* vGPU type name is defined as GVTg_Vx_y which contains
- * physical GPU generation type (e.g V4 as BDW server, V5 as
- * SKL server).
- *
- * Depend on physical SKU resource, might see vGPU types like
- * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create
- * different types of vGPU on same physical GPU depending on
- * available resource. Each vGPU type will have "avail_instance"
- * to indicate how many vGPU instance can be created for this
- * type.
- *
- */
- low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
- high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
- num_types = ARRAY_SIZE(vgpu_types);
+ unsigned int low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
+ unsigned int high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
+ unsigned int num_types = ARRAY_SIZE(intel_vgpu_configs);
+ unsigned int i;
gvt->types = kcalloc(num_types, sizeof(struct intel_vgpu_type),
GFP_KERNEL);
if (!gvt->types)
return -ENOMEM;
- min_low = MB_TO_BYTES(32);
- for (i = 0; i < num_types; ++i) {
- if (low_avail / vgpu_types[i].low_mm == 0)
- break;
-
- gvt->types[i].low_gm_size = vgpu_types[i].low_mm;
- gvt->types[i].high_gm_size = vgpu_types[i].high_mm;
- gvt->types[i].fence = vgpu_types[i].fence;
+ gvt->mdev_types = kcalloc(num_types, sizeof(*gvt->mdev_types),
+ GFP_KERNEL);
+ if (!gvt->mdev_types)
+ goto out_free_types;
- if (vgpu_types[i].weight < 1 ||
- vgpu_types[i].weight > VGPU_MAX_WEIGHT)
- return -EINVAL;
+ for (i = 0; i < num_types; ++i) {
+ const struct intel_vgpu_config *conf = &intel_vgpu_configs[i];
- gvt->types[i].weight = vgpu_types[i].weight;
- gvt->types[i].resolution = vgpu_types[i].edid;
- gvt->types[i].avail_instance = min(low_avail / vgpu_types[i].low_mm,
- high_avail / vgpu_types[i].high_mm);
+ if (low_avail / conf->low_mm == 0)
+ break;
+ if (conf->weight < 1 || conf->weight > VGPU_MAX_WEIGHT)
+ goto out_free_mdev_types;
- if (GRAPHICS_VER(gvt->gt->i915) == 8)
- sprintf(gvt->types[i].name, "GVTg_V4_%s",
- vgpu_types[i].name);
- else if (GRAPHICS_VER(gvt->gt->i915) == 9)
- sprintf(gvt->types[i].name, "GVTg_V5_%s",
- vgpu_types[i].name);
+ sprintf(gvt->types[i].name, "GVTg_V%u_%s",
+ GRAPHICS_VER(gvt->gt->i915) == 8 ? 4 : 5, conf->name);
+ gvt->types[i].conf = conf;
gvt_dbg_core("type[%d]: %s avail %u low %u high %u fence %u weight %u res %s\n",
i, gvt->types[i].name,
- gvt->types[i].avail_instance,
- gvt->types[i].low_gm_size,
- gvt->types[i].high_gm_size, gvt->types[i].fence,
- gvt->types[i].weight,
- vgpu_edid_str(gvt->types[i].resolution));
+ min(low_avail / conf->low_mm,
+ high_avail / conf->high_mm),
+ conf->low_mm, conf->high_mm, conf->fence,
+ conf->weight, vgpu_edid_str(conf->edid));
+
+ gvt->mdev_types[i] = &gvt->types[i].type;
+ gvt->mdev_types[i]->sysfs_name = gvt->types[i].name;
}
gvt->num_types = i;
return 0;
-}
-void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt)
-{
+out_free_mdev_types:
+ kfree(gvt->mdev_types);
+out_free_types:
kfree(gvt->types);
+ return -EINVAL;
}
-static void intel_gvt_update_vgpu_types(struct intel_gvt *gvt)
+void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt)
{
- int i;
- unsigned int low_gm_avail, high_gm_avail, fence_avail;
- unsigned int low_gm_min, high_gm_min, fence_min;
-
- /* Need to depend on maxium hw resource size but keep on
- * static config for now.
- */
- low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE -
- gvt->gm.vgpu_allocated_low_gm_size;
- high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE -
- gvt->gm.vgpu_allocated_high_gm_size;
- fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
- gvt->fence.vgpu_allocated_fence_num;
-
- for (i = 0; i < gvt->num_types; i++) {
- low_gm_min = low_gm_avail / gvt->types[i].low_gm_size;
- high_gm_min = high_gm_avail / gvt->types[i].high_gm_size;
- fence_min = fence_avail / gvt->types[i].fence;
- gvt->types[i].avail_instance = min(min(low_gm_min, high_gm_min),
- fence_min);
-
- gvt_dbg_core("update type[%d]: %s avail %u low %u high %u fence %u\n",
- i, gvt->types[i].name,
- gvt->types[i].avail_instance, gvt->types[i].low_gm_size,
- gvt->types[i].high_gm_size, gvt->types[i].fence);
- }
+ kfree(gvt->mdev_types);
+ kfree(gvt->types);
}
/**
intel_vgpu_clean_mmio(vgpu);
intel_vgpu_dmabuf_cleanup(vgpu);
mutex_unlock(&vgpu->vgpu_lock);
-
- mutex_lock(&gvt->lock);
- intel_gvt_update_vgpu_types(gvt);
- mutex_unlock(&gvt->lock);
-
- vfree(vgpu);
}
#define IDLE_VGPU_IDR 0
vfree(vgpu);
}
-static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
- struct intel_vgpu_creation_params *param)
+int intel_gvt_create_vgpu(struct intel_vgpu *vgpu,
+ const struct intel_vgpu_config *conf)
{
+ struct intel_gvt *gvt = vgpu->gvt;
struct drm_i915_private *dev_priv = gvt->gt->i915;
- struct intel_vgpu *vgpu;
int ret;
- gvt_dbg_core("low %llu MB high %llu MB fence %llu\n",
- param->low_gm_sz, param->high_gm_sz,
- param->fence_sz);
-
- vgpu = vzalloc(sizeof(*vgpu));
- if (!vgpu)
- return ERR_PTR(-ENOMEM);
+ gvt_dbg_core("low %u MB high %u MB fence %u\n",
+ BYTES_TO_MB(conf->low_mm), BYTES_TO_MB(conf->high_mm),
+ conf->fence);
+ mutex_lock(&gvt->lock);
ret = idr_alloc(&gvt->vgpu_idr, vgpu, IDLE_VGPU_IDR + 1, GVT_MAX_VGPU,
GFP_KERNEL);
if (ret < 0)
- goto out_free_vgpu;
+ goto out_unlock;;
vgpu->id = ret;
- vgpu->gvt = gvt;
- vgpu->sched_ctl.weight = param->weight;
+ vgpu->sched_ctl.weight = conf->weight;
mutex_init(&vgpu->vgpu_lock);
mutex_init(&vgpu->dmabuf_lock);
INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head);
INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL);
idr_init_base(&vgpu->object_idr, 1);
- intel_vgpu_init_cfg_space(vgpu, param->primary);
+ intel_vgpu_init_cfg_space(vgpu, 1);
vgpu->d3_entered = false;
ret = intel_vgpu_init_mmio(vgpu);
if (ret)
goto out_clean_idr;
- ret = intel_vgpu_alloc_resource(vgpu, param);
+ ret = intel_vgpu_alloc_resource(vgpu, conf);
if (ret)
goto out_clean_vgpu_mmio;
if (ret)
goto out_clean_gtt;
- ret = intel_vgpu_init_display(vgpu, param->resolution);
+ ret = intel_vgpu_init_display(vgpu, conf->edid);
if (ret)
goto out_clean_opregion;
if (ret)
goto out_clean_sched_policy;
- return vgpu;
+ intel_gvt_update_reg_whitelist(vgpu);
+ mutex_unlock(&gvt->lock);
+ return 0;
out_clean_sched_policy:
intel_vgpu_clean_sched_policy(vgpu);
intel_vgpu_clean_mmio(vgpu);
out_clean_idr:
idr_remove(&gvt->vgpu_idr, vgpu->id);
-out_free_vgpu:
- vfree(vgpu);
- return ERR_PTR(ret);
-}
-
-/**
- * intel_gvt_create_vgpu - create a virtual GPU
- * @gvt: GVT device
- * @type: type of the vGPU to create
- *
- * This function is called when user wants to create a virtual GPU.
- *
- * Returns:
- * pointer to intel_vgpu, error pointer if failed.
- */
-struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
- struct intel_vgpu_type *type)
-{
- struct intel_vgpu_creation_params param;
- struct intel_vgpu *vgpu;
-
- param.primary = 1;
- param.low_gm_sz = type->low_gm_size;
- param.high_gm_sz = type->high_gm_size;
- param.fence_sz = type->fence;
- param.weight = type->weight;
- param.resolution = type->resolution;
-
- /* XXX current param based on MB */
- param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz);
- param.high_gm_sz = BYTES_TO_MB(param.high_gm_sz);
-
- mutex_lock(&gvt->lock);
- vgpu = __intel_gvt_create_vgpu(gvt, ¶m);
- if (!IS_ERR(vgpu)) {
- /* calculate left instance change for types */
- intel_gvt_update_vgpu_types(gvt);
- intel_gvt_update_reg_whitelist(vgpu);
- }
+out_unlock:
mutex_unlock(&gvt->lock);
-
- return vgpu;
+ return ret;
}
/**
range = round_down(end - len, align) - round_up(start, align);
if (range) {
if (sizeof(unsigned long) == sizeof(u64)) {
- addr = get_random_long();
+ addr = get_random_u64();
} else {
- addr = get_random_int();
+ addr = get_random_u32();
if (range > U32_MAX) {
addr <<= 32;
- addr |= get_random_int();
+ addr |= get_random_u32();
}
}
div64_u64_rem(addr, range, &addr);
#define TRANS_PSR_IIR(tran) _MMIO_TRANS2(tran, _PSR_IIR_A)
#define _EDP_PSR_TRANS_SHIFT(trans) ((trans) == TRANSCODER_EDP ? \
0 : ((trans) - TRANSCODER_A + 1) * 8)
-#define EDP_PSR_TRANS_MASK(trans) (0x7 << _EDP_PSR_TRANS_SHIFT(trans))
-#define EDP_PSR_ERROR(trans) (0x4 << _EDP_PSR_TRANS_SHIFT(trans))
-#define EDP_PSR_POST_EXIT(trans) (0x2 << _EDP_PSR_TRANS_SHIFT(trans))
-#define EDP_PSR_PRE_ENTRY(trans) (0x1 << _EDP_PSR_TRANS_SHIFT(trans))
+#define TGL_PSR_MASK REG_GENMASK(2, 0)
+#define TGL_PSR_ERROR REG_BIT(2)
+#define TGL_PSR_POST_EXIT REG_BIT(1)
+#define TGL_PSR_PRE_ENTRY REG_BIT(0)
+#define EDP_PSR_MASK(trans) (TGL_PSR_MASK << \
+ _EDP_PSR_TRANS_SHIFT(trans))
+#define EDP_PSR_ERROR(trans) (TGL_PSR_ERROR << \
+ _EDP_PSR_TRANS_SHIFT(trans))
+#define EDP_PSR_POST_EXIT(trans) (TGL_PSR_POST_EXIT << \
+ _EDP_PSR_TRANS_SHIFT(trans))
+#define EDP_PSR_PRE_ENTRY(trans) (TGL_PSR_PRE_ENTRY << \
+ _EDP_PSR_TRANS_SHIFT(trans))
#define _SRD_AUX_DATA_A 0x60814
#define _SRD_AUX_DATA_EDP 0x6f814
int err = 0;
while (!i915_selftest.random_seed)
- i915_selftest.random_seed = get_random_int();
+ i915_selftest.random_seed = get_random_u32();
i915_selftest.timeout_jiffies =
i915_selftest.timeout_ms ?
}
}
-static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm,
- struct vm_fault *vmf, struct migrate_vma *args,
- dma_addr_t *dma_addr)
+static int nouveau_dmem_copy_one(struct nouveau_drm *drm, struct page *spage,
+ struct page *dpage, dma_addr_t *dma_addr)
{
struct device *dev = drm->dev->dev;
- struct page *dpage, *spage;
- struct nouveau_svmm *svmm;
-
- spage = migrate_pfn_to_page(args->src[0]);
- if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE))
- return 0;
- dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address);
- if (!dpage)
- return VM_FAULT_SIGBUS;
lock_page(dpage);
*dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, *dma_addr))
- goto error_free_page;
+ return -EIO;
- svmm = spage->zone_device_data;
- mutex_lock(&svmm->mutex);
- nouveau_svmm_invalidate(svmm, args->start, args->end);
if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,
- NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))
- goto error_dma_unmap;
- mutex_unlock(&svmm->mutex);
+ NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) {
+ dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ return -EIO;
+ }
- args->dst[0] = migrate_pfn(page_to_pfn(dpage));
return 0;
-
-error_dma_unmap:
- mutex_unlock(&svmm->mutex);
- dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
-error_free_page:
- __free_page(dpage);
- return VM_FAULT_SIGBUS;
}
static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
struct nouveau_drm *drm = page_to_drm(vmf->page);
struct nouveau_dmem *dmem = drm->dmem;
struct nouveau_fence *fence;
+ struct nouveau_svmm *svmm;
+ struct page *spage, *dpage;
unsigned long src = 0, dst = 0;
dma_addr_t dma_addr = 0;
- vm_fault_t ret;
+ vm_fault_t ret = 0;
struct migrate_vma args = {
.vma = vmf->vma,
.start = vmf->address,
if (!args.cpages)
return 0;
- ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr);
- if (ret || dst == 0)
+ spage = migrate_pfn_to_page(src);
+ if (!spage || !(src & MIGRATE_PFN_MIGRATE))
+ goto done;
+
+ dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address);
+ if (!dpage)
+ goto done;
+
+ dst = migrate_pfn(page_to_pfn(dpage));
+
+ svmm = spage->zone_device_data;
+ mutex_lock(&svmm->mutex);
+ nouveau_svmm_invalidate(svmm, args.start, args.end);
+ ret = nouveau_dmem_copy_one(drm, spage, dpage, &dma_addr);
+ mutex_unlock(&svmm->mutex);
+ if (ret) {
+ ret = VM_FAULT_SIGBUS;
goto done;
+ }
nouveau_fence_new(dmem->migrate.chan, false, &fence);
migrate_vma_pages(&args);
return NULL;
}
- lock_page(page);
+ zone_device_page_init(page);
return page;
}
mutex_unlock(&drm->dmem->mutex);
}
+/*
+ * Evict all pages mapping a chunk.
+ */
+static void
+nouveau_dmem_evict_chunk(struct nouveau_dmem_chunk *chunk)
+{
+ unsigned long i, npages = range_len(&chunk->pagemap.range) >> PAGE_SHIFT;
+ unsigned long *src_pfns, *dst_pfns;
+ dma_addr_t *dma_addrs;
+ struct nouveau_fence *fence;
+
+ src_pfns = kcalloc(npages, sizeof(*src_pfns), GFP_KERNEL);
+ dst_pfns = kcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL);
+ dma_addrs = kcalloc(npages, sizeof(*dma_addrs), GFP_KERNEL);
+
+ migrate_device_range(src_pfns, chunk->pagemap.range.start >> PAGE_SHIFT,
+ npages);
+
+ for (i = 0; i < npages; i++) {
+ if (src_pfns[i] & MIGRATE_PFN_MIGRATE) {
+ struct page *dpage;
+
+ /*
+ * _GFP_NOFAIL because the GPU is going away and there
+ * is nothing sensible we can do if we can't copy the
+ * data back.
+ */
+ dpage = alloc_page(GFP_HIGHUSER | __GFP_NOFAIL);
+ dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
+ nouveau_dmem_copy_one(chunk->drm,
+ migrate_pfn_to_page(src_pfns[i]), dpage,
+ &dma_addrs[i]);
+ }
+ }
+
+ nouveau_fence_new(chunk->drm->dmem->migrate.chan, false, &fence);
+ migrate_device_pages(src_pfns, dst_pfns, npages);
+ nouveau_dmem_fence_done(&fence);
+ migrate_device_finalize(src_pfns, dst_pfns, npages);
+ kfree(src_pfns);
+ kfree(dst_pfns);
+ for (i = 0; i < npages; i++)
+ dma_unmap_page(chunk->drm->dev->dev, dma_addrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
+ kfree(dma_addrs);
+}
+
void
nouveau_dmem_fini(struct nouveau_drm *drm)
{
mutex_lock(&drm->dmem->mutex);
list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) {
+ nouveau_dmem_evict_chunk(chunk);
nouveau_bo_unpin(chunk->bo);
nouveau_bo_ref(NULL, &chunk->bo);
+ WARN_ON(chunk->callocated);
list_del(&chunk->list);
memunmap_pages(&chunk->pagemap);
release_mem_region(chunk->pagemap.range.start,
static int drm_buddy_init_test(struct kunit *test)
{
while (!random_seed)
- random_seed = get_random_int();
+ random_seed = get_random_u32();
return 0;
}
static int drm_mm_init_test(struct kunit *test)
{
while (!random_seed)
- random_seed = get_random_int();
+ random_seed = get_random_u32();
return 0;
}
[ABS_HAT3Y] = "Hat 3Y", [ABS_PRESSURE] = "Pressure",
[ABS_DISTANCE] = "Distance", [ABS_TILT_X] = "XTilt",
[ABS_TILT_Y] = "YTilt", [ABS_TOOL_WIDTH] = "ToolWidth",
- [ABS_VOLUME] = "Volume", [ABS_MISC] = "Misc",
+ [ABS_VOLUME] = "Volume", [ABS_PROFILE] = "Profile",
+ [ABS_MISC] = "Misc",
[ABS_MT_TOUCH_MAJOR] = "MTMajor",
[ABS_MT_TOUCH_MINOR] = "MTMinor",
[ABS_MT_WIDTH_MAJOR] = "MTMajorW",
void vmbus_on_event(unsigned long data)
{
struct vmbus_channel *channel = (void *) data;
- unsigned long time_limit = jiffies + 2;
+ void (*callback_fn)(void *context);
trace_vmbus_on_event(channel);
hv_debug_delay_test(channel, INTERRUPT_DELAY);
- do {
- void (*callback_fn)(void *);
- /* A channel once created is persistent even when
- * there is no driver handling the device. An
- * unloading driver sets the onchannel_callback to NULL.
- */
- callback_fn = READ_ONCE(channel->onchannel_callback);
- if (unlikely(callback_fn == NULL))
- return;
-
- (*callback_fn)(channel->channel_callback_context);
+ /* A channel once created is persistent even when
+ * there is no driver handling the device. An
+ * unloading driver sets the onchannel_callback to NULL.
+ */
+ callback_fn = READ_ONCE(channel->onchannel_callback);
+ if (unlikely(!callback_fn))
+ return;
- if (channel->callback_mode != HV_CALL_BATCHED)
- return;
+ (*callback_fn)(channel->channel_callback_context);
- if (likely(hv_end_read(&channel->inbound) == 0))
- return;
+ if (channel->callback_mode != HV_CALL_BATCHED)
+ return;
- hv_begin_read(&channel->inbound);
- } while (likely(time_before(jiffies, time_limit)));
+ if (likely(hv_end_read(&channel->inbound) == 0))
+ return;
- /* The time limit (2 jiffies) has been reached */
+ hv_begin_read(&channel->inbound);
tasklet_schedule(&channel->callback_event);
}
static struct acpi_device *hv_acpi_dev;
-static struct completion probe_event;
-
static int hyperv_cpuhp_online;
static void *hv_panic_page;
return;
INIT_WORK(&ctx->work, vmbus_onmessage_work);
- memcpy(&ctx->msg, &msg_copy, sizeof(msg->header) + payload_size);
+ ctx->msg.header = msg_copy.header;
+ memcpy(&ctx->msg.payload, msg_copy.u.payload, payload_size);
/*
* The host can generate a rescind message while we
}
/**
- * __vmbus_child_driver_register() - Register a vmbus's driver
+ * __vmbus_driver_register() - Register a vmbus's driver
* @hv_driver: Pointer to driver structure you want to register
* @owner: owner module of the drv
* @mod_name: module name string
child_device_obj->channel = channel;
guid_copy(&child_device_obj->dev_type, type);
guid_copy(&child_device_obj->dev_instance, instance);
- child_device_obj->vendor_id = 0x1414; /* MSFT vendor ID */
+ child_device_obj->vendor_id = PCI_VENDOR_ID_MICROSOFT;
return child_device_obj;
}
ret_val = 0;
acpi_walk_err:
- complete(&probe_event);
if (ret_val)
vmbus_acpi_remove(device);
return ret_val;
.remove = vmbus_acpi_remove,
},
.drv.pm = &vmbus_bus_pm,
+ .drv.probe_type = PROBE_FORCE_SYNCHRONOUS,
};
static void hv_kexec_handler(void)
static int __init hv_acpi_init(void)
{
- int ret, t;
+ int ret;
if (!hv_is_hyperv_initialized())
return -ENODEV;
if (hv_root_partition)
return 0;
- init_completion(&probe_event);
-
/*
* Get ACPI resources first.
*/
if (ret)
return ret;
- t = wait_for_completion_timeout(&probe_event, 5*HZ);
- if (t == 0) {
- ret = -ETIMEDOUT;
+ if (!hv_acpi_dev) {
+ ret = -ENODEV;
goto cleanup;
}
u32 command, irq_handled = 0;
struct i2c_client *slave = bus->slave;
u8 value;
+ int ret;
if (!slave)
return 0;
break;
case ASPEED_I2C_SLAVE_WRITE_REQUESTED:
bus->slave_state = ASPEED_I2C_SLAVE_WRITE_RECEIVED;
- i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value);
+ ret = i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value);
+ /*
+ * Slave ACK's on this address phase already but as the backend driver
+ * returns an errno, the bus driver should nack the next incoming byte.
+ */
+ if (ret < 0)
+ writel(ASPEED_I2CD_M_S_RX_CMD_LAST, bus->base + ASPEED_I2C_CMD_REG);
break;
case ASPEED_I2C_SLAVE_WRITE_RECEIVED:
i2c_slave_event(slave, I2C_SLAVE_WRITE_RECEIVED, &value);
* status codes
*/
#define STATUS_IDLE 0x0
-#define STATUS_WRITE_IN_PROGRESS 0x1
-#define STATUS_READ_IN_PROGRESS 0x2
+#define STATUS_ACTIVE 0x1
+#define STATUS_WRITE_IN_PROGRESS 0x2
+#define STATUS_READ_IN_PROGRESS 0x4
/*
* operation modes
static inline void __i2c_dw_enable(struct dw_i2c_dev *dev)
{
+ dev->status |= STATUS_ACTIVE;
regmap_write(dev->map, DW_IC_ENABLE, 1);
}
static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev)
{
regmap_write(dev->map, DW_IC_ENABLE, 0);
+ dev->status &= ~STATUS_ACTIVE;
}
void __i2c_dw_disable(struct dw_i2c_dev *dev);
u32 stat;
stat = i2c_dw_read_clear_intrbits(dev);
+
+ if (!(dev->status & STATUS_ACTIVE)) {
+ /*
+ * Unexpected interrupt in driver point of view. State
+ * variables are either unset or stale so acknowledge and
+ * disable interrupts for suppressing further interrupts if
+ * interrupt really came from this HW (E.g. firmware has left
+ * the HW active).
+ */
+ regmap_write(dev->map, DW_IC_INTR_MASK, 0);
+ return 0;
+ }
+
if (stat & DW_IC_INTR_TX_ABRT) {
dev->cmd_err |= DW_IC_ERR_TX_ABRT;
dev->status = STATUS_IDLE;
void __iomem *p2 = i2c->i2c_base + SMBUS_STATUS_REG_OFF;
void __iomem *p1 = i2c->i2c_base + SMB_GPR_REG;
u8 regval;
- u8 ret;
+ int ret;
ret = set_sys_lock(i2c);
if (ret == -EPERM) {
};
static const struct of_device_id cci_dt_match[] = {
+ { .compatible = "qcom,msm8226-cci", .data = &cci_v1_data},
{ .compatible = "qcom,msm8916-cci", .data = &cci_v1_data},
{ .compatible = "qcom,msm8974-cci", .data = &cci_v1_5_data},
{ .compatible = "qcom,msm8996-cci", .data = &cci_v2_data},
i3c_bus_set_addr_slot_status(&master->bus,
dev->info.dyn_addr,
I3C_ADDR_SLOT_I3C_DEV);
+ if (old_dyn_addr)
+ i3c_bus_set_addr_slot_status(&master->bus, old_dyn_addr,
+ I3C_ADDR_SLOT_FREE);
}
if (master->ops->reattach_i3c_dev) {
i3c_master_free_i3c_dev(olddev);
}
- ret = i3c_master_reattach_i3c_dev(newdev, old_dyn_addr);
- if (ret)
- goto err_detach_dev;
-
/*
* Depending on our previous state, the expected dynamic address might
* differ:
inet_get_local_port_range(net, &low, &high);
remaining = (high - low) + 1;
- rover = prandom_u32() % remaining + low;
+ rover = prandom_u32_max(remaining) + low;
retry:
if (last_used_port != rover) {
struct rdma_bind_list *bind_list;
&ep->com.remote_addr;
int ret;
enum chip_type adapter_type = ep->com.dev->rdev.lldi.adapter_type;
- u32 isn = (prandom_u32() & ~7UL) - 1;
+ u32 isn = (get_random_u32() & ~7UL) - 1;
struct net_device *netdev;
u64 params;
}
if (!is_t4(adapter_type)) {
- u32 isn = (prandom_u32() & ~7UL) - 1;
+ u32 isn = (get_random_u32() & ~7UL) - 1;
skb = get_skb(skb, roundup(sizeof(*rpl5), 16), GFP_KERNEL);
rpl5 = __skb_put_zero(skb, roundup(sizeof(*rpl5), 16));
if (obj < alloc->max) {
if (alloc->flags & C4IW_ID_TABLE_F_RANDOM)
- alloc->last += prandom_u32() % RANDOM_SKIP;
+ alloc->last += prandom_u32_max(RANDOM_SKIP);
else
alloc->last = obj + 1;
if (alloc->last >= alloc->max)
alloc->start = start;
alloc->flags = flags;
if (flags & C4IW_ID_TABLE_F_RANDOM)
- alloc->last = prandom_u32() % RANDOM_SKIP;
+ alloc->last = prandom_u32_max(RANDOM_SKIP);
else
alloc->last = 0;
alloc->max = num;
int i;
for (i = 0; i < RXE_NUM_TID_FLOWS; i++) {
- rcd->flows[i].generation = mask_generation(prandom_u32());
+ rcd->flows[i].generation = mask_generation(get_random_u32());
kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, i);
}
}
u16 sport;
if (!fl)
- sport = get_random_u32() %
- (IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 -
- IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) +
+ sport = prandom_u32_max(IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 -
+ IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) +
IB_ROCE_UDP_ENCAP_VALID_PORT_MIN;
else
sport = rdma_flow_label_to_udp_sport(fl);
__be64 mlx4_ib_gen_node_guid(void)
{
#define NODE_GUID_HI ((u64) (((u64)IB_OPENIB_OUI) << 40))
- return cpu_to_be64(NODE_GUID_HI | prandom_u32());
+ return cpu_to_be64(NODE_GUID_HI | get_random_u32());
}
__be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx)
goto err_qp;
}
- psn = prandom_u32() & 0xffffff;
+ psn = get_random_u32() & 0xffffff;
ret = ipoib_cm_modify_rx_qp(dev, cm_id, p->qp, psn);
if (ret)
goto err_modify;
rtrs_clt_stop_and_destroy_conns(clt_path);
queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork,
msecs_to_jiffies(delay_ms +
- prandom_u32() %
- RTRS_RECONNECT_SEED));
+ prandom_u32_max(RTRS_RECONNECT_SEED)));
}
static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt,
* Copyright (c) 2006 Dmitry Torokhov <dtor@mail.ru>
*/
-/*
- */
-
/* #define DEBUG */
#include <linux/input.h>
* Copyright (c) 2006 Dmitry Torokhov <dtor@mail.ru>
*/
-/*
- */
-
/* #define DEBUG */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
* EMU10k1 - SB Live / Audigy - gameport driver for Linux
*/
-/*
- */
-
#include <asm/io.h>
#include <linux/module.h>
* PDPI Lightning 4 gamecard driver for Linux.
*/
-/*
- */
-
#include <asm/io.h>
#include <linux/delay.h>
#include <linux/errno.h>
* NS558 based standard IBM game port driver for Linux
*/
-/*
- */
-
#include <asm/io.h>
#include <linux/module.h>
}
/*
- * These codes are copied from from hid-ids.h, unfortunately there is no common
+ * These codes are copied from hid-ids.h, unfortunately there is no common
* usb_ids/bt_ids.h header.
*/
#define USB_VENDOR_ID_SONY 0x054c
* FP-Gaming Assassin 3D joystick driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
struct adc_joystick_axis *axes;
struct iio_channel *chans;
int num_chans;
+ bool polled;
};
+static void adc_joystick_poll(struct input_dev *input)
+{
+ struct adc_joystick *joy = input_get_drvdata(input);
+ int i, val, ret;
+
+ for (i = 0; i < joy->num_chans; i++) {
+ ret = iio_read_channel_raw(&joy->chans[i], &val);
+ if (ret < 0)
+ return;
+ input_report_abs(input, joy->axes[i].code, val);
+ }
+ input_sync(input);
+}
+
static int adc_joystick_handle(const void *data, void *private)
{
struct adc_joystick *joy = private;
int error;
int bits;
int i;
+ unsigned int poll_interval;
joy = devm_kzalloc(dev, sizeof(*joy), GFP_KERNEL);
if (!joy)
return error;
}
- /* Count how many channels we got. NULL terminated. */
+ error = device_property_read_u32(dev, "poll-interval", &poll_interval);
+ if (error) {
+ /* -EINVAL means the property is absent. */
+ if (error != -EINVAL)
+ return error;
+ } else if (poll_interval == 0) {
+ dev_err(dev, "Unable to get poll-interval\n");
+ return -EINVAL;
+ } else {
+ joy->polled = true;
+ }
+
+ /*
+ * Count how many channels we got. NULL terminated.
+ * Do not check the storage size if using polling.
+ */
for (i = 0; joy->chans[i].indio_dev; i++) {
+ if (joy->polled)
+ continue;
bits = joy->chans[i].channel->scan_type.storagebits;
if (!bits || bits > 16) {
dev_err(dev, "Unsupported channel storage size\n");
joy->input = input;
input->name = pdev->name;
input->id.bustype = BUS_HOST;
- input->open = adc_joystick_open;
- input->close = adc_joystick_close;
error = adc_joystick_set_axes(dev, joy);
if (error)
return error;
- joy->buffer = iio_channel_get_all_cb(dev, adc_joystick_handle, joy);
- if (IS_ERR(joy->buffer)) {
- dev_err(dev, "Unable to allocate callback buffer\n");
- return PTR_ERR(joy->buffer);
- }
+ if (joy->polled) {
+ input_setup_polling(input, adc_joystick_poll);
+ input_set_poll_interval(input, poll_interval);
+ } else {
+ input->open = adc_joystick_open;
+ input->close = adc_joystick_close;
+
+ joy->buffer = iio_channel_get_all_cb(dev, adc_joystick_handle,
+ joy);
+ if (IS_ERR(joy->buffer)) {
+ dev_err(dev, "Unable to allocate callback buffer\n");
+ return PTR_ERR(joy->buffer);
+ }
- error = devm_add_action_or_reset(dev, adc_joystick_cleanup, joy->buffer);
- if (error) {
- dev_err(dev, "Unable to add action\n");
- return error;
+ error = devm_add_action_or_reset(dev, adc_joystick_cleanup,
+ joy->buffer);
+ if (error) {
+ dev_err(dev, "Unable to add action\n");
+ return error;
+ }
}
input_set_drvdata(input, joy);
* Logitech ADI joystick family driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/module.h>
* Driver for Amiga joysticks for Linux/m68k
*/
-/*
- */
-
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/kernel.h>
* Analog joystick and gamepad driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/module.h>
* Creative Labs Blaster GamePad Cobra driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
* Atari, Amstrad, Commodore, Amiga, Sega, etc. joystick driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/delay.h>
* Raphael Assenat
*/
-/*
- */
-
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kernel.h>
* Genius Flight 2000 joystick driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/slab.h>
* Gravis/Kensington GrIP protocol joystick and gamepad driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
* Guillemot Digital Interface Protocol driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
* InterAct digital gamepad/joystick driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
* out of the joystick port into the syslog ...
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/gameport.h>
#include <linux/kernel.h>
* Magellan and Space Mouse 6dof controller driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
* Microsoft SideWinder joystick family driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/module.h>
* SpaceTec SpaceBall 2003/3003/4000 FLX driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
* SpaceTec SpaceOrb 360 and Avenger 6dof controller driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
* Gravis Stinger gamepad driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
* ThrustMaster DirectConnect (BSP) joystick family driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/slab.h>
* TurboGraFX parallel port interface driver for Linux.
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/parport.h>
#include <linux/input.h>
* Arndt Schoenewald <arndt@quelltext.com>
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
* Logitech WingMan Warrior joystick driver for Linux
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
* Later changes can be tracked in SCM.
*/
+#include <linux/bits.h>
#include <linux/kernel.h>
#include <linux/input.h>
#include <linux/rcupdate.h>
#define MAP_TRIGGERS_TO_BUTTONS (1 << 1)
#define MAP_STICKS_TO_NULL (1 << 2)
#define MAP_SELECT_BUTTON (1 << 3)
+#define MAP_PADDLES (1 << 4)
+#define MAP_PROFILE_BUTTON (1 << 5)
+
#define DANCEPAD_MAP_CONFIG (MAP_DPAD_TO_BUTTONS | \
MAP_TRIGGERS_TO_BUTTONS | MAP_STICKS_TO_NULL)
#define XTYPE_XBOXONE 3
#define XTYPE_UNKNOWN 4
+/* Send power-off packet to xpad360w after holding the mode button for this many
+ * seconds
+ */
+#define XPAD360W_POWEROFF_TIMEOUT 5
+
+#define PKT_XB 0
+#define PKT_XBE1 1
+#define PKT_XBE2_FW_OLD 2
+#define PKT_XBE2_FW_5_EARLY 3
+#define PKT_XBE2_FW_5_11 4
+
static bool dpad_to_buttons;
module_param(dpad_to_buttons, bool, S_IRUGO);
MODULE_PARM_DESC(dpad_to_buttons, "Map D-PAD to buttons rather than axes for unknown pads");
char *name;
u8 mapping;
u8 xtype;
+ u8 packet_type;
} xpad_device[] = {
{ 0x0079, 0x18d4, "GPD Win 2 X-Box Controller", 0, XTYPE_XBOX360 },
+ { 0x03eb, 0xff01, "Wooting One (Legacy)", 0, XTYPE_XBOX360 },
+ { 0x03eb, 0xff02, "Wooting Two (Legacy)", 0, XTYPE_XBOX360 },
{ 0x044f, 0x0f00, "Thrustmaster Wheel", 0, XTYPE_XBOX },
{ 0x044f, 0x0f03, "Thrustmaster Wheel", 0, XTYPE_XBOX },
{ 0x044f, 0x0f07, "Thrustmaster, Inc. Controller", 0, XTYPE_XBOX },
{ 0x045e, 0x0291, "Xbox 360 Wireless Receiver (XBOX)", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W },
{ 0x045e, 0x02d1, "Microsoft X-Box One pad", 0, XTYPE_XBOXONE },
{ 0x045e, 0x02dd, "Microsoft X-Box One pad (Firmware 2015)", 0, XTYPE_XBOXONE },
- { 0x045e, 0x02e3, "Microsoft X-Box One Elite pad", 0, XTYPE_XBOXONE },
+ { 0x045e, 0x02e3, "Microsoft X-Box One Elite pad", MAP_PADDLES, XTYPE_XBOXONE },
+ { 0x045e, 0x0b00, "Microsoft X-Box One Elite 2 pad", MAP_PADDLES, XTYPE_XBOXONE },
{ 0x045e, 0x02ea, "Microsoft X-Box One S pad", 0, XTYPE_XBOXONE },
{ 0x045e, 0x0719, "Xbox 360 Wireless Receiver", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W },
+ { 0x045e, 0x0b0a, "Microsoft X-Box Adaptive Controller", MAP_PROFILE_BUTTON, XTYPE_XBOXONE },
{ 0x045e, 0x0b12, "Microsoft Xbox Series S|X Controller", MAP_SELECT_BUTTON, XTYPE_XBOXONE },
{ 0x046d, 0xc21d, "Logitech Gamepad F310", 0, XTYPE_XBOX360 },
{ 0x046d, 0xc21e, "Logitech Gamepad F510", 0, XTYPE_XBOX360 },
{ 0x0f0d, 0x0063, "Hori Real Arcade Pro Hayabusa (USA) Xbox One", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE },
{ 0x0f0d, 0x0067, "HORIPAD ONE", 0, XTYPE_XBOXONE },
{ 0x0f0d, 0x0078, "Hori Real Arcade Pro V Kai Xbox One", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE },
+ { 0x0f0d, 0x00c5, "Hori Fighting Commander ONE", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE },
{ 0x0f30, 0x010b, "Philips Recoil", 0, XTYPE_XBOX },
{ 0x0f30, 0x0202, "Joytech Advanced Controller", 0, XTYPE_XBOX },
{ 0x0f30, 0x8888, "BigBen XBMiniPad Controller", 0, XTYPE_XBOX },
{ 0x1430, 0x8888, "TX6500+ Dance Pad (first generation)", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX },
{ 0x1430, 0xf801, "RedOctane Controller", 0, XTYPE_XBOX360 },
{ 0x146b, 0x0601, "BigBen Interactive XBOX 360 Controller", 0, XTYPE_XBOX360 },
+ { 0x146b, 0x0604, "Bigben Interactive DAIJA Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
{ 0x1532, 0x0037, "Razer Sabertooth", 0, XTYPE_XBOX360 },
{ 0x1532, 0x0a00, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE },
{ 0x1532, 0x0a03, "Razer Wildcat", 0, XTYPE_XBOXONE },
{ 0x24c6, 0x5502, "Hori Fighting Stick VX Alt", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
{ 0x24c6, 0x5503, "Hori Fighting Edge", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
{ 0x24c6, 0x5506, "Hori SOULCALIBUR V Stick", 0, XTYPE_XBOX360 },
+ { 0x24c6, 0x5510, "Hori Fighting Commander ONE (Xbox 360/PC Mode)", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
{ 0x24c6, 0x550d, "Hori GEM Xbox controller", 0, XTYPE_XBOX360 },
{ 0x24c6, 0x550e, "Hori Real Arcade Pro V Kai 360", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 },
{ 0x24c6, 0x551a, "PowerA FUSION Pro Controller", 0, XTYPE_XBOXONE },
{ 0x24c6, 0x5b03, "Thrustmaster Ferrari 458 Racing Wheel", 0, XTYPE_XBOX360 },
{ 0x24c6, 0x5d04, "Razer Sabertooth", 0, XTYPE_XBOX360 },
{ 0x24c6, 0xfafe, "Rock Candy Gamepad for Xbox 360", 0, XTYPE_XBOX360 },
+ { 0x2563, 0x058d, "OneXPlayer Gamepad", 0, XTYPE_XBOX360 },
+ { 0x2dc8, 0x2000, "8BitDo Pro 2 Wired Controller fox Xbox", 0, XTYPE_XBOXONE },
+ { 0x31e3, 0x1100, "Wooting One", 0, XTYPE_XBOX360 },
+ { 0x31e3, 0x1200, "Wooting Two", 0, XTYPE_XBOX360 },
+ { 0x31e3, 0x1210, "Wooting Lekker", 0, XTYPE_XBOX360 },
+ { 0x31e3, 0x1220, "Wooting Two HE", 0, XTYPE_XBOX360 },
+ { 0x31e3, 0x1300, "Wooting 60HE (AVR)", 0, XTYPE_XBOX360 },
+ { 0x31e3, 0x1310, "Wooting 60HE (ARM)", 0, XTYPE_XBOX360 },
{ 0x3285, 0x0607, "Nacon GC-100", 0, XTYPE_XBOX360 },
{ 0x3767, 0x0101, "Fanatec Speedster 3 Forceshock Wheel", 0, XTYPE_XBOX },
{ 0xffff, 0xffff, "Chinese-made Xbox Controller", 0, XTYPE_XBOX },
-1
};
+/* used when the controller has extra paddle buttons */
+static const signed short xpad_btn_paddles[] = {
+ BTN_TRIGGER_HAPPY5, BTN_TRIGGER_HAPPY6, /* paddle upper right, lower right */
+ BTN_TRIGGER_HAPPY7, BTN_TRIGGER_HAPPY8, /* paddle upper left, lower left */
+ -1 /* terminating entry */
+};
+
/*
* Xbox 360 has a vendor-specific class, so we cannot match it with only
* USB_INTERFACE_INFO (also specifically refused by USB subsystem), so we
static const struct usb_device_id xpad_table[] = {
{ USB_INTERFACE_INFO('X', 'B', 0) }, /* X-Box USB-IF not approved class */
XPAD_XBOX360_VENDOR(0x0079), /* GPD Win 2 Controller */
+ XPAD_XBOX360_VENDOR(0x03eb), /* Wooting Keyboards (Legacy) */
XPAD_XBOX360_VENDOR(0x044f), /* Thrustmaster X-Box 360 controllers */
XPAD_XBOX360_VENDOR(0x045e), /* Microsoft X-Box 360 controllers */
XPAD_XBOXONE_VENDOR(0x045e), /* Microsoft X-Box One controllers */
{ USB_DEVICE(0x0738, 0x4540) }, /* Mad Catz Beat Pad */
XPAD_XBOXONE_VENDOR(0x0738), /* Mad Catz FightStick TE 2 */
XPAD_XBOX360_VENDOR(0x07ff), /* Mad Catz GamePad */
+ XPAD_XBOX360_VENDOR(0x0c12), /* Zeroplus X-Box 360 controllers */
XPAD_XBOX360_VENDOR(0x0e6f), /* 0x0e6f X-Box 360 controllers */
XPAD_XBOXONE_VENDOR(0x0e6f), /* 0x0e6f X-Box One controllers */
XPAD_XBOX360_VENDOR(0x0f0d), /* Hori Controllers */
XPAD_XBOXONE_VENDOR(0x20d6), /* PowerA Controllers */
XPAD_XBOX360_VENDOR(0x24c6), /* PowerA Controllers */
XPAD_XBOXONE_VENDOR(0x24c6), /* PowerA Controllers */
+ XPAD_XBOX360_VENDOR(0x2563), /* OneXPlayer Gamepad */
+ XPAD_XBOX360_VENDOR(0x260d), /* Dareu H101 */
+ XPAD_XBOXONE_VENDOR(0x2dc8), /* 8BitDo Pro 2 Wired Controller for Xbox */
XPAD_XBOXONE_VENDOR(0x2e24), /* Hyperkin Duke X-Box One pad */
XPAD_XBOX360_VENDOR(0x2f24), /* GameSir Controllers */
+ XPAD_XBOX360_VENDOR(0x31e3), /* Wooting Keyboards */
XPAD_XBOX360_VENDOR(0x3285), /* Nacon GC-100 */
{ }
};
.len = ARRAY_SIZE(_data), \
}
+/*
+ * starting with xbox one, the game input protocol is used
+ * magic numbers are taken from
+ * - https://github.com/xpadneo/gip-dissector/blob/main/src/gip-dissector.lua
+ * - https://github.com/medusalix/xone/blob/master/bus/protocol.c
+ */
+#define GIP_CMD_ACK 0x01
+#define GIP_CMD_IDENTIFY 0x04
+#define GIP_CMD_POWER 0x05
+#define GIP_CMD_AUTHENTICATE 0x06
+#define GIP_CMD_VIRTUAL_KEY 0x07
+#define GIP_CMD_RUMBLE 0x09
+#define GIP_CMD_LED 0x0a
+#define GIP_CMD_FIRMWARE 0x0c
+#define GIP_CMD_INPUT 0x20
+
+#define GIP_SEQ0 0x00
+
+#define GIP_OPT_ACK 0x10
+#define GIP_OPT_INTERNAL 0x20
+
+/*
+ * length of the command payload encoded with
+ * https://en.wikipedia.org/wiki/LEB128
+ * which is a no-op for N < 128
+ */
+#define GIP_PL_LEN(N) (N)
+
+/*
+ * payload specific defines
+ */
+#define GIP_PWR_ON 0x00
+#define GIP_LED_ON 0x01
+
+#define GIP_MOTOR_R BIT(0)
+#define GIP_MOTOR_L BIT(1)
+#define GIP_MOTOR_RT BIT(2)
+#define GIP_MOTOR_LT BIT(3)
+#define GIP_MOTOR_ALL (GIP_MOTOR_R | GIP_MOTOR_L | GIP_MOTOR_RT | GIP_MOTOR_LT)
/*
* This packet is required for all Xbox One pads with 2015
* or later firmware installed (or present from the factory).
*/
-static const u8 xboxone_fw2015_init[] = {
- 0x05, 0x20, 0x00, 0x01, 0x00
+static const u8 xboxone_power_on[] = {
+ GIP_CMD_POWER, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(1), GIP_PWR_ON
};
/*
* Bluetooth mode.
*/
static const u8 xboxone_s_init[] = {
- 0x05, 0x20, 0x00, 0x0f, 0x06
+ GIP_CMD_POWER, GIP_OPT_INTERNAL, GIP_SEQ0, 0x0f, 0x06
+};
+
+/*
+ * This packet is required to get additional input data
+ * from Xbox One Elite Series 2 (0x045e:0x0b00) pads.
+ * We mostly do this right now to get paddle data
+ */
+static const u8 extra_input_packet_init[] = {
+ 0x4d, 0x10, 0x01, 0x02, 0x07, 0x00
};
/*
* (0x0e6f:0x0165) to finish initialization and for Hori pads
* (0x0f0d:0x0067) to make the analog sticks work.
*/
-static const u8 xboxone_hori_init[] = {
- 0x01, 0x20, 0x00, 0x09, 0x00, 0x04, 0x20, 0x3a,
- 0x00, 0x00, 0x00, 0x80, 0x00
+static const u8 xboxone_hori_ack_id[] = {
+ GIP_CMD_ACK, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(9),
+ 0x00, GIP_CMD_IDENTIFY, GIP_OPT_INTERNAL, 0x3a, 0x00, 0x00, 0x00, 0x80, 0x00
};
/*
* sending input reports. These pads include: (0x0e6f:0x02ab),
* (0x0e6f:0x02a4), (0x0e6f:0x02a6).
*/
-static const u8 xboxone_pdp_init1[] = {
- 0x0a, 0x20, 0x00, 0x03, 0x00, 0x01, 0x14
+static const u8 xboxone_pdp_led_on[] = {
+ GIP_CMD_LED, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(3), 0x00, GIP_LED_ON, 0x14
};
/*
* sending input reports. These pads include: (0x0e6f:0x02ab),
* (0x0e6f:0x02a4), (0x0e6f:0x02a6).
*/
-static const u8 xboxone_pdp_init2[] = {
- 0x06, 0x20, 0x00, 0x02, 0x01, 0x00
+static const u8 xboxone_pdp_auth[] = {
+ GIP_CMD_AUTHENTICATE, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(2), 0x01, 0x00
};
/*
* sending input reports. One of those pads is (0x24c6:0x543a).
*/
static const u8 xboxone_rumblebegin_init[] = {
- 0x09, 0x00, 0x00, 0x09, 0x00, 0x0F, 0x00, 0x00,
- 0x1D, 0x1D, 0xFF, 0x00, 0x00
+ GIP_CMD_RUMBLE, 0x00, GIP_SEQ0, GIP_PL_LEN(9),
+ 0x00, GIP_MOTOR_ALL, 0x00, 0x00, 0x1D, 0x1D, 0xFF, 0x00, 0x00
};
/*
* spin up to enough speed to actually vibrate the gamepad.
*/
static const u8 xboxone_rumbleend_init[] = {
- 0x09, 0x00, 0x00, 0x09, 0x00, 0x0F, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00
+ GIP_CMD_RUMBLE, 0x00, GIP_SEQ0, GIP_PL_LEN(9),
+ 0x00, GIP_MOTOR_ALL, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
/*
* packet is going to be sent.
*/
static const struct xboxone_init_packet xboxone_init_packets[] = {
- XBOXONE_INIT_PKT(0x0e6f, 0x0165, xboxone_hori_init),
- XBOXONE_INIT_PKT(0x0f0d, 0x0067, xboxone_hori_init),
- XBOXONE_INIT_PKT(0x0000, 0x0000, xboxone_fw2015_init),
+ XBOXONE_INIT_PKT(0x0e6f, 0x0165, xboxone_hori_ack_id),
+ XBOXONE_INIT_PKT(0x0f0d, 0x0067, xboxone_hori_ack_id),
+ XBOXONE_INIT_PKT(0x0000, 0x0000, xboxone_power_on),
XBOXONE_INIT_PKT(0x045e, 0x02ea, xboxone_s_init),
XBOXONE_INIT_PKT(0x045e, 0x0b00, xboxone_s_init),
- XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_init1),
- XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_init2),
+ XBOXONE_INIT_PKT(0x045e, 0x0b00, extra_input_packet_init),
+ XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_led_on),
+ XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_auth),
XBOXONE_INIT_PKT(0x24c6, 0x541a, xboxone_rumblebegin_init),
XBOXONE_INIT_PKT(0x24c6, 0x542a, xboxone_rumblebegin_init),
XBOXONE_INIT_PKT(0x24c6, 0x543a, xboxone_rumblebegin_init),
int mapping; /* map d-pad to buttons or to axes */
int xtype; /* type of xbox device */
+ int packet_type; /* type of the extended packet */
int pad_nr; /* the order x360 pads were attached */
const char *name; /* name of the device */
struct work_struct work; /* init/remove device from callback */
+ time64_t mode_btn_down_ts;
};
static int xpad_init_input(struct usb_xpad *xpad);
static void xpad_deinit_input(struct usb_xpad *xpad);
static void xpadone_ack_mode_report(struct usb_xpad *xpad, u8 seq_num);
+static void xpad360w_poweroff_controller(struct usb_xpad *xpad);
/*
* xpad_process_packet
/* digital pad */
if (xpad->mapping & MAP_DPAD_TO_BUTTONS) {
/* dpad as buttons (left, right, up, down) */
- input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & 0x04);
- input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & 0x08);
- input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & 0x01);
- input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & 0x02);
+ input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & BIT(2));
+ input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & BIT(3));
+ input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & BIT(1));
} else {
input_report_abs(dev, ABS_HAT0X,
!!(data[2] & 0x08) - !!(data[2] & 0x04));
}
/* start/back buttons and stick press left/right */
- input_report_key(dev, BTN_START, data[2] & 0x10);
- input_report_key(dev, BTN_SELECT, data[2] & 0x20);
- input_report_key(dev, BTN_THUMBL, data[2] & 0x40);
- input_report_key(dev, BTN_THUMBR, data[2] & 0x80);
+ input_report_key(dev, BTN_START, data[2] & BIT(4));
+ input_report_key(dev, BTN_SELECT, data[2] & BIT(5));
+ input_report_key(dev, BTN_THUMBL, data[2] & BIT(6));
+ input_report_key(dev, BTN_THUMBR, data[2] & BIT(7));
/* "analog" buttons A, B, X, Y */
input_report_key(dev, BTN_A, data[4]);
input_report_key(dev, BTN_C, data[8]);
input_report_key(dev, BTN_Z, data[9]);
+ /* Profile button has a value of 0-3, so it is reported as an axis */
+ if (xpad->mapping & MAP_PROFILE_BUTTON)
+ input_report_abs(dev, ABS_PROFILE, data[34]);
+
input_sync(dev);
}
/* digital pad */
if (xpad->mapping & MAP_DPAD_TO_BUTTONS) {
/* dpad as buttons (left, right, up, down) */
- input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & 0x04);
- input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & 0x08);
- input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & 0x01);
- input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & 0x02);
+ input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & BIT(2));
+ input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & BIT(3));
+ input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & BIT(1));
}
/*
}
/* start/back buttons */
- input_report_key(dev, BTN_START, data[2] & 0x10);
- input_report_key(dev, BTN_SELECT, data[2] & 0x20);
+ input_report_key(dev, BTN_START, data[2] & BIT(4));
+ input_report_key(dev, BTN_SELECT, data[2] & BIT(5));
/* stick press left/right */
- input_report_key(dev, BTN_THUMBL, data[2] & 0x40);
- input_report_key(dev, BTN_THUMBR, data[2] & 0x80);
+ input_report_key(dev, BTN_THUMBL, data[2] & BIT(6));
+ input_report_key(dev, BTN_THUMBR, data[2] & BIT(7));
/* buttons A,B,X,Y,TL,TR and MODE */
- input_report_key(dev, BTN_A, data[3] & 0x10);
- input_report_key(dev, BTN_B, data[3] & 0x20);
- input_report_key(dev, BTN_X, data[3] & 0x40);
- input_report_key(dev, BTN_Y, data[3] & 0x80);
- input_report_key(dev, BTN_TL, data[3] & 0x01);
- input_report_key(dev, BTN_TR, data[3] & 0x02);
- input_report_key(dev, BTN_MODE, data[3] & 0x04);
+ input_report_key(dev, BTN_A, data[3] & BIT(4));
+ input_report_key(dev, BTN_B, data[3] & BIT(5));
+ input_report_key(dev, BTN_X, data[3] & BIT(6));
+ input_report_key(dev, BTN_Y, data[3] & BIT(7));
+ input_report_key(dev, BTN_TL, data[3] & BIT(0));
+ input_report_key(dev, BTN_TR, data[3] & BIT(1));
+ input_report_key(dev, BTN_MODE, data[3] & BIT(2));
if (!(xpad->mapping & MAP_STICKS_TO_NULL)) {
/* left stick */
}
input_sync(dev);
+
+ /* XBOX360W controllers can't be turned off without driver assistance */
+ if (xpad->xtype == XTYPE_XBOX360W) {
+ if (xpad->mode_btn_down_ts > 0 && xpad->pad_present &&
+ ((ktime_get_seconds() - xpad->mode_btn_down_ts) >=
+ XPAD360W_POWEROFF_TIMEOUT)) {
+ xpad360w_poweroff_controller(xpad);
+ xpad->mode_btn_down_ts = 0;
+ return;
+ }
+
+ /* mode button down/up */
+ if (data[3] & BIT(2))
+ xpad->mode_btn_down_ts = ktime_get_seconds();
+ else
+ xpad->mode_btn_down_ts = 0;
+ }
}
static void xpad_presence_work(struct work_struct *work)
static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data)
{
struct input_dev *dev = xpad->dev;
+ bool do_sync = false;
/* the xbox button has its own special report */
- if (data[0] == 0X07) {
+ if (data[0] == GIP_CMD_VIRTUAL_KEY) {
/*
* The Xbox One S controller requires these reports to be
* acked otherwise it continues sending them forever and
* won't report further mode button events.
*/
- if (data[1] == 0x30)
+ if (data[1] == (GIP_OPT_ACK | GIP_OPT_INTERNAL))
xpadone_ack_mode_report(xpad, data[2]);
- input_report_key(dev, BTN_MODE, data[4] & 0x01);
+ input_report_key(dev, BTN_MODE, data[4] & GENMASK(1, 0));
input_sync(dev);
- return;
- }
- /* check invalid packet */
- else if (data[0] != 0X20)
- return;
- /* menu/view buttons */
- input_report_key(dev, BTN_START, data[4] & 0x04);
- input_report_key(dev, BTN_SELECT, data[4] & 0x08);
- if (xpad->mapping & MAP_SELECT_BUTTON)
- input_report_key(dev, KEY_RECORD, data[22] & 0x01);
+ do_sync = true;
+ } else if (data[0] == GIP_CMD_FIRMWARE) {
+ /* Some packet formats force us to use this separate to poll paddle inputs */
+ if (xpad->packet_type == PKT_XBE2_FW_5_11) {
+ /* Mute paddles if controller is in a custom profile slot
+ * Checked by looking at the active profile slot to
+ * verify it's the default slot
+ */
+ if (data[19] != 0)
+ data[18] = 0;
- /* buttons A,B,X,Y */
- input_report_key(dev, BTN_A, data[4] & 0x10);
- input_report_key(dev, BTN_B, data[4] & 0x20);
- input_report_key(dev, BTN_X, data[4] & 0x40);
- input_report_key(dev, BTN_Y, data[4] & 0x80);
+ /* Elite Series 2 split packet paddle bits */
+ input_report_key(dev, BTN_TRIGGER_HAPPY5, data[18] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY6, data[18] & BIT(1));
+ input_report_key(dev, BTN_TRIGGER_HAPPY7, data[18] & BIT(2));
+ input_report_key(dev, BTN_TRIGGER_HAPPY8, data[18] & BIT(3));
- /* digital pad */
- if (xpad->mapping & MAP_DPAD_TO_BUTTONS) {
- /* dpad as buttons (left, right, up, down) */
- input_report_key(dev, BTN_TRIGGER_HAPPY1, data[5] & 0x04);
- input_report_key(dev, BTN_TRIGGER_HAPPY2, data[5] & 0x08);
- input_report_key(dev, BTN_TRIGGER_HAPPY3, data[5] & 0x01);
- input_report_key(dev, BTN_TRIGGER_HAPPY4, data[5] & 0x02);
- } else {
- input_report_abs(dev, ABS_HAT0X,
- !!(data[5] & 0x08) - !!(data[5] & 0x04));
- input_report_abs(dev, ABS_HAT0Y,
- !!(data[5] & 0x02) - !!(data[5] & 0x01));
- }
-
- /* TL/TR */
- input_report_key(dev, BTN_TL, data[5] & 0x10);
- input_report_key(dev, BTN_TR, data[5] & 0x20);
+ do_sync = true;
+ }
+ } else if (data[0] == GIP_CMD_INPUT) { /* The main valid packet type for inputs */
+ /* menu/view buttons */
+ input_report_key(dev, BTN_START, data[4] & BIT(2));
+ input_report_key(dev, BTN_SELECT, data[4] & BIT(3));
+ if (xpad->mapping & MAP_SELECT_BUTTON)
+ input_report_key(dev, KEY_RECORD, data[22] & BIT(0));
+
+ /* buttons A,B,X,Y */
+ input_report_key(dev, BTN_A, data[4] & BIT(4));
+ input_report_key(dev, BTN_B, data[4] & BIT(5));
+ input_report_key(dev, BTN_X, data[4] & BIT(6));
+ input_report_key(dev, BTN_Y, data[4] & BIT(7));
+
+ /* digital pad */
+ if (xpad->mapping & MAP_DPAD_TO_BUTTONS) {
+ /* dpad as buttons (left, right, up, down) */
+ input_report_key(dev, BTN_TRIGGER_HAPPY1, data[5] & BIT(2));
+ input_report_key(dev, BTN_TRIGGER_HAPPY2, data[5] & BIT(3));
+ input_report_key(dev, BTN_TRIGGER_HAPPY3, data[5] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY4, data[5] & BIT(1));
+ } else {
+ input_report_abs(dev, ABS_HAT0X,
+ !!(data[5] & 0x08) - !!(data[5] & 0x04));
+ input_report_abs(dev, ABS_HAT0Y,
+ !!(data[5] & 0x02) - !!(data[5] & 0x01));
+ }
- /* stick press left/right */
- input_report_key(dev, BTN_THUMBL, data[5] & 0x40);
- input_report_key(dev, BTN_THUMBR, data[5] & 0x80);
+ /* TL/TR */
+ input_report_key(dev, BTN_TL, data[5] & BIT(4));
+ input_report_key(dev, BTN_TR, data[5] & BIT(5));
+
+ /* stick press left/right */
+ input_report_key(dev, BTN_THUMBL, data[5] & BIT(6));
+ input_report_key(dev, BTN_THUMBR, data[5] & BIT(7));
+
+ if (!(xpad->mapping & MAP_STICKS_TO_NULL)) {
+ /* left stick */
+ input_report_abs(dev, ABS_X,
+ (__s16) le16_to_cpup((__le16 *)(data + 10)));
+ input_report_abs(dev, ABS_Y,
+ ~(__s16) le16_to_cpup((__le16 *)(data + 12)));
+
+ /* right stick */
+ input_report_abs(dev, ABS_RX,
+ (__s16) le16_to_cpup((__le16 *)(data + 14)));
+ input_report_abs(dev, ABS_RY,
+ ~(__s16) le16_to_cpup((__le16 *)(data + 16)));
+ }
- if (!(xpad->mapping & MAP_STICKS_TO_NULL)) {
- /* left stick */
- input_report_abs(dev, ABS_X,
- (__s16) le16_to_cpup((__le16 *)(data + 10)));
- input_report_abs(dev, ABS_Y,
- ~(__s16) le16_to_cpup((__le16 *)(data + 12)));
+ /* triggers left/right */
+ if (xpad->mapping & MAP_TRIGGERS_TO_BUTTONS) {
+ input_report_key(dev, BTN_TL2,
+ (__u16) le16_to_cpup((__le16 *)(data + 6)));
+ input_report_key(dev, BTN_TR2,
+ (__u16) le16_to_cpup((__le16 *)(data + 8)));
+ } else {
+ input_report_abs(dev, ABS_Z,
+ (__u16) le16_to_cpup((__le16 *)(data + 6)));
+ input_report_abs(dev, ABS_RZ,
+ (__u16) le16_to_cpup((__le16 *)(data + 8)));
+ }
- /* right stick */
- input_report_abs(dev, ABS_RX,
- (__s16) le16_to_cpup((__le16 *)(data + 14)));
- input_report_abs(dev, ABS_RY,
- ~(__s16) le16_to_cpup((__le16 *)(data + 16)));
- }
+ /* paddle handling */
+ /* based on SDL's SDL_hidapi_xboxone.c */
+ if (xpad->mapping & MAP_PADDLES) {
+ if (xpad->packet_type == PKT_XBE1) {
+ /* Mute paddles if controller has a custom mapping applied.
+ * Checked by comparing the current mapping
+ * config against the factory mapping config
+ */
+ if (memcmp(&data[4], &data[18], 2) != 0)
+ data[32] = 0;
+
+ /* OG Elite Series Controller paddle bits */
+ input_report_key(dev, BTN_TRIGGER_HAPPY5, data[32] & BIT(1));
+ input_report_key(dev, BTN_TRIGGER_HAPPY6, data[32] & BIT(3));
+ input_report_key(dev, BTN_TRIGGER_HAPPY7, data[32] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY8, data[32] & BIT(2));
+ } else if (xpad->packet_type == PKT_XBE2_FW_OLD) {
+ /* Mute paddles if controller has a custom mapping applied.
+ * Checked by comparing the current mapping
+ * config against the factory mapping config
+ */
+ if (data[19] != 0)
+ data[18] = 0;
+
+ /* Elite Series 2 4.x firmware paddle bits */
+ input_report_key(dev, BTN_TRIGGER_HAPPY5, data[18] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY6, data[18] & BIT(1));
+ input_report_key(dev, BTN_TRIGGER_HAPPY7, data[18] & BIT(2));
+ input_report_key(dev, BTN_TRIGGER_HAPPY8, data[18] & BIT(3));
+ } else if (xpad->packet_type == PKT_XBE2_FW_5_EARLY) {
+ /* Mute paddles if controller has a custom mapping applied.
+ * Checked by comparing the current mapping
+ * config against the factory mapping config
+ */
+ if (data[23] != 0)
+ data[22] = 0;
+
+ /* Elite Series 2 5.x firmware paddle bits
+ * (before the packet was split)
+ */
+ input_report_key(dev, BTN_TRIGGER_HAPPY5, data[22] & BIT(0));
+ input_report_key(dev, BTN_TRIGGER_HAPPY6, data[22] & BIT(1));
+ input_report_key(dev, BTN_TRIGGER_HAPPY7, data[22] & BIT(2));
+ input_report_key(dev, BTN_TRIGGER_HAPPY8, data[22] & BIT(3));
+ }
+ }
- /* triggers left/right */
- if (xpad->mapping & MAP_TRIGGERS_TO_BUTTONS) {
- input_report_key(dev, BTN_TL2,
- (__u16) le16_to_cpup((__le16 *)(data + 6)));
- input_report_key(dev, BTN_TR2,
- (__u16) le16_to_cpup((__le16 *)(data + 8)));
- } else {
- input_report_abs(dev, ABS_Z,
- (__u16) le16_to_cpup((__le16 *)(data + 6)));
- input_report_abs(dev, ABS_RZ,
- (__u16) le16_to_cpup((__le16 *)(data + 8)));
+ do_sync = true;
}
- input_sync(dev);
+ if (do_sync)
+ input_sync(dev);
}
static void xpad_irq_in(struct urb *urb)
struct xpad_output_packet *packet =
&xpad->out_packets[XPAD_OUT_CMD_IDX];
static const u8 mode_report_ack[] = {
- 0x01, 0x20, 0x00, 0x09, 0x00, 0x07, 0x20, 0x02,
- 0x00, 0x00, 0x00, 0x00, 0x00
+ GIP_CMD_ACK, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(9),
+ 0x00, GIP_CMD_VIRTUAL_KEY, GIP_OPT_INTERNAL, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00
};
spin_lock_irqsave(&xpad->odata_lock, flags);
break;
case XTYPE_XBOXONE:
- packet->data[0] = 0x09; /* activate rumble */
+ packet->data[0] = GIP_CMD_RUMBLE; /* activate rumble */
packet->data[1] = 0x00;
packet->data[2] = xpad->odata_serial++;
- packet->data[3] = 0x09;
+ packet->data[3] = GIP_PL_LEN(9);
packet->data[4] = 0x00;
- packet->data[5] = 0x0F;
- packet->data[6] = 0x00;
- packet->data[7] = 0x00;
+ packet->data[5] = GIP_MOTOR_ALL;
+ packet->data[6] = 0x00; /* left trigger */
+ packet->data[7] = 0x00; /* right trigger */
packet->data[8] = strong / 512; /* left actuator */
packet->data[9] = weak / 512; /* right actuator */
packet->data[10] = 0xFF; /* on period */
case ABS_HAT0Y: /* the d-pad (only if dpad is mapped to axes */
input_set_abs_params(input_dev, abs, -1, 1, 0, 0);
break;
+ case ABS_PROFILE: /* 4 value profile button (such as on XAC) */
+ input_set_abs_params(input_dev, abs, 0, 4, 0, 0);
+ break;
default:
input_set_abs_params(input_dev, abs, 0, 0, 0, 0);
break;
xpad_btn_pad[i]);
}
+ /* set up paddles if the controller has them */
+ if (xpad->mapping & MAP_PADDLES) {
+ for (i = 0; xpad_btn_paddles[i] >= 0; i++)
+ input_set_capability(input_dev, EV_KEY, xpad_btn_paddles[i]);
+ }
+
/*
* This should be a simple else block. However historically
* xbox360w has mapped DPAD to buttons while xbox360 did not. This
xpad_set_up_abs(input_dev, xpad_abs_triggers[i]);
}
+ /* setup profile button as an axis with 4 possible values */
+ if (xpad->mapping & MAP_PROFILE_BUTTON)
+ xpad_set_up_abs(input_dev, ABS_PROFILE);
+
error = xpad_init_ff(xpad);
if (error)
goto err_free_input;
xpad->mapping = xpad_device[i].mapping;
xpad->xtype = xpad_device[i].xtype;
xpad->name = xpad_device[i].name;
+ xpad->packet_type = PKT_XB;
INIT_WORK(&xpad->work, xpad_presence_work);
if (xpad->xtype == XTYPE_UNKNOWN) {
usb_set_intfdata(intf, xpad);
+ /* Packet type detection */
+ if (le16_to_cpu(udev->descriptor.idVendor) == 0x045e) { /* Microsoft controllers */
+ if (le16_to_cpu(udev->descriptor.idProduct) == 0x02e3) {
+ /* The original elite controller always uses the oldest
+ * type of extended packet
+ */
+ xpad->packet_type = PKT_XBE1;
+ } else if (le16_to_cpu(udev->descriptor.idProduct) == 0x0b00) {
+ /* The elite 2 controller has seen multiple packet
+ * revisions. These are tied to specific firmware
+ * versions
+ */
+ if (le16_to_cpu(udev->descriptor.bcdDevice) < 0x0500) {
+ /* This is the format that the Elite 2 used
+ * prior to the BLE update
+ */
+ xpad->packet_type = PKT_XBE2_FW_OLD;
+ } else if (le16_to_cpu(udev->descriptor.bcdDevice) <
+ 0x050b) {
+ /* This is the format that the Elite 2 used
+ * prior to the update that split the packet
+ */
+ xpad->packet_type = PKT_XBE2_FW_5_EARLY;
+ } else {
+ /* The split packet format that was introduced
+ * in firmware v5.11
+ */
+ xpad->packet_type = PKT_XBE2_FW_5_11;
+ }
+ }
+ }
+
if (xpad->xtype == XTYPE_XBOX360W) {
/*
* Submit the int URB immediately rather than waiting for open
.disconnect = xpad_disconnect,
.suspend = xpad_suspend,
.resume = xpad_resume,
- .reset_resume = xpad_resume,
.id_table = xpad_table,
};
* coder :-(
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
config KEYBOARD_ADP5588
tristate "ADP5588/87 I2C QWERTY Keypad and IO Expander"
depends on I2C
+ select GPIOLIB
+ select GPIOLIB_IRQCHIP
+ select INPUT_MATRIXKMAP
help
Say Y here if you want to use a ADP5588/87 attached to your
system I2C bus.
config KEYBOARD_CLPS711X
tristate "CLPS711X Keypad support"
- depends on OF_GPIO && (ARCH_CLPS711X || COMPILE_TEST)
+ depends on ARCH_CLPS711X || COMPILE_TEST
select INPUT_MATRIXKMAP
help
Say Y here to enable the matrix keypad on the Cirrus Logic
To compile this driver as a module, choose M here; the
module will be called opencores-kbd.
+config KEYBOARD_PINEPHONE
+ tristate "Pine64 PinePhone Keyboard"
+ depends on I2C && REGULATOR
+ select CRC8
+ select INPUT_MATRIXKMAP
+ help
+ Say Y here to enable support for the keyboard in the Pine64 PinePhone
+ keyboard case. This driver supports the FLOSS firmware available at
+ https://megous.com/git/pinephone-keyboard/
+
+ To compile this driver as a module, choose M here; the
+ module will be called pinephone-keyboard.
+
config KEYBOARD_PXA27x
tristate "PXA27x/PXA3xx keypad support"
depends on PXA27x || PXA3xx || ARCH_MMP
obj-$(CONFIG_KEYBOARD_OMAP) += omap-keypad.o
obj-$(CONFIG_KEYBOARD_OMAP4) += omap4-keypad.o
obj-$(CONFIG_KEYBOARD_OPENCORES) += opencores-kbd.o
+obj-$(CONFIG_KEYBOARD_PINEPHONE) += pinephone-keyboard.o
obj-$(CONFIG_KEYBOARD_PMIC8XXX) += pmic8xxx-keypad.o
obj-$(CONFIG_KEYBOARD_PXA27x) += pxa27x_keypad.o
obj-$(CONFIG_KEYBOARD_PXA930_ROTARY) += pxa930_rotary.o
* Copyright (C) 2008-2010 Analog Devices Inc.
*/
+#include <linux/bits.h>
#include <linux/delay.h>
#include <linux/errno.h>
+#include <linux/gpio/consumer.h>
#include <linux/gpio/driver.h>
#include <linux/i2c.h>
#include <linux/input.h>
+#include <linux/input/matrix_keypad.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/ktime.h>
#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/pinctrl/pinconf-generic.h>
#include <linux/platform_device.h>
#include <linux/pm.h>
+#include <linux/regulator/consumer.h>
#include <linux/slab.h>
#include <linux/timekeeping.h>
-#include <linux/platform_data/adp5588.h>
+#define DEV_ID 0x00 /* Device ID */
+#define CFG 0x01 /* Configuration Register1 */
+#define INT_STAT 0x02 /* Interrupt Status Register */
+#define KEY_LCK_EC_STAT 0x03 /* Key Lock and Event Counter Register */
+#define KEY_EVENTA 0x04 /* Key Event Register A */
+#define KEY_EVENTB 0x05 /* Key Event Register B */
+#define KEY_EVENTC 0x06 /* Key Event Register C */
+#define KEY_EVENTD 0x07 /* Key Event Register D */
+#define KEY_EVENTE 0x08 /* Key Event Register E */
+#define KEY_EVENTF 0x09 /* Key Event Register F */
+#define KEY_EVENTG 0x0A /* Key Event Register G */
+#define KEY_EVENTH 0x0B /* Key Event Register H */
+#define KEY_EVENTI 0x0C /* Key Event Register I */
+#define KEY_EVENTJ 0x0D /* Key Event Register J */
+#define KP_LCK_TMR 0x0E /* Keypad Lock1 to Lock2 Timer */
+#define UNLOCK1 0x0F /* Unlock Key1 */
+#define UNLOCK2 0x10 /* Unlock Key2 */
+#define GPIO_INT_STAT1 0x11 /* GPIO Interrupt Status */
+#define GPIO_INT_STAT2 0x12 /* GPIO Interrupt Status */
+#define GPIO_INT_STAT3 0x13 /* GPIO Interrupt Status */
+#define GPIO_DAT_STAT1 0x14 /* GPIO Data Status, Read twice to clear */
+#define GPIO_DAT_STAT2 0x15 /* GPIO Data Status, Read twice to clear */
+#define GPIO_DAT_STAT3 0x16 /* GPIO Data Status, Read twice to clear */
+#define GPIO_DAT_OUT1 0x17 /* GPIO DATA OUT */
+#define GPIO_DAT_OUT2 0x18 /* GPIO DATA OUT */
+#define GPIO_DAT_OUT3 0x19 /* GPIO DATA OUT */
+#define GPIO_INT_EN1 0x1A /* GPIO Interrupt Enable */
+#define GPIO_INT_EN2 0x1B /* GPIO Interrupt Enable */
+#define GPIO_INT_EN3 0x1C /* GPIO Interrupt Enable */
+#define KP_GPIO1 0x1D /* Keypad or GPIO Selection */
+#define KP_GPIO2 0x1E /* Keypad or GPIO Selection */
+#define KP_GPIO3 0x1F /* Keypad or GPIO Selection */
+#define GPI_EM1 0x20 /* GPI Event Mode 1 */
+#define GPI_EM2 0x21 /* GPI Event Mode 2 */
+#define GPI_EM3 0x22 /* GPI Event Mode 3 */
+#define GPIO_DIR1 0x23 /* GPIO Data Direction */
+#define GPIO_DIR2 0x24 /* GPIO Data Direction */
+#define GPIO_DIR3 0x25 /* GPIO Data Direction */
+#define GPIO_INT_LVL1 0x26 /* GPIO Edge/Level Detect */
+#define GPIO_INT_LVL2 0x27 /* GPIO Edge/Level Detect */
+#define GPIO_INT_LVL3 0x28 /* GPIO Edge/Level Detect */
+#define DEBOUNCE_DIS1 0x29 /* Debounce Disable */
+#define DEBOUNCE_DIS2 0x2A /* Debounce Disable */
+#define DEBOUNCE_DIS3 0x2B /* Debounce Disable */
+#define GPIO_PULL1 0x2C /* GPIO Pull Disable */
+#define GPIO_PULL2 0x2D /* GPIO Pull Disable */
+#define GPIO_PULL3 0x2E /* GPIO Pull Disable */
+#define CMP_CFG_STAT 0x30 /* Comparator Configuration and Status Register */
+#define CMP_CONFG_SENS1 0x31 /* Sensor1 Comparator Configuration Register */
+#define CMP_CONFG_SENS2 0x32 /* L2 Light Sensor Reference Level, Output Falling for Sensor 1 */
+#define CMP1_LVL2_TRIP 0x33 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 1 */
+#define CMP1_LVL2_HYS 0x34 /* L3 Light Sensor Reference Level, Output Falling For Sensor 1 */
+#define CMP1_LVL3_TRIP 0x35 /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 1 */
+#define CMP1_LVL3_HYS 0x36 /* Sensor 2 Comparator Configuration Register */
+#define CMP2_LVL2_TRIP 0x37 /* L2 Light Sensor Reference Level, Output Falling for Sensor 2 */
+#define CMP2_LVL2_HYS 0x38 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 2 */
+#define CMP2_LVL3_TRIP 0x39 /* L3 Light Sensor Reference Level, Output Falling For Sensor 2 */
+#define CMP2_LVL3_HYS 0x3A /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 2 */
+#define CMP1_ADC_DAT_R1 0x3B /* Comparator 1 ADC data Register1 */
+#define CMP1_ADC_DAT_R2 0x3C /* Comparator 1 ADC data Register2 */
+#define CMP2_ADC_DAT_R1 0x3D /* Comparator 2 ADC data Register1 */
+#define CMP2_ADC_DAT_R2 0x3E /* Comparator 2 ADC data Register2 */
+
+#define ADP5588_DEVICE_ID_MASK 0xF
+
+ /* Configuration Register1 */
+#define ADP5588_AUTO_INC BIT(7)
+#define ADP5588_GPIEM_CFG BIT(6)
+#define ADP5588_OVR_FLOW_M BIT(5)
+#define ADP5588_INT_CFG BIT(4)
+#define ADP5588_OVR_FLOW_IEN BIT(3)
+#define ADP5588_K_LCK_IM BIT(2)
+#define ADP5588_GPI_IEN BIT(1)
+#define ADP5588_KE_IEN BIT(0)
+
+/* Interrupt Status Register */
+#define ADP5588_CMP2_INT BIT(5)
+#define ADP5588_CMP1_INT BIT(4)
+#define ADP5588_OVR_FLOW_INT BIT(3)
+#define ADP5588_K_LCK_INT BIT(2)
+#define ADP5588_GPI_INT BIT(1)
+#define ADP5588_KE_INT BIT(0)
+
+/* Key Lock and Event Counter Register */
+#define ADP5588_K_LCK_EN BIT(6)
+#define ADP5588_LCK21 0x30
+#define ADP5588_KEC GENMASK(3, 0)
+
+#define ADP5588_MAXGPIO 18
+#define ADP5588_BANK(offs) ((offs) >> 3)
+#define ADP5588_BIT(offs) (1u << ((offs) & 0x7))
+
+/* Put one of these structures in i2c_board_info platform_data */
+
+/*
+ * 128 so it fits matrix-keymap maximum number of keys when the full
+ * 10cols * 8rows are used.
+ */
+#define ADP5588_KEYMAPSIZE 128
+
+#define GPI_PIN_ROW0 97
+#define GPI_PIN_ROW1 98
+#define GPI_PIN_ROW2 99
+#define GPI_PIN_ROW3 100
+#define GPI_PIN_ROW4 101
+#define GPI_PIN_ROW5 102
+#define GPI_PIN_ROW6 103
+#define GPI_PIN_ROW7 104
+#define GPI_PIN_COL0 105
+#define GPI_PIN_COL1 106
+#define GPI_PIN_COL2 107
+#define GPI_PIN_COL3 108
+#define GPI_PIN_COL4 109
+#define GPI_PIN_COL5 110
+#define GPI_PIN_COL6 111
+#define GPI_PIN_COL7 112
+#define GPI_PIN_COL8 113
+#define GPI_PIN_COL9 114
+
+#define GPI_PIN_ROW_BASE GPI_PIN_ROW0
+#define GPI_PIN_ROW_END GPI_PIN_ROW7
+#define GPI_PIN_COL_BASE GPI_PIN_COL0
+#define GPI_PIN_COL_END GPI_PIN_COL9
+
+#define GPI_PIN_BASE GPI_PIN_ROW_BASE
+#define GPI_PIN_END GPI_PIN_COL_END
+
+#define ADP5588_ROWS_MAX (GPI_PIN_ROW7 - GPI_PIN_ROW0 + 1)
+#define ADP5588_COLS_MAX (GPI_PIN_COL9 - GPI_PIN_COL0 + 1)
+
+#define ADP5588_GPIMAPSIZE_MAX (GPI_PIN_END - GPI_PIN_BASE + 1)
/* Key Event Register xy */
-#define KEY_EV_PRESSED (1 << 7)
-#define KEY_EV_MASK (0x7F)
+#define KEY_EV_PRESSED BIT(7)
+#define KEY_EV_MASK GENMASK(6, 0)
-#define KP_SEL(x) (0xFFFF >> (16 - x)) /* 2^x-1 */
+#define KP_SEL(x) (BIT(x) - 1) /* 2^x-1 */
#define KEYP_MAX_EVENT 10
#define WA_DELAYED_READOUT_REVID(rev) ((rev) < 4)
#define WA_DELAYED_READOUT_TIME 25
+#define ADP5588_INVALID_HWIRQ (~0UL)
+
struct adp5588_kpad {
struct i2c_client *client;
struct input_dev *input;
ktime_t irq_time;
unsigned long delay;
+ u32 row_shift;
+ u32 rows;
+ u32 cols;
+ u32 unlock_keys[2];
+ int nkeys_unlock;
unsigned short keycode[ADP5588_KEYMAPSIZE];
- const struct adp5588_gpi_map *gpimap;
- unsigned short gpimapsize;
-#ifdef CONFIG_GPIOLIB
unsigned char gpiomap[ADP5588_MAXGPIO];
struct gpio_chip gc;
struct mutex gpio_lock; /* Protect cached dir, dat_out */
u8 dat_out[3];
u8 dir[3];
-#endif
+ u8 int_en[3];
+ u8 irq_mask[3];
+ u8 pull_dis[3];
};
static int adp5588_read(struct i2c_client *client, u8 reg)
return i2c_smbus_write_byte_data(client, reg, val);
}
-#ifdef CONFIG_GPIOLIB
-static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned off)
+static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned int off)
{
struct adp5588_kpad *kpad = gpiochip_get_data(chip);
unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]);
}
static void adp5588_gpio_set_value(struct gpio_chip *chip,
- unsigned off, int val)
+ unsigned int off, int val)
{
struct adp5588_kpad *kpad = gpiochip_get_data(chip);
unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]);
else
kpad->dat_out[bank] &= ~bit;
- adp5588_write(kpad->client, GPIO_DAT_OUT1 + bank,
- kpad->dat_out[bank]);
+ adp5588_write(kpad->client, GPIO_DAT_OUT1 + bank, kpad->dat_out[bank]);
mutex_unlock(&kpad->gpio_lock);
}
-static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned off)
+static int adp5588_gpio_set_config(struct gpio_chip *chip, unsigned int off,
+ unsigned long config)
+{
+ struct adp5588_kpad *kpad = gpiochip_get_data(chip);
+ unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]);
+ unsigned int bit = ADP5588_BIT(kpad->gpiomap[off]);
+ bool pull_disable;
+ int ret;
+
+ switch (pinconf_to_config_param(config)) {
+ case PIN_CONFIG_BIAS_PULL_UP:
+ pull_disable = false;
+ break;
+ case PIN_CONFIG_BIAS_DISABLE:
+ pull_disable = true;
+ break;
+ default:
+ return -ENOTSUPP;
+ }
+
+ mutex_lock(&kpad->gpio_lock);
+
+ if (pull_disable)
+ kpad->pull_dis[bank] |= bit;
+ else
+ kpad->pull_dis[bank] &= bit;
+
+ ret = adp5588_write(kpad->client, GPIO_PULL1 + bank,
+ kpad->pull_dis[bank]);
+
+ mutex_unlock(&kpad->gpio_lock);
+
+ return ret;
+}
+
+static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned int off)
{
struct adp5588_kpad *kpad = gpiochip_get_data(chip);
unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]);
}
static int adp5588_gpio_direction_output(struct gpio_chip *chip,
- unsigned off, int val)
+ unsigned int off, int val)
{
struct adp5588_kpad *kpad = gpiochip_get_data(chip);
unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]);
kpad->dat_out[bank] &= ~bit;
ret = adp5588_write(kpad->client, GPIO_DAT_OUT1 + bank,
- kpad->dat_out[bank]);
- ret |= adp5588_write(kpad->client, GPIO_DIR1 + bank,
- kpad->dir[bank]);
+ kpad->dat_out[bank]);
+ if (ret)
+ goto out_unlock;
+
+ ret = adp5588_write(kpad->client, GPIO_DIR1 + bank, kpad->dir[bank]);
+out_unlock:
mutex_unlock(&kpad->gpio_lock);
return ret;
}
-static int adp5588_build_gpiomap(struct adp5588_kpad *kpad,
- const struct adp5588_kpad_platform_data *pdata)
+static int adp5588_build_gpiomap(struct adp5588_kpad *kpad)
{
bool pin_used[ADP5588_MAXGPIO];
int n_unused = 0;
memset(pin_used, 0, sizeof(pin_used));
- for (i = 0; i < pdata->rows; i++)
+ for (i = 0; i < kpad->rows; i++)
pin_used[i] = true;
- for (i = 0; i < pdata->cols; i++)
+ for (i = 0; i < kpad->cols; i++)
pin_used[i + GPI_PIN_COL_BASE - GPI_PIN_BASE] = true;
- for (i = 0; i < kpad->gpimapsize; i++)
- pin_used[kpad->gpimap[i].pin - GPI_PIN_BASE] = true;
-
for (i = 0; i < ADP5588_MAXGPIO; i++)
if (!pin_used[i])
kpad->gpiomap[n_unused++] = i;
return n_unused;
}
-static void adp5588_gpio_do_teardown(void *_kpad)
+static void adp5588_irq_bus_lock(struct irq_data *d)
{
- struct adp5588_kpad *kpad = _kpad;
- struct device *dev = &kpad->client->dev;
- const struct adp5588_kpad_platform_data *pdata = dev_get_platdata(dev);
- const struct adp5588_gpio_platform_data *gpio_data = pdata->gpio_data;
- int error;
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct adp5588_kpad *kpad = gpiochip_get_data(gc);
- error = gpio_data->teardown(kpad->client,
- kpad->gc.base, kpad->gc.ngpio,
- gpio_data->context);
- if (error)
- dev_warn(&kpad->client->dev, "teardown failed %d\n", error);
+ mutex_lock(&kpad->gpio_lock);
}
+static void adp5588_irq_bus_sync_unlock(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct adp5588_kpad *kpad = gpiochip_get_data(gc);
+ int i;
+
+ for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) {
+ if (kpad->int_en[i] ^ kpad->irq_mask[i]) {
+ kpad->int_en[i] = kpad->irq_mask[i];
+ adp5588_write(kpad->client, GPI_EM1 + i, kpad->int_en[i]);
+ }
+ }
+
+ mutex_unlock(&kpad->gpio_lock);
+}
+
+static void adp5588_irq_mask(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct adp5588_kpad *kpad = gpiochip_get_data(gc);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ unsigned long real_irq = kpad->gpiomap[hwirq];
+
+ kpad->irq_mask[ADP5588_BANK(real_irq)] &= ~ADP5588_BIT(real_irq);
+ gpiochip_disable_irq(gc, hwirq);
+}
+
+static void adp5588_irq_unmask(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct adp5588_kpad *kpad = gpiochip_get_data(gc);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ unsigned long real_irq = kpad->gpiomap[hwirq];
+
+ gpiochip_enable_irq(gc, hwirq);
+ kpad->irq_mask[ADP5588_BANK(real_irq)] |= ADP5588_BIT(real_irq);
+}
+
+static int adp5588_irq_set_type(struct irq_data *d, unsigned int type)
+{
+ if (!(type & IRQ_TYPE_EDGE_BOTH))
+ return -EINVAL;
+
+ irq_set_handler_locked(d, handle_edge_irq);
+
+ return 0;
+}
+
+static const struct irq_chip adp5588_irq_chip = {
+ .name = "adp5588",
+ .irq_mask = adp5588_irq_mask,
+ .irq_unmask = adp5588_irq_unmask,
+ .irq_bus_lock = adp5588_irq_bus_lock,
+ .irq_bus_sync_unlock = adp5588_irq_bus_sync_unlock,
+ .irq_set_type = adp5588_irq_set_type,
+ .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_IMMUTABLE,
+ GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
static int adp5588_gpio_add(struct adp5588_kpad *kpad)
{
struct device *dev = &kpad->client->dev;
- const struct adp5588_kpad_platform_data *pdata = dev_get_platdata(dev);
- const struct adp5588_gpio_platform_data *gpio_data = pdata->gpio_data;
+ struct gpio_irq_chip *girq;
int i, error;
- if (!gpio_data)
- return 0;
-
- kpad->gc.ngpio = adp5588_build_gpiomap(kpad, pdata);
+ kpad->gc.ngpio = adp5588_build_gpiomap(kpad);
if (kpad->gc.ngpio == 0) {
dev_info(dev, "No unused gpios left to export\n");
return 0;
}
+ kpad->gc.parent = &kpad->client->dev;
kpad->gc.direction_input = adp5588_gpio_direction_input;
kpad->gc.direction_output = adp5588_gpio_direction_output;
kpad->gc.get = adp5588_gpio_get_value;
kpad->gc.set = adp5588_gpio_set_value;
+ kpad->gc.set_config = adp5588_gpio_set_config;
kpad->gc.can_sleep = 1;
- kpad->gc.base = gpio_data->gpio_start;
+ kpad->gc.base = -1;
kpad->gc.label = kpad->client->name;
kpad->gc.owner = THIS_MODULE;
- kpad->gc.names = gpio_data->names;
+
+ girq = &kpad->gc.irq;
+ gpio_irq_chip_set_chip(girq, &adp5588_irq_chip);
+ girq->handler = handle_bad_irq;
+ girq->threaded = true;
mutex_init(&kpad->gpio_lock);
kpad->dat_out[i] = adp5588_read(kpad->client,
GPIO_DAT_OUT1 + i);
kpad->dir[i] = adp5588_read(kpad->client, GPIO_DIR1 + i);
+ kpad->pull_dis[i] = adp5588_read(kpad->client, GPIO_PULL1 + i);
}
- if (gpio_data->setup) {
- error = gpio_data->setup(kpad->client,
- kpad->gc.base, kpad->gc.ngpio,
- gpio_data->context);
- if (error)
- dev_warn(dev, "setup failed: %d\n", error);
- }
+ return 0;
+}
- if (gpio_data->teardown) {
- error = devm_add_action(dev, adp5588_gpio_do_teardown, kpad);
- if (error)
- dev_warn(dev, "failed to schedule teardown: %d\n",
- error);
- }
+static unsigned long adp5588_gpiomap_get_hwirq(struct device *dev,
+ const u8 *map, unsigned int gpio,
+ unsigned int ngpios)
+{
+ unsigned int hwirq;
- return 0;
+ for (hwirq = 0; hwirq < ngpios; hwirq++)
+ if (map[hwirq] == gpio)
+ return hwirq;
+
+ /* should never happen */
+ dev_warn_ratelimited(dev, "could not find the hwirq for gpio(%u)\n", gpio);
+
+ return ADP5588_INVALID_HWIRQ;
}
-#else
-static inline int adp5588_gpio_add(struct adp5588_kpad *kpad)
+static void adp5588_gpio_irq_handle(struct adp5588_kpad *kpad, int key_val,
+ int key_press)
{
- return 0;
+ unsigned int irq, gpio = key_val - GPI_PIN_BASE, irq_type;
+ struct i2c_client *client = kpad->client;
+ struct irq_data *irqd;
+ unsigned long hwirq;
+
+ hwirq = adp5588_gpiomap_get_hwirq(&client->dev, kpad->gpiomap,
+ gpio, kpad->gc.ngpio);
+ if (hwirq == ADP5588_INVALID_HWIRQ) {
+ dev_err(&client->dev, "Could not get hwirq for key(%u)\n", key_val);
+ return;
+ }
+
+ irq = irq_find_mapping(kpad->gc.irq.domain, hwirq);
+ if (!irq)
+ return;
+
+ irqd = irq_get_irq_data(irq);
+ if (!irqd) {
+ dev_err(&client->dev, "Could not get irq(%u) data\n", irq);
+ return;
+ }
+
+ irq_type = irqd_get_trigger_type(irqd);
+
+ /*
+ * Default is active low which means key_press is asserted on
+ * the falling edge.
+ */
+ if ((irq_type & IRQ_TYPE_EDGE_RISING && !key_press) ||
+ (irq_type & IRQ_TYPE_EDGE_FALLING && key_press))
+ handle_nested_irq(irq);
}
-#endif
static void adp5588_report_events(struct adp5588_kpad *kpad, int ev_cnt)
{
- int i, j;
+ int i;
for (i = 0; i < ev_cnt; i++) {
- int key = adp5588_read(kpad->client, Key_EVENTA + i);
+ int key = adp5588_read(kpad->client, KEY_EVENTA + i);
int key_val = key & KEY_EV_MASK;
+ int key_press = key & KEY_EV_PRESSED;
if (key_val >= GPI_PIN_BASE && key_val <= GPI_PIN_END) {
- for (j = 0; j < kpad->gpimapsize; j++) {
- if (key_val == kpad->gpimap[j].pin) {
- input_report_switch(kpad->input,
- kpad->gpimap[j].sw_evt,
- key & KEY_EV_PRESSED);
- break;
- }
- }
+ /* gpio line used as IRQ source */
+ adp5588_gpio_irq_handle(kpad, key_val, key_press);
} else {
+ int row = (key_val - 1) / ADP5588_COLS_MAX;
+ int col = (key_val - 1) % ADP5588_COLS_MAX;
+ int code = MATRIX_SCAN_CODE(row, col, kpad->row_shift);
+
+ dev_dbg_ratelimited(&kpad->client->dev,
+ "report key(%d) r(%d) c(%d) code(%d)\n",
+ key_val, row, col, kpad->keycode[code]);
+
input_report_key(kpad->input,
- kpad->keycode[key_val - 1],
- key & KEY_EV_PRESSED);
+ kpad->keycode[code], key_press);
}
}
}
return IRQ_HANDLED;
}
-static int adp5588_setup(struct i2c_client *client)
+static int adp5588_setup(struct adp5588_kpad *kpad)
{
- const struct adp5588_kpad_platform_data *pdata =
- dev_get_platdata(&client->dev);
- const struct adp5588_gpio_platform_data *gpio_data = pdata->gpio_data;
+ struct i2c_client *client = kpad->client;
int i, ret;
- unsigned char evt_mode1 = 0, evt_mode2 = 0, evt_mode3 = 0;
-
- ret = adp5588_write(client, KP_GPIO1, KP_SEL(pdata->rows));
- ret |= adp5588_write(client, KP_GPIO2, KP_SEL(pdata->cols) & 0xFF);
- ret |= adp5588_write(client, KP_GPIO3, KP_SEL(pdata->cols) >> 8);
- if (pdata->en_keylock) {
- ret |= adp5588_write(client, UNLOCK1, pdata->unlock_key1);
- ret |= adp5588_write(client, UNLOCK2, pdata->unlock_key2);
- ret |= adp5588_write(client, KEY_LCK_EC_STAT, ADP5588_K_LCK_EN);
- }
+ ret = adp5588_write(client, KP_GPIO1, KP_SEL(kpad->rows));
+ if (ret)
+ return ret;
- for (i = 0; i < KEYP_MAX_EVENT; i++)
- ret |= adp5588_read(client, Key_EVENTA);
+ ret = adp5588_write(client, KP_GPIO2, KP_SEL(kpad->cols) & 0xFF);
+ if (ret)
+ return ret;
- for (i = 0; i < pdata->gpimapsize; i++) {
- unsigned short pin = pdata->gpimap[i].pin;
+ ret = adp5588_write(client, KP_GPIO3, KP_SEL(kpad->cols) >> 8);
+ if (ret)
+ return ret;
- if (pin <= GPI_PIN_ROW_END) {
- evt_mode1 |= (1 << (pin - GPI_PIN_ROW_BASE));
- } else {
- evt_mode2 |= ((1 << (pin - GPI_PIN_COL_BASE)) & 0xFF);
- evt_mode3 |= ((1 << (pin - GPI_PIN_COL_BASE)) >> 8);
- }
+ for (i = 0; i < kpad->nkeys_unlock; i++) {
+ ret = adp5588_write(client, UNLOCK1 + i, kpad->unlock_keys[i]);
+ if (ret)
+ return ret;
}
- if (pdata->gpimapsize) {
- ret |= adp5588_write(client, GPI_EM1, evt_mode1);
- ret |= adp5588_write(client, GPI_EM2, evt_mode2);
- ret |= adp5588_write(client, GPI_EM3, evt_mode3);
+ if (kpad->nkeys_unlock) {
+ ret = adp5588_write(client, KEY_LCK_EC_STAT, ADP5588_K_LCK_EN);
+ if (ret)
+ return ret;
}
- if (gpio_data) {
- for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) {
- int pull_mask = gpio_data->pullup_dis_mask;
-
- ret |= adp5588_write(client, GPIO_PULL1 + i,
- (pull_mask >> (8 * i)) & 0xFF);
- }
+ for (i = 0; i < KEYP_MAX_EVENT; i++) {
+ ret = adp5588_read(client, KEY_EVENTA);
+ if (ret)
+ return ret;
}
- ret |= adp5588_write(client, INT_STAT,
- ADP5588_CMP2_INT | ADP5588_CMP1_INT |
- ADP5588_OVR_FLOW_INT | ADP5588_K_LCK_INT |
- ADP5588_GPI_INT | ADP5588_KE_INT); /* Status is W1C */
+ ret = adp5588_write(client, INT_STAT,
+ ADP5588_CMP2_INT | ADP5588_CMP1_INT |
+ ADP5588_OVR_FLOW_INT | ADP5588_K_LCK_INT |
+ ADP5588_GPI_INT | ADP5588_KE_INT); /* Status is W1C */
+ if (ret)
+ return ret;
- ret |= adp5588_write(client, CFG, ADP5588_INT_CFG |
- ADP5588_OVR_FLOW_IEN |
- ADP5588_KE_IEN);
+ return adp5588_write(client, CFG, ADP5588_INT_CFG |
+ ADP5588_OVR_FLOW_IEN | ADP5588_KE_IEN);
+}
+
+static int adp5588_fw_parse(struct adp5588_kpad *kpad)
+{
+ struct i2c_client *client = kpad->client;
+ int ret, i;
- if (ret < 0) {
- dev_err(&client->dev, "Write Error\n");
+ ret = matrix_keypad_parse_properties(&client->dev, &kpad->rows,
+ &kpad->cols);
+ if (ret)
return ret;
+
+ if (kpad->rows > ADP5588_ROWS_MAX || kpad->cols > ADP5588_COLS_MAX) {
+ dev_err(&client->dev, "Invalid nr of rows(%u) or cols(%u)\n",
+ kpad->rows, kpad->cols);
+ return -EINVAL;
}
- return 0;
-}
+ ret = matrix_keypad_build_keymap(NULL, NULL, kpad->rows, kpad->cols,
+ kpad->keycode, kpad->input);
+ if (ret)
+ return ret;
-static void adp5588_report_switch_state(struct adp5588_kpad *kpad)
-{
- int gpi_stat1 = adp5588_read(kpad->client, GPIO_DAT_STAT1);
- int gpi_stat2 = adp5588_read(kpad->client, GPIO_DAT_STAT2);
- int gpi_stat3 = adp5588_read(kpad->client, GPIO_DAT_STAT3);
- int gpi_stat_tmp, pin_loc;
- int i;
+ kpad->row_shift = get_count_order(kpad->cols);
- for (i = 0; i < kpad->gpimapsize; i++) {
- unsigned short pin = kpad->gpimap[i].pin;
+ if (device_property_read_bool(&client->dev, "autorepeat"))
+ __set_bit(EV_REP, kpad->input->evbit);
- if (pin <= GPI_PIN_ROW_END) {
- gpi_stat_tmp = gpi_stat1;
- pin_loc = pin - GPI_PIN_ROW_BASE;
- } else if ((pin - GPI_PIN_COL_BASE) < 8) {
- gpi_stat_tmp = gpi_stat2;
- pin_loc = pin - GPI_PIN_COL_BASE;
- } else {
- gpi_stat_tmp = gpi_stat3;
- pin_loc = pin - GPI_PIN_COL_BASE - 8;
- }
+ kpad->nkeys_unlock = device_property_count_u32(&client->dev,
+ "adi,unlock-keys");
+ if (kpad->nkeys_unlock <= 0) {
+ /* so that we don't end up enabling key lock */
+ kpad->nkeys_unlock = 0;
+ return 0;
+ }
- if (gpi_stat_tmp < 0) {
- dev_err(&kpad->client->dev,
- "Can't read GPIO_DAT_STAT switch %d default to OFF\n",
- pin);
- gpi_stat_tmp = 0;
+ if (kpad->nkeys_unlock > ARRAY_SIZE(kpad->unlock_keys)) {
+ dev_err(&client->dev, "number of unlock keys(%d) > (%zu)\n",
+ kpad->nkeys_unlock, ARRAY_SIZE(kpad->unlock_keys));
+ return -EINVAL;
+ }
+
+ ret = device_property_read_u32_array(&client->dev, "adi,unlock-keys",
+ kpad->unlock_keys,
+ kpad->nkeys_unlock);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < kpad->nkeys_unlock; i++) {
+ /*
+ * Even though it should be possible (as stated in the datasheet)
+ * to use GPIs (which are part of the keys event) as unlock keys,
+ * it was not working at all and was leading to overflow events
+ * at some point. Hence, for now, let's just allow keys which are
+ * part of keypad matrix to be used and if a reliable way of
+ * using GPIs is found, this condition can be removed/lightened.
+ */
+ if (kpad->unlock_keys[i] >= kpad->cols * kpad->rows) {
+ dev_err(&client->dev, "Invalid unlock key(%d)\n",
+ kpad->unlock_keys[i]);
+ return -EINVAL;
}
- input_report_switch(kpad->input,
- kpad->gpimap[i].sw_evt,
- !(gpi_stat_tmp & (1 << pin_loc)));
+ /*
+ * Firmware properties keys start from 0 but on the device they
+ * start from 1.
+ */
+ kpad->unlock_keys[i] += 1;
}
- input_sync(kpad->input);
+ return 0;
}
+static void adp5588_disable_regulator(void *reg)
+{
+ regulator_disable(reg);
+}
static int adp5588_probe(struct i2c_client *client,
const struct i2c_device_id *id)
{
struct adp5588_kpad *kpad;
- const struct adp5588_kpad_platform_data *pdata =
- dev_get_platdata(&client->dev);
struct input_dev *input;
+ struct gpio_desc *gpio;
+ struct regulator *vcc;
unsigned int revid;
- int ret, i;
+ int ret;
int error;
if (!i2c_check_functionality(client->adapter,
- I2C_FUNC_SMBUS_BYTE_DATA)) {
+ I2C_FUNC_SMBUS_BYTE_DATA)) {
dev_err(&client->dev, "SMBUS Byte Data not Supported\n");
return -EIO;
}
- if (!pdata) {
- dev_err(&client->dev, "no platform data?\n");
- return -EINVAL;
- }
-
- if (!pdata->rows || !pdata->cols || !pdata->keymap) {
- dev_err(&client->dev, "no rows, cols or keymap from pdata\n");
- return -EINVAL;
- }
-
- if (pdata->keymapsize != ADP5588_KEYMAPSIZE) {
- dev_err(&client->dev, "invalid keymapsize\n");
- return -EINVAL;
- }
-
- if (!pdata->gpimap && pdata->gpimapsize) {
- dev_err(&client->dev, "invalid gpimap from pdata\n");
- return -EINVAL;
- }
-
- if (pdata->gpimapsize > ADP5588_GPIMAPSIZE_MAX) {
- dev_err(&client->dev, "invalid gpimapsize\n");
- return -EINVAL;
- }
-
- for (i = 0; i < pdata->gpimapsize; i++) {
- unsigned short pin = pdata->gpimap[i].pin;
-
- if (pin < GPI_PIN_BASE || pin > GPI_PIN_END) {
- dev_err(&client->dev, "invalid gpi pin data\n");
- return -EINVAL;
- }
-
- if (pin <= GPI_PIN_ROW_END) {
- if (pin - GPI_PIN_ROW_BASE + 1 <= pdata->rows) {
- dev_err(&client->dev, "invalid gpi row data\n");
- return -EINVAL;
- }
- } else {
- if (pin - GPI_PIN_COL_BASE + 1 <= pdata->cols) {
- dev_err(&client->dev, "invalid gpi col data\n");
- return -EINVAL;
- }
- }
- }
-
- if (!client->irq) {
- dev_err(&client->dev, "no IRQ?\n");
- return -EINVAL;
- }
-
kpad = devm_kzalloc(&client->dev, sizeof(*kpad), GFP_KERNEL);
if (!kpad)
return -ENOMEM;
kpad->client = client;
kpad->input = input;
+ error = adp5588_fw_parse(kpad);
+ if (error)
+ return error;
+
+ vcc = devm_regulator_get(&client->dev, "vcc");
+ if (IS_ERR(vcc))
+ return PTR_ERR(vcc);
+
+ error = regulator_enable(vcc);
+ if (error)
+ return error;
+
+ error = devm_add_action_or_reset(&client->dev,
+ adp5588_disable_regulator, vcc);
+ if (error)
+ return error;
+
+ gpio = devm_gpiod_get_optional(&client->dev, "reset", GPIOD_OUT_HIGH);
+ if (IS_ERR(gpio))
+ return PTR_ERR(gpio);
+
+ if (gpio) {
+ fsleep(30);
+ gpiod_set_value_cansleep(gpio, 0);
+ fsleep(60);
+ }
+
ret = adp5588_read(client, DEV_ID);
if (ret < 0)
return ret;
- revid = (u8) ret & ADP5588_DEVICE_ID_MASK;
+ revid = ret & ADP5588_DEVICE_ID_MASK;
if (WA_DELAYED_READOUT_REVID(revid))
kpad->delay = msecs_to_jiffies(WA_DELAYED_READOUT_TIME);
input->id.product = 0x0001;
input->id.version = revid;
- input->keycodesize = sizeof(kpad->keycode[0]);
- input->keycodemax = pdata->keymapsize;
- input->keycode = kpad->keycode;
-
- memcpy(kpad->keycode, pdata->keymap,
- pdata->keymapsize * input->keycodesize);
-
- kpad->gpimap = pdata->gpimap;
- kpad->gpimapsize = pdata->gpimapsize;
-
- /* setup input device */
- __set_bit(EV_KEY, input->evbit);
-
- if (pdata->repeat)
- __set_bit(EV_REP, input->evbit);
-
- for (i = 0; i < input->keycodemax; i++)
- if (kpad->keycode[i] <= KEY_MAX)
- __set_bit(kpad->keycode[i], input->keybit);
- __clear_bit(KEY_RESERVED, input->keybit);
-
- if (kpad->gpimapsize)
- __set_bit(EV_SW, input->evbit);
- for (i = 0; i < kpad->gpimapsize; i++)
- __set_bit(kpad->gpimap[i].sw_evt, input->swbit);
-
error = input_register_device(input);
if (error) {
dev_err(&client->dev, "unable to register input device: %d\n",
return error;
}
+ error = adp5588_setup(kpad);
+ if (error)
+ return error;
+
+ error = adp5588_gpio_add(kpad);
+ if (error)
+ return error;
+
error = devm_request_threaded_irq(&client->dev, client->irq,
adp5588_hard_irq, adp5588_thread_irq,
IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
return error;
}
- error = adp5588_setup(client);
- if (error)
- return error;
-
- if (kpad->gpimapsize)
- adp5588_report_switch_state(kpad);
-
- error = adp5588_gpio_add(kpad);
- if (error)
- return error;
-
dev_info(&client->dev, "Rev.%d keypad, irq %d\n", revid, client->irq);
return 0;
}
/* all resources will be freed by devm */
}
-static int __maybe_unused adp5588_suspend(struct device *dev)
+static int adp5588_suspend(struct device *dev)
{
struct i2c_client *client = to_i2c_client(dev);
return 0;
}
-static int __maybe_unused adp5588_resume(struct device *dev)
+static int adp5588_resume(struct device *dev)
{
struct i2c_client *client = to_i2c_client(dev);
return 0;
}
-static SIMPLE_DEV_PM_OPS(adp5588_dev_pm_ops, adp5588_suspend, adp5588_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(adp5588_dev_pm_ops, adp5588_suspend, adp5588_resume);
static const struct i2c_device_id adp5588_id[] = {
{ "adp5588-keys", 0 },
};
MODULE_DEVICE_TABLE(i2c, adp5588_id);
+static const struct of_device_id adp5588_of_match[] = {
+ { .compatible = "adi,adp5588" },
+ { .compatible = "adi,adp5587" },
+ {}
+};
+MODULE_DEVICE_TABLE(of, adp5588_of_match);
+
static struct i2c_driver adp5588_driver = {
.driver = {
.name = KBUILD_MODNAME,
- .pm = &adp5588_dev_pm_ops,
+ .of_match_table = adp5588_of_match,
+ .pm = pm_sleep_ptr(&adp5588_dev_pm_ops),
},
.probe = adp5588_probe,
.remove = adp5588_remove,
* Amiga keyboard driver for Linux/m68k
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/init.h>
#include <linux/input.h>
};
/**
- * struct touchpad_info - touchpad info response.
+ * struct touchpad_info_protocol - touchpad info response.
* message.type = 0x1020, message.length = 0x006e
*
* @unknown1: unknown
struct command_protocol_mt_init init_mt_command;
struct command_protocol_capsl capsl_command;
struct command_protocol_bl bl_command;
- u8 data[0];
+ DECLARE_FLEX_ARRAY(u8, data);
};
};
* This driver only deals with handing key events off to the input layer.
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/init.h>
#include <linux/input.h>
return attr->mode;
}
-static struct attribute_group atkbd_attribute_group = {
+static const struct attribute_group atkbd_attribute_group = {
.attrs = atkbd_attributes,
.is_visible = atkbd_attr_is_visible,
};
+__ATTRIBUTE_GROUPS(atkbd_attribute);
+
static const unsigned int xl_table[] = {
ATKBD_RET_BAT, ATKBD_RET_ERR, ATKBD_RET_ACK,
ATKBD_RET_NAK, ATKBD_RET_HANJA, ATKBD_RET_HANGEUL,
{
struct atkbd *atkbd = serio_get_drvdata(serio);
- sysfs_remove_group(&serio->dev.kobj, &atkbd_attribute_group);
-
atkbd_disable(atkbd);
input_unregister_device(atkbd->dev);
atkbd_set_keycode_table(atkbd);
atkbd_set_device_attrs(atkbd);
- err = sysfs_create_group(&serio->dev.kobj, &atkbd_attribute_group);
- if (err)
- goto fail3;
-
atkbd_enable(atkbd);
if (serio->write)
atkbd_activate(atkbd);
err = input_register_device(atkbd->dev);
if (err)
- goto fail4;
+ goto fail3;
return 0;
- fail4: sysfs_remove_group(&serio->dev.kobj, &atkbd_attribute_group);
fail3: serio_close(serio);
fail2: serio_set_drvdata(serio, NULL);
fail1: input_free_device(dev);
static struct serio_driver atkbd_drv = {
.driver = {
- .name = "atkbd",
+ .name = "atkbd",
+ .dev_groups = atkbd_attribute_groups,
},
.description = DRIVER_DESC,
.id_table = atkbd_serio_ids,
*/
#include <linux/input.h>
+#include <linux/mod_devicetable.h>
#include <linux/module.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
#include <linux/platform_device.h>
+#include <linux/property.h>
#include <linux/regmap.h>
#include <linux/sched.h>
#include <linux/input/matrix_keypad.h>
{
struct clps711x_keypad_data *priv;
struct device *dev = &pdev->dev;
- struct device_node *np = dev->of_node;
struct input_dev *input;
u32 poll_interval;
int i, err;
if (!priv)
return -ENOMEM;
- priv->syscon = syscon_regmap_lookup_by_phandle(np, "syscon");
+ priv->syscon = syscon_regmap_lookup_by_phandle(dev->of_node, "syscon");
if (IS_ERR(priv->syscon))
return PTR_ERR(priv->syscon);
- priv->row_count = of_gpio_named_count(np, "row-gpios");
+ priv->row_count = gpiod_count(dev, "row");
if (priv->row_count < 1)
return -EINVAL;
return PTR_ERR(data->desc);
}
- err = of_property_read_u32(np, "poll-interval", &poll_interval);
+ err = device_property_read_u32(dev, "poll-interval", &poll_interval);
if (err)
return err;
return err;
input_set_capability(input, EV_MSC, MSC_SCAN);
- if (of_property_read_bool(np, "autorepeat"))
+ if (device_property_read_bool(dev, "autorepeat"))
__set_bit(EV_REP, input->evbit);
/* Set all columns to low */
#include <linux/interrupt.h>
#include <linux/clk.h>
#include <linux/io.h>
+#include <linux/input.h>
#include <linux/input/matrix_keypad.h>
#include <linux/slab.h>
#include <linux/soc/cirrus/ep93xx.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/err.h>
+#include <linux/input.h>
#include <linux/input/matrix_keypad.h>
#include <linux/interrupt.h>
#include <linux/io.h>
* http://www.vt100.net/manx/details?pn=EK-104AA-TM-001;id=21;cp=1
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/module.h>
*/
switch (lk->id[4]) {
case 1:
- strlcpy(lk->name, "DEC LK201 keyboard", sizeof(lk->name));
+ strscpy(lk->name, "DEC LK201 keyboard", sizeof(lk->name));
if (lk201_compose_is_alt)
lk->keycode[0xb1] = KEY_LEFTALT;
break;
case 2:
- strlcpy(lk->name, "DEC LK401 keyboard", sizeof(lk->name));
+ strscpy(lk->name, "DEC LK401 keyboard", sizeof(lk->name));
break;
default:
- strlcpy(lk->name, "Unknown DEC keyboard", sizeof(lk->name));
+ strscpy(lk->name, "Unknown DEC keyboard", sizeof(lk->name));
printk(KERN_ERR
"lkkbd: keyboard on %s is unknown, please report to "
"Jan-Benedict Glaw <jbglaw@lug-owl.de>\n", lk->phys);
lk->ctrlclick_volume = ctrlclick_volume;
memcpy(lk->keycode, lkkbd_keycode, sizeof(lk->keycode));
- strlcpy(lk->name, "DEC LK keyboard", sizeof(lk->name));
+ strscpy(lk->name, "DEC LK keyboard", sizeof(lk->name));
snprintf(lk->phys, sizeof(lk->phys), "%s/input0", serio->phys);
input_dev->name = lk->name;
* Copyright (C) 2012 Wolfram Sang, Pengutronix <kernel@pengutronix.de>
*/
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/irq.h>
#include <linux/i2c.h>
-#include <linux/interrupt.h>
+#include <linux/input.h>
#include <linux/input/matrix_keypad.h>
#include <linux/input/lm8333.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/slab.h>
#define LM8333_FIFO_READ 0x20
#define LM8333_DEBOUNCE 0x22
#include <linux/types.h>
#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
#include <linux/platform_device.h>
#include <linux/input.h>
#include <linux/irq.h>
return ERR_PTR(-ENOMEM);
}
- pdata->num_row_gpios = nrow = of_gpio_named_count(np, "row-gpios");
- pdata->num_col_gpios = ncol = of_gpio_named_count(np, "col-gpios");
- if (nrow <= 0 || ncol <= 0) {
+ pdata->num_row_gpios = nrow = gpiod_count(dev, "row");
+ pdata->num_col_gpios = ncol = gpiod_count(dev, "col");
+ if (nrow < 0 || ncol < 0) {
dev_err(dev, "number of keypad rows/columns not specified\n");
return ERR_PTR(-EINVAL);
}
*/
#include <linux/bitops.h>
#include <linux/clk.h>
+#include <linux/input.h>
#include <linux/input/matrix_keypad.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#define MTK_KPD_DEBOUNCE_MASK GENMASK(13, 0)
#define MTK_KPD_DEBOUNCE_MAX_MS 256
#define MTK_KPD_SEL 0x0020
+#define MTK_KPD_SEL_DOUBLE_KP_MODE BIT(0)
#define MTK_KPD_SEL_COL GENMASK(15, 10)
#define MTK_KPD_SEL_ROW GENMASK(9, 4)
#define MTK_KPD_SEL_COLMASK(c) GENMASK((c) + 9, 10)
struct clk *clk;
u32 n_rows;
u32 n_cols;
+ void (*calc_row_col)(unsigned int key,
+ unsigned int *row, unsigned int *col);
DECLARE_BITMAP(keymap_state, MTK_KPD_NUM_BITS);
};
continue;
key = bit_nr / 32 * 16 + bit_nr % 32;
- row = key / 9;
- col = key % 9;
+ keypad->calc_row_col(key, &row, &col);
scancode = MATRIX_SCAN_CODE(row, col, row_shift);
/* 1: not pressed, 0: pressed */
clk_disable_unprepare(data);
}
+static void mt6779_keypad_calc_row_col_single(unsigned int key,
+ unsigned int *row,
+ unsigned int *col)
+{
+ *row = key / 9;
+ *col = key % 9;
+}
+
+static void mt6779_keypad_calc_row_col_double(unsigned int key,
+ unsigned int *row,
+ unsigned int *col)
+{
+ *row = key / 13;
+ *col = (key % 13) / 2;
+}
+
static int mt6779_keypad_pdrv_probe(struct platform_device *pdev)
{
struct mt6779_keypad *keypad;
void __iomem *base;
int irq;
u32 debounce;
+ u32 keys_per_group;
bool wakeup;
int error;
return -EINVAL;
}
+ if (device_property_read_u32(&pdev->dev, "mediatek,keys-per-group",
+ &keys_per_group))
+ keys_per_group = 1;
+
+ switch (keys_per_group) {
+ case 1:
+ keypad->calc_row_col = mt6779_keypad_calc_row_col_single;
+ break;
+ case 2:
+ keypad->calc_row_col = mt6779_keypad_calc_row_col_double;
+ break;
+ default:
+ dev_err(&pdev->dev,
+ "Invalid keys-per-group: %d\n", keys_per_group);
+ return -EINVAL;
+ }
+
wakeup = device_property_read_bool(&pdev->dev, "wakeup-source");
dev_dbg(&pdev->dev, "n_row=%d n_col=%d debounce=%d\n",
regmap_write(keypad->regmap, MTK_KPD_DEBOUNCE,
(debounce * (1 << 5)) & MTK_KPD_DEBOUNCE_MASK);
+ if (keys_per_group == 2)
+ regmap_update_bits(keypad->regmap, MTK_KPD_SEL,
+ MTK_KPD_SEL_DOUBLE_KP_MODE,
+ MTK_KPD_SEL_DOUBLE_KP_MODE);
+
regmap_update_bits(keypad->regmap, MTK_KPD_SEL, MTK_KPD_SEL_ROW,
MTK_KPD_SEL_ROWMASK(keypad->n_rows));
regmap_update_bits(keypad->regmap, MTK_KPD_SEL, MTK_KPD_SEL_COL,
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/mfd/mt6323/registers.h>
+#include <linux/mfd/mt6331/registers.h>
#include <linux/mfd/mt6358/registers.h>
#include <linux/mfd/mt6397/core.h>
#include <linux/mfd/mt6397/registers.h>
#define MTK_PMIC_PWRKEY_RST BIT(6)
#define MTK_PMIC_HOMEKEY_RST BIT(5)
+#define MTK_PMIC_MT6331_RST_DU_MASK GENMASK(13, 12)
+#define MTK_PMIC_MT6331_PWRKEY_RST BIT(9)
+#define MTK_PMIC_MT6331_HOMEKEY_RST BIT(8)
+
#define MTK_PMIC_PWRKEY_INDEX 0
#define MTK_PMIC_HOMEKEY_INDEX 1
#define MTK_PMIC_MAX_KEY_COUNT 2
.rst_lprst_mask = MTK_PMIC_RST_DU_MASK,
};
+static const struct mtk_pmic_regs mt6331_regs = {
+ .keys_regs[MTK_PMIC_PWRKEY_INDEX] =
+ MTK_PMIC_KEYS_REGS(MT6331_TOPSTATUS, 0x2,
+ MT6331_INT_MISC_CON, 0x4,
+ MTK_PMIC_MT6331_PWRKEY_RST),
+ .keys_regs[MTK_PMIC_HOMEKEY_INDEX] =
+ MTK_PMIC_KEYS_REGS(MT6331_TOPSTATUS, 0x4,
+ MT6331_INT_MISC_CON, 0x2,
+ MTK_PMIC_MT6331_HOMEKEY_RST),
+ .pmic_rst_reg = MT6331_TOP_RST_MISC,
+ .rst_lprst_mask = MTK_PMIC_MT6331_RST_DU_MASK,
+};
+
static const struct mtk_pmic_regs mt6358_regs = {
.keys_regs[MTK_PMIC_PWRKEY_INDEX] =
MTK_PMIC_KEYS_REGS(MT6358_TOPSTATUS,
}, {
.compatible = "mediatek,mt6323-keys",
.data = &mt6323_regs,
+ }, {
+ .compatible = "mediatek,mt6331-keys",
+ .data = &mt6331_regs,
}, {
.compatible = "mediatek,mt6358-keys",
.data = &mt6358_regs,
* Newton keyboard driver for Linux
*/
-/*
- */
-
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/input.h>
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Copyright (C) 2021-2022 Samuel Holland <samuel@sholland.org>
+
+#include <linux/crc8.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/input/matrix_keypad.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+#include <linux/types.h>
+
+#define DRV_NAME "pinephone-keyboard"
+
+#define PPKB_CRC8_POLYNOMIAL 0x07
+
+#define PPKB_DEVICE_ID_HI 0x00
+#define PPKB_DEVICE_ID_HI_VALUE 'K'
+#define PPKB_DEVICE_ID_LO 0x01
+#define PPKB_DEVICE_ID_LO_VALUE 'B'
+#define PPKB_FW_REVISION 0x02
+#define PPKB_FW_FEATURES 0x03
+#define PPKB_MATRIX_SIZE 0x06
+#define PPKB_SCAN_CRC 0x07
+#define PPKB_SCAN_DATA 0x08
+#define PPKB_SYS_CONFIG 0x20
+#define PPKB_SYS_CONFIG_DISABLE_SCAN BIT(0)
+#define PPKB_SYS_SMBUS_COMMAND 0x21
+#define PPKB_SYS_SMBUS_DATA 0x22
+#define PPKB_SYS_COMMAND 0x23
+#define PPKB_SYS_COMMAND_SMBUS_READ 0x91
+#define PPKB_SYS_COMMAND_SMBUS_WRITE 0xa1
+
+#define PPKB_ROWS 6
+#define PPKB_COLS 12
+
+/* Size of the scan buffer, including the CRC byte at the beginning. */
+#define PPKB_BUF_LEN (1 + PPKB_COLS)
+
+static const uint32_t ppkb_keymap[] = {
+ KEY(0, 0, KEY_ESC),
+ KEY(0, 1, KEY_1),
+ KEY(0, 2, KEY_2),
+ KEY(0, 3, KEY_3),
+ KEY(0, 4, KEY_4),
+ KEY(0, 5, KEY_5),
+ KEY(0, 6, KEY_6),
+ KEY(0, 7, KEY_7),
+ KEY(0, 8, KEY_8),
+ KEY(0, 9, KEY_9),
+ KEY(0, 10, KEY_0),
+ KEY(0, 11, KEY_BACKSPACE),
+
+ KEY(1, 0, KEY_TAB),
+ KEY(1, 1, KEY_Q),
+ KEY(1, 2, KEY_W),
+ KEY(1, 3, KEY_E),
+ KEY(1, 4, KEY_R),
+ KEY(1, 5, KEY_T),
+ KEY(1, 6, KEY_Y),
+ KEY(1, 7, KEY_U),
+ KEY(1, 8, KEY_I),
+ KEY(1, 9, KEY_O),
+ KEY(1, 10, KEY_P),
+ KEY(1, 11, KEY_ENTER),
+
+ KEY(2, 0, KEY_LEFTMETA),
+ KEY(2, 1, KEY_A),
+ KEY(2, 2, KEY_S),
+ KEY(2, 3, KEY_D),
+ KEY(2, 4, KEY_F),
+ KEY(2, 5, KEY_G),
+ KEY(2, 6, KEY_H),
+ KEY(2, 7, KEY_J),
+ KEY(2, 8, KEY_K),
+ KEY(2, 9, KEY_L),
+ KEY(2, 10, KEY_SEMICOLON),
+
+ KEY(3, 0, KEY_LEFTSHIFT),
+ KEY(3, 1, KEY_Z),
+ KEY(3, 2, KEY_X),
+ KEY(3, 3, KEY_C),
+ KEY(3, 4, KEY_V),
+ KEY(3, 5, KEY_B),
+ KEY(3, 6, KEY_N),
+ KEY(3, 7, KEY_M),
+ KEY(3, 8, KEY_COMMA),
+ KEY(3, 9, KEY_DOT),
+ KEY(3, 10, KEY_SLASH),
+
+ KEY(4, 1, KEY_LEFTCTRL),
+ KEY(4, 4, KEY_SPACE),
+ KEY(4, 6, KEY_APOSTROPHE),
+ KEY(4, 8, KEY_RIGHTBRACE),
+ KEY(4, 9, KEY_LEFTBRACE),
+
+ KEY(5, 2, KEY_FN),
+ KEY(5, 3, KEY_LEFTALT),
+ KEY(5, 5, KEY_RIGHTALT),
+
+ /* FN layer */
+ KEY(PPKB_ROWS + 0, 0, KEY_FN_ESC),
+ KEY(PPKB_ROWS + 0, 1, KEY_F1),
+ KEY(PPKB_ROWS + 0, 2, KEY_F2),
+ KEY(PPKB_ROWS + 0, 3, KEY_F3),
+ KEY(PPKB_ROWS + 0, 4, KEY_F4),
+ KEY(PPKB_ROWS + 0, 5, KEY_F5),
+ KEY(PPKB_ROWS + 0, 6, KEY_F6),
+ KEY(PPKB_ROWS + 0, 7, KEY_F7),
+ KEY(PPKB_ROWS + 0, 8, KEY_F8),
+ KEY(PPKB_ROWS + 0, 9, KEY_F9),
+ KEY(PPKB_ROWS + 0, 10, KEY_F10),
+ KEY(PPKB_ROWS + 0, 11, KEY_DELETE),
+
+ KEY(PPKB_ROWS + 1, 10, KEY_PAGEUP),
+
+ KEY(PPKB_ROWS + 2, 0, KEY_SYSRQ),
+ KEY(PPKB_ROWS + 2, 9, KEY_PAGEDOWN),
+ KEY(PPKB_ROWS + 2, 10, KEY_INSERT),
+
+ KEY(PPKB_ROWS + 3, 0, KEY_LEFTSHIFT),
+ KEY(PPKB_ROWS + 3, 8, KEY_HOME),
+ KEY(PPKB_ROWS + 3, 9, KEY_UP),
+ KEY(PPKB_ROWS + 3, 10, KEY_END),
+
+ KEY(PPKB_ROWS + 4, 1, KEY_LEFTCTRL),
+ KEY(PPKB_ROWS + 4, 6, KEY_LEFT),
+ KEY(PPKB_ROWS + 4, 8, KEY_RIGHT),
+ KEY(PPKB_ROWS + 4, 9, KEY_DOWN),
+
+ KEY(PPKB_ROWS + 5, 3, KEY_LEFTALT),
+ KEY(PPKB_ROWS + 5, 5, KEY_RIGHTALT),
+};
+
+static const struct matrix_keymap_data ppkb_keymap_data = {
+ .keymap = ppkb_keymap,
+ .keymap_size = ARRAY_SIZE(ppkb_keymap),
+};
+
+struct pinephone_keyboard {
+ struct i2c_adapter adapter;
+ struct input_dev *input;
+ u8 buf[2][PPKB_BUF_LEN];
+ u8 crc_table[CRC8_TABLE_SIZE];
+ u8 fn_state[PPKB_COLS];
+ bool buf_swap;
+ bool fn_pressed;
+};
+
+static int ppkb_adap_smbus_xfer(struct i2c_adapter *adap, u16 addr,
+ unsigned short flags, char read_write,
+ u8 command, int size,
+ union i2c_smbus_data *data)
+{
+ struct i2c_client *client = adap->algo_data;
+ u8 buf[3];
+ int ret;
+
+ buf[0] = command;
+ buf[1] = data->byte;
+ buf[2] = read_write == I2C_SMBUS_READ ? PPKB_SYS_COMMAND_SMBUS_READ
+ : PPKB_SYS_COMMAND_SMBUS_WRITE;
+
+ ret = i2c_smbus_write_i2c_block_data(client, PPKB_SYS_SMBUS_COMMAND,
+ sizeof(buf), buf);
+ if (ret)
+ return ret;
+
+ /* Read back the command status until it passes or fails. */
+ do {
+ usleep_range(300, 500);
+ ret = i2c_smbus_read_byte_data(client, PPKB_SYS_COMMAND);
+ } while (ret == buf[2]);
+ if (ret < 0)
+ return ret;
+ /* Commands return 0x00 on success and 0xff on failure. */
+ if (ret)
+ return -EIO;
+
+ if (read_write == I2C_SMBUS_READ) {
+ ret = i2c_smbus_read_byte_data(client, PPKB_SYS_SMBUS_DATA);
+ if (ret < 0)
+ return ret;
+
+ data->byte = ret;
+ }
+
+ return 0;
+}
+
+static u32 ppkg_adap_functionality(struct i2c_adapter *adap)
+{
+ return I2C_FUNC_SMBUS_BYTE_DATA;
+}
+
+static const struct i2c_algorithm ppkb_adap_algo = {
+ .smbus_xfer = ppkb_adap_smbus_xfer,
+ .functionality = ppkg_adap_functionality,
+};
+
+static void ppkb_update(struct i2c_client *client)
+{
+ struct pinephone_keyboard *ppkb = i2c_get_clientdata(client);
+ unsigned short *keymap = ppkb->input->keycode;
+ int row_shift = get_count_order(PPKB_COLS);
+ u8 *old_buf = ppkb->buf[!ppkb->buf_swap];
+ u8 *new_buf = ppkb->buf[ppkb->buf_swap];
+ int col, crc, ret, row;
+ struct device *dev = &client->dev;
+
+ ret = i2c_smbus_read_i2c_block_data(client, PPKB_SCAN_CRC,
+ PPKB_BUF_LEN, new_buf);
+ if (ret != PPKB_BUF_LEN) {
+ dev_err(dev, "Failed to read scan data: %d\n", ret);
+ return;
+ }
+
+ crc = crc8(ppkb->crc_table, &new_buf[1], PPKB_COLS, CRC8_INIT_VALUE);
+ if (crc != new_buf[0]) {
+ dev_err(dev, "Bad scan data (%02x != %02x)\n", crc, new_buf[0]);
+ return;
+ }
+
+ ppkb->buf_swap = !ppkb->buf_swap;
+
+ for (col = 0; col < PPKB_COLS; ++col) {
+ u8 old = old_buf[1 + col];
+ u8 new = new_buf[1 + col];
+ u8 changed = old ^ new;
+
+ if (!changed)
+ continue;
+
+ for (row = 0; row < PPKB_ROWS; ++row) {
+ u8 mask = BIT(row);
+ u8 value = new & mask;
+ unsigned short code;
+ bool fn_state;
+
+ if (!(changed & mask))
+ continue;
+
+ /*
+ * Save off the FN key state when the key was pressed,
+ * and use that to determine the code during a release.
+ */
+ fn_state = value ? ppkb->fn_pressed : ppkb->fn_state[col] & mask;
+ if (fn_state)
+ ppkb->fn_state[col] ^= mask;
+
+ /* The FN layer is a second set of rows. */
+ code = MATRIX_SCAN_CODE(fn_state ? PPKB_ROWS + row : row,
+ col, row_shift);
+ input_event(ppkb->input, EV_MSC, MSC_SCAN, code);
+ input_report_key(ppkb->input, keymap[code], value);
+ if (keymap[code] == KEY_FN)
+ ppkb->fn_pressed = value;
+ }
+ }
+ input_sync(ppkb->input);
+}
+
+static irqreturn_t ppkb_irq_thread(int irq, void *data)
+{
+ struct i2c_client *client = data;
+
+ ppkb_update(client);
+
+ return IRQ_HANDLED;
+}
+
+static int ppkb_set_scan(struct i2c_client *client, bool enable)
+{
+ struct device *dev = &client->dev;
+ int ret, val;
+
+ ret = i2c_smbus_read_byte_data(client, PPKB_SYS_CONFIG);
+ if (ret < 0) {
+ dev_err(dev, "Failed to read config: %d\n", ret);
+ return ret;
+ }
+
+ if (enable)
+ val = ret & ~PPKB_SYS_CONFIG_DISABLE_SCAN;
+ else
+ val = ret | PPKB_SYS_CONFIG_DISABLE_SCAN;
+
+ ret = i2c_smbus_write_byte_data(client, PPKB_SYS_CONFIG, val);
+ if (ret) {
+ dev_err(dev, "Failed to write config: %d\n", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int ppkb_open(struct input_dev *input)
+{
+ struct i2c_client *client = input_get_drvdata(input);
+ int error;
+
+ error = ppkb_set_scan(client, true);
+ if (error)
+ return error;
+
+ return 0;
+}
+
+static void ppkb_close(struct input_dev *input)
+{
+ struct i2c_client *client = input_get_drvdata(input);
+
+ ppkb_set_scan(client, false);
+}
+
+static void ppkb_regulator_disable(void *regulator)
+{
+ regulator_disable(regulator);
+}
+
+static int ppkb_probe(struct i2c_client *client)
+{
+ struct device *dev = &client->dev;
+ unsigned int phys_rows, phys_cols;
+ struct pinephone_keyboard *ppkb;
+ struct regulator *vbat_supply;
+ u8 info[PPKB_MATRIX_SIZE + 1];
+ struct device_node *i2c_bus;
+ int ret;
+ int error;
+
+ vbat_supply = devm_regulator_get(dev, "vbat");
+ error = PTR_ERR_OR_ZERO(vbat_supply);
+ if (error) {
+ dev_err(dev, "Failed to get VBAT supply: %d\n", error);
+ return error;
+ }
+
+ error = regulator_enable(vbat_supply);
+ if (error) {
+ dev_err(dev, "Failed to enable VBAT: %d\n", error);
+ return error;
+ }
+
+ error = devm_add_action_or_reset(dev, ppkb_regulator_disable,
+ vbat_supply);
+ if (error)
+ return error;
+
+ ret = i2c_smbus_read_i2c_block_data(client, 0, sizeof(info), info);
+ if (ret != sizeof(info)) {
+ error = ret < 0 ? ret : -EIO;
+ dev_err(dev, "Failed to read device ID: %d\n", error);
+ return error;
+ }
+
+ if (info[PPKB_DEVICE_ID_HI] != PPKB_DEVICE_ID_HI_VALUE ||
+ info[PPKB_DEVICE_ID_LO] != PPKB_DEVICE_ID_LO_VALUE) {
+ dev_warn(dev, "Unexpected device ID: %#02x %#02x\n",
+ info[PPKB_DEVICE_ID_HI], info[PPKB_DEVICE_ID_LO]);
+ return -ENODEV;
+ }
+
+ dev_info(dev, "Found firmware version %d.%d features %#x\n",
+ info[PPKB_FW_REVISION] >> 4,
+ info[PPKB_FW_REVISION] & 0xf,
+ info[PPKB_FW_FEATURES]);
+
+ phys_rows = info[PPKB_MATRIX_SIZE] & 0xf;
+ phys_cols = info[PPKB_MATRIX_SIZE] >> 4;
+ if (phys_rows != PPKB_ROWS || phys_cols != PPKB_COLS) {
+ dev_err(dev, "Unexpected keyboard size %ux%u\n",
+ phys_rows, phys_cols);
+ return -EINVAL;
+ }
+
+ /* Disable scan by default to save power. */
+ error = ppkb_set_scan(client, false);
+ if (error)
+ return error;
+
+ ppkb = devm_kzalloc(dev, sizeof(*ppkb), GFP_KERNEL);
+ if (!ppkb)
+ return -ENOMEM;
+
+ i2c_set_clientdata(client, ppkb);
+
+ i2c_bus = of_get_child_by_name(dev->of_node, "i2c");
+ if (i2c_bus) {
+ ppkb->adapter.owner = THIS_MODULE;
+ ppkb->adapter.algo = &ppkb_adap_algo;
+ ppkb->adapter.algo_data = client;
+ ppkb->adapter.dev.parent = dev;
+ ppkb->adapter.dev.of_node = i2c_bus;
+ strscpy(ppkb->adapter.name, DRV_NAME, sizeof(ppkb->adapter.name));
+
+ error = devm_i2c_add_adapter(dev, &ppkb->adapter);
+ if (error) {
+ dev_err(dev, "Failed to add I2C adapter: %d\n", error);
+ return error;
+ }
+ }
+
+ crc8_populate_msb(ppkb->crc_table, PPKB_CRC8_POLYNOMIAL);
+
+ ppkb->input = devm_input_allocate_device(dev);
+ if (!ppkb->input)
+ return -ENOMEM;
+
+ input_set_drvdata(ppkb->input, client);
+
+ ppkb->input->name = "PinePhone Keyboard";
+ ppkb->input->phys = DRV_NAME "/input0";
+ ppkb->input->id.bustype = BUS_I2C;
+ ppkb->input->open = ppkb_open;
+ ppkb->input->close = ppkb_close;
+
+ input_set_capability(ppkb->input, EV_MSC, MSC_SCAN);
+ __set_bit(EV_REP, ppkb->input->evbit);
+
+ error = matrix_keypad_build_keymap(&ppkb_keymap_data, NULL,
+ 2 * PPKB_ROWS, PPKB_COLS, NULL,
+ ppkb->input);
+ if (error) {
+ dev_err(dev, "Failed to build keymap: %d\n", error);
+ return error;
+ }
+
+ error = input_register_device(ppkb->input);
+ if (error) {
+ dev_err(dev, "Failed to register input: %d\n", error);
+ return error;
+ }
+
+ error = devm_request_threaded_irq(dev, client->irq,
+ NULL, ppkb_irq_thread,
+ IRQF_ONESHOT, client->name, client);
+ if (error) {
+ dev_err(dev, "Failed to request IRQ: %d\n", error);
+ return error;
+ }
+
+ return 0;
+}
+
+static const struct of_device_id ppkb_of_match[] = {
+ { .compatible = "pine64,pinephone-keyboard" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, ppkb_of_match);
+
+static struct i2c_driver ppkb_driver = {
+ .probe_new = ppkb_probe,
+ .driver = {
+ .name = DRV_NAME,
+ .of_match_table = ppkb_of_match,
+ },
+};
+module_i2c_driver(ppkb_driver);
+
+MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
+MODULE_DESCRIPTION("Pine64 PinePhone keyboard driver");
+MODULE_LICENSE("GPL");
* Based on sh_keysc.c, copyright 2008 Magnus Damm
*/
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/platform_device.h>
#include <linux/clk.h>
-#include <linux/io.h>
+#include <linux/input.h>
#include <linux/input/matrix_keypad.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
#define ST_KEYSCAN_MAXKEYS 16
* by Justin Cormack
*/
-/*
- */
-
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/input.h>
* Sun keyboard driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/sched.h>
#include <linux/slab.h>
#define TC3589x_KBD_INT_CLR 0x1
/**
- * struct tc35893_keypad_platform_data - platform specific keypad data
+ * struct tc3589x_keypad_platform_data - platform specific keypad data
* @keymap_data: matrix scan code table for keycodes
* @krow: mask for available rows, value is 0xFF
* @kcol: mask for available columns, value is 0xFF
* XT keyboard driver for Linux
*/
-/*
- */
-
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/input.h>
To compile this driver as a module, choose M here: the
module will be called adxl34x-spi.
+config INPUT_IBM_PANEL
+ tristate "IBM Operation Panel driver"
+ depends on I2C && I2C_SLAVE
+ help
+ Say Y here if you have an IBM Operation Panel connected to your system
+ over I2C. The panel is typically connected only to a system's service
+ processor (BMC).
+
+ If unsure, say N.
+
+ The Operation Panel is a controller with some buttons and an LCD
+ display that allows someone with physical access to the system to
+ perform various administrative tasks. This driver only supports the part
+ of the controller that sends commands to the system.
+
+ To compile this driver as a module, choose M here: the module will be
+ called ibm-panel.
+
config INPUT_IMS_PCU
tristate "IMS Passenger Control Unit driver"
depends on USB
To compile this driver as a module, choose M here. The module will
be called sc27xx_vibra.
+config INPUT_RT5120_PWRKEY
+ tristate "RT5120 PMIC power key support"
+ depends on MFD_RT5120 || COMPILE_TEST
+ help
+ This enables support for RT5120 PMIC power key driver.
+
+ To compile this driver as a module, choose M here. the module will
+ be called rt5120-pwrkey.
+
config INPUT_STPMIC1_ONKEY
tristate "STPMIC1 PMIC Onkey support"
depends on MFD_STPMIC1
obj-$(CONFIG_INPUT_GPIO_VIBRA) += gpio-vibra.o
obj-$(CONFIG_INPUT_HISI_POWERKEY) += hisi_powerkey.o
obj-$(CONFIG_HP_SDC_RTC) += hp_sdc_rtc.o
+obj-$(CONFIG_INPUT_IBM_PANEL) += ibm-panel.o
obj-$(CONFIG_INPUT_IMS_PCU) += ims-pcu.o
obj-$(CONFIG_INPUT_IQS269A) += iqs269a.o
obj-$(CONFIG_INPUT_IQS626A) += iqs626a.o
obj-$(CONFIG_INPUT_RB532_BUTTON) += rb532_button.o
obj-$(CONFIG_INPUT_REGULATOR_HAPTIC) += regulator-haptic.o
obj-$(CONFIG_INPUT_RETU_PWRBUTTON) += retu-pwrbutton.o
+obj-$(CONFIG_INPUT_RT5120_PWRKEY) += rt5120-pwrkey.o
obj-$(CONFIG_INPUT_AXP20X_PEK) += axp20x-pek.o
obj-$(CONFIG_INPUT_GPIO_ROTARY_ENCODER) += rotary_encoder.o
obj-$(CONFIG_INPUT_RK805_PWRKEY) += rk805-pwrkey.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) IBM Corporation 2020
+ */
+
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/input.h>
+#include <linux/kernel.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/spinlock.h>
+
+#define DEVICE_NAME "ibm-panel"
+#define PANEL_KEYCODES_COUNT 3
+
+struct ibm_panel {
+ u8 idx;
+ u8 command[11];
+ u32 keycodes[PANEL_KEYCODES_COUNT];
+ spinlock_t lock; /* protects writes to idx and command */
+ struct input_dev *input;
+};
+
+static u8 ibm_panel_calculate_checksum(struct ibm_panel *panel)
+{
+ u8 chksum;
+ u16 sum = 0;
+ unsigned int i;
+
+ for (i = 0; i < sizeof(panel->command) - 1; ++i) {
+ sum += panel->command[i];
+ if (sum & 0xff00) {
+ sum &= 0xff;
+ sum++;
+ }
+ }
+
+ chksum = sum & 0xff;
+ chksum = ~chksum;
+ chksum++;
+
+ return chksum;
+}
+
+static void ibm_panel_process_command(struct ibm_panel *panel)
+{
+ u8 button;
+ u8 chksum;
+
+ if (panel->command[0] != 0xff && panel->command[1] != 0xf0) {
+ dev_dbg(&panel->input->dev, "command invalid: %02x %02x\n",
+ panel->command[0], panel->command[1]);
+ return;
+ }
+
+ chksum = ibm_panel_calculate_checksum(panel);
+ if (chksum != panel->command[sizeof(panel->command) - 1]) {
+ dev_dbg(&panel->input->dev,
+ "command failed checksum: %u != %u\n", chksum,
+ panel->command[sizeof(panel->command) - 1]);
+ return;
+ }
+
+ button = panel->command[2] & 0xf;
+ if (button < PANEL_KEYCODES_COUNT) {
+ input_report_key(panel->input, panel->keycodes[button],
+ !(panel->command[2] & 0x80));
+ input_sync(panel->input);
+ } else {
+ dev_dbg(&panel->input->dev, "unknown button %u\n",
+ button);
+ }
+}
+
+static int ibm_panel_i2c_slave_cb(struct i2c_client *client,
+ enum i2c_slave_event event, u8 *val)
+{
+ unsigned long flags;
+ struct ibm_panel *panel = i2c_get_clientdata(client);
+
+ dev_dbg(&panel->input->dev, "event: %u data: %02x\n", event, *val);
+
+ spin_lock_irqsave(&panel->lock, flags);
+
+ switch (event) {
+ case I2C_SLAVE_STOP:
+ if (panel->idx == sizeof(panel->command))
+ ibm_panel_process_command(panel);
+ else
+ dev_dbg(&panel->input->dev,
+ "command incorrect size %u\n", panel->idx);
+ fallthrough;
+ case I2C_SLAVE_WRITE_REQUESTED:
+ panel->idx = 0;
+ break;
+ case I2C_SLAVE_WRITE_RECEIVED:
+ if (panel->idx < sizeof(panel->command))
+ panel->command[panel->idx++] = *val;
+ else
+ /*
+ * The command is too long and therefore invalid, so set the index
+ * to it's largest possible value. When a STOP is finally received,
+ * the command will be rejected upon processing.
+ */
+ panel->idx = U8_MAX;
+ break;
+ case I2C_SLAVE_READ_REQUESTED:
+ case I2C_SLAVE_READ_PROCESSED:
+ *val = 0xff;
+ break;
+ default:
+ break;
+ }
+
+ spin_unlock_irqrestore(&panel->lock, flags);
+
+ return 0;
+}
+
+static int ibm_panel_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct ibm_panel *panel;
+ int i;
+ int error;
+
+ panel = devm_kzalloc(&client->dev, sizeof(*panel), GFP_KERNEL);
+ if (!panel)
+ return -ENOMEM;
+
+ spin_lock_init(&panel->lock);
+
+ panel->input = devm_input_allocate_device(&client->dev);
+ if (!panel->input)
+ return -ENOMEM;
+
+ panel->input->name = client->name;
+ panel->input->id.bustype = BUS_I2C;
+
+ error = device_property_read_u32_array(&client->dev,
+ "linux,keycodes",
+ panel->keycodes,
+ PANEL_KEYCODES_COUNT);
+ if (error) {
+ /*
+ * Use gamepad buttons as defaults for compatibility with
+ * existing applications.
+ */
+ panel->keycodes[0] = BTN_NORTH;
+ panel->keycodes[1] = BTN_SOUTH;
+ panel->keycodes[2] = BTN_SELECT;
+ }
+
+ for (i = 0; i < PANEL_KEYCODES_COUNT; ++i)
+ input_set_capability(panel->input, EV_KEY, panel->keycodes[i]);
+
+ error = input_register_device(panel->input);
+ if (error) {
+ dev_err(&client->dev,
+ "Failed to register input device: %d\n", error);
+ return error;
+ }
+
+ i2c_set_clientdata(client, panel);
+ error = i2c_slave_register(client, ibm_panel_i2c_slave_cb);
+ if (error) {
+ dev_err(&client->dev,
+ "Failed to register as i2c slave: %d\n", error);
+ return error;
+ }
+
+ return 0;
+}
+
+static void ibm_panel_remove(struct i2c_client *client)
+{
+ i2c_slave_unregister(client);
+}
+
+static const struct of_device_id ibm_panel_match[] = {
+ { .compatible = "ibm,op-panel" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, ibm_panel_match);
+
+static struct i2c_driver ibm_panel_driver = {
+ .driver = {
+ .name = DEVICE_NAME,
+ .of_match_table = ibm_panel_match,
+ },
+ .probe = ibm_panel_probe,
+ .remove = ibm_panel_remove,
+};
+module_i2c_driver(ibm_panel_driver);
+
+MODULE_AUTHOR("Eddie James <eajames@linux.ibm.com>");
+MODULE_DESCRIPTION("IBM Operation Panel Driver");
+MODULE_LICENSE("GPL");
error = ims_pcu_execute_command(pcu, JUMP_TO_BTLDR, NULL, 0);
if (error) {
dev_err(pcu->dev,
- "Failure when sending JUMP TO BOOLTLOADER command, error: %d\n",
+ "Failure when sending JUMP TO BOOTLOADER command, error: %d\n",
error);
return error;
}
static int iqs7222_force_comms(struct iqs7222_private *iqs7222)
{
- u8 msg_buf[] = { 0xFF, 0x00, };
+ u8 msg_buf[] = { 0xFF, };
int ret;
/*
if (!chan_node)
return 0;
- if (dev_desc->allow_offset) {
- sys_setup[dev_desc->allow_offset] |= BIT(chan_index);
- if (fwnode_property_present(chan_node, "azoteq,ulp-allow"))
- sys_setup[dev_desc->allow_offset] &= ~BIT(chan_index);
- }
+ if (dev_desc->allow_offset &&
+ fwnode_property_present(chan_node, "azoteq,ulp-allow"))
+ sys_setup[dev_desc->allow_offset] &= ~BIT(chan_index);
chan_setup[0] |= IQS7222_CHAN_SETUP_0_CHAN_EN;
u16 *sys_setup = iqs7222->sys_setup;
int error, i;
+ if (dev_desc->allow_offset)
+ sys_setup[dev_desc->allow_offset] = U16_MAX;
+
if (dev_desc->event_offset)
sys_setup[dev_desc->event_offset] = IQS7222_EVENT_MASK_ATI;
int k = 2 + j * (num_chan > 16 ? 2 : 1);
u16 state = le16_to_cpu(status[k + i / 16]);
+ if (!iqs7222->kp_type[i][j])
+ continue;
+
input_event(iqs7222->keypad,
iqs7222->kp_type[i][j],
iqs7222->kp_code[i][j],
}
if (udev->manufacturer)
- strlcpy(remote->name, udev->manufacturer, sizeof(remote->name));
+ strscpy(remote->name, udev->manufacturer, sizeof(remote->name));
if (udev->product) {
if (udev->manufacturer)
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 Richtek Technology Corp.
+ * Author: ChiYuan Huang <cy_huang@richtek.com>
+ */
+
+#include <linux/bits.h>
+#include <linux/input.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define RT5120_REG_INTSTAT 0x1E
+#define RT5120_PWRKEYSTAT_MASK BIT(7)
+
+struct rt5120_priv {
+ struct regmap *regmap;
+ struct input_dev *input;
+};
+
+static irqreturn_t rt5120_pwrkey_handler(int irq, void *devid)
+{
+ struct rt5120_priv *priv = devid;
+ unsigned int stat;
+ int error;
+
+ error = regmap_read(priv->regmap, RT5120_REG_INTSTAT, &stat);
+ if (error)
+ return IRQ_NONE;
+
+ input_report_key(priv->input, KEY_POWER,
+ !(stat & RT5120_PWRKEYSTAT_MASK));
+ input_sync(priv->input);
+
+ return IRQ_HANDLED;
+}
+
+static int rt5120_pwrkey_probe(struct platform_device *pdev)
+{
+ struct rt5120_priv *priv;
+ struct device *dev = &pdev->dev;
+ int press_irq, release_irq;
+ int error;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->regmap = dev_get_regmap(dev->parent, NULL);
+ if (!priv->regmap) {
+ dev_err(dev, "Failed to init regmap\n");
+ return -ENODEV;
+ }
+
+ press_irq = platform_get_irq_byname(pdev, "pwrkey-press");
+ if (press_irq < 0)
+ return press_irq;
+
+ release_irq = platform_get_irq_byname(pdev, "pwrkey-release");
+ if (release_irq < 0)
+ return release_irq;
+
+ /* Make input device be device resource managed */
+ priv->input = devm_input_allocate_device(dev);
+ if (!priv->input)
+ return -ENOMEM;
+
+ priv->input->name = "rt5120_pwrkey";
+ priv->input->phys = "rt5120_pwrkey/input0";
+ priv->input->id.bustype = BUS_I2C;
+ input_set_capability(priv->input, EV_KEY, KEY_POWER);
+
+ error = input_register_device(priv->input);
+ if (error) {
+ dev_err(dev, "Failed to register input device: %d\n", error);
+ return error;
+ }
+
+ error = devm_request_threaded_irq(dev, press_irq,
+ NULL, rt5120_pwrkey_handler,
+ 0, "pwrkey-press", priv);
+ if (error) {
+ dev_err(dev,
+ "Failed to register pwrkey press irq: %d\n", error);
+ return error;
+ }
+
+ error = devm_request_threaded_irq(dev, release_irq,
+ NULL, rt5120_pwrkey_handler,
+ 0, "pwrkey-release", priv);
+ if (error) {
+ dev_err(dev,
+ "Failed to register pwrkey release irq: %d\n", error);
+ return error;
+ }
+
+ return 0;
+}
+
+static const struct of_device_id r5120_pwrkey_match_table[] = {
+ { .compatible = "richtek,rt5120-pwrkey" },
+ {}
+};
+MODULE_DEVICE_TABLE(of, r5120_pwrkey_match_table);
+
+static struct platform_driver rt5120_pwrkey_driver = {
+ .driver = {
+ .name = "rt5120-pwrkey",
+ .of_match_table = r5120_pwrkey_match_table,
+ },
+ .probe = rt5120_pwrkey_probe,
+};
+module_platform_driver(rt5120_pwrkey_driver);
+
+MODULE_AUTHOR("ChiYuan Huang <cy_huang@richtek.com>");
+MODULE_DESCRIPTION("Richtek RT5120 power key driver");
+MODULE_LICENSE("GPL");
#include <linux/errno.h>
#include <linux/input.h>
#include <linux/interrupt.h>
+#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/mfd/twl.h>
static SIMPLE_DEV_PM_OPS(twl4030_vibra_pm_ops,
twl4030_vibra_suspend, twl4030_vibra_resume);
-static bool twl4030_vibra_check_coexist(struct twl4030_vibra_data *pdata,
- struct device_node *parent)
+static bool twl4030_vibra_check_coexist(struct device_node *parent)
{
struct device_node *node;
- if (pdata && pdata->coexist)
- return true;
-
node = of_get_child_by_name(parent, "codec");
if (node) {
of_node_put(node);
static int twl4030_vibra_probe(struct platform_device *pdev)
{
- struct twl4030_vibra_data *pdata = dev_get_platdata(&pdev->dev);
struct device_node *twl4030_core_node = pdev->dev.parent->of_node;
struct vibra_info *info;
int ret;
- if (!pdata && !twl4030_core_node) {
- dev_dbg(&pdev->dev, "platform_data not available\n");
+ if (!twl4030_core_node) {
+ dev_dbg(&pdev->dev, "twl4030 OF node is missing\n");
return -EINVAL;
}
return -ENOMEM;
info->dev = &pdev->dev;
- info->coexist = twl4030_vibra_check_coexist(pdata, twl4030_core_node);
+ info->coexist = twl4030_vibra_check_coexist(twl4030_core_node);
INIT_WORK(&info->play_work, vibra_play_work);
info->input_dev = devm_input_allocate_device(&pdev->dev);
return error;
}
- error = devm_device_add_groups(dev, elan_sysfs_groups);
- if (error) {
- dev_err(dev, "failed to create sysfs attributes: %d\n", error);
- return error;
- }
-
error = input_register_device(data->input);
if (error) {
dev_err(dev, "failed to register input device: %d\n", error);
.acpi_match_table = ACPI_PTR(elan_acpi_id),
.of_match_table = of_match_ptr(elan_of_match),
.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+ .dev_groups = elan_sysfs_groups,
},
.probe = elan_probe,
.id_table = elan_id,
/*
* We queue work instead of doing recalibration right here
- * to avoid adding locking to to hgpk_force_recalibrate()
+ * to avoid adding locking to hgpk_force_recalibrate()
* since workqueue provides serialization.
*/
psmouse_queue_work(psmouse, &priv->recalib_wq, 0);
strlen(hgpk_mode_name));
if (hgpk_default_mode == HGPK_MODE_INVALID) {
hgpk_default_mode = HGPK_MODE_MOUSE;
- strlcpy(hgpk_mode_name, hgpk_mode_names[HGPK_MODE_MOUSE],
+ strscpy(hgpk_mode_name, hgpk_mode_names[HGPK_MODE_MOUSE],
sizeof(hgpk_mode_name));
}
}
* Inport (ATI XL and Microsoft) busmouse driver for Linux
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/ioport.h>
#include <linux/init.h>
* Logitech Bus Mouse Driver for Linux
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/ioport.h>
* IBM PC110 touchpad driver for Linux
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/errno.h>
(void *) offsetof(struct psmouse, resync_time),
psmouse_show_int_attr, psmouse_set_int_attr);
-static struct attribute *psmouse_attributes[] = {
+static struct attribute *psmouse_dev_attrs[] = {
&psmouse_attr_protocol.dattr.attr,
&psmouse_attr_rate.dattr.attr,
&psmouse_attr_resolution.dattr.attr,
NULL
};
-static const struct attribute_group psmouse_attribute_group = {
- .attrs = psmouse_attributes,
-};
+ATTRIBUTE_GROUPS(psmouse_dev);
/*
* psmouse_mutex protects all operations changing state of mouse
struct psmouse *psmouse = serio_get_drvdata(serio);
struct psmouse *parent = NULL;
- sysfs_remove_group(&serio->dev.kobj, &psmouse_attribute_group);
-
mutex_lock(&psmouse_mutex);
psmouse_set_state(psmouse, PSMOUSE_CMD_MODE);
if (parent && parent->pt_activate)
parent->pt_activate(parent);
- error = sysfs_create_group(&serio->dev.kobj, &psmouse_attribute_group);
- if (error)
- goto err_pt_deactivate;
-
/*
* PS/2 devices having SMBus companions should stay disabled
* on PS/2 side, in order to have SMBus part operable.
mutex_unlock(&psmouse_mutex);
return retval;
- err_pt_deactivate:
- if (parent && parent->pt_deactivate)
- parent->pt_deactivate(parent);
- if (input_dev) {
- input_unregister_device(input_dev);
- input_dev = NULL; /* so we don't try to free it below */
- }
err_protocol_disconnect:
if (psmouse->disconnect)
psmouse->disconnect(psmouse);
static struct serio_driver psmouse_drv = {
.driver = {
- .name = "psmouse",
+ .name = "psmouse",
+ .dev_groups = psmouse_dev_groups,
},
.description = DRIVER_DESC,
.id_table = psmouse_serio_ids,
* Serial mouse driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/slab.h>
"LEN0099", /* X1 Extreme Gen 1 / P1 Gen 1 */
"LEN009b", /* T580 */
"LEN0402", /* X1 Extreme Gen 2 / P1 Gen 2 */
+ "LEN040f", /* P1 Gen 3 */
"LEN200f", /* T450s */
"LEN2044", /* L470 */
"LEN2054", /* E480 */
}
serio->id.type = SERIO_PS_PSTHRU;
- strlcpy(serio->name, "Synaptics pass-through", sizeof(serio->name));
- strlcpy(serio->phys, "synaptics-pt/serio0", sizeof(serio->phys));
+ strscpy(serio->name, "Synaptics pass-through", sizeof(serio->name));
+ strscpy(serio->phys, "synaptics-pt/serio0", sizeof(serio->phys));
serio->write = synaptics_pt_write;
serio->start = synaptics_pt_start;
serio->stop = synaptics_pt_stop;
synusb->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
if (udev->manufacturer)
- strlcpy(synusb->name, udev->manufacturer,
+ strscpy(synusb->name, udev->manufacturer,
sizeof(synusb->name));
if (udev->product) {
* Later on, I had access to the device's documentation (referenced below).
*/
-/*
- */
-
/*
* Building an adaptor to DE9 / DB25 RS232
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
{
switch (mouse->type) {
case 0x02:
- strlcpy(mouse->name, "DEC VSXXX-AA/-GA mouse",
+ strscpy(mouse->name, "DEC VSXXX-AA/-GA mouse",
sizeof(mouse->name));
break;
case 0x04:
- strlcpy(mouse->name, "DEC VSXXX-AB digitizer",
+ strscpy(mouse->name, "DEC VSXXX-AB digitizer",
sizeof(mouse->name));
break;
serio->close = rmi_f03_pt_close;
serio->port_data = f03;
- strlcpy(serio->name, "RMI4 PS/2 pass-through", sizeof(serio->name));
+ strscpy(serio->name, "RMI4 PS/2 pass-through", sizeof(serio->name));
snprintf(serio->phys, sizeof(serio->phys), "%s/serio0",
dev_name(&f03->fn->dev));
serio->dev.parent = &f03->fn->dev;
complete(&f34->v5.cmd_done);
} else {
ret = rmi_read_block(f34->fn->rmi_dev,
- f34->fn->fd.data_base_addr +
- f34->v7.off.flash_status,
- &status, sizeof(status));
- rmi_dbg(RMI_DEBUG_FN, &fn->dev, "%s: status: %#02x, ret: %d\n",
+ f34->fn->fd.data_base_addr +
+ V7_COMMAND_OFFSET,
+ &status, sizeof(status));
+ rmi_dbg(RMI_DEBUG_FN, &f34->fn->dev, "%s: cmd: %#02x, ret: %d\n",
__func__, status, ret);
- if (!ret && !(status & 0x1f))
+ if (!ret && status == CMD_V7_IDLE)
complete(&f34->v7.cmd_done);
}
f34 = dev_get_drvdata(&fn->dev);
if (f34->bl_version == 5)
- return scnprintf(buf, PAGE_SIZE, "%c%c\n",
- f34->bootloader_id[0],
- f34->bootloader_id[1]);
+ return sysfs_emit(buf, "%c%c\n",
+ f34->bootloader_id[0],
+ f34->bootloader_id[1]);
else
- return scnprintf(buf, PAGE_SIZE, "V%d.%d\n",
- f34->bootloader_id[1],
- f34->bootloader_id[0]);
+ return sysfs_emit(buf, "V%d.%d\n",
+ f34->bootloader_id[1],
+ f34->bootloader_id[0]);
}
return 0;
if (fn) {
f34 = dev_get_drvdata(&fn->dev);
- return scnprintf(buf, PAGE_SIZE, "%s\n", f34->configuration_id);
+ return sysfs_emit(buf, "%s\n", f34->configuration_id);
}
return 0;
f34 = dev_get_drvdata(&data->f34_container->dev);
- if (f34->bl_version == 7) {
+ if (f34->bl_version >= 7) {
if (data->pdt_props & HAS_BSR) {
dev_err(dev, "%s: LTS not supported\n", __func__);
return -ENODEV;
}
/* Enter flash mode */
- if (f34->bl_version == 7)
+ if (f34->bl_version >= 7)
ret = rmi_f34v7_start_reflash(f34, fw);
else
ret = rmi_f34_enable_flash(f34);
f34 = dev_get_drvdata(&data->f34_container->dev);
/* Perform firmware update */
- if (f34->bl_version == 7)
+ if (f34->bl_version >= 7)
ret = rmi_f34v7_do_reflash(f34, fw);
else
ret = rmi_f34_update_firmware(f34, fw);
if (data->f34_container)
update_status = rmi_f34_status(data->f34_container);
- return scnprintf(buf, PAGE_SIZE, "%d\n", update_status);
+ return sysfs_emit(buf, "%d\n", update_status);
}
static DEVICE_ATTR(update_fw_status, 0444,
struct physical_address phyaddr;
};
-struct register_offset {
- u8 properties;
- u8 properties_2;
- u8 block_size;
- u8 block_count;
- u8 gc_block_count;
- u8 flash_status;
- u8 partition_id;
- u8 block_number;
- u8 transfer_length;
- u8 flash_cmd;
- u8 payload;
-};
-
struct rmi_f34_firmware {
__le32 checksum;
u8 pad1[3];
struct f34v7_data {
bool has_display_cfg;
bool has_guest_code;
- bool force_update;
bool in_bl_mode;
u8 *read_config_buf;
size_t read_config_buf_size;
u16 payload_length;
u8 partitions;
u16 partition_table_bytes;
- bool new_partition_table;
- struct register_offset off;
struct block_count blkcount;
struct physical_address phyaddr;
struct image_metadata img;
int ret;
ret = rmi_read_block(f34->fn->rmi_dev,
- f34->fn->fd.data_base_addr + f34->v7.off.flash_status,
+ f34->fn->fd.data_base_addr + V7_FLASH_STATUS_OFFSET,
&status,
sizeof(status));
if (ret < 0) {
}
ret = rmi_read_block(f34->fn->rmi_dev,
- f34->fn->fd.data_base_addr + f34->v7.off.flash_cmd,
+ f34->fn->fd.data_base_addr + V7_COMMAND_OFFSET,
&command,
sizeof(command));
if (ret < 0) {
return 0;
}
+static int rmi_f34v7_check_command_status(struct f34_data *f34, int timeout_ms)
+{
+ int ret;
+
+ ret = rmi_f34v7_wait_for_idle(f34, timeout_ms);
+ if (ret < 0)
+ return ret;
+
+ ret = rmi_f34v7_read_flash_status(f34);
+ if (ret < 0)
+ return ret;
+
+ if (f34->v7.flash_status != 0x00)
+ return -EIO;
+
+ return 0;
+}
+
static int rmi_f34v7_write_command_single_transaction(struct f34_data *f34,
u8 cmd)
{
data_1_5.payload[1] = f34->bootloader_id[1];
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.partition_id,
+ base + V7_PARTITION_ID_OFFSET,
&data_1_5, sizeof(data_1_5));
if (ret < 0) {
dev_err(&f34->fn->dev,
__func__, command);
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.flash_cmd,
+ base + V7_COMMAND_OFFSET,
&command, sizeof(command));
if (ret < 0) {
dev_err(&f34->fn->dev, "%s: Failed to write flash command\n",
}
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.partition_id,
+ base + V7_PARTITION_ID_OFFSET,
&partition, sizeof(partition));
if (ret < 0) {
dev_err(&f34->fn->dev, "%s: Failed to write partition ID\n",
return ret;
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.block_number,
+ base + V7_BLOCK_NUMBER_OFFSET,
&block_number, sizeof(block_number));
if (ret < 0) {
dev_err(&f34->fn->dev, "%s: Failed to write block number\n",
put_unaligned_le16(f34->v7.flash_config_length, &length);
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.transfer_length,
+ base + V7_TRANSFER_LENGTH_OFFSET,
&length, sizeof(length));
if (ret < 0) {
dev_err(&f34->fn->dev, "%s: Failed to write transfer length\n",
return ret;
}
+ /*
+ * rmi_f34v7_check_command_status() can't be used here, as this
+ * function is called before IRQs are available
+ */
timeout = msecs_to_jiffies(F34_WRITE_WAIT_MS);
while (time_before(jiffies, timeout)) {
usleep_range(5000, 6000);
}
ret = rmi_read_block(f34->fn->rmi_dev,
- base + f34->v7.off.payload,
+ base + V7_PAYLOAD_OFFSET,
f34->v7.read_config_buf,
f34->v7.partition_table_bytes);
if (ret < 0) {
rmi_dbg(RMI_DEBUG_FN, &f34->fn->dev, "%s: f34->v7.block_size = %d\n",
__func__, f34->v7.block_size);
- f34->v7.off.flash_status = V7_FLASH_STATUS_OFFSET;
- f34->v7.off.partition_id = V7_PARTITION_ID_OFFSET;
- f34->v7.off.block_number = V7_BLOCK_NUMBER_OFFSET;
- f34->v7.off.transfer_length = V7_TRANSFER_LENGTH_OFFSET;
- f34->v7.off.flash_cmd = V7_COMMAND_OFFSET;
- f34->v7.off.payload = V7_PAYLOAD_OFFSET;
-
f34->v7.has_display_cfg = query_1_7.partition_support[1] & HAS_DISP_CFG;
f34->v7.has_guest_code =
query_1_7.partition_support[1] & HAS_GUEST_CODE;
return 0;
}
-static int rmi_f34v7_check_ui_firmware_size(struct f34_data *f34)
-{
- u16 block_count;
-
- block_count = f34->v7.img.ui_firmware.size / f34->v7.block_size;
- f34->update_size += block_count;
-
- if (block_count != f34->v7.blkcount.ui_firmware) {
- dev_err(&f34->fn->dev,
- "UI firmware size mismatch: %d != %d\n",
- block_count, f34->v7.blkcount.ui_firmware);
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int rmi_f34v7_check_ui_config_size(struct f34_data *f34)
-{
- u16 block_count;
-
- block_count = f34->v7.img.ui_config.size / f34->v7.block_size;
- f34->update_size += block_count;
-
- if (block_count != f34->v7.blkcount.ui_config) {
- dev_err(&f34->fn->dev, "UI config size mismatch\n");
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int rmi_f34v7_check_dp_config_size(struct f34_data *f34)
-{
- u16 block_count;
-
- block_count = f34->v7.img.dp_config.size / f34->v7.block_size;
- f34->update_size += block_count;
-
- if (block_count != f34->v7.blkcount.dp_config) {
- dev_err(&f34->fn->dev, "Display config size mismatch\n");
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int rmi_f34v7_check_guest_code_size(struct f34_data *f34)
-{
- u16 block_count;
-
- block_count = f34->v7.img.guest_code.size / f34->v7.block_size;
- f34->update_size += block_count;
-
- if (block_count != f34->v7.blkcount.guest_code) {
- dev_err(&f34->fn->dev, "Guest code size mismatch\n");
- return -EINVAL;
- }
-
- return 0;
-}
-
static int rmi_f34v7_check_bl_config_size(struct f34_data *f34)
{
u16 block_count;
return 0;
}
-static int rmi_f34v7_erase_config(struct f34_data *f34)
-{
- int ret;
-
- dev_info(&f34->fn->dev, "Erasing config...\n");
-
- init_completion(&f34->v7.cmd_done);
-
- switch (f34->v7.config_area) {
- case v7_UI_CONFIG_AREA:
- ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_UI_CONFIG);
- if (ret < 0)
- return ret;
- break;
- case v7_DP_CONFIG_AREA:
- ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_DISP_CONFIG);
- if (ret < 0)
- return ret;
- break;
- case v7_BL_CONFIG_AREA:
- ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_BL_CONFIG);
- if (ret < 0)
- return ret;
- break;
- }
-
- ret = rmi_f34v7_wait_for_idle(f34, F34_ERASE_WAIT_MS);
- if (ret < 0)
- return ret;
-
- return 0;
-}
-
-static int rmi_f34v7_erase_guest_code(struct f34_data *f34)
-{
- int ret;
-
- dev_info(&f34->fn->dev, "Erasing guest code...\n");
-
- init_completion(&f34->v7.cmd_done);
-
- ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_GUEST_CODE);
- if (ret < 0)
- return ret;
-
- ret = rmi_f34v7_wait_for_idle(f34, F34_ERASE_WAIT_MS);
- if (ret < 0)
- return ret;
-
- return 0;
-}
-
static int rmi_f34v7_erase_all(struct f34_data *f34)
{
int ret;
init_completion(&f34->v7.cmd_done);
- ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_UI_FIRMWARE);
- if (ret < 0)
- return ret;
-
- ret = rmi_f34v7_wait_for_idle(f34, F34_ERASE_WAIT_MS);
+ ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_ALL);
if (ret < 0)
return ret;
- f34->v7.config_area = v7_UI_CONFIG_AREA;
- ret = rmi_f34v7_erase_config(f34);
+ ret = rmi_f34v7_check_command_status(f34, F34_ERASE_WAIT_MS);
if (ret < 0)
return ret;
- if (f34->v7.has_display_cfg) {
- f34->v7.config_area = v7_DP_CONFIG_AREA;
- ret = rmi_f34v7_erase_config(f34);
- if (ret < 0)
- return ret;
- }
-
- if (f34->v7.new_partition_table && f34->v7.has_guest_code) {
- ret = rmi_f34v7_erase_guest_code(f34);
- if (ret < 0)
- return ret;
- }
-
return 0;
}
return ret;
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.block_number,
+ base + V7_BLOCK_NUMBER_OFFSET,
&block_number, sizeof(block_number));
if (ret < 0) {
dev_err(&f34->fn->dev, "%s: Failed to write block number\n",
put_unaligned_le16(transfer, &length);
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.transfer_length,
+ base + V7_TRANSFER_LENGTH_OFFSET,
&length, sizeof(length));
if (ret < 0) {
dev_err(&f34->fn->dev,
if (ret < 0)
return ret;
- ret = rmi_f34v7_wait_for_idle(f34, F34_ENABLE_WAIT_MS);
+ ret = rmi_f34v7_check_command_status(f34, F34_ENABLE_WAIT_MS);
if (ret < 0)
return ret;
ret = rmi_read_block(f34->fn->rmi_dev,
- base + f34->v7.off.payload,
+ base + V7_PAYLOAD_OFFSET,
&f34->v7.read_config_buf[index],
transfer * f34->v7.block_size);
if (ret < 0) {
return ret;
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.block_number,
+ base + V7_BLOCK_NUMBER_OFFSET,
&block_number, sizeof(block_number));
if (ret < 0) {
dev_err(&f34->fn->dev, "%s: Failed to write block number\n",
init_completion(&f34->v7.cmd_done);
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.transfer_length,
+ base + V7_TRANSFER_LENGTH_OFFSET,
&length, sizeof(length));
if (ret < 0) {
dev_err(&f34->fn->dev,
return ret;
ret = rmi_write_block(f34->fn->rmi_dev,
- base + f34->v7.off.payload,
+ base + V7_PAYLOAD_OFFSET,
block_ptr, transfer * f34->v7.block_size);
if (ret < 0) {
dev_err(&f34->fn->dev,
return ret;
}
- ret = rmi_f34v7_wait_for_idle(f34, F34_ENABLE_WAIT_MS);
+ ret = rmi_f34v7_check_command_status(f34, F34_ENABLE_WAIT_MS);
if (ret < 0)
return ret;
init_completion(&f34->v7.cmd_done);
- ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_FLASH_CONFIG);
- if (ret < 0)
- return ret;
-
- rmi_dbg(RMI_DEBUG_FN, &f34->fn->dev,
- "%s: Erase flash config command written\n", __func__);
-
- ret = rmi_f34v7_wait_for_idle(f34, F34_WRITE_WAIT_MS);
- if (ret < 0)
- return ret;
-
ret = rmi_f34v7_write_config(f34);
if (ret < 0)
return ret;
if (ret < 0)
return ret;
- ret = rmi_f34v7_erase_config(f34);
- if (ret < 0)
- return ret;
-
ret = rmi_f34v7_write_flash_config(f34);
if (ret < 0)
return ret;
blk_count, v7_CMD_WRITE_FW);
}
-static void rmi_f34v7_compare_partition_tables(struct f34_data *f34)
-{
- if (f34->v7.phyaddr.ui_firmware != f34->v7.img.phyaddr.ui_firmware) {
- f34->v7.new_partition_table = true;
- return;
- }
-
- if (f34->v7.phyaddr.ui_config != f34->v7.img.phyaddr.ui_config) {
- f34->v7.new_partition_table = true;
- return;
- }
-
- if (f34->v7.has_display_cfg &&
- f34->v7.phyaddr.dp_config != f34->v7.img.phyaddr.dp_config) {
- f34->v7.new_partition_table = true;
- return;
- }
-
- if (f34->v7.has_guest_code &&
- f34->v7.phyaddr.guest_code != f34->v7.img.phyaddr.guest_code) {
- f34->v7.new_partition_table = true;
- return;
- }
-
- f34->v7.new_partition_table = false;
-}
-
static void rmi_f34v7_parse_img_header_10_bl_container(struct f34_data *f34,
const void *image)
{
rmi_f34v7_parse_partition_table(f34, f34->v7.img.fl_config.data,
&f34->v7.img.blkcount, &f34->v7.img.phyaddr);
- rmi_f34v7_compare_partition_tables(f34);
-
return 0;
}
ret = rmi_f34v7_parse_image_info(f34);
if (ret < 0)
- goto fail;
-
- if (!f34->v7.new_partition_table) {
- ret = rmi_f34v7_check_ui_firmware_size(f34);
- if (ret < 0)
- goto fail;
-
- ret = rmi_f34v7_check_ui_config_size(f34);
- if (ret < 0)
- goto fail;
-
- if (f34->v7.has_display_cfg &&
- f34->v7.img.contains_display_cfg) {
- ret = rmi_f34v7_check_dp_config_size(f34);
- if (ret < 0)
- goto fail;
- }
+ return ret;
- if (f34->v7.has_guest_code && f34->v7.img.contains_guest_code) {
- ret = rmi_f34v7_check_guest_code_size(f34);
- if (ret < 0)
- goto fail;
- }
- } else {
- ret = rmi_f34v7_check_bl_config_size(f34);
- if (ret < 0)
- goto fail;
- }
+ ret = rmi_f34v7_check_bl_config_size(f34);
+ if (ret < 0)
+ return ret;
ret = rmi_f34v7_erase_all(f34);
if (ret < 0)
- goto fail;
+ return ret;
- if (f34->v7.new_partition_table) {
- ret = rmi_f34v7_write_partition_table(f34);
- if (ret < 0)
- goto fail;
- dev_info(&f34->fn->dev, "%s: Partition table programmed\n",
- __func__);
- }
+ ret = rmi_f34v7_write_partition_table(f34);
+ if (ret < 0)
+ return ret;
+ dev_info(&f34->fn->dev, "%s: Partition table programmed\n", __func__);
+
+ /*
+ * Reset to reload partition table - as the previous firmware has been
+ * erased, we remain in bootloader mode.
+ */
+ ret = rmi_scan_pdt(f34->fn->rmi_dev, NULL, rmi_initial_reset);
+ if (ret < 0)
+ dev_warn(&f34->fn->dev, "RMI reset failed!\n");
dev_info(&f34->fn->dev, "Writing firmware (%d bytes)...\n",
f34->v7.img.ui_firmware.size);
ret = rmi_f34v7_write_firmware(f34);
if (ret < 0)
- goto fail;
+ return ret;
dev_info(&f34->fn->dev, "Writing config (%d bytes)...\n",
f34->v7.img.ui_config.size);
f34->v7.config_area = v7_UI_CONFIG_AREA;
ret = rmi_f34v7_write_ui_config(f34);
if (ret < 0)
- goto fail;
+ return ret;
if (f34->v7.has_display_cfg && f34->v7.img.contains_display_cfg) {
dev_info(&f34->fn->dev, "Writing display config...\n");
ret = rmi_f34v7_write_dp_config(f34);
if (ret < 0)
- goto fail;
+ return ret;
}
- if (f34->v7.new_partition_table) {
- if (f34->v7.has_guest_code && f34->v7.img.contains_guest_code) {
- dev_info(&f34->fn->dev, "Writing guest code...\n");
+ if (f34->v7.has_guest_code && f34->v7.img.contains_guest_code) {
+ dev_info(&f34->fn->dev, "Writing guest code...\n");
- ret = rmi_f34v7_write_guest_code(f34);
- if (ret < 0)
- goto fail;
- }
+ ret = rmi_f34v7_write_guest_code(f34);
+ if (ret < 0)
+ return ret;
}
-fail:
- return ret;
+ return 0;
}
static int rmi_f34v7_enter_flash_prog(struct f34_data *f34)
if (ret < 0)
return ret;
- if (f34->v7.in_bl_mode)
+ if (f34->v7.in_bl_mode) {
+ dev_info(&f34->fn->dev, "%s: Device in bootloader mode\n",
+ __func__);
return 0;
+ }
init_completion(&f34->v7.cmd_done);
if (ret < 0)
return ret;
- ret = rmi_f34v7_wait_for_idle(f34, F34_ENABLE_WAIT_MS);
+ ret = rmi_f34v7_check_command_status(f34, F34_ENABLE_WAIT_MS);
if (ret < 0)
return ret;
{
int ret = 0;
- f34->fn->rmi_dev->driver->set_irq_bits(f34->fn->rmi_dev, f34->fn->irq_mask);
-
f34->v7.config_area = v7_UI_CONFIG_AREA;
f34->v7.image = fw->data;
ret = rmi_f34v7_parse_image_info(f34);
if (ret < 0)
- goto exit;
-
- if (!f34->v7.force_update && f34->v7.new_partition_table) {
- dev_err(&f34->fn->dev, "%s: Partition table mismatch\n",
- __func__);
- ret = -EINVAL;
- goto exit;
- }
+ return ret;
dev_info(&f34->fn->dev, "Firmware image OK\n");
- ret = rmi_f34v7_read_flash_status(f34);
- if (ret < 0)
- goto exit;
-
- if (f34->v7.in_bl_mode) {
- dev_info(&f34->fn->dev, "%s: Device in bootloader mode\n",
- __func__);
- }
-
- rmi_f34v7_enter_flash_prog(f34);
-
- return 0;
-
-exit:
- return ret;
+ return rmi_f34v7_enter_flash_prog(f34);
}
int rmi_f34v7_probe(struct f34_data *f34)
if (ret < 0)
return ret;
- f34->v7.force_update = true;
return 0;
}
{
struct f54_data *f54 = video_drvdata(file);
- strlcpy(cap->driver, F54_NAME, sizeof(cap->driver));
- strlcpy(cap->card, SYNAPTICS_INPUT_DEVICE_NAME, sizeof(cap->card));
+ strscpy(cap->driver, F54_NAME, sizeof(cap->driver));
+ strscpy(cap->card, SYNAPTICS_INPUT_DEVICE_NAME, sizeof(cap->card));
snprintf(cap->bus_info, sizeof(cap->bus_info),
"rmi4:%s", dev_name(&f54->fn->dev));
i->type = V4L2_INPUT_TYPE_TOUCH;
- strlcpy(i->name, rmi_f54_report_type_names[reptype], sizeof(i->name));
+ strscpy(i->name, rmi_f54_report_type_names[reptype], sizeof(i->name));
return 0;
}
rmi_f54_set_input(f54, 0);
/* register video device */
- strlcpy(f54->v4l2.name, F54_NAME, sizeof(f54->v4l2.name));
+ strscpy(f54->v4l2.name, F54_NAME, sizeof(f54->v4l2.name));
ret = v4l2_device_register(&fn->dev, &f54->v4l2);
if (ret) {
dev_err(&fn->dev, "Unable to register video dev.\n");
serio->write = altera_ps2_write;
serio->open = altera_ps2_open;
serio->close = altera_ps2_close;
- strlcpy(serio->name, dev_name(&pdev->dev), sizeof(serio->name));
- strlcpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys));
+ strscpy(serio->name, dev_name(&pdev->dev), sizeof(serio->name));
+ strscpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys));
serio->port_data = ps2if;
serio->dev.parent = &pdev->dev;
ps2if->io = serio;
io->write = amba_kmi_write;
io->open = amba_kmi_open;
io->close = amba_kmi_close;
- strlcpy(io->name, dev_name(&dev->dev), sizeof(io->name));
- strlcpy(io->phys, dev_name(&dev->dev), sizeof(io->phys));
+ strscpy(io->name, dev_name(&dev->dev), sizeof(io->name));
+ strscpy(io->phys, dev_name(&dev->dev), sizeof(io->phys));
io->port_data = kmi;
io->dev.parent = &dev->dev;
serio->id.type = SERIO_8042;
serio->open = ams_delta_serio_open;
serio->close = ams_delta_serio_close;
- strlcpy(serio->name, "AMS DELTA keyboard adapter", sizeof(serio->name));
- strlcpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys));
+ strscpy(serio->name, "AMS DELTA keyboard adapter", sizeof(serio->name));
+ strscpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys));
serio->dev.parent = &pdev->dev;
serio->port_data = priv;
priv->io->close = apbps2_close;
priv->io->write = apbps2_write;
priv->io->port_data = priv;
- strlcpy(priv->io->name, "APBPS2 PS/2", sizeof(priv->io->name));
+ strscpy(priv->io->name, "APBPS2 PS/2", sizeof(priv->io->name));
snprintf(priv->io->phys, sizeof(priv->io->phys),
"apbps2_%d", apbps2_idx++);
* 82C710 C&T mouse port chip driver for Linux
*/
-/*
- */
-
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/ioport.h>
ct82c710_port->open = ct82c710_open;
ct82c710_port->close = ct82c710_close;
ct82c710_port->write = ct82c710_write;
- strlcpy(ct82c710_port->name, "C&T 82c710 mouse port",
+ strscpy(ct82c710_port->name, "C&T 82c710 mouse port",
sizeof(ct82c710_port->name));
snprintf(ct82c710_port->phys, sizeof(ct82c710_port->phys),
"isa%16llx/serio0", (unsigned long long)CT82C710_DATA);
snprintf(serio->name, sizeof(serio->name), "gsc-ps2-%s",
(ps2port->id == GSC_ID_KEYBOARD) ? "keyboard" : "mouse");
- strlcpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys));
+ strscpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys));
serio->id.type = SERIO_8042;
serio->write = gscps2_write;
serio->open = gscps2_open;
hv_serio->dev.parent = &hv_dev->device;
hv_serio->id.type = SERIO_8042_XL;
hv_serio->port_data = kbd_dev;
- strlcpy(hv_serio->name, dev_name(&hv_dev->device),
+ strscpy(hv_serio->name, dev_name(&hv_dev->device),
sizeof(hv_serio->name));
- strlcpy(hv_serio->phys, dev_name(&hv_dev->device),
+ strscpy(hv_serio->phys, dev_name(&hv_dev->device),
sizeof(hv_serio->phys));
hv_serio->start = hv_kbd_start;
/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef _I8042_X86IA64IO_H
-#define _I8042_X86IA64IO_H
+#ifndef _I8042_ACPIPNPIO_H
+#define _I8042_ACPIPNPIO_H
+#include <linux/acpi.h>
#ifdef CONFIG_X86
#include <asm/x86_init.h>
static void i8042_pnp_id_to_string(struct pnp_id *id, char *dst, int dst_size)
{
- strlcpy(dst, "PNP:", dst_size);
+ strscpy(dst, "PNP:", dst_size);
while (id) {
strlcat(dst, " ", dst_size);
if (pnp_irq_valid(dev,0))
i8042_pnp_kbd_irq = pnp_irq(dev, 0);
- strlcpy(i8042_pnp_kbd_name, did->id, sizeof(i8042_pnp_kbd_name));
+ strscpy(i8042_pnp_kbd_name, did->id, sizeof(i8042_pnp_kbd_name));
if (strlen(pnp_dev_name(dev))) {
strlcat(i8042_pnp_kbd_name, ":", sizeof(i8042_pnp_kbd_name));
strlcat(i8042_pnp_kbd_name, pnp_dev_name(dev), sizeof(i8042_pnp_kbd_name));
if (pnp_irq_valid(dev, 0))
i8042_pnp_aux_irq = pnp_irq(dev, 0);
- strlcpy(i8042_pnp_aux_name, did->id, sizeof(i8042_pnp_aux_name));
+ strscpy(i8042_pnp_aux_name, did->id, sizeof(i8042_pnp_aux_name));
if (strlen(pnp_dev_name(dev))) {
strlcat(i8042_pnp_aux_name, ":", sizeof(i8042_pnp_aux_name));
strlcat(i8042_pnp_aux_name, pnp_dev_name(dev), sizeof(i8042_pnp_aux_name));
return -ENODEV;
#else
pr_info("PNP: No PS/2 controller found.\n");
+#if defined(__loongarch__)
+ if (acpi_disabled == 0)
+ return -ENODEV;
+#else
if (x86_platform.legacy.i8042 !=
X86_LEGACY_I8042_EXPECTED_PRESENT)
return -ENODEV;
+#endif
pr_info("Probing ports directly.\n");
return 0;
#endif
i8042_pnp_exit();
}
-#endif /* _I8042_X86IA64IO_H */
+#endif /* _I8042_ACPIPNPIO_H */
#define _I8042_SPARCIO_H
#include <linux/of_device.h>
+#include <linux/types.h>
#include <asm/io.h>
#include <asm/oplib.h>
.remove = sparc_i8042_remove,
};
-static int __init i8042_platform_init(void)
+static bool i8042_is_mr_coffee(void)
{
- struct device_node *root = of_find_node_by_path("/");
- const char *name = of_get_property(root, "name", NULL);
+ struct device_node *root;
+ const char *name;
+ bool is_mr_coffee;
+
+ root = of_find_node_by_path("/");
+
+ name = of_get_property(root, "name", NULL);
+ is_mr_coffee = name && !strcmp(name, "SUNW,JavaStation-1");
- if (name && !strcmp(name, "SUNW,JavaStation-1")) {
+ of_node_put(root);
+
+ return is_mr_coffee;
+}
+
+static int __init i8042_platform_init(void)
+{
+ if (i8042_is_mr_coffee()) {
/* Hardcoded values for MrCoffee. */
i8042_kbd_irq = i8042_aux_irq = 13 | 0x20;
kbd_iobase = ioremap(0x71300060, 8);
static inline void i8042_platform_exit(void)
{
- struct device_node *root = of_find_node_by_path("/");
- const char *name = of_get_property(root, "name", NULL);
-
- if (!name || strcmp(name, "SUNW,JavaStation-1"))
+ if (!i8042_is_mr_coffee())
platform_driver_unregister(&sparc_i8042_driver);
}
serio->ps2_cmd_mutex = &i8042_mutex;
serio->port_data = port;
serio->dev.parent = &i8042_platform_device->dev;
- strlcpy(serio->name, "i8042 KBD port", sizeof(serio->name));
- strlcpy(serio->phys, I8042_KBD_PHYS_DESC, sizeof(serio->phys));
- strlcpy(serio->firmware_id, i8042_kbd_firmware_id,
+ strscpy(serio->name, "i8042 KBD port", sizeof(serio->name));
+ strscpy(serio->phys, I8042_KBD_PHYS_DESC, sizeof(serio->phys));
+ strscpy(serio->firmware_id, i8042_kbd_firmware_id,
sizeof(serio->firmware_id));
set_primary_fwnode(&serio->dev, i8042_kbd_fwnode);
serio->port_data = port;
serio->dev.parent = &i8042_platform_device->dev;
if (idx < 0) {
- strlcpy(serio->name, "i8042 AUX port", sizeof(serio->name));
- strlcpy(serio->phys, I8042_AUX_PHYS_DESC, sizeof(serio->phys));
- strlcpy(serio->firmware_id, i8042_aux_firmware_id,
+ strscpy(serio->name, "i8042 AUX port", sizeof(serio->name));
+ strscpy(serio->phys, I8042_AUX_PHYS_DESC, sizeof(serio->phys));
+ strscpy(serio->firmware_id, i8042_aux_firmware_id,
sizeof(serio->firmware_id));
serio->close = i8042_port_close;
} else {
snprintf(serio->name, sizeof(serio->name), "i8042 AUX%d port", idx);
snprintf(serio->phys, sizeof(serio->phys), I8042_MUX_PHYS_DESC, idx + 1);
- strlcpy(serio->firmware_id, i8042_aux_firmware_id,
+ strscpy(serio->firmware_id, i8042_aux_firmware_id,
sizeof(serio->firmware_id));
}
#include "i8042-snirm.h"
#elif defined(CONFIG_SPARC)
#include "i8042-sparcio.h"
-#elif defined(CONFIG_X86) || defined(CONFIG_IA64)
-#include "i8042-x86ia64io.h"
+#elif defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH)
+#include "i8042-acpipnpio.h"
#else
#include "i8042-io.h"
#endif
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/input.h>
+#include <linux/kmsan-checks.h>
#include <linux/serio.h>
#include <linux/i8042.h>
#include <linux/libps2.h>
serio_pause_rx(ps2dev->serio);
- if (param)
+ if (param) {
for (i = 0; i < receive; i++)
param[i] = ps2dev->cmdbuf[(receive - 1) - i];
+ kmsan_unpoison_memory(param, receive);
+ }
if (ps2dev->cmdcnt &&
(command != PS2_CMD_RESET_BAT || ps2dev->cmdcnt != 1)) {
kb_serio->close = olpc_apsp_close;
kb_serio->port_data = priv;
kb_serio->dev.parent = &pdev->dev;
- strlcpy(kb_serio->name, "sp keyboard", sizeof(kb_serio->name));
- strlcpy(kb_serio->phys, "sp/serio0", sizeof(kb_serio->phys));
+ strscpy(kb_serio->name, "sp keyboard", sizeof(kb_serio->name));
+ strscpy(kb_serio->phys, "sp/serio0", sizeof(kb_serio->phys));
priv->kbio = kb_serio;
serio_register_port(kb_serio);
pad_serio->close = olpc_apsp_close;
pad_serio->port_data = priv;
pad_serio->dev.parent = &pdev->dev;
- strlcpy(pad_serio->name, "sp touchpad", sizeof(pad_serio->name));
- strlcpy(pad_serio->phys, "sp/serio1", sizeof(pad_serio->phys));
+ strscpy(pad_serio->name, "sp touchpad", sizeof(pad_serio->name));
+ strscpy(pad_serio->phys, "sp/serio1", sizeof(pad_serio->phys));
priv->padio = pad_serio;
serio_register_port(pad_serio);
if (serio) {
serio->id.type = parkbd_mode;
serio->write = parkbd_write;
- strlcpy(serio->name, "PARKBD AT/XT keyboard adapter", sizeof(serio->name));
+ strscpy(serio->name, "PARKBD AT/XT keyboard adapter", sizeof(serio->name));
snprintf(serio->phys, sizeof(serio->phys), "%s/serio0", parkbd_dev->port->name);
}
serio->write = pcips2_write;
serio->open = pcips2_open;
serio->close = pcips2_close;
- strlcpy(serio->name, pci_name(dev), sizeof(serio->name));
- strlcpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys));
+ strscpy(serio->name, pci_name(dev), sizeof(serio->name));
+ strscpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys));
serio->port_data = ps2if;
serio->dev.parent = &dev->dev;
ps2if->io = serio;
serio->write = drvdata->write_enable ? ps2_gpio_write : NULL;
serio->port_data = drvdata;
serio->dev.parent = dev;
- strlcpy(serio->name, dev_name(dev), sizeof(serio->name));
- strlcpy(serio->phys, dev_name(dev), sizeof(serio->phys));
+ strscpy(serio->name, dev_name(dev), sizeof(serio->name));
+ strscpy(serio->phys, dev_name(dev), sizeof(serio->phys));
drvdata->serio = serio;
drvdata->dev = dev;
if (!serio)
return -ENOMEM;
- strlcpy(serio->name, "TQC PS/2 Multiplexer", sizeof(serio->name));
+ strscpy(serio->name, "TQC PS/2 Multiplexer", sizeof(serio->name));
snprintf(serio->phys, sizeof(serio->phys),
"%s/port%d", mx_serio->phys, i);
serio->id.type = SERIO_8042;
* Q40 PS/2 keyboard controller driver for Linux/m68k
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/serio.h>
#include <linux/interrupt.h>
port->close = q40kbd_close;
port->port_data = q40kbd;
port->dev.parent = &pdev->dev;
- strlcpy(port->name, "Q40 Kbd Port", sizeof(port->name));
- strlcpy(port->phys, "Q40", sizeof(port->phys));
+ strscpy(port->name, "Q40 Kbd Port", sizeof(port->name));
+ strscpy(port->phys, "Q40", sizeof(port->phys));
q40kbd_stop();
* Acorn RiscPC PS/2 keyboard controller driver for Linux/ARM
*/
-/*
- */
-
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/serio.h>
serio->close = rpckbd_close;
serio->dev.parent = &dev->dev;
serio->port_data = rpckbd;
- strlcpy(serio->name, "RiscPC PS/2 kbd port", sizeof(serio->name));
- strlcpy(serio->phys, "rpckbd/serio0", sizeof(serio->phys));
+ strscpy(serio->name, "RiscPC PS/2 kbd port", sizeof(serio->name));
+ strscpy(serio->phys, "rpckbd/serio0", sizeof(serio->phys));
platform_set_drvdata(dev, serio);
serio_register_port(serio);
serio->write = ps2_write;
serio->open = ps2_open;
serio->close = ps2_close;
- strlcpy(serio->name, dev_name(&dev->dev), sizeof(serio->name));
- strlcpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys));
+ strscpy(serio->name, dev_name(&dev->dev), sizeof(serio->name));
+ strscpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys));
serio->port_data = ps2if;
serio->dev.parent = &dev->dev;
ps2if->io = serio;
* Copyright (c) 2003 Daniele Bellucci
*/
-/*
- */
-
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/stddef.h>
if (!serio)
return -ENOMEM;
- strlcpy(serio->name, "Serial port", sizeof(serio->name));
+ strscpy(serio->name, "Serial port", sizeof(serio->name));
snprintf(serio->phys, sizeof(serio->phys), "%s/serio0", tty_name(tty));
serio->id = serport->id;
serio->id.type = SERIO_RS232;
serio->close = sun4i_ps2_close;
serio->port_data = drvdata;
serio->dev.parent = dev;
- strlcpy(serio->name, dev_name(dev), sizeof(serio->name));
- strlcpy(serio->phys, dev_name(dev), sizeof(serio->phys));
+ strscpy(serio->name, dev_name(dev), sizeof(serio->name));
+ strscpy(serio->phys, dev_name(dev), sizeof(serio->phys));
/* shutoff interrupt */
writel(0, drvdata->reg_base + PS2_REG_GCTL);
* v3.2 - Added sysfs support
*/
-/*
- */
-
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
acecad->input = input_dev;
if (dev->manufacturer)
- strlcpy(acecad->name, dev->manufacturer, sizeof(acecad->name));
+ strscpy(acecad->name, dev->manufacturer, sizeof(acecad->name));
if (dev->product) {
if (dev->manufacturer)
static DEVICE_ATTR(firmware_code, S_IRUGO, show_firmwareCode, NULL);
-static struct attribute *aiptek_attributes[] = {
+static struct attribute *aiptek_dev_attrs[] = {
&dev_attr_size.attr,
&dev_attr_pointer_mode.attr,
&dev_attr_coordinate_mode.attr,
NULL
};
-static const struct attribute_group aiptek_attribute_group = {
- .attrs = aiptek_attributes,
-};
+ATTRIBUTE_GROUPS(aiptek_dev);
/***********************************************************************
* This routine is called when a tablet has been identified. It basically
*/
usb_set_intfdata(intf, aiptek);
- /* Set up the sysfs files
- */
- err = sysfs_create_group(&intf->dev.kobj, &aiptek_attribute_group);
- if (err) {
- dev_warn(&intf->dev, "cannot create sysfs group err: %d\n",
- err);
- goto fail3;
- }
-
/* Register the tablet as an Input Device
*/
err = input_register_device(aiptek->inputdev);
if (err) {
dev_warn(&intf->dev,
"input_register_device returned err: %d\n", err);
- goto fail4;
+ goto fail3;
}
return 0;
- fail4: sysfs_remove_group(&intf->dev.kobj, &aiptek_attribute_group);
fail3: usb_free_urb(aiptek->urb);
fail2: usb_free_coherent(usbdev, AIPTEK_PACKET_LENGTH, aiptek->data,
aiptek->data_dma);
*/
usb_kill_urb(aiptek->urb);
input_unregister_device(aiptek->inputdev);
- sysfs_remove_group(&intf->dev.kobj, &aiptek_attribute_group);
usb_free_urb(aiptek->urb);
usb_free_coherent(interface_to_usbdev(intf),
AIPTEK_PACKET_LENGTH,
.probe = aiptek_probe,
.disconnect = aiptek_disconnect,
.id_table = aiptek_ids,
+ .dev_groups = aiptek_dev_groups,
};
module_usb_driver(aiptek_driver);
* Copyright (c) 2010 Xing Wei <weixing@hanwang.com.cn>
*/
-/*
- */
-
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/slab.h>
usb_make_path(dev, hanwang->phys, sizeof(hanwang->phys));
strlcat(hanwang->phys, "/input0", sizeof(hanwang->phys));
- strlcpy(hanwang->name, hanwang->features->name, sizeof(hanwang->name));
+ strscpy(hanwang->name, hanwang->features->name, sizeof(hanwang->name));
input_dev->name = hanwang->name;
input_dev->phys = hanwang->phys;
usb_to_input_id(dev, &input_dev->id);
pegasus->irq->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
if (dev->manufacturer)
- strlcpy(pegasus->name, dev->manufacturer,
+ strscpy(pegasus->name, dev->manufacturer,
sizeof(pegasus->name));
if (dev->product) {
config TOUCHSCREEN_COLIBRI_VF50
tristate "Toradex Colibri on board touchscreen driver"
- depends on IIO && VF610_ADC
+ depends on IIO
depends on GPIOLIB || COMPILE_TEST
help
Say Y here if you have a Colibri VF50 and plan to use
{
struct mxt_data *data = video_drvdata(file);
- strlcpy(cap->driver, "atmel_mxt_ts", sizeof(cap->driver));
- strlcpy(cap->card, "atmel_mxt_ts touch", sizeof(cap->card));
+ strscpy(cap->driver, "atmel_mxt_ts", sizeof(cap->driver));
+ strscpy(cap->card, "atmel_mxt_ts touch", sizeof(cap->card));
snprintf(cap->bus_info, sizeof(cap->bus_info),
"I2C:%s", dev_name(&data->client->dev));
return 0;
switch (i->index) {
case MXT_V4L_INPUT_REFS:
- strlcpy(i->name, "Mutual Capacitance References",
+ strscpy(i->name, "Mutual Capacitance References",
sizeof(i->name));
break;
case MXT_V4L_INPUT_DELTAS:
- strlcpy(i->name, "Mutual Capacitance Deltas", sizeof(i->name));
+ strscpy(i->name, "Mutual Capacitance Deltas", sizeof(i->name));
break;
}
* Copyright (c) 2008 QUALCOMM USA, INC.
*/
+#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/i2c.h>
#include <linux/mutex.h>
#include <linux/delay.h>
-#include <linux/gpio.h>
-#include <linux/input/auo-pixcir-ts.h>
+#include <linux/gpio/consumer.h>
#include <linux/of.h>
-#include <linux/of_gpio.h>
+#include <linux/property.h>
/*
* Coordinate calculation:
#define AUO_PIXCIR_INT_RELEASE (1 << 4)
#define AUO_PIXCIR_INT_ENABLE (1 << 3)
#define AUO_PIXCIR_INT_POL_HIGH (1 << 2)
+
+/*
+ * Interrupt modes:
+ * periodical: interrupt is asserted periodicaly
+ * compare coordinates: interrupt is asserted when coordinates change
+ * indicate touch: interrupt is asserted during touch
+ */
+#define AUO_PIXCIR_INT_PERIODICAL 0x00
+#define AUO_PIXCIR_INT_COMP_COORD 0x01
+#define AUO_PIXCIR_INT_TOUCH_IND 0x02
#define AUO_PIXCIR_INT_MODE_MASK 0x03
/*
struct auo_pixcir_ts {
struct i2c_client *client;
struct input_dev *input;
- const struct auo_pixcir_ts_platdata *pdata;
+ struct gpio_desc *gpio_int;
+ struct gpio_desc *gpio_rst;
char phys[32];
- /* special handling for touch_indicate interupt mode */
+ unsigned int x_max;
+ unsigned int y_max;
+
+ /* special handling for touch_indicate interrupt mode */
bool touch_ind_mode;
wait_queue_head_t wait;
struct auo_point_t *point)
{
struct i2c_client *client = ts->client;
- const struct auo_pixcir_ts_platdata *pdata = ts->pdata;
uint8_t raw_coord[8];
uint8_t raw_area[4];
int i, ret;
point[i].coord_y =
raw_coord[4 * i + 3] << 8 | raw_coord[4 * i + 2];
- if (point[i].coord_x > pdata->x_max ||
- point[i].coord_y > pdata->y_max) {
+ if (point[i].coord_x > ts->x_max ||
+ point[i].coord_y > ts->y_max) {
dev_warn(&client->dev, "coordinates (%d,%d) invalid\n",
point[i].coord_x, point[i].coord_y);
point[i].coord_x = point[i].coord_y = 0;
static irqreturn_t auo_pixcir_interrupt(int irq, void *dev_id)
{
struct auo_pixcir_ts *ts = dev_id;
- const struct auo_pixcir_ts_platdata *pdata = ts->pdata;
struct auo_point_t point[AUO_PIXCIR_REPORT_POINTS];
int i;
int ret;
/* check for up event in touch touch_ind_mode */
if (ts->touch_ind_mode) {
- if (gpio_get_value(pdata->gpio_int) == 0) {
+ if (gpiod_get_value_cansleep(ts->gpio_int) == 0) {
input_mt_sync(ts->input);
input_report_key(ts->input, BTN_TOUCH, 0);
input_sync(ts->input);
return 0;
}
-static int auo_pixcir_int_config(struct auo_pixcir_ts *ts,
- int int_setting)
+static int auo_pixcir_int_config(struct auo_pixcir_ts *ts, int int_setting)
{
struct i2c_client *client = ts->client;
- const struct auo_pixcir_ts_platdata *pdata = ts->pdata;
int ret;
ret = i2c_smbus_read_byte_data(client, AUO_PIXCIR_REG_INT_SETTING);
return ret;
}
- ts->touch_ind_mode = pdata->int_setting == AUO_PIXCIR_INT_TOUCH_IND;
+ ts->touch_ind_mode = int_setting == AUO_PIXCIR_INT_TOUCH_IND;
return 0;
}
static SIMPLE_DEV_PM_OPS(auo_pixcir_pm_ops,
auo_pixcir_suspend, auo_pixcir_resume);
-#ifdef CONFIG_OF
-static struct auo_pixcir_ts_platdata *auo_pixcir_parse_dt(struct device *dev)
-{
- struct auo_pixcir_ts_platdata *pdata;
- struct device_node *np = dev->of_node;
-
- if (!np)
- return ERR_PTR(-ENOENT);
-
- pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
- if (!pdata)
- return ERR_PTR(-ENOMEM);
-
- pdata->gpio_int = of_get_gpio(np, 0);
- if (!gpio_is_valid(pdata->gpio_int)) {
- dev_err(dev, "failed to get interrupt gpio\n");
- return ERR_PTR(-EINVAL);
- }
-
- pdata->gpio_rst = of_get_gpio(np, 1);
- if (!gpio_is_valid(pdata->gpio_rst)) {
- dev_err(dev, "failed to get reset gpio\n");
- return ERR_PTR(-EINVAL);
- }
-
- if (of_property_read_u32(np, "x-size", &pdata->x_max)) {
- dev_err(dev, "failed to get x-size property\n");
- return ERR_PTR(-EINVAL);
- }
-
- if (of_property_read_u32(np, "y-size", &pdata->y_max)) {
- dev_err(dev, "failed to get y-size property\n");
- return ERR_PTR(-EINVAL);
- }
-
- /* default to asserting the interrupt when the screen is touched */
- pdata->int_setting = AUO_PIXCIR_INT_TOUCH_IND;
-
- return pdata;
-}
-#else
-static struct auo_pixcir_ts_platdata *auo_pixcir_parse_dt(struct device *dev)
-{
- return ERR_PTR(-EINVAL);
-}
-#endif
-
static void auo_pixcir_reset(void *data)
{
struct auo_pixcir_ts *ts = data;
- gpio_set_value(ts->pdata->gpio_rst, 0);
+ gpiod_set_value_cansleep(ts->gpio_rst, 1);
}
static int auo_pixcir_probe(struct i2c_client *client,
const struct i2c_device_id *id)
{
- const struct auo_pixcir_ts_platdata *pdata;
struct auo_pixcir_ts *ts;
struct input_dev *input_dev;
int version;
int error;
- pdata = dev_get_platdata(&client->dev);
- if (!pdata) {
- pdata = auo_pixcir_parse_dt(&client->dev);
- if (IS_ERR(pdata))
- return PTR_ERR(pdata);
- }
-
- ts = devm_kzalloc(&client->dev,
- sizeof(struct auo_pixcir_ts), GFP_KERNEL);
+ ts = devm_kzalloc(&client->dev, sizeof(*ts), GFP_KERNEL);
if (!ts)
return -ENOMEM;
return -ENOMEM;
}
- ts->pdata = pdata;
ts->client = client;
ts->input = input_dev;
ts->touch_ind_mode = 0;
snprintf(ts->phys, sizeof(ts->phys),
"%s/input0", dev_name(&client->dev));
+ if (device_property_read_u32(&client->dev, "x-size", &ts->x_max)) {
+ dev_err(&client->dev, "failed to get x-size property\n");
+ return -EINVAL;
+ }
+
+ if (device_property_read_u32(&client->dev, "y-size", &ts->y_max)) {
+ dev_err(&client->dev, "failed to get y-size property\n");
+ return -EINVAL;
+ }
+
input_dev->name = "AUO-Pixcir touchscreen";
input_dev->phys = ts->phys;
input_dev->id.bustype = BUS_I2C;
__set_bit(BTN_TOUCH, input_dev->keybit);
/* For single touch */
- input_set_abs_params(input_dev, ABS_X, 0, pdata->x_max, 0, 0);
- input_set_abs_params(input_dev, ABS_Y, 0, pdata->y_max, 0, 0);
+ input_set_abs_params(input_dev, ABS_X, 0, ts->x_max, 0, 0);
+ input_set_abs_params(input_dev, ABS_Y, 0, ts->y_max, 0, 0);
/* For multi touch */
- input_set_abs_params(input_dev, ABS_MT_POSITION_X, 0,
- pdata->x_max, 0, 0);
- input_set_abs_params(input_dev, ABS_MT_POSITION_Y, 0,
- pdata->y_max, 0, 0);
- input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, 0,
- AUO_PIXCIR_MAX_AREA, 0, 0);
- input_set_abs_params(input_dev, ABS_MT_TOUCH_MINOR, 0,
- AUO_PIXCIR_MAX_AREA, 0, 0);
+ input_set_abs_params(input_dev, ABS_MT_POSITION_X, 0, ts->x_max, 0, 0);
+ input_set_abs_params(input_dev, ABS_MT_POSITION_Y, 0, ts->y_max, 0, 0);
+ input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR,
+ 0, AUO_PIXCIR_MAX_AREA, 0, 0);
+ input_set_abs_params(input_dev, ABS_MT_TOUCH_MINOR,
+ 0, AUO_PIXCIR_MAX_AREA, 0, 0);
input_set_abs_params(input_dev, ABS_MT_ORIENTATION, 0, 1, 0, 0);
input_set_drvdata(ts->input, ts);
- error = devm_gpio_request_one(&client->dev, pdata->gpio_int,
- GPIOF_DIR_IN, "auo_pixcir_ts_int");
+ ts->gpio_int = devm_gpiod_get_index(&client->dev, NULL, 0, GPIOD_IN);
+ error = PTR_ERR_OR_ZERO(ts->gpio_int);
if (error) {
- dev_err(&client->dev, "request of gpio %d failed, %d\n",
- pdata->gpio_int, error);
+ dev_err(&client->dev,
+ "request of int gpio failed: %d\n", error);
return error;
}
- error = devm_gpio_request_one(&client->dev, pdata->gpio_rst,
- GPIOF_DIR_OUT | GPIOF_INIT_HIGH,
- "auo_pixcir_ts_rst");
+ gpiod_set_consumer_name(ts->gpio_int, "auo_pixcir_ts_int");
+
+ /* Take the chip out of reset */
+ ts->gpio_rst = devm_gpiod_get_index(&client->dev, NULL, 1,
+ GPIOD_OUT_LOW);
+ error = PTR_ERR_OR_ZERO(ts->gpio_rst);
if (error) {
- dev_err(&client->dev, "request of gpio %d failed, %d\n",
- pdata->gpio_rst, error);
+ dev_err(&client->dev,
+ "request of reset gpio failed: %d\n", error);
return error;
}
+ gpiod_set_consumer_name(ts->gpio_rst, "auo_pixcir_ts_rst");
+
error = devm_add_action_or_reset(&client->dev, auo_pixcir_reset, ts);
if (error) {
dev_err(&client->dev, "failed to register reset action, %d\n",
dev_info(&client->dev, "firmware version 0x%X\n", version);
- error = auo_pixcir_int_config(ts, pdata->int_setting);
+ /* default to asserting the interrupt when the screen is touched */
+ error = auo_pixcir_int_config(ts, AUO_PIXCIR_INT_TOUCH_IND);
if (error)
return error;
error = devm_request_threaded_irq(&client->dev, client->irq,
NULL, auo_pixcir_interrupt,
- IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+ IRQF_ONESHOT,
input_dev->name, ts);
if (error) {
dev_err(&client->dev, "irq %d requested failed, %d\n",
static int icn8505_probe_acpi(struct icn8505_data *icn8505, struct device *dev)
{
- struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
- const char *subsys = "unknown";
- struct acpi_device *adev;
- union acpi_object *obj;
- acpi_status status;
-
- adev = ACPI_COMPANION(dev);
- if (!adev)
- return -ENODEV;
+ const char *subsys;
+ int error;
- status = acpi_evaluate_object(adev->handle, "_SUB", NULL, &buffer);
- if (ACPI_SUCCESS(status)) {
- obj = buffer.pointer;
- if (obj->type == ACPI_TYPE_STRING)
- subsys = obj->string.pointer;
- else
- dev_warn(dev, "Warning ACPI _SUB did not return a string\n");
- } else {
- dev_warn(dev, "Warning ACPI _SUB failed: %#x\n", status);
- buffer.pointer = NULL;
- }
+ subsys = acpi_get_subsystem_id(ACPI_HANDLE(dev));
+ error = PTR_ERR_OR_ZERO(subsys);
+ if (error == -ENODATA)
+ subsys = "unknown";
+ else if (error)
+ return error;
snprintf(icn8505->firmware_name, sizeof(icn8505->firmware_name),
"chipone/icn8505-%s.fw", subsys);
- kfree(buffer.pointer);
+ kfree_const(subsys);
return 0;
}
p = strchr(rdbuf, '*');
if (p)
*p++ = '\0';
- strlcpy(model_name, rdbuf + 1, EDT_NAME_LEN);
- strlcpy(fw_version, p ? p : "", EDT_NAME_LEN);
+ strscpy(model_name, rdbuf + 1, EDT_NAME_LEN);
+ strscpy(fw_version, p ? p : "", EDT_NAME_LEN);
} else if (!strncasecmp(rdbuf, "EP0", 3)) {
tsdata->version = EDT_M12;
p = strchr(rdbuf, '*');
if (p)
*p++ = '\0';
- strlcpy(model_name, rdbuf, EDT_NAME_LEN);
- strlcpy(fw_version, p ? p : "", EDT_NAME_LEN);
+ strscpy(model_name, rdbuf, EDT_NAME_LEN);
+ strscpy(fw_version, p ? p : "", EDT_NAME_LEN);
} else {
/* If it is not an EDT M06/M12 touchscreen, then the model
* detection is a bit hairy. The different ft5x06
if (error)
return error;
- strlcpy(fw_version, rdbuf, 2);
+ strscpy(fw_version, rdbuf, 2);
error = edt_ft5x06_ts_readwrite(client, 1, "\xA8",
1, rdbuf);
1, rdbuf);
if (error)
return error;
- strlcpy(fw_version, rdbuf, 1);
+ strscpy(fw_version, rdbuf, 1);
snprintf(model_name, EDT_NAME_LEN,
"EVERVISION-FT5726NEi");
break;
* Gunze AHL-51S touchscreen driver for Linux
*/
-/*
- */
-
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/module.h>
{
struct sur40_state *sur40 = video_drvdata(file);
- strlcpy(cap->driver, DRIVER_SHORT, sizeof(cap->driver));
- strlcpy(cap->card, DRIVER_LONG, sizeof(cap->card));
+ strscpy(cap->driver, DRIVER_SHORT, sizeof(cap->driver));
+ strscpy(cap->card, DRIVER_LONG, sizeof(cap->card));
usb_make_path(sur40->usbdev, cap->bus_info, sizeof(cap->bus_info));
return 0;
}
return -EINVAL;
i->type = V4L2_INPUT_TYPE_TOUCH;
i->std = V4L2_STD_UNKNOWN;
- strlcpy(i->name, "In-Cell Sensor", sizeof(i->name));
+ strscpy(i->name, "In-Cell Sensor", sizeof(i->name));
i->capabilities = 0;
return 0;
}
usbtouch->input = input_dev;
if (udev->manufacturer)
- strlcpy(usbtouch->name, udev->manufacturer, sizeof(usbtouch->name));
+ strscpy(usbtouch->name, udev->manufacturer, sizeof(usbtouch->name));
if (udev->product) {
if (udev->manufacturer)
/* For backwards-compatibility we compose the basename based on
* capabilities and then just append the tool type
*/
- strlcpy(basename, "Wacom Serial", sizeof(basename));
+ strscpy(basename, "Wacom Serial", sizeof(basename));
err_pen = w8001_setup_pen(w8001, basename, sizeof(basename));
err_touch = w8001_setup_touch(w8001, basename, sizeof(basename));
}
if (!err_pen) {
- strlcpy(w8001->pen_name, basename, sizeof(w8001->pen_name));
+ strscpy(w8001->pen_name, basename, sizeof(w8001->pen_name));
strlcat(w8001->pen_name, " Pen", sizeof(w8001->pen_name));
input_dev_pen->name = w8001->pen_name;
}
if (!err_touch) {
- strlcpy(w8001->touch_name, basename, sizeof(w8001->touch_name));
+ strscpy(w8001->touch_name, basename, sizeof(w8001->touch_name));
strlcat(w8001->touch_name, " Finger",
sizeof(w8001->touch_name));
input_dev_touch->name = w8001->touch_name;
config IRQCHIP
def_bool y
- depends on OF_IRQ
+ depends on (OF_IRQ || ACPI_GENERIC_GSI)
config ARM_GIC
bool
help
Support for the i.MX INTMUX interrupt multiplexer.
+config IMX_MU_MSI
+ tristate "i.MX MU used as MSI controller"
+ depends on OF && HAS_IOMEM
+ depends on ARCH_MXC || COMPILE_TEST
+ default m if ARCH_MXC
+ select IRQ_DOMAIN
+ select IRQ_DOMAIN_HIERARCHY
+ select GENERIC_MSI_IRQ_DOMAIN
+ help
+ Provide a driver for the i.MX Messaging Unit block used as a
+ CPU-to-CPU MSI controller. This requires a specially crafted DT
+ to make use of this driver.
+
+ If unsure, say N
+
config LS1X_IRQ
bool "Loongson-1 Interrupt Controller"
depends on MACH_LOONGSON32
obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o
obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o
obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o
+obj-$(CONFIG_IMX_MU_MSI) += irq-imx-mu-msi.o
obj-$(CONFIG_MADERA_IRQ) += irq-madera.o
obj-$(CONFIG_LS1X_IRQ) += irq-ls1x.o
obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o
u64 typer = gic_read_typer(ptr + GICR_TYPER);
u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
- /* Boot-time cleanip */
+ /* Boot-time cleanup */
if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
u64 val;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Freescale MU used as MSI controller
+ *
+ * Copyright (c) 2018 Pengutronix, Oleksij Rempel <o.rempel@pengutronix.de>
+ * Copyright 2022 NXP
+ * Frank Li <Frank.Li@nxp.com>
+ * Peng Fan <peng.fan@nxp.com>
+ *
+ * Based on drivers/mailbox/imx-mailbox.c
+ */
+
+#include <linux/clk.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_domain.h>
+#include <linux/spinlock.h>
+
+#define IMX_MU_CHANS 4
+
+enum imx_mu_xcr {
+ IMX_MU_GIER,
+ IMX_MU_GCR,
+ IMX_MU_TCR,
+ IMX_MU_RCR,
+ IMX_MU_xCR_MAX,
+};
+
+enum imx_mu_xsr {
+ IMX_MU_SR,
+ IMX_MU_GSR,
+ IMX_MU_TSR,
+ IMX_MU_RSR,
+ IMX_MU_xSR_MAX
+};
+
+enum imx_mu_type {
+ IMX_MU_V2 = BIT(1),
+};
+
+/* Receive Interrupt Enable */
+#define IMX_MU_xCR_RIEn(data, x) ((data->cfg->type) & IMX_MU_V2 ? BIT(x) : BIT(24 + (3 - (x))))
+#define IMX_MU_xSR_RFn(data, x) ((data->cfg->type) & IMX_MU_V2 ? BIT(x) : BIT(24 + (3 - (x))))
+
+struct imx_mu_dcfg {
+ enum imx_mu_type type;
+ u32 xTR; /* Transmit Register0 */
+ u32 xRR; /* Receive Register0 */
+ u32 xSR[IMX_MU_xSR_MAX]; /* Status Registers */
+ u32 xCR[IMX_MU_xCR_MAX]; /* Control Registers */
+};
+
+struct imx_mu_msi {
+ raw_spinlock_t lock;
+ struct irq_domain *msi_domain;
+ void __iomem *regs;
+ phys_addr_t msiir_addr;
+ const struct imx_mu_dcfg *cfg;
+ unsigned long used;
+ struct clk *clk;
+};
+
+static void imx_mu_write(struct imx_mu_msi *msi_data, u32 val, u32 offs)
+{
+ iowrite32(val, msi_data->regs + offs);
+}
+
+static u32 imx_mu_read(struct imx_mu_msi *msi_data, u32 offs)
+{
+ return ioread32(msi_data->regs + offs);
+}
+
+static u32 imx_mu_xcr_rmw(struct imx_mu_msi *msi_data, enum imx_mu_xcr type, u32 set, u32 clr)
+{
+ unsigned long flags;
+ u32 val;
+
+ raw_spin_lock_irqsave(&msi_data->lock, flags);
+ val = imx_mu_read(msi_data, msi_data->cfg->xCR[type]);
+ val &= ~clr;
+ val |= set;
+ imx_mu_write(msi_data, val, msi_data->cfg->xCR[type]);
+ raw_spin_unlock_irqrestore(&msi_data->lock, flags);
+
+ return val;
+}
+
+static void imx_mu_msi_parent_mask_irq(struct irq_data *data)
+{
+ struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data);
+
+ imx_mu_xcr_rmw(msi_data, IMX_MU_RCR, 0, IMX_MU_xCR_RIEn(msi_data, data->hwirq));
+}
+
+static void imx_mu_msi_parent_unmask_irq(struct irq_data *data)
+{
+ struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data);
+
+ imx_mu_xcr_rmw(msi_data, IMX_MU_RCR, IMX_MU_xCR_RIEn(msi_data, data->hwirq), 0);
+}
+
+static void imx_mu_msi_parent_ack_irq(struct irq_data *data)
+{
+ struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data);
+
+ imx_mu_read(msi_data, msi_data->cfg->xRR + data->hwirq * 4);
+}
+
+static struct irq_chip imx_mu_msi_irq_chip = {
+ .name = "MU-MSI",
+ .irq_ack = irq_chip_ack_parent,
+};
+
+static struct msi_domain_ops imx_mu_msi_irq_ops = {
+};
+
+static struct msi_domain_info imx_mu_msi_domain_info = {
+ .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
+ .ops = &imx_mu_msi_irq_ops,
+ .chip = &imx_mu_msi_irq_chip,
+};
+
+static void imx_mu_msi_parent_compose_msg(struct irq_data *data,
+ struct msi_msg *msg)
+{
+ struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data);
+ u64 addr = msi_data->msiir_addr + 4 * data->hwirq;
+
+ msg->address_hi = upper_32_bits(addr);
+ msg->address_lo = lower_32_bits(addr);
+ msg->data = data->hwirq;
+}
+
+static int imx_mu_msi_parent_set_affinity(struct irq_data *irq_data,
+ const struct cpumask *mask, bool force)
+{
+ return -EINVAL;
+}
+
+static struct irq_chip imx_mu_msi_parent_chip = {
+ .name = "MU",
+ .irq_mask = imx_mu_msi_parent_mask_irq,
+ .irq_unmask = imx_mu_msi_parent_unmask_irq,
+ .irq_ack = imx_mu_msi_parent_ack_irq,
+ .irq_compose_msi_msg = imx_mu_msi_parent_compose_msg,
+ .irq_set_affinity = imx_mu_msi_parent_set_affinity,
+};
+
+static int imx_mu_msi_domain_irq_alloc(struct irq_domain *domain,
+ unsigned int virq,
+ unsigned int nr_irqs,
+ void *args)
+{
+ struct imx_mu_msi *msi_data = domain->host_data;
+ unsigned long flags;
+ int pos, err = 0;
+
+ WARN_ON(nr_irqs != 1);
+
+ raw_spin_lock_irqsave(&msi_data->lock, flags);
+ pos = find_first_zero_bit(&msi_data->used, IMX_MU_CHANS);
+ if (pos < IMX_MU_CHANS)
+ __set_bit(pos, &msi_data->used);
+ else
+ err = -ENOSPC;
+ raw_spin_unlock_irqrestore(&msi_data->lock, flags);
+
+ if (err)
+ return err;
+
+ irq_domain_set_info(domain, virq, pos,
+ &imx_mu_msi_parent_chip, msi_data,
+ handle_edge_irq, NULL, NULL);
+ return 0;
+}
+
+static void imx_mu_msi_domain_irq_free(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs)
+{
+ struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+ struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(d);
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&msi_data->lock, flags);
+ __clear_bit(d->hwirq, &msi_data->used);
+ raw_spin_unlock_irqrestore(&msi_data->lock, flags);
+}
+
+static const struct irq_domain_ops imx_mu_msi_domain_ops = {
+ .alloc = imx_mu_msi_domain_irq_alloc,
+ .free = imx_mu_msi_domain_irq_free,
+};
+
+static void imx_mu_msi_irq_handler(struct irq_desc *desc)
+{
+ struct imx_mu_msi *msi_data = irq_desc_get_handler_data(desc);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ u32 status;
+ int i;
+
+ status = imx_mu_read(msi_data, msi_data->cfg->xSR[IMX_MU_RSR]);
+
+ chained_irq_enter(chip, desc);
+ for (i = 0; i < IMX_MU_CHANS; i++) {
+ if (status & IMX_MU_xSR_RFn(msi_data, i))
+ generic_handle_domain_irq(msi_data->msi_domain, i);
+ }
+ chained_irq_exit(chip, desc);
+}
+
+static int imx_mu_msi_domains_init(struct imx_mu_msi *msi_data, struct device *dev)
+{
+ struct fwnode_handle *fwnodes = dev_fwnode(dev);
+ struct irq_domain *parent;
+
+ /* Initialize MSI domain parent */
+ parent = irq_domain_create_linear(fwnodes,
+ IMX_MU_CHANS,
+ &imx_mu_msi_domain_ops,
+ msi_data);
+ if (!parent) {
+ dev_err(dev, "failed to create IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
+
+ msi_data->msi_domain = platform_msi_create_irq_domain(fwnodes,
+ &imx_mu_msi_domain_info,
+ parent);
+
+ if (!msi_data->msi_domain) {
+ dev_err(dev, "failed to create MSI domain\n");
+ irq_domain_remove(parent);
+ return -ENOMEM;
+ }
+
+ irq_domain_set_pm_device(msi_data->msi_domain, dev);
+
+ return 0;
+}
+
+/* Register offset of different version MU IP */
+static const struct imx_mu_dcfg imx_mu_cfg_imx6sx = {
+ .type = 0,
+ .xTR = 0x0,
+ .xRR = 0x10,
+ .xSR = {
+ [IMX_MU_SR] = 0x20,
+ [IMX_MU_GSR] = 0x20,
+ [IMX_MU_TSR] = 0x20,
+ [IMX_MU_RSR] = 0x20,
+ },
+ .xCR = {
+ [IMX_MU_GIER] = 0x24,
+ [IMX_MU_GCR] = 0x24,
+ [IMX_MU_TCR] = 0x24,
+ [IMX_MU_RCR] = 0x24,
+ },
+};
+
+static const struct imx_mu_dcfg imx_mu_cfg_imx7ulp = {
+ .type = 0,
+ .xTR = 0x20,
+ .xRR = 0x40,
+ .xSR = {
+ [IMX_MU_SR] = 0x60,
+ [IMX_MU_GSR] = 0x60,
+ [IMX_MU_TSR] = 0x60,
+ [IMX_MU_RSR] = 0x60,
+ },
+ .xCR = {
+ [IMX_MU_GIER] = 0x64,
+ [IMX_MU_GCR] = 0x64,
+ [IMX_MU_TCR] = 0x64,
+ [IMX_MU_RCR] = 0x64,
+ },
+};
+
+static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = {
+ .type = IMX_MU_V2,
+ .xTR = 0x200,
+ .xRR = 0x280,
+ .xSR = {
+ [IMX_MU_SR] = 0xC,
+ [IMX_MU_GSR] = 0x118,
+ [IMX_MU_TSR] = 0x124,
+ [IMX_MU_RSR] = 0x12C,
+ },
+ .xCR = {
+ [IMX_MU_GIER] = 0x110,
+ [IMX_MU_GCR] = 0x114,
+ [IMX_MU_TCR] = 0x120,
+ [IMX_MU_RCR] = 0x128
+ },
+};
+
+static int __init imx_mu_of_init(struct device_node *dn,
+ struct device_node *parent,
+ const struct imx_mu_dcfg *cfg)
+{
+ struct platform_device *pdev = of_find_device_by_node(dn);
+ struct device_link *pd_link_a;
+ struct device_link *pd_link_b;
+ struct imx_mu_msi *msi_data;
+ struct resource *res;
+ struct device *pd_a;
+ struct device *pd_b;
+ struct device *dev;
+ int ret;
+ int irq;
+
+ dev = &pdev->dev;
+
+ msi_data = devm_kzalloc(&pdev->dev, sizeof(*msi_data), GFP_KERNEL);
+ if (!msi_data)
+ return -ENOMEM;
+
+ msi_data->cfg = cfg;
+
+ msi_data->regs = devm_platform_ioremap_resource_byname(pdev, "processor-a-side");
+ if (IS_ERR(msi_data->regs)) {
+ dev_err(&pdev->dev, "failed to initialize 'regs'\n");
+ return PTR_ERR(msi_data->regs);
+ }
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "processor-b-side");
+ if (!res)
+ return -EIO;
+
+ msi_data->msiir_addr = res->start + msi_data->cfg->xTR;
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq <= 0)
+ return -ENODEV;
+
+ platform_set_drvdata(pdev, msi_data);
+
+ msi_data->clk = devm_clk_get(dev, NULL);
+ if (IS_ERR(msi_data->clk))
+ return PTR_ERR(msi_data->clk);
+
+ pd_a = dev_pm_domain_attach_by_name(dev, "processor-a-side");
+ if (IS_ERR(pd_a))
+ return PTR_ERR(pd_a);
+
+ pd_b = dev_pm_domain_attach_by_name(dev, "processor-b-side");
+ if (IS_ERR(pd_b))
+ return PTR_ERR(pd_b);
+
+ pd_link_a = device_link_add(dev, pd_a,
+ DL_FLAG_STATELESS |
+ DL_FLAG_PM_RUNTIME |
+ DL_FLAG_RPM_ACTIVE);
+
+ if (!pd_link_a) {
+ dev_err(dev, "Failed to add device_link to mu a.\n");
+ goto err_pd_a;
+ }
+
+ pd_link_b = device_link_add(dev, pd_b,
+ DL_FLAG_STATELESS |
+ DL_FLAG_PM_RUNTIME |
+ DL_FLAG_RPM_ACTIVE);
+
+
+ if (!pd_link_b) {
+ dev_err(dev, "Failed to add device_link to mu a.\n");
+ goto err_pd_b;
+ }
+
+ ret = imx_mu_msi_domains_init(msi_data, dev);
+ if (ret)
+ goto err_dm_init;
+
+ pm_runtime_enable(dev);
+
+ irq_set_chained_handler_and_data(irq,
+ imx_mu_msi_irq_handler,
+ msi_data);
+
+ return 0;
+
+err_dm_init:
+ device_link_remove(dev, pd_b);
+err_pd_b:
+ device_link_remove(dev, pd_a);
+err_pd_a:
+ return -EINVAL;
+}
+
+static int __maybe_unused imx_mu_runtime_suspend(struct device *dev)
+{
+ struct imx_mu_msi *priv = dev_get_drvdata(dev);
+
+ clk_disable_unprepare(priv->clk);
+
+ return 0;
+}
+
+static int __maybe_unused imx_mu_runtime_resume(struct device *dev)
+{
+ struct imx_mu_msi *priv = dev_get_drvdata(dev);
+ int ret;
+
+ ret = clk_prepare_enable(priv->clk);
+ if (ret)
+ dev_err(dev, "failed to enable clock\n");
+
+ return ret;
+}
+
+static const struct dev_pm_ops imx_mu_pm_ops = {
+ SET_RUNTIME_PM_OPS(imx_mu_runtime_suspend,
+ imx_mu_runtime_resume, NULL)
+};
+
+static int __init imx_mu_imx7ulp_of_init(struct device_node *dn,
+ struct device_node *parent)
+{
+ return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx7ulp);
+}
+
+static int __init imx_mu_imx6sx_of_init(struct device_node *dn,
+ struct device_node *parent)
+{
+ return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx6sx);
+}
+
+static int __init imx_mu_imx8ulp_of_init(struct device_node *dn,
+ struct device_node *parent)
+{
+ return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx8ulp);
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(imx_mu_msi)
+IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_of_init)
+IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_of_init)
+IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_of_init)
+IRQCHIP_PLATFORM_DRIVER_END(imx_mu_msi, .pm = &imx_mu_pm_ops)
+
+
+MODULE_AUTHOR("Frank Li <Frank.Li@nxp.com>");
+MODULE_DESCRIPTION("Freescale MU MSI controller driver");
+MODULE_LICENSE("GPL");
#include <linux/irqchip.h>
#include <linux/irqdomain.h>
#include <linux/of.h>
-#include <linux/mfd/syscon.h>
-#include <linux/regmap.h>
+#include <linux/of_address.h>
#include <linux/slab.h>
#include <dt-bindings/interrupt-controller/arm-gic.h>
#define LS1021A_SCFGREVCR 0x200
struct ls_extirq_data {
- struct regmap *syscon;
- u32 intpcr;
+ void __iomem *intpcr;
+ raw_spinlock_t lock;
+ bool big_endian;
bool is_ls1021a_or_ls1043a;
u32 nirq;
struct irq_fwspec map[MAXIRQ];
};
+static void ls_extirq_intpcr_rmw(struct ls_extirq_data *priv, u32 mask,
+ u32 value)
+{
+ u32 intpcr;
+
+ /*
+ * Serialize concurrent calls to ls_extirq_set_type() from multiple
+ * IRQ descriptors, making sure the read-modify-write is atomic.
+ */
+ raw_spin_lock(&priv->lock);
+
+ if (priv->big_endian)
+ intpcr = ioread32be(priv->intpcr);
+ else
+ intpcr = ioread32(priv->intpcr);
+
+ intpcr &= ~mask;
+ intpcr |= value;
+
+ if (priv->big_endian)
+ iowrite32be(intpcr, priv->intpcr);
+ else
+ iowrite32(intpcr, priv->intpcr);
+
+ raw_spin_unlock(&priv->lock);
+}
+
static int
ls_extirq_set_type(struct irq_data *data, unsigned int type)
{
default:
return -EINVAL;
}
- regmap_update_bits(priv->syscon, priv->intpcr, mask, value);
+
+ ls_extirq_intpcr_rmw(priv, mask, value);
return irq_chip_set_type_parent(data, type);
}
static int __init
ls_extirq_of_init(struct device_node *node, struct device_node *parent)
{
-
struct irq_domain *domain, *parent_domain;
struct ls_extirq_data *priv;
int ret;
parent_domain = irq_find_host(parent);
if (!parent_domain) {
pr_err("Cannot find parent domain\n");
- return -ENODEV;
+ ret = -ENODEV;
+ goto err_irq_find_host;
}
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv)
- return -ENOMEM;
-
- priv->syscon = syscon_node_to_regmap(node->parent);
- if (IS_ERR(priv->syscon)) {
- ret = PTR_ERR(priv->syscon);
- pr_err("Failed to lookup parent regmap\n");
- goto out;
+ if (!priv) {
+ ret = -ENOMEM;
+ goto err_alloc_priv;
}
- ret = of_property_read_u32(node, "reg", &priv->intpcr);
- if (ret) {
- pr_err("Missing INTPCR offset value\n");
- goto out;
+
+ /*
+ * All extirq OF nodes are under a scfg/syscon node with
+ * the 'ranges' property
+ */
+ priv->intpcr = of_iomap(node, 0);
+ if (!priv->intpcr) {
+ pr_err("Cannot ioremap OF node %pOF\n", node);
+ ret = -ENOMEM;
+ goto err_iomap;
}
ret = ls_extirq_parse_map(priv, node);
if (ret)
- goto out;
+ goto err_parse_map;
+ priv->big_endian = of_device_is_big_endian(parent);
priv->is_ls1021a_or_ls1043a = of_device_is_compatible(node, "fsl,ls1021a-extirq") ||
of_device_is_compatible(node, "fsl,ls1043a-extirq");
+ raw_spin_lock_init(&priv->lock);
domain = irq_domain_add_hierarchy(parent_domain, 0, priv->nirq, node,
&extirq_domain_ops, priv);
- if (!domain)
+ if (!domain) {
ret = -ENOMEM;
+ goto err_add_hierarchy;
+ }
-out:
- if (ret)
- kfree(priv);
+ return 0;
+
+err_add_hierarchy:
+err_parse_map:
+ iounmap(priv->intpcr);
+err_iomap:
+ kfree(priv);
+err_alloc_priv:
+err_irq_find_host:
return ret;
}
#define RTL_ICTL_IRR2 0x10
#define RTL_ICTL_IRR3 0x14
+#define RTL_ICTL_NUM_INPUTS 32
+
#define REG(x) (realtek_ictl_base + x)
static DEFINE_RAW_SPINLOCK(irq_lock);
static void __iomem *realtek_ictl_base;
+/*
+ * IRR0-IRR3 store 4 bits per interrupt, but Realtek uses inverted numbering,
+ * placing IRQ 31 in the first four bits. A routing value of '0' means the
+ * interrupt is left disconnected. Routing values {1..15} connect to output
+ * lines {0..14}.
+ */
+#define IRR_OFFSET(idx) (4 * (3 - (idx * 4) / 32))
+#define IRR_SHIFT(idx) ((idx * 4) % 32)
+
+static void write_irr(void __iomem *irr0, int idx, u32 value)
+{
+ unsigned int offset = IRR_OFFSET(idx);
+ unsigned int shift = IRR_SHIFT(idx);
+ u32 irr;
+
+ irr = readl(irr0 + offset) & ~(0xf << shift);
+ irr |= (value & 0xf) << shift;
+ writel(irr, irr0 + offset);
+}
+
static void realtek_ictl_unmask_irq(struct irq_data *i)
{
unsigned long flags;
static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw)
{
+ unsigned long flags;
+
irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq);
+ raw_spin_lock_irqsave(&irq_lock, flags);
+ write_irr(REG(RTL_ICTL_IRR0), hw, 1);
+ raw_spin_unlock_irqrestore(&irq_lock, flags);
+
return 0;
}
chained_irq_exit(chip, desc);
}
-/*
- * SoC interrupts are cascaded to MIPS CPU interrupts according to the
- * interrupt-map in the device tree. Each SoC interrupt gets 4 bits for
- * the CPU interrupt in an Interrupt Routing Register. Max 32 SoC interrupts
- * thus go into 4 IRRs. A routing value of '0' means the interrupt is left
- * disconnected. Routing values {1..15} connect to output lines {0..14}.
- */
-static int __init map_interrupts(struct device_node *node, struct irq_domain *domain)
-{
- struct device_node *cpu_ictl;
- const __be32 *imap;
- u32 imaplen, soc_int, cpu_int, tmp, regs[4];
- int ret, i, irr_regs[] = {
- RTL_ICTL_IRR3,
- RTL_ICTL_IRR2,
- RTL_ICTL_IRR1,
- RTL_ICTL_IRR0,
- };
- u8 mips_irqs_set;
-
- ret = of_property_read_u32(node, "#address-cells", &tmp);
- if (ret || tmp)
- return -EINVAL;
-
- imap = of_get_property(node, "interrupt-map", &imaplen);
- if (!imap || imaplen % 3)
- return -EINVAL;
-
- mips_irqs_set = 0;
- memset(regs, 0, sizeof(regs));
- for (i = 0; i < imaplen; i += 3 * sizeof(u32)) {
- soc_int = be32_to_cpup(imap);
- if (soc_int > 31)
- return -EINVAL;
-
- cpu_ictl = of_find_node_by_phandle(be32_to_cpup(imap + 1));
- if (!cpu_ictl)
- return -EINVAL;
- ret = of_property_read_u32(cpu_ictl, "#interrupt-cells", &tmp);
- of_node_put(cpu_ictl);
- if (ret || tmp != 1)
- return -EINVAL;
-
- cpu_int = be32_to_cpup(imap + 2);
- if (cpu_int > 7 || cpu_int < 2)
- return -EINVAL;
-
- if (!(mips_irqs_set & BIT(cpu_int))) {
- irq_set_chained_handler_and_data(cpu_int, realtek_irq_dispatch,
- domain);
- mips_irqs_set |= BIT(cpu_int);
- }
-
- /* Use routing values (1..6) for CPU interrupts (2..7) */
- regs[(soc_int * 4) / 32] |= (cpu_int - 1) << (soc_int * 4) % 32;
- imap += 3;
- }
-
- for (i = 0; i < 4; i++)
- writel(regs[i], REG(irr_regs[i]));
-
- return 0;
-}
-
static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent)
{
+ struct of_phandle_args oirq;
struct irq_domain *domain;
- int ret;
+ unsigned int soc_irq;
+ int parent_irq;
realtek_ictl_base = of_iomap(node, 0);
if (!realtek_ictl_base)
return -ENXIO;
- /* Disable all cascaded interrupts */
+ /* Disable all cascaded interrupts and clear routing */
writel(0, REG(RTL_ICTL_GIMR));
+ for (soc_irq = 0; soc_irq < RTL_ICTL_NUM_INPUTS; soc_irq++)
+ write_irr(REG(RTL_ICTL_IRR0), soc_irq, 0);
+
+ if (WARN_ON(!of_irq_count(node))) {
+ /*
+ * If DT contains no parent interrupts, assume MIPS CPU IRQ 2
+ * (HW0) is connected to the first output. This is the case for
+ * all known hardware anyway. "interrupt-map" is deprecated, so
+ * don't bother trying to parse that.
+ */
+ oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller");
+ oirq.args_count = 1;
+ oirq.args[0] = 2;
+
+ parent_irq = irq_create_of_mapping(&oirq);
+
+ of_node_put(oirq.np);
+ } else {
+ parent_irq = of_irq_get(node, 0);
+ }
- domain = irq_domain_add_simple(node, 32, 0,
- &irq_domain_ops, NULL);
+ if (parent_irq < 0)
+ return parent_irq;
+ else if (!parent_irq)
+ return -ENODEV;
- ret = map_interrupts(node, domain);
- if (ret) {
- pr_err("invalid interrupt map\n");
- return ret;
- }
+ domain = irq_domain_add_linear(node, RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL);
+ if (!domain)
+ return -ENOMEM;
+
+ irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain);
return 0;
}
static void __exit
HFC_cleanup(void)
{
- if (timer_pending(&hfc_tl))
- del_timer_sync(&hfc_tl);
+ del_timer_sync(&hfc_tl);
pci_unregister_driver(&hfc_driver);
}
struct pca963x *chip;
struct led_classdev led_cdev;
int led_num; /* 0 .. 15 potentially */
+ bool blinking;
u8 gdc;
u8 gfrq;
};
switch (brightness) {
case LED_FULL:
- val = (ledout & ~mask) | (PCA963X_LED_ON << shift);
+ if (led->blinking) {
+ val = (ledout & ~mask) | (PCA963X_LED_GRP_PWM << shift);
+ ret = i2c_smbus_write_byte_data(client,
+ PCA963X_PWM_BASE +
+ led->led_num,
+ LED_FULL);
+ } else {
+ val = (ledout & ~mask) | (PCA963X_LED_ON << shift);
+ }
ret = i2c_smbus_write_byte_data(client, ledout_addr, val);
break;
case LED_OFF:
val = ledout & ~mask;
ret = i2c_smbus_write_byte_data(client, ledout_addr, val);
+ led->blinking = false;
break;
default:
ret = i2c_smbus_write_byte_data(client,
if (ret < 0)
return ret;
- val = (ledout & ~mask) | (PCA963X_LED_PWM << shift);
+ if (led->blinking)
+ val = (ledout & ~mask) | (PCA963X_LED_GRP_PWM << shift);
+ else
+ val = (ledout & ~mask) | (PCA963X_LED_PWM << shift);
+
ret = i2c_smbus_write_byte_data(client, ledout_addr, val);
break;
}
}
mutex_unlock(&led->chip->mutex);
+ led->blinking = true;
}
static int pca963x_power_state(struct pca963x_led *led)
led->gfrq = gfrq;
pca963x_blink(led);
+ led->led_cdev.brightness = LED_FULL;
+ pca963x_led_set(led_cdev, LED_FULL);
*delay_on = time_on;
*delay_off = time_off;
led->led_cdev.brightness_set_blocking = pca963x_led_set;
if (hw_blink)
led->led_cdev.blink_set = pca963x_blink_set;
+ led->blinking = false;
init_data.fwnode = child;
/* for backwards compatibility */
}
if (bypass_torture_test(dc)) {
- if ((get_random_int() & 3) == 3)
+ if (prandom_u32_max(4) == 3)
goto skip;
else
goto rescale;
}
create:
if (create_super) {
- log->last_cp_seq = prandom_u32();
+ log->last_cp_seq = get_random_u32();
cp = 0;
r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq);
/*
g = tpg_colors[col].g;
b = tpg_colors[col].b;
} else if (tpg->pattern == TPG_PAT_NOISE) {
- r = g = b = prandom_u32_max(256);
+ r = g = b = get_random_u8();
} else if (k == TPG_COLOR_RANDOM) {
r = g = b = tpg->qual_offset + prandom_u32_max(196);
} else if (k >= TPG_COLOR_RAMP) {
break;
case 2:
rds.block |= V4L2_RDS_BLOCK_ERROR;
- rds.lsb = prandom_u32_max(256);
- rds.msb = prandom_u32_max(256);
+ rds.lsb = get_random_u8();
+ rds.msb = get_random_u8();
break;
case 3: /* Skip block altogether */
if (i)
/* Fill 10% of the values within range -3 and 3, zero the others */
for (i = 0; i < size; i++) {
- unsigned int rand = get_random_int();
+ unsigned int rand = get_random_u32();
if (rand % 10)
tch_buf[i] = 0;
static inline int get_random_pressure(void)
{
- return get_random_int() % VIVID_PRESSURE_LIMIT;
+ return prandom_u32_max(VIVID_PRESSURE_LIMIT);
}
static void vivid_tch_buf_set(struct v4l2_pix_format *f,
return;
if (test_pat_idx == 0)
- dev->tch_pat_random = get_random_int();
+ dev->tch_pat_random = get_random_u32();
rand = dev->tch_pat_random;
switch (test_pattern) {
mmput(mm);
}
-static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)
-{
- struct mm_struct *mm;
-
- mm = get_mem_context(ctx);
- if (mm == NULL) {
- pr_devel("cxl_prefault_one unable to get mm %i\n",
- pid_nr(ctx->pid));
- return;
- }
-
- cxl_fault_segment(ctx, mm, ea);
-
- mmput(mm);
-}
-
static u64 next_segment(u64 ea, u64 vsid)
{
if (vsid & SLB_VSID_B_1T)
return ea + 1;
}
-static void cxl_prefault_vma(struct cxl_context *ctx)
+static void cxl_prefault_vma(struct cxl_context *ctx, struct mm_struct *mm)
{
u64 ea, last_esid = 0;
struct copro_slb slb;
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
int rc;
- struct mm_struct *mm;
-
- mm = get_mem_context(ctx);
- if (mm == NULL) {
- pr_devel("cxl_prefault_vm unable to get mm %i\n",
- pid_nr(ctx->pid));
- return;
- }
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
for (ea = vma->vm_start; ea < vma->vm_end;
ea = next_segment(ea, slb.vsid)) {
rc = copro_calculate_slb(mm, ea, &slb);
}
}
mmap_read_unlock(mm);
-
- mmput(mm);
}
void cxl_prefault(struct cxl_context *ctx, u64 wed)
{
+ struct mm_struct *mm = get_mem_context(ctx);
+
+ if (mm == NULL) {
+ pr_devel("cxl_prefault unable to get mm %i\n",
+ pid_nr(ctx->pid));
+ return;
+ }
+
switch (ctx->afu->prefault_mode) {
case CXL_PREFAULT_WED:
- cxl_prefault_one(ctx, wed);
+ cxl_fault_segment(ctx, mm, wed);
break;
case CXL_PREFAULT_ALL:
- cxl_prefault_vma(ctx);
+ cxl_prefault_vma(ctx, mm);
break;
default:
break;
}
+
+ mmput(mm);
}
static inline int gaudi2_get_non_zero_random_int(void)
{
- int rand = get_random_int();
+ int rand = get_random_u32();
return rand ? rand : 1;
}
{
struct mmc_blk_data *md = mq->blkdata;
struct mmc_card *card = md->queue.card;
+ unsigned int arg = card->erase_arg;
- mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, card->erase_arg);
+ if (mmc_card_broken_sd_discard(card))
+ arg = SD_ERASE_ARG;
+
+ mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg);
}
static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
#define EXT_CSD_REV_ANY (-1u)
#define CID_MANFID_SANDISK 0x2
+#define CID_MANFID_SANDISK_SD 0x3
#define CID_MANFID_ATP 0x9
#define CID_MANFID_TOSHIBA 0x11
#define CID_MANFID_MICRON 0x13
return c->quirks & MMC_QUIRK_BROKEN_HPI;
}
+static inline int mmc_card_broken_sd_discard(const struct mmc_card *c)
+{
+ return c->quirks & MMC_QUIRK_BROKEN_SD_DISCARD;
+}
+
#endif
!should_fail(&host->fail_mmc_request, data->blksz * data->blocks))
return;
- data->error = data_errors[prandom_u32() % ARRAY_SIZE(data_errors)];
- data->bytes_xfered = (prandom_u32() % (data->bytes_xfered >> 9)) << 9;
+ data->error = data_errors[prandom_u32_max(ARRAY_SIZE(data_errors))];
+ data->bytes_xfered = prandom_u32_max(data->bytes_xfered >> 9) << 9;
}
#else /* CONFIG_FAIL_MMC_REQUEST */
MMC_FIXUP("V10016", CID_MANFID_KINGSTON, CID_OEMID_ANY, add_quirk_mmc,
MMC_QUIRK_TRIM_BROKEN),
+ /*
+ * Some SD cards reports discard support while they don't
+ */
+ MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_SD, 0x5344, add_quirk_sd,
+ MMC_QUIRK_BROKEN_SD_DISCARD),
+
END_FIXUP
};
* Try to inject the error at random points during the data transfer.
*/
hrtimer_start(&host->fault_timer,
- ms_to_ktime(prandom_u32() % 25),
+ ms_to_ktime(prandom_u32_max(25)),
HRTIMER_MODE_REL);
}
struct clk *ref_clk = priv->clk;
unsigned int freq, diff, best_freq = 0, diff_min = ~0;
unsigned int new_clock, clkh_shift = 0;
+ unsigned int new_upper_limit;
int i;
/*
* greater than, new_clock. As we can divide by 1 << i for
* any i in [0, 9] we want the input clock to be as close as
* possible, but no greater than, new_clock << i.
+ *
+ * Add an upper limit of 1/1024 rate higher to the clock rate to fix
+ * clk rate jumping to lower rate due to rounding error (eg: RZ/G2L has
+ * 3 clk sources 533.333333 MHz, 400 MHz and 266.666666 MHz. The request
+ * for 533.333333 MHz will selects a slower 400 MHz due to rounding
+ * error (533333333 Hz / 4 * 4 = 533333332 Hz < 533333333 Hz)).
*/
for (i = min(9, ilog2(UINT_MAX / new_clock)); i >= 0; i--) {
freq = clk_round_rate(ref_clk, new_clock << i);
- if (freq > (new_clock << i)) {
+ new_upper_limit = (new_clock << i) + ((new_clock << i) >> 10);
+ if (freq > new_upper_limit) {
/* Too fast; look for a slightly slower option */
freq = clk_round_rate(ref_clk, (new_clock << i) / 4 * 3);
- if (freq > (new_clock << i))
+ if (freq > new_upper_limit)
continue;
}
static void renesas_sdhi_set_clock(struct tmio_mmc_host *host,
unsigned int new_clock)
{
+ unsigned int clk_margin;
u32 clk = 0, clock;
sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, ~CLK_CTL_SCLKEN &
host->mmc->actual_clock = renesas_sdhi_clk_update(host, new_clock);
clock = host->mmc->actual_clock / 512;
- for (clk = 0x80000080; new_clock >= (clock << 1); clk >>= 1)
+ /*
+ * Add a margin of 1/1024 rate higher to the clock rate in order
+ * to avoid clk variable setting a value of 0 due to the margin
+ * provided for actual_clock in renesas_sdhi_clk_update().
+ */
+ clk_margin = new_clock >> 10;
+ for (clk = 0x80000080; new_clock + clk_margin >= (clock << 1); clk >>= 1)
clock <<= 1;
/* 1/1 clock is option */
static unsigned int sdhci_sprd_get_min_clock(struct sdhci_host *host)
{
- return 400000;
+ return 100000;
}
static void sdhci_sprd_set_uhs_signaling(struct sdhci_host *host,
dev_err(dev, "failed to set clk rate to %luHz: %d\n",
host_clk, err);
- tegra_host->curr_clk_rate = host_clk;
+ tegra_host->curr_clk_rate = clk_get_rate(pltfm_host->clk);
if (tegra_host->ddr_signaling)
host->max_clk = host_clk;
else
unsigned int page_no = ns->regs.row;
if (ns_read_error(page_no)) {
- prandom_bytes(ns->buf.byte, num);
+ get_random_bytes(ns->buf.byte, num);
NS_WARN("simulating read error in page %u\n", page_no);
return 1;
}
static void ns_do_bit_flips(struct nandsim *ns, int num)
{
- if (bitflips && prandom_u32() < (1 << 22)) {
+ if (bitflips && get_random_u16() < (1 << 6)) {
int flips = 1;
if (bitflips > 1)
- flips = (prandom_u32() % (int) bitflips) + 1;
+ flips = prandom_u32_max(bitflips) + 1;
while (flips--) {
- int pos = prandom_u32() % (num * 8);
+ int pos = prandom_u32_max(num * 8);
ns->buf.byte[pos / 8] ^= (1 << (pos % 8));
NS_WARN("read_page: flipping bit %d in page %d "
"reading from %d ecc: corrected=%u failed=%u\n",
static void single_bit_error_data(void *error_data, void *correct_data,
size_t size)
{
- unsigned int offset = prandom_u32() % (size * BITS_PER_BYTE);
+ unsigned int offset = prandom_u32_max(size * BITS_PER_BYTE);
memcpy(error_data, correct_data, size);
__change_bit_le(offset, error_data);
{
unsigned int offset[2];
- offset[0] = prandom_u32() % (size * BITS_PER_BYTE);
+ offset[0] = prandom_u32_max(size * BITS_PER_BYTE);
do {
- offset[1] = prandom_u32() % (size * BITS_PER_BYTE);
+ offset[1] = prandom_u32_max(size * BITS_PER_BYTE);
} while (offset[0] == offset[1]);
memcpy(error_data, correct_data, size);
static unsigned int random_ecc_bit(size_t size)
{
- unsigned int offset = prandom_u32() % (3 * BITS_PER_BYTE);
+ unsigned int offset = prandom_u32_max(3 * BITS_PER_BYTE);
if (size == 256) {
/*
* and 17th bit) in ECC code for 256 byte data block
*/
while (offset == 16 || offset == 17)
- offset = prandom_u32() % (3 * BITS_PER_BYTE);
+ offset = prandom_u32_max(3 * BITS_PER_BYTE);
}
return offset;
goto error;
}
- prandom_bytes(correct_data, size);
+ get_random_bytes(correct_data, size);
ecc_sw_hamming_calculate(correct_data, size, correct_ecc, sm_order);
for (i = 0; i < ARRAY_SIZE(nand_ecc_test); i++) {
nand_ecc_test[i].prepare(error_data, error_ecc,
if (!iobuf)
goto out;
- prandom_bytes(iobuf, mtd->erasesize);
+ get_random_bytes(iobuf, mtd->erasesize);
bbt = kzalloc(ebcnt, GFP_KERNEL);
if (!bbt)
unsigned int eb;
again:
- eb = prandom_u32();
/* Read or write up 2 eraseblocks at a time - hence 'ebcnt - 1' */
- eb %= (ebcnt - 1);
+ eb = prandom_u32_max(ebcnt - 1);
if (bbt[eb])
goto again;
return eb;
static int rand_offs(void)
{
- unsigned int offs;
-
- offs = prandom_u32();
- offs %= bufsize;
- return offs;
+ return prandom_u32_max(bufsize);
}
static int rand_len(int offs)
{
- unsigned int len;
-
- len = prandom_u32();
- len %= (bufsize - offs);
- return len;
+ return prandom_u32_max(bufsize - offs);
}
static int do_read(void)
static int do_operation(void)
{
- if (prandom_u32() & 1)
+ if (prandom_u32_max(2))
return do_read();
else
return do_write();
goto out;
for (i = 0; i < ebcnt; i++)
offsets[i] = mtd->erasesize;
- prandom_bytes(writebuf, bufsize);
+ get_random_bytes(writebuf, bufsize);
bbt = kzalloc(ebcnt, GFP_KERNEL);
if (!bbt)
ret = blk_mq_alloc_tag_set(&dev->tag_set);
if (ret) {
dev_err(disk_to_dev(dev->gd), "blk_mq_alloc_tag_set failed");
- goto out_free_dev;;
+ goto out_free_dev;
}
/*
* Create one workqueue per volume (per registered block device).
- * Rembember workqueues are cheap, they're not threads.
+ * Remember workqueues are cheap, they're not threads.
*/
dev->wq = alloc_workqueue("%s", 0, 0, gd->disk_name);
if (!dev->wq) {
* @ubi_num: number to assign to the new UBI device
* @vid_hdr_offset: VID header offset
* @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs
+ * @disable_fm: whether disable fastmap
*
* This function attaches MTD device @mtd_dev to UBI and assign @ubi_num number
* to the newly created UBI device, unless @ubi_num is %UBI_DEV_NUM_AUTO, in
* automatically. Returns the new UBI device number in case of success and a
* negative error code in case of failure.
*
+ * If @disable_fm is true, ubi doesn't create new fastmap even the module param
+ * 'fm_autoconvert' is set, and existed old fastmap will be destroyed after
+ * doing full scanning.
+ *
* Note, the invocations of this function has to be serialized by the
* @ubi_devices_mutex.
*/
int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
- int vid_hdr_offset, int max_beb_per1024)
+ int vid_hdr_offset, int max_beb_per1024, bool disable_fm)
{
struct ubi_device *ubi;
int i, err;
UBI_FM_MIN_POOL_SIZE);
ubi->fm_wl_pool.max_size = ubi->fm_pool.max_size / 2;
- ubi->fm_disabled = !fm_autoconvert;
+ ubi->fm_disabled = (!fm_autoconvert || disable_fm) ? 1 : 0;
if (fm_debug)
ubi_enable_dbg_chk_fastmap(ubi);
if (!ubi->fm_buf)
goto out_free;
#endif
- err = ubi_attach(ubi, 0);
+ err = ubi_attach(ubi, disable_fm ? 1 : 0);
if (err) {
ubi_err(ubi, "failed to attach mtd%d, error %d",
mtd->index, err);
mutex_lock(&ubi_devices_mutex);
err = ubi_attach_mtd_dev(mtd, p->ubi_num,
- p->vid_hdr_offs, p->max_beb_per1024);
+ p->vid_hdr_offs, p->max_beb_per1024,
+ false);
mutex_unlock(&ubi_devices_mutex);
if (err < 0) {
pr_err("UBI error: cannot attach mtd%d\n",
* @req: volumes re-name request
*
* This is a helper function for the volume re-name IOCTL which validates the
- * the request, opens the volume and calls corresponding volumes management
+ * request, opens the volume and calls corresponding volumes management
* function. Returns zero in case of success and a negative error code in case
* of failure.
*/
*/
mutex_lock(&ubi_devices_mutex);
err = ubi_attach_mtd_dev(mtd, req.ubi_num, req.vid_hdr_offset,
- req.max_beb_per1024);
+ req.max_beb_per1024, !!req.disable_fm);
mutex_unlock(&ubi_devices_mutex);
if (err < 0)
put_mtd_device(mtd);
if (ubi->dbg.power_cut_max > ubi->dbg.power_cut_min) {
range = ubi->dbg.power_cut_max - ubi->dbg.power_cut_min;
- ubi->dbg.power_cut_counter += prandom_u32() % range;
+ ubi->dbg.power_cut_counter += prandom_u32_max(range);
}
return 0;
}
static inline int ubi_dbg_is_bitflip(const struct ubi_device *ubi)
{
if (ubi->dbg.emulate_bitflips)
- return !(prandom_u32() % 200);
+ return !prandom_u32_max(200);
return 0;
}
static inline int ubi_dbg_is_write_failure(const struct ubi_device *ubi)
{
if (ubi->dbg.emulate_io_failures)
- return !(prandom_u32() % 500);
+ return !prandom_u32_max(500);
return 0;
}
static inline int ubi_dbg_is_erase_failure(const struct ubi_device *ubi)
{
if (ubi->dbg.emulate_io_failures)
- return !(prandom_u32() % 400);
+ return !prandom_u32_max(400);
return 0;
}
*
* This function locks a logical eraseblock for writing if there is no
* contention and does nothing if there is contention. Returns %0 in case of
- * success, %1 in case of contention, and and a negative error code in case of
+ * success, %1 in case of contention, and a negative error code in case of
* failure.
*/
static int leb_write_trylock(struct ubi_device *ubi, int vol_id, int lnum)
if (!ubi_dbg_chk_fastmap(ubi))
return NULL;
- ret = kcalloc(BITS_TO_LONGS(ubi->peb_count), sizeof(unsigned long),
- GFP_KERNEL);
+ ret = bitmap_zalloc(ubi->peb_count, GFP_KERNEL);
if (!ret)
return ERR_PTR(-ENOMEM);
*/
static inline void free_seen(unsigned long *seen)
{
- kfree(seen);
+ bitmap_free(seen);
}
/**
if (!ubi->fast_attach)
return 0;
- vol->checkmap = kcalloc(BITS_TO_LONGS(leb_count), sizeof(unsigned long),
- GFP_KERNEL);
+ vol->checkmap = bitmap_zalloc(leb_count, GFP_KERNEL);
if (!vol->checkmap)
return -ENOMEM;
void ubi_fastmap_destroy_checkmap(struct ubi_volume *vol)
{
- kfree(vol->checkmap);
+ bitmap_free(vol->checkmap);
}
/**
* @ubi: UBI device description object
* @pnum: the physical eraseblock number to check
*
- * This function returns zero if the erase counter header is all right and and
+ * This function returns zero if the erase counter header is all right and
* a negative error code if not or if an error occurred.
*/
static int self_check_peb_ec_hdr(const struct ubi_device *ubi, int pnum)
* is changed radically. This field is duplicated in the volume identifier
* header.
*
- * The @vid_hdr_offset and @data_offset fields contain the offset of the the
+ * The @vid_hdr_offset and @data_offset fields contain the offset of the
* volume identifier header and user data, relative to the beginning of the
* physical eraseblock. These values have to be the same for all physical
* eraseblocks.
* Error codes returned by the I/O sub-system.
*
* UBI_IO_FF: the read region of flash contains only 0xFFs
- * UBI_IO_FF_BITFLIPS: the same as %UBI_IO_FF, but also also there was a data
+ * UBI_IO_FF_BITFLIPS: the same as %UBI_IO_FF, but also there was a data
* integrity error reported by the MTD driver
* (uncorrectable ECC error in case of NAND)
* UBI_IO_BAD_HDR: the EC or VID header is corrupted (bad magic or CRC)
/**
* struct ubi_volume - UBI volume description data structure.
- * @dev: device object to make use of the the Linux device model
+ * @dev: device object to make use of the Linux device model
* @cdev: character device object to create character device
* @ubi: reference to the UBI device description object
* @vol_id: volume ID
/**
* struct ubi_device - UBI device description structure
- * @dev: UBI device object to use the the Linux device model
+ * @dev: UBI device object to use the Linux device model
* @cdev: character device object to create character device
* @ubi_num: UBI device number
* @ubi_name: UBI device name
/* build.c */
int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
- int vid_hdr_offset, int max_beb_per1024);
+ int vid_hdr_offset, int max_beb_per1024,
+ bool disable_fm);
int ubi_detach_mtd_dev(int ubi_num, int anyway);
struct ubi_device *ubi_get_device(int ubi_num);
void ubi_put_device(struct ubi_device *ubi);
* @ubi: UBI device description object
* @vol_id: volume ID
*
- * Returns zero if volume is all right and a a negative error code if not.
+ * Returns zero if volume is all right and a negative error code if not.
*/
static int self_check_volume(struct ubi_device *ubi, int vol_id)
{
* self_check_volumes - check information about all volumes.
* @ubi: UBI device description object
*
- * Returns zero if volumes are all right and a a negative error code if not.
+ * Returns zero if volumes are all right and a negative error code if not.
*/
static int self_check_volumes(struct ubi_device *ubi)
{
* refill_wl_user_pool().
* @ubi: UBI device description object
*
- * This function returns a a wear leveling entry in case of success and
+ * This function returns a wear leveling entry in case of success and
* NULL in case of failure.
*/
static struct ubi_wl_entry *wl_get_wle(struct ubi_device *ubi)
/**
* sync_erase - synchronously erase a physical eraseblock.
* @ubi: UBI device description object
- * @e: the the physical eraseblock to erase
+ * @e: the physical eraseblock to erase
* @torture: if the physical eraseblock has to be tortured
*
* This function returns zero in case of success and a negative error code in
/*
* If the ubi->scrub tree is not empty, scrubbing is needed, and the
- * the WL worker has to be scheduled anyway.
+ * WL worker has to be scheduled anyway.
*/
if (!ubi->scrub.rb_node) {
#ifdef CONFIG_MTD_UBI_FASTMAP
* ubi_bitflip_check - Check an eraseblock for bitflips and scrub it if needed.
* @ubi: UBI device description object
* @pnum: the physical eraseblock to schedule
- * @force: dont't read the block, assume bitflips happened and take action.
+ * @force: don't read the block, assume bitflips happened and take action.
*
* This function reads the given eraseblock and checks if bitflips occured.
* In case of bitflips, the eraseblock is scheduled for scrubbing.
tristate "WireGuard secure network tunnel"
depends on NET && INET
depends on IPV6 || !IPV6
+ depends on !KMSAN # KMSAN doesn't support the crypto configs below
select NET_UDP_TUNNEL
select DST_CACHE
select CRYPTO
switch (packets_per_slave) {
case 0:
- slave_id = prandom_u32();
+ slave_id = get_random_u32();
break;
case 1:
slave_id = this_cpu_inc_return(*bond->rr_tx_counter);
extern const struct kvaser_usb_dev_ops kvaser_usb_hydra_dev_ops;
extern const struct kvaser_usb_dev_ops kvaser_usb_leaf_dev_ops;
+void kvaser_usb_unlink_tx_urbs(struct kvaser_usb_net_priv *priv);
+
int kvaser_usb_recv_cmd(const struct kvaser_usb *dev, void *cmd, int len,
int *actual_len);
/* This method might sleep. Do not call it in the atomic context
* of URB completions.
*/
-static void kvaser_usb_unlink_tx_urbs(struct kvaser_usb_net_priv *priv)
+void kvaser_usb_unlink_tx_urbs(struct kvaser_usb_net_priv *priv)
{
usb_kill_anchored_urbs(&priv->tx_submitted);
kvaser_usb_reset_tx_urb_contexts(priv);
init_usb_anchor(&priv->tx_submitted);
init_completion(&priv->start_comp);
init_completion(&priv->stop_comp);
+ init_completion(&priv->flush_comp);
priv->can.ctrlmode_supported = 0;
priv->dev = dev;
{
int err;
- init_completion(&priv->flush_comp);
+ reinit_completion(&priv->flush_comp);
err = kvaser_usb_hydra_send_simple_cmd(priv->dev, CMD_FLUSH_QUEUE,
priv->channel);
} u;
} __packed;
+#define CMD_SIZE_ANY 0xff
+#define kvaser_fsize(field) sizeof_field(struct kvaser_cmd, field)
+
+static const u8 kvaser_usb_leaf_cmd_sizes_leaf[] = {
+ [CMD_START_CHIP_REPLY] = kvaser_fsize(u.simple),
+ [CMD_STOP_CHIP_REPLY] = kvaser_fsize(u.simple),
+ [CMD_GET_CARD_INFO_REPLY] = kvaser_fsize(u.cardinfo),
+ [CMD_TX_ACKNOWLEDGE] = kvaser_fsize(u.tx_acknowledge_header),
+ [CMD_GET_SOFTWARE_INFO_REPLY] = kvaser_fsize(u.leaf.softinfo),
+ [CMD_RX_STD_MESSAGE] = kvaser_fsize(u.leaf.rx_can),
+ [CMD_RX_EXT_MESSAGE] = kvaser_fsize(u.leaf.rx_can),
+ [CMD_LEAF_LOG_MESSAGE] = kvaser_fsize(u.leaf.log_message),
+ [CMD_CHIP_STATE_EVENT] = kvaser_fsize(u.leaf.chip_state_event),
+ [CMD_CAN_ERROR_EVENT] = kvaser_fsize(u.leaf.error_event),
+ /* ignored events: */
+ [CMD_FLUSH_QUEUE_REPLY] = CMD_SIZE_ANY,
+};
+
+static const u8 kvaser_usb_leaf_cmd_sizes_usbcan[] = {
+ [CMD_START_CHIP_REPLY] = kvaser_fsize(u.simple),
+ [CMD_STOP_CHIP_REPLY] = kvaser_fsize(u.simple),
+ [CMD_GET_CARD_INFO_REPLY] = kvaser_fsize(u.cardinfo),
+ [CMD_TX_ACKNOWLEDGE] = kvaser_fsize(u.tx_acknowledge_header),
+ [CMD_GET_SOFTWARE_INFO_REPLY] = kvaser_fsize(u.usbcan.softinfo),
+ [CMD_RX_STD_MESSAGE] = kvaser_fsize(u.usbcan.rx_can),
+ [CMD_RX_EXT_MESSAGE] = kvaser_fsize(u.usbcan.rx_can),
+ [CMD_CHIP_STATE_EVENT] = kvaser_fsize(u.usbcan.chip_state_event),
+ [CMD_CAN_ERROR_EVENT] = kvaser_fsize(u.usbcan.error_event),
+ /* ignored events: */
+ [CMD_USBCAN_CLOCK_OVERFLOW_EVENT] = CMD_SIZE_ANY,
+};
+
/* Summary of a kvaser error event, for a unified Leaf/Usbcan error
* handling. Some discrepancies between the two families exist:
*
.bittiming_const = &kvaser_usb_flexc_bittiming_const,
};
+static int kvaser_usb_leaf_verify_size(const struct kvaser_usb *dev,
+ const struct kvaser_cmd *cmd)
+{
+ /* buffer size >= cmd->len ensured by caller */
+ u8 min_size = 0;
+
+ switch (dev->driver_info->family) {
+ case KVASER_LEAF:
+ if (cmd->id < ARRAY_SIZE(kvaser_usb_leaf_cmd_sizes_leaf))
+ min_size = kvaser_usb_leaf_cmd_sizes_leaf[cmd->id];
+ break;
+ case KVASER_USBCAN:
+ if (cmd->id < ARRAY_SIZE(kvaser_usb_leaf_cmd_sizes_usbcan))
+ min_size = kvaser_usb_leaf_cmd_sizes_usbcan[cmd->id];
+ break;
+ }
+
+ if (min_size == CMD_SIZE_ANY)
+ return 0;
+
+ if (min_size) {
+ min_size += CMD_HEADER_LEN;
+ if (cmd->len >= min_size)
+ return 0;
+
+ dev_err_ratelimited(&dev->intf->dev,
+ "Received command %u too short (size %u, needed %u)",
+ cmd->id, cmd->len, min_size);
+ return -EIO;
+ }
+
+ dev_warn_ratelimited(&dev->intf->dev,
+ "Unhandled command (%d, size %d)\n",
+ cmd->id, cmd->len);
+ return -EINVAL;
+}
+
static void *
kvaser_usb_leaf_frame_to_cmd(const struct kvaser_usb_net_priv *priv,
const struct sk_buff *skb, int *cmd_len,
end:
kfree(buf);
+ if (err == 0)
+ err = kvaser_usb_leaf_verify_size(dev, cmd);
+
return err;
}
static void kvaser_usb_leaf_handle_command(const struct kvaser_usb *dev,
const struct kvaser_cmd *cmd)
{
+ if (kvaser_usb_leaf_verify_size(dev, cmd) < 0)
+ return;
+
switch (cmd->id) {
case CMD_START_CHIP_REPLY:
kvaser_usb_leaf_start_chip_reply(dev, cmd);
switch (mode) {
case CAN_MODE_START:
+ kvaser_usb_unlink_tx_urbs(priv);
+
err = kvaser_usb_leaf_simple_cmd_async(priv, CMD_START_CHIP);
if (err)
return err;
+
+ priv->can.state = CAN_STATE_ERROR_ACTIVE;
break;
default:
return -EOPNOTSUPP;
return ret;
}
+static bool adin1110_port_dev_check(const struct net_device *dev)
+{
+ return dev->netdev_ops == &adin1110_netdev_ops;
+}
+
static int adin1110_netdevice_event(struct notifier_block *unused,
unsigned long event, void *ptr)
{
struct netdev_notifier_changeupper_info *info = ptr;
int ret = 0;
+ if (!adin1110_port_dev_check(dev))
+ return NOTIFY_DONE;
+
switch (event) {
case NETDEV_CHANGEUPPER:
if (netif_is_bridge_master(info->upper_dev)) {
phy_disconnect(data);
}
-static bool adin1110_port_dev_check(const struct net_device *dev)
-{
- return dev->netdev_ops == &adin1110_netdev_ops;
-}
-
static int adin1110_port_set_forwarding_state(struct adin1110_port_priv *port_priv)
{
struct adin1110_priv *priv = port_priv->priv;
obj-$(CONFIG_BGMAC_PLATFORM) += bgmac-platform.o
obj-$(CONFIG_SYSTEMPORT) += bcmsysport.o
obj-$(CONFIG_BNXT) += bnxt/
-
-# FIXME: temporarily silence -Warray-bounds on non W=1+ builds
-ifndef KBUILD_EXTRA_WARN
-CFLAGS_tg3.o += -Wno-array-bounds
-endif
/* Number of Receive hardware descriptor words */
#define SP_NUM_HW_RX_DESC_WORDS 1024
-#define SP_LT_NUM_HW_RX_DESC_WORDS 256
+#define SP_LT_NUM_HW_RX_DESC_WORDS 512
/* Internal linked-list RAM size */
#define SP_NUM_TX_DESC 1536
if (bp->vnic_info[i].rss_hash_key) {
if (i == 0)
- prandom_bytes(vnic->rss_hash_key,
+ get_random_bytes(vnic->rss_hash_key,
HW_HASH_KEY_SIZE);
else
memcpy(vnic->rss_hash_key,
for (i = 0; i < MAX_CM_SK_TBL_SZ; i++)
atomic_set(&cp->csk_tbl[i].ref_count, 0);
- port_id = prandom_u32();
- port_id %= CNIC_LOCAL_PORT_RANGE;
+ port_id = prandom_u32_max(CNIC_LOCAL_PORT_RANGE);
if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE,
CNIC_LOCAL_PORT_MIN, port_id)) {
cnic_cm_free_mem(dev);
{
u32 seed;
- seed = prandom_u32();
+ seed = get_random_u32();
cnic_ctx_wr(dev, 45, 0, seed);
return 0;
}
opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));
rpl5->opt0 = cpu_to_be64(opt0);
rpl5->opt2 = cpu_to_be32(opt2);
- rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
+ rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);
set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
tp->write_seq = snd_isn;
tp->snd_nxt = snd_isn;
tp->snd_una = snd_isn;
- inet_sk(sk)->inet_id = prandom_u32();
+ inet_sk(sk)->inet_id = get_random_u16();
assign_rxopt(sk, opt);
if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
current_timeo = *timeo_p;
noblock = (*timeo_p ? false : true);
if (csk_mem_free(cdev, sk)) {
- current_timeo = (prandom_u32() % (HZ / 5)) + 2;
- vm_wait = (prandom_u32() % (HZ / 5)) + 2;
+ current_timeo = prandom_u32_max(HZ / 5) + 2;
+ vm_wait = prandom_u32_max(HZ / 5) + 2;
}
add_wait_queue(sk_sleep(sk), &wait);
#include <linux/math64.h>
#include <linux/refcount.h>
#include <net/pkt_cls.h>
-#include <net/pkt_sched.h>
#include <net/tc_act/tc_gate.h>
static u16 enetc_get_max_gcl_len(struct enetc_hw *hw)
mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_ENB, 0xff);
mcs->tx_sa_active = alloc_mem(mcs, mcs->hw->sc_entries);
- if (!mcs->tx_sa_active)
+ if (!mcs->tx_sa_active) {
+ ret = -ENOMEM;
goto exit;
+ }
return ret;
exit:
default:
ret = -EINVAL;
goto fail;
- };
+ }
mutex_unlock(&mbox->lock);
sc_req = otx2_mbox_alloc_msg_mcs_rx_sc_cam_write(mbox);
if (!sc_req) {
- return -ENOMEM;
+ ret = -ENOMEM;
goto fail;
}
req = otx2_mbox_alloc_msg_mcs_flowid_ena_entry(mbox);
if (!req) {
- return -ENOMEM;
+ ret = -ENOMEM;
goto fail;
}
return 0;
fail:
dev_err(pfvf->dev, "Cannot notify PN wrapped event\n");
+ mutex_unlock(&mbox->lock);
return 0;
}
err = register_netdev(netdev);
if (err) {
dev_err(dev, "Failed to register netdevice\n");
- goto err_del_mcam_entries;
+ goto err_mcs_free;
}
err = otx2_wq_init(pf);
otx2_mcam_flow_del(pf);
err_unreg_netdev:
unregister_netdev(netdev);
+err_mcs_free:
+ cn10k_mcs_free(pf);
err_del_mcam_entries:
otx2_mcam_flow_del(pf);
err_ptp_destroy:
list_for_each_entry(binding, &block->binding_list, list) {
err = prestera_span_rule_add(binding, port, block->ingress);
+ if (err == -EEXIST)
+ return err;
if (err)
goto rollback;
}
refcount_inc(&nh_grp->refcount);
} else {
nh_grp = __prestera_nexthop_group_create(sw, key);
- if (IS_ERR(nh_grp))
- return ERR_CAST(nh_grp);
+ if (!nh_grp)
+ return ERR_PTR(-ENOMEM);
refcount_set(&nh_grp->refcount, 1);
}
case PRESTERA_FIB_TYPE_UC_NH:
fib_node->info.nh_grp = prestera_nexthop_group_get(sw,
nh_grp_key);
- if (!fib_node->info.nh_grp)
+ if (IS_ERR(fib_node->info.nh_grp))
goto err_nh_grp_get;
grp_id = fib_node->info.nh_grp->grp_id;
entry = prestera_span_entry_find_by_id(sw->span, span_id);
if (!entry)
- return false;
+ return -ENOENT;
if (!refcount_dec_and_test(&entry->ref_count))
return 0;
{
int err;
+ if (binding->span_id == PRESTERA_SPAN_INVALID_ID)
+ return -ENOENT;
+
err = prestera_hw_span_unbind(binding->port, ingress);
if (err)
return err;
endif
obj-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_ops.o
obj-$(CONFIG_NET_MEDIATEK_STAR_EMAC) += mtk_star_emac.o
-
-# FIXME: temporarily silence -Warray-bounds on non W=1+ builds
-ifndef KBUILD_EXTRA_WARN
-CFLAGS_mtk_ppe.o += -Wno-array-bounds
-endif
struct mlx5e_flow_meters *flow_meters;
u8 cir_man, cir_exp, cbs_man, cbs_exp;
struct mlx5_aso_wqe *aso_wqe;
+ unsigned long expires;
struct mlx5_aso *aso;
u64 rate, burst;
u8 ds_cnt;
mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl);
/* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */
- err = mlx5_aso_poll_cq(aso, true, 10);
+ expires = jiffies + msecs_to_jiffies(10);
+ do {
+ err = mlx5_aso_poll_cq(aso, true);
+ if (err)
+ usleep_range(2, 10);
+ } while (err && time_is_after_jiffies(expires));
mutex_unlock(&flow_meters->aso_lock);
return err;
MLX5_ACCESS_ASO_OPC_MOD_MACSEC);
macsec_aso_build_ctrl(aso, &aso_wqe->aso_ctrl, in);
mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl);
- err = mlx5_aso_poll_cq(maso, false, 10);
+ err = mlx5_aso_poll_cq(maso, false);
mutex_unlock(&aso->aso_lock);
return err;
macsec_aso_build_wqe_ctrl_seg(aso, &aso_wqe->aso_ctrl, NULL);
mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl);
- err = mlx5_aso_poll_cq(maso, false, 10);
+ err = mlx5_aso_poll_cq(maso, false);
if (err)
goto err_out;
WRITE_ONCE(doorbell_cseg, NULL);
}
-int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms)
+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
{
struct mlx5_aso_cq *cq = &aso->cq;
struct mlx5_cqe64 *cqe;
- unsigned long expires;
cqe = mlx5_cqwq_get_cqe(&cq->wq);
-
- expires = jiffies + msecs_to_jiffies(interval_ms);
- while (!cqe && time_is_after_jiffies(expires)) {
- usleep_range(2, 10);
- cqe = mlx5_cqwq_get_cqe(&cq->wq);
- }
-
if (!cqe)
return -ETIMEDOUT;
u32 obj_id, u32 opc_mode);
void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
struct mlx5_wqe_ctrl_seg *doorbell_cseg);
-int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms);
+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data);
struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn);
void mlx5_aso_destroy(struct mlx5_aso *aso);
if (ipv6_tun) {
key_layer_two |= NFP_FLOWER_LAYER2_TUN_IPV6;
key_size +=
- sizeof(struct nfp_flower_ipv6_udp_tun);
+ sizeof(struct nfp_flower_ipv6_gre_tun);
} else {
key_size +=
- sizeof(struct nfp_flower_ipv4_udp_tun);
+ sizeof(struct nfp_flower_ipv4_gre_tun);
}
if (enc_op.key) {
u64 test_reg;
u64 rnd;
- rnd = prandom_u32();
+ rnd = get_random_u32();
rnd >>= 1;
rocker_write32(rocker, TEST_REG, rnd);
test_reg = rocker_read32(rocker, TEST_REG);
return -EIO;
}
- rnd = prandom_u32();
+ rnd = get_random_u32();
rnd <<= 31;
- rnd |= prandom_u32();
+ rnd |= get_random_u32();
rocker_write64(rocker, TEST_REG64, rnd);
test_reg = rocker_read64(rocker, TEST_REG64);
if (test_reg != rnd * 2) {
if (err)
goto unmap;
- prandom_bytes(buf, ROCKER_TEST_DMA_BUF_SIZE);
+ get_random_bytes(buf, ROCKER_TEST_DMA_BUF_SIZE);
for (i = 0; i < ROCKER_TEST_DMA_BUF_SIZE; i++)
expect[i] = ~buf[i];
err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_INVERT,
hpreg_res = devm_request_region(&pdev->dev, pci_resource_start(pdev, 0),
pci_resource_len(pdev, 0), DRV_NAME);
- if (IS_ERR(hpreg_res)) {
- err = PTR_ERR(hpreg_res);
+ if (!hpreg_res) {
+ err = -EBUSY;
dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting.\n");
goto err_out_clear_quattro;
}
port = am65_common_get_port(common, i);
dl_port = &port->devlink_port;
- attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
+ if (port->ndev)
+ attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
+ else
+ attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
attrs.phys.port_number = port->port_id;
attrs.switch_id.id_len = sizeof(resource_size_t);
memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
if ((--bc->hdlctx.slotcnt) > 0)
return 0;
bc->hdlctx.slotcnt = bc->ch_params.slottime;
- if ((prandom_u32() % 256) > bc->ch_params.ppersist)
+ if (get_random_u8() > bc->ch_params.ppersist)
return 0;
}
}
if ((--s->hdlctx.slotcnt) > 0)
return;
s->hdlctx.slotcnt = s->ch_params.slottime;
- if ((prandom_u32() % 256) > s->ch_params.ppersist)
+ if (get_random_u8() > s->ch_params.ppersist)
return;
start_tx(dev, s);
}
yp->slotcnt = yp->slot / 10;
/* is random > persist ? */
- if ((prandom_u32() % 256) > yp->pers)
+ if (get_random_u8() > yp->pers)
return;
yam_start_tx(dev, yp);
u32 vf_alloc;
/* Serial number of the VF to team with */
u32 vf_serial;
-
+ /* completion variable to confirm vf association */
+ struct completion vf_add;
/* Is the current data path through the VF NIC? */
bool data_path_is_vf;
net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated;
net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial;
+
+ if (net_device_ctx->vf_alloc)
+ complete(&net_device_ctx->vf_add);
+
netdev_info(ndev, "VF slot %u %s\n",
net_device_ctx->vf_serial,
net_device_ctx->vf_alloc ? "added" : "removed");
}
+ /* Fallback path to check synthetic vf with
+ * help of mac addr
+ */
+ list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) {
+ ndev = hv_get_drvdata(ndev_ctx->device_ctx);
+ if (ether_addr_equal(vf_netdev->perm_addr, ndev->perm_addr)) {
+ netdev_notice(vf_netdev,
+ "falling back to mac addr based matching\n");
+ return ndev;
+ }
+ }
+
netdev_notice(vf_netdev,
"no netdev found for vf serial:%u\n", serial);
return NULL;
if (net_device_ctx->data_path_is_vf == vf_is_up)
return NOTIFY_OK;
+ if (vf_is_up && !net_device_ctx->vf_alloc) {
+ netdev_info(ndev, "Waiting for the VF association from host\n");
+ wait_for_completion(&net_device_ctx->vf_add);
+ }
+
ret = netvsc_switch_datapath(ndev, vf_is_up);
if (ret) {
netvsc_vf_setxdp(vf_netdev, NULL);
+ reinit_completion(&net_device_ctx->vf_add);
netdev_rx_handler_unregister(vf_netdev);
netdev_upper_dev_unlink(vf_netdev, ndev);
RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
+ init_completion(&net_device_ctx->vf_add);
spin_lock_init(&net_device_ctx->lock);
INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
{
ether_setup(dev);
- dev->min_mtu = 0;
+ /* ether_setup() has set dev->min_mtu to ETH_MIN_MTU. */
dev->max_mtu = ETH_MAX_MTU;
dev->priv_flags &= ~IFF_TX_SKB_SHARING;
netif_keep_dst(dev);
static int qca808x_phy_ms_random_seed_set(struct phy_device *phydev)
{
- u16 seed_value = (prandom_u32() % QCA808X_MASTER_SLAVE_SEED_RANGE);
+ u16 seed_value = prandom_u32_max(QCA808X_MASTER_SLAVE_SEED_RANGE);
return at803x_debug_reg_mask(phydev, QCA808X_PHY_DEBUG_LOCAL_SEED,
QCA808X_MASTER_SLAVE_SEED_CFG,
return phy_clear_bits(phydev, MII_BMCR, BMCR_ANENABLE | BMCR_SPEED100);
}
-static int ksz886x_cable_test_result_trans(u16 status, u16 mask)
+static __always_inline int ksz886x_cable_test_result_trans(u16 status, u16 mask)
{
switch (FIELD_GET(mask, status)) {
case KSZ8081_LMD_STAT_NORMAL:
}
}
-static bool ksz886x_cable_test_failed(u16 status, u16 mask)
+static __always_inline bool ksz886x_cable_test_failed(u16 status, u16 mask)
{
return FIELD_GET(mask, status) ==
KSZ8081_LMD_STAT_FAIL;
}
-static bool ksz886x_cable_test_fault_length_valid(u16 status, u16 mask)
+static __always_inline bool ksz886x_cable_test_fault_length_valid(u16 status, u16 mask)
{
switch (FIELD_GET(mask, status)) {
case KSZ8081_LMD_STAT_OPEN:
return false;
}
-static int ksz886x_cable_test_fault_length(struct phy_device *phydev, u16 status, u16 data_mask)
+static __always_inline int ksz886x_cable_test_fault_length(struct phy_device *phydev,
+ u16 status, u16 data_mask)
{
int dt;
case SFF8024_ECC_100GBASE_SR4_25GBASE_SR:
phylink_set(modes, 100000baseSR4_Full);
phylink_set(modes, 25000baseSR_Full);
+ __set_bit(PHY_INTERFACE_MODE_25GBASER, interfaces);
break;
case SFF8024_ECC_100GBASE_LR4_25GBASE_LR:
case SFF8024_ECC_100GBASE_ER4_25GBASE_ER:
case SFF8024_ECC_25GBASE_CR_S:
case SFF8024_ECC_25GBASE_CR_N:
phylink_set(modes, 25000baseCR_Full);
+ __set_bit(PHY_INTERFACE_MODE_25GBASER, interfaces);
break;
case SFF8024_ECC_10GBASE_T_SFI:
case SFF8024_ECC_10GBASE_T_SR:
break;
case SFF8024_ECC_5GBASE_T:
phylink_set(modes, 5000baseT_Full);
+ __set_bit(PHY_INTERFACE_MODE_5GBASER, interfaces);
break;
case SFF8024_ECC_2_5GBASE_T:
phylink_set(modes, 2500baseT_Full);
config PSE_REGULATOR
tristate "Regulator based PSE controller"
+ depends on REGULATOR || COMPILE_TEST
help
This module provides support for simple regulator based Ethernet Power
Sourcing Equipment without automatic classification support. For
/* I like... big packets and I cannot lie! */
bool big_packets;
+ /* number of sg entries allocated for big packets */
+ unsigned int big_packets_num_skbfrags;
+
/* Host will merge rx buffers for big packets (shake it! shake it!) */
bool mergeable_rx_bufs;
char *p;
int i, err, offset;
- sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
+ sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2);
- /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
- for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
+ /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */
+ for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) {
first = get_a_page(rq, gfp);
if (!first) {
if (list)
/* chain first in list head */
first->private = (unsigned long)list;
- err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
+ err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2,
first, gfp);
if (err < 0)
give_pages(rq, first);
return 0;
}
+static bool virtnet_check_guest_gso(const struct virtnet_info *vi)
+{
+ return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
+ virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
+ virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
+ virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO);
+}
+
+static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu)
+{
+ bool guest_gso = virtnet_check_guest_gso(vi);
+
+ /* If device can receive ANY guest GSO packets, regardless of mtu,
+ * allocate packets of maximum size, otherwise limit it to only
+ * mtu size worth only.
+ */
+ if (mtu > ETH_DATA_LEN || guest_gso) {
+ vi->big_packets = true;
+ vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE);
+ }
+}
+
static int virtnet_probe(struct virtio_device *vdev)
{
int i, err = -ENOMEM;
struct net_device *dev;
struct virtnet_info *vi;
u16 max_queue_pairs;
- int mtu;
+ int mtu = 0;
/* Find if host supports multiqueue/rss virtio_net device */
max_queue_pairs = 1;
INIT_WORK(&vi->config_work, virtnet_config_changed_work);
spin_lock_init(&vi->refill_lock);
- /* If we can receive ANY GSO packets, we must allocate large ones. */
- if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
- virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
- virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
- virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
- vi->big_packets = true;
-
if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
vi->mergeable_rx_bufs = true;
dev->mtu = mtu;
dev->max_mtu = mtu;
-
- /* TODO: size buffers correctly in this case. */
- if (dev->mtu > ETH_DATA_LEN)
- vi->big_packets = true;
}
+ virtnet_set_big_packets(vi, mtu);
+
if (vi->any_header_sg)
dev->needed_headroom = vi->hdr_len;
mutex_lock(&mutex);
for (i = 0; i < NUM_RAND_ROUTES; ++i) {
- prandom_bytes(ip, 4);
+ get_random_bytes(ip, 4);
cidr = prandom_u32_max(32) + 1;
peer = peers[prandom_u32_max(NUM_PEERS)];
if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr,
}
for (j = 0; j < NUM_MUTATED_ROUTES; ++j) {
memcpy(mutated, ip, 4);
- prandom_bytes(mutate_mask, 4);
+ get_random_bytes(mutate_mask, 4);
mutate_amount = prandom_u32_max(32);
for (k = 0; k < mutate_amount / 8; ++k)
mutate_mask[k] = 0xff;
for (k = 0; k < 4; ++k)
mutated[k] = (mutated[k] & mutate_mask[k]) |
(~mutate_mask[k] &
- prandom_u32_max(256));
+ get_random_u8());
cidr = prandom_u32_max(32) + 1;
peer = peers[prandom_u32_max(NUM_PEERS)];
if (wg_allowedips_insert_v4(&t,
}
for (i = 0; i < NUM_RAND_ROUTES; ++i) {
- prandom_bytes(ip, 16);
+ get_random_bytes(ip, 16);
cidr = prandom_u32_max(128) + 1;
peer = peers[prandom_u32_max(NUM_PEERS)];
if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr,
}
for (j = 0; j < NUM_MUTATED_ROUTES; ++j) {
memcpy(mutated, ip, 16);
- prandom_bytes(mutate_mask, 16);
+ get_random_bytes(mutate_mask, 16);
mutate_amount = prandom_u32_max(128);
for (k = 0; k < mutate_amount / 8; ++k)
mutate_mask[k] = 0xff;
for (k = 0; k < 4; ++k)
mutated[k] = (mutated[k] & mutate_mask[k]) |
(~mutate_mask[k] &
- prandom_u32_max(256));
+ get_random_u8());
cidr = prandom_u32_max(128) + 1;
peer = peers[prandom_u32_max(NUM_PEERS)];
if (wg_allowedips_insert_v6(&t,
for (j = 0;; ++j) {
for (i = 0; i < NUM_QUERIES; ++i) {
- prandom_bytes(ip, 4);
+ get_random_bytes(ip, 4);
if (lookup(t.root4, 32, ip) != horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) {
horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip);
pr_err("allowedips random v4 self-test: FAIL\n");
goto free;
}
- prandom_bytes(ip, 16);
+ get_random_bytes(ip, 16);
if (lookup(t.root6, 128, ip) != horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) {
pr_err("allowedips random v6 self-test: FAIL\n");
goto free;
struct cfg80211_chan_def def;
const struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap;
enum nl80211_band band;
- u16 *he_mcs_mask;
+ u16 he_mcs_mask[NL80211_HE_NSS_MAX];
u8 max_nss, he_mcs;
u16 he_tx_mcs = 0, v = 0;
int i, he_nss, nss_idx;
return;
band = def.chan->band;
- he_mcs_mask = arvif->bitrate_mask.control[band].he_mcs;
+ memcpy(he_mcs_mask, arvif->bitrate_mask.control[band].he_mcs,
+ sizeof(he_mcs_mask));
if (ath11k_peer_assoc_h_he_masked(he_mcs_mask))
return;
if (afx_hdl->is_listen && afx_hdl->my_listen_chan)
/* 100ms ~ 300ms */
err = brcmf_p2p_discover_listen(p2p, afx_hdl->my_listen_chan,
- 100 * (1 + prandom_u32() % 3));
+ 100 * (1 + prandom_u32_max(3)));
else
err = brcmf_p2p_act_frm_search(p2p, afx_hdl->peer_listen_chan);
memcpy(pfn_mac.mac, mac_addr, ETH_ALEN);
for (i = 0; i < ETH_ALEN; i++) {
pfn_mac.mac[i] &= mac_mask[i];
- pfn_mac.mac[i] |= get_random_int() & ~(mac_mask[i]);
+ pfn_mac.mac[i] |= get_random_u8() & ~(mac_mask[i]);
}
/* Clear multi bit */
pfn_mac.mac[0] &= 0xFE;
iwl_mvm_mac_ap_iterator, &data);
if (data.beacon_device_ts) {
- u32 rand = (prandom_u32() % (64 - 36)) + 36;
+ u32 rand = prandom_u32_max(64 - 36) + 36;
mvmvif->ap_beacon_time = data.beacon_device_ts +
ieee80211_tu_to_usec(data.beacon_int * rand /
100);
iwl_mvm_txq_from_tid(sta, tid);
mvmtxq->txq_id = IWL_MVM_INVALID_QUEUE;
+ list_del_init(&mvmtxq->list);
}
/* Regardless if this is a reserved TXQ for a STA - mark it as false */
mvmsta->tid_data[tid].txq_id = IWL_MVM_INVALID_QUEUE;
mvmtxq->txq_id = IWL_MVM_INVALID_QUEUE;
+ list_del_init(&mvmtxq->list);
}
mvmsta->tfd_queue_msk &= ~BIT(queue); /* Don't use this queue anymore */
}
rx_status.rate_idx = nla_get_u32(info->attrs[HWSIM_ATTR_RX_RATE]);
+ if (rx_status.rate_idx >= data2->hw->wiphy->bands[rx_status.band]->n_bitrates)
+ goto out;
rx_status.signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
hdr = (void *)skb->data;
tx_info->pkt_len = pkt_len;
mwifiex_form_mgmt_frame(skb, buf, len);
- *cookie = prandom_u32() | 1;
+ *cookie = get_random_u32() | 1;
if (ieee80211_is_action(mgmt->frame_control))
skb = mwifiex_clone_skb_for_tx_status(priv,
duration);
if (!ret) {
- *cookie = prandom_u32() | 1;
+ *cookie = get_random_u32() | 1;
priv->roc_cfg.cookie = *cookie;
priv->roc_cfg.chan = *chan;
skb_reserve(skb, q->buf_offset);
- if (q == &dev->q_rx[MT_RXQ_MCU]) {
- u32 *rxfce = (u32 *)skb->cb;
- *rxfce = info;
- }
+ *(u32 *)skb->cb = info;
__skb_put(skb, len);
done++;
u32 rxd1 = le32_to_cpu(rxd[1]);
u32 rxd2 = le32_to_cpu(rxd[2]);
u32 csum_mask = MT_RXD0_NORMAL_IP_SUM | MT_RXD0_NORMAL_UDP_TCP_SUM;
+ u32 csum_status = *(u32 *)skb->cb;
bool unicast, hdr_trans, remove_pad, insert_ccmp_hdr = false;
u16 hdr_gap;
int phy_idx;
spin_unlock_bh(&dev->sta_poll_lock);
}
- if ((rxd0 & csum_mask) == csum_mask)
+ if (mt76_is_mmio(&dev->mt76) && (rxd0 & csum_mask) == csum_mask &&
+ !(csum_status & (BIT(0) | BIT(2) | BIT(3))))
skb->ip_summed = CHECKSUM_UNNECESSARY;
if (rxd2 & MT_RXD2_NORMAL_FCS_ERR)
* When header translation failure is indicated,
* the hardware will insert an extra 2-byte field
* containing the data length after the protocol
- * type field.
+ * type field. This happens either when the LLC-SNAP
+ * pattern did not match, or if a VLAN header was
+ * detected.
*/
pad_start = 12;
if (get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q)
pad_start += 4;
-
- if (get_unaligned_be16(skb->data + pad_start) !=
- skb->len - pad_start - 2)
+ else
pad_start = 0;
}
u8 remove_pad, amsdu_info;
u8 mode = 0, qos_ctl = 0;
struct mt7915_sta *msta = NULL;
+ u32 csum_status = *(u32 *)skb->cb;
bool hdr_trans;
u16 hdr_gap;
u16 seq_ctrl = 0;
if (!sband->channels)
return -EINVAL;
- if ((rxd0 & csum_mask) == csum_mask)
+ if ((rxd0 & csum_mask) == csum_mask &&
+ !(csum_status & (BIT(0) | BIT(2) | BIT(3))))
skb->ip_summed = CHECKSUM_UNNECESSARY;
if (rxd1 & MT_RXD1_NORMAL_FCS_ERR)
* When header translation failure is indicated,
* the hardware will insert an extra 2-byte field
* containing the data length after the protocol
- * type field.
+ * type field. This happens either when the LLC-SNAP
+ * pattern did not match, or if a VLAN header was
+ * detected.
*/
pad_start = 12;
if (get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q)
pad_start += 4;
-
- if (get_unaligned_be16(skb->data + pad_start) !=
- skb->len - pad_start - 2)
+ else
pad_start = 0;
}
struct mt76_phy *mphy = &dev->mt76.phy;
struct mt7921_phy *phy = &dev->phy;
struct ieee80211_supported_band *sband;
+ u32 csum_status = *(u32 *)skb->cb;
u32 rxd0 = le32_to_cpu(rxd[0]);
u32 rxd1 = le32_to_cpu(rxd[1]);
u32 rxd2 = le32_to_cpu(rxd[2]);
if (!sband->channels)
return -EINVAL;
- if ((rxd0 & csum_mask) == csum_mask)
+ if (mt76_is_mmio(&dev->mt76) && (rxd0 & csum_mask) == csum_mask &&
+ !(csum_status & (BIT(0) | BIT(2) | BIT(3))))
skb->ip_summed = CHECKSUM_UNNECESSARY;
if (rxd1 & MT_RXD1_NORMAL_FCS_ERR)
.skb = skb,
.info = IEEE80211_SKB_CB(skb),
};
+ struct ieee80211_rate_status rs = {};
struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb);
struct mt76_wcid *wcid;
wcid = rcu_dereference(dev->wcid[cb->wcid]);
if (wcid) {
status.sta = wcid_to_sta(wcid);
- status.rates = NULL;
- status.n_rates = 0;
+ if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) {
+ rs.rate_idx = wcid->rate;
+ status.rates = &rs;
+ status.n_rates = 1;
+ } else {
+ status.n_rates = 0;
+ }
}
hw = mt76_tx_status_get_hw(dev, skb);
const u8 *vendor_ie;
int ret = 0;
- *cookie = prandom_u32();
+ *cookie = get_random_u32();
priv->tx_cookie = *cookie;
mgmt = (const struct ieee80211_mgmt *)buf;
{
struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
const struct ieee80211_mgmt *mgmt_frame = (void *)params->buf;
- u32 short_cookie = prandom_u32();
+ u32 short_cookie = get_random_u32();
u16 flags = 0;
u16 freq;
edca = &priv->edca.params[i];
score = ((edca->aifns + edca->cwmin) << 16) +
((edca->cwmax - edca->cwmin) *
- (get_random_int() & 0xFFFF));
+ get_random_u16());
if (score < best && (winner < 0 || i != 3)) {
best = score;
winner = i;
wl1271_warning("Fuse mac address is zero. using random mac");
/* Use TI oui and a random nic */
oui_addr = WLCORE_TI_OUI_ADDRESS;
- nic_addr = get_random_int();
+ nic_addr = get_random_u32();
} else {
oui_addr = wl->fuse_oui_addr;
/* fuse has the BD_ADDR, the WLAN addresses are the next two */
const uuid_t *nd_dev_to_uuid(struct device *dev)
{
- if (!dev)
- return &uuid_null;
-
- if (is_namespace_pmem(dev)) {
+ if (dev && is_namespace_pmem(dev)) {
struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
return nspm->uuid;
- } else
- return &uuid_null;
+ }
+ return &uuid_null;
}
EXPORT_SYMBOL(nd_dev_to_uuid);
*
* BLK-space is valid as long as it does not precede a PMEM
* allocation in a given region. PMEM-space must be contiguous
- * and adjacent to an existing existing allocation (if one
+ * and adjacent to an existing allocation (if one
* exists). If reserving PMEM any space is valid.
*/
static void space_valid(struct nd_region *nd_region, struct nvdimm_drvdata *ndd,
{
struct nd_region *nd_region = to_nd_region(dev->parent);
unsigned long long val;
- uuid_t **uuid = NULL;
int rc;
rc = kstrtoull(buf, 0, &val);
if (rc >= 0)
rc = nd_namespace_label_update(nd_region, dev);
- if (is_namespace_pmem(dev)) {
+ /* setting size zero == 'delete namespace' */
+ if (rc == 0 && val == 0 && is_namespace_pmem(dev)) {
struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
- uuid = &nspm->uuid;
- }
-
- if (rc == 0 && val == 0 && uuid) {
- /* setting size zero == 'delete namespace' */
- kfree(*uuid);
- *uuid = NULL;
+ kfree(nspm->uuid);
+ nspm->uuid = NULL;
}
dev_dbg(dev, "%llx %s (%d)\n", val, rc < 0 ? "fail" : "success", rc);
struct nd_namespace_common *ndns);
#if IS_ENABLED(CONFIG_ND_CLAIM)
/* max struct page size independent of kernel config */
-#define MAX_STRUCT_PAGE_SIZE 64
+#define MAX_STRUCT_PAGE_SIZE 128
int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap);
#else
static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
* when populating the vmemmap. This *should* be equal to
* PMD_SIZE for most architectures.
*
- * Also make sure size of struct page is less than 64. We
+ * Also make sure size of struct page is less than 128. We
* want to make sure we use large enough size here so that
* we don't have a dynamic reserve space depending on
* struct page size. But we also want to make sure we notice
{
struct nd_region *nd_region = to_nd_region(dev);
unsigned long val, dpa;
- u32 remainder;
+ u32 mappings, remainder;
int rc;
rc = kstrtoul(buf, 0, &val);
if (rc)
return rc;
- if (!nd_region->ndr_mappings)
- return -ENXIO;
-
/*
* Ensure space-align is evenly divisible by the region
* interleave-width because the kernel typically has no facility
* contribute to the tail capacity in system-physical-address
* space for the namespace.
*/
- dpa = div_u64_rem(val, nd_region->ndr_mappings, &remainder);
+ mappings = max_t(u32, 1, nd_region->ndr_mappings);
+ dpa = div_u64_rem(val, mappings, &remainder);
if (!is_power_of_2(dpa) || dpa < PAGE_SIZE
|| val > region_size(nd_region) || remainder)
return -EINVAL;
return rc;
}
/**
- * nvdimm_flush - flush any posted write queues between the cpu and pmem media
+ * generic_nvdimm_flush() - flush any posted write queues between the cpu and pmem media
* @nd_region: interleaved pmem region
*/
int generic_nvdimm_flush(struct nd_region *nd_region)
return rc;
}
-void __nvdimm_security_overwrite_query(struct nvdimm *nvdimm)
+static void __nvdimm_security_overwrite_query(struct nvdimm *nvdimm)
{
struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nvdimm->dev);
int rc;
mutex_lock(&nvme_dhchap_mutex);
if (!nvme_dhchap_seqnum)
- nvme_dhchap_seqnum = prandom_u32();
+ nvme_dhchap_seqnum = get_random_u32();
else {
nvme_dhchap_seqnum++;
if (!nvme_dhchap_seqnum)
for_each_node(node)
rcu_assign_pointer(head->current_path[node], NULL);
+ kblockd_schedule_work(&head->requeue_work);
}
static bool nvme_path_is_disabled(struct nvme_ns *ns)
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */
.driver_data = NVME_QUIRK_BOGUS_NID, },
+ { PCI_DEVICE(0x1e49, 0x0021), /* ZHITAI TiPro5000 NVMe SSD */
+ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
{ PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */
.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
{ PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */
.driver_data = NVME_QUIRK_BOGUS_NID, },
+ { PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */
+ .driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061),
.driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, },
{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065),
{
struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
- cancel_work_sync(&ctrl->err_work);
+ flush_work(&ctrl->err_work);
cancel_delayed_work_sync(&ctrl->reconnect_work);
}
static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
{
- cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
+ flush_work(&to_tcp_ctrl(ctrl)->err_work);
cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
}
}
if (p0 + function_len < pos) {
- printk(KERN_ERR "eisa_enumerator: function %d length mis-match "
+ printk(KERN_ERR "eisa_enumerator: function %d length mismatch "
"got %d, expected %d\n",
num_func, pos-p0, function_len);
res=-1;
}
if (pos != es->config_data_length) {
- printk(KERN_ERR "eisa_enumerator: config data length mis-match got %d, expected %d\n",
+ printk(KERN_ERR "eisa_enumerator: config data length mismatch got %d, expected %d\n",
pos, es->config_data_length);
res=-1;
}
if (num_func != es->num_functions) {
- printk(KERN_ERR "eisa_enumerator: number of functions mis-match got %d, expected %d\n",
+ printk(KERN_ERR "eisa_enumerator: number of functions mismatch got %d, expected %d\n",
num_func, es->num_functions);
res=-2;
}
}
if (es->eisa_slot_id != id) {
print_eisa_id(id_string, id);
- printk(KERN_ERR "EISA slot %d id mis-match: got %s",
+ printk(KERN_ERR "EISA slot %d id mismatch: got %s",
slot, id_string);
print_eisa_id(id_string, es->eisa_slot_id);
IMX7D,
IMX8MQ,
IMX8MM,
+ IMX8MP,
};
#define IMX6_PCIE_FLAG_IMX6_PHY BIT(0)
enum imx6_pcie_variants variant;
u32 flags;
int dbi_length;
+ const char *gpr;
};
struct imx6_pcie {
static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie)
{
WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ &&
- imx6_pcie->drvdata->variant != IMX8MM);
+ imx6_pcie->drvdata->variant != IMX8MM &&
+ imx6_pcie->drvdata->variant != IMX8MP);
return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;
}
{
switch (imx6_pcie->drvdata->variant) {
case IMX8MM:
+ case IMX8MP:
/*
* The PHY initialization had been done in the PHY
* driver, break here directly.
break;
case IMX8MM:
case IMX8MQ:
+ case IMX8MP:
ret = clk_prepare_enable(imx6_pcie->pcie_aux);
if (ret) {
dev_err(dev, "unable to enable pcie_aux clock\n");
break;
case IMX8MM:
case IMX8MQ:
+ case IMX8MP:
clk_disable_unprepare(imx6_pcie->pcie_aux);
break;
default:
reset_control_assert(imx6_pcie->pciephy_reset);
fallthrough;
case IMX8MM:
+ case IMX8MP:
reset_control_assert(imx6_pcie->apps_reset);
break;
case IMX6SX:
break;
case IMX6Q: /* Nothing to do */
case IMX8MM:
+ case IMX8MP:
break;
}
case IMX7D:
case IMX8MQ:
case IMX8MM:
+ case IMX8MP:
reset_control_deassert(imx6_pcie->apps_reset);
break;
}
case IMX7D:
case IMX8MQ:
case IMX8MM:
+ case IMX8MP:
reset_control_assert(imx6_pcie->apps_reset);
break;
}
}
if (imx6_pcie->phy) {
- ret = phy_power_on(imx6_pcie->phy);
+ ret = phy_init(imx6_pcie->phy);
if (ret) {
dev_err(dev, "pcie PHY power up failed\n");
goto err_clk_disable;
}
if (imx6_pcie->phy) {
- ret = phy_init(imx6_pcie->phy);
+ ret = phy_power_on(imx6_pcie->phy);
if (ret) {
dev_err(dev, "waiting for PHY ready timeout!\n");
goto err_phy_off;
err_phy_off:
if (imx6_pcie->phy)
- phy_power_off(imx6_pcie->phy);
+ phy_exit(imx6_pcie->phy);
err_clk_disable:
imx6_pcie_clk_disable(imx6_pcie);
err_reg_disable:
}
break;
case IMX8MM:
+ case IMX8MP:
imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux");
if (IS_ERR(imx6_pcie->pcie_aux))
return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux),
/* Grab GPR config register range */
imx6_pcie->iomuxc_gpr =
- syscon_regmap_lookup_by_compatible("fsl,imx6q-iomuxc-gpr");
+ syscon_regmap_lookup_by_compatible(imx6_pcie->drvdata->gpr);
if (IS_ERR(imx6_pcie->iomuxc_gpr)) {
dev_err(dev, "unable to find iomuxc registers\n");
return PTR_ERR(imx6_pcie->iomuxc_gpr);
.flags = IMX6_PCIE_FLAG_IMX6_PHY |
IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE,
.dbi_length = 0x200,
+ .gpr = "fsl,imx6q-iomuxc-gpr",
},
[IMX6SX] = {
.variant = IMX6SX,
.flags = IMX6_PCIE_FLAG_IMX6_PHY |
IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE |
IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
+ .gpr = "fsl,imx6q-iomuxc-gpr",
},
[IMX6QP] = {
.variant = IMX6QP,
IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE |
IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
.dbi_length = 0x200,
+ .gpr = "fsl,imx6q-iomuxc-gpr",
},
[IMX7D] = {
.variant = IMX7D,
.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
+ .gpr = "fsl,imx7d-iomuxc-gpr",
},
[IMX8MQ] = {
.variant = IMX8MQ,
+ .gpr = "fsl,imx8mq-iomuxc-gpr",
},
[IMX8MM] = {
.variant = IMX8MM,
.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
+ .gpr = "fsl,imx8mm-iomuxc-gpr",
+ },
+ [IMX8MP] = {
+ .variant = IMX8MP,
+ .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
+ .gpr = "fsl,imx8mp-iomuxc-gpr",
},
};
{ .compatible = "fsl,imx7d-pcie", .data = &drvdata[IMX7D], },
{ .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], },
{ .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], },
+ { .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], },
{},
};
irq_domain_remove(pp->msi_domain);
irq_domain_remove(pp->irq_domain);
-
- if (pp->msi_data) {
- struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
- struct device *dev = pci->dev;
-
- dma_unmap_page(dev, pp->msi_data, PAGE_SIZE, DMA_FROM_DEVICE);
- if (pp->msi_page)
- __free_page(pp->msi_page);
- }
}
static void dw_pcie_msi_init(struct dw_pcie_rp *pp)
struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
struct device *dev = pci->dev;
struct platform_device *pdev = to_platform_device(dev);
+ u64 *msi_vaddr;
int ret;
u32 ctrl, num_ctrls;
dw_chained_msi_isr, pp);
}
- ret = dma_set_mask(dev, DMA_BIT_MASK(32));
+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
if (ret)
dev_warn(dev, "Failed to set DMA mask to 32-bit. Devices with only 32-bit MSI support may not work properly\n");
- pp->msi_page = alloc_page(GFP_DMA32);
- pp->msi_data = dma_map_page(dev, pp->msi_page, 0,
- PAGE_SIZE, DMA_FROM_DEVICE);
- ret = dma_mapping_error(dev, pp->msi_data);
- if (ret) {
- dev_err(pci->dev, "Failed to map MSI data\n");
- __free_page(pp->msi_page);
- pp->msi_page = NULL;
- pp->msi_data = 0;
+ msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data,
+ GFP_KERNEL);
+ if (!msi_vaddr) {
+ dev_err(dev, "Failed to alloc and map MSI data\n");
dw_pcie_free_msi(pp);
-
- return ret;
+ return -ENOMEM;
}
return 0;
struct irq_domain *irq_domain;
struct irq_domain *msi_domain;
dma_addr_t msi_data;
- struct page *msi_page;
struct irq_chip *msi_irq_chip;
u32 num_vectors;
u32 irq_mask[MAX_MSI_CTRLS];
#include <linux/delay.h>
#include <linux/err.h>
#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
#include <linux/interrupt.h>
#include <linux/mfd/syscon.h>
#include <linux/of_address.h>
struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
- struct device_node *np = dev->of_node;
char name[32];
int ret, i;
/* This is an optional property */
- ret = of_gpio_named_count(np, "hisilicon,clken-gpios");
+ ret = gpiod_count(dev, "hisilicon,clken");
if (ret < 0)
return 0;
*/
#include <linux/clk.h>
+#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/gpio/consumer.h>
#include <linux/mfd/syscon.h>
#define PARF_SYS_CTRL 0x00
#define PARF_DB_CTRL 0x10
#define PARF_PM_CTRL 0x20
+#define PARF_MHI_CLOCK_RESET_CTRL 0x174
#define PARF_MHI_BASE_ADDR_LOWER 0x178
#define PARF_MHI_BASE_ADDR_UPPER 0x17c
#define PARF_DEBUG_INT_EN 0x190
#define PARF_ATU_BASE_ADDR 0x634
#define PARF_ATU_BASE_ADDR_HI 0x638
#define PARF_SRIS_MODE 0x644
+#define PARF_DEBUG_CNT_PM_LINKST_IN_L2 0xc04
+#define PARF_DEBUG_CNT_PM_LINKST_IN_L1 0xc0c
+#define PARF_DEBUG_CNT_PM_LINKST_IN_L0S 0xc10
+#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1 0xc84
+#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2 0xc88
#define PARF_DEVICE_TYPE 0x1000
#define PARF_BDF_TO_SID_CFG 0x2c00
#define PARF_PM_CTRL_READY_ENTR_L23 BIT(2)
#define PARF_PM_CTRL_REQ_NOT_ENTR_L1 BIT(5)
+/* PARF_MHI_CLOCK_RESET_CTRL fields */
+#define PARF_MSTR_AXI_CLK_EN BIT(1)
+
/* PARF_AXI_MSTR_RD_HALT_NO_WRITES register fields */
#define PARF_AXI_MSTR_RD_HALT_NO_WRITE_EN BIT(0)
/* PARF_SYS_CTRL register fields */
#define PARF_SYS_CTRL_AUX_PWR_DET BIT(4)
#define PARF_SYS_CTRL_CORE_CLK_CGC_DIS BIT(6)
+#define PARF_SYS_CTRL_MSTR_ACLK_CGC_DIS BIT(10)
#define PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE BIT(11)
/* PARF_DB_CTRL register fields */
QCOM_PCIE_EP_LINK_DOWN,
};
-static struct clk_bulk_data qcom_pcie_ep_clks[] = {
- { .id = "cfg" },
- { .id = "aux" },
- { .id = "bus_master" },
- { .id = "bus_slave" },
- { .id = "ref" },
- { .id = "sleep" },
- { .id = "slave_q2a" },
-};
-
+/**
+ * struct qcom_pcie_ep - Qualcomm PCIe Endpoint Controller
+ * @pci: Designware PCIe controller struct
+ * @parf: Qualcomm PCIe specific PARF register base
+ * @elbi: Designware PCIe specific ELBI register base
+ * @mmio: MMIO register base
+ * @perst_map: PERST regmap
+ * @mmio_res: MMIO region resource
+ * @core_reset: PCIe Endpoint core reset
+ * @reset: PERST# GPIO
+ * @wake: WAKE# GPIO
+ * @phy: PHY controller block
+ * @debugfs: PCIe Endpoint Debugfs directory
+ * @clks: PCIe clocks
+ * @num_clks: PCIe clocks count
+ * @perst_en: Flag for PERST enable
+ * @perst_sep_en: Flag for PERST separation enable
+ * @link_status: PCIe Link status
+ * @global_irq: Qualcomm PCIe specific Global IRQ
+ * @perst_irq: PERST# IRQ
+ */
struct qcom_pcie_ep {
struct dw_pcie pci;
void __iomem *parf;
void __iomem *elbi;
+ void __iomem *mmio;
struct regmap *perst_map;
struct resource *mmio_res;
struct gpio_desc *reset;
struct gpio_desc *wake;
struct phy *phy;
+ struct dentry *debugfs;
+
+ struct clk_bulk_data *clks;
+ int num_clks;
u32 perst_en;
u32 perst_sep_en;
*/
static void qcom_pcie_ep_configure_tcsr(struct qcom_pcie_ep *pcie_ep)
{
- regmap_write(pcie_ep->perst_map, pcie_ep->perst_en, 0);
- regmap_write(pcie_ep->perst_map, pcie_ep->perst_sep_en, 0);
+ if (pcie_ep->perst_map) {
+ regmap_write(pcie_ep->perst_map, pcie_ep->perst_en, 0);
+ regmap_write(pcie_ep->perst_map, pcie_ep->perst_sep_en, 0);
+ }
}
static int qcom_pcie_dw_link_up(struct dw_pcie *pci)
{
int ret;
- ret = clk_bulk_prepare_enable(ARRAY_SIZE(qcom_pcie_ep_clks),
- qcom_pcie_ep_clks);
+ ret = clk_bulk_prepare_enable(pcie_ep->num_clks, pcie_ep->clks);
if (ret)
return ret;
err_phy_exit:
phy_exit(pcie_ep->phy);
err_disable_clk:
- clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks),
- qcom_pcie_ep_clks);
+ clk_bulk_disable_unprepare(pcie_ep->num_clks, pcie_ep->clks);
return ret;
}
{
phy_power_off(pcie_ep->phy);
phy_exit(pcie_ep->phy);
- clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks),
- qcom_pcie_ep_clks);
+ clk_bulk_disable_unprepare(pcie_ep->num_clks, pcie_ep->clks);
}
static int qcom_pcie_perst_deassert(struct dw_pcie *pci)
val &= ~PARF_Q2A_FLUSH_EN;
writel_relaxed(val, pcie_ep->parf + PARF_Q2A_FLUSH);
- /* Disable DBI Wakeup, core clock CGC and enable AUX power */
+ /*
+ * Disable Master AXI clock during idle. Do not allow DBI access
+ * to take the core out of L1. Disable core clock gating that
+ * gates PIPE clock from propagating to core clock. Report to the
+ * host that Vaux is present.
+ */
val = readl_relaxed(pcie_ep->parf + PARF_SYS_CTRL);
+ val &= ~PARF_SYS_CTRL_MSTR_ACLK_CGC_DIS;
val |= PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE |
PARF_SYS_CTRL_CORE_CLK_CGC_DIS |
PARF_SYS_CTRL_AUX_PWR_DET;
pcie_ep->parf + PARF_MHI_BASE_ADDR_LOWER);
writel_relaxed(0, pcie_ep->parf + PARF_MHI_BASE_ADDR_UPPER);
+ /* Gate Master AXI clock to MHI bus during L1SS */
+ val = readl_relaxed(pcie_ep->parf + PARF_MHI_CLOCK_RESET_CTRL);
+ val &= ~PARF_MSTR_AXI_CLK_EN;
+ val = readl_relaxed(pcie_ep->parf + PARF_MHI_CLOCK_RESET_CTRL);
+
dw_pcie_ep_init_notify(&pcie_ep->pci.ep);
/* Enable LTSSM */
pcie_ep->mmio_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"mmio");
+ if (!pcie_ep->mmio_res) {
+ dev_err(dev, "Failed to get mmio resource\n");
+ return -EINVAL;
+ }
+
+ pcie_ep->mmio = devm_pci_remap_cfg_resource(dev, pcie_ep->mmio_res);
+ if (IS_ERR(pcie_ep->mmio))
+ return PTR_ERR(pcie_ep->mmio);
syscon = of_parse_phandle(dev->of_node, "qcom,perst-regs", 0);
if (!syscon) {
- dev_err(dev, "Failed to parse qcom,perst-regs\n");
- return -EINVAL;
+ dev_dbg(dev, "PERST separation not available\n");
+ return 0;
}
pcie_ep->perst_map = syscon_node_to_regmap(syscon);
ret = qcom_pcie_ep_get_io_resources(pdev, pcie_ep);
if (ret) {
- dev_err(&pdev->dev, "Failed to get io resources %d\n", ret);
+ dev_err(dev, "Failed to get io resources %d\n", ret);
return ret;
}
- ret = devm_clk_bulk_get(dev, ARRAY_SIZE(qcom_pcie_ep_clks),
- qcom_pcie_ep_clks);
- if (ret)
- return ret;
+ pcie_ep->num_clks = devm_clk_bulk_get_all(dev, &pcie_ep->clks);
+ if (pcie_ep->num_clks < 0) {
+ dev_err(dev, "Failed to get clocks\n");
+ return pcie_ep->num_clks;
+ }
pcie_ep->core_reset = devm_reset_control_get_exclusive(dev, "core");
if (IS_ERR(pcie_ep->core_reset))
if (IS_ERR(pcie_ep->wake))
return PTR_ERR(pcie_ep->wake);
- pcie_ep->phy = devm_phy_optional_get(&pdev->dev, "pciephy");
+ pcie_ep->phy = devm_phy_optional_get(dev, "pciephy");
if (IS_ERR(pcie_ep->phy))
ret = PTR_ERR(pcie_ep->phy);
static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev,
struct qcom_pcie_ep *pcie_ep)
{
- int irq, ret;
+ int ret;
- irq = platform_get_irq_byname(pdev, "global");
- if (irq < 0)
- return irq;
+ pcie_ep->global_irq = platform_get_irq_byname(pdev, "global");
+ if (pcie_ep->global_irq < 0)
+ return pcie_ep->global_irq;
- ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+ ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->global_irq, NULL,
qcom_pcie_ep_global_irq_thread,
IRQF_ONESHOT,
"global_irq", pcie_ep);
"perst_irq", pcie_ep);
if (ret) {
dev_err(&pdev->dev, "Failed to request PERST IRQ\n");
- disable_irq(irq);
+ disable_irq(pcie_ep->global_irq);
return ret;
}
}
}
+static int qcom_pcie_ep_link_transition_count(struct seq_file *s, void *data)
+{
+ struct qcom_pcie_ep *pcie_ep = (struct qcom_pcie_ep *)
+ dev_get_drvdata(s->private);
+
+ seq_printf(s, "L0s transition count: %u\n",
+ readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L0S));
+
+ seq_printf(s, "L1 transition count: %u\n",
+ readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L1));
+
+ seq_printf(s, "L1.1 transition count: %u\n",
+ readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1));
+
+ seq_printf(s, "L1.2 transition count: %u\n",
+ readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2));
+
+ seq_printf(s, "L2 transition count: %u\n",
+ readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L2));
+
+ return 0;
+}
+
+static void qcom_pcie_ep_init_debugfs(struct qcom_pcie_ep *pcie_ep)
+{
+ struct dw_pcie *pci = &pcie_ep->pci;
+
+ debugfs_create_devm_seqfile(pci->dev, "link_transition_count", pcie_ep->debugfs,
+ qcom_pcie_ep_link_transition_count);
+}
+
static const struct pci_epc_features qcom_pcie_epc_features = {
.linkup_notifier = true,
.core_init_notifier = true,
{
struct device *dev = &pdev->dev;
struct qcom_pcie_ep *pcie_ep;
+ char *name;
int ret;
pcie_ep = devm_kzalloc(dev, sizeof(*pcie_ep), GFP_KERNEL);
if (ret)
goto err_disable_resources;
+ name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
+ if (!name) {
+ ret = -ENOMEM;
+ goto err_disable_irqs;
+ }
+
+ pcie_ep->debugfs = debugfs_create_dir(name, NULL);
+ qcom_pcie_ep_init_debugfs(pcie_ep);
+
return 0;
+err_disable_irqs:
+ disable_irq(pcie_ep->global_irq);
+ disable_irq(pcie_ep->perst_irq);
+
err_disable_resources:
qcom_pcie_disable_resources(pcie_ep);
{
struct qcom_pcie_ep *pcie_ep = platform_get_drvdata(pdev);
+ disable_irq(pcie_ep->global_irq);
+ disable_irq(pcie_ep->perst_irq);
+
+ debugfs_remove_recursive(pcie_ep->debugfs);
+
if (pcie_ep->link_status == QCOM_PCIE_EP_LINK_DISABLED)
return 0;
static const struct of_device_id qcom_pcie_ep_match[] = {
{ .compatible = "qcom,sdx55-pcie-ep", },
+ { .compatible = "qcom,sm8450-pcie-ep", },
{ }
};
+MODULE_DEVICE_TABLE(of, qcom_pcie_ep_match);
static struct platform_driver qcom_pcie_ep_driver = {
.probe = qcom_pcie_ep_probe,
/* 6 clocks typically, 7 for sm8250 */
struct qcom_pcie_resources_2_7_0 {
- struct clk_bulk_data clks[9];
+ struct clk_bulk_data clks[12];
int num_clks;
struct regulator_bulk_data supplies[2];
struct reset_control *pci_reset;
int (*init)(struct qcom_pcie *pcie);
int (*post_init)(struct qcom_pcie *pcie);
void (*deinit)(struct qcom_pcie *pcie);
- void (*post_deinit)(struct qcom_pcie *pcie);
void (*ltssm_enable)(struct qcom_pcie *pcie);
int (*config_sid)(struct qcom_pcie *pcie);
};
struct qcom_pcie_cfg {
const struct qcom_pcie_ops *ops;
- unsigned int has_tbu_clk:1;
- unsigned int has_ddrss_sf_tbu_clk:1;
- unsigned int has_aggre0_clk:1;
- unsigned int has_aggre1_clk:1;
};
struct qcom_pcie {
struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0;
struct dw_pcie *pci = pcie->pci;
struct device *dev = pci->dev;
+ unsigned int num_clks, num_opt_clks;
unsigned int idx;
int ret;
res->clks[idx++].id = "bus_master";
res->clks[idx++].id = "bus_slave";
res->clks[idx++].id = "slave_q2a";
- if (pcie->cfg->has_tbu_clk)
- res->clks[idx++].id = "tbu";
- if (pcie->cfg->has_ddrss_sf_tbu_clk)
- res->clks[idx++].id = "ddrss_sf_tbu";
- if (pcie->cfg->has_aggre0_clk)
- res->clks[idx++].id = "aggre0";
- if (pcie->cfg->has_aggre1_clk)
- res->clks[idx++].id = "aggre1";
+ num_clks = idx;
+
+ ret = devm_clk_bulk_get(dev, num_clks, res->clks);
+ if (ret < 0)
+ return ret;
+
+ res->clks[idx++].id = "tbu";
+ res->clks[idx++].id = "ddrss_sf_tbu";
+ res->clks[idx++].id = "aggre0";
+ res->clks[idx++].id = "aggre1";
+ res->clks[idx++].id = "noc_aggr_4";
+ res->clks[idx++].id = "noc_aggr_south_sf";
+ res->clks[idx++].id = "cnoc_qx";
+
+ num_opt_clks = idx - num_clks;
res->num_clks = idx;
- ret = devm_clk_bulk_get(dev, res->num_clks, res->clks);
+ ret = devm_clk_bulk_get_optional(dev, num_opt_clks, res->clks + num_clks);
if (ret < 0)
return ret;
if (pcie->cfg->ops->config_sid) {
ret = pcie->cfg->ops->config_sid(pcie);
if (ret)
- goto err;
+ goto err_assert_reset;
}
return 0;
-err:
+err_assert_reset:
qcom_ep_reset_assert(pcie);
- if (pcie->cfg->ops->post_deinit)
- pcie->cfg->ops->post_deinit(pcie);
err_disable_phy:
phy_power_off(pcie->phy);
err_deinit:
.ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
};
-static const struct qcom_pcie_cfg apq8084_cfg = {
+static const struct qcom_pcie_cfg cfg_1_0_0 = {
.ops = &ops_1_0_0,
};
-static const struct qcom_pcie_cfg ipq8064_cfg = {
+static const struct qcom_pcie_cfg cfg_1_9_0 = {
+ .ops = &ops_1_9_0,
+};
+
+static const struct qcom_pcie_cfg cfg_2_1_0 = {
.ops = &ops_2_1_0,
};
-static const struct qcom_pcie_cfg msm8996_cfg = {
+static const struct qcom_pcie_cfg cfg_2_3_2 = {
.ops = &ops_2_3_2,
};
-static const struct qcom_pcie_cfg ipq8074_cfg = {
+static const struct qcom_pcie_cfg cfg_2_3_3 = {
.ops = &ops_2_3_3,
};
-static const struct qcom_pcie_cfg ipq4019_cfg = {
+static const struct qcom_pcie_cfg cfg_2_4_0 = {
.ops = &ops_2_4_0,
};
-static const struct qcom_pcie_cfg sdm845_cfg = {
+static const struct qcom_pcie_cfg cfg_2_7_0 = {
.ops = &ops_2_7_0,
- .has_tbu_clk = true,
-};
-
-static const struct qcom_pcie_cfg sm8150_cfg = {
- /* sm8150 has qcom IP rev 1.5.0. However 1.5.0 ops are same as
- * 1.9.0, so reuse the same.
- */
- .ops = &ops_1_9_0,
-};
-
-static const struct qcom_pcie_cfg sm8250_cfg = {
- .ops = &ops_1_9_0,
- .has_tbu_clk = true,
- .has_ddrss_sf_tbu_clk = true,
-};
-
-static const struct qcom_pcie_cfg sm8450_pcie0_cfg = {
- .ops = &ops_1_9_0,
- .has_ddrss_sf_tbu_clk = true,
- .has_aggre0_clk = true,
- .has_aggre1_clk = true,
-};
-
-static const struct qcom_pcie_cfg sm8450_pcie1_cfg = {
- .ops = &ops_1_9_0,
- .has_ddrss_sf_tbu_clk = true,
- .has_aggre1_clk = true,
-};
-
-static const struct qcom_pcie_cfg sc7280_cfg = {
- .ops = &ops_1_9_0,
- .has_tbu_clk = true,
-};
-
-static const struct qcom_pcie_cfg sc8180x_cfg = {
- .ops = &ops_1_9_0,
- .has_tbu_clk = true,
};
-static const struct qcom_pcie_cfg ipq6018_cfg = {
+static const struct qcom_pcie_cfg cfg_2_9_0 = {
.ops = &ops_2_9_0,
};
}
static const struct of_device_id qcom_pcie_match[] = {
- { .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
- { .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
- { .compatible = "qcom,pcie-ipq8064-v2", .data = &ipq8064_cfg },
- { .compatible = "qcom,pcie-apq8064", .data = &ipq8064_cfg },
- { .compatible = "qcom,pcie-msm8996", .data = &msm8996_cfg },
- { .compatible = "qcom,pcie-ipq8074", .data = &ipq8074_cfg },
- { .compatible = "qcom,pcie-ipq4019", .data = &ipq4019_cfg },
- { .compatible = "qcom,pcie-qcs404", .data = &ipq4019_cfg },
- { .compatible = "qcom,pcie-sdm845", .data = &sdm845_cfg },
- { .compatible = "qcom,pcie-sm8150", .data = &sm8150_cfg },
- { .compatible = "qcom,pcie-sm8250", .data = &sm8250_cfg },
- { .compatible = "qcom,pcie-sc8180x", .data = &sc8180x_cfg },
- { .compatible = "qcom,pcie-sm8450-pcie0", .data = &sm8450_pcie0_cfg },
- { .compatible = "qcom,pcie-sm8450-pcie1", .data = &sm8450_pcie1_cfg },
- { .compatible = "qcom,pcie-sc7280", .data = &sc7280_cfg },
- { .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
+ { .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 },
+ { .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 },
+ { .compatible = "qcom,pcie-ipq4019", .data = &cfg_2_4_0 },
+ { .compatible = "qcom,pcie-ipq6018", .data = &cfg_2_9_0 },
+ { .compatible = "qcom,pcie-ipq8064", .data = &cfg_2_1_0 },
+ { .compatible = "qcom,pcie-ipq8064-v2", .data = &cfg_2_1_0 },
+ { .compatible = "qcom,pcie-ipq8074", .data = &cfg_2_3_3 },
+ { .compatible = "qcom,pcie-msm8996", .data = &cfg_2_3_2 },
+ { .compatible = "qcom,pcie-qcs404", .data = &cfg_2_4_0 },
+ { .compatible = "qcom,pcie-sa8540p", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sc7280", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sc8180x", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sc8280xp", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sdm845", .data = &cfg_2_7_0 },
+ { .compatible = "qcom,pcie-sm8150", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sm8250", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 },
+ { .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 },
{ }
};
#define PCIE_CORE_DEV_ID_REG 0x0
#define PCIE_CORE_CMD_STATUS_REG 0x4
#define PCIE_CORE_DEV_REV_REG 0x8
+#define PCIE_CORE_SSDEV_ID_REG 0x2c
#define PCIE_CORE_PCIEXP_CAP 0xc0
#define PCIE_CORE_PCIERR_CAP 0x100
#define PCIE_CORE_ERR_CAPCTL_REG 0x118
/* Indicates supports for Completion Retry Status */
bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS);
+ bridge->subsystem_vendor_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) & 0xffff;
+ bridge->subsystem_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) >> 16;
bridge->has_pcie = true;
+ bridge->pcie_start = PCIE_CORE_PCIEXP_CAP;
bridge->data = pcie;
bridge->ops = &advk_pci_bridge_emul_ops;
#define FARADAY_PCI_DMA_MEM2_BASE 0x00000000
#define FARADAY_PCI_DMA_MEM3_BASE 0x00000000
-/* Defines for PCI configuration command register */
-#define PCI_CONF_ENABLE BIT(31)
-#define PCI_CONF_WHERE(r) ((r) & 0xFC)
-#define PCI_CONF_BUS(b) (((b) & 0xFF) << 16)
-#define PCI_CONF_DEVICE(d) (((d) & 0x1F) << 11)
-#define PCI_CONF_FUNCTION(f) (((f) & 0x07) << 8)
-
/**
* struct faraday_pci_variant - encodes IP block differences
* @cascaded_irq: this host has cascaded IRQs from an interrupt controller
unsigned int fn, int config, int size,
u32 *value)
{
- writel(PCI_CONF_BUS(bus_number) |
- PCI_CONF_DEVICE(PCI_SLOT(fn)) |
- PCI_CONF_FUNCTION(PCI_FUNC(fn)) |
- PCI_CONF_WHERE(config) |
- PCI_CONF_ENABLE,
+ writel(PCI_CONF1_ADDRESS(bus_number, PCI_SLOT(fn),
+ PCI_FUNC(fn), config),
p->base + FTPCI_CONFIG);
*value = readl(p->base + FTPCI_DATA);
{
int ret = PCIBIOS_SUCCESSFUL;
- writel(PCI_CONF_BUS(bus_number) |
- PCI_CONF_DEVICE(PCI_SLOT(fn)) |
- PCI_CONF_FUNCTION(PCI_FUNC(fn)) |
- PCI_CONF_WHERE(config) |
- PCI_CONF_ENABLE,
+ writel(PCI_CONF1_ADDRESS(bus_number, PCI_SLOT(fn),
+ PCI_FUNC(fn), config),
p->base + FTPCI_CONFIG);
switch (size) {
/* Are the new iobase/iolimit values invalid? */
if (conf->iolimit < conf->iobase ||
- conf->iolimitupper < conf->iobaseupper)
+ le16_to_cpu(conf->iolimitupper) < le16_to_cpu(conf->iobaseupper))
return mvebu_pcie_set_window(port, port->io_target, port->io_attr,
&desired, &port->iowin);
* is the CPU address.
*/
desired.remap = ((conf->iobase & 0xF0) << 8) |
- (conf->iobaseupper << 16);
+ (le16_to_cpu(conf->iobaseupper) << 16);
desired.base = port->pcie->io.start + desired.remap;
desired.size = ((0xFFF | ((conf->iolimit & 0xF0) << 8) |
- (conf->iolimitupper << 16)) -
+ (le16_to_cpu(conf->iolimitupper) << 16)) -
desired.remap) +
1;
struct pci_bridge_emul_conf *conf = &port->bridge.conf;
/* Are the new membase/memlimit values invalid? */
- if (conf->memlimit < conf->membase)
+ if (le16_to_cpu(conf->memlimit) < le16_to_cpu(conf->membase))
return mvebu_pcie_set_window(port, port->mem_target, port->mem_attr,
&desired, &port->memwin);
* window to setup, according to the PCI-to-PCI bridge
* specifications.
*/
- desired.base = ((conf->membase & 0xFFF0) << 16);
- desired.size = (((conf->memlimit & 0xFFF0) << 16) | 0xFFFFF) -
+ desired.base = ((le16_to_cpu(conf->membase) & 0xFFF0) << 16);
+ desired.size = (((le16_to_cpu(conf->memlimit) & 0xFFF0) << 16) | 0xFFFFF) -
desired.base + 1;
return mvebu_pcie_set_window(port, port->mem_target, port->mem_attr, &desired,
bridge->subsystem_vendor_id = ssdev_id & 0xffff;
bridge->subsystem_id = ssdev_id >> 16;
bridge->has_pcie = true;
+ bridge->pcie_start = PCIE_CAP_PCIEXP;
bridge->data = port;
bridge->ops = &mvebu_pci_bridge_emul_ops;
* address (access to which generates correct config transaction) falls in
* this 4 KiB region.
*/
-static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn,
- unsigned int where)
-{
- return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) |
- (PCI_FUNC(devfn) << 8) | (where & 0xff);
-}
-
static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
unsigned int devfn,
int where)
unsigned int offset;
u32 base;
- offset = tegra_pcie_conf_offset(bus->number, devfn, where);
+ offset = PCI_CONF1_EXT_ADDRESS(bus->number, PCI_SLOT(devfn),
+ PCI_FUNC(devfn), where) &
+ ~PCI_CONF1_ENABLE;
/* move 4 KiB window to offset within the FPCI region */
base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8);
u32 stat, idx;
int ret, i;
- reset = gpiod_get_from_of_node(np, "reset-gpios", 0,
- GPIOD_OUT_LOW, "PERST#");
+ reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset",
+ GPIOD_OUT_LOW, "PERST#");
if (IS_ERR(reset))
return PTR_ERR(reset);
.probe = mtk_pcie_probe,
.remove = mtk_pcie_remove,
.driver = {
- .name = "mtk-pcie",
+ .name = "mtk-pcie-gen3",
.of_match_table = mtk_pcie_of_match,
.pm = &mtk_pcie_pm_ops,
},
#include <linux/reset.h>
#include <linux/sys_soc.h>
+#include "../pci.h"
+
/* MediaTek-specific configuration registers */
#define PCIE_FTS_NUM 0x70c
#define PCIE_FTS_NUM_MASK GENMASK(15, 8)
writel_relaxed(val, port->base + reg);
}
-static inline u32 mt7621_pcie_get_cfgaddr(unsigned int bus, unsigned int slot,
- unsigned int func, unsigned int where)
-{
- return (((where & 0xf00) >> 8) << 24) | (bus << 16) | (slot << 11) |
- (func << 8) | (where & 0xfc) | 0x80000000;
-}
-
static void __iomem *mt7621_pcie_map_bus(struct pci_bus *bus,
unsigned int devfn, int where)
{
struct mt7621_pcie *pcie = bus->sysdata;
- u32 address = mt7621_pcie_get_cfgaddr(bus->number, PCI_SLOT(devfn),
- PCI_FUNC(devfn), where);
+ u32 address = PCI_CONF1_EXT_ADDRESS(bus->number, PCI_SLOT(devfn),
+ PCI_FUNC(devfn), where);
writel_relaxed(address, pcie->base + RALINK_PCI_CONFIG_ADDR);
static u32 read_config(struct mt7621_pcie *pcie, unsigned int dev, u32 reg)
{
- u32 address = mt7621_pcie_get_cfgaddr(0, dev, 0, reg);
+ u32 address = PCI_CONF1_EXT_ADDRESS(0, dev, 0, reg);
pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR);
return pcie_read(pcie, RALINK_PCI_CONFIG_DATA);
static void write_config(struct mt7621_pcie *pcie, unsigned int dev,
u32 reg, u32 val)
{
- u32 address = mt7621_pcie_get_cfgaddr(0, dev, 0, reg);
+ u32 address = PCI_CONF1_EXT_ADDRESS(0, dev, 0, reg);
pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR);
pcie_write(pcie, val, RALINK_PCI_CONFIG_DATA);
desc.pci.msi_attrib.can_mask = !pci_msi_ignore_mask &&
!desc.pci.msi_attrib.is_virtual;
- if (!desc.pci.msi_attrib.can_mask) {
+ if (desc.pci.msi_attrib.can_mask) {
addr = pci_msix_desc_addr(&desc);
desc.pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
}
if (!closest_pdevs)
return NULL;
- while ((pdev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
+ for_each_pci_dev(pdev) {
if (!pci_has_p2pmem(pdev))
continue;
#define PCI_BRIDGE_CONF_END PCI_STD_HEADER_SIZEOF
#define PCI_CAP_SSID_SIZEOF (PCI_SSVID_DEVICE_ID + 2)
-#define PCI_CAP_SSID_START PCI_BRIDGE_CONF_END
-#define PCI_CAP_SSID_END (PCI_CAP_SSID_START + PCI_CAP_SSID_SIZEOF)
#define PCI_CAP_PCIE_SIZEOF (PCI_EXP_SLTSTA2 + 2)
-#define PCI_CAP_PCIE_START PCI_CAP_SSID_END
-#define PCI_CAP_PCIE_END (PCI_CAP_PCIE_START + PCI_CAP_PCIE_SIZEOF)
/**
* struct pci_bridge_reg_behavior - register bits behaviors
switch (reg) {
case PCI_CAP_LIST_ID:
*value = PCI_CAP_ID_SSVID |
- (bridge->has_pcie ? (PCI_CAP_PCIE_START << 8) : 0);
+ ((bridge->pcie_start > bridge->ssid_start) ? (bridge->pcie_start << 8) : 0);
return PCI_BRIDGE_EMUL_HANDLED;
case PCI_SSVID_VENDOR_ID:
if (!bridge->pci_regs_behavior)
return -ENOMEM;
- if (bridge->subsystem_vendor_id)
- bridge->conf.capabilities_pointer = PCI_CAP_SSID_START;
- else if (bridge->has_pcie)
- bridge->conf.capabilities_pointer = PCI_CAP_PCIE_START;
- else
- bridge->conf.capabilities_pointer = 0;
+ /* If ssid_start and pcie_start were not specified then choose the lowest possible value. */
+ if (!bridge->ssid_start && !bridge->pcie_start) {
+ if (bridge->subsystem_vendor_id)
+ bridge->ssid_start = PCI_BRIDGE_CONF_END;
+ if (bridge->has_pcie)
+ bridge->pcie_start = bridge->ssid_start + PCI_CAP_SSID_SIZEOF;
+ } else if (!bridge->ssid_start && bridge->subsystem_vendor_id) {
+ if (bridge->pcie_start - PCI_BRIDGE_CONF_END >= PCI_CAP_SSID_SIZEOF)
+ bridge->ssid_start = PCI_BRIDGE_CONF_END;
+ else
+ bridge->ssid_start = bridge->pcie_start + PCI_CAP_PCIE_SIZEOF;
+ } else if (!bridge->pcie_start && bridge->has_pcie) {
+ if (bridge->ssid_start - PCI_BRIDGE_CONF_END >= PCI_CAP_PCIE_SIZEOF)
+ bridge->pcie_start = PCI_BRIDGE_CONF_END;
+ else
+ bridge->pcie_start = bridge->ssid_start + PCI_CAP_SSID_SIZEOF;
+ }
+
+ bridge->conf.capabilities_pointer = min(bridge->ssid_start, bridge->pcie_start);
if (bridge->conf.capabilities_pointer)
bridge->conf.status |= cpu_to_le16(PCI_STATUS_CAP_LIST);
if (bridge->has_pcie) {
bridge->pcie_conf.cap_id = PCI_CAP_ID_EXP;
+ bridge->pcie_conf.next = (bridge->ssid_start > bridge->pcie_start) ?
+ bridge->ssid_start : 0;
bridge->pcie_conf.cap |= cpu_to_le16(PCI_EXP_TYPE_ROOT_PORT << 4);
bridge->pcie_cap_regs_behavior =
kmemdup(pcie_cap_regs_behavior,
read_op = bridge->ops->read_base;
cfgspace = (__le32 *) &bridge->conf;
behavior = bridge->pci_regs_behavior;
- } else if (reg >= PCI_CAP_SSID_START && reg < PCI_CAP_SSID_END && bridge->subsystem_vendor_id) {
+ } else if (reg >= bridge->ssid_start && reg < bridge->ssid_start + PCI_CAP_SSID_SIZEOF &&
+ bridge->subsystem_vendor_id) {
/* Emulated PCI Bridge Subsystem Vendor ID capability */
- reg -= PCI_CAP_SSID_START;
+ reg -= bridge->ssid_start;
read_op = pci_bridge_emul_read_ssid;
cfgspace = NULL;
behavior = NULL;
- } else if (reg >= PCI_CAP_PCIE_START && reg < PCI_CAP_PCIE_END && bridge->has_pcie) {
+ } else if (reg >= bridge->pcie_start && reg < bridge->pcie_start + PCI_CAP_PCIE_SIZEOF &&
+ bridge->has_pcie) {
/* Our emulated PCIe capability */
- reg -= PCI_CAP_PCIE_START;
+ reg -= bridge->pcie_start;
read_op = bridge->ops->read_pcie;
cfgspace = (__le32 *) &bridge->pcie_conf;
behavior = bridge->pcie_cap_regs_behavior;
write_op = bridge->ops->write_base;
cfgspace = (__le32 *) &bridge->conf;
behavior = bridge->pci_regs_behavior;
- } else if (reg >= PCI_CAP_PCIE_START && reg < PCI_CAP_PCIE_END && bridge->has_pcie) {
+ } else if (reg >= bridge->pcie_start && reg < bridge->pcie_start + PCI_CAP_PCIE_SIZEOF &&
+ bridge->has_pcie) {
/* Our emulated PCIe capability */
- reg -= PCI_CAP_PCIE_START;
+ reg -= bridge->pcie_start;
write_op = bridge->ops->write_pcie;
cfgspace = (__le32 *) &bridge->pcie_conf;
behavior = bridge->pcie_cap_regs_behavior;
struct pci_bridge_reg_behavior *pci_regs_behavior;
struct pci_bridge_reg_behavior *pcie_cap_regs_behavior;
void *data;
+ u8 pcie_start;
+ u8 ssid_start;
bool has_pcie;
u16 subsystem_vendor_id;
u16 subsystem_id;
pci_dev->skip_bus_pm = false;
+ /*
+ * Disabling PTM allows some systems, e.g., Intel mobile chips
+ * since Coffee Lake, to enter a lower-power PM state.
+ */
+ pci_suspend_ptm(pci_dev);
+
if (pci_has_legacy_pm_support(pci_dev))
return pci_legacy_suspend(dev, PMSG_SUSPEND);
}
}
- if (pci_dev->skip_bus_pm) {
+ if (!pci_dev->state_saved) {
+ pci_save_state(pci_dev);
+
/*
- * Either the device is a bridge with a child in D0 below it, or
- * the function is running for the second time in a row without
- * going through full resume, which is possible only during
- * suspend-to-idle in a spurious wakeup case. The device should
- * be in D0 at this point, but if it is a bridge, it may be
- * necessary to save its state.
+ * If the device is a bridge with a child in D0 below it,
+ * it needs to stay in D0, so check skip_bus_pm to avoid
+ * putting it into a low-power state in that case.
*/
- if (!pci_dev->state_saved)
- pci_save_state(pci_dev);
- } else if (!pci_dev->state_saved) {
- pci_save_state(pci_dev);
- if (pci_power_manageable(pci_dev))
+ if (!pci_dev->skip_bus_pm && pci_power_manageable(pci_dev))
pci_prepare_to_sleep(pci_dev);
}
if (pci_dev->state_saved)
pci_restore_standard_config(pci_dev);
+ pci_resume_ptm(pci_dev);
+
if (pci_has_legacy_pm_support(pci_dev))
return pci_legacy_resume(dev);
pci_power_t prev = pci_dev->current_state;
int error;
+ pci_suspend_ptm(pci_dev);
+
/*
* If pci_dev->driver is not set (unbound), we leave the device in D0,
* but it may go to D3cold when the bridge above it runtime suspends.
* D3cold when the bridge above it runtime suspended.
*/
pci_pm_default_resume_early(pci_dev);
+ pci_resume_ptm(pci_dev);
if (!pci_dev->driver)
return 0;
#include <linux/pm_runtime.h>
#include <linux/msi.h>
#include <linux/of.h>
+#include <linux/aperture.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
.is_visible = pci_dev_reset_attr_is_visible,
};
+#define pci_dev_resource_resize_attr(n) \
+static ssize_t resource##n##_resize_show(struct device *dev, \
+ struct device_attribute *attr, \
+ char * buf) \
+{ \
+ struct pci_dev *pdev = to_pci_dev(dev); \
+ ssize_t ret; \
+ \
+ pci_config_pm_runtime_get(pdev); \
+ \
+ ret = sysfs_emit(buf, "%016llx\n", \
+ (u64)pci_rebar_get_possible_sizes(pdev, n)); \
+ \
+ pci_config_pm_runtime_put(pdev); \
+ \
+ return ret; \
+} \
+ \
+static ssize_t resource##n##_resize_store(struct device *dev, \
+ struct device_attribute *attr,\
+ const char *buf, size_t count)\
+{ \
+ struct pci_dev *pdev = to_pci_dev(dev); \
+ unsigned long size, flags; \
+ int ret, i; \
+ u16 cmd; \
+ \
+ if (kstrtoul(buf, 0, &size) < 0) \
+ return -EINVAL; \
+ \
+ device_lock(dev); \
+ if (dev->driver) { \
+ ret = -EBUSY; \
+ goto unlock; \
+ } \
+ \
+ pci_config_pm_runtime_get(pdev); \
+ \
+ if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) { \
+ ret = aperture_remove_conflicting_pci_devices(pdev, \
+ "resourceN_resize"); \
+ if (ret) \
+ goto pm_put; \
+ } \
+ \
+ pci_read_config_word(pdev, PCI_COMMAND, &cmd); \
+ pci_write_config_word(pdev, PCI_COMMAND, \
+ cmd & ~PCI_COMMAND_MEMORY); \
+ \
+ flags = pci_resource_flags(pdev, n); \
+ \
+ pci_remove_resource_files(pdev); \
+ \
+ for (i = 0; i < PCI_STD_NUM_BARS; i++) { \
+ if (pci_resource_len(pdev, i) && \
+ pci_resource_flags(pdev, i) == flags) \
+ pci_release_resource(pdev, i); \
+ } \
+ \
+ ret = pci_resize_resource(pdev, n, size); \
+ \
+ pci_assign_unassigned_bus_resources(pdev->bus); \
+ \
+ if (pci_create_resource_files(pdev)) \
+ pci_warn(pdev, "Failed to recreate resource files after BAR resizing\n");\
+ \
+ pci_write_config_word(pdev, PCI_COMMAND, cmd); \
+pm_put: \
+ pci_config_pm_runtime_put(pdev); \
+unlock: \
+ device_unlock(dev); \
+ \
+ return ret ? ret : count; \
+} \
+static DEVICE_ATTR_RW(resource##n##_resize)
+
+pci_dev_resource_resize_attr(0);
+pci_dev_resource_resize_attr(1);
+pci_dev_resource_resize_attr(2);
+pci_dev_resource_resize_attr(3);
+pci_dev_resource_resize_attr(4);
+pci_dev_resource_resize_attr(5);
+
+static struct attribute *resource_resize_attrs[] = {
+ &dev_attr_resource0_resize.attr,
+ &dev_attr_resource1_resize.attr,
+ &dev_attr_resource2_resize.attr,
+ &dev_attr_resource3_resize.attr,
+ &dev_attr_resource4_resize.attr,
+ &dev_attr_resource5_resize.attr,
+ NULL,
+};
+
+static umode_t resource_resize_is_visible(struct kobject *kobj,
+ struct attribute *a, int n)
+{
+ struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+
+ return pci_rebar_get_current_size(pdev, n) < 0 ? 0 : a->mode;
+}
+
+static const struct attribute_group pci_dev_resource_resize_group = {
+ .attrs = resource_resize_attrs,
+ .is_visible = resource_resize_is_visible,
+};
+
int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev)
{
if (!sysfs_initialized)
#ifdef CONFIG_ACPI
&pci_dev_acpi_attr_group,
#endif
+ &pci_dev_resource_resize_group,
NULL,
};
static void pci_dev_d3_sleep(struct pci_dev *dev)
{
- unsigned int delay = dev->d3hot_delay;
-
- if (delay < pci_pm_d3hot_delay)
- delay = pci_pm_d3hot_delay;
-
- if (delay)
- msleep(delay);
+ unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
+ unsigned int upper;
+
+ if (delay_ms) {
+ /* Use a 20% upper bound, 1ms minimum */
+ upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U);
+ usleep_range(delay_ms * USEC_PER_MSEC,
+ (delay_ms + upper) * USEC_PER_MSEC);
+ }
}
bool pci_reset_supported(struct pci_dev *dev)
return i;
pci_save_ltr_state(dev);
+ pci_save_aspm_l1ss_state(dev);
pci_save_dpc_state(dev);
pci_save_aer_state(dev);
pci_save_ptm_state(dev);
* LTR itself (in the PCIe capability).
*/
pci_restore_ltr_state(dev);
+ pci_restore_aspm_l1ss_state(dev);
pci_restore_pcie_state(dev);
pci_restore_pasid_state(dev);
if (target_state == PCI_POWER_ERROR)
return -EIO;
- /*
- * There are systems (for example, Intel mobile chips since Coffee
- * Lake) where the power drawn while suspended can be significantly
- * reduced by disabling PTM on PCIe root ports as this allows the
- * port to enter a lower-power PM state and the SoC to reach a
- * lower-power idle state as a whole.
- */
- if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
- pci_disable_ptm(dev);
-
pci_enable_wake(dev, target_state, wakeup);
error = pci_set_power_state(dev, target_state);
- if (error) {
+ if (error)
pci_enable_wake(dev, target_state, false);
- pci_restore_ptm_state(dev);
- }
return error;
}
if (target_state == PCI_POWER_ERROR)
return -EIO;
- /*
- * There are systems (for example, Intel mobile chips since Coffee
- * Lake) where the power drawn while suspended can be significantly
- * reduced by disabling PTM on PCIe root ports as this allows the
- * port to enter a lower-power PM state and the SoC to reach a
- * lower-power idle state as a whole.
- */
- if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
- pci_disable_ptm(dev);
-
__pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
error = pci_set_power_state(dev, target_state);
- if (error) {
+ if (error)
pci_enable_wake(dev, target_state, false);
- pci_restore_ptm_state(dev);
- }
return error;
}
if (error)
pci_err(dev, "unable to allocate suspend buffer for LTR\n");
+ error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_L1SS,
+ 2 * sizeof(u32));
+ if (error)
+ pci_err(dev, "unable to allocate suspend buffer for ASPM-L1SS\n");
+
pci_allocate_vc_save_buffers(dev);
}
#endif /* CONFIG_PCI_IOV */
#ifdef CONFIG_PCIE_PTM
+void pci_ptm_init(struct pci_dev *dev);
void pci_save_ptm_state(struct pci_dev *dev);
void pci_restore_ptm_state(struct pci_dev *dev);
-void pci_disable_ptm(struct pci_dev *dev);
+void pci_suspend_ptm(struct pci_dev *dev);
+void pci_resume_ptm(struct pci_dev *dev);
#else
+static inline void pci_ptm_init(struct pci_dev *dev) { }
static inline void pci_save_ptm_state(struct pci_dev *dev) { }
static inline void pci_restore_ptm_state(struct pci_dev *dev) { }
-static inline void pci_disable_ptm(struct pci_dev *dev) { }
+static inline void pci_suspend_ptm(struct pci_dev *dev) { }
+static inline void pci_resume_ptm(struct pci_dev *dev) { }
#endif
unsigned long pci_cardbus_resource_alignment(struct resource *);
void pcie_aspm_init_link_state(struct pci_dev *pdev);
void pcie_aspm_exit_link_state(struct pci_dev *pdev);
void pcie_aspm_powersave_config_link(struct pci_dev *pdev);
+void pci_save_aspm_l1ss_state(struct pci_dev *dev);
+void pci_restore_aspm_l1ss_state(struct pci_dev *dev);
#else
static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { }
static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev) { }
static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { }
+static inline void pci_save_aspm_l1ss_state(struct pci_dev *dev) { }
+static inline void pci_restore_aspm_l1ss_state(struct pci_dev *dev) { }
#endif
#ifdef CONFIG_PCIE_ECRC
static inline void pcie_ecrc_get_policy(char *str) { }
#endif
-#ifdef CONFIG_PCIE_PTM
-void pci_ptm_init(struct pci_dev *dev);
-#else
-static inline void pci_ptm_init(struct pci_dev *dev) { }
-#endif
-
struct pci_dev_reset_methods {
u16 vendor;
u16 device;
}
#endif
+/*
+ * Config Address for PCI Configuration Mechanism #1
+ *
+ * See PCI Local Bus Specification, Revision 3.0,
+ * Section 3.2.2.3.2, Figure 3-2, p. 50.
+ */
+
+#define PCI_CONF1_BUS_SHIFT 16 /* Bus number */
+#define PCI_CONF1_DEV_SHIFT 11 /* Device number */
+#define PCI_CONF1_FUNC_SHIFT 8 /* Function number */
+
+#define PCI_CONF1_BUS_MASK 0xff
+#define PCI_CONF1_DEV_MASK 0x1f
+#define PCI_CONF1_FUNC_MASK 0x7
+#define PCI_CONF1_REG_MASK 0xfc /* Limit aligned offset to a maximum of 256B */
+
+#define PCI_CONF1_ENABLE BIT(31)
+#define PCI_CONF1_BUS(x) (((x) & PCI_CONF1_BUS_MASK) << PCI_CONF1_BUS_SHIFT)
+#define PCI_CONF1_DEV(x) (((x) & PCI_CONF1_DEV_MASK) << PCI_CONF1_DEV_SHIFT)
+#define PCI_CONF1_FUNC(x) (((x) & PCI_CONF1_FUNC_MASK) << PCI_CONF1_FUNC_SHIFT)
+#define PCI_CONF1_REG(x) ((x) & PCI_CONF1_REG_MASK)
+
+#define PCI_CONF1_ADDRESS(bus, dev, func, reg) \
+ (PCI_CONF1_ENABLE | \
+ PCI_CONF1_BUS(bus) | \
+ PCI_CONF1_DEV(dev) | \
+ PCI_CONF1_FUNC(func) | \
+ PCI_CONF1_REG(reg))
+
+/*
+ * Extension of PCI Config Address for accessing extended PCIe registers
+ *
+ * No standardized specification, but used on lot of non-ECAM-compliant ARM SoCs
+ * or on AMD Barcelona and new CPUs. Reserved bits [27:24] of PCI Config Address
+ * are used for specifying additional 4 high bits of PCI Express register.
+ */
+
+#define PCI_CONF1_EXT_REG_SHIFT 16
+#define PCI_CONF1_EXT_REG_MASK 0xf00
+#define PCI_CONF1_EXT_REG(x) (((x) & PCI_CONF1_EXT_REG_MASK) << PCI_CONF1_EXT_REG_SHIFT)
+
+#define PCI_CONF1_EXT_ADDRESS(bus, dev, func, reg) \
+ (PCI_CONF1_ADDRESS(bus, dev, func, reg) | \
+ PCI_CONF1_EXT_REG(reg))
+
#endif /* DRIVERS_PCI_H */
*/
#include <linux/kernel.h>
+#include <linux/math.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
return 0;
}
+/*
+ * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1
+ * register. Ports enter L1.2 when the most recent LTR value is greater
+ * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we
+ * don't enter L1.2 too aggressively.
+ *
+ * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.
+ */
static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
{
- u32 threshold_ns = threshold_us * 1000;
+ u64 threshold_ns = (u64) threshold_us * 1000;
- /* See PCIe r3.1, sec 7.33.3 and sec 6.18 */
- if (threshold_ns < 32) {
- *scale = 0;
+ /*
+ * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max
+ * value of 0x3ff.
+ */
+ if (threshold_ns <= 0x3ff * 1) {
+ *scale = 0; /* Value times 1ns */
*value = threshold_ns;
- } else if (threshold_ns < 1024) {
- *scale = 1;
- *value = threshold_ns >> 5;
- } else if (threshold_ns < 32768) {
- *scale = 2;
- *value = threshold_ns >> 10;
- } else if (threshold_ns < 1048576) {
- *scale = 3;
- *value = threshold_ns >> 15;
- } else if (threshold_ns < 33554432) {
- *scale = 4;
- *value = threshold_ns >> 20;
+ } else if (threshold_ns <= 0x3ff * 32) {
+ *scale = 1; /* Value times 32ns */
+ *value = roundup(threshold_ns, 32) / 32;
+ } else if (threshold_ns <= 0x3ff * 1024) {
+ *scale = 2; /* Value times 1024ns */
+ *value = roundup(threshold_ns, 1024) / 1024;
+ } else if (threshold_ns <= 0x3ff * 32768) {
+ *scale = 3; /* Value times 32768ns */
+ *value = roundup(threshold_ns, 32768) / 32768;
+ } else if (threshold_ns <= 0x3ff * 1048576) {
+ *scale = 4; /* Value times 1048576ns */
+ *value = roundup(threshold_ns, 1048576) / 1048576;
+ } else if (threshold_ns <= 0x3ff * (u64) 33554432) {
+ *scale = 5; /* Value times 33554432ns */
+ *value = roundup(threshold_ns, 33554432) / 33554432;
} else {
*scale = 5;
- *value = threshold_ns >> 25;
+ *value = 0x3ff; /* Max representable value */
}
}
pci_write_config_dword(pdev, pos, val);
}
+static void aspm_program_l1ss(struct pci_dev *dev, u32 ctl1, u32 ctl2)
+{
+ u16 l1ss = dev->l1ss;
+ u32 l1_2_enable;
+
+ /*
+ * Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be
+ * programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1.
+ */
+ pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL2, ctl2);
+
+ /*
+ * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in
+ * PCI_L1SS_CTL1 must be programmed *before* setting the L1.2
+ * enable bits, even though they're all in PCI_L1SS_CTL1.
+ */
+ l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK;
+ ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;
+
+ pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, ctl1);
+ if (l1_2_enable)
+ pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1,
+ ctl1 | l1_2_enable);
+}
+
/* Calculate L1.2 PM substate timing parameters */
static void aspm_calc_l1ss_info(struct pcie_link_state *link,
u32 parent_l1ss_cap, u32 child_l1ss_cap)
u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
u32 ctl1 = 0, ctl2 = 0;
u32 pctl1, pctl2, cctl1, cctl2;
- u32 pl1_2_enables, cl1_2_enables;
if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
return;
ctl2 == pctl2 && ctl2 == cctl2)
return;
- /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
- pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
- cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
+ pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
+ pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
+ aspm_program_l1ss(parent, pctl1, ctl2);
+
+ cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE);
+ cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |
+ PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
+ PCI_L1SS_CTL1_LTR_L12_TH_SCALE));
+ aspm_program_l1ss(child, cctl1, ctl2);
+}
- if (pl1_2_enables || cl1_2_enables) {
- pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
- PCI_L1SS_CTL1_L1_2_MASK, 0);
- pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
- PCI_L1SS_CTL1_L1_2_MASK, 0);
- }
+static void aspm_l1ss_init(struct pcie_link_state *link)
+{
+ struct pci_dev *child = link->downstream, *parent = link->pdev;
+ u32 parent_l1ss_cap, child_l1ss_cap;
+ u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
- /* Program T_POWER_ON times in both ports */
- pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
- pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
+ if (!parent->l1ss || !child->l1ss)
+ return;
- /* Program Common_Mode_Restore_Time in upstream device */
- pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
- PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
+ /* Setup L1 substate */
+ pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
+ &parent_l1ss_cap);
+ pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
+ &child_l1ss_cap);
- /* Program LTR_L1.2_THRESHOLD time in both ports */
- pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
- PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
- PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
- pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
- PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
- PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
-
- if (pl1_2_enables || cl1_2_enables) {
- pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
- pl1_2_enables);
- pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
- cl1_2_enables);
- }
+ if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
+ parent_l1ss_cap = 0;
+ if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
+ child_l1ss_cap = 0;
+
+ /*
+ * If we don't have LTR for the entire path from the Root Complex
+ * to this device, we can't use ASPM L1.2 because it relies on the
+ * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
+ */
+ if (!child->ltr_path)
+ child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
+
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
+ link->aspm_support |= ASPM_STATE_L1_1;
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
+ link->aspm_support |= ASPM_STATE_L1_2;
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
+ link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
+ if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
+ link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
+
+ if (parent_l1ss_cap)
+ pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
+ &parent_l1ss_ctl1);
+ if (child_l1ss_cap)
+ pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
+ &child_l1ss_ctl1);
+
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
+ link->aspm_enabled |= ASPM_STATE_L1_1;
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
+ link->aspm_enabled |= ASPM_STATE_L1_2;
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
+ link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
+ if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
+ link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
+
+ if (link->aspm_support & ASPM_STATE_L1SS)
+ aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
}
static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
struct pci_dev *child = link->downstream, *parent = link->pdev;
u32 parent_lnkcap, child_lnkcap;
u16 parent_lnkctl, child_lnkctl;
- u32 parent_l1ss_cap, child_l1ss_cap;
- u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
struct pci_bus *linkbus = parent->subordinate;
if (blacklist) {
if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
link->aspm_enabled |= ASPM_STATE_L1;
- /* Setup L1 substate */
- pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
- &parent_l1ss_cap);
- pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
- &child_l1ss_cap);
-
- if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
- parent_l1ss_cap = 0;
- if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
- child_l1ss_cap = 0;
-
- /*
- * If we don't have LTR for the entire path from the Root Complex
- * to this device, we can't use ASPM L1.2 because it relies on the
- * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
- */
- if (!child->ltr_path)
- child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
-
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
- link->aspm_support |= ASPM_STATE_L1_1;
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
- link->aspm_support |= ASPM_STATE_L1_2;
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
- link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
- if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
- link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
-
- if (parent_l1ss_cap)
- pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
- &parent_l1ss_ctl1);
- if (child_l1ss_cap)
- pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
- &child_l1ss_ctl1);
-
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
- link->aspm_enabled |= ASPM_STATE_L1_1;
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
- link->aspm_enabled |= ASPM_STATE_L1_2;
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
- link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
- if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
- link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
-
- if (link->aspm_support & ASPM_STATE_L1SS)
- aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
+ aspm_l1ss_init(link);
/* Save default state */
link->aspm_default = link->aspm_enabled;
PCI_L1SS_CTL1_L1SS_MASK, val);
}
+void pci_save_aspm_l1ss_state(struct pci_dev *dev)
+{
+ struct pci_cap_saved_state *save_state;
+ u16 l1ss = dev->l1ss;
+ u32 *cap;
+
+ if (!l1ss)
+ return;
+
+ save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS);
+ if (!save_state)
+ return;
+
+ cap = (u32 *)&save_state->cap.data[0];
+ pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL2, cap++);
+ pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL1, cap++);
+}
+
+void pci_restore_aspm_l1ss_state(struct pci_dev *dev)
+{
+ struct pci_cap_saved_state *save_state;
+ u32 *cap, ctl1, ctl2;
+ u16 l1ss = dev->l1ss;
+
+ if (!l1ss)
+ return;
+
+ save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS);
+ if (!save_state)
+ return;
+
+ cap = (u32 *)&save_state->cap.data[0];
+ ctl2 = *cap++;
+ ctl1 = *cap;
+ aspm_program_l1ss(dev, ctl1, ctl2);
+}
+
static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
{
pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
return;
pdev->dpc_rp_extensions = true;
- pdev->dpc_rp_log_size = (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8;
- if (pdev->dpc_rp_log_size < 4 || pdev->dpc_rp_log_size > 9) {
- pci_err(pdev, "RP PIO log size %u is invalid\n",
- pdev->dpc_rp_log_size);
- pdev->dpc_rp_log_size = 0;
+
+ /* Quirks may set dpc_rp_log_size if device or firmware is buggy */
+ if (!pdev->dpc_rp_log_size) {
+ pdev->dpc_rp_log_size =
+ (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8;
+ if (pdev->dpc_rp_log_size < 4 || pdev->dpc_rp_log_size > 9) {
+ pci_err(pdev, "RP PIO log size %u is invalid\n",
+ pdev->dpc_rp_log_size);
+ pdev->dpc_rp_log_size = 0;
+ }
}
}
#include <linux/pci.h>
#include "../pci.h"
-static void pci_ptm_info(struct pci_dev *dev)
+/*
+ * If the next upstream device supports PTM, return it; otherwise return
+ * NULL. PTM Messages are local, so both link partners must support it.
+ */
+static struct pci_dev *pci_upstream_ptm(struct pci_dev *dev)
{
- char clock_desc[8];
+ struct pci_dev *ups = pci_upstream_bridge(dev);
- switch (dev->ptm_granularity) {
- case 0:
- snprintf(clock_desc, sizeof(clock_desc), "unknown");
- break;
- case 255:
- snprintf(clock_desc, sizeof(clock_desc), ">254ns");
- break;
- default:
- snprintf(clock_desc, sizeof(clock_desc), "%uns",
- dev->ptm_granularity);
- break;
- }
- pci_info(dev, "PTM enabled%s, %s granularity\n",
- dev->ptm_root ? " (root)" : "", clock_desc);
+ /*
+ * Switch Downstream Ports are not permitted to have a PTM
+ * capability; their PTM behavior is controlled by the Upstream
+ * Port (PCIe r5.0, sec 7.9.16), so if the upstream bridge is a
+ * Switch Downstream Port, look up one more level.
+ */
+ if (ups && pci_pcie_type(ups) == PCI_EXP_TYPE_DOWNSTREAM)
+ ups = pci_upstream_bridge(ups);
+
+ if (ups && ups->ptm_cap)
+ return ups;
+
+ return NULL;
}
-void pci_disable_ptm(struct pci_dev *dev)
+/*
+ * Find the PTM Capability (if present) and extract the information we need
+ * to use it.
+ */
+void pci_ptm_init(struct pci_dev *dev)
{
- int ptm;
- u16 ctrl;
+ u16 ptm;
+ u32 cap;
+ struct pci_dev *ups;
if (!pci_is_pcie(dev))
return;
if (!ptm)
return;
- pci_read_config_word(dev, ptm + PCI_PTM_CTRL, &ctrl);
- ctrl &= ~(PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT);
- pci_write_config_word(dev, ptm + PCI_PTM_CTRL, ctrl);
+ dev->ptm_cap = ptm;
+ pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_PTM, sizeof(u32));
+
+ pci_read_config_dword(dev, ptm + PCI_PTM_CAP, &cap);
+ dev->ptm_granularity = (cap & PCI_PTM_GRANULARITY_MASK) >> 8;
+
+ /*
+ * Per the spec recommendation (PCIe r6.0, sec 7.9.15.3), select the
+ * furthest upstream Time Source as the PTM Root. For Endpoints,
+ * "the Effective Granularity is the maximum Local Clock Granularity
+ * reported by the PTM Root and all intervening PTM Time Sources."
+ */
+ ups = pci_upstream_ptm(dev);
+ if (ups) {
+ if (ups->ptm_granularity == 0)
+ dev->ptm_granularity = 0;
+ else if (ups->ptm_granularity > dev->ptm_granularity)
+ dev->ptm_granularity = ups->ptm_granularity;
+ } else if (cap & PCI_PTM_CAP_ROOT) {
+ dev->ptm_root = 1;
+ } else if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
+
+ /*
+ * Per sec 7.9.15.3, this should be the Local Clock
+ * Granularity of the associated Time Source. But it
+ * doesn't say how to find that Time Source.
+ */
+ dev->ptm_granularity = 0;
+ }
+
+ if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
+ pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM)
+ pci_enable_ptm(dev, NULL);
}
void pci_save_ptm_state(struct pci_dev *dev)
{
- int ptm;
+ u16 ptm = dev->ptm_cap;
struct pci_cap_saved_state *save_state;
- u16 *cap;
+ u32 *cap;
- if (!pci_is_pcie(dev))
- return;
-
- ptm = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
if (!ptm)
return;
if (!save_state)
return;
- cap = (u16 *)&save_state->cap.data[0];
- pci_read_config_word(dev, ptm + PCI_PTM_CTRL, cap);
+ cap = (u32 *)&save_state->cap.data[0];
+ pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, cap);
}
void pci_restore_ptm_state(struct pci_dev *dev)
{
+ u16 ptm = dev->ptm_cap;
struct pci_cap_saved_state *save_state;
- int ptm;
- u16 *cap;
+ u32 *cap;
- if (!pci_is_pcie(dev))
+ if (!ptm)
return;
save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_PTM);
- ptm = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
- if (!save_state || !ptm)
+ if (!save_state)
return;
- cap = (u16 *)&save_state->cap.data[0];
- pci_write_config_word(dev, ptm + PCI_PTM_CTRL, *cap);
+ cap = (u32 *)&save_state->cap.data[0];
+ pci_write_config_dword(dev, ptm + PCI_PTM_CTRL, *cap);
}
-void pci_ptm_init(struct pci_dev *dev)
+/* Enable PTM in the Control register if possible */
+static int __pci_enable_ptm(struct pci_dev *dev)
{
- int pos;
- u32 cap, ctrl;
- u8 local_clock;
+ u16 ptm = dev->ptm_cap;
struct pci_dev *ups;
+ u32 ctrl;
- if (!pci_is_pcie(dev))
- return;
-
- /*
- * Enable PTM only on interior devices (root ports, switch ports,
- * etc.) on the assumption that it causes no link traffic until an
- * endpoint enables it.
- */
- if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT ||
- pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END))
- return;
+ if (!ptm)
+ return -EINVAL;
/*
- * Switch Downstream Ports are not permitted to have a PTM
- * capability; their PTM behavior is controlled by the Upstream
- * Port (PCIe r5.0, sec 7.9.16).
+ * A device uses local PTM Messages to request time information
+ * from a PTM Root that's farther upstream. Every device along the
+ * path must support PTM and have it enabled so it can handle the
+ * messages. Therefore, if this device is not a PTM Root, the
+ * upstream link partner must have PTM enabled before we can enable
+ * PTM.
*/
- ups = pci_upstream_bridge(dev);
- if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM &&
- ups && ups->ptm_enabled) {
- dev->ptm_granularity = ups->ptm_granularity;
- dev->ptm_enabled = 1;
- return;
+ if (!dev->ptm_root) {
+ ups = pci_upstream_ptm(dev);
+ if (!ups || !ups->ptm_enabled)
+ return -EINVAL;
}
- pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
- if (!pos)
- return;
-
- pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_PTM, sizeof(u16));
-
- pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap);
- local_clock = (cap & PCI_PTM_GRANULARITY_MASK) >> 8;
-
- /*
- * There's no point in enabling PTM unless it's enabled in the
- * upstream device or this device can be a PTM Root itself. Per
- * the spec recommendation (PCIe r3.1, sec 7.32.3), select the
- * furthest upstream Time Source as the PTM Root.
- */
- if (ups && ups->ptm_enabled) {
- ctrl = PCI_PTM_CTRL_ENABLE;
- if (ups->ptm_granularity == 0)
- dev->ptm_granularity = 0;
- else if (ups->ptm_granularity > local_clock)
- dev->ptm_granularity = ups->ptm_granularity;
- } else {
- if (cap & PCI_PTM_CAP_ROOT) {
- ctrl = PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT;
- dev->ptm_root = 1;
- dev->ptm_granularity = local_clock;
- } else
- return;
- }
+ pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, &ctrl);
+ ctrl |= PCI_PTM_CTRL_ENABLE;
+ ctrl &= ~PCI_PTM_GRANULARITY_MASK;
ctrl |= dev->ptm_granularity << 8;
- pci_write_config_dword(dev, pos + PCI_PTM_CTRL, ctrl);
- dev->ptm_enabled = 1;
+ if (dev->ptm_root)
+ ctrl |= PCI_PTM_CTRL_ROOT;
- pci_ptm_info(dev);
+ pci_write_config_dword(dev, ptm + PCI_PTM_CTRL, ctrl);
+ return 0;
}
+/**
+ * pci_enable_ptm() - Enable Precision Time Measurement
+ * @dev: PCI device
+ * @granularity: pointer to return granularity
+ *
+ * Enable Precision Time Measurement for @dev. If successful and
+ * @granularity is non-NULL, return the Effective Granularity.
+ *
+ * Return: zero if successful, or -EINVAL if @dev lacks a PTM Capability or
+ * is not a PTM Root and lacks an upstream path of PTM-enabled devices.
+ */
int pci_enable_ptm(struct pci_dev *dev, u8 *granularity)
{
- int pos;
- u32 cap, ctrl;
- struct pci_dev *ups;
-
- if (!pci_is_pcie(dev))
- return -EINVAL;
-
- pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
- if (!pos)
- return -EINVAL;
-
- pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap);
- if (!(cap & PCI_PTM_CAP_REQ))
- return -EINVAL;
-
- /*
- * For a PCIe Endpoint, PTM is only useful if the endpoint can
- * issue PTM requests to upstream devices that have PTM enabled.
- *
- * For Root Complex Integrated Endpoints, there is no upstream
- * device, so there must be some implementation-specific way to
- * associate the endpoint with a time source.
- */
- if (pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT) {
- ups = pci_upstream_bridge(dev);
- if (!ups || !ups->ptm_enabled)
- return -EINVAL;
+ int rc;
+ char clock_desc[8];
- dev->ptm_granularity = ups->ptm_granularity;
- } else if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
- dev->ptm_granularity = 0;
- } else
- return -EINVAL;
+ rc = __pci_enable_ptm(dev);
+ if (rc)
+ return rc;
- ctrl = PCI_PTM_CTRL_ENABLE;
- ctrl |= dev->ptm_granularity << 8;
- pci_write_config_dword(dev, pos + PCI_PTM_CTRL, ctrl);
dev->ptm_enabled = 1;
- pci_ptm_info(dev);
-
if (granularity)
*granularity = dev->ptm_granularity;
+
+ switch (dev->ptm_granularity) {
+ case 0:
+ snprintf(clock_desc, sizeof(clock_desc), "unknown");
+ break;
+ case 255:
+ snprintf(clock_desc, sizeof(clock_desc), ">254ns");
+ break;
+ default:
+ snprintf(clock_desc, sizeof(clock_desc), "%uns",
+ dev->ptm_granularity);
+ break;
+ }
+ pci_info(dev, "PTM enabled%s, %s granularity\n",
+ dev->ptm_root ? " (root)" : "", clock_desc);
+
return 0;
}
EXPORT_SYMBOL(pci_enable_ptm);
+static void __pci_disable_ptm(struct pci_dev *dev)
+{
+ u16 ptm = dev->ptm_cap;
+ u32 ctrl;
+
+ if (!ptm)
+ return;
+
+ pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, &ctrl);
+ ctrl &= ~(PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT);
+ pci_write_config_dword(dev, ptm + PCI_PTM_CTRL, ctrl);
+}
+
+/**
+ * pci_disable_ptm() - Disable Precision Time Measurement
+ * @dev: PCI device
+ *
+ * Disable Precision Time Measurement for @dev.
+ */
+void pci_disable_ptm(struct pci_dev *dev)
+{
+ if (dev->ptm_enabled) {
+ __pci_disable_ptm(dev);
+ dev->ptm_enabled = 0;
+ }
+}
+EXPORT_SYMBOL(pci_disable_ptm);
+
+/*
+ * Disable PTM, but preserve dev->ptm_enabled so we silently re-enable it on
+ * resume if necessary.
+ */
+void pci_suspend_ptm(struct pci_dev *dev)
+{
+ if (dev->ptm_enabled)
+ __pci_disable_ptm(dev);
+}
+
+/* If PTM was enabled before suspend, re-enable it when resuming */
+void pci_resume_ptm(struct pci_dev *dev)
+{
+ if (dev->ptm_enabled)
+ __pci_enable_ptm(dev);
+}
+
bool pcie_ptm_enabled(struct pci_dev *dev)
{
if (!dev)
if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
!is_cardbus && !broken) {
- unsigned int cmax;
+ unsigned int cmax, buses;
/*
* Bus already configured by firmware, process it in the
child->bridge_ctl = bctl;
}
- cmax = pci_scan_child_bus(child);
+ buses = subordinate - secondary;
+ cmax = pci_scan_child_bus_extend(child, buses);
if (cmax > subordinate)
pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
subordinate, cmax);
* hotplug bridges too much during the second scan below.
*/
used_buses++;
- if (cmax - max > 1)
- used_buses += cmax - max - 1;
+ if (max - cmax > 1)
+ used_buses += max - cmax - 1;
}
/* Scan bridges that need to be reconfigured */
unsigned int buses = 0;
if (!hotplug_bridges && normal_bridges == 1) {
-
/*
* There is only one bridge on the bus (upstream
* port) so it gets all available buses which it
*/
buses = available_buses;
} else if (dev->is_hotplug_bridge) {
-
/*
* Distribute the extra buses between hotplug
* bridges if any.
/*
* Make sure a hotplug bridge has at least the minimum requested
* number of buses but allow it to grow up to the maximum available
- * bus number of there is room.
+ * bus number if there is room.
*/
if (bus->self && bus->self->is_hotplug_bridge) {
used_buses = max_t(unsigned int, available_buses,
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x56c0, aspm_l1_acceptable_latency);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x56c1, aspm_l1_acceptable_latency);
#endif
+
+#ifdef CONFIG_PCIE_DPC
+/*
+ * Intel Tiger Lake and Alder Lake BIOS has a bug that clears the DPC
+ * RP PIO Log Size of the integrated Thunderbolt PCIe Root Ports.
+ */
+static void dpc_log_size(struct pci_dev *dev)
+{
+ u16 dpc, val;
+
+ dpc = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC);
+ if (!dpc)
+ return;
+
+ pci_read_config_word(dev, dpc + PCI_EXP_DPC_CAP, &val);
+ if (!(val & PCI_EXP_DPC_CAP_RP_EXT))
+ return;
+
+ if (!((val & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8)) {
+ pci_info(dev, "Overriding RP PIO Log Size to 4\n");
+ dev->dpc_rp_log_size = 4;
+ }
+}
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x461f, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x462f, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x463f, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x466e, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a23, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a25, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a27, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a29, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a2b, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a2d, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a2f, dpc_log_size);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a31, dpc_log_size);
+#endif
}
#endif
-/*
- * First try will not touch PCI bridge res.
- * Second and later try will clear small leaf bridge res.
- * Will stop till to the max depth if can not find good one.
- */
-void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
-{
- LIST_HEAD(realloc_head);
- /* List of resources that want additional resources */
- struct list_head *add_list = NULL;
- int tried_times = 0;
- enum release_type rel_type = leaf_only;
- LIST_HEAD(fail_head);
- struct pci_dev_resource *fail_res;
- int pci_try_num = 1;
- enum enable_type enable_local;
-
- /* Don't realloc if asked to do so */
- enable_local = pci_realloc_detect(bus, pci_realloc_enable);
- if (pci_realloc_enabled(enable_local)) {
- int max_depth = pci_bus_get_depth(bus);
-
- pci_try_num = max_depth + 1;
- dev_info(&bus->dev, "max bus depth: %d pci_try_num: %d\n",
- max_depth, pci_try_num);
- }
-
-again:
- /*
- * Last try will use add_list, otherwise will try good to have as must
- * have, so can realloc parent bridge resource
- */
- if (tried_times + 1 == pci_try_num)
- add_list = &realloc_head;
- /*
- * Depth first, calculate sizes and alignments of all subordinate buses.
- */
- __pci_bus_size_bridges(bus, add_list);
-
- /* Depth last, allocate resources and update the hardware. */
- __pci_bus_assign_resources(bus, add_list, &fail_head);
- if (add_list)
- BUG_ON(!list_empty(add_list));
- tried_times++;
-
- /* Any device complain? */
- if (list_empty(&fail_head))
- goto dump;
-
- if (tried_times >= pci_try_num) {
- if (enable_local == undefined)
- dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
- else if (enable_local == auto_enabled)
- dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
-
- free_list(&fail_head);
- goto dump;
- }
-
- dev_info(&bus->dev, "No. %d try to assign unassigned res\n",
- tried_times + 1);
-
- /* Third times and later will not check if it is leaf */
- if ((tried_times + 1) > 2)
- rel_type = whole_subtree;
-
- /*
- * Try to release leaf bridge's resources that doesn't fit resource of
- * child device under that bridge.
- */
- list_for_each_entry(fail_res, &fail_head, list)
- pci_bus_release_bridge_resources(fail_res->dev->bus,
- fail_res->flags & PCI_RES_TYPE_MASK,
- rel_type);
-
- /* Restore size and flags */
- list_for_each_entry(fail_res, &fail_head, list) {
- struct resource *res = fail_res->res;
- int idx;
-
- res->start = fail_res->start;
- res->end = fail_res->end;
- res->flags = fail_res->flags;
-
- if (pci_is_bridge(fail_res->dev)) {
- idx = res - &fail_res->dev->resource[0];
- if (idx >= PCI_BRIDGE_RESOURCES &&
- idx <= PCI_BRIDGE_RESOURCE_END)
- res->flags = 0;
- }
- }
- free_list(&fail_head);
-
- goto again;
-
-dump:
- /* Dump the resource on buses */
- pci_bus_dump_resources(bus);
-}
-
-void __init pci_assign_unassigned_resources(void)
-{
- struct pci_bus *root_bus;
-
- list_for_each_entry(root_bus, &pci_root_buses, node) {
- pci_assign_unassigned_root_bus_resources(root_bus);
-
- /* Make sure the root bridge has a companion ACPI device */
- if (ACPI_HANDLE(root_bus->bridge))
- acpi_ioapic_add(ACPI_HANDLE(root_bus->bridge));
- }
-}
-
static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res,
struct list_head *add_list,
resource_size_t new_size)
}
static void pci_bridge_distribute_available_resources(struct pci_dev *bridge,
- struct list_head *add_list)
+ struct list_head *add_list)
{
struct resource available_io, available_mmio, available_mmio_pref;
available_mmio_pref);
}
+/*
+ * First try will not touch PCI bridge res.
+ * Second and later try will clear small leaf bridge res.
+ * Will stop till to the max depth if can not find good one.
+ */
+void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
+{
+ LIST_HEAD(realloc_head);
+ /* List of resources that want additional resources */
+ struct list_head *add_list = NULL;
+ int tried_times = 0;
+ enum release_type rel_type = leaf_only;
+ LIST_HEAD(fail_head);
+ struct pci_dev_resource *fail_res;
+ int pci_try_num = 1;
+ enum enable_type enable_local;
+
+ /* Don't realloc if asked to do so */
+ enable_local = pci_realloc_detect(bus, pci_realloc_enable);
+ if (pci_realloc_enabled(enable_local)) {
+ int max_depth = pci_bus_get_depth(bus);
+
+ pci_try_num = max_depth + 1;
+ dev_info(&bus->dev, "max bus depth: %d pci_try_num: %d\n",
+ max_depth, pci_try_num);
+ }
+
+again:
+ /*
+ * Last try will use add_list, otherwise will try good to have as must
+ * have, so can realloc parent bridge resource
+ */
+ if (tried_times + 1 == pci_try_num)
+ add_list = &realloc_head;
+ /*
+ * Depth first, calculate sizes and alignments of all subordinate buses.
+ */
+ __pci_bus_size_bridges(bus, add_list);
+
+ /* Depth last, allocate resources and update the hardware. */
+ __pci_bus_assign_resources(bus, add_list, &fail_head);
+ if (add_list)
+ BUG_ON(!list_empty(add_list));
+ tried_times++;
+
+ /* Any device complain? */
+ if (list_empty(&fail_head))
+ goto dump;
+
+ if (tried_times >= pci_try_num) {
+ if (enable_local == undefined)
+ dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n");
+ else if (enable_local == auto_enabled)
+ dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n");
+
+ free_list(&fail_head);
+ goto dump;
+ }
+
+ dev_info(&bus->dev, "No. %d try to assign unassigned res\n",
+ tried_times + 1);
+
+ /* Third times and later will not check if it is leaf */
+ if ((tried_times + 1) > 2)
+ rel_type = whole_subtree;
+
+ /*
+ * Try to release leaf bridge's resources that doesn't fit resource of
+ * child device under that bridge.
+ */
+ list_for_each_entry(fail_res, &fail_head, list)
+ pci_bus_release_bridge_resources(fail_res->dev->bus,
+ fail_res->flags & PCI_RES_TYPE_MASK,
+ rel_type);
+
+ /* Restore size and flags */
+ list_for_each_entry(fail_res, &fail_head, list) {
+ struct resource *res = fail_res->res;
+ int idx;
+
+ res->start = fail_res->start;
+ res->end = fail_res->end;
+ res->flags = fail_res->flags;
+
+ if (pci_is_bridge(fail_res->dev)) {
+ idx = res - &fail_res->dev->resource[0];
+ if (idx >= PCI_BRIDGE_RESOURCES &&
+ idx <= PCI_BRIDGE_RESOURCE_END)
+ res->flags = 0;
+ }
+ }
+ free_list(&fail_head);
+
+ goto again;
+
+dump:
+ /* Dump the resource on buses */
+ pci_bus_dump_resources(bus);
+}
+
+void __init pci_assign_unassigned_resources(void)
+{
+ struct pci_bus *root_bus;
+
+ list_for_each_entry(root_bus, &pci_root_buses, node) {
+ pci_assign_unassigned_root_bus_resources(root_bus);
+
+ /* Make sure the root bridge has a companion ACPI device */
+ if (ACPI_HANDLE(root_bus->bridge))
+ acpi_ioapic_add(ACPI_HANDLE(root_bus->bridge));
+ }
+}
+
void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
{
struct pci_bus *parent = bridge->subordinate;
root = pci_find_parent_resource(dev, res);
if (!root) {
+ /*
+ * If dev is behind a bridge, accesses will only reach it
+ * if res is inside the relevant bridge window.
+ */
+ if (pci_upstream_bridge(dev))
+ return -ENXIO;
+
+ /*
+ * On the root bus, assume the host bridge will forward
+ * everything.
+ */
if (res->flags & IORESOURCE_IO)
root = &ioport_resource;
else
int err;
struct pci_bus *b;
-#ifndef CONFIG_PCI_DOMAINS
- if (domain != 0) {
- dev_err(&pdev->xdev->dev,
- "PCI Root in non-zero PCI Domain! domain=%d\n", domain);
- dev_err(&pdev->xdev->dev,
- "Please compile with CONFIG_PCI_DOMAINS\n");
- return -EINVAL;
- }
-#endif
-
- dev_info(&pdev->xdev->dev, "Rescanning PCI Frontend Bus %04x:%02x\n",
- domain, bus);
-
b = pci_find_bus(domain, bus);
if (!b)
/* If the bus is unknown, create it. */
return pcifront_scan_root(pdev, domain, bus);
+ dev_info(&pdev->xdev->dev, "Rescanning PCI Frontend Bus %04x:%02x\n",
+ domain, bus);
+
err = pcifront_scan_bus(pdev, domain, bus, b);
/* Claim resources before going "live" with our devices */
return err;
}
-static int pcifront_try_connect(struct pcifront_device *pdev)
+static void pcifront_connect(struct pcifront_device *pdev)
{
- int err = -EFAULT;
+ int err;
int i, num_roots, len;
char str[64];
unsigned int domain, bus;
-
- /* Only connect once */
- if (xenbus_read_driver_state(pdev->xdev->nodename) !=
- XenbusStateInitialised)
- goto out;
-
- err = pcifront_connect_and_init_dma(pdev);
- if (err && err != -EEXIST) {
- xenbus_dev_fatal(pdev->xdev, err,
- "Error setting up PCI Frontend");
- goto out;
- }
-
err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend,
"root_num", "%d", &num_roots);
if (err == -ENOENT) {
xenbus_dev_error(pdev->xdev, err,
"No PCI Roots found, trying 0000:00");
- err = pcifront_scan_root(pdev, 0, 0);
+ err = pcifront_rescan_root(pdev, 0, 0);
if (err) {
xenbus_dev_fatal(pdev->xdev, err,
"Error scanning PCI root 0000:00");
- goto out;
+ return;
}
num_roots = 0;
} else if (err != 1) {
- if (err == 0)
- err = -EINVAL;
- xenbus_dev_fatal(pdev->xdev, err,
+ xenbus_dev_fatal(pdev->xdev, err >= 0 ? -EINVAL : err,
"Error reading number of PCI roots");
- goto out;
+ return;
}
for (i = 0; i < num_roots; i++) {
len = snprintf(str, sizeof(str), "root-%d", i);
- if (unlikely(len >= (sizeof(str) - 1))) {
- err = -ENOMEM;
- goto out;
- }
+ if (unlikely(len >= (sizeof(str) - 1)))
+ return;
err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
"%x:%x", &domain, &bus);
if (err != 2) {
- if (err >= 0)
- err = -EINVAL;
- xenbus_dev_fatal(pdev->xdev, err,
+ xenbus_dev_fatal(pdev->xdev, err >= 0 ? -EINVAL : err,
"Error reading PCI root %d", i);
- goto out;
+ return;
}
- err = pcifront_scan_root(pdev, domain, bus);
+ err = pcifront_rescan_root(pdev, domain, bus);
if (err) {
xenbus_dev_fatal(pdev->xdev, err,
"Error scanning PCI root %04x:%02x",
domain, bus);
- goto out;
+ return;
}
}
- err = xenbus_switch_state(pdev->xdev, XenbusStateConnected);
+ xenbus_switch_state(pdev->xdev, XenbusStateConnected);
+}
-out:
- return err;
+static void pcifront_try_connect(struct pcifront_device *pdev)
+{
+ int err;
+
+ /* Only connect once */
+ if (xenbus_read_driver_state(pdev->xdev->nodename) !=
+ XenbusStateInitialised)
+ return;
+
+ err = pcifront_connect_and_init_dma(pdev);
+ if (err && err != -EEXIST) {
+ xenbus_dev_fatal(pdev->xdev, err,
+ "Error setting up PCI Frontend");
+ return;
+ }
+
+ pcifront_connect(pdev);
}
static int pcifront_try_disconnect(struct pcifront_device *pdev)
return err;
}
-static int pcifront_attach_devices(struct pcifront_device *pdev)
+static void pcifront_attach_devices(struct pcifront_device *pdev)
{
- int err = -EFAULT;
- int i, num_roots, len;
- unsigned int domain, bus;
- char str[64];
-
- if (xenbus_read_driver_state(pdev->xdev->nodename) !=
+ if (xenbus_read_driver_state(pdev->xdev->nodename) ==
XenbusStateReconfiguring)
- goto out;
-
- err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend,
- "root_num", "%d", &num_roots);
- if (err == -ENOENT) {
- xenbus_dev_error(pdev->xdev, err,
- "No PCI Roots found, trying 0000:00");
- err = pcifront_rescan_root(pdev, 0, 0);
- if (err) {
- xenbus_dev_fatal(pdev->xdev, err,
- "Error scanning PCI root 0000:00");
- goto out;
- }
- num_roots = 0;
- } else if (err != 1) {
- if (err == 0)
- err = -EINVAL;
- xenbus_dev_fatal(pdev->xdev, err,
- "Error reading number of PCI roots");
- goto out;
- }
-
- for (i = 0; i < num_roots; i++) {
- len = snprintf(str, sizeof(str), "root-%d", i);
- if (unlikely(len >= (sizeof(str) - 1))) {
- err = -ENOMEM;
- goto out;
- }
-
- err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
- "%x:%x", &domain, &bus);
- if (err != 2) {
- if (err >= 0)
- err = -EINVAL;
- xenbus_dev_fatal(pdev->xdev, err,
- "Error reading PCI root %d", i);
- goto out;
- }
-
- err = pcifront_rescan_root(pdev, domain, bus);
- if (err) {
- xenbus_dev_fatal(pdev->xdev, err,
- "Error scanning PCI root %04x:%02x",
- domain, bus);
- goto out;
- }
- }
-
- xenbus_switch_state(pdev->xdev, XenbusStateConnected);
-
-out:
- return err;
+ pcifront_connect(pdev);
}
static int pcifront_detach_devices(struct pcifront_device *pdev)
{
int err = 0;
int i, num_devs;
+ enum xenbus_state state;
unsigned int domain, bus, slot, func;
struct pci_dev *pci_dev;
char str[64];
- if (xenbus_read_driver_state(pdev->xdev->nodename) !=
- XenbusStateConnected)
+ state = xenbus_read_driver_state(pdev->xdev->nodename);
+ if (state == XenbusStateInitialised) {
+ dev_dbg(&pdev->xdev->dev, "Handle skipped connect.\n");
+ /* We missed Connected and need to initialize. */
+ err = pcifront_connect_and_init_dma(pdev);
+ if (err && err != -EEXIST) {
+ xenbus_dev_fatal(pdev->xdev, err,
+ "Error setting up PCI Frontend");
+ goto out;
+ }
+
+ goto out_switch_state;
+ } else if (state != XenbusStateConnected) {
goto out;
+ }
err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, "num_devs", "%d",
&num_devs);
domain, bus, slot, func);
}
+ out_switch_state:
err = xenbus_switch_state(pdev->xdev, XenbusStateReconfiguring);
out:
config ALIBABA_UNCORE_DRW_PMU
tristate "Alibaba T-Head Yitian 710 DDR Sub-system Driveway PMU driver"
- depends on ARM64 || COMPILE_TEST
+ depends on (ARM64 && ACPI) || COMPILE_TEST
help
Support for Driveway PMU events monitoring on Yitian 710 DDR
Sub-system.
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
drw_pmu->cfg_base = devm_ioremap_resource(&pdev->dev, res);
- if (!drw_pmu->cfg_base)
- return -ENOMEM;
+ if (IS_ERR(drw_pmu->cfg_base))
+ return PTR_ERR(drw_pmu->cfg_base);
name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
(u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node);
struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
- /* Enable the access for TIME csr only from the user mode now */
- csr_write(CSR_SCOUNTEREN, 0x2);
+ /*
+ * Enable the access for CYCLE, TIME, and INSTRET CSRs from userspace,
+ * as is necessary to maintain uABI compatibility.
+ */
+ csr_write(CSR_SCOUNTEREN, 0x7);
/* Stop all the counters so that they can be enabled from perf */
pmu_sbi_stop_all(pmu);
bool clkreq_unused;
};
-static int imx8_pcie_phy_init(struct phy *phy)
+static int imx8_pcie_phy_power_on(struct phy *phy)
{
int ret;
u32 val, pad_mode;
return ret;
}
-static int imx8_pcie_phy_power_on(struct phy *phy)
+static int imx8_pcie_phy_init(struct phy *phy)
{
struct imx8_pcie_phy *imx8_phy = phy_get_drvdata(phy);
return clk_prepare_enable(imx8_phy->clk);
}
-static int imx8_pcie_phy_power_off(struct phy *phy)
+static int imx8_pcie_phy_exit(struct phy *phy)
{
struct imx8_pcie_phy *imx8_phy = phy_get_drvdata(phy);
static const struct phy_ops imx8_pcie_phy_ops = {
.init = imx8_pcie_phy_init,
+ .exit = imx8_pcie_phy_exit,
.power_on = imx8_pcie_phy_power_on,
- .power_off = imx8_pcie_phy_power_off,
.owner = THIS_MODULE,
};
help
Pinctrl driver for Bitmain BM1880 SoC.
+config PINCTRL_CY8C95X0
+ tristate "Cypress CY8C95X0 I2C pinctrl and GPIO driver"
+ depends on I2C
+ select GPIOLIB
+ select GPIOLIB_IRQCHIP
+ select PINMUX
+ select PINCONF
+ select GENERIC_PINCONF
+ select REGMAP_I2C
+ help
+ Support for 20/40/60 pin Cypress Cy8C95x0 pinctrl/gpio I2C expander.
+ This driver can also be built as a module. If so, the module will be
+ called pinctrl-cy8c95x0.
+
config PINCTRL_DA850_PUPD
tristate "TI DA850/OMAP-L138/AM18XX pull-up and pull-down groups"
depends on OF && (ARCH_DAVINCI_DA850 || COMPILE_TEST)
select GENERIC_PINMUX_FUNCTIONS
select OF_GPIO
select REGMAP_MMIO
+ help
+ Support for the internal GPIO interfaces on Microsemi Ocelot and
+ Jaguar2 SoCs.
+
+ If conpiled as a module, the module name will be pinctrl-ocelot.
config PINCTRL_OXNAS
bool
select PINCONF
select GPIOLIB_IRQCHIP
-config PINCTRL_STARFIVE
- tristate "Pinctrl and GPIO driver for the StarFive JH7100 SoC"
- depends on SOC_STARFIVE || COMPILE_TEST
- depends on OF
- default SOC_STARFIVE
- select GENERIC_PINCTRL_GROUPS
- select GENERIC_PINMUX_FUNCTIONS
- select GENERIC_PINCONF
- select GPIOLIB
- select GPIOLIB_IRQCHIP
- select OF_GPIO
- help
- Say yes here to support pin control on the StarFive JH7100 SoC.
- This also provides an interface to the GPIO pins not used by other
- peripherals supporting inputs, outputs, configuring pull-up/pull-down
- and interrupts on input changes.
-
config PINCTRL_STMFX
tristate "STMicroelectronics STMFX GPIO expander pinctrl driver"
depends on I2C
source "drivers/pinctrl/samsung/Kconfig"
source "drivers/pinctrl/spear/Kconfig"
source "drivers/pinctrl/sprd/Kconfig"
+source "drivers/pinctrl/starfive/Kconfig"
source "drivers/pinctrl/stm32/Kconfig"
source "drivers/pinctrl/sunplus/Kconfig"
source "drivers/pinctrl/sunxi/Kconfig"
obj-$(CONFIG_PINCTRL_AT91PIO4) += pinctrl-at91-pio4.o
obj-$(CONFIG_PINCTRL_AXP209) += pinctrl-axp209.o
obj-$(CONFIG_PINCTRL_BM1880) += pinctrl-bm1880.o
+obj-$(CONFIG_PINCTRL_CY8C95X0) += pinctrl-cy8c95x0.o
obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o
obj-$(CONFIG_PINCTRL_DA9062) += pinctrl-da9062.o
obj-$(CONFIG_PINCTRL_DIGICOLOR) += pinctrl-digicolor.o
obj-$(CONFIG_PINCTRL_ROCKCHIP) += pinctrl-rockchip.o
obj-$(CONFIG_PINCTRL_SINGLE) += pinctrl-single.o
obj-$(CONFIG_PINCTRL_ST) += pinctrl-st.o
-obj-$(CONFIG_PINCTRL_STARFIVE) += pinctrl-starfive.o
obj-$(CONFIG_PINCTRL_STMFX) += pinctrl-stmfx.o
obj-$(CONFIG_PINCTRL_SX150X) += pinctrl-sx150x.o
obj-$(CONFIG_PINCTRL_TB10X) += pinctrl-tb10x.o
obj-$(CONFIG_PINCTRL_SAMSUNG) += samsung/
obj-$(CONFIG_PINCTRL_SPEAR) += spear/
obj-y += sprd/
+obj-$(CONFIG_SOC_STARFIVE) += starfive/
obj-$(CONFIG_PINCTRL_STM32) += stm32/
obj-y += sunplus/
obj-$(CONFIG_PINCTRL_SUNXI) += sunxi/
static int aspeed_sig_expr_disable(struct aspeed_pinmux_data *ctx,
const struct aspeed_sig_expr *expr)
{
- int ret;
-
pr_debug("Disabling signal %s for %s\n", expr->signal,
expr->function);
- ret = aspeed_sig_expr_eval(ctx, expr, true);
- if (ret < 0)
- return ret;
-
- if (ret)
- return aspeed_sig_expr_set(ctx, expr, false);
-
- return 0;
+ return aspeed_sig_expr_set(ctx, expr, false);
}
/**
#define BCM6318_PAD_REG 0x54
#define BCM6328_PAD_MASK GENMASK(3, 0)
-struct bcm6318_pingroup {
- const char *name;
- const unsigned * const pins;
- const unsigned num_pins;
-};
-
struct bcm6318_function {
const char *name;
const char * const *groups;
static unsigned gpio48_pins[] = { 48 };
static unsigned gpio49_pins[] = { 49 };
-#define BCM6318_GROUP(n) \
- { \
- .name = #n, \
- .pins = n##_pins, \
- .num_pins = ARRAY_SIZE(n##_pins), \
- }
-
-static struct bcm6318_pingroup bcm6318_groups[] = {
- BCM6318_GROUP(gpio0),
- BCM6318_GROUP(gpio1),
- BCM6318_GROUP(gpio2),
- BCM6318_GROUP(gpio3),
- BCM6318_GROUP(gpio4),
- BCM6318_GROUP(gpio5),
- BCM6318_GROUP(gpio6),
- BCM6318_GROUP(gpio7),
- BCM6318_GROUP(gpio8),
- BCM6318_GROUP(gpio9),
- BCM6318_GROUP(gpio10),
- BCM6318_GROUP(gpio11),
- BCM6318_GROUP(gpio12),
- BCM6318_GROUP(gpio13),
- BCM6318_GROUP(gpio14),
- BCM6318_GROUP(gpio15),
- BCM6318_GROUP(gpio16),
- BCM6318_GROUP(gpio17),
- BCM6318_GROUP(gpio18),
- BCM6318_GROUP(gpio19),
- BCM6318_GROUP(gpio20),
- BCM6318_GROUP(gpio21),
- BCM6318_GROUP(gpio22),
- BCM6318_GROUP(gpio23),
- BCM6318_GROUP(gpio24),
- BCM6318_GROUP(gpio25),
- BCM6318_GROUP(gpio26),
- BCM6318_GROUP(gpio27),
- BCM6318_GROUP(gpio28),
- BCM6318_GROUP(gpio29),
- BCM6318_GROUP(gpio30),
- BCM6318_GROUP(gpio31),
- BCM6318_GROUP(gpio32),
- BCM6318_GROUP(gpio33),
- BCM6318_GROUP(gpio34),
- BCM6318_GROUP(gpio35),
- BCM6318_GROUP(gpio36),
- BCM6318_GROUP(gpio37),
- BCM6318_GROUP(gpio38),
- BCM6318_GROUP(gpio39),
- BCM6318_GROUP(gpio40),
- BCM6318_GROUP(gpio41),
- BCM6318_GROUP(gpio42),
- BCM6318_GROUP(gpio43),
- BCM6318_GROUP(gpio44),
- BCM6318_GROUP(gpio45),
- BCM6318_GROUP(gpio46),
- BCM6318_GROUP(gpio47),
- BCM6318_GROUP(gpio48),
- BCM6318_GROUP(gpio49),
+static struct pingroup bcm6318_groups[] = {
+ BCM_PIN_GROUP(gpio0),
+ BCM_PIN_GROUP(gpio1),
+ BCM_PIN_GROUP(gpio2),
+ BCM_PIN_GROUP(gpio3),
+ BCM_PIN_GROUP(gpio4),
+ BCM_PIN_GROUP(gpio5),
+ BCM_PIN_GROUP(gpio6),
+ BCM_PIN_GROUP(gpio7),
+ BCM_PIN_GROUP(gpio8),
+ BCM_PIN_GROUP(gpio9),
+ BCM_PIN_GROUP(gpio10),
+ BCM_PIN_GROUP(gpio11),
+ BCM_PIN_GROUP(gpio12),
+ BCM_PIN_GROUP(gpio13),
+ BCM_PIN_GROUP(gpio14),
+ BCM_PIN_GROUP(gpio15),
+ BCM_PIN_GROUP(gpio16),
+ BCM_PIN_GROUP(gpio17),
+ BCM_PIN_GROUP(gpio18),
+ BCM_PIN_GROUP(gpio19),
+ BCM_PIN_GROUP(gpio20),
+ BCM_PIN_GROUP(gpio21),
+ BCM_PIN_GROUP(gpio22),
+ BCM_PIN_GROUP(gpio23),
+ BCM_PIN_GROUP(gpio24),
+ BCM_PIN_GROUP(gpio25),
+ BCM_PIN_GROUP(gpio26),
+ BCM_PIN_GROUP(gpio27),
+ BCM_PIN_GROUP(gpio28),
+ BCM_PIN_GROUP(gpio29),
+ BCM_PIN_GROUP(gpio30),
+ BCM_PIN_GROUP(gpio31),
+ BCM_PIN_GROUP(gpio32),
+ BCM_PIN_GROUP(gpio33),
+ BCM_PIN_GROUP(gpio34),
+ BCM_PIN_GROUP(gpio35),
+ BCM_PIN_GROUP(gpio36),
+ BCM_PIN_GROUP(gpio37),
+ BCM_PIN_GROUP(gpio38),
+ BCM_PIN_GROUP(gpio39),
+ BCM_PIN_GROUP(gpio40),
+ BCM_PIN_GROUP(gpio41),
+ BCM_PIN_GROUP(gpio42),
+ BCM_PIN_GROUP(gpio43),
+ BCM_PIN_GROUP(gpio44),
+ BCM_PIN_GROUP(gpio45),
+ BCM_PIN_GROUP(gpio46),
+ BCM_PIN_GROUP(gpio47),
+ BCM_PIN_GROUP(gpio48),
+ BCM_PIN_GROUP(gpio49),
};
/* GPIO_MODE */
static int bcm6318_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
unsigned group, const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
*pins = bcm6318_groups[group].pins;
- *num_pins = bcm6318_groups[group].num_pins;
+ *npins = bcm6318_groups[group].npins;
return 0;
}
unsigned selector, unsigned group)
{
struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
- const struct bcm6318_pingroup *pg = &bcm6318_groups[group];
+ const struct pingroup *pg = &bcm6318_groups[group];
const struct bcm6318_function *f = &bcm6318_funcs[selector];
bcm6318_rmw_mux(pc, pg->pins[0], f->mode_val, f->mux_val);
BCM63268_BASEMODE,
};
-struct bcm63268_pingroup {
- const char *name;
- const unsigned * const pins;
- const unsigned num_pins;
-};
-
struct bcm63268_function {
const char *name;
const char * const *groups;
static unsigned vdsl_phy2_grp_pins[] = { 24, 25 };
static unsigned vdsl_phy3_grp_pins[] = { 26, 27 };
-#define BCM63268_GROUP(n) \
- { \
- .name = #n, \
- .pins = n##_pins, \
- .num_pins = ARRAY_SIZE(n##_pins), \
- }
-
-static struct bcm63268_pingroup bcm63268_groups[] = {
- BCM63268_GROUP(gpio0),
- BCM63268_GROUP(gpio1),
- BCM63268_GROUP(gpio2),
- BCM63268_GROUP(gpio3),
- BCM63268_GROUP(gpio4),
- BCM63268_GROUP(gpio5),
- BCM63268_GROUP(gpio6),
- BCM63268_GROUP(gpio7),
- BCM63268_GROUP(gpio8),
- BCM63268_GROUP(gpio9),
- BCM63268_GROUP(gpio10),
- BCM63268_GROUP(gpio11),
- BCM63268_GROUP(gpio12),
- BCM63268_GROUP(gpio13),
- BCM63268_GROUP(gpio14),
- BCM63268_GROUP(gpio15),
- BCM63268_GROUP(gpio16),
- BCM63268_GROUP(gpio17),
- BCM63268_GROUP(gpio18),
- BCM63268_GROUP(gpio19),
- BCM63268_GROUP(gpio20),
- BCM63268_GROUP(gpio21),
- BCM63268_GROUP(gpio22),
- BCM63268_GROUP(gpio23),
- BCM63268_GROUP(gpio24),
- BCM63268_GROUP(gpio25),
- BCM63268_GROUP(gpio26),
- BCM63268_GROUP(gpio27),
- BCM63268_GROUP(gpio28),
- BCM63268_GROUP(gpio29),
- BCM63268_GROUP(gpio30),
- BCM63268_GROUP(gpio31),
- BCM63268_GROUP(gpio32),
- BCM63268_GROUP(gpio33),
- BCM63268_GROUP(gpio34),
- BCM63268_GROUP(gpio35),
- BCM63268_GROUP(gpio36),
- BCM63268_GROUP(gpio37),
- BCM63268_GROUP(gpio38),
- BCM63268_GROUP(gpio39),
- BCM63268_GROUP(gpio40),
- BCM63268_GROUP(gpio41),
- BCM63268_GROUP(gpio42),
- BCM63268_GROUP(gpio43),
- BCM63268_GROUP(gpio44),
- BCM63268_GROUP(gpio45),
- BCM63268_GROUP(gpio46),
- BCM63268_GROUP(gpio47),
- BCM63268_GROUP(gpio48),
- BCM63268_GROUP(gpio49),
- BCM63268_GROUP(gpio50),
- BCM63268_GROUP(gpio51),
+static struct pingroup bcm63268_groups[] = {
+ BCM_PIN_GROUP(gpio0),
+ BCM_PIN_GROUP(gpio1),
+ BCM_PIN_GROUP(gpio2),
+ BCM_PIN_GROUP(gpio3),
+ BCM_PIN_GROUP(gpio4),
+ BCM_PIN_GROUP(gpio5),
+ BCM_PIN_GROUP(gpio6),
+ BCM_PIN_GROUP(gpio7),
+ BCM_PIN_GROUP(gpio8),
+ BCM_PIN_GROUP(gpio9),
+ BCM_PIN_GROUP(gpio10),
+ BCM_PIN_GROUP(gpio11),
+ BCM_PIN_GROUP(gpio12),
+ BCM_PIN_GROUP(gpio13),
+ BCM_PIN_GROUP(gpio14),
+ BCM_PIN_GROUP(gpio15),
+ BCM_PIN_GROUP(gpio16),
+ BCM_PIN_GROUP(gpio17),
+ BCM_PIN_GROUP(gpio18),
+ BCM_PIN_GROUP(gpio19),
+ BCM_PIN_GROUP(gpio20),
+ BCM_PIN_GROUP(gpio21),
+ BCM_PIN_GROUP(gpio22),
+ BCM_PIN_GROUP(gpio23),
+ BCM_PIN_GROUP(gpio24),
+ BCM_PIN_GROUP(gpio25),
+ BCM_PIN_GROUP(gpio26),
+ BCM_PIN_GROUP(gpio27),
+ BCM_PIN_GROUP(gpio28),
+ BCM_PIN_GROUP(gpio29),
+ BCM_PIN_GROUP(gpio30),
+ BCM_PIN_GROUP(gpio31),
+ BCM_PIN_GROUP(gpio32),
+ BCM_PIN_GROUP(gpio33),
+ BCM_PIN_GROUP(gpio34),
+ BCM_PIN_GROUP(gpio35),
+ BCM_PIN_GROUP(gpio36),
+ BCM_PIN_GROUP(gpio37),
+ BCM_PIN_GROUP(gpio38),
+ BCM_PIN_GROUP(gpio39),
+ BCM_PIN_GROUP(gpio40),
+ BCM_PIN_GROUP(gpio41),
+ BCM_PIN_GROUP(gpio42),
+ BCM_PIN_GROUP(gpio43),
+ BCM_PIN_GROUP(gpio44),
+ BCM_PIN_GROUP(gpio45),
+ BCM_PIN_GROUP(gpio46),
+ BCM_PIN_GROUP(gpio47),
+ BCM_PIN_GROUP(gpio48),
+ BCM_PIN_GROUP(gpio49),
+ BCM_PIN_GROUP(gpio50),
+ BCM_PIN_GROUP(gpio51),
/* multi pin groups */
- BCM63268_GROUP(nand_grp),
- BCM63268_GROUP(dectpd_grp),
- BCM63268_GROUP(vdsl_phy0_grp),
- BCM63268_GROUP(vdsl_phy1_grp),
- BCM63268_GROUP(vdsl_phy2_grp),
- BCM63268_GROUP(vdsl_phy3_grp),
+ BCM_PIN_GROUP(nand_grp),
+ BCM_PIN_GROUP(dectpd_grp),
+ BCM_PIN_GROUP(vdsl_phy0_grp),
+ BCM_PIN_GROUP(vdsl_phy1_grp),
+ BCM_PIN_GROUP(vdsl_phy2_grp),
+ BCM_PIN_GROUP(vdsl_phy3_grp),
};
static const char * const led_groups[] = {
static int bcm63268_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
unsigned group,
const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
*pins = bcm63268_groups[group].pins;
- *num_pins = bcm63268_groups[group].num_pins;
+ *npins = bcm63268_groups[group].npins;
return 0;
}
unsigned selector, unsigned group)
{
struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
- const struct bcm63268_pingroup *pg = &bcm63268_groups[group];
+ const struct pingroup *pg = &bcm63268_groups[group];
const struct bcm63268_function *f = &bcm63268_funcs[selector];
unsigned i;
unsigned int reg;
unsigned int val, mask;
- for (i = 0; i < pg->num_pins; i++)
+ for (i = 0; i < pg->npins; i++)
bcm63268_set_gpio(pc, pg->pins[i]);
switch (f->reg) {
#define BCM6328_MUX_OTHER_REG 0x24
#define BCM6328_MUX_MASK GENMASK(1, 0)
-struct bcm6328_pingroup {
- const char *name;
- const unsigned * const pins;
- const unsigned num_pins;
-};
-
struct bcm6328_function {
const char *name;
const char * const *groups;
static unsigned hsspi_cs1_pins[] = { 36 };
static unsigned usb_port1_pins[] = { 38 };
-#define BCM6328_GROUP(n) \
- { \
- .name = #n, \
- .pins = n##_pins, \
- .num_pins = ARRAY_SIZE(n##_pins), \
- }
-
-static struct bcm6328_pingroup bcm6328_groups[] = {
- BCM6328_GROUP(gpio0),
- BCM6328_GROUP(gpio1),
- BCM6328_GROUP(gpio2),
- BCM6328_GROUP(gpio3),
- BCM6328_GROUP(gpio4),
- BCM6328_GROUP(gpio5),
- BCM6328_GROUP(gpio6),
- BCM6328_GROUP(gpio7),
- BCM6328_GROUP(gpio8),
- BCM6328_GROUP(gpio9),
- BCM6328_GROUP(gpio10),
- BCM6328_GROUP(gpio11),
- BCM6328_GROUP(gpio12),
- BCM6328_GROUP(gpio13),
- BCM6328_GROUP(gpio14),
- BCM6328_GROUP(gpio15),
- BCM6328_GROUP(gpio16),
- BCM6328_GROUP(gpio17),
- BCM6328_GROUP(gpio18),
- BCM6328_GROUP(gpio19),
- BCM6328_GROUP(gpio20),
- BCM6328_GROUP(gpio21),
- BCM6328_GROUP(gpio22),
- BCM6328_GROUP(gpio23),
- BCM6328_GROUP(gpio24),
- BCM6328_GROUP(gpio25),
- BCM6328_GROUP(gpio26),
- BCM6328_GROUP(gpio27),
- BCM6328_GROUP(gpio28),
- BCM6328_GROUP(gpio29),
- BCM6328_GROUP(gpio30),
- BCM6328_GROUP(gpio31),
-
- BCM6328_GROUP(hsspi_cs1),
- BCM6328_GROUP(usb_port1),
+static struct pingroup bcm6328_groups[] = {
+ BCM_PIN_GROUP(gpio0),
+ BCM_PIN_GROUP(gpio1),
+ BCM_PIN_GROUP(gpio2),
+ BCM_PIN_GROUP(gpio3),
+ BCM_PIN_GROUP(gpio4),
+ BCM_PIN_GROUP(gpio5),
+ BCM_PIN_GROUP(gpio6),
+ BCM_PIN_GROUP(gpio7),
+ BCM_PIN_GROUP(gpio8),
+ BCM_PIN_GROUP(gpio9),
+ BCM_PIN_GROUP(gpio10),
+ BCM_PIN_GROUP(gpio11),
+ BCM_PIN_GROUP(gpio12),
+ BCM_PIN_GROUP(gpio13),
+ BCM_PIN_GROUP(gpio14),
+ BCM_PIN_GROUP(gpio15),
+ BCM_PIN_GROUP(gpio16),
+ BCM_PIN_GROUP(gpio17),
+ BCM_PIN_GROUP(gpio18),
+ BCM_PIN_GROUP(gpio19),
+ BCM_PIN_GROUP(gpio20),
+ BCM_PIN_GROUP(gpio21),
+ BCM_PIN_GROUP(gpio22),
+ BCM_PIN_GROUP(gpio23),
+ BCM_PIN_GROUP(gpio24),
+ BCM_PIN_GROUP(gpio25),
+ BCM_PIN_GROUP(gpio26),
+ BCM_PIN_GROUP(gpio27),
+ BCM_PIN_GROUP(gpio28),
+ BCM_PIN_GROUP(gpio29),
+ BCM_PIN_GROUP(gpio30),
+ BCM_PIN_GROUP(gpio31),
+
+ BCM_PIN_GROUP(hsspi_cs1),
+ BCM_PIN_GROUP(usb_port1),
};
/* GPIO_MODE */
static int bcm6328_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
unsigned group, const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
*pins = bcm6328_groups[group].pins;
- *num_pins = bcm6328_groups[group].num_pins;
+ *npins = bcm6328_groups[group].npins;
return 0;
}
unsigned selector, unsigned group)
{
struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
- const struct bcm6328_pingroup *pg = &bcm6328_groups[group];
+ const struct pingroup *pg = &bcm6328_groups[group];
const struct bcm6328_function *f = &bcm6328_funcs[selector];
bcm6328_rmw_mux(pc, pg->pins[0], f->mode_val, f->mux_val);
#define BCM6358_MODE_MUX_SYS_IRQ BIT(15)
struct bcm6358_pingroup {
- const char *name;
- const unsigned * const pins;
- const unsigned num_pins;
+ struct pingroup grp;
const uint16_t mode_val;
#define BCM6358_GPIO_MUX_GROUP(n, bit, dir) \
{ \
- .name = #n, \
- .pins = n##_pins, \
- .num_pins = ARRAY_SIZE(n##_pins), \
+ .grp = BCM_PIN_GROUP(n), \
.mode_val = BCM6358_MODE_MUX_##bit, \
.direction = dir, \
}
static const char *bcm6358_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
unsigned group)
{
- return bcm6358_groups[group].name;
+ return bcm6358_groups[group].grp.name;
}
static int bcm6358_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
unsigned group, const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
- *pins = bcm6358_groups[group].pins;
- *num_pins = bcm6358_groups[group].num_pins;
+ *pins = bcm6358_groups[group].grp.pins;
+ *npins = bcm6358_groups[group].grp.npins;
return 0;
}
unsigned int mask = val;
unsigned pin;
- for (pin = 0; pin < pg->num_pins; pin++)
+ for (pin = 0; pin < pg->grp.npins; pin++)
mask |= (unsigned long)bcm6358_pins[pin].drv_data;
regmap_field_update_bits(priv->overlays, mask, val);
- for (pin = 0; pin < pg->num_pins; pin++) {
+ for (pin = 0; pin < pg->grp.npins; pin++) {
struct pinctrl_gpio_range *range;
unsigned int hw_gpio = bcm6358_pins[pin].number;
BCM6362_BASEMODE,
};
-struct bcm6362_pingroup {
- const char *name;
- const unsigned * const pins;
- const unsigned num_pins;
-};
-
struct bcm6362_function {
const char *name;
const char * const *groups;
18, 19, 20, 21, 22, 23, 27,
};
-#define BCM6362_GROUP(n) \
- { \
- .name = #n, \
- .pins = n##_pins, \
- .num_pins = ARRAY_SIZE(n##_pins), \
- }
-
-static struct bcm6362_pingroup bcm6362_groups[] = {
- BCM6362_GROUP(gpio0),
- BCM6362_GROUP(gpio1),
- BCM6362_GROUP(gpio2),
- BCM6362_GROUP(gpio3),
- BCM6362_GROUP(gpio4),
- BCM6362_GROUP(gpio5),
- BCM6362_GROUP(gpio6),
- BCM6362_GROUP(gpio7),
- BCM6362_GROUP(gpio8),
- BCM6362_GROUP(gpio9),
- BCM6362_GROUP(gpio10),
- BCM6362_GROUP(gpio11),
- BCM6362_GROUP(gpio12),
- BCM6362_GROUP(gpio13),
- BCM6362_GROUP(gpio14),
- BCM6362_GROUP(gpio15),
- BCM6362_GROUP(gpio16),
- BCM6362_GROUP(gpio17),
- BCM6362_GROUP(gpio18),
- BCM6362_GROUP(gpio19),
- BCM6362_GROUP(gpio20),
- BCM6362_GROUP(gpio21),
- BCM6362_GROUP(gpio22),
- BCM6362_GROUP(gpio23),
- BCM6362_GROUP(gpio24),
- BCM6362_GROUP(gpio25),
- BCM6362_GROUP(gpio26),
- BCM6362_GROUP(gpio27),
- BCM6362_GROUP(gpio28),
- BCM6362_GROUP(gpio29),
- BCM6362_GROUP(gpio30),
- BCM6362_GROUP(gpio31),
- BCM6362_GROUP(gpio32),
- BCM6362_GROUP(gpio33),
- BCM6362_GROUP(gpio34),
- BCM6362_GROUP(gpio35),
- BCM6362_GROUP(gpio36),
- BCM6362_GROUP(gpio37),
- BCM6362_GROUP(gpio38),
- BCM6362_GROUP(gpio39),
- BCM6362_GROUP(gpio40),
- BCM6362_GROUP(gpio41),
- BCM6362_GROUP(gpio42),
- BCM6362_GROUP(gpio43),
- BCM6362_GROUP(gpio44),
- BCM6362_GROUP(gpio45),
- BCM6362_GROUP(gpio46),
- BCM6362_GROUP(gpio47),
- BCM6362_GROUP(nand_grp),
+static struct pingroup bcm6362_groups[] = {
+ BCM_PIN_GROUP(gpio0),
+ BCM_PIN_GROUP(gpio1),
+ BCM_PIN_GROUP(gpio2),
+ BCM_PIN_GROUP(gpio3),
+ BCM_PIN_GROUP(gpio4),
+ BCM_PIN_GROUP(gpio5),
+ BCM_PIN_GROUP(gpio6),
+ BCM_PIN_GROUP(gpio7),
+ BCM_PIN_GROUP(gpio8),
+ BCM_PIN_GROUP(gpio9),
+ BCM_PIN_GROUP(gpio10),
+ BCM_PIN_GROUP(gpio11),
+ BCM_PIN_GROUP(gpio12),
+ BCM_PIN_GROUP(gpio13),
+ BCM_PIN_GROUP(gpio14),
+ BCM_PIN_GROUP(gpio15),
+ BCM_PIN_GROUP(gpio16),
+ BCM_PIN_GROUP(gpio17),
+ BCM_PIN_GROUP(gpio18),
+ BCM_PIN_GROUP(gpio19),
+ BCM_PIN_GROUP(gpio20),
+ BCM_PIN_GROUP(gpio21),
+ BCM_PIN_GROUP(gpio22),
+ BCM_PIN_GROUP(gpio23),
+ BCM_PIN_GROUP(gpio24),
+ BCM_PIN_GROUP(gpio25),
+ BCM_PIN_GROUP(gpio26),
+ BCM_PIN_GROUP(gpio27),
+ BCM_PIN_GROUP(gpio28),
+ BCM_PIN_GROUP(gpio29),
+ BCM_PIN_GROUP(gpio30),
+ BCM_PIN_GROUP(gpio31),
+ BCM_PIN_GROUP(gpio32),
+ BCM_PIN_GROUP(gpio33),
+ BCM_PIN_GROUP(gpio34),
+ BCM_PIN_GROUP(gpio35),
+ BCM_PIN_GROUP(gpio36),
+ BCM_PIN_GROUP(gpio37),
+ BCM_PIN_GROUP(gpio38),
+ BCM_PIN_GROUP(gpio39),
+ BCM_PIN_GROUP(gpio40),
+ BCM_PIN_GROUP(gpio41),
+ BCM_PIN_GROUP(gpio42),
+ BCM_PIN_GROUP(gpio43),
+ BCM_PIN_GROUP(gpio44),
+ BCM_PIN_GROUP(gpio45),
+ BCM_PIN_GROUP(gpio46),
+ BCM_PIN_GROUP(gpio47),
+ BCM_PIN_GROUP(nand_grp),
};
static const char * const led_groups[] = {
static int bcm6362_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
unsigned group, const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
*pins = bcm6362_groups[group].pins;
- *num_pins = bcm6362_groups[group].num_pins;
+ *npins = bcm6362_groups[group].npins;
return 0;
}
unsigned selector, unsigned group)
{
struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
- const struct bcm6362_pingroup *pg = &bcm6362_groups[group];
+ const struct pingroup *pg = &bcm6362_groups[group];
const struct bcm6362_function *f = &bcm6362_funcs[selector];
unsigned i;
unsigned int reg;
unsigned int val, mask;
- for (i = 0; i < pg->num_pins; i++)
+ for (i = 0; i < pg->npins; i++)
bcm6362_set_gpio(pc, pg->pins[i]);
switch (f->reg) {
#define BCM6368_BASEMODE_GPIO 0x0
#define BCM6368_BASEMODE_UART1 0x1
-struct bcm6368_pingroup {
- const char *name;
- const unsigned * const pins;
- const unsigned num_pins;
-};
-
struct bcm6368_function {
const char *name;
const char * const *groups;
static unsigned gpio31_pins[] = { 31 };
static unsigned uart1_grp_pins[] = { 30, 31, 32, 33 };
-#define BCM6368_GROUP(n) \
- { \
- .name = #n, \
- .pins = n##_pins, \
- .num_pins = ARRAY_SIZE(n##_pins), \
- }
-
-static struct bcm6368_pingroup bcm6368_groups[] = {
- BCM6368_GROUP(gpio0),
- BCM6368_GROUP(gpio1),
- BCM6368_GROUP(gpio2),
- BCM6368_GROUP(gpio3),
- BCM6368_GROUP(gpio4),
- BCM6368_GROUP(gpio5),
- BCM6368_GROUP(gpio6),
- BCM6368_GROUP(gpio7),
- BCM6368_GROUP(gpio8),
- BCM6368_GROUP(gpio9),
- BCM6368_GROUP(gpio10),
- BCM6368_GROUP(gpio11),
- BCM6368_GROUP(gpio12),
- BCM6368_GROUP(gpio13),
- BCM6368_GROUP(gpio14),
- BCM6368_GROUP(gpio15),
- BCM6368_GROUP(gpio16),
- BCM6368_GROUP(gpio17),
- BCM6368_GROUP(gpio18),
- BCM6368_GROUP(gpio19),
- BCM6368_GROUP(gpio20),
- BCM6368_GROUP(gpio21),
- BCM6368_GROUP(gpio22),
- BCM6368_GROUP(gpio23),
- BCM6368_GROUP(gpio24),
- BCM6368_GROUP(gpio25),
- BCM6368_GROUP(gpio26),
- BCM6368_GROUP(gpio27),
- BCM6368_GROUP(gpio28),
- BCM6368_GROUP(gpio29),
- BCM6368_GROUP(gpio30),
- BCM6368_GROUP(gpio31),
- BCM6368_GROUP(uart1_grp),
+static struct pingroup bcm6368_groups[] = {
+ BCM_PIN_GROUP(gpio0),
+ BCM_PIN_GROUP(gpio1),
+ BCM_PIN_GROUP(gpio2),
+ BCM_PIN_GROUP(gpio3),
+ BCM_PIN_GROUP(gpio4),
+ BCM_PIN_GROUP(gpio5),
+ BCM_PIN_GROUP(gpio6),
+ BCM_PIN_GROUP(gpio7),
+ BCM_PIN_GROUP(gpio8),
+ BCM_PIN_GROUP(gpio9),
+ BCM_PIN_GROUP(gpio10),
+ BCM_PIN_GROUP(gpio11),
+ BCM_PIN_GROUP(gpio12),
+ BCM_PIN_GROUP(gpio13),
+ BCM_PIN_GROUP(gpio14),
+ BCM_PIN_GROUP(gpio15),
+ BCM_PIN_GROUP(gpio16),
+ BCM_PIN_GROUP(gpio17),
+ BCM_PIN_GROUP(gpio18),
+ BCM_PIN_GROUP(gpio19),
+ BCM_PIN_GROUP(gpio20),
+ BCM_PIN_GROUP(gpio21),
+ BCM_PIN_GROUP(gpio22),
+ BCM_PIN_GROUP(gpio23),
+ BCM_PIN_GROUP(gpio24),
+ BCM_PIN_GROUP(gpio25),
+ BCM_PIN_GROUP(gpio26),
+ BCM_PIN_GROUP(gpio27),
+ BCM_PIN_GROUP(gpio28),
+ BCM_PIN_GROUP(gpio29),
+ BCM_PIN_GROUP(gpio30),
+ BCM_PIN_GROUP(gpio31),
+ BCM_PIN_GROUP(uart1_grp),
};
static const char * const analog_afe_0_groups[] = {
static int bcm6368_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
unsigned group, const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
*pins = bcm6368_groups[group].pins;
- *num_pins = bcm6368_groups[group].num_pins;
+ *npins = bcm6368_groups[group].npins;
return 0;
}
{
struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
struct bcm6368_priv *priv = pc->driver_data;
- const struct bcm6368_pingroup *pg = &bcm6368_groups[group];
+ const struct pingroup *pg = &bcm6368_groups[group];
const struct bcm6368_function *fun = &bcm6368_funcs[selector];
int i, pin;
if (fun->basemode) {
unsigned int mask = 0;
- for (i = 0; i < pg->num_pins; i++) {
+ for (i = 0; i < pg->npins; i++) {
pin = pg->pins[i];
if (pin < BCM63XX_BANK_GPIOS)
mask |= BIT(pin);
BIT(pin));
}
- for (pin = 0; pin < pg->num_pins; pin++) {
+ for (pin = 0; pin < pg->npins; pin++) {
struct pinctrl_gpio_range *range;
int hw_gpio = bcm6368_pins[pin].number;
unsigned int ngpios;
};
+#define BCM_PIN_GROUP(n) PINCTRL_PINGROUP(#n, n##_pins, ARRAY_SIZE(n##_pins))
+
struct bcm63xx_pinctrl {
struct device *dev;
struct regmap *regs;
res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"cru_gpio_control");
ns_pinctrl->base = devm_ioremap_resource(dev, res);
- if (IS_ERR(ns_pinctrl->base)) {
- dev_err(dev, "Failed to map pinctrl regs\n");
+ if (IS_ERR(ns_pinctrl->base))
return PTR_ERR(ns_pinctrl->base);
- }
memcpy(pctldesc, &ns_pinctrl_desc, sizeof(*pctldesc));
for (i = 0; i < pctrl->desc->ngroups; i++) {
desc_group = pctrl->desc->groups + i;
- /* compute the maxiumum number of functions a group can have */
+ /* compute the maximum number of functions a group can have */
max_functions += 1 << (desc_group->bit_width + 1);
}
config PINCTRL_IMX8MM
tristate "IMX8MM pinctrl driver"
- depends on ARCH_MXC
+ depends on OF
+ depends on SOC_IMX8M
select PINCTRL_IMX
help
Say Y here to enable the imx8mm pinctrl driver
config PINCTRL_IMX8MN
tristate "IMX8MN pinctrl driver"
- depends on ARCH_MXC
+ depends on OF
+ depends on SOC_IMX8M
select PINCTRL_IMX
help
Say Y here to enable the imx8mn pinctrl driver
config PINCTRL_IMX8MP
tristate "IMX8MP pinctrl driver"
- depends on ARCH_MXC
+ depends on OF
+ depends on SOC_IMX8M
select PINCTRL_IMX
help
Say Y here to enable the imx8mp pinctrl driver
config PINCTRL_IMX8MQ
tristate "IMX8MQ pinctrl driver"
- depends on ARCH_MXC
+ depends on OF
+ depends on SOC_IMX8M
select PINCTRL_IMX
help
Say Y here to enable the imx8mq pinctrl driver
default ARM64 && ARCH_MEDIATEK
select PINCTRL_MTK_PARIS
+config PINCTRL_MT8188
+ bool "MediaTek MT8188 pin control"
+ depends on OF
+ depends on ARM64 || COMPILE_TEST
+ default ARM64 && ARCH_MEDIATEK
+ select PINCTRL_MTK_PARIS
+ help
+ Say yes here to support pin controller and gpio driver
+ on MediaTek MT8188 SoC.
+ In MTK platform, we support virtual gpio and use it to
+ map specific eint which doesn't have real gpio pin.
+
config PINCTRL_MT8192
bool "Mediatek MT8192 pin control"
depends on OF
obj-$(CONFIG_PINCTRL_MT8173) += pinctrl-mt8173.o
obj-$(CONFIG_PINCTRL_MT8183) += pinctrl-mt8183.o
obj-$(CONFIG_PINCTRL_MT8186) += pinctrl-mt8186.o
+obj-$(CONFIG_PINCTRL_MT8188) += pinctrl-mt8188.o
obj-$(CONFIG_PINCTRL_MT8192) += pinctrl-mt8192.o
obj-$(CONFIG_PINCTRL_MT8195) += pinctrl-mt8195.o
obj-$(CONFIG_PINCTRL_MT8365) += pinctrl-mt8365.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 MediaTek Inc.
+ * Author: Hui Liu <hui.liu@mediatek.com>
+ *
+ */
+
+#include <linux/module.h>
+#include "pinctrl-mtk-mt8188.h"
+#include "pinctrl-paris.h"
+
+/* MT8188 have multiple bases to program pin configuration listed as the below:
+ * iocfg[0]:0x10005000, iocfg[1]:0x11c00000, iocfg[2]:0x11e10000,
+ * iocfg[3]:0x11e20000, iocfg[4]:0x11ea0000
+ * _i_based could be used to indicate what base the pin should be mapped into.
+ */
+
+#define PIN_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \
+ PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, \
+ 32, 0)
+
+#define PINS_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \
+ PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, \
+ 32, 1)
+
+static const struct mtk_pin_field_calc mt8188_pin_mode_range[] = {
+ PIN_FIELD(0, 177, 0x0300, 0x10, 0, 4),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_dir_range[] = {
+ PIN_FIELD(0, 177, 0x0000, 0x10, 0, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_di_range[] = {
+ PIN_FIELD(0, 177, 0x0200, 0x10, 0, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_do_range[] = {
+ PIN_FIELD(0, 177, 0x0100, 0x10, 0, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_smt_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x0170, 0x10, 8, 1),
+ PIN_FIELD_BASE(1, 1, 1, 0x0170, 0x10, 9, 1),
+ PIN_FIELD_BASE(2, 2, 1, 0x0170, 0x10, 10, 1),
+ PIN_FIELD_BASE(3, 3, 1, 0x0170, 0x10, 11, 1),
+ PIN_FIELD_BASE(4, 4, 1, 0x0170, 0x10, 18, 1),
+ PIN_FIELD_BASE(5, 5, 1, 0x0170, 0x10, 18, 1),
+ PIN_FIELD_BASE(6, 6, 1, 0x0170, 0x10, 18, 1),
+ PIN_FIELD_BASE(7, 7, 1, 0x0170, 0x10, 12, 1),
+ PIN_FIELD_BASE(8, 8, 1, 0x0170, 0x10, 13, 1),
+ PIN_FIELD_BASE(9, 9, 1, 0x0170, 0x10, 14, 1),
+ PIN_FIELD_BASE(10, 10, 1, 0x0170, 0x10, 15, 1),
+ PIN_FIELD_BASE(11, 11, 1, 0x0170, 0x10, 19, 1),
+ PIN_FIELD_BASE(12, 12, 2, 0x0160, 0x10, 12, 1),
+ PIN_FIELD_BASE(13, 13, 2, 0x0160, 0x10, 13, 1),
+ PIN_FIELD_BASE(14, 14, 2, 0x0160, 0x10, 14, 1),
+ PIN_FIELD_BASE(15, 15, 2, 0x0160, 0x10, 15, 1),
+ PIN_FIELD_BASE(16, 16, 3, 0x00d0, 0x10, 10, 1),
+ PIN_FIELD_BASE(17, 17, 3, 0x00d0, 0x10, 10, 1),
+ PIN_FIELD_BASE(18, 18, 4, 0x00e0, 0x10, 9, 1),
+ PIN_FIELD_BASE(19, 19, 4, 0x00e0, 0x10, 9, 1),
+ PIN_FIELD_BASE(20, 20, 4, 0x00e0, 0x10, 9, 1),
+ PIN_FIELD_BASE(21, 21, 4, 0x00e0, 0x10, 9, 1),
+ PIN_FIELD_BASE(22, 22, 4, 0x00e0, 0x10, 0, 1),
+ PIN_FIELD_BASE(23, 23, 4, 0x00e0, 0x10, 1, 1),
+ PIN_FIELD_BASE(24, 24, 4, 0x00e0, 0x10, 2, 1),
+ PIN_FIELD_BASE(25, 25, 1, 0x0170, 0x10, 17, 1),
+ PIN_FIELD_BASE(26, 26, 1, 0x0170, 0x10, 17, 1),
+ PIN_FIELD_BASE(27, 27, 1, 0x0170, 0x10, 17, 1),
+ PIN_FIELD_BASE(28, 28, 1, 0x0170, 0x10, 18, 1),
+ PIN_FIELD_BASE(29, 29, 1, 0x0170, 0x10, 16, 1),
+ PIN_FIELD_BASE(30, 30, 1, 0x0170, 0x10, 17, 1),
+ PIN_FIELD_BASE(31, 31, 1, 0x0170, 0x10, 19, 1),
+ PIN_FIELD_BASE(32, 32, 1, 0x0170, 0x10, 19, 1),
+ PIN_FIELD_BASE(33, 33, 1, 0x0170, 0x10, 20, 1),
+ PIN_FIELD_BASE(34, 34, 1, 0x0170, 0x10, 20, 1),
+ PIN_FIELD_BASE(35, 35, 1, 0x0170, 0x10, 19, 1),
+ PIN_FIELD_BASE(36, 36, 1, 0x0170, 0x10, 20, 1),
+ PIN_FIELD_BASE(37, 37, 1, 0x0170, 0x10, 21, 1),
+ PIN_FIELD_BASE(38, 38, 1, 0x0170, 0x10, 20, 1),
+ PIN_FIELD_BASE(39, 39, 1, 0x0170, 0x10, 21, 1),
+ PIN_FIELD_BASE(40, 40, 1, 0x0170, 0x10, 21, 1),
+ PIN_FIELD_BASE(41, 41, 1, 0x0170, 0x10, 21, 1),
+ PIN_FIELD_BASE(42, 42, 2, 0x0160, 0x10, 21, 1),
+ PIN_FIELD_BASE(43, 43, 2, 0x0160, 0x10, 22, 1),
+ PIN_FIELD_BASE(44, 44, 2, 0x0160, 0x10, 21, 1),
+ PIN_FIELD_BASE(45, 45, 2, 0x0160, 0x10, 22, 1),
+ PIN_FIELD_BASE(46, 46, 3, 0x00d0, 0x10, 10, 1),
+ PIN_FIELD_BASE(47, 47, 1, 0x0170, 0x10, 16, 1),
+ PIN_FIELD_BASE(48, 48, 1, 0x0170, 0x10, 16, 1),
+ PIN_FIELD_BASE(49, 49, 1, 0x0170, 0x10, 16, 1),
+ PIN_FIELD_BASE(50, 50, 3, 0x00d0, 0x10, 10, 1),
+ PIN_FIELD_BASE(51, 51, 3, 0x00d0, 0x10, 11, 1),
+ PIN_FIELD_BASE(52, 52, 3, 0x00d0, 0x10, 11, 1),
+ PIN_FIELD_BASE(53, 53, 3, 0x00d0, 0x10, 11, 1),
+ PIN_FIELD_BASE(54, 54, 3, 0x00d0, 0x10, 11, 1),
+ PIN_FIELD_BASE(55, 55, 1, 0x0170, 0x10, 25, 1),
+ PIN_FIELD_BASE(56, 56, 1, 0x0170, 0x10, 28, 1),
+ PIN_FIELD_BASE(57, 57, 2, 0x0160, 0x10, 29, 1),
+ PIN_FIELD_BASE(58, 58, 2, 0x0160, 0x10, 31, 1),
+ PIN_FIELD_BASE(59, 59, 1, 0x0170, 0x10, 26, 1),
+ PIN_FIELD_BASE(60, 60, 1, 0x0170, 0x10, 29, 1),
+ PIN_FIELD_BASE(61, 61, 1, 0x0170, 0x10, 27, 1),
+ PIN_FIELD_BASE(62, 62, 1, 0x0170, 0x10, 30, 1),
+ PIN_FIELD_BASE(63, 63, 2, 0x0160, 0x10, 30, 1),
+ PIN_FIELD_BASE(64, 64, 2, 0x0170, 0x10, 0, 1),
+ PIN_FIELD_BASE(65, 65, 4, 0x00e0, 0x10, 10, 1),
+ PIN_FIELD_BASE(66, 66, 4, 0x00e0, 0x10, 12, 1),
+ PIN_FIELD_BASE(67, 67, 4, 0x00e0, 0x10, 11, 1),
+ PIN_FIELD_BASE(68, 68, 4, 0x00e0, 0x10, 13, 1),
+ PIN_FIELD_BASE(69, 69, 1, 0x0180, 0x10, 0, 1),
+ PIN_FIELD_BASE(70, 70, 1, 0x0170, 0x10, 31, 1),
+ PIN_FIELD_BASE(71, 71, 1, 0x0180, 0x10, 4, 1),
+ PIN_FIELD_BASE(72, 72, 1, 0x0180, 0x10, 3, 1),
+ PIN_FIELD_BASE(73, 73, 1, 0x0180, 0x10, 1, 1),
+ PIN_FIELD_BASE(74, 74, 1, 0x0180, 0x10, 2, 1),
+ PIN_FIELD_BASE(75, 75, 1, 0x0180, 0x10, 6, 1),
+ PIN_FIELD_BASE(76, 76, 1, 0x0180, 0x10, 5, 1),
+ PIN_FIELD_BASE(77, 77, 1, 0x0180, 0x10, 8, 1),
+ PIN_FIELD_BASE(78, 78, 1, 0x0180, 0x10, 7, 1),
+ PIN_FIELD_BASE(79, 79, 4, 0x00e0, 0x10, 15, 1),
+ PIN_FIELD_BASE(80, 80, 4, 0x00e0, 0x10, 14, 1),
+ PIN_FIELD_BASE(81, 81, 4, 0x00e0, 0x10, 17, 1),
+ PIN_FIELD_BASE(82, 82, 4, 0x00e0, 0x10, 16, 1),
+ PIN_FIELD_BASE(83, 83, 2, 0x0160, 0x10, 26, 1),
+ PIN_FIELD_BASE(84, 84, 2, 0x0160, 0x10, 26, 1),
+ PIN_FIELD_BASE(85, 85, 2, 0x0160, 0x10, 27, 1),
+ PIN_FIELD_BASE(86, 86, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(87, 87, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(88, 88, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(89, 89, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(90, 90, 2, 0x0160, 0x10, 27, 1),
+ PIN_FIELD_BASE(91, 91, 2, 0x0160, 0x10, 27, 1),
+ PIN_FIELD_BASE(92, 92, 2, 0x0160, 0x10, 18, 1),
+ PIN_FIELD_BASE(93, 93, 2, 0x0160, 0x10, 18, 1),
+ PIN_FIELD_BASE(94, 94, 2, 0x0160, 0x10, 18, 1),
+ PIN_FIELD_BASE(95, 95, 2, 0x0160, 0x10, 18, 1),
+ PIN_FIELD_BASE(96, 96, 2, 0x0160, 0x10, 22, 1),
+ PIN_FIELD_BASE(97, 97, 2, 0x0160, 0x10, 23, 1),
+ PIN_FIELD_BASE(98, 98, 2, 0x0160, 0x10, 24, 1),
+ PIN_FIELD_BASE(99, 99, 2, 0x0160, 0x10, 22, 1),
+ PIN_FIELD_BASE(100, 100, 2, 0x0160, 0x10, 16, 1),
+ PIN_FIELD_BASE(101, 101, 2, 0x0160, 0x10, 23, 1),
+ PIN_FIELD_BASE(102, 102, 2, 0x0160, 0x10, 23, 1),
+ PIN_FIELD_BASE(103, 103, 2, 0x0160, 0x10, 23, 1),
+ PIN_FIELD_BASE(104, 104, 2, 0x0160, 0x10, 24, 1),
+ PIN_FIELD_BASE(105, 105, 2, 0x0160, 0x10, 24, 1),
+ PIN_FIELD_BASE(106, 106, 2, 0x0160, 0x10, 24, 1),
+ PIN_FIELD_BASE(107, 107, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(108, 108, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(109, 109, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(110, 110, 2, 0x0160, 0x10, 17, 1),
+ PIN_FIELD_BASE(111, 111, 2, 0x0160, 0x10, 19, 1),
+ PIN_FIELD_BASE(112, 112, 2, 0x0160, 0x10, 19, 1),
+ PIN_FIELD_BASE(113, 113, 2, 0x0160, 0x10, 19, 1),
+ PIN_FIELD_BASE(114, 114, 2, 0x0160, 0x10, 19, 1),
+ PIN_FIELD_BASE(115, 115, 2, 0x0160, 0x10, 20, 1),
+ PIN_FIELD_BASE(116, 116, 2, 0x0160, 0x10, 20, 1),
+ PIN_FIELD_BASE(117, 117, 2, 0x0160, 0x10, 20, 1),
+ PIN_FIELD_BASE(118, 118, 2, 0x0160, 0x10, 20, 1),
+ PIN_FIELD_BASE(119, 119, 2, 0x0160, 0x10, 21, 1),
+ PIN_FIELD_BASE(120, 120, 2, 0x0160, 0x10, 21, 1),
+ PIN_FIELD_BASE(121, 121, 3, 0x00d0, 0x10, 6, 1),
+ PIN_FIELD_BASE(122, 122, 3, 0x00d0, 0x10, 9, 1),
+ PIN_FIELD_BASE(123, 123, 3, 0x00d0, 0x10, 8, 1),
+ PIN_FIELD_BASE(124, 124, 3, 0x00d0, 0x10, 7, 1),
+ PIN_FIELD_BASE(125, 125, 2, 0x0160, 0x10, 25, 1),
+ PIN_FIELD_BASE(126, 126, 2, 0x0160, 0x10, 25, 1),
+ PIN_FIELD_BASE(127, 127, 2, 0x0160, 0x10, 25, 1),
+ PIN_FIELD_BASE(128, 128, 2, 0x0160, 0x10, 25, 1),
+ PIN_FIELD_BASE(129, 129, 2, 0x0160, 0x10, 26, 1),
+ PIN_FIELD_BASE(130, 130, 2, 0x0160, 0x10, 26, 1),
+ PIN_FIELD_BASE(131, 131, 1, 0x0170, 0x10, 0, 1),
+ PIN_FIELD_BASE(132, 132, 1, 0x0170, 0x10, 1, 1),
+ PIN_FIELD_BASE(133, 133, 1, 0x0170, 0x10, 6, 1),
+ PIN_FIELD_BASE(134, 134, 1, 0x0170, 0x10, 7, 1),
+ PIN_FIELD_BASE(135, 135, 1, 0x0170, 0x10, 22, 1),
+ PIN_FIELD_BASE(136, 136, 1, 0x0170, 0x10, 22, 1),
+ PIN_FIELD_BASE(137, 137, 1, 0x0170, 0x10, 22, 1),
+ PIN_FIELD_BASE(138, 138, 1, 0x0170, 0x10, 22, 1),
+ PIN_FIELD_BASE(139, 139, 1, 0x0170, 0x10, 23, 1),
+ PIN_FIELD_BASE(140, 140, 1, 0x0170, 0x10, 23, 1),
+ PIN_FIELD_BASE(141, 141, 1, 0x0170, 0x10, 23, 1),
+ PIN_FIELD_BASE(142, 142, 1, 0x0170, 0x10, 23, 1),
+ PIN_FIELD_BASE(143, 143, 1, 0x0170, 0x10, 2, 1),
+ PIN_FIELD_BASE(144, 144, 1, 0x0170, 0x10, 3, 1),
+ PIN_FIELD_BASE(145, 145, 1, 0x0170, 0x10, 4, 1),
+ PIN_FIELD_BASE(146, 146, 1, 0x0170, 0x10, 5, 1),
+ PIN_FIELD_BASE(147, 147, 1, 0x0170, 0x10, 24, 1),
+ PIN_FIELD_BASE(148, 148, 1, 0x0170, 0x10, 24, 1),
+ PIN_FIELD_BASE(149, 149, 1, 0x0170, 0x10, 24, 1),
+ PIN_FIELD_BASE(150, 150, 1, 0x0170, 0x10, 24, 1),
+ PIN_FIELD_BASE(151, 151, 2, 0x0160, 0x10, 9, 1),
+ PIN_FIELD_BASE(152, 152, 2, 0x0160, 0x10, 8, 1),
+ PIN_FIELD_BASE(153, 153, 2, 0x0160, 0x10, 7, 1),
+ PIN_FIELD_BASE(154, 154, 2, 0x0160, 0x10, 6, 1),
+ PIN_FIELD_BASE(155, 155, 2, 0x0160, 0x10, 11, 1),
+ PIN_FIELD_BASE(156, 156, 2, 0x0160, 0x10, 1, 1),
+ PIN_FIELD_BASE(157, 157, 2, 0x0160, 0x10, 0, 1),
+ PIN_FIELD_BASE(158, 158, 2, 0x0160, 0x10, 5, 1),
+ PIN_FIELD_BASE(159, 159, 2, 0x0160, 0x10, 4, 1),
+ PIN_FIELD_BASE(160, 160, 2, 0x0160, 0x10, 3, 1),
+ PIN_FIELD_BASE(161, 161, 2, 0x0160, 0x10, 2, 1),
+ PIN_FIELD_BASE(162, 162, 2, 0x0160, 0x10, 10, 1),
+ PIN_FIELD_BASE(163, 163, 4, 0x00e0, 0x10, 4, 1),
+ PIN_FIELD_BASE(164, 164, 4, 0x00e0, 0x10, 3, 1),
+ PIN_FIELD_BASE(165, 165, 4, 0x00e0, 0x10, 5, 1),
+ PIN_FIELD_BASE(166, 166, 4, 0x00e0, 0x10, 6, 1),
+ PIN_FIELD_BASE(167, 167, 4, 0x00e0, 0x10, 7, 1),
+ PIN_FIELD_BASE(168, 168, 4, 0x00e0, 0x10, 8, 1),
+ PIN_FIELD_BASE(169, 169, 3, 0x00d0, 0x10, 1, 1),
+ PIN_FIELD_BASE(170, 170, 3, 0x00d0, 0x10, 0, 1),
+ PIN_FIELD_BASE(171, 171, 3, 0x00d0, 0x10, 2, 1),
+ PIN_FIELD_BASE(172, 172, 3, 0x00d0, 0x10, 3, 1),
+ PIN_FIELD_BASE(173, 173, 3, 0x00d0, 0x10, 4, 1),
+ PIN_FIELD_BASE(174, 174, 3, 0x00d0, 0x10, 5, 1),
+ PIN_FIELD_BASE(175, 175, 2, 0x0160, 0x10, 28, 1),
+ PIN_FIELD_BASE(176, 176, 2, 0x0160, 0x10, 28, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_ies_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x0080, 0x10, 26, 1),
+ PIN_FIELD_BASE(1, 1, 1, 0x0080, 0x10, 27, 1),
+ PIN_FIELD_BASE(2, 2, 1, 0x0080, 0x10, 28, 1),
+ PIN_FIELD_BASE(3, 3, 1, 0x0080, 0x10, 29, 1),
+ PIN_FIELD_BASE(4, 4, 1, 0x0080, 0x10, 30, 1),
+ PIN_FIELD_BASE(5, 5, 1, 0x0080, 0x10, 31, 1),
+ PIN_FIELD_BASE(6, 6, 1, 0x0090, 0x10, 0, 1),
+ PIN_FIELD_BASE(7, 7, 1, 0x0090, 0x10, 1, 1),
+ PIN_FIELD_BASE(8, 8, 1, 0x0090, 0x10, 2, 1),
+ PIN_FIELD_BASE(9, 9, 1, 0x0090, 0x10, 3, 1),
+ PIN_FIELD_BASE(10, 10, 1, 0x0090, 0x10, 4, 1),
+ PIN_FIELD_BASE(11, 11, 1, 0x0090, 0x10, 5, 1),
+ PIN_FIELD_BASE(12, 12, 2, 0x0070, 0x10, 24, 1),
+ PIN_FIELD_BASE(13, 13, 2, 0x0070, 0x10, 25, 1),
+ PIN_FIELD_BASE(14, 14, 2, 0x0070, 0x10, 26, 1),
+ PIN_FIELD_BASE(15, 15, 2, 0x0070, 0x10, 27, 1),
+ PIN_FIELD_BASE(16, 16, 3, 0x0040, 0x10, 1, 1),
+ PIN_FIELD_BASE(17, 17, 3, 0x0040, 0x10, 2, 1),
+ PIN_FIELD_BASE(18, 18, 4, 0x0050, 0x10, 3, 1),
+ PIN_FIELD_BASE(19, 19, 4, 0x0050, 0x10, 5, 1),
+ PIN_FIELD_BASE(20, 20, 4, 0x0050, 0x10, 4, 1),
+ PIN_FIELD_BASE(21, 21, 4, 0x0050, 0x10, 6, 1),
+ PIN_FIELD_BASE(22, 22, 4, 0x0050, 0x10, 0, 1),
+ PIN_FIELD_BASE(23, 23, 4, 0x0050, 0x10, 1, 1),
+ PIN_FIELD_BASE(24, 24, 4, 0x0050, 0x10, 2, 1),
+ PIN_FIELD_BASE(25, 25, 1, 0x0080, 0x10, 23, 1),
+ PIN_FIELD_BASE(26, 26, 1, 0x0080, 0x10, 22, 1),
+ PIN_FIELD_BASE(27, 27, 1, 0x0080, 0x10, 25, 1),
+ PIN_FIELD_BASE(28, 28, 1, 0x0080, 0x10, 24, 1),
+ PIN_FIELD_BASE(29, 29, 1, 0x0080, 0x10, 0, 1),
+ PIN_FIELD_BASE(30, 30, 1, 0x0080, 0x10, 1, 1),
+ PIN_FIELD_BASE(31, 31, 1, 0x0090, 0x10, 31, 1),
+ PIN_FIELD_BASE(32, 32, 1, 0x0090, 0x10, 30, 1),
+ PIN_FIELD_BASE(33, 33, 1, 0x00a0, 0x10, 1, 1),
+ PIN_FIELD_BASE(34, 34, 1, 0x00a0, 0x10, 0, 1),
+ PIN_FIELD_BASE(35, 35, 1, 0x00a0, 0x10, 3, 1),
+ PIN_FIELD_BASE(36, 36, 1, 0x00a0, 0x10, 2, 1),
+ PIN_FIELD_BASE(37, 37, 1, 0x0090, 0x10, 9, 1),
+ PIN_FIELD_BASE(38, 38, 1, 0x0090, 0x10, 6, 1),
+ PIN_FIELD_BASE(39, 39, 1, 0x0090, 0x10, 7, 1),
+ PIN_FIELD_BASE(40, 40, 1, 0x0090, 0x10, 8, 1),
+ PIN_FIELD_BASE(41, 41, 1, 0x0090, 0x10, 10, 1),
+ PIN_FIELD_BASE(42, 42, 2, 0x0080, 0x10, 10, 1),
+ PIN_FIELD_BASE(43, 43, 2, 0x0080, 0x10, 11, 1),
+ PIN_FIELD_BASE(44, 44, 2, 0x0080, 0x10, 12, 1),
+ PIN_FIELD_BASE(45, 45, 2, 0x0080, 0x10, 13, 1),
+ PIN_FIELD_BASE(46, 46, 3, 0x0040, 0x10, 0, 1),
+ PIN_FIELD_BASE(47, 47, 1, 0x0090, 0x10, 13, 1),
+ PIN_FIELD_BASE(48, 48, 1, 0x0090, 0x10, 12, 1),
+ PIN_FIELD_BASE(49, 49, 1, 0x0090, 0x10, 11, 1),
+ PIN_FIELD_BASE(50, 50, 3, 0x0040, 0x10, 5, 1),
+ PIN_FIELD_BASE(51, 51, 3, 0x0040, 0x10, 4, 1),
+ PIN_FIELD_BASE(52, 52, 3, 0x0040, 0x10, 3, 1),
+ PIN_FIELD_BASE(53, 53, 3, 0x0040, 0x10, 6, 1),
+ PIN_FIELD_BASE(54, 54, 3, 0x0040, 0x10, 7, 1),
+ PIN_FIELD_BASE(55, 55, 1, 0x0090, 0x10, 14, 1),
+ PIN_FIELD_BASE(56, 56, 1, 0x0090, 0x10, 17, 1),
+ PIN_FIELD_BASE(57, 57, 2, 0x0080, 0x10, 22, 1),
+ PIN_FIELD_BASE(58, 58, 2, 0x0080, 0x10, 25, 1),
+ PIN_FIELD_BASE(59, 59, 1, 0x0090, 0x10, 15, 1),
+ PIN_FIELD_BASE(60, 60, 1, 0x0090, 0x10, 18, 1),
+ PIN_FIELD_BASE(61, 61, 1, 0x0090, 0x10, 16, 1),
+ PIN_FIELD_BASE(62, 62, 1, 0x0090, 0x10, 19, 1),
+ PIN_FIELD_BASE(63, 63, 2, 0x0080, 0x10, 23, 1),
+ PIN_FIELD_BASE(64, 64, 2, 0x0080, 0x10, 26, 1),
+ PIN_FIELD_BASE(65, 65, 4, 0x0050, 0x10, 13, 1),
+ PIN_FIELD_BASE(66, 66, 4, 0x0050, 0x10, 15, 1),
+ PIN_FIELD_BASE(67, 67, 4, 0x0050, 0x10, 14, 1),
+ PIN_FIELD_BASE(68, 68, 4, 0x0050, 0x10, 16, 1),
+ PIN_FIELD_BASE(69, 69, 1, 0x0090, 0x10, 21, 1),
+ PIN_FIELD_BASE(70, 70, 1, 0x0090, 0x10, 20, 1),
+ PIN_FIELD_BASE(71, 71, 1, 0x0090, 0x10, 25, 1),
+ PIN_FIELD_BASE(72, 72, 1, 0x0090, 0x10, 24, 1),
+ PIN_FIELD_BASE(73, 73, 1, 0x0090, 0x10, 22, 1),
+ PIN_FIELD_BASE(74, 74, 1, 0x0090, 0x10, 23, 1),
+ PIN_FIELD_BASE(75, 75, 1, 0x0090, 0x10, 27, 1),
+ PIN_FIELD_BASE(76, 76, 1, 0x0090, 0x10, 26, 1),
+ PIN_FIELD_BASE(77, 77, 1, 0x0090, 0x10, 29, 1),
+ PIN_FIELD_BASE(78, 78, 1, 0x0090, 0x10, 28, 1),
+ PIN_FIELD_BASE(79, 79, 4, 0x0050, 0x10, 18, 1),
+ PIN_FIELD_BASE(80, 80, 4, 0x0050, 0x10, 17, 1),
+ PIN_FIELD_BASE(81, 81, 4, 0x0050, 0x10, 20, 1),
+ PIN_FIELD_BASE(82, 82, 4, 0x0050, 0x10, 19, 1),
+ PIN_FIELD_BASE(83, 83, 2, 0x0080, 0x10, 30, 1),
+ PIN_FIELD_BASE(84, 84, 2, 0x0080, 0x10, 29, 1),
+ PIN_FIELD_BASE(85, 85, 2, 0x0080, 0x10, 31, 1),
+ PIN_FIELD_BASE(86, 86, 2, 0x0090, 0x10, 1, 1),
+ PIN_FIELD_BASE(87, 87, 2, 0x0090, 0x10, 0, 1),
+ PIN_FIELD_BASE(88, 88, 2, 0x0090, 0x10, 2, 1),
+ PIN_FIELD_BASE(89, 89, 2, 0x0090, 0x10, 4, 1),
+ PIN_FIELD_BASE(90, 90, 2, 0x0090, 0x10, 3, 1),
+ PIN_FIELD_BASE(91, 91, 2, 0x0090, 0x10, 5, 1),
+ PIN_FIELD_BASE(92, 92, 2, 0x0080, 0x10, 19, 1),
+ PIN_FIELD_BASE(93, 93, 2, 0x0080, 0x10, 18, 1),
+ PIN_FIELD_BASE(94, 94, 2, 0x0080, 0x10, 21, 1),
+ PIN_FIELD_BASE(95, 95, 2, 0x0080, 0x10, 20, 1),
+ PIN_FIELD_BASE(96, 96, 2, 0x0080, 0x10, 15, 1),
+ PIN_FIELD_BASE(97, 97, 2, 0x0080, 0x10, 16, 1),
+ PIN_FIELD_BASE(98, 98, 2, 0x0080, 0x10, 24, 1),
+ PIN_FIELD_BASE(99, 99, 2, 0x0080, 0x10, 14, 1),
+ PIN_FIELD_BASE(100, 100, 2, 0x0080, 0x10, 17, 1),
+ PIN_FIELD_BASE(101, 101, 2, 0x0070, 0x10, 0, 1),
+ PIN_FIELD_BASE(102, 102, 2, 0x0070, 0x10, 5, 1),
+ PIN_FIELD_BASE(103, 103, 2, 0x0070, 0x10, 3, 1),
+ PIN_FIELD_BASE(104, 104, 2, 0x0070, 0x10, 4, 1),
+ PIN_FIELD_BASE(105, 105, 2, 0x0070, 0x10, 1, 1),
+ PIN_FIELD_BASE(106, 106, 2, 0x0070, 0x10, 2, 1),
+ PIN_FIELD_BASE(107, 107, 2, 0x0080, 0x10, 1, 1),
+ PIN_FIELD_BASE(108, 108, 2, 0x0070, 0x10, 28, 1),
+ PIN_FIELD_BASE(109, 109, 2, 0x0080, 0x10, 2, 1),
+ PIN_FIELD_BASE(110, 110, 2, 0x0070, 0x10, 29, 1),
+ PIN_FIELD_BASE(111, 111, 2, 0x0070, 0x10, 30, 1),
+ PIN_FIELD_BASE(112, 112, 2, 0x0070, 0x10, 31, 1),
+ PIN_FIELD_BASE(113, 113, 2, 0x0080, 0x10, 0, 1),
+ PIN_FIELD_BASE(114, 114, 2, 0x0080, 0x10, 8, 1),
+ PIN_FIELD_BASE(115, 115, 2, 0x0080, 0x10, 3, 1),
+ PIN_FIELD_BASE(116, 116, 2, 0x0080, 0x10, 9, 1),
+ PIN_FIELD_BASE(117, 117, 2, 0x0080, 0x10, 4, 1),
+ PIN_FIELD_BASE(118, 118, 2, 0x0080, 0x10, 5, 1),
+ PIN_FIELD_BASE(119, 119, 2, 0x0080, 0x10, 6, 1),
+ PIN_FIELD_BASE(120, 120, 2, 0x0080, 0x10, 7, 1),
+ PIN_FIELD_BASE(121, 121, 3, 0x0040, 0x10, 14, 1),
+ PIN_FIELD_BASE(122, 122, 3, 0x0040, 0x10, 17, 1),
+ PIN_FIELD_BASE(123, 123, 3, 0x0040, 0x10, 16, 1),
+ PIN_FIELD_BASE(124, 124, 3, 0x0040, 0x10, 15, 1),
+ PIN_FIELD_BASE(125, 125, 2, 0x0070, 0x10, 6, 1),
+ PIN_FIELD_BASE(126, 126, 2, 0x0070, 0x10, 7, 1),
+ PIN_FIELD_BASE(127, 127, 2, 0x0070, 0x10, 8, 1),
+ PIN_FIELD_BASE(128, 128, 2, 0x0070, 0x10, 9, 1),
+ PIN_FIELD_BASE(129, 129, 2, 0x0070, 0x10, 10, 1),
+ PIN_FIELD_BASE(130, 130, 2, 0x0070, 0x10, 11, 1),
+ PIN_FIELD_BASE(131, 131, 1, 0x0080, 0x10, 3, 1),
+ PIN_FIELD_BASE(132, 132, 1, 0x0080, 0x10, 4, 1),
+ PIN_FIELD_BASE(133, 133, 1, 0x0080, 0x10, 11, 1),
+ PIN_FIELD_BASE(134, 134, 1, 0x0080, 0x10, 12, 1),
+ PIN_FIELD_BASE(135, 135, 1, 0x0080, 0x10, 13, 1),
+ PIN_FIELD_BASE(136, 136, 1, 0x0080, 0x10, 14, 1),
+ PIN_FIELD_BASE(137, 137, 1, 0x0080, 0x10, 15, 1),
+ PIN_FIELD_BASE(138, 138, 1, 0x0080, 0x10, 16, 1),
+ PIN_FIELD_BASE(139, 139, 1, 0x0080, 0x10, 17, 1),
+ PIN_FIELD_BASE(140, 140, 1, 0x0080, 0x10, 18, 1),
+ PIN_FIELD_BASE(141, 141, 1, 0x0080, 0x10, 5, 1),
+ PIN_FIELD_BASE(142, 142, 1, 0x0080, 0x10, 6, 1),
+ PIN_FIELD_BASE(143, 143, 1, 0x0080, 0x10, 7, 1),
+ PIN_FIELD_BASE(144, 144, 1, 0x0080, 0x10, 8, 1),
+ PIN_FIELD_BASE(145, 145, 1, 0x0080, 0x10, 9, 1),
+ PIN_FIELD_BASE(146, 146, 1, 0x0080, 0x10, 10, 1),
+ PIN_FIELD_BASE(147, 147, 1, 0x0080, 0x10, 20, 1),
+ PIN_FIELD_BASE(148, 148, 1, 0x0080, 0x10, 21, 1),
+ PIN_FIELD_BASE(149, 149, 1, 0x0080, 0x10, 19, 1),
+ PIN_FIELD_BASE(150, 150, 1, 0x0080, 0x10, 2, 1),
+ PIN_FIELD_BASE(151, 151, 2, 0x0070, 0x10, 21, 1),
+ PIN_FIELD_BASE(152, 152, 2, 0x0070, 0x10, 20, 1),
+ PIN_FIELD_BASE(153, 153, 2, 0x0070, 0x10, 19, 1),
+ PIN_FIELD_BASE(154, 154, 2, 0x0070, 0x10, 18, 1),
+ PIN_FIELD_BASE(155, 155, 2, 0x0070, 0x10, 23, 1),
+ PIN_FIELD_BASE(156, 156, 2, 0x0070, 0x10, 13, 1),
+ PIN_FIELD_BASE(157, 157, 2, 0x0070, 0x10, 12, 1),
+ PIN_FIELD_BASE(158, 158, 2, 0x0070, 0x10, 17, 1),
+ PIN_FIELD_BASE(159, 159, 2, 0x0070, 0x10, 16, 1),
+ PIN_FIELD_BASE(160, 160, 2, 0x0070, 0x10, 15, 1),
+ PIN_FIELD_BASE(161, 161, 2, 0x0070, 0x10, 14, 1),
+ PIN_FIELD_BASE(162, 162, 2, 0x0070, 0x10, 22, 1),
+ PIN_FIELD_BASE(163, 163, 4, 0x0050, 0x10, 8, 1),
+ PIN_FIELD_BASE(164, 164, 4, 0x0050, 0x10, 7, 1),
+ PIN_FIELD_BASE(165, 165, 4, 0x0050, 0x10, 9, 1),
+ PIN_FIELD_BASE(166, 166, 4, 0x0050, 0x10, 10, 1),
+ PIN_FIELD_BASE(167, 167, 4, 0x0050, 0x10, 11, 1),
+ PIN_FIELD_BASE(168, 168, 4, 0x0050, 0x10, 12, 1),
+ PIN_FIELD_BASE(169, 169, 3, 0x0040, 0x10, 9, 1),
+ PIN_FIELD_BASE(170, 170, 3, 0x0040, 0x10, 8, 1),
+ PIN_FIELD_BASE(171, 171, 3, 0x0040, 0x10, 10, 1),
+ PIN_FIELD_BASE(172, 172, 3, 0x0040, 0x10, 11, 1),
+ PIN_FIELD_BASE(173, 173, 3, 0x0040, 0x10, 12, 1),
+ PIN_FIELD_BASE(174, 174, 3, 0x0040, 0x10, 13, 1),
+ PIN_FIELD_BASE(175, 175, 2, 0x0080, 0x10, 27, 1),
+ PIN_FIELD_BASE(176, 176, 2, 0x0080, 0x10, 28, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_tdsel_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x01b0, 0x10, 0, 4),
+ PIN_FIELD_BASE(1, 1, 1, 0x01b0, 0x10, 4, 4),
+ PIN_FIELD_BASE(2, 2, 1, 0x01b0, 0x10, 8, 4),
+ PIN_FIELD_BASE(3, 3, 1, 0x01b0, 0x10, 12, 4),
+ PIN_FIELD_BASE(4, 4, 1, 0x01c0, 0x10, 16, 4),
+ PIN_FIELD_BASE(5, 5, 1, 0x01c0, 0x10, 20, 4),
+ PIN_FIELD_BASE(6, 6, 1, 0x01c0, 0x10, 20, 4),
+ PIN_FIELD_BASE(7, 7, 1, 0x01b0, 0x10, 16, 4),
+ PIN_FIELD_BASE(8, 8, 1, 0x01b0, 0x10, 20, 4),
+ PIN_FIELD_BASE(9, 9, 1, 0x01b0, 0x10, 24, 4),
+ PIN_FIELD_BASE(10, 10, 1, 0x01b0, 0x10, 28, 4),
+ PIN_FIELD_BASE(11, 11, 1, 0x01c0, 0x10, 20, 4),
+ PIN_FIELD_BASE(12, 12, 2, 0x0190, 0x10, 16, 4),
+ PIN_FIELD_BASE(13, 13, 2, 0x0190, 0x10, 20, 4),
+ PIN_FIELD_BASE(14, 14, 2, 0x0190, 0x10, 24, 4),
+ PIN_FIELD_BASE(15, 15, 2, 0x0190, 0x10, 28, 4),
+ PIN_FIELD_BASE(16, 16, 3, 0x0100, 0x10, 8, 4),
+ PIN_FIELD_BASE(17, 17, 3, 0x0100, 0x10, 8, 4),
+ PIN_FIELD_BASE(18, 18, 4, 0x0110, 0x10, 4, 4),
+ PIN_FIELD_BASE(19, 19, 4, 0x0110, 0x10, 8, 4),
+ PIN_FIELD_BASE(20, 20, 4, 0x0110, 0x10, 8, 4),
+ PIN_FIELD_BASE(21, 21, 4, 0x0110, 0x10, 8, 4),
+ PIN_FIELD_BASE(22, 22, 4, 0x0100, 0x10, 0, 4),
+ PIN_FIELD_BASE(23, 23, 4, 0x0100, 0x10, 4, 4),
+ PIN_FIELD_BASE(24, 24, 4, 0x0100, 0x10, 8, 4),
+ PIN_FIELD_BASE(25, 25, 1, 0x01c0, 0x10, 8, 4),
+ PIN_FIELD_BASE(26, 26, 1, 0x01c0, 0x10, 8, 4),
+ PIN_FIELD_BASE(27, 27, 1, 0x01c0, 0x10, 8, 4),
+ PIN_FIELD_BASE(28, 28, 1, 0x01c0, 0x10, 12, 4),
+ PIN_FIELD_BASE(29, 29, 1, 0x01c0, 0x10, 0, 4),
+ PIN_FIELD_BASE(30, 30, 1, 0x01c0, 0x10, 8, 4),
+ PIN_FIELD_BASE(31, 31, 1, 0x01c0, 0x10, 20, 4),
+ PIN_FIELD_BASE(32, 32, 1, 0x01c0, 0x10, 24, 4),
+ PIN_FIELD_BASE(33, 33, 1, 0x01c0, 0x10, 24, 4),
+ PIN_FIELD_BASE(34, 34, 1, 0x01c0, 0x10, 28, 4),
+ PIN_FIELD_BASE(35, 35, 1, 0x01c0, 0x10, 24, 4),
+ PIN_FIELD_BASE(36, 36, 1, 0x01c0, 0x10, 24, 4),
+ PIN_FIELD_BASE(37, 37, 1, 0x01c0, 0x10, 28, 4),
+ PIN_FIELD_BASE(38, 38, 1, 0x01c0, 0x10, 28, 4),
+ PIN_FIELD_BASE(39, 39, 1, 0x01c0, 0x10, 28, 4),
+ PIN_FIELD_BASE(40, 40, 1, 0x01d0, 0x10, 0, 4),
+ PIN_FIELD_BASE(41, 41, 1, 0x01d0, 0x10, 0, 4),
+ PIN_FIELD_BASE(42, 42, 2, 0x01a0, 0x10, 16, 4),
+ PIN_FIELD_BASE(43, 43, 2, 0x01a0, 0x10, 20, 4),
+ PIN_FIELD_BASE(44, 44, 2, 0x01a0, 0x10, 16, 4),
+ PIN_FIELD_BASE(45, 45, 2, 0x01a0, 0x10, 20, 4),
+ PIN_FIELD_BASE(46, 46, 3, 0x0100, 0x10, 8, 4),
+ PIN_FIELD_BASE(47, 47, 1, 0x01c0, 0x10, 0, 4),
+ PIN_FIELD_BASE(48, 48, 1, 0x01c0, 0x10, 0, 4),
+ PIN_FIELD_BASE(49, 49, 1, 0x01c0, 0x10, 0, 4),
+ PIN_FIELD_BASE(50, 50, 3, 0x0100, 0x10, 8, 4),
+ PIN_FIELD_BASE(51, 51, 3, 0x0100, 0x10, 12, 4),
+ PIN_FIELD_BASE(52, 52, 3, 0x0100, 0x10, 12, 4),
+ PIN_FIELD_BASE(53, 53, 3, 0x0100, 0x10, 12, 4),
+ PIN_FIELD_BASE(54, 54, 3, 0x0100, 0x10, 12, 4),
+ PIN_FIELD_BASE(55, 55, 1, 0x01c0, 0x10, 12, 4),
+ PIN_FIELD_BASE(56, 56, 1, 0x01c0, 0x10, 12, 4),
+ PIN_FIELD_BASE(57, 57, 2, 0x01a0, 0x10, 24, 4),
+ PIN_FIELD_BASE(58, 58, 2, 0x01a0, 0x10, 24, 4),
+ PIN_FIELD_BASE(59, 59, 1, 0x01c0, 0x10, 16, 4),
+ PIN_FIELD_BASE(60, 60, 1, 0x01c0, 0x10, 12, 4),
+ PIN_FIELD_BASE(61, 61, 1, 0x01c0, 0x10, 16, 4),
+ PIN_FIELD_BASE(62, 62, 1, 0x01c0, 0x10, 16, 4),
+ PIN_FIELD_BASE(63, 63, 2, 0x01a0, 0x10, 20, 4),
+ PIN_FIELD_BASE(64, 64, 2, 0x01a0, 0x10, 20, 4),
+ PIN_FIELD_BASE(65, 65, 4, 0x0110, 0x10, 12, 4),
+ PIN_FIELD_BASE(66, 66, 4, 0x0110, 0x10, 8, 4),
+ PIN_FIELD_BASE(67, 67, 4, 0x0110, 0x10, 12, 4),
+ PIN_FIELD_BASE(68, 68, 4, 0x0110, 0x10, 12, 4),
+ PIN_FIELD_BASE(69, 69, 1, 0x01d0, 0x10, 16, 4),
+ PIN_FIELD_BASE(70, 70, 1, 0x01d0, 0x10, 12, 4),
+ PIN_FIELD_BASE(71, 71, 1, 0x01e0, 0x10, 0, 4),
+ PIN_FIELD_BASE(72, 72, 1, 0x01d0, 0x10, 28, 4),
+ PIN_FIELD_BASE(73, 73, 1, 0x01d0, 0x10, 20, 4),
+ PIN_FIELD_BASE(74, 74, 1, 0x01d0, 0x10, 24, 4),
+ PIN_FIELD_BASE(75, 75, 1, 0x01e0, 0x10, 8, 4),
+ PIN_FIELD_BASE(76, 76, 1, 0x01e0, 0x10, 4, 4),
+ PIN_FIELD_BASE(77, 77, 1, 0x01e0, 0x10, 16, 4),
+ PIN_FIELD_BASE(78, 78, 1, 0x01e0, 0x10, 12, 4),
+ PIN_FIELD_BASE(79, 79, 4, 0x0110, 0x10, 20, 4),
+ PIN_FIELD_BASE(80, 80, 4, 0x0110, 0x10, 16, 4),
+ PIN_FIELD_BASE(81, 81, 4, 0x0110, 0x10, 28, 4),
+ PIN_FIELD_BASE(82, 82, 4, 0x0110, 0x10, 24, 4),
+ PIN_FIELD_BASE(83, 83, 2, 0x01b0, 0x10, 8, 4),
+ PIN_FIELD_BASE(84, 84, 2, 0x01b0, 0x10, 8, 4),
+ PIN_FIELD_BASE(85, 85, 2, 0x01b0, 0x10, 12, 4),
+ PIN_FIELD_BASE(86, 86, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(87, 87, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(88, 88, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(89, 89, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(90, 90, 2, 0x01b0, 0x10, 12, 4),
+ PIN_FIELD_BASE(91, 91, 2, 0x01b0, 0x10, 12, 4),
+ PIN_FIELD_BASE(92, 92, 2, 0x01a0, 0x10, 4, 4),
+ PIN_FIELD_BASE(93, 93, 2, 0x01a0, 0x10, 4, 4),
+ PIN_FIELD_BASE(94, 94, 2, 0x01a0, 0x10, 4, 4),
+ PIN_FIELD_BASE(95, 95, 2, 0x01a0, 0x10, 4, 4),
+ PIN_FIELD_BASE(96, 96, 2, 0x01a0, 0x10, 24, 4),
+ PIN_FIELD_BASE(97, 97, 2, 0x01a0, 0x10, 28, 4),
+ PIN_FIELD_BASE(98, 98, 2, 0x01b0, 0x10, 0, 4),
+ PIN_FIELD_BASE(99, 99, 2, 0x01a0, 0x10, 24, 4),
+ PIN_FIELD_BASE(100, 100, 2, 0x01b0, 0x10, 20, 4),
+ PIN_FIELD_BASE(101, 101, 2, 0x01a0, 0x10, 28, 4),
+ PIN_FIELD_BASE(102, 102, 2, 0x01a0, 0x10, 28, 4),
+ PIN_FIELD_BASE(103, 103, 2, 0x01a0, 0x10, 28, 4),
+ PIN_FIELD_BASE(104, 104, 2, 0x01b0, 0x10, 0, 4),
+ PIN_FIELD_BASE(105, 105, 2, 0x01b0, 0x10, 0, 4),
+ PIN_FIELD_BASE(106, 106, 2, 0x01b0, 0x10, 0, 4),
+ PIN_FIELD_BASE(107, 107, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(108, 108, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(109, 109, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(110, 110, 2, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(111, 111, 2, 0x01a0, 0x10, 8, 4),
+ PIN_FIELD_BASE(112, 112, 2, 0x01a0, 0x10, 8, 4),
+ PIN_FIELD_BASE(113, 113, 2, 0x01a0, 0x10, 8, 4),
+ PIN_FIELD_BASE(114, 114, 2, 0x01a0, 0x10, 8, 4),
+ PIN_FIELD_BASE(115, 115, 2, 0x01a0, 0x10, 12, 4),
+ PIN_FIELD_BASE(116, 116, 2, 0x01a0, 0x10, 12, 4),
+ PIN_FIELD_BASE(117, 117, 2, 0x01a0, 0x10, 12, 4),
+ PIN_FIELD_BASE(118, 118, 2, 0x01a0, 0x10, 12, 4),
+ PIN_FIELD_BASE(119, 119, 2, 0x01a0, 0x10, 16, 4),
+ PIN_FIELD_BASE(120, 120, 2, 0x01a0, 0x10, 16, 4),
+ PIN_FIELD_BASE(121, 121, 3, 0x00f0, 0x10, 24, 4),
+ PIN_FIELD_BASE(122, 122, 3, 0x0100, 0x10, 4, 4),
+ PIN_FIELD_BASE(123, 123, 3, 0x0100, 0x10, 0, 4),
+ PIN_FIELD_BASE(124, 124, 3, 0x00f0, 0x10, 28, 4),
+ PIN_FIELD_BASE(125, 125, 2, 0x01b0, 0x10, 4, 4),
+ PIN_FIELD_BASE(126, 126, 2, 0x01b0, 0x10, 4, 4),
+ PIN_FIELD_BASE(127, 127, 2, 0x01b0, 0x10, 4, 4),
+ PIN_FIELD_BASE(128, 128, 2, 0x01b0, 0x10, 4, 4),
+ PIN_FIELD_BASE(129, 129, 2, 0x01b0, 0x10, 8, 4),
+ PIN_FIELD_BASE(130, 130, 2, 0x01b0, 0x10, 8, 4),
+ PIN_FIELD_BASE(131, 131, 1, 0x01a0, 0x10, 0, 4),
+ PIN_FIELD_BASE(132, 132, 1, 0x01a0, 0x10, 20, 4),
+ PIN_FIELD_BASE(133, 133, 1, 0x01a0, 0x10, 24, 4),
+ PIN_FIELD_BASE(134, 134, 1, 0x01a0, 0x10, 28, 4),
+ PIN_FIELD_BASE(135, 135, 1, 0x01d0, 0x10, 0, 4),
+ PIN_FIELD_BASE(136, 136, 1, 0x01d0, 0x10, 0, 4),
+ PIN_FIELD_BASE(137, 137, 1, 0x01d0, 0x10, 4, 4),
+ PIN_FIELD_BASE(138, 138, 1, 0x01d0, 0x10, 4, 4),
+ PIN_FIELD_BASE(139, 139, 1, 0x01d0, 0x10, 4, 4),
+ PIN_FIELD_BASE(140, 140, 1, 0x01d0, 0x10, 4, 4),
+ PIN_FIELD_BASE(141, 141, 1, 0x01d0, 0x10, 8, 4),
+ PIN_FIELD_BASE(142, 142, 1, 0x01d0, 0x10, 8, 4),
+ PIN_FIELD_BASE(143, 143, 1, 0x01a0, 0x10, 4, 4),
+ PIN_FIELD_BASE(144, 144, 1, 0x01a0, 0x10, 8, 4),
+ PIN_FIELD_BASE(145, 145, 1, 0x01a0, 0x10, 12, 4),
+ PIN_FIELD_BASE(146, 146, 1, 0x01a0, 0x10, 16, 4),
+ PIN_FIELD_BASE(147, 147, 1, 0x01d0, 0x10, 8, 4),
+ PIN_FIELD_BASE(148, 148, 1, 0x01d0, 0x10, 8, 4),
+ PIN_FIELD_BASE(149, 149, 1, 0x01c0, 0x10, 4, 4),
+ PIN_FIELD_BASE(150, 150, 1, 0x01c0, 0x10, 4, 4),
+ PIN_FIELD_BASE(151, 151, 2, 0x0190, 0x10, 4, 4),
+ PIN_FIELD_BASE(152, 152, 2, 0x0190, 0x10, 0, 4),
+ PIN_FIELD_BASE(153, 153, 2, 0x0180, 0x10, 28, 4),
+ PIN_FIELD_BASE(154, 154, 2, 0x0180, 0x10, 24, 4),
+ PIN_FIELD_BASE(155, 155, 2, 0x0190, 0x10, 12, 4),
+ PIN_FIELD_BASE(156, 156, 2, 0x0180, 0x10, 4, 4),
+ PIN_FIELD_BASE(157, 157, 2, 0x0180, 0x10, 0, 4),
+ PIN_FIELD_BASE(158, 158, 2, 0x0180, 0x10, 20, 4),
+ PIN_FIELD_BASE(159, 159, 2, 0x0180, 0x10, 16, 4),
+ PIN_FIELD_BASE(160, 160, 2, 0x0180, 0x10, 12, 4),
+ PIN_FIELD_BASE(161, 161, 2, 0x0180, 0x10, 8, 4),
+ PIN_FIELD_BASE(162, 162, 2, 0x0190, 0x10, 8, 4),
+ PIN_FIELD_BASE(163, 163, 4, 0x0100, 0x10, 16, 4),
+ PIN_FIELD_BASE(164, 164, 4, 0x0100, 0x10, 12, 4),
+ PIN_FIELD_BASE(165, 165, 4, 0x0100, 0x10, 20, 4),
+ PIN_FIELD_BASE(166, 166, 4, 0x0100, 0x10, 24, 4),
+ PIN_FIELD_BASE(167, 167, 4, 0x0100, 0x10, 28, 4),
+ PIN_FIELD_BASE(168, 168, 4, 0x0110, 0x10, 0, 4),
+ PIN_FIELD_BASE(169, 169, 3, 0x00f0, 0x10, 4, 4),
+ PIN_FIELD_BASE(170, 170, 3, 0x00f0, 0x10, 0, 4),
+ PIN_FIELD_BASE(171, 171, 3, 0x00f0, 0x10, 8, 4),
+ PIN_FIELD_BASE(172, 172, 3, 0x00f0, 0x10, 12, 4),
+ PIN_FIELD_BASE(173, 173, 3, 0x00f0, 0x10, 16, 4),
+ PIN_FIELD_BASE(174, 174, 3, 0x00f0, 0x10, 20, 4),
+ PIN_FIELD_BASE(175, 175, 2, 0x01b0, 0x10, 16, 4),
+ PIN_FIELD_BASE(176, 176, 2, 0x01b0, 0x10, 16, 4),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_rdsel_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x0130, 0x10, 18, 2),
+ PIN_FIELD_BASE(1, 1, 1, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(2, 2, 1, 0x0130, 0x10, 22, 2),
+ PIN_FIELD_BASE(3, 3, 1, 0x0130, 0x10, 24, 2),
+ PIN_FIELD_BASE(4, 4, 1, 0x0140, 0x10, 14, 2),
+ PIN_FIELD_BASE(5, 5, 1, 0x0140, 0x10, 16, 2),
+ PIN_FIELD_BASE(6, 6, 1, 0x0140, 0x10, 16, 2),
+ PIN_FIELD_BASE(7, 7, 1, 0x0130, 0x10, 26, 2),
+ PIN_FIELD_BASE(8, 8, 1, 0x0130, 0x10, 28, 2),
+ PIN_FIELD_BASE(9, 9, 1, 0x0130, 0x10, 30, 2),
+ PIN_FIELD_BASE(10, 10, 1, 0x0140, 0x10, 0, 2),
+ PIN_FIELD_BASE(11, 11, 1, 0x0140, 0x10, 16, 2),
+ PIN_FIELD_BASE(12, 12, 2, 0x0130, 0x10, 12, 2),
+ PIN_FIELD_BASE(13, 13, 2, 0x0130, 0x10, 14, 2),
+ PIN_FIELD_BASE(14, 14, 2, 0x0130, 0x10, 16, 2),
+ PIN_FIELD_BASE(15, 15, 2, 0x0130, 0x10, 18, 2),
+ PIN_FIELD_BASE(16, 16, 3, 0x00b0, 0x10, 14, 2),
+ PIN_FIELD_BASE(17, 17, 3, 0x00b0, 0x10, 14, 2),
+ PIN_FIELD_BASE(18, 18, 4, 0x00c0, 0x10, 12, 2),
+ PIN_FIELD_BASE(19, 19, 4, 0x00c0, 0x10, 12, 2),
+ PIN_FIELD_BASE(20, 20, 4, 0x00c0, 0x10, 12, 2),
+ PIN_FIELD_BASE(21, 21, 4, 0x00c0, 0x10, 12, 2),
+ PIN_FIELD_BASE(22, 22, 4, 0x00b0, 0x10, 0, 2),
+ PIN_FIELD_BASE(23, 23, 4, 0x00b0, 0x10, 2, 2),
+ PIN_FIELD_BASE(24, 24, 4, 0x00b0, 0x10, 4, 2),
+ PIN_FIELD_BASE(25, 25, 1, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(26, 26, 1, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(27, 27, 1, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(28, 28, 1, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(29, 29, 1, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(30, 30, 1, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(31, 31, 1, 0x0140, 0x10, 16, 2),
+ PIN_FIELD_BASE(32, 32, 1, 0x0140, 0x10, 18, 2),
+ PIN_FIELD_BASE(33, 33, 1, 0x0140, 0x10, 18, 2),
+ PIN_FIELD_BASE(34, 34, 1, 0x0140, 0x10, 20, 2),
+ PIN_FIELD_BASE(35, 35, 1, 0x0140, 0x10, 18, 2),
+ PIN_FIELD_BASE(36, 36, 1, 0x0140, 0x10, 18, 2),
+ PIN_FIELD_BASE(37, 37, 1, 0x0140, 0x10, 20, 2),
+ PIN_FIELD_BASE(38, 38, 1, 0x0140, 0x10, 20, 2),
+ PIN_FIELD_BASE(39, 39, 1, 0x0140, 0x10, 20, 2),
+ PIN_FIELD_BASE(40, 40, 1, 0x0140, 0x10, 22, 2),
+ PIN_FIELD_BASE(41, 41, 1, 0x0140, 0x10, 22, 2),
+ PIN_FIELD_BASE(42, 42, 2, 0x0130, 0x10, 30, 2),
+ PIN_FIELD_BASE(43, 43, 2, 0x0140, 0x10, 0, 2),
+ PIN_FIELD_BASE(44, 44, 2, 0x0130, 0x10, 30, 2),
+ PIN_FIELD_BASE(45, 45, 2, 0x0140, 0x10, 0, 2),
+ PIN_FIELD_BASE(46, 46, 3, 0x00b0, 0x10, 14, 2),
+ PIN_FIELD_BASE(47, 47, 1, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(48, 48, 1, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(49, 49, 1, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(50, 50, 3, 0x00b0, 0x10, 14, 2),
+ PIN_FIELD_BASE(51, 51, 3, 0x00b0, 0x10, 16, 2),
+ PIN_FIELD_BASE(52, 52, 3, 0x00b0, 0x10, 16, 2),
+ PIN_FIELD_BASE(53, 53, 3, 0x00b0, 0x10, 16, 2),
+ PIN_FIELD_BASE(54, 54, 3, 0x00b0, 0x10, 16, 2),
+ PIN_FIELD_BASE(55, 55, 1, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(56, 56, 1, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(57, 57, 2, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(58, 58, 2, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(59, 59, 1, 0x0140, 0x10, 14, 2),
+ PIN_FIELD_BASE(60, 60, 1, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(61, 61, 1, 0x0140, 0x10, 14, 2),
+ PIN_FIELD_BASE(62, 62, 1, 0x0140, 0x10, 14, 2),
+ PIN_FIELD_BASE(63, 63, 2, 0x0140, 0x10, 0, 2),
+ PIN_FIELD_BASE(64, 64, 2, 0x0140, 0x10, 0, 2),
+ PIN_FIELD_BASE(65, 65, 4, 0x00c0, 0x10, 14, 2),
+ PIN_FIELD_BASE(66, 66, 4, 0x00c0, 0x10, 14, 2),
+ PIN_FIELD_BASE(67, 67, 4, 0x00c0, 0x10, 14, 2),
+ PIN_FIELD_BASE(68, 68, 4, 0x00c0, 0x10, 14, 2),
+ PIN_FIELD_BASE(69, 69, 1, 0x0150, 0x10, 14, 2),
+ PIN_FIELD_BASE(70, 70, 1, 0x0150, 0x10, 12, 2),
+ PIN_FIELD_BASE(71, 71, 1, 0x0150, 0x10, 22, 2),
+ PIN_FIELD_BASE(72, 72, 1, 0x0150, 0x10, 20, 2),
+ PIN_FIELD_BASE(73, 73, 1, 0x0150, 0x10, 16, 2),
+ PIN_FIELD_BASE(74, 74, 1, 0x0150, 0x10, 18, 2),
+ PIN_FIELD_BASE(75, 75, 1, 0x0150, 0x10, 26, 2),
+ PIN_FIELD_BASE(76, 76, 1, 0x0150, 0x10, 24, 2),
+ PIN_FIELD_BASE(77, 77, 1, 0x0150, 0x10, 30, 2),
+ PIN_FIELD_BASE(78, 78, 1, 0x0150, 0x10, 28, 2),
+ PIN_FIELD_BASE(79, 79, 4, 0x00c0, 0x10, 18, 2),
+ PIN_FIELD_BASE(80, 80, 4, 0x00c0, 0x10, 16, 2),
+ PIN_FIELD_BASE(81, 81, 4, 0x00c0, 0x10, 22, 2),
+ PIN_FIELD_BASE(82, 82, 4, 0x00c0, 0x10, 20, 2),
+ PIN_FIELD_BASE(83, 83, 2, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(84, 84, 2, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(85, 85, 2, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(86, 86, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(87, 87, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(88, 88, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(89, 89, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(90, 90, 2, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(91, 91, 2, 0x0140, 0x10, 12, 2),
+ PIN_FIELD_BASE(92, 92, 2, 0x0130, 0x10, 22, 2),
+ PIN_FIELD_BASE(93, 93, 2, 0x0130, 0x10, 22, 2),
+ PIN_FIELD_BASE(94, 94, 2, 0x0130, 0x10, 22, 2),
+ PIN_FIELD_BASE(95, 95, 2, 0x0130, 0x10, 22, 2),
+ PIN_FIELD_BASE(96, 96, 2, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(97, 97, 2, 0x0140, 0x10, 4, 2),
+ PIN_FIELD_BASE(98, 98, 2, 0x0140, 0x10, 6, 2),
+ PIN_FIELD_BASE(99, 99, 2, 0x0140, 0x10, 2, 2),
+ PIN_FIELD_BASE(100, 100, 2, 0x0140, 0x10, 16, 2),
+ PIN_FIELD_BASE(101, 101, 2, 0x0140, 0x10, 4, 2),
+ PIN_FIELD_BASE(102, 102, 2, 0x0140, 0x10, 4, 2),
+ PIN_FIELD_BASE(103, 103, 2, 0x0140, 0x10, 4, 2),
+ PIN_FIELD_BASE(104, 104, 2, 0x0140, 0x10, 6, 2),
+ PIN_FIELD_BASE(105, 105, 2, 0x0140, 0x10, 6, 2),
+ PIN_FIELD_BASE(106, 106, 2, 0x0140, 0x10, 6, 2),
+ PIN_FIELD_BASE(107, 107, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(108, 108, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(109, 109, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(110, 110, 2, 0x0130, 0x10, 20, 2),
+ PIN_FIELD_BASE(111, 111, 2, 0x0130, 0x10, 24, 2),
+ PIN_FIELD_BASE(112, 112, 2, 0x0130, 0x10, 24, 2),
+ PIN_FIELD_BASE(113, 113, 2, 0x0130, 0x10, 24, 2),
+ PIN_FIELD_BASE(114, 114, 2, 0x0130, 0x10, 24, 2),
+ PIN_FIELD_BASE(115, 115, 2, 0x0130, 0x10, 28, 2),
+ PIN_FIELD_BASE(116, 116, 2, 0x0130, 0x10, 28, 2),
+ PIN_FIELD_BASE(117, 117, 2, 0x0130, 0x10, 28, 2),
+ PIN_FIELD_BASE(118, 118, 2, 0x0130, 0x10, 28, 2),
+ PIN_FIELD_BASE(119, 119, 2, 0x0130, 0x10, 30, 2),
+ PIN_FIELD_BASE(120, 120, 2, 0x0130, 0x10, 30, 2),
+ PIN_FIELD_BASE(121, 121, 3, 0x00b0, 0x10, 6, 2),
+ PIN_FIELD_BASE(122, 122, 3, 0x00b0, 0x10, 12, 2),
+ PIN_FIELD_BASE(123, 123, 3, 0x00b0, 0x10, 10, 2),
+ PIN_FIELD_BASE(124, 124, 3, 0x00b0, 0x10, 8, 2),
+ PIN_FIELD_BASE(125, 125, 2, 0x0140, 0x10, 8, 2),
+ PIN_FIELD_BASE(126, 126, 2, 0x0140, 0x10, 8, 2),
+ PIN_FIELD_BASE(127, 127, 2, 0x0140, 0x10, 8, 2),
+ PIN_FIELD_BASE(128, 128, 2, 0x0140, 0x10, 8, 2),
+ PIN_FIELD_BASE(129, 129, 2, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(130, 130, 2, 0x0140, 0x10, 10, 2),
+ PIN_FIELD_BASE(131, 131, 1, 0x0120, 0x10, 0, 6),
+ PIN_FIELD_BASE(132, 132, 1, 0x0130, 0x10, 0, 6),
+ PIN_FIELD_BASE(133, 133, 1, 0x0130, 0x10, 6, 6),
+ PIN_FIELD_BASE(134, 134, 1, 0x0130, 0x10, 12, 6),
+ PIN_FIELD_BASE(135, 135, 1, 0x0140, 0x10, 24, 6),
+ PIN_FIELD_BASE(136, 136, 1, 0x0140, 0x10, 24, 6),
+ PIN_FIELD_BASE(137, 137, 1, 0x0150, 0x10, 0, 6),
+ PIN_FIELD_BASE(138, 138, 1, 0x0150, 0x10, 0, 6),
+ PIN_FIELD_BASE(139, 139, 1, 0x0150, 0x10, 0, 6),
+ PIN_FIELD_BASE(140, 140, 1, 0x0150, 0x10, 0, 6),
+ PIN_FIELD_BASE(141, 141, 1, 0x0150, 0x10, 6, 6),
+ PIN_FIELD_BASE(142, 142, 1, 0x0150, 0x10, 6, 6),
+ PIN_FIELD_BASE(143, 143, 1, 0x0120, 0x10, 6, 6),
+ PIN_FIELD_BASE(144, 144, 1, 0x0120, 0x10, 12, 6),
+ PIN_FIELD_BASE(145, 145, 1, 0x0120, 0x10, 18, 6),
+ PIN_FIELD_BASE(146, 146, 1, 0x0120, 0x10, 24, 6),
+ PIN_FIELD_BASE(147, 147, 1, 0x0150, 0x10, 6, 6),
+ PIN_FIELD_BASE(148, 148, 1, 0x0150, 0x10, 6, 6),
+ PIN_FIELD_BASE(149, 149, 1, 0x0140, 0x10, 4, 6),
+ PIN_FIELD_BASE(150, 150, 1, 0x0140, 0x10, 4, 6),
+ PIN_FIELD_BASE(151, 151, 2, 0x0120, 0x10, 24, 6),
+ PIN_FIELD_BASE(152, 152, 2, 0x0120, 0x10, 18, 6),
+ PIN_FIELD_BASE(153, 153, 2, 0x0120, 0x10, 12, 6),
+ PIN_FIELD_BASE(154, 154, 2, 0x0120, 0x10, 6, 6),
+ PIN_FIELD_BASE(155, 155, 2, 0x0130, 0x10, 6, 6),
+ PIN_FIELD_BASE(156, 156, 2, 0x0110, 0x10, 6, 6),
+ PIN_FIELD_BASE(157, 157, 2, 0x0110, 0x10, 0, 6),
+ PIN_FIELD_BASE(158, 158, 2, 0x0120, 0x10, 0, 6),
+ PIN_FIELD_BASE(159, 159, 2, 0x0110, 0x10, 24, 6),
+ PIN_FIELD_BASE(160, 160, 2, 0x0110, 0x10, 18, 6),
+ PIN_FIELD_BASE(161, 161, 2, 0x0110, 0x10, 12, 6),
+ PIN_FIELD_BASE(162, 162, 2, 0x0130, 0x10, 0, 6),
+ PIN_FIELD_BASE(163, 163, 4, 0x00b0, 0x10, 12, 6),
+ PIN_FIELD_BASE(164, 164, 4, 0x00b0, 0x10, 6, 6),
+ PIN_FIELD_BASE(165, 165, 4, 0x00b0, 0x10, 18, 6),
+ PIN_FIELD_BASE(166, 166, 4, 0x00b0, 0x10, 24, 6),
+ PIN_FIELD_BASE(167, 167, 4, 0x00c0, 0x10, 0, 6),
+ PIN_FIELD_BASE(168, 168, 4, 0x00c0, 0x10, 6, 6),
+ PIN_FIELD_BASE(169, 169, 3, 0x00a0, 0x10, 6, 6),
+ PIN_FIELD_BASE(170, 170, 3, 0x00a0, 0x10, 0, 6),
+ PIN_FIELD_BASE(171, 171, 3, 0x00a0, 0x10, 12, 6),
+ PIN_FIELD_BASE(172, 172, 3, 0x00a0, 0x10, 18, 6),
+ PIN_FIELD_BASE(173, 173, 3, 0x00a0, 0x10, 24, 6),
+ PIN_FIELD_BASE(174, 174, 3, 0x00b0, 0x10, 0, 6),
+ PIN_FIELD_BASE(175, 175, 2, 0x0140, 0x10, 14, 2),
+ PIN_FIELD_BASE(176, 176, 2, 0x0140, 0x10, 14, 2),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_pupd_range[] = {
+ PIN_FIELD_BASE(42, 42, 2, 0x00c0, 0x10, 12, 1),
+ PIN_FIELD_BASE(43, 43, 2, 0x00c0, 0x10, 13, 1),
+ PIN_FIELD_BASE(44, 44, 2, 0x00c0, 0x10, 14, 1),
+ PIN_FIELD_BASE(45, 45, 2, 0x00c0, 0x10, 15, 1),
+ PIN_FIELD_BASE(131, 131, 1, 0x00d0, 0x10, 1, 1),
+ PIN_FIELD_BASE(132, 132, 1, 0x00d0, 0x10, 2, 1),
+ PIN_FIELD_BASE(133, 133, 1, 0x00d0, 0x10, 9, 1),
+ PIN_FIELD_BASE(134, 134, 1, 0x00d0, 0x10, 10, 1),
+ PIN_FIELD_BASE(135, 135, 1, 0x00d0, 0x10, 11, 1),
+ PIN_FIELD_BASE(136, 136, 1, 0x00d0, 0x10, 12, 1),
+ PIN_FIELD_BASE(137, 137, 1, 0x00d0, 0x10, 13, 1),
+ PIN_FIELD_BASE(138, 138, 1, 0x00d0, 0x10, 14, 1),
+ PIN_FIELD_BASE(139, 139, 1, 0x00d0, 0x10, 15, 1),
+ PIN_FIELD_BASE(140, 140, 1, 0x00d0, 0x10, 16, 1),
+ PIN_FIELD_BASE(141, 141, 1, 0x00d0, 0x10, 3, 1),
+ PIN_FIELD_BASE(142, 142, 1, 0x00d0, 0x10, 4, 1),
+ PIN_FIELD_BASE(143, 143, 1, 0x00d0, 0x10, 5, 1),
+ PIN_FIELD_BASE(144, 144, 1, 0x00d0, 0x10, 6, 1),
+ PIN_FIELD_BASE(145, 145, 1, 0x00d0, 0x10, 7, 1),
+ PIN_FIELD_BASE(146, 146, 1, 0x00d0, 0x10, 8, 1),
+ PIN_FIELD_BASE(147, 147, 1, 0x00d0, 0x10, 18, 1),
+ PIN_FIELD_BASE(148, 148, 1, 0x00d0, 0x10, 19, 1),
+ PIN_FIELD_BASE(149, 149, 1, 0x00d0, 0x10, 17, 1),
+ PIN_FIELD_BASE(150, 150, 1, 0x00d0, 0x10, 0, 1),
+ PIN_FIELD_BASE(151, 151, 2, 0x00c0, 0x10, 9, 1),
+ PIN_FIELD_BASE(152, 152, 2, 0x00c0, 0x10, 8, 1),
+ PIN_FIELD_BASE(153, 153, 2, 0x00c0, 0x10, 7, 1),
+ PIN_FIELD_BASE(154, 154, 2, 0x00c0, 0x10, 6, 1),
+ PIN_FIELD_BASE(155, 155, 2, 0x00c0, 0x10, 11, 1),
+ PIN_FIELD_BASE(156, 156, 2, 0x00c0, 0x10, 1, 1),
+ PIN_FIELD_BASE(157, 157, 2, 0x00c0, 0x10, 0, 1),
+ PIN_FIELD_BASE(158, 158, 2, 0x00c0, 0x10, 5, 1),
+ PIN_FIELD_BASE(159, 159, 2, 0x00c0, 0x10, 4, 1),
+ PIN_FIELD_BASE(160, 160, 2, 0x00c0, 0x10, 3, 1),
+ PIN_FIELD_BASE(161, 161, 2, 0x00c0, 0x10, 2, 1),
+ PIN_FIELD_BASE(162, 162, 2, 0x00c0, 0x10, 10, 1),
+ PIN_FIELD_BASE(163, 163, 4, 0x0070, 0x10, 1, 1),
+ PIN_FIELD_BASE(164, 164, 4, 0x0070, 0x10, 0, 1),
+ PIN_FIELD_BASE(165, 165, 4, 0x0070, 0x10, 2, 1),
+ PIN_FIELD_BASE(166, 166, 4, 0x0070, 0x10, 3, 1),
+ PIN_FIELD_BASE(167, 167, 4, 0x0070, 0x10, 4, 1),
+ PIN_FIELD_BASE(168, 168, 4, 0x0070, 0x10, 5, 1),
+ PIN_FIELD_BASE(169, 169, 3, 0x0060, 0x10, 1, 1),
+ PIN_FIELD_BASE(170, 170, 3, 0x0060, 0x10, 0, 1),
+ PIN_FIELD_BASE(171, 171, 3, 0x0060, 0x10, 2, 1),
+ PIN_FIELD_BASE(172, 172, 3, 0x0060, 0x10, 3, 1),
+ PIN_FIELD_BASE(173, 173, 3, 0x0060, 0x10, 4, 1),
+ PIN_FIELD_BASE(174, 174, 3, 0x0060, 0x10, 5, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_r0_range[] = {
+ PIN_FIELD_BASE(42, 42, 2, 0x00f0, 0x10, 12, 1),
+ PIN_FIELD_BASE(43, 43, 2, 0x00f0, 0x10, 13, 1),
+ PIN_FIELD_BASE(44, 44, 2, 0x00f0, 0x10, 14, 1),
+ PIN_FIELD_BASE(45, 45, 2, 0x00f0, 0x10, 15, 1),
+ PIN_FIELD_BASE(131, 131, 1, 0x0100, 0x10, 1, 1),
+ PIN_FIELD_BASE(132, 132, 1, 0x0100, 0x10, 2, 1),
+ PIN_FIELD_BASE(133, 133, 1, 0x0100, 0x10, 9, 1),
+ PIN_FIELD_BASE(134, 134, 1, 0x0100, 0x10, 10, 1),
+ PIN_FIELD_BASE(135, 135, 1, 0x0100, 0x10, 11, 1),
+ PIN_FIELD_BASE(136, 136, 1, 0x0100, 0x10, 12, 1),
+ PIN_FIELD_BASE(137, 137, 1, 0x0100, 0x10, 13, 1),
+ PIN_FIELD_BASE(138, 138, 1, 0x0100, 0x10, 14, 1),
+ PIN_FIELD_BASE(139, 139, 1, 0x0100, 0x10, 15, 1),
+ PIN_FIELD_BASE(140, 140, 1, 0x0100, 0x10, 16, 1),
+ PIN_FIELD_BASE(141, 141, 1, 0x0100, 0x10, 3, 1),
+ PIN_FIELD_BASE(142, 142, 1, 0x0100, 0x10, 4, 1),
+ PIN_FIELD_BASE(143, 143, 1, 0x0100, 0x10, 5, 1),
+ PIN_FIELD_BASE(144, 144, 1, 0x0100, 0x10, 6, 1),
+ PIN_FIELD_BASE(145, 145, 1, 0x0100, 0x10, 7, 1),
+ PIN_FIELD_BASE(146, 146, 1, 0x0100, 0x10, 8, 1),
+ PIN_FIELD_BASE(147, 147, 1, 0x0100, 0x10, 18, 1),
+ PIN_FIELD_BASE(148, 148, 1, 0x0100, 0x10, 19, 1),
+ PIN_FIELD_BASE(149, 149, 1, 0x0100, 0x10, 17, 1),
+ PIN_FIELD_BASE(150, 150, 1, 0x0100, 0x10, 0, 1),
+ PIN_FIELD_BASE(151, 151, 2, 0x00f0, 0x10, 9, 1),
+ PIN_FIELD_BASE(152, 152, 2, 0x00f0, 0x10, 8, 1),
+ PIN_FIELD_BASE(153, 153, 2, 0x00f0, 0x10, 7, 1),
+ PIN_FIELD_BASE(154, 154, 2, 0x00f0, 0x10, 6, 1),
+ PIN_FIELD_BASE(155, 155, 2, 0x00f0, 0x10, 11, 1),
+ PIN_FIELD_BASE(156, 156, 2, 0x00f0, 0x10, 1, 1),
+ PIN_FIELD_BASE(157, 157, 2, 0x00f0, 0x10, 0, 1),
+ PIN_FIELD_BASE(158, 158, 2, 0x00f0, 0x10, 5, 1),
+ PIN_FIELD_BASE(159, 159, 2, 0x00f0, 0x10, 4, 1),
+ PIN_FIELD_BASE(160, 160, 2, 0x00f0, 0x10, 3, 1),
+ PIN_FIELD_BASE(161, 161, 2, 0x00f0, 0x10, 2, 1),
+ PIN_FIELD_BASE(162, 162, 2, 0x00f0, 0x10, 10, 1),
+ PIN_FIELD_BASE(163, 163, 4, 0x0090, 0x10, 1, 1),
+ PIN_FIELD_BASE(164, 164, 4, 0x0090, 0x10, 0, 1),
+ PIN_FIELD_BASE(165, 165, 4, 0x0090, 0x10, 2, 1),
+ PIN_FIELD_BASE(166, 166, 4, 0x0090, 0x10, 3, 1),
+ PIN_FIELD_BASE(167, 167, 4, 0x0090, 0x10, 4, 1),
+ PIN_FIELD_BASE(168, 168, 4, 0x0090, 0x10, 5, 1),
+ PIN_FIELD_BASE(169, 169, 3, 0x0080, 0x10, 1, 1),
+ PIN_FIELD_BASE(170, 170, 3, 0x0080, 0x10, 0, 1),
+ PIN_FIELD_BASE(171, 171, 3, 0x0080, 0x10, 2, 1),
+ PIN_FIELD_BASE(172, 172, 3, 0x0080, 0x10, 3, 1),
+ PIN_FIELD_BASE(173, 173, 3, 0x0080, 0x10, 4, 1),
+ PIN_FIELD_BASE(174, 174, 3, 0x0080, 0x10, 5, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_r1_range[] = {
+ PIN_FIELD_BASE(42, 42, 2, 0x0100, 0x10, 12, 1),
+ PIN_FIELD_BASE(43, 43, 2, 0x0100, 0x10, 13, 1),
+ PIN_FIELD_BASE(44, 44, 2, 0x0100, 0x10, 14, 1),
+ PIN_FIELD_BASE(45, 45, 2, 0x0100, 0x10, 15, 1),
+ PIN_FIELD_BASE(131, 131, 1, 0x0110, 0x10, 1, 1),
+ PIN_FIELD_BASE(132, 132, 1, 0x0110, 0x10, 2, 1),
+ PIN_FIELD_BASE(133, 133, 1, 0x0110, 0x10, 9, 1),
+ PIN_FIELD_BASE(134, 134, 1, 0x0110, 0x10, 10, 1),
+ PIN_FIELD_BASE(135, 135, 1, 0x0110, 0x10, 11, 1),
+ PIN_FIELD_BASE(136, 136, 1, 0x0110, 0x10, 12, 1),
+ PIN_FIELD_BASE(137, 137, 1, 0x0110, 0x10, 13, 1),
+ PIN_FIELD_BASE(138, 138, 1, 0x0110, 0x10, 14, 1),
+ PIN_FIELD_BASE(139, 139, 1, 0x0110, 0x10, 15, 1),
+ PIN_FIELD_BASE(140, 140, 1, 0x0110, 0x10, 16, 1),
+ PIN_FIELD_BASE(141, 141, 1, 0x0110, 0x10, 3, 1),
+ PIN_FIELD_BASE(142, 142, 1, 0x0110, 0x10, 4, 1),
+ PIN_FIELD_BASE(143, 143, 1, 0x0110, 0x10, 5, 1),
+ PIN_FIELD_BASE(144, 144, 1, 0x0110, 0x10, 6, 1),
+ PIN_FIELD_BASE(145, 145, 1, 0x0110, 0x10, 7, 1),
+ PIN_FIELD_BASE(146, 146, 1, 0x0110, 0x10, 8, 1),
+ PIN_FIELD_BASE(147, 147, 1, 0x0110, 0x10, 18, 1),
+ PIN_FIELD_BASE(148, 148, 1, 0x0110, 0x10, 19, 1),
+ PIN_FIELD_BASE(149, 149, 1, 0x0110, 0x10, 17, 1),
+ PIN_FIELD_BASE(150, 150, 1, 0x0110, 0x10, 0, 1),
+ PIN_FIELD_BASE(151, 151, 2, 0x0100, 0x10, 9, 1),
+ PIN_FIELD_BASE(152, 152, 2, 0x0100, 0x10, 8, 1),
+ PIN_FIELD_BASE(153, 153, 2, 0x0100, 0x10, 7, 1),
+ PIN_FIELD_BASE(154, 154, 2, 0x0100, 0x10, 6, 1),
+ PIN_FIELD_BASE(155, 155, 2, 0x0100, 0x10, 11, 1),
+ PIN_FIELD_BASE(156, 156, 2, 0x0100, 0x10, 1, 1),
+ PIN_FIELD_BASE(157, 157, 2, 0x0100, 0x10, 0, 1),
+ PIN_FIELD_BASE(158, 158, 2, 0x0100, 0x10, 5, 1),
+ PIN_FIELD_BASE(159, 159, 2, 0x0100, 0x10, 4, 1),
+ PIN_FIELD_BASE(160, 160, 2, 0x0100, 0x10, 3, 1),
+ PIN_FIELD_BASE(161, 161, 2, 0x0100, 0x10, 2, 1),
+ PIN_FIELD_BASE(162, 162, 2, 0x0100, 0x10, 10, 1),
+ PIN_FIELD_BASE(163, 163, 4, 0x00a0, 0x10, 1, 1),
+ PIN_FIELD_BASE(164, 164, 4, 0x00a0, 0x10, 0, 1),
+ PIN_FIELD_BASE(165, 165, 4, 0x00a0, 0x10, 2, 1),
+ PIN_FIELD_BASE(166, 166, 4, 0x00a0, 0x10, 3, 1),
+ PIN_FIELD_BASE(167, 167, 4, 0x00a0, 0x10, 4, 1),
+ PIN_FIELD_BASE(168, 168, 4, 0x00a0, 0x10, 5, 1),
+ PIN_FIELD_BASE(169, 169, 3, 0x0090, 0x10, 1, 1),
+ PIN_FIELD_BASE(170, 170, 3, 0x0090, 0x10, 0, 1),
+ PIN_FIELD_BASE(171, 171, 3, 0x0090, 0x10, 2, 1),
+ PIN_FIELD_BASE(172, 172, 3, 0x0090, 0x10, 3, 1),
+ PIN_FIELD_BASE(173, 173, 3, 0x0090, 0x10, 4, 1),
+ PIN_FIELD_BASE(174, 174, 3, 0x0090, 0x10, 5, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_pu_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x00e0, 0x10, 6, 1),
+ PIN_FIELD_BASE(1, 1, 1, 0x00e0, 0x10, 7, 1),
+ PIN_FIELD_BASE(2, 2, 1, 0x00e0, 0x10, 8, 1),
+ PIN_FIELD_BASE(3, 3, 1, 0x00e0, 0x10, 9, 1),
+ PIN_FIELD_BASE(4, 4, 1, 0x00e0, 0x10, 10, 1),
+ PIN_FIELD_BASE(5, 5, 1, 0x00e0, 0x10, 11, 1),
+ PIN_FIELD_BASE(6, 6, 1, 0x00e0, 0x10, 12, 1),
+ PIN_FIELD_BASE(7, 7, 1, 0x00e0, 0x10, 13, 1),
+ PIN_FIELD_BASE(8, 8, 1, 0x00e0, 0x10, 14, 1),
+ PIN_FIELD_BASE(9, 9, 1, 0x00e0, 0x10, 15, 1),
+ PIN_FIELD_BASE(10, 10, 1, 0x00e0, 0x10, 16, 1),
+ PIN_FIELD_BASE(11, 11, 1, 0x00e0, 0x10, 17, 1),
+ PIN_FIELD_BASE(12, 12, 2, 0x00d0, 0x10, 12, 1),
+ PIN_FIELD_BASE(13, 13, 2, 0x00d0, 0x10, 13, 1),
+ PIN_FIELD_BASE(14, 14, 2, 0x00d0, 0x10, 14, 1),
+ PIN_FIELD_BASE(15, 15, 2, 0x00d0, 0x10, 15, 1),
+ PIN_FIELD_BASE(16, 16, 3, 0x0070, 0x10, 1, 1),
+ PIN_FIELD_BASE(17, 17, 3, 0x0070, 0x10, 2, 1),
+ PIN_FIELD_BASE(18, 18, 4, 0x0080, 0x10, 3, 1),
+ PIN_FIELD_BASE(19, 19, 4, 0x0080, 0x10, 5, 1),
+ PIN_FIELD_BASE(20, 20, 4, 0x0080, 0x10, 4, 1),
+ PIN_FIELD_BASE(21, 21, 4, 0x0080, 0x10, 6, 1),
+ PIN_FIELD_BASE(22, 22, 4, 0x0080, 0x10, 0, 1),
+ PIN_FIELD_BASE(23, 23, 4, 0x0080, 0x10, 1, 1),
+ PIN_FIELD_BASE(24, 24, 4, 0x0080, 0x10, 2, 1),
+ PIN_FIELD_BASE(25, 25, 1, 0x00e0, 0x10, 3, 1),
+ PIN_FIELD_BASE(26, 26, 1, 0x00e0, 0x10, 2, 1),
+ PIN_FIELD_BASE(27, 27, 1, 0x00e0, 0x10, 5, 1),
+ PIN_FIELD_BASE(28, 28, 1, 0x00e0, 0x10, 4, 1),
+ PIN_FIELD_BASE(29, 29, 1, 0x00e0, 0x10, 0, 1),
+ PIN_FIELD_BASE(30, 30, 1, 0x00e0, 0x10, 1, 1),
+ PIN_FIELD_BASE(31, 31, 1, 0x00f0, 0x10, 11, 1),
+ PIN_FIELD_BASE(32, 32, 1, 0x00f0, 0x10, 10, 1),
+ PIN_FIELD_BASE(33, 33, 1, 0x00f0, 0x10, 13, 1),
+ PIN_FIELD_BASE(34, 34, 1, 0x00f0, 0x10, 12, 1),
+ PIN_FIELD_BASE(35, 35, 1, 0x00f0, 0x10, 15, 1),
+ PIN_FIELD_BASE(36, 36, 1, 0x00f0, 0x10, 14, 1),
+ PIN_FIELD_BASE(37, 37, 1, 0x00e0, 0x10, 21, 1),
+ PIN_FIELD_BASE(38, 38, 1, 0x00e0, 0x10, 18, 1),
+ PIN_FIELD_BASE(39, 39, 1, 0x00e0, 0x10, 19, 1),
+ PIN_FIELD_BASE(40, 40, 1, 0x00e0, 0x10, 20, 1),
+ PIN_FIELD_BASE(41, 41, 1, 0x00e0, 0x10, 22, 1),
+ PIN_FIELD_BASE(46, 46, 3, 0x0070, 0x10, 0, 1),
+ PIN_FIELD_BASE(47, 47, 1, 0x00e0, 0x10, 25, 1),
+ PIN_FIELD_BASE(48, 48, 1, 0x00e0, 0x10, 24, 1),
+ PIN_FIELD_BASE(49, 49, 1, 0x00e0, 0x10, 23, 1),
+ PIN_FIELD_BASE(50, 50, 3, 0x0070, 0x10, 5, 1),
+ PIN_FIELD_BASE(51, 51, 3, 0x0070, 0x10, 4, 1),
+ PIN_FIELD_BASE(52, 52, 3, 0x0070, 0x10, 3, 1),
+ PIN_FIELD_BASE(53, 53, 3, 0x0070, 0x10, 6, 1),
+ PIN_FIELD_BASE(54, 54, 3, 0x0070, 0x10, 7, 1),
+ PIN_FIELD_BASE(55, 55, 1, 0x00e0, 0x10, 26, 1),
+ PIN_FIELD_BASE(56, 56, 1, 0x00e0, 0x10, 29, 1),
+ PIN_FIELD_BASE(57, 57, 2, 0x00e0, 0x10, 6, 1),
+ PIN_FIELD_BASE(58, 58, 2, 0x00e0, 0x10, 9, 1),
+ PIN_FIELD_BASE(59, 59, 1, 0x00e0, 0x10, 27, 1),
+ PIN_FIELD_BASE(60, 60, 1, 0x00e0, 0x10, 30, 1),
+ PIN_FIELD_BASE(61, 61, 1, 0x00e0, 0x10, 28, 1),
+ PIN_FIELD_BASE(62, 62, 1, 0x00e0, 0x10, 31, 1),
+ PIN_FIELD_BASE(63, 63, 2, 0x00e0, 0x10, 7, 1),
+ PIN_FIELD_BASE(64, 64, 2, 0x00e0, 0x10, 10, 1),
+ PIN_FIELD_BASE(65, 65, 4, 0x0080, 0x10, 7, 1),
+ PIN_FIELD_BASE(66, 66, 4, 0x0080, 0x10, 9, 1),
+ PIN_FIELD_BASE(67, 67, 4, 0x0080, 0x10, 8, 1),
+ PIN_FIELD_BASE(68, 68, 4, 0x0080, 0x10, 10, 1),
+ PIN_FIELD_BASE(69, 69, 1, 0x00f0, 0x10, 1, 1),
+ PIN_FIELD_BASE(70, 70, 1, 0x00f0, 0x10, 0, 1),
+ PIN_FIELD_BASE(71, 71, 1, 0x00f0, 0x10, 5, 1),
+ PIN_FIELD_BASE(72, 72, 1, 0x00f0, 0x10, 4, 1),
+ PIN_FIELD_BASE(73, 73, 1, 0x00f0, 0x10, 2, 1),
+ PIN_FIELD_BASE(74, 74, 1, 0x00f0, 0x10, 3, 1),
+ PIN_FIELD_BASE(75, 75, 1, 0x00f0, 0x10, 7, 1),
+ PIN_FIELD_BASE(76, 76, 1, 0x00f0, 0x10, 6, 1),
+ PIN_FIELD_BASE(77, 77, 1, 0x00f0, 0x10, 9, 1),
+ PIN_FIELD_BASE(78, 78, 1, 0x00f0, 0x10, 8, 1),
+ PIN_FIELD_BASE(79, 79, 4, 0x0080, 0x10, 12, 1),
+ PIN_FIELD_BASE(80, 80, 4, 0x0080, 0x10, 11, 1),
+ PIN_FIELD_BASE(81, 81, 4, 0x0080, 0x10, 14, 1),
+ PIN_FIELD_BASE(82, 82, 4, 0x0080, 0x10, 13, 1),
+ PIN_FIELD_BASE(83, 83, 2, 0x00e0, 0x10, 16, 1),
+ PIN_FIELD_BASE(84, 84, 2, 0x00e0, 0x10, 15, 1),
+ PIN_FIELD_BASE(85, 85, 2, 0x00e0, 0x10, 17, 1),
+ PIN_FIELD_BASE(86, 86, 2, 0x00e0, 0x10, 19, 1),
+ PIN_FIELD_BASE(87, 87, 2, 0x00e0, 0x10, 18, 1),
+ PIN_FIELD_BASE(88, 88, 2, 0x00e0, 0x10, 20, 1),
+ PIN_FIELD_BASE(89, 89, 2, 0x00e0, 0x10, 22, 1),
+ PIN_FIELD_BASE(90, 90, 2, 0x00e0, 0x10, 21, 1),
+ PIN_FIELD_BASE(91, 91, 2, 0x00e0, 0x10, 23, 1),
+ PIN_FIELD_BASE(92, 92, 2, 0x00e0, 0x10, 3, 1),
+ PIN_FIELD_BASE(93, 93, 2, 0x00e0, 0x10, 2, 1),
+ PIN_FIELD_BASE(94, 94, 2, 0x00e0, 0x10, 5, 1),
+ PIN_FIELD_BASE(95, 95, 2, 0x00e0, 0x10, 4, 1),
+ PIN_FIELD_BASE(96, 96, 2, 0x00d0, 0x10, 31, 1),
+ PIN_FIELD_BASE(97, 97, 2, 0x00e0, 0x10, 0, 1),
+ PIN_FIELD_BASE(98, 98, 2, 0x00e0, 0x10, 8, 1),
+ PIN_FIELD_BASE(99, 99, 2, 0x00d0, 0x10, 30, 1),
+ PIN_FIELD_BASE(100, 100, 2, 0x00e0, 0x10, 1, 1),
+ PIN_FIELD_BASE(101, 101, 2, 0x00d0, 0x10, 0, 1),
+ PIN_FIELD_BASE(102, 102, 2, 0x00d0, 0x10, 5, 1),
+ PIN_FIELD_BASE(103, 103, 2, 0x00d0, 0x10, 3, 1),
+ PIN_FIELD_BASE(104, 104, 2, 0x00d0, 0x10, 4, 1),
+ PIN_FIELD_BASE(105, 105, 2, 0x00d0, 0x10, 1, 1),
+ PIN_FIELD_BASE(106, 106, 2, 0x00d0, 0x10, 2, 1),
+ PIN_FIELD_BASE(107, 107, 2, 0x00d0, 0x10, 21, 1),
+ PIN_FIELD_BASE(108, 108, 2, 0x00d0, 0x10, 16, 1),
+ PIN_FIELD_BASE(109, 109, 2, 0x00d0, 0x10, 22, 1),
+ PIN_FIELD_BASE(110, 110, 2, 0x00d0, 0x10, 17, 1),
+ PIN_FIELD_BASE(111, 111, 2, 0x00d0, 0x10, 18, 1),
+ PIN_FIELD_BASE(112, 112, 2, 0x00d0, 0x10, 19, 1),
+ PIN_FIELD_BASE(113, 113, 2, 0x00d0, 0x10, 20, 1),
+ PIN_FIELD_BASE(114, 114, 2, 0x00d0, 0x10, 28, 1),
+ PIN_FIELD_BASE(115, 115, 2, 0x00d0, 0x10, 23, 1),
+ PIN_FIELD_BASE(116, 116, 2, 0x00d0, 0x10, 29, 1),
+ PIN_FIELD_BASE(117, 117, 2, 0x00d0, 0x10, 24, 1),
+ PIN_FIELD_BASE(118, 118, 2, 0x00d0, 0x10, 25, 1),
+ PIN_FIELD_BASE(119, 119, 2, 0x00d0, 0x10, 26, 1),
+ PIN_FIELD_BASE(120, 120, 2, 0x00d0, 0x10, 27, 1),
+ PIN_FIELD_BASE(121, 121, 3, 0x0070, 0x10, 8, 1),
+ PIN_FIELD_BASE(122, 122, 3, 0x0070, 0x10, 11, 1),
+ PIN_FIELD_BASE(123, 123, 3, 0x0070, 0x10, 10, 1),
+ PIN_FIELD_BASE(124, 124, 3, 0x0070, 0x10, 9, 1),
+ PIN_FIELD_BASE(125, 125, 2, 0x00d0, 0x10, 6, 1),
+ PIN_FIELD_BASE(126, 126, 2, 0x00d0, 0x10, 7, 1),
+ PIN_FIELD_BASE(127, 127, 2, 0x00d0, 0x10, 8, 1),
+ PIN_FIELD_BASE(128, 128, 2, 0x00d0, 0x10, 9, 1),
+ PIN_FIELD_BASE(129, 129, 2, 0x00d0, 0x10, 10, 1),
+ PIN_FIELD_BASE(130, 130, 2, 0x00d0, 0x10, 11, 1),
+ PIN_FIELD_BASE(175, 175, 2, 0x00e0, 0x10, 11, 1),
+ PIN_FIELD_BASE(176, 176, 2, 0x00e0, 0x10, 12, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_pd_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x00b0, 0x10, 6, 1),
+ PIN_FIELD_BASE(1, 1, 1, 0x00b0, 0x10, 7, 1),
+ PIN_FIELD_BASE(2, 2, 1, 0x00b0, 0x10, 8, 1),
+ PIN_FIELD_BASE(3, 3, 1, 0x00b0, 0x10, 9, 1),
+ PIN_FIELD_BASE(4, 4, 1, 0x00b0, 0x10, 10, 1),
+ PIN_FIELD_BASE(5, 5, 1, 0x00b0, 0x10, 11, 1),
+ PIN_FIELD_BASE(6, 6, 1, 0x00b0, 0x10, 12, 1),
+ PIN_FIELD_BASE(7, 7, 1, 0x00b0, 0x10, 13, 1),
+ PIN_FIELD_BASE(8, 8, 1, 0x00b0, 0x10, 14, 1),
+ PIN_FIELD_BASE(9, 9, 1, 0x00b0, 0x10, 15, 1),
+ PIN_FIELD_BASE(10, 10, 1, 0x00b0, 0x10, 16, 1),
+ PIN_FIELD_BASE(11, 11, 1, 0x00b0, 0x10, 17, 1),
+ PIN_FIELD_BASE(12, 12, 2, 0x00a0, 0x10, 12, 1),
+ PIN_FIELD_BASE(13, 13, 2, 0x00a0, 0x10, 13, 1),
+ PIN_FIELD_BASE(14, 14, 2, 0x00a0, 0x10, 14, 1),
+ PIN_FIELD_BASE(15, 15, 2, 0x00a0, 0x10, 15, 1),
+ PIN_FIELD_BASE(16, 16, 3, 0x0050, 0x10, 1, 1),
+ PIN_FIELD_BASE(17, 17, 3, 0x0050, 0x10, 2, 1),
+ PIN_FIELD_BASE(18, 18, 4, 0x0060, 0x10, 3, 1),
+ PIN_FIELD_BASE(19, 19, 4, 0x0060, 0x10, 5, 1),
+ PIN_FIELD_BASE(20, 20, 4, 0x0060, 0x10, 4, 1),
+ PIN_FIELD_BASE(21, 21, 4, 0x0060, 0x10, 6, 1),
+ PIN_FIELD_BASE(22, 22, 4, 0x0060, 0x10, 0, 1),
+ PIN_FIELD_BASE(23, 23, 4, 0x0060, 0x10, 1, 1),
+ PIN_FIELD_BASE(24, 24, 4, 0x0060, 0x10, 2, 1),
+ PIN_FIELD_BASE(25, 25, 1, 0x00b0, 0x10, 3, 1),
+ PIN_FIELD_BASE(26, 26, 1, 0x00b0, 0x10, 2, 1),
+ PIN_FIELD_BASE(27, 27, 1, 0x00b0, 0x10, 5, 1),
+ PIN_FIELD_BASE(28, 28, 1, 0x00b0, 0x10, 4, 1),
+ PIN_FIELD_BASE(29, 29, 1, 0x00b0, 0x10, 0, 1),
+ PIN_FIELD_BASE(30, 30, 1, 0x00b0, 0x10, 1, 1),
+ PIN_FIELD_BASE(31, 31, 1, 0x00c0, 0x10, 11, 1),
+ PIN_FIELD_BASE(32, 32, 1, 0x00c0, 0x10, 10, 1),
+ PIN_FIELD_BASE(33, 33, 1, 0x00c0, 0x10, 13, 1),
+ PIN_FIELD_BASE(34, 34, 1, 0x00c0, 0x10, 12, 1),
+ PIN_FIELD_BASE(35, 35, 1, 0x00c0, 0x10, 15, 1),
+ PIN_FIELD_BASE(36, 36, 1, 0x00c0, 0x10, 14, 1),
+ PIN_FIELD_BASE(37, 37, 1, 0x00b0, 0x10, 21, 1),
+ PIN_FIELD_BASE(38, 38, 1, 0x00b0, 0x10, 18, 1),
+ PIN_FIELD_BASE(39, 39, 1, 0x00b0, 0x10, 19, 1),
+ PIN_FIELD_BASE(40, 40, 1, 0x00b0, 0x10, 20, 1),
+ PIN_FIELD_BASE(41, 41, 1, 0x00b0, 0x10, 22, 1),
+ PIN_FIELD_BASE(46, 46, 3, 0x0050, 0x10, 0, 1),
+ PIN_FIELD_BASE(47, 47, 1, 0x00b0, 0x10, 25, 1),
+ PIN_FIELD_BASE(48, 48, 1, 0x00b0, 0x10, 24, 1),
+ PIN_FIELD_BASE(49, 49, 1, 0x00b0, 0x10, 23, 1),
+ PIN_FIELD_BASE(50, 50, 3, 0x0050, 0x10, 5, 1),
+ PIN_FIELD_BASE(51, 51, 3, 0x0050, 0x10, 4, 1),
+ PIN_FIELD_BASE(52, 52, 3, 0x0050, 0x10, 3, 1),
+ PIN_FIELD_BASE(53, 53, 3, 0x0050, 0x10, 6, 1),
+ PIN_FIELD_BASE(54, 54, 3, 0x0050, 0x10, 7, 1),
+ PIN_FIELD_BASE(55, 55, 1, 0x00b0, 0x10, 26, 1),
+ PIN_FIELD_BASE(56, 56, 1, 0x00b0, 0x10, 29, 1),
+ PIN_FIELD_BASE(57, 57, 2, 0x00b0, 0x10, 6, 1),
+ PIN_FIELD_BASE(58, 58, 2, 0x00b0, 0x10, 9, 1),
+ PIN_FIELD_BASE(59, 59, 1, 0x00b0, 0x10, 27, 1),
+ PIN_FIELD_BASE(60, 60, 1, 0x00b0, 0x10, 30, 1),
+ PIN_FIELD_BASE(61, 61, 1, 0x00b0, 0x10, 28, 1),
+ PIN_FIELD_BASE(62, 62, 1, 0x00b0, 0x10, 31, 1),
+ PIN_FIELD_BASE(63, 63, 2, 0x00b0, 0x10, 7, 1),
+ PIN_FIELD_BASE(64, 64, 2, 0x00b0, 0x10, 10, 1),
+ PIN_FIELD_BASE(65, 65, 4, 0x0060, 0x10, 7, 1),
+ PIN_FIELD_BASE(66, 66, 4, 0x0060, 0x10, 9, 1),
+ PIN_FIELD_BASE(67, 67, 4, 0x0060, 0x10, 8, 1),
+ PIN_FIELD_BASE(68, 68, 4, 0x0060, 0x10, 10, 1),
+ PIN_FIELD_BASE(69, 69, 1, 0x00c0, 0x10, 1, 1),
+ PIN_FIELD_BASE(70, 70, 1, 0x00c0, 0x10, 0, 1),
+ PIN_FIELD_BASE(71, 71, 1, 0x00c0, 0x10, 5, 1),
+ PIN_FIELD_BASE(72, 72, 1, 0x00c0, 0x10, 4, 1),
+ PIN_FIELD_BASE(73, 73, 1, 0x00c0, 0x10, 2, 1),
+ PIN_FIELD_BASE(74, 74, 1, 0x00c0, 0x10, 3, 1),
+ PIN_FIELD_BASE(75, 75, 1, 0x00c0, 0x10, 7, 1),
+ PIN_FIELD_BASE(76, 76, 1, 0x00c0, 0x10, 6, 1),
+ PIN_FIELD_BASE(77, 77, 1, 0x00c0, 0x10, 9, 1),
+ PIN_FIELD_BASE(78, 78, 1, 0x00c0, 0x10, 8, 1),
+ PIN_FIELD_BASE(79, 79, 4, 0x0060, 0x10, 12, 1),
+ PIN_FIELD_BASE(80, 80, 4, 0x0060, 0x10, 11, 1),
+ PIN_FIELD_BASE(81, 81, 4, 0x0060, 0x10, 14, 1),
+ PIN_FIELD_BASE(82, 82, 4, 0x0060, 0x10, 13, 1),
+ PIN_FIELD_BASE(83, 83, 2, 0x00b0, 0x10, 16, 1),
+ PIN_FIELD_BASE(84, 84, 2, 0x00b0, 0x10, 15, 1),
+ PIN_FIELD_BASE(85, 85, 2, 0x00b0, 0x10, 17, 1),
+ PIN_FIELD_BASE(86, 86, 2, 0x00b0, 0x10, 19, 1),
+ PIN_FIELD_BASE(87, 87, 2, 0x00b0, 0x10, 18, 1),
+ PIN_FIELD_BASE(88, 88, 2, 0x00b0, 0x10, 20, 1),
+ PIN_FIELD_BASE(89, 89, 2, 0x00b0, 0x10, 22, 1),
+ PIN_FIELD_BASE(90, 90, 2, 0x00b0, 0x10, 21, 1),
+ PIN_FIELD_BASE(91, 91, 2, 0x00b0, 0x10, 23, 1),
+ PIN_FIELD_BASE(92, 92, 2, 0x00b0, 0x10, 3, 1),
+ PIN_FIELD_BASE(93, 93, 2, 0x00b0, 0x10, 2, 1),
+ PIN_FIELD_BASE(94, 94, 2, 0x00b0, 0x10, 5, 1),
+ PIN_FIELD_BASE(95, 95, 2, 0x00b0, 0x10, 4, 1),
+ PIN_FIELD_BASE(96, 96, 2, 0x00a0, 0x10, 31, 1),
+ PIN_FIELD_BASE(97, 97, 2, 0x00b0, 0x10, 0, 1),
+ PIN_FIELD_BASE(98, 98, 2, 0x00b0, 0x10, 8, 1),
+ PIN_FIELD_BASE(99, 99, 2, 0x00a0, 0x10, 30, 1),
+ PIN_FIELD_BASE(100, 100, 2, 0x00b0, 0x10, 1, 1),
+ PIN_FIELD_BASE(101, 101, 2, 0x00a0, 0x10, 0, 1),
+ PIN_FIELD_BASE(102, 102, 2, 0x00a0, 0x10, 5, 1),
+ PIN_FIELD_BASE(103, 103, 2, 0x00a0, 0x10, 3, 1),
+ PIN_FIELD_BASE(104, 104, 2, 0x00a0, 0x10, 4, 1),
+ PIN_FIELD_BASE(105, 105, 2, 0x00a0, 0x10, 1, 1),
+ PIN_FIELD_BASE(106, 106, 2, 0x00a0, 0x10, 2, 1),
+ PIN_FIELD_BASE(107, 107, 2, 0x00a0, 0x10, 21, 1),
+ PIN_FIELD_BASE(108, 108, 2, 0x00a0, 0x10, 16, 1),
+ PIN_FIELD_BASE(109, 109, 2, 0x00a0, 0x10, 22, 1),
+ PIN_FIELD_BASE(110, 110, 2, 0x00a0, 0x10, 17, 1),
+ PIN_FIELD_BASE(111, 111, 2, 0x00a0, 0x10, 18, 1),
+ PIN_FIELD_BASE(112, 112, 2, 0x00a0, 0x10, 19, 1),
+ PIN_FIELD_BASE(113, 113, 2, 0x00a0, 0x10, 20, 1),
+ PIN_FIELD_BASE(114, 114, 2, 0x00a0, 0x10, 28, 1),
+ PIN_FIELD_BASE(115, 115, 2, 0x00a0, 0x10, 23, 1),
+ PIN_FIELD_BASE(116, 116, 2, 0x00a0, 0x10, 29, 1),
+ PIN_FIELD_BASE(117, 117, 2, 0x00a0, 0x10, 24, 1),
+ PIN_FIELD_BASE(118, 118, 2, 0x00a0, 0x10, 25, 1),
+ PIN_FIELD_BASE(119, 119, 2, 0x00a0, 0x10, 26, 1),
+ PIN_FIELD_BASE(120, 120, 2, 0x00a0, 0x10, 27, 1),
+ PIN_FIELD_BASE(121, 121, 3, 0x0050, 0x10, 8, 1),
+ PIN_FIELD_BASE(122, 122, 3, 0x0050, 0x10, 11, 1),
+ PIN_FIELD_BASE(123, 123, 3, 0x0050, 0x10, 10, 1),
+ PIN_FIELD_BASE(124, 124, 3, 0x0050, 0x10, 9, 1),
+ PIN_FIELD_BASE(125, 125, 2, 0x00a0, 0x10, 6, 1),
+ PIN_FIELD_BASE(126, 126, 2, 0x00a0, 0x10, 7, 1),
+ PIN_FIELD_BASE(127, 127, 2, 0x00a0, 0x10, 8, 1),
+ PIN_FIELD_BASE(128, 128, 2, 0x00a0, 0x10, 9, 1),
+ PIN_FIELD_BASE(129, 129, 2, 0x00a0, 0x10, 10, 1),
+ PIN_FIELD_BASE(130, 130, 2, 0x00a0, 0x10, 11, 1),
+ PIN_FIELD_BASE(175, 175, 2, 0x00b0, 0x10, 11, 1),
+ PIN_FIELD_BASE(176, 176, 2, 0x00b0, 0x10, 12, 1),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_drv_range[] = {
+ PIN_FIELD_BASE(0, 0, 1, 0x0000, 0x10, 24, 3),
+ PIN_FIELD_BASE(1, 1, 1, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(2, 2, 1, 0x0010, 0x10, 0, 3),
+ PIN_FIELD_BASE(3, 3, 1, 0x0010, 0x10, 3, 3),
+ PIN_FIELD_BASE(4, 4, 1, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(5, 5, 1, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(6, 6, 1, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(7, 7, 1, 0x0010, 0x10, 6, 3),
+ PIN_FIELD_BASE(8, 8, 1, 0x0010, 0x10, 9, 3),
+ PIN_FIELD_BASE(9, 9, 1, 0x0010, 0x10, 12, 3),
+ PIN_FIELD_BASE(10, 10, 1, 0x0010, 0x10, 15, 3),
+ PIN_FIELD_BASE(11, 11, 1, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(12, 12, 2, 0x0010, 0x10, 24, 3),
+ PIN_FIELD_BASE(13, 13, 2, 0x0010, 0x10, 27, 3),
+ PIN_FIELD_BASE(14, 14, 2, 0x0020, 0x10, 0, 3),
+ PIN_FIELD_BASE(15, 15, 2, 0x0020, 0x10, 3, 3),
+ PIN_FIELD_BASE(16, 16, 3, 0x0010, 0x10, 15, 3),
+ PIN_FIELD_BASE(17, 17, 3, 0x0010, 0x10, 15, 3),
+ PIN_FIELD_BASE(18, 18, 4, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(19, 19, 4, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(20, 20, 4, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(21, 21, 4, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(22, 22, 4, 0x0000, 0x10, 0, 3),
+ PIN_FIELD_BASE(23, 23, 4, 0x0000, 0x10, 3, 3),
+ PIN_FIELD_BASE(24, 24, 4, 0x0000, 0x10, 6, 3),
+ PIN_FIELD_BASE(25, 25, 1, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(26, 26, 1, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(27, 27, 1, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(28, 28, 1, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(29, 29, 1, 0x0020, 0x10, 3, 3),
+ PIN_FIELD_BASE(30, 30, 1, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(31, 31, 1, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(32, 32, 1, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(33, 33, 1, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(34, 34, 1, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(35, 35, 1, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(36, 36, 1, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(37, 37, 1, 0x0010, 0x10, 27, 3),
+ PIN_FIELD_BASE(38, 38, 1, 0x0010, 0x10, 18, 3),
+ PIN_FIELD_BASE(39, 39, 1, 0x0010, 0x10, 21, 3),
+ PIN_FIELD_BASE(40, 40, 1, 0x0010, 0x10, 24, 3),
+ PIN_FIELD_BASE(41, 41, 1, 0x0020, 0x10, 0, 3),
+ PIN_FIELD_BASE(42, 42, 2, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(43, 43, 2, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(44, 44, 2, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(45, 45, 2, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(46, 46, 3, 0x0010, 0x10, 15, 3),
+ PIN_FIELD_BASE(47, 47, 1, 0x0020, 0x10, 3, 3),
+ PIN_FIELD_BASE(48, 48, 1, 0x0020, 0x10, 3, 3),
+ PIN_FIELD_BASE(49, 49, 1, 0x0020, 0x10, 3, 3),
+ PIN_FIELD_BASE(50, 50, 3, 0x0000, 0x10, 6, 3),
+ PIN_FIELD_BASE(51, 51, 3, 0x0000, 0x10, 3, 3),
+ PIN_FIELD_BASE(52, 52, 3, 0x0000, 0x10, 0, 3),
+ PIN_FIELD_BASE(53, 53, 3, 0x0000, 0x10, 9, 3),
+ PIN_FIELD_BASE(54, 54, 3, 0x0000, 0x10, 12, 3),
+ PIN_FIELD_BASE(55, 55, 1, 0x0020, 0x10, 27, 3),
+ PIN_FIELD_BASE(56, 56, 1, 0x0030, 0x10, 6, 3),
+ PIN_FIELD_BASE(57, 57, 2, 0x0030, 0x10, 9, 3),
+ PIN_FIELD_BASE(58, 58, 2, 0x0030, 0x10, 15, 3),
+ PIN_FIELD_BASE(59, 59, 1, 0x0030, 0x10, 0, 3),
+ PIN_FIELD_BASE(60, 60, 1, 0x0030, 0x10, 9, 3),
+ PIN_FIELD_BASE(61, 61, 1, 0x0030, 0x10, 3, 3),
+ PIN_FIELD_BASE(62, 62, 1, 0x0030, 0x10, 12, 3),
+ PIN_FIELD_BASE(63, 63, 2, 0x0030, 0x10, 12, 3),
+ PIN_FIELD_BASE(64, 64, 2, 0x0030, 0x10, 18, 3),
+ PIN_FIELD_BASE(65, 65, 4, 0x0010, 0x10, 0, 3),
+ PIN_FIELD_BASE(66, 66, 4, 0x0010, 0x10, 6, 3),
+ PIN_FIELD_BASE(67, 67, 4, 0x0010, 0x10, 3, 3),
+ PIN_FIELD_BASE(68, 68, 4, 0x0010, 0x10, 9, 3),
+ PIN_FIELD_BASE(69, 69, 1, 0x0030, 0x10, 18, 3),
+ PIN_FIELD_BASE(70, 70, 1, 0x0030, 0x10, 15, 3),
+ PIN_FIELD_BASE(71, 71, 1, 0x0040, 0x10, 0, 3),
+ PIN_FIELD_BASE(72, 72, 1, 0x0030, 0x10, 27, 3),
+ PIN_FIELD_BASE(73, 73, 1, 0x0030, 0x10, 21, 3),
+ PIN_FIELD_BASE(74, 74, 1, 0x0030, 0x10, 24, 3),
+ PIN_FIELD_BASE(75, 75, 1, 0x0040, 0x10, 6, 3),
+ PIN_FIELD_BASE(76, 76, 1, 0x0040, 0x10, 3, 3),
+ PIN_FIELD_BASE(77, 77, 1, 0x0040, 0x10, 12, 3),
+ PIN_FIELD_BASE(78, 78, 1, 0x0040, 0x10, 9, 3),
+ PIN_FIELD_BASE(79, 79, 4, 0x0010, 0x10, 15, 3),
+ PIN_FIELD_BASE(80, 80, 4, 0x0010, 0x10, 12, 3),
+ PIN_FIELD_BASE(81, 81, 4, 0x0010, 0x10, 21, 3),
+ PIN_FIELD_BASE(82, 82, 4, 0x0010, 0x10, 18, 3),
+ PIN_FIELD_BASE(83, 83, 2, 0x0030, 0x10, 0, 3),
+ PIN_FIELD_BASE(84, 84, 2, 0x0020, 0x10, 27, 3),
+ PIN_FIELD_BASE(85, 85, 2, 0x0030, 0x10, 0, 3),
+ PIN_FIELD_BASE(86, 86, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(87, 87, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(88, 88, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(89, 89, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(90, 90, 2, 0x0030, 0x10, 0, 3),
+ PIN_FIELD_BASE(91, 91, 2, 0x0030, 0x10, 0, 3),
+ PIN_FIELD_BASE(92, 92, 2, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(93, 93, 2, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(94, 94, 2, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(95, 95, 2, 0x0020, 0x10, 9, 3),
+ PIN_FIELD_BASE(96, 96, 2, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(97, 97, 2, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(98, 98, 2, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(99, 99, 2, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(100, 100, 2, 0x0030, 0x10, 6, 3),
+ PIN_FIELD_BASE(101, 101, 2, 0x0000, 0x10, 0, 3),
+ PIN_FIELD_BASE(102, 102, 2, 0x0000, 0x10, 15, 3),
+ PIN_FIELD_BASE(103, 103, 2, 0x0000, 0x10, 9, 3),
+ PIN_FIELD_BASE(104, 104, 2, 0x0000, 0x10, 12, 3),
+ PIN_FIELD_BASE(105, 105, 2, 0x0000, 0x10, 3, 3),
+ PIN_FIELD_BASE(106, 106, 2, 0x0000, 0x10, 6, 3),
+ PIN_FIELD_BASE(107, 107, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(108, 108, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(109, 109, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(110, 110, 2, 0x0020, 0x10, 6, 3),
+ PIN_FIELD_BASE(111, 111, 2, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(112, 112, 2, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(113, 113, 2, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(114, 114, 2, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(115, 115, 2, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(116, 116, 2, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(117, 117, 2, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(118, 118, 2, 0x0020, 0x10, 12, 3),
+ PIN_FIELD_BASE(119, 119, 2, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(120, 120, 2, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(121, 121, 3, 0x0010, 0x10, 3, 3),
+ PIN_FIELD_BASE(122, 122, 3, 0x0010, 0x10, 12, 3),
+ PIN_FIELD_BASE(123, 123, 3, 0x0010, 0x10, 9, 3),
+ PIN_FIELD_BASE(124, 124, 3, 0x0010, 0x10, 6, 3),
+ PIN_FIELD_BASE(125, 125, 2, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(126, 126, 2, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(127, 127, 2, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(128, 128, 2, 0x0020, 0x10, 27, 3),
+ PIN_FIELD_BASE(129, 129, 2, 0x0020, 0x10, 27, 3),
+ PIN_FIELD_BASE(130, 130, 2, 0x0020, 0x10, 27, 3),
+ PIN_FIELD_BASE(131, 131, 1, 0x0000, 0x10, 0, 3),
+ PIN_FIELD_BASE(132, 132, 1, 0x0000, 0x10, 15, 3),
+ PIN_FIELD_BASE(133, 133, 1, 0x0000, 0x10, 18, 3),
+ PIN_FIELD_BASE(134, 134, 1, 0x0000, 0x10, 21, 3),
+ PIN_FIELD_BASE(135, 135, 1, 0x0020, 0x10, 15, 3),
+ PIN_FIELD_BASE(136, 136, 1, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(137, 137, 1, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(138, 138, 1, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(139, 139, 1, 0x0020, 0x10, 18, 3),
+ PIN_FIELD_BASE(140, 140, 1, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(141, 141, 1, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(142, 142, 1, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(143, 143, 1, 0x0000, 0x10, 3, 3),
+ PIN_FIELD_BASE(144, 144, 1, 0x0000, 0x10, 6, 3),
+ PIN_FIELD_BASE(145, 145, 1, 0x0000, 0x10, 9, 3),
+ PIN_FIELD_BASE(146, 146, 1, 0x0000, 0x10, 12, 3),
+ PIN_FIELD_BASE(147, 147, 1, 0x0020, 0x10, 21, 3),
+ PIN_FIELD_BASE(148, 148, 1, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(149, 149, 1, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(150, 150, 1, 0x0020, 0x10, 24, 3),
+ PIN_FIELD_BASE(151, 151, 2, 0x0010, 0x10, 15, 3),
+ PIN_FIELD_BASE(152, 152, 2, 0x0010, 0x10, 12, 3),
+ PIN_FIELD_BASE(153, 153, 2, 0x0010, 0x10, 9, 3),
+ PIN_FIELD_BASE(154, 154, 2, 0x0010, 0x10, 6, 3),
+ PIN_FIELD_BASE(155, 155, 2, 0x0010, 0x10, 21, 3),
+ PIN_FIELD_BASE(156, 156, 2, 0x0000, 0x10, 21, 3),
+ PIN_FIELD_BASE(157, 157, 2, 0x0000, 0x10, 18, 3),
+ PIN_FIELD_BASE(158, 158, 2, 0x0010, 0x10, 3, 3),
+ PIN_FIELD_BASE(159, 159, 2, 0x0010, 0x10, 0, 3),
+ PIN_FIELD_BASE(160, 160, 2, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(161, 161, 2, 0x0000, 0x10, 24, 3),
+ PIN_FIELD_BASE(162, 162, 2, 0x0010, 0x10, 18, 3),
+ PIN_FIELD_BASE(163, 163, 4, 0x0000, 0x10, 12, 3),
+ PIN_FIELD_BASE(164, 164, 4, 0x0000, 0x10, 9, 3),
+ PIN_FIELD_BASE(165, 165, 4, 0x0000, 0x10, 15, 3),
+ PIN_FIELD_BASE(166, 166, 4, 0x0000, 0x10, 18, 3),
+ PIN_FIELD_BASE(167, 167, 4, 0x0000, 0x10, 21, 3),
+ PIN_FIELD_BASE(168, 168, 4, 0x0000, 0x10, 24, 3),
+ PIN_FIELD_BASE(169, 169, 3, 0x0000, 0x10, 18, 3),
+ PIN_FIELD_BASE(170, 170, 3, 0x0000, 0x10, 15, 3),
+ PIN_FIELD_BASE(171, 171, 3, 0x0000, 0x10, 21, 3),
+ PIN_FIELD_BASE(172, 172, 3, 0x0000, 0x10, 24, 3),
+ PIN_FIELD_BASE(173, 173, 3, 0x0000, 0x10, 27, 3),
+ PIN_FIELD_BASE(174, 174, 3, 0x0010, 0x10, 0, 3),
+ PIN_FIELD_BASE(175, 175, 2, 0x0030, 0x10, 3, 3),
+ PIN_FIELD_BASE(176, 176, 2, 0x0030, 0x10, 3, 3),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_drv_adv_range[] = {
+ PIN_FIELD_BASE(53, 53, 3, 0x0020, 0x10, 0, 3),
+ PIN_FIELD_BASE(54, 54, 3, 0x0020, 0x10, 3, 3),
+ PIN_FIELD_BASE(55, 55, 1, 0x0060, 0x10, 0, 3),
+ PIN_FIELD_BASE(56, 56, 1, 0x0060, 0x10, 9, 3),
+ PIN_FIELD_BASE(57, 57, 2, 0x0050, 0x10, 0, 3),
+ PIN_FIELD_BASE(58, 58, 2, 0x0050, 0x10, 6, 3),
+ PIN_FIELD_BASE(59, 59, 1, 0x0060, 0x10, 3, 3),
+ PIN_FIELD_BASE(60, 60, 1, 0x0060, 0x10, 12, 3),
+ PIN_FIELD_BASE(61, 61, 1, 0x0060, 0x10, 6, 3),
+ PIN_FIELD_BASE(62, 62, 1, 0x0060, 0x10, 15, 3),
+ PIN_FIELD_BASE(63, 63, 2, 0x0050, 0x10, 3, 3),
+ PIN_FIELD_BASE(64, 64, 2, 0x0050, 0x10, 9, 3),
+ PIN_FIELD_BASE(65, 65, 4, 0x0030, 0x10, 0, 3),
+ PIN_FIELD_BASE(66, 66, 4, 0x0030, 0x10, 6, 3),
+ PIN_FIELD_BASE(67, 67, 4, 0x0030, 0x10, 3, 3),
+ PIN_FIELD_BASE(68, 68, 4, 0x0030, 0x10, 9, 3),
+ PIN_FIELD_BASE(175, 175, 2, 0x0050, 0x10, 12, 3),
+ PIN_FIELD_BASE(176, 176, 2, 0x0050, 0x10, 15, 3),
+};
+
+static const struct mtk_pin_field_calc mt8188_pin_rsel_range[] = {
+ PIN_FIELD_BASE(53, 53, 3, 0x00c0, 0x10, 0, 3),
+ PIN_FIELD_BASE(54, 54, 3, 0x00c0, 0x10, 3, 3),
+ PIN_FIELD_BASE(55, 55, 1, 0x0160, 0x10, 0, 3),
+ PIN_FIELD_BASE(56, 56, 1, 0x0160, 0x10, 9, 3),
+ PIN_FIELD_BASE(57, 57, 2, 0x0150, 0x10, 0, 3),
+ PIN_FIELD_BASE(58, 58, 2, 0x0150, 0x10, 6, 3),
+ PIN_FIELD_BASE(59, 59, 1, 0x0160, 0x10, 3, 3),
+ PIN_FIELD_BASE(60, 60, 1, 0x0160, 0x10, 12, 3),
+ PIN_FIELD_BASE(61, 61, 1, 0x0160, 0x10, 6, 3),
+ PIN_FIELD_BASE(62, 62, 1, 0x0160, 0x10, 15, 3),
+ PIN_FIELD_BASE(63, 63, 2, 0x0150, 0x10, 3, 3),
+ PIN_FIELD_BASE(64, 64, 2, 0x0150, 0x10, 9, 3),
+ PIN_FIELD_BASE(65, 65, 4, 0x00d0, 0x10, 0, 3),
+ PIN_FIELD_BASE(66, 66, 4, 0x00d0, 0x10, 6, 3),
+ PIN_FIELD_BASE(67, 67, 4, 0x00d0, 0x10, 3, 3),
+ PIN_FIELD_BASE(68, 68, 4, 0x00d0, 0x10, 9, 3),
+ PIN_FIELD_BASE(175, 175, 2, 0x0150, 0x10, 12, 3),
+ PIN_FIELD_BASE(176, 176, 2, 0x0150, 0x10, 15, 3),
+};
+
+static const struct mtk_pin_rsel mt8188_pin_rsel_val_range[] = {
+ PIN_RSEL(53, 68, 0x0, 75000, 75000),
+ PIN_RSEL(53, 68, 0x1, 10000, 5000),
+ PIN_RSEL(53, 68, 0x2, 5000, 75000),
+ PIN_RSEL(53, 68, 0x3, 4000, 5000),
+ PIN_RSEL(53, 68, 0x4, 3000, 75000),
+ PIN_RSEL(53, 68, 0x5, 2000, 5000),
+ PIN_RSEL(53, 68, 0x6, 1500, 75000),
+ PIN_RSEL(53, 68, 0x7, 1000, 5000),
+ PIN_RSEL(175, 176, 0x0, 75000, 75000),
+ PIN_RSEL(175, 176, 0x1, 10000, 5000),
+ PIN_RSEL(175, 176, 0x2, 5000, 75000),
+ PIN_RSEL(175, 176, 0x3, 4000, 5000),
+ PIN_RSEL(175, 176, 0x4, 3000, 75000),
+ PIN_RSEL(175, 176, 0x5, 2000, 5000),
+ PIN_RSEL(175, 176, 0x6, 1500, 75000),
+ PIN_RSEL(175, 176, 0x7, 1000, 5000),
+};
+
+static const unsigned int mt8188_pull_type[] = {
+ MTK_PULL_PU_PD_TYPE, /*0*/
+ MTK_PULL_PU_PD_TYPE, /*1*/
+ MTK_PULL_PU_PD_TYPE, /*2*/
+ MTK_PULL_PU_PD_TYPE, /*3*/
+ MTK_PULL_PU_PD_TYPE, /*4*/
+ MTK_PULL_PU_PD_TYPE, /*5*/
+ MTK_PULL_PU_PD_TYPE, /*6*/
+ MTK_PULL_PU_PD_TYPE, /*7*/
+ MTK_PULL_PU_PD_TYPE, /*8*/
+ MTK_PULL_PU_PD_TYPE, /*9*/
+ MTK_PULL_PU_PD_TYPE, /*10*/
+ MTK_PULL_PU_PD_TYPE, /*11*/
+ MTK_PULL_PU_PD_TYPE, /*12*/
+ MTK_PULL_PU_PD_TYPE, /*13*/
+ MTK_PULL_PU_PD_TYPE, /*14*/
+ MTK_PULL_PU_PD_TYPE, /*15*/
+ MTK_PULL_PU_PD_TYPE, /*16*/
+ MTK_PULL_PU_PD_TYPE, /*17*/
+ MTK_PULL_PU_PD_TYPE, /*18*/
+ MTK_PULL_PU_PD_TYPE, /*19*/
+ MTK_PULL_PU_PD_TYPE, /*20*/
+ MTK_PULL_PU_PD_TYPE, /*21*/
+ MTK_PULL_PU_PD_TYPE, /*22*/
+ MTK_PULL_PU_PD_TYPE, /*23*/
+ MTK_PULL_PU_PD_TYPE, /*24*/
+ MTK_PULL_PU_PD_TYPE, /*25*/
+ MTK_PULL_PU_PD_TYPE, /*26*/
+ MTK_PULL_PU_PD_TYPE, /*27*/
+ MTK_PULL_PU_PD_TYPE, /*28*/
+ MTK_PULL_PU_PD_TYPE, /*29*/
+ MTK_PULL_PU_PD_TYPE, /*30*/
+ MTK_PULL_PU_PD_TYPE, /*31*/
+ MTK_PULL_PU_PD_TYPE, /*32*/
+ MTK_PULL_PU_PD_TYPE, /*33*/
+ MTK_PULL_PU_PD_TYPE, /*34*/
+ MTK_PULL_PU_PD_TYPE, /*35*/
+ MTK_PULL_PU_PD_TYPE, /*36*/
+ MTK_PULL_PU_PD_TYPE, /*37*/
+ MTK_PULL_PU_PD_TYPE, /*38*/
+ MTK_PULL_PU_PD_TYPE, /*39*/
+ MTK_PULL_PU_PD_TYPE, /*40*/
+ MTK_PULL_PU_PD_TYPE, /*41*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*42*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*43*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*44*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*45*/
+ MTK_PULL_PU_PD_TYPE, /*46*/
+ MTK_PULL_PU_PD_TYPE, /*47*/
+ MTK_PULL_PU_PD_TYPE, /*48*/
+ MTK_PULL_PU_PD_TYPE, /*49*/
+ MTK_PULL_PU_PD_TYPE, /*50*/
+ MTK_PULL_PU_PD_TYPE, /*51*/
+ MTK_PULL_PU_PD_TYPE, /*52*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*53*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*54*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*55*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*56*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*57*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*58*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*59*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*60*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*61*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*62*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*63*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*64*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*65*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*66*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*67*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*68*/
+ MTK_PULL_PU_PD_TYPE, /*69*/
+ MTK_PULL_PU_PD_TYPE, /*70*/
+ MTK_PULL_PU_PD_TYPE, /*71*/
+ MTK_PULL_PU_PD_TYPE, /*72*/
+ MTK_PULL_PU_PD_TYPE, /*73*/
+ MTK_PULL_PU_PD_TYPE, /*74*/
+ MTK_PULL_PU_PD_TYPE, /*75*/
+ MTK_PULL_PU_PD_TYPE, /*76*/
+ MTK_PULL_PU_PD_TYPE, /*77*/
+ MTK_PULL_PU_PD_TYPE, /*78*/
+ MTK_PULL_PU_PD_TYPE, /*79*/
+ MTK_PULL_PU_PD_TYPE, /*80*/
+ MTK_PULL_PU_PD_TYPE, /*81*/
+ MTK_PULL_PU_PD_TYPE, /*82*/
+ MTK_PULL_PU_PD_TYPE, /*83*/
+ MTK_PULL_PU_PD_TYPE, /*84*/
+ MTK_PULL_PU_PD_TYPE, /*85*/
+ MTK_PULL_PU_PD_TYPE, /*86*/
+ MTK_PULL_PU_PD_TYPE, /*87*/
+ MTK_PULL_PU_PD_TYPE, /*88*/
+ MTK_PULL_PU_PD_TYPE, /*89*/
+ MTK_PULL_PU_PD_TYPE, /*90*/
+ MTK_PULL_PU_PD_TYPE, /*91*/
+ MTK_PULL_PU_PD_TYPE, /*92*/
+ MTK_PULL_PU_PD_TYPE, /*93*/
+ MTK_PULL_PU_PD_TYPE, /*94*/
+ MTK_PULL_PU_PD_TYPE, /*95*/
+ MTK_PULL_PU_PD_TYPE, /*96*/
+ MTK_PULL_PU_PD_TYPE, /*97*/
+ MTK_PULL_PU_PD_TYPE, /*98*/
+ MTK_PULL_PU_PD_TYPE, /*99*/
+ MTK_PULL_PU_PD_TYPE, /*100*/
+ MTK_PULL_PU_PD_TYPE, /*101*/
+ MTK_PULL_PU_PD_TYPE, /*102*/
+ MTK_PULL_PU_PD_TYPE, /*103*/
+ MTK_PULL_PU_PD_TYPE, /*104*/
+ MTK_PULL_PU_PD_TYPE, /*105*/
+ MTK_PULL_PU_PD_TYPE, /*106*/
+ MTK_PULL_PU_PD_TYPE, /*107*/
+ MTK_PULL_PU_PD_TYPE, /*108*/
+ MTK_PULL_PU_PD_TYPE, /*109*/
+ MTK_PULL_PU_PD_TYPE, /*110*/
+ MTK_PULL_PU_PD_TYPE, /*111*/
+ MTK_PULL_PU_PD_TYPE, /*112*/
+ MTK_PULL_PU_PD_TYPE, /*113*/
+ MTK_PULL_PU_PD_TYPE, /*114*/
+ MTK_PULL_PU_PD_TYPE, /*115*/
+ MTK_PULL_PU_PD_TYPE, /*116*/
+ MTK_PULL_PU_PD_TYPE, /*117*/
+ MTK_PULL_PU_PD_TYPE, /*118*/
+ MTK_PULL_PU_PD_TYPE, /*119*/
+ MTK_PULL_PU_PD_TYPE, /*120*/
+ MTK_PULL_PU_PD_TYPE, /*121*/
+ MTK_PULL_PU_PD_TYPE, /*122*/
+ MTK_PULL_PU_PD_TYPE, /*123*/
+ MTK_PULL_PU_PD_TYPE, /*124*/
+ MTK_PULL_PU_PD_TYPE, /*125*/
+ MTK_PULL_PU_PD_TYPE, /*126*/
+ MTK_PULL_PU_PD_TYPE, /*127*/
+ MTK_PULL_PU_PD_TYPE, /*128*/
+ MTK_PULL_PU_PD_TYPE, /*129*/
+ MTK_PULL_PU_PD_TYPE, /*130*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*131*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*132*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*133*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*134*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*135*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*136*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*137*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*138*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*139*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*140*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*141*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*142*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*143*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*144*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*145*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*146*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*147*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*148*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*149*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*150*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*151*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*152*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*153*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*154*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*155*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*156*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*157*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*158*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*159*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*160*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*161*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*162*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*163*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*164*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*165*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*166*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*167*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*168*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*169*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*170*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*171*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*172*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*173*/
+ MTK_PULL_PUPD_R1R0_TYPE, /*174*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*175*/
+ MTK_PULL_PU_PD_RSEL_TYPE, /*176*/
+};
+
+static const struct mtk_pin_reg_calc mt8188_reg_cals[PINCTRL_PIN_REG_MAX] = {
+ [PINCTRL_PIN_REG_MODE] = MTK_RANGE(mt8188_pin_mode_range),
+ [PINCTRL_PIN_REG_DIR] = MTK_RANGE(mt8188_pin_dir_range),
+ [PINCTRL_PIN_REG_DI] = MTK_RANGE(mt8188_pin_di_range),
+ [PINCTRL_PIN_REG_DO] = MTK_RANGE(mt8188_pin_do_range),
+ [PINCTRL_PIN_REG_SMT] = MTK_RANGE(mt8188_pin_smt_range),
+ [PINCTRL_PIN_REG_IES] = MTK_RANGE(mt8188_pin_ies_range),
+ [PINCTRL_PIN_REG_TDSEL] = MTK_RANGE(mt8188_pin_tdsel_range),
+ [PINCTRL_PIN_REG_RDSEL] = MTK_RANGE(mt8188_pin_rdsel_range),
+ [PINCTRL_PIN_REG_PUPD] = MTK_RANGE(mt8188_pin_pupd_range),
+ [PINCTRL_PIN_REG_R0] = MTK_RANGE(mt8188_pin_r0_range),
+ [PINCTRL_PIN_REG_R1] = MTK_RANGE(mt8188_pin_r1_range),
+ [PINCTRL_PIN_REG_PU] = MTK_RANGE(mt8188_pin_pu_range),
+ [PINCTRL_PIN_REG_PD] = MTK_RANGE(mt8188_pin_pd_range),
+ [PINCTRL_PIN_REG_DRV] = MTK_RANGE(mt8188_pin_drv_range),
+ [PINCTRL_PIN_REG_DRV_ADV] = MTK_RANGE(mt8188_pin_drv_adv_range),
+ [PINCTRL_PIN_REG_RSEL] = MTK_RANGE(mt8188_pin_rsel_range),
+};
+
+static const char * const mt8188_pinctrl_register_base_name[] = {
+ "iocfg0", "iocfg_rm", "iocfg_lt", "iocfg_lm", "iocfg_rt",
+};
+
+static const struct mtk_eint_hw mt8188_eint_hw = {
+ .port_mask = 0xf,
+ .ports = 7,
+ .ap_num = 225,
+ .db_cnt = 32,
+};
+
+static const struct mtk_pin_soc mt8188_data = {
+ .reg_cal = mt8188_reg_cals,
+ .pins = mtk_pins_mt8188,
+ .npins = ARRAY_SIZE(mtk_pins_mt8188),
+ .ngrps = ARRAY_SIZE(mtk_pins_mt8188),
+ .eint_hw = &mt8188_eint_hw,
+ .nfuncs = 8,
+ .gpio_m = 0,
+ .base_names = mt8188_pinctrl_register_base_name,
+ .nbase_names = ARRAY_SIZE(mt8188_pinctrl_register_base_name),
+ .pull_type = mt8188_pull_type,
+ .pin_rsel = mt8188_pin_rsel_val_range,
+ .npin_rsel = ARRAY_SIZE(mt8188_pin_rsel_val_range),
+ .bias_set_combo = mtk_pinconf_bias_set_combo,
+ .bias_get_combo = mtk_pinconf_bias_get_combo,
+ .drive_set = mtk_pinconf_drive_set_rev1,
+ .drive_get = mtk_pinconf_drive_get_rev1,
+ .adv_drive_set = mtk_pinconf_adv_drive_set_raw,
+ .adv_drive_get = mtk_pinconf_adv_drive_get_raw,
+};
+
+static const struct of_device_id mt8188_pinctrl_of_match[] = {
+ { .compatible = "mediatek,mt8188-pinctrl", .data = &mt8188_data },
+ { }
+};
+
+static struct platform_driver mt8188_pinctrl_driver = {
+ .driver = {
+ .name = "mt8188-pinctrl",
+ .of_match_table = mt8188_pinctrl_of_match,
+ .pm = &mtk_paris_pinctrl_pm_ops
+ },
+ .probe = mtk_paris_pinctrl_probe,
+};
+
+static int __init mt8188_pinctrl_init(void)
+{
+ return platform_driver_register(&mt8188_pinctrl_driver);
+}
+
+arch_initcall(mt8188_pinctrl_init);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("MediaTek MT8188 Pinctrl Driver");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2022 MediaTek Inc.
+ * Author: Hui Liu <hui.liu@mediatek.com>
+ *
+ */
+
+#ifndef __PINCTRL_MTK_MT8188_H
+#define __PINCTRL_MTK_MT8188_H
+
+#include "pinctrl-paris.h"
+
+static const struct mtk_pin_desc mtk_pins_mt8188[] = {
+ MTK_PIN(
+ 0, "GPIO0",
+ MTK_EINT_FUNCTION(0, 0),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO0"),
+ MTK_FUNCTION(1, "B0_TP_GPIO0_AO"),
+ MTK_FUNCTION(2, "O_SPIM5_CSB"),
+ MTK_FUNCTION(3, "O_UTXD1"),
+ MTK_FUNCTION(4, "O_DMIC3_CLK"),
+ MTK_FUNCTION(5, "B0_I2SIN_MCK"),
+ MTK_FUNCTION(6, "O_I2SO2_MCK"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A0")
+ ),
+
+ MTK_PIN(
+ 1, "GPIO1",
+ MTK_EINT_FUNCTION(0, 1),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO1"),
+ MTK_FUNCTION(1, "B0_TP_GPIO1_AO"),
+ MTK_FUNCTION(2, "O_SPIM5_CLK"),
+ MTK_FUNCTION(3, "I1_URXD1"),
+ MTK_FUNCTION(4, "I0_DMIC3_DAT"),
+ MTK_FUNCTION(5, "B0_I2SIN_BCK"),
+ MTK_FUNCTION(6, "B0_I2SO2_BCK"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A1")
+ ),
+
+ MTK_PIN(
+ 2, "GPIO2",
+ MTK_EINT_FUNCTION(0, 2),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO2"),
+ MTK_FUNCTION(1, "B0_TP_GPIO2_AO"),
+ MTK_FUNCTION(2, "B0_SPIM5_MOSI"),
+ MTK_FUNCTION(3, "O_URTS1"),
+ MTK_FUNCTION(4, "I0_DMIC3_DAT_R"),
+ MTK_FUNCTION(5, "B0_I2SIN_WS"),
+ MTK_FUNCTION(6, "B0_I2SO2_WS"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A2")
+ ),
+
+ MTK_PIN(
+ 3, "GPIO3",
+ MTK_EINT_FUNCTION(0, 3),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO3"),
+ MTK_FUNCTION(1, "B0_TP_GPIO3_AO"),
+ MTK_FUNCTION(2, "B0_SPIM5_MISO"),
+ MTK_FUNCTION(3, "I1_UCTS1"),
+ MTK_FUNCTION(4, "O_DMIC4_CLK"),
+ MTK_FUNCTION(5, "I0_I2SIN_D0"),
+ MTK_FUNCTION(6, "O_I2SO2_D0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A3")
+ ),
+
+ MTK_PIN(
+ 4, "GPIO4",
+ MTK_EINT_FUNCTION(0, 4),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO4"),
+ MTK_FUNCTION(1, "B0_TP_GPIO4_AO"),
+ MTK_FUNCTION(2, "I0_SPDIF_IN2"),
+ MTK_FUNCTION(3, "O_I2SO1_MCK"),
+ MTK_FUNCTION(4, "I0_DMIC4_DAT"),
+ MTK_FUNCTION(5, "I0_I2SIN_D1"),
+ MTK_FUNCTION(6, "O_I2SO2_D1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A4")
+ ),
+
+ MTK_PIN(
+ 5, "GPIO5",
+ MTK_EINT_FUNCTION(0, 5),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO5"),
+ MTK_FUNCTION(1, "B0_TP_GPIO5_AO"),
+ MTK_FUNCTION(2, "I0_SPDIF_IN1"),
+ MTK_FUNCTION(3, "O_I2SO1_BCK"),
+ MTK_FUNCTION(4, "I0_DMIC4_DAT_R"),
+ MTK_FUNCTION(5, "I0_I2SIN_D2"),
+ MTK_FUNCTION(6, "O_I2SO2_D2"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A5")
+ ),
+
+ MTK_PIN(
+ 6, "GPIO6",
+ MTK_EINT_FUNCTION(0, 6),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO6"),
+ MTK_FUNCTION(1, "B0_TP_GPIO6_AO"),
+ MTK_FUNCTION(2, "I0_SPDIF_IN0"),
+ MTK_FUNCTION(3, "O_I2SO1_WS"),
+ MTK_FUNCTION(4, "O_DMIC1_CLK"),
+ MTK_FUNCTION(5, "I0_I2SIN_D3"),
+ MTK_FUNCTION(6, "O_I2SO2_D3"),
+ MTK_FUNCTION(7, "B0_MD32_0_GPIO0")
+ ),
+
+ MTK_PIN(
+ 7, "GPIO7",
+ MTK_EINT_FUNCTION(0, 7),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO7"),
+ MTK_FUNCTION(1, "B0_TP_GPIO7_AO"),
+ MTK_FUNCTION(2, "O_SPIM3_CSB"),
+ MTK_FUNCTION(3, "B0_TDMIN_MCK"),
+ MTK_FUNCTION(4, "I0_DMIC1_DAT"),
+ MTK_FUNCTION(5, "O_CMVREF0"),
+ MTK_FUNCTION(6, "O_CLKM0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A6")
+ ),
+
+ MTK_PIN(
+ 8, "GPIO8",
+ MTK_EINT_FUNCTION(0, 8),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO8"),
+ MTK_FUNCTION(1, "B0_TP_GPIO0_AO"),
+ MTK_FUNCTION(2, "O_SPIM3_CLK"),
+ MTK_FUNCTION(3, "B0_TDMIN_BCK"),
+ MTK_FUNCTION(4, "I0_DMIC1_DAT_R"),
+ MTK_FUNCTION(5, "O_CMVREF1"),
+ MTK_FUNCTION(6, "O_CLKM1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A7")
+ ),
+
+ MTK_PIN(
+ 9, "GPIO9",
+ MTK_EINT_FUNCTION(0, 9),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO9"),
+ MTK_FUNCTION(1, "B0_TP_GPIO1_AO"),
+ MTK_FUNCTION(2, "B0_SPIM3_MOSI"),
+ MTK_FUNCTION(3, "B0_TDMIN_LRCK"),
+ MTK_FUNCTION(4, "O_DMIC2_CLK"),
+ MTK_FUNCTION(5, "O_CMFLASH0"),
+ MTK_FUNCTION(6, "O_PWM_0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A8")
+ ),
+
+ MTK_PIN(
+ 10, "GPIO10",
+ MTK_EINT_FUNCTION(0, 10),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO10"),
+ MTK_FUNCTION(1, "B0_TP_GPIO2_AO"),
+ MTK_FUNCTION(2, "B0_SPIM3_MISO"),
+ MTK_FUNCTION(3, "I0_TDMIN_DI"),
+ MTK_FUNCTION(4, "I0_DMIC2_DAT"),
+ MTK_FUNCTION(5, "O_CMFLASH1"),
+ MTK_FUNCTION(6, "O_PWM_1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A9")
+ ),
+
+ MTK_PIN(
+ 11, "GPIO11",
+ MTK_EINT_FUNCTION(0, 11),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO11"),
+ MTK_FUNCTION(1, "B0_TP_GPIO3_AO"),
+ MTK_FUNCTION(2, "O_SPDIF_OUT"),
+ MTK_FUNCTION(3, "O_I2SO1_D0"),
+ MTK_FUNCTION(4, "I0_DMIC2_DAT_R"),
+ MTK_FUNCTION(5, "I0_DVFSRC_EXT_REQ"),
+ MTK_FUNCTION(6, "O_CMVREF6"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A10")
+ ),
+
+ MTK_PIN(
+ 12, "GPIO12",
+ MTK_EINT_FUNCTION(0, 12),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO12"),
+ MTK_FUNCTION(1, "B0_TP_GPIO4_AO"),
+ MTK_FUNCTION(2, "O_SPIM4_CSB"),
+ MTK_FUNCTION(3, "B1_JTMS_SEL3"),
+ MTK_FUNCTION(4, "B1_APU_JTAG_TMS"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_TMS"),
+ MTK_FUNCTION(6, "I0_IPU_JTAG_TMS"),
+ MTK_FUNCTION(7, "I0_HDMITX20_HTPLG")
+ ),
+
+ MTK_PIN(
+ 13, "GPIO13",
+ MTK_EINT_FUNCTION(0, 13),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO13"),
+ MTK_FUNCTION(1, "B0_TP_GPIO5_AO"),
+ MTK_FUNCTION(2, "O_SPIM4_CLK"),
+ MTK_FUNCTION(3, "I0_JTCK_SEL3"),
+ MTK_FUNCTION(4, "I0_APU_JTAG_TCK"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_TCK"),
+ MTK_FUNCTION(6, "I0_IPU_JTAG_TCK"),
+ MTK_FUNCTION(7, "B1_HDMITX20_CEC")
+ ),
+
+ MTK_PIN(
+ 14, "GPIO14",
+ MTK_EINT_FUNCTION(0, 14),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO14"),
+ MTK_FUNCTION(1, "B0_TP_GPIO6_AO"),
+ MTK_FUNCTION(2, "B0_SPIM4_MOSI"),
+ MTK_FUNCTION(3, "I1_JTDI_SEL3"),
+ MTK_FUNCTION(4, "I1_APU_JTAG_TDI"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_TDI"),
+ MTK_FUNCTION(6, "I0_IPU_JTAG_TDI"),
+ MTK_FUNCTION(7, "B1_HDMITX20_SCL")
+ ),
+
+ MTK_PIN(
+ 15, "GPIO15",
+ MTK_EINT_FUNCTION(0, 15),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO15"),
+ MTK_FUNCTION(1, "B0_TP_GPIO7_AO"),
+ MTK_FUNCTION(2, "B0_SPIM4_MISO"),
+ MTK_FUNCTION(3, "O_JTDO_SEL3"),
+ MTK_FUNCTION(4, "O_APU_JTAG_TDO"),
+ MTK_FUNCTION(5, "O_VPU_UDI_TDO"),
+ MTK_FUNCTION(6, "O_IPU_JTAG_TDO"),
+ MTK_FUNCTION(7, "B1_HDMITX20_SDA")
+ ),
+
+ MTK_PIN(
+ 16, "GPIO16",
+ MTK_EINT_FUNCTION(0, 16),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO16"),
+ MTK_FUNCTION(1, "B0_TP_GPIO0_AO"),
+ MTK_FUNCTION(2, "O_UTXD3"),
+ MTK_FUNCTION(3, "I1_JTRSTn_SEL3"),
+ MTK_FUNCTION(4, "I0_APU_JTAG_TRST"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_NTRST"),
+ MTK_FUNCTION(6, "I0_IPU_JTAG_TRST"),
+ MTK_FUNCTION(7, "O_HDMITX20_PWR5V")
+ ),
+
+ MTK_PIN(
+ 17, "GPIO17",
+ MTK_EINT_FUNCTION(0, 17),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO17"),
+ MTK_FUNCTION(1, "B0_TP_GPIO1_AO"),
+ MTK_FUNCTION(2, "I1_URXD3"),
+ MTK_FUNCTION(3, "O_CMFLASH2"),
+ MTK_FUNCTION(4, "I0_EDP_TX_HPD"),
+ MTK_FUNCTION(5, "I0_DVFSRC_EXT_REQ"),
+ MTK_FUNCTION(6, "O_CMVREF7"),
+ MTK_FUNCTION(7, "B0_MD32_0_GPIO1")
+ ),
+
+ MTK_PIN(
+ 18, "GPIO18",
+ MTK_EINT_FUNCTION(0, 18),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO18"),
+ MTK_FUNCTION(1, "B0_TP_GPIO2_AO"),
+ MTK_FUNCTION(2, "O_CMFLASH0"),
+ MTK_FUNCTION(3, "O_CMVREF4"),
+ MTK_FUNCTION(4, "B0_TDMIN_MCK"),
+ MTK_FUNCTION(5, "O_UTXD1"),
+ MTK_FUNCTION(6, "O_TP_UTXD1_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A11")
+ ),
+
+ MTK_PIN(
+ 19, "GPIO19",
+ MTK_EINT_FUNCTION(0, 19),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO19"),
+ MTK_FUNCTION(1, "B0_TP_GPIO3_AO"),
+ MTK_FUNCTION(2, "O_CMFLASH1"),
+ MTK_FUNCTION(3, "O_CMVREF5"),
+ MTK_FUNCTION(4, "B0_TDMIN_BCK"),
+ MTK_FUNCTION(5, "I1_URXD1"),
+ MTK_FUNCTION(6, "I1_TP_URXD1_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A12")
+ ),
+
+ MTK_PIN(
+ 20, "GPIO20",
+ MTK_EINT_FUNCTION(0, 20),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO20"),
+ MTK_FUNCTION(1, "B0_TP_GPIO4_AO"),
+ MTK_FUNCTION(2, "O_CMFLASH2"),
+ MTK_FUNCTION(3, "O_CLKM2"),
+ MTK_FUNCTION(4, "B0_TDMIN_LRCK"),
+ MTK_FUNCTION(5, "O_URTS1"),
+ MTK_FUNCTION(6, "O_TP_URTS1_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A13")
+ ),
+
+ MTK_PIN(
+ 21, "GPIO21",
+ MTK_EINT_FUNCTION(0, 21),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO21"),
+ MTK_FUNCTION(1, "B0_TP_GPIO5_AO"),
+ MTK_FUNCTION(2, "O_CMFLASH3"),
+ MTK_FUNCTION(3, "O_CLKM3"),
+ MTK_FUNCTION(4, "I0_TDMIN_DI"),
+ MTK_FUNCTION(5, "I1_UCTS1"),
+ MTK_FUNCTION(6, "I1_TP_UCTS1_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A14")
+ ),
+
+ MTK_PIN(
+ 22, "GPIO22",
+ MTK_EINT_FUNCTION(0, 22),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO22"),
+ MTK_FUNCTION(1, "O_CMMCLK0"),
+ MTK_FUNCTION(5, "B0_TP_GPIO6_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A15")
+ ),
+
+ MTK_PIN(
+ 23, "GPIO23",
+ MTK_EINT_FUNCTION(0, 23),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO23"),
+ MTK_FUNCTION(1, "O_CMMCLK1"),
+ MTK_FUNCTION(3, "O_PWM_2"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL"),
+ MTK_FUNCTION(5, "B0_TP_GPIO7_AO"),
+ MTK_FUNCTION(6, "I0_DP_TX_HPD"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A16")
+ ),
+
+ MTK_PIN(
+ 24, "GPIO24",
+ MTK_EINT_FUNCTION(0, 24),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO24"),
+ MTK_FUNCTION(1, "O_CMMCLK2"),
+ MTK_FUNCTION(3, "O_PWM_3"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA"),
+ MTK_FUNCTION(5, "I0_DVFSRC_EXT_REQ"),
+ MTK_FUNCTION(6, "I0_EDP_TX_HPD"),
+ MTK_FUNCTION(7, "B0_MD32_0_GPIO2")
+ ),
+
+ MTK_PIN(
+ 25, "GPIO25",
+ MTK_EINT_FUNCTION(0, 25),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO25"),
+ MTK_FUNCTION(1, "O_LCM_RST"),
+ MTK_FUNCTION(2, "O_LCM1_RST"),
+ MTK_FUNCTION(3, "I0_DP_TX_HPD")
+ ),
+
+ MTK_PIN(
+ 26, "GPIO26",
+ MTK_EINT_FUNCTION(0, 26),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO26"),
+ MTK_FUNCTION(1, "I0_DSI_TE"),
+ MTK_FUNCTION(2, "I0_DSI1_TE"),
+ MTK_FUNCTION(3, "I0_EDP_TX_HPD")
+ ),
+
+ MTK_PIN(
+ 27, "GPIO27",
+ MTK_EINT_FUNCTION(0, 27),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO27"),
+ MTK_FUNCTION(1, "O_LCM1_RST"),
+ MTK_FUNCTION(2, "O_LCM_RST"),
+ MTK_FUNCTION(3, "I0_DP_TX_HPD"),
+ MTK_FUNCTION(4, "O_CMVREF2"),
+ MTK_FUNCTION(5, "O_mbistwriteen_trigger"),
+ MTK_FUNCTION(6, "O_PWM_2"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A17")
+ ),
+
+ MTK_PIN(
+ 28, "GPIO28",
+ MTK_EINT_FUNCTION(0, 28),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO28"),
+ MTK_FUNCTION(1, "I0_DSI1_TE"),
+ MTK_FUNCTION(2, "I0_DSI_TE"),
+ MTK_FUNCTION(3, "I0_EDP_TX_HPD"),
+ MTK_FUNCTION(4, "O_CMVREF3"),
+ MTK_FUNCTION(5, "O_mbistreaden_trigger"),
+ MTK_FUNCTION(6, "O_PWM_3"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A18")
+ ),
+
+ MTK_PIN(
+ 29, "GPIO29",
+ MTK_EINT_FUNCTION(0, 29),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO29"),
+ MTK_FUNCTION(1, "O_DISP_PWM0"),
+ MTK_FUNCTION(2, "O_DISP_PWM1")
+ ),
+
+ MTK_PIN(
+ 30, "GPIO30",
+ MTK_EINT_FUNCTION(0, 30),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO30"),
+ MTK_FUNCTION(1, "O_DISP_PWM1"),
+ MTK_FUNCTION(2, "O_DISP_PWM0"),
+ MTK_FUNCTION(3, "O_CMFLASH3"),
+ MTK_FUNCTION(4, "O_PWM_1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A19")
+ ),
+
+ MTK_PIN(
+ 31, "GPIO31",
+ MTK_EINT_FUNCTION(0, 31),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO31"),
+ MTK_FUNCTION(1, "O_UTXD0"),
+ MTK_FUNCTION(2, "O_TP_UTXD1_AO"),
+ MTK_FUNCTION(3, "O_ADSP_UTXD0"),
+ MTK_FUNCTION(4, "O_TP_UTXD2_AO"),
+ MTK_FUNCTION(5, "O_MD32_0_TXD"),
+ MTK_FUNCTION(6, "O_MD32_1_TXD"),
+ MTK_FUNCTION(7, "O_SSPM_UTXD_AO")
+ ),
+
+ MTK_PIN(
+ 32, "GPIO32",
+ MTK_EINT_FUNCTION(0, 32),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO32"),
+ MTK_FUNCTION(1, "I1_URXD0"),
+ MTK_FUNCTION(2, "I1_TP_URXD1_AO"),
+ MTK_FUNCTION(3, "I1_ADSP_URXD0"),
+ MTK_FUNCTION(4, "I1_TP_URXD2_AO"),
+ MTK_FUNCTION(5, "I1_MD32_0_RXD"),
+ MTK_FUNCTION(6, "I1_MD32_1_RXD"),
+ MTK_FUNCTION(7, "I1_SSPM_URXD_AO")
+ ),
+
+ MTK_PIN(
+ 33, "GPIO33",
+ MTK_EINT_FUNCTION(0, 33),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO33"),
+ MTK_FUNCTION(1, "O_UTXD1"),
+ MTK_FUNCTION(2, "O_URTS2"),
+ MTK_FUNCTION(3, "O_ADSP_UTXD0"),
+ MTK_FUNCTION(4, "O_TP_UTXD1_AO"),
+ MTK_FUNCTION(5, "O_mbistwriteen_trigger"),
+ MTK_FUNCTION(6, "O_MD32_0_TXD"),
+ MTK_FUNCTION(7, "O_SSPM_UTXD_AO")
+ ),
+
+ MTK_PIN(
+ 34, "GPIO34",
+ MTK_EINT_FUNCTION(0, 34),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO34"),
+ MTK_FUNCTION(1, "I1_URXD1"),
+ MTK_FUNCTION(2, "I1_UCTS2"),
+ MTK_FUNCTION(3, "I1_ADSP_URXD0"),
+ MTK_FUNCTION(4, "I1_TP_URXD1_AO"),
+ MTK_FUNCTION(5, "O_mbistreaden_trigger"),
+ MTK_FUNCTION(6, "I1_MD32_0_RXD"),
+ MTK_FUNCTION(7, "I1_SSPM_URXD_AO")
+ ),
+
+ MTK_PIN(
+ 35, "GPIO35",
+ MTK_EINT_FUNCTION(0, 35),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO35"),
+ MTK_FUNCTION(1, "O_UTXD2"),
+ MTK_FUNCTION(2, "O_URTS1"),
+ MTK_FUNCTION(3, "O_ADSP_UTXD0"),
+ MTK_FUNCTION(4, "O_TP_URTS1_AO"),
+ MTK_FUNCTION(5, "O_TP_UTXD2_AO"),
+ MTK_FUNCTION(6, "O_MD32_1_TXD"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A20")
+ ),
+
+ MTK_PIN(
+ 36, "GPIO36",
+ MTK_EINT_FUNCTION(0, 36),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO36"),
+ MTK_FUNCTION(1, "I1_URXD2"),
+ MTK_FUNCTION(2, "I1_UCTS1"),
+ MTK_FUNCTION(3, "I1_ADSP_URXD0"),
+ MTK_FUNCTION(4, "I1_TP_UCTS1_AO"),
+ MTK_FUNCTION(5, "I1_TP_URXD2_AO"),
+ MTK_FUNCTION(6, "I1_MD32_1_RXD"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A21")
+ ),
+
+ MTK_PIN(
+ 37, "GPIO37",
+ MTK_EINT_FUNCTION(0, 37),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO37"),
+ MTK_FUNCTION(1, "B1_JTMS_SEL1"),
+ MTK_FUNCTION(2, "I0_UDI_TMS"),
+ MTK_FUNCTION(3, "I1_SPM_JTAG_TMS"),
+ MTK_FUNCTION(4, "I1_ADSP_JTAG0_TMS"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG0_TMS"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TMS"),
+ MTK_FUNCTION(7, "I1_MCUPM_JTAG_TMS")
+ ),
+
+ MTK_PIN(
+ 38, "GPIO38",
+ MTK_EINT_FUNCTION(0, 38),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO38"),
+ MTK_FUNCTION(1, "I0_JTCK_SEL1"),
+ MTK_FUNCTION(2, "I0_UDI_TCK"),
+ MTK_FUNCTION(3, "I1_SPM_JTAG_TCK"),
+ MTK_FUNCTION(4, "I0_ADSP_JTAG0_TCK"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG0_TCK"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TCK"),
+ MTK_FUNCTION(7, "I1_MCUPM_JTAG_TCK")
+ ),
+
+ MTK_PIN(
+ 39, "GPIO39",
+ MTK_EINT_FUNCTION(0, 39),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO39"),
+ MTK_FUNCTION(1, "I1_JTDI_SEL1"),
+ MTK_FUNCTION(2, "I0_UDI_TDI"),
+ MTK_FUNCTION(3, "I1_SPM_JTAG_TDI"),
+ MTK_FUNCTION(4, "I1_ADSP_JTAG0_TDI"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG0_TDI"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TDI"),
+ MTK_FUNCTION(7, "I1_MCUPM_JTAG_TDI")
+ ),
+
+ MTK_PIN(
+ 40, "GPIO40",
+ MTK_EINT_FUNCTION(0, 40),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO40"),
+ MTK_FUNCTION(1, "O_JTDO_SEL1"),
+ MTK_FUNCTION(2, "O_UDI_TDO"),
+ MTK_FUNCTION(3, "O_SPM_JTAG_TDO"),
+ MTK_FUNCTION(4, "O_ADSP_JTAG0_TDO"),
+ MTK_FUNCTION(5, "O_SCP_JTAG0_TDO"),
+ MTK_FUNCTION(6, "O_CCU0_JTAG_TDO"),
+ MTK_FUNCTION(7, "O_MCUPM_JTAG_TDO")
+ ),
+
+ MTK_PIN(
+ 41, "GPIO41",
+ MTK_EINT_FUNCTION(0, 41),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO41"),
+ MTK_FUNCTION(1, "I1_JTRSTn_SEL1"),
+ MTK_FUNCTION(2, "I0_UDI_NTRST"),
+ MTK_FUNCTION(3, "I0_SPM_JTAG_TRSTN"),
+ MTK_FUNCTION(4, "I1_ADSP_JTAG0_TRSTN"),
+ MTK_FUNCTION(5, "I0_SCP_JTAG0_TRSTN"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TRST"),
+ MTK_FUNCTION(7, "I0_MCUPM_JTAG_TRSTN")
+ ),
+
+ MTK_PIN(
+ 42, "GPIO42",
+ MTK_EINT_FUNCTION(0, 42),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO42"),
+ MTK_FUNCTION(1, "B1_KPCOL0")
+ ),
+
+ MTK_PIN(
+ 43, "GPIO43",
+ MTK_EINT_FUNCTION(0, 43),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO43"),
+ MTK_FUNCTION(1, "B1_KPCOL1"),
+ MTK_FUNCTION(2, "I0_DP_TX_HPD"),
+ MTK_FUNCTION(3, "O_CMFLASH2"),
+ MTK_FUNCTION(4, "I0_DVFSRC_EXT_REQ"),
+ MTK_FUNCTION(7, "O_mbistwriteen_trigger")
+ ),
+
+ MTK_PIN(
+ 44, "GPIO44",
+ MTK_EINT_FUNCTION(0, 44),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO44"),
+ MTK_FUNCTION(1, "B1_KPROW0")
+ ),
+
+ MTK_PIN(
+ 45, "GPIO45",
+ MTK_EINT_FUNCTION(0, 45),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO45"),
+ MTK_FUNCTION(1, "B1_KPROW1"),
+ MTK_FUNCTION(2, "I0_EDP_TX_HPD"),
+ MTK_FUNCTION(3, "O_CMFLASH3"),
+ MTK_FUNCTION(4, "B0_I2SIN_MCK"),
+ MTK_FUNCTION(7, "O_mbistreaden_trigger")
+ ),
+
+ MTK_PIN(
+ 46, "GPIO46",
+ MTK_EINT_FUNCTION(0, 46),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO46"),
+ MTK_FUNCTION(1, "I0_DP_TX_HPD"),
+ MTK_FUNCTION(2, "O_PWM_0"),
+ MTK_FUNCTION(3, "I0_VBUSVALID_2P"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A22")
+ ),
+
+ MTK_PIN(
+ 47, "GPIO47",
+ MTK_EINT_FUNCTION(0, 47),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO47"),
+ MTK_FUNCTION(1, "I1_WAKEN"),
+ MTK_FUNCTION(6, "O_GDU_TROOPS_DET0")
+ ),
+
+ MTK_PIN(
+ 48, "GPIO48",
+ MTK_EINT_FUNCTION(0, 48),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO48"),
+ MTK_FUNCTION(1, "O_PERSTN"),
+ MTK_FUNCTION(6, "O_GDU_TROOPS_DET1")
+ ),
+
+ MTK_PIN(
+ 49, "GPIO49",
+ MTK_EINT_FUNCTION(0, 49),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO49"),
+ MTK_FUNCTION(1, "B1_CLKREQN"),
+ MTK_FUNCTION(6, "O_GDU_TROOPS_DET2")
+ ),
+
+ MTK_PIN(
+ 50, "GPIO50",
+ MTK_EINT_FUNCTION(0, 50),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO50"),
+ MTK_FUNCTION(1, "O_HDMITX20_PWR5V"),
+ MTK_FUNCTION(3, "I1_IDDIG_1P"),
+ MTK_FUNCTION(4, "I1_SCP_JTAG1_TMS"),
+ MTK_FUNCTION(5, "I1_SSPM_JTAG_TMS"),
+ MTK_FUNCTION(6, "I1_MD32_0_JTAG_TMS"),
+ MTK_FUNCTION(7, "I1_MD32_1_JTAG_TMS")
+ ),
+
+ MTK_PIN(
+ 51, "GPIO51",
+ MTK_EINT_FUNCTION(0, 51),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO51"),
+ MTK_FUNCTION(1, "I0_HDMITX20_HTPLG"),
+ MTK_FUNCTION(2, "I0_EDP_TX_HPD"),
+ MTK_FUNCTION(3, "O_USB_DRVVBUS_1P"),
+ MTK_FUNCTION(4, "I1_SCP_JTAG1_TCK"),
+ MTK_FUNCTION(5, "I1_SSPM_JTAG_TCK"),
+ MTK_FUNCTION(6, "I1_MD32_0_JTAG_TCK"),
+ MTK_FUNCTION(7, "I1_MD32_1_JTAG_TCK")
+ ),
+
+ MTK_PIN(
+ 52, "GPIO52",
+ MTK_EINT_FUNCTION(0, 52),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO52"),
+ MTK_FUNCTION(1, "B1_HDMITX20_CEC"),
+ MTK_FUNCTION(3, "I0_VBUSVALID_1P"),
+ MTK_FUNCTION(4, "I1_SCP_JTAG1_TDI"),
+ MTK_FUNCTION(5, "I1_SSPM_JTAG_TDI"),
+ MTK_FUNCTION(6, "I1_MD32_0_JTAG_TDI"),
+ MTK_FUNCTION(7, "I1_MD32_1_JTAG_TDI")
+ ),
+
+ MTK_PIN(
+ 53, "GPIO53",
+ MTK_EINT_FUNCTION(0, 53),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO53"),
+ MTK_FUNCTION(1, "B1_HDMITX20_SCL"),
+ MTK_FUNCTION(3, "I1_IDDIG_2P"),
+ MTK_FUNCTION(4, "O_SCP_JTAG1_TDO"),
+ MTK_FUNCTION(5, "O_SSPM_JTAG_TDO"),
+ MTK_FUNCTION(6, "O_MD32_0_JTAG_TDO"),
+ MTK_FUNCTION(7, "O_MD32_1_JTAG_TDO")
+ ),
+
+ MTK_PIN(
+ 54, "GPIO54",
+ MTK_EINT_FUNCTION(0, 54),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO54"),
+ MTK_FUNCTION(1, "B1_HDMITX20_SDA"),
+ MTK_FUNCTION(3, "O_USB_DRVVBUS_2P"),
+ MTK_FUNCTION(4, "I0_SCP_JTAG1_TRSTN"),
+ MTK_FUNCTION(5, "I0_SSPM_JTAG_TRSTN"),
+ MTK_FUNCTION(6, "I1_MD32_0_JTAG_TRST"),
+ MTK_FUNCTION(7, "I1_MD32_1_JTAG_TRST")
+ ),
+
+ MTK_PIN(
+ 55, "GPIO55",
+ MTK_EINT_FUNCTION(0, 55),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO55"),
+ MTK_FUNCTION(1, "B1_SCL0"),
+ MTK_FUNCTION(2, "B1_SCP_SCL0"),
+ MTK_FUNCTION(3, "B1_SCP_SCL1"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL")
+ ),
+
+ MTK_PIN(
+ 56, "GPIO56",
+ MTK_EINT_FUNCTION(0, 56),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO56"),
+ MTK_FUNCTION(1, "B1_SDA0"),
+ MTK_FUNCTION(2, "B1_SCP_SDA0"),
+ MTK_FUNCTION(3, "B1_SCP_SDA1"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA")
+ ),
+
+ MTK_PIN(
+ 57, "GPIO57",
+ MTK_EINT_FUNCTION(0, 57),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO57"),
+ MTK_FUNCTION(1, "B1_SCL1")
+ ),
+
+ MTK_PIN(
+ 58, "GPIO58",
+ MTK_EINT_FUNCTION(0, 58),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO58"),
+ MTK_FUNCTION(1, "B1_SDA1")
+ ),
+
+ MTK_PIN(
+ 59, "GPIO59",
+ MTK_EINT_FUNCTION(0, 59),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO59"),
+ MTK_FUNCTION(1, "B1_SCL2"),
+ MTK_FUNCTION(2, "B1_SCP_SCL0"),
+ MTK_FUNCTION(3, "B1_SCP_SCL1")
+ ),
+
+ MTK_PIN(
+ 60, "GPIO60",
+ MTK_EINT_FUNCTION(0, 60),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO60"),
+ MTK_FUNCTION(1, "B1_SDA2"),
+ MTK_FUNCTION(2, "B1_SCP_SDA0"),
+ MTK_FUNCTION(3, "B1_SCP_SDA1")
+ ),
+
+ MTK_PIN(
+ 61, "GPIO61",
+ MTK_EINT_FUNCTION(0, 61),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO61"),
+ MTK_FUNCTION(1, "B1_SCL3"),
+ MTK_FUNCTION(2, "B1_SCP_SCL0"),
+ MTK_FUNCTION(3, "B1_SCP_SCL1"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL")
+ ),
+
+ MTK_PIN(
+ 62, "GPIO62",
+ MTK_EINT_FUNCTION(0, 62),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO62"),
+ MTK_FUNCTION(1, "B1_SDA3"),
+ MTK_FUNCTION(2, "B1_SCP_SDA0"),
+ MTK_FUNCTION(3, "B1_SCP_SDA1"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA")
+ ),
+
+ MTK_PIN(
+ 63, "GPIO63",
+ MTK_EINT_FUNCTION(0, 63),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO63"),
+ MTK_FUNCTION(1, "B1_SCL4")
+ ),
+
+ MTK_PIN(
+ 64, "GPIO64",
+ MTK_EINT_FUNCTION(0, 64),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO64"),
+ MTK_FUNCTION(1, "B1_SDA4")
+ ),
+
+ MTK_PIN(
+ 65, "GPIO65",
+ MTK_EINT_FUNCTION(0, 65),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO65"),
+ MTK_FUNCTION(1, "B1_SCL5"),
+ MTK_FUNCTION(2, "B1_SCP_SCL0"),
+ MTK_FUNCTION(3, "B1_SCP_SCL1")
+ ),
+
+ MTK_PIN(
+ 66, "GPIO66",
+ MTK_EINT_FUNCTION(0, 66),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO66"),
+ MTK_FUNCTION(1, "B1_SDA5"),
+ MTK_FUNCTION(2, "B1_SCP_SDA0"),
+ MTK_FUNCTION(3, "B1_SCP_SDA1")
+ ),
+
+ MTK_PIN(
+ 67, "GPIO67",
+ MTK_EINT_FUNCTION(0, 67),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO67"),
+ MTK_FUNCTION(1, "B1_SCL6"),
+ MTK_FUNCTION(2, "B1_SCP_SCL0"),
+ MTK_FUNCTION(3, "B1_SCP_SCL1"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL")
+ ),
+
+ MTK_PIN(
+ 68, "GPIO68",
+ MTK_EINT_FUNCTION(0, 68),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO68"),
+ MTK_FUNCTION(1, "B1_SDA6"),
+ MTK_FUNCTION(2, "B1_SCP_SDA0"),
+ MTK_FUNCTION(3, "B1_SCP_SDA1"),
+ MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA")
+ ),
+
+ MTK_PIN(
+ 69, "GPIO69",
+ MTK_EINT_FUNCTION(0, 69),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO69"),
+ MTK_FUNCTION(1, "O_SPIM0_CSB"),
+ MTK_FUNCTION(2, "O_SCP_SPI0_CS"),
+ MTK_FUNCTION(3, "O_DMIC3_CLK"),
+ MTK_FUNCTION(4, "B0_MD32_1_GPIO0"),
+ MTK_FUNCTION(5, "O_CMVREF0"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP0_0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A23")
+ ),
+
+ MTK_PIN(
+ 70, "GPIO70",
+ MTK_EINT_FUNCTION(0, 70),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO70"),
+ MTK_FUNCTION(1, "O_SPIM0_CLK"),
+ MTK_FUNCTION(2, "O_SCP_SPI0_CK"),
+ MTK_FUNCTION(3, "I0_DMIC3_DAT"),
+ MTK_FUNCTION(4, "B0_MD32_1_GPIO1"),
+ MTK_FUNCTION(5, "O_CMVREF1"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP0_1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A24")
+ ),
+
+ MTK_PIN(
+ 71, "GPIO71",
+ MTK_EINT_FUNCTION(0, 71),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO71"),
+ MTK_FUNCTION(1, "B0_SPIM0_MOSI"),
+ MTK_FUNCTION(2, "O_SCP_SPI0_MO"),
+ MTK_FUNCTION(3, "I0_DMIC3_DAT_R"),
+ MTK_FUNCTION(4, "B0_MD32_1_GPIO2"),
+ MTK_FUNCTION(5, "O_CMVREF2"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP0_2"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A25")
+ ),
+
+ MTK_PIN(
+ 72, "GPIO72",
+ MTK_EINT_FUNCTION(0, 72),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO72"),
+ MTK_FUNCTION(1, "B0_SPIM0_MISO"),
+ MTK_FUNCTION(2, "I0_SCP_SPI0_MI"),
+ MTK_FUNCTION(3, "O_DMIC4_CLK"),
+ MTK_FUNCTION(5, "O_CMVREF3"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP1_0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A26")
+ ),
+
+ MTK_PIN(
+ 73, "GPIO73",
+ MTK_EINT_FUNCTION(0, 73),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO73"),
+ MTK_FUNCTION(1, "B0_SPIM0_MIO2"),
+ MTK_FUNCTION(2, "O_UTXD3"),
+ MTK_FUNCTION(3, "I0_DMIC4_DAT"),
+ MTK_FUNCTION(4, "O_CLKM0"),
+ MTK_FUNCTION(5, "O_CMVREF4"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP1_1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A27")
+ ),
+
+ MTK_PIN(
+ 74, "GPIO74",
+ MTK_EINT_FUNCTION(0, 74),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO74"),
+ MTK_FUNCTION(1, "B0_SPIM0_MIO3"),
+ MTK_FUNCTION(2, "I1_URXD3"),
+ MTK_FUNCTION(3, "I0_DMIC4_DAT_R"),
+ MTK_FUNCTION(4, "O_CLKM1"),
+ MTK_FUNCTION(5, "O_CMVREF5"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP1_2"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A28")
+ ),
+
+ MTK_PIN(
+ 75, "GPIO75",
+ MTK_EINT_FUNCTION(0, 75),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO75"),
+ MTK_FUNCTION(1, "O_SPIM1_CSB"),
+ MTK_FUNCTION(2, "O_SCP_SPI1_A_CS"),
+ MTK_FUNCTION(3, "B0_TDMIN_MCK"),
+ MTK_FUNCTION(4, "B1_SCP_SCL0"),
+ MTK_FUNCTION(5, "O_CMVREF6"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP2_0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A29")
+ ),
+
+ MTK_PIN(
+ 76, "GPIO76",
+ MTK_EINT_FUNCTION(0, 76),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO76"),
+ MTK_FUNCTION(1, "O_SPIM1_CLK"),
+ MTK_FUNCTION(2, "O_SCP_SPI1_A_CK"),
+ MTK_FUNCTION(3, "B0_TDMIN_BCK"),
+ MTK_FUNCTION(4, "B1_SCP_SDA0"),
+ MTK_FUNCTION(5, "O_CMVREF7"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP2_1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A30")
+ ),
+
+ MTK_PIN(
+ 77, "GPIO77",
+ MTK_EINT_FUNCTION(0, 77),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO77"),
+ MTK_FUNCTION(1, "B0_SPIM1_MOSI"),
+ MTK_FUNCTION(2, "O_SCP_SPI1_A_MO"),
+ MTK_FUNCTION(3, "B0_TDMIN_LRCK"),
+ MTK_FUNCTION(4, "B1_SCP_SCL1"),
+ MTK_FUNCTION(6, "O_GDU_SUM_TROOP2_2"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A31")
+ ),
+
+ MTK_PIN(
+ 78, "GPIO78",
+ MTK_EINT_FUNCTION(0, 78),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO78"),
+ MTK_FUNCTION(1, "B0_SPIM1_MISO"),
+ MTK_FUNCTION(2, "I0_SCP_SPI1_A_MI"),
+ MTK_FUNCTION(3, "I0_TDMIN_DI"),
+ MTK_FUNCTION(4, "B1_SCP_SDA1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_A32")
+ ),
+
+ MTK_PIN(
+ 79, "GPIO79",
+ MTK_EINT_FUNCTION(0, 79),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO79"),
+ MTK_FUNCTION(1, "O_SPIM2_CSB"),
+ MTK_FUNCTION(2, "O_SCP_SPI2_CS"),
+ MTK_FUNCTION(3, "O_I2SO1_MCK"),
+ MTK_FUNCTION(4, "O_UTXD2"),
+ MTK_FUNCTION(5, "O_TP_UTXD2_AO"),
+ MTK_FUNCTION(6, "B0_PCM_SYNC"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B0")
+ ),
+
+ MTK_PIN(
+ 80, "GPIO80",
+ MTK_EINT_FUNCTION(0, 80),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO80"),
+ MTK_FUNCTION(1, "O_SPIM2_CLK"),
+ MTK_FUNCTION(2, "O_SCP_SPI2_CK"),
+ MTK_FUNCTION(3, "O_I2SO1_BCK"),
+ MTK_FUNCTION(4, "I1_URXD2"),
+ MTK_FUNCTION(5, "I1_TP_URXD2_AO"),
+ MTK_FUNCTION(6, "B0_PCM_CLK"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B1")
+ ),
+
+ MTK_PIN(
+ 81, "GPIO81",
+ MTK_EINT_FUNCTION(0, 81),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO81"),
+ MTK_FUNCTION(1, "B0_SPIM2_MOSI"),
+ MTK_FUNCTION(2, "O_SCP_SPI2_MO"),
+ MTK_FUNCTION(3, "O_I2SO1_WS"),
+ MTK_FUNCTION(4, "O_URTS2"),
+ MTK_FUNCTION(5, "O_TP_URTS2_AO"),
+ MTK_FUNCTION(6, "O_PCM_DO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B2")
+ ),
+
+ MTK_PIN(
+ 82, "GPIO82",
+ MTK_EINT_FUNCTION(0, 82),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO82"),
+ MTK_FUNCTION(1, "B0_SPIM2_MISO"),
+ MTK_FUNCTION(2, "I0_SCP_SPI2_MI"),
+ MTK_FUNCTION(3, "O_I2SO1_D0"),
+ MTK_FUNCTION(4, "I1_UCTS2"),
+ MTK_FUNCTION(5, "I1_TP_UCTS2_AO"),
+ MTK_FUNCTION(6, "I0_PCM_DI"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B3")
+ ),
+
+ MTK_PIN(
+ 83, "GPIO83",
+ MTK_EINT_FUNCTION(0, 83),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO83"),
+ MTK_FUNCTION(1, "I1_IDDIG")
+ ),
+
+ MTK_PIN(
+ 84, "GPIO84",
+ MTK_EINT_FUNCTION(0, 84),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO84"),
+ MTK_FUNCTION(1, "O_USB_DRVVBUS")
+ ),
+
+ MTK_PIN(
+ 85, "GPIO85",
+ MTK_EINT_FUNCTION(0, 85),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO85"),
+ MTK_FUNCTION(1, "I0_VBUSVALID")
+ ),
+
+ MTK_PIN(
+ 86, "GPIO86",
+ MTK_EINT_FUNCTION(0, 86),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO86"),
+ MTK_FUNCTION(1, "I1_IDDIG_1P"),
+ MTK_FUNCTION(2, "O_UTXD1"),
+ MTK_FUNCTION(3, "O_URTS2"),
+ MTK_FUNCTION(4, "O_PWM_2"),
+ MTK_FUNCTION(5, "B0_TP_GPIO4_AO"),
+ MTK_FUNCTION(6, "O_AUXIF_ST0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B4")
+ ),
+
+ MTK_PIN(
+ 87, "GPIO87",
+ MTK_EINT_FUNCTION(0, 87),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO87"),
+ MTK_FUNCTION(1, "O_USB_DRVVBUS_1P"),
+ MTK_FUNCTION(2, "I1_URXD1"),
+ MTK_FUNCTION(3, "I1_UCTS2"),
+ MTK_FUNCTION(4, "O_PWM_3"),
+ MTK_FUNCTION(5, "B0_TP_GPIO5_AO"),
+ MTK_FUNCTION(6, "O_AUXIF_CLK0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B5")
+ ),
+
+ MTK_PIN(
+ 88, "GPIO88",
+ MTK_EINT_FUNCTION(0, 88),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO88"),
+ MTK_FUNCTION(1, "I0_VBUSVALID_1P"),
+ MTK_FUNCTION(2, "O_UTXD2"),
+ MTK_FUNCTION(3, "O_URTS1"),
+ MTK_FUNCTION(4, "O_CLKM2"),
+ MTK_FUNCTION(5, "B0_TP_GPIO6_AO"),
+ MTK_FUNCTION(6, "O_AUXIF_ST1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B6")
+ ),
+
+ MTK_PIN(
+ 89, "GPIO89",
+ MTK_EINT_FUNCTION(0, 89),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO89"),
+ MTK_FUNCTION(1, "I1_IDDIG_2P"),
+ MTK_FUNCTION(2, "I1_URXD2"),
+ MTK_FUNCTION(3, "I1_UCTS1"),
+ MTK_FUNCTION(4, "O_CLKM3"),
+ MTK_FUNCTION(5, "B0_TP_GPIO7_AO"),
+ MTK_FUNCTION(6, "O_AUXIF_CLK1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B7")
+ ),
+
+ MTK_PIN(
+ 90, "GPIO90",
+ MTK_EINT_FUNCTION(0, 90),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO90"),
+ MTK_FUNCTION(1, "O_USB_DRVVBUS_2P"),
+ MTK_FUNCTION(2, "O_UTXD3"),
+ MTK_FUNCTION(3, "O_ADSP_UTXD0"),
+ MTK_FUNCTION(4, "O_SSPM_UTXD_AO"),
+ MTK_FUNCTION(5, "O_MD32_0_TXD"),
+ MTK_FUNCTION(6, "O_MD32_1_TXD"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B8")
+ ),
+
+ MTK_PIN(
+ 91, "GPIO91",
+ MTK_EINT_FUNCTION(0, 91),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO91"),
+ MTK_FUNCTION(1, "I0_VBUSVALID_2P"),
+ MTK_FUNCTION(2, "I1_URXD3"),
+ MTK_FUNCTION(3, "I1_ADSP_URXD0"),
+ MTK_FUNCTION(4, "I1_SSPM_URXD_AO"),
+ MTK_FUNCTION(5, "I1_MD32_0_RXD"),
+ MTK_FUNCTION(6, "I1_MD32_1_RXD"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B9")
+ ),
+
+ MTK_PIN(
+ 92, "GPIO92",
+ MTK_EINT_FUNCTION(0, 92),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO92"),
+ MTK_FUNCTION(1, "O_PWRAP_SPI0_CSN")
+ ),
+
+ MTK_PIN(
+ 93, "GPIO93",
+ MTK_EINT_FUNCTION(0, 93),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO93"),
+ MTK_FUNCTION(1, "O_PWRAP_SPI0_CK")
+ ),
+
+ MTK_PIN(
+ 94, "GPIO94",
+ MTK_EINT_FUNCTION(0, 94),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO94"),
+ MTK_FUNCTION(1, "B0_PWRAP_SPI0_MO"),
+ MTK_FUNCTION(2, "B0_PWRAP_SPI0_MI")
+ ),
+
+ MTK_PIN(
+ 95, "GPIO95",
+ MTK_EINT_FUNCTION(0, 95),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO95"),
+ MTK_FUNCTION(1, "B0_PWRAP_SPI0_MI"),
+ MTK_FUNCTION(2, "B0_PWRAP_SPI0_MO")
+ ),
+
+ MTK_PIN(
+ 96, "GPIO96",
+ MTK_EINT_FUNCTION(0, 96),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO96"),
+ MTK_FUNCTION(1, "O_SRCLKENA0")
+ ),
+
+ MTK_PIN(
+ 97, "GPIO97",
+ MTK_EINT_FUNCTION(0, 97),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO97"),
+ MTK_FUNCTION(1, "O_SRCLKENA1")
+ ),
+
+ MTK_PIN(
+ 98, "GPIO98",
+ MTK_EINT_FUNCTION(0, 98),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO98"),
+ MTK_FUNCTION(1, "O_SCP_VREQ_VAO"),
+ MTK_FUNCTION(2, "I0_DVFSRC_EXT_REQ")
+ ),
+
+ MTK_PIN(
+ 99, "GPIO99",
+ MTK_EINT_FUNCTION(0, 99),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO99"),
+ MTK_FUNCTION(1, "I0_RTC32K_CK")
+ ),
+
+ MTK_PIN(
+ 100, "GPIO100",
+ MTK_EINT_FUNCTION(0, 100),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO100"),
+ MTK_FUNCTION(1, "O_WATCHDOG")
+ ),
+
+ MTK_PIN(
+ 101, "GPIO101",
+ MTK_EINT_FUNCTION(0, 101),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO101"),
+ MTK_FUNCTION(1, "O_AUD_CLK_MOSI"),
+ MTK_FUNCTION(2, "O_I2SO1_MCK"),
+ MTK_FUNCTION(3, "B0_I2SIN_BCK")
+ ),
+
+ MTK_PIN(
+ 102, "GPIO102",
+ MTK_EINT_FUNCTION(0, 102),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO102"),
+ MTK_FUNCTION(1, "O_AUD_SYNC_MOSI"),
+ MTK_FUNCTION(2, "O_I2SO1_BCK"),
+ MTK_FUNCTION(3, "B0_I2SIN_WS")
+ ),
+
+ MTK_PIN(
+ 103, "GPIO103",
+ MTK_EINT_FUNCTION(0, 103),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO103"),
+ MTK_FUNCTION(1, "O_AUD_DAT_MOSI0"),
+ MTK_FUNCTION(2, "O_I2SO1_WS"),
+ MTK_FUNCTION(3, "I0_I2SIN_D0")
+ ),
+
+ MTK_PIN(
+ 104, "GPIO104",
+ MTK_EINT_FUNCTION(0, 104),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO104"),
+ MTK_FUNCTION(1, "O_AUD_DAT_MOSI1"),
+ MTK_FUNCTION(2, "O_I2SO1_D0"),
+ MTK_FUNCTION(3, "I0_I2SIN_D1")
+ ),
+
+ MTK_PIN(
+ 105, "GPIO105",
+ MTK_EINT_FUNCTION(0, 105),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO105"),
+ MTK_FUNCTION(1, "I0_AUD_DAT_MISO0"),
+ MTK_FUNCTION(2, "I0_VOW_DAT_MISO"),
+ MTK_FUNCTION(3, "I0_I2SIN_D2")
+ ),
+
+ MTK_PIN(
+ 106, "GPIO106",
+ MTK_EINT_FUNCTION(0, 106),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO106"),
+ MTK_FUNCTION(1, "I0_AUD_DAT_MISO1"),
+ MTK_FUNCTION(2, "I0_VOW_CLK_MISO"),
+ MTK_FUNCTION(3, "I0_I2SIN_D3")
+ ),
+
+ MTK_PIN(
+ 107, "GPIO107",
+ MTK_EINT_FUNCTION(0, 107),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO107"),
+ MTK_FUNCTION(1, "B0_I2SIN_MCK"),
+ MTK_FUNCTION(2, "I0_SPLIN_MCK"),
+ MTK_FUNCTION(3, "I0_SPDIF_IN0"),
+ MTK_FUNCTION(4, "O_CMVREF4"),
+ MTK_FUNCTION(5, "O_AUXIF_ST0"),
+ MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR0")
+ ),
+
+ MTK_PIN(
+ 108, "GPIO108",
+ MTK_EINT_FUNCTION(0, 108),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO108"),
+ MTK_FUNCTION(1, "B0_I2SIN_BCK"),
+ MTK_FUNCTION(2, "I0_SPLIN_LRCK"),
+ MTK_FUNCTION(3, "O_DMIC4_CLK"),
+ MTK_FUNCTION(4, "O_CMVREF5"),
+ MTK_FUNCTION(5, "O_AUXIF_CLK0"),
+ MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR1"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B10")
+ ),
+
+ MTK_PIN(
+ 109, "GPIO109",
+ MTK_EINT_FUNCTION(0, 109),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO109"),
+ MTK_FUNCTION(1, "B0_I2SIN_WS"),
+ MTK_FUNCTION(2, "I0_SPLIN_BCK"),
+ MTK_FUNCTION(3, "I0_DMIC4_DAT"),
+ MTK_FUNCTION(4, "O_CMVREF6"),
+ MTK_FUNCTION(5, "O_AUXIF_ST1"),
+ MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR2"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B11")
+ ),
+
+ MTK_PIN(
+ 110, "GPIO110",
+ MTK_EINT_FUNCTION(0, 110),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO110"),
+ MTK_FUNCTION(1, "I0_I2SIN_D0"),
+ MTK_FUNCTION(2, "I0_SPLIN_D0"),
+ MTK_FUNCTION(3, "I0_DMIC4_DAT_R"),
+ MTK_FUNCTION(4, "O_CMVREF7"),
+ MTK_FUNCTION(5, "O_AUXIF_CLK1"),
+ MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR3"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B12")
+ ),
+
+ MTK_PIN(
+ 111, "GPIO111",
+ MTK_EINT_FUNCTION(0, 111),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO111"),
+ MTK_FUNCTION(1, "I0_I2SIN_D1"),
+ MTK_FUNCTION(2, "I0_SPLIN_D1"),
+ MTK_FUNCTION(3, "O_DMIC3_CLK"),
+ MTK_FUNCTION(4, "O_SPDIF_OUT"),
+ MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR4"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B13")
+ ),
+
+ MTK_PIN(
+ 112, "GPIO112",
+ MTK_EINT_FUNCTION(0, 112),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO112"),
+ MTK_FUNCTION(1, "I0_I2SIN_D2"),
+ MTK_FUNCTION(2, "I0_SPLIN_D2"),
+ MTK_FUNCTION(3, "I0_DMIC3_DAT"),
+ MTK_FUNCTION(4, "B0_TDMIN_MCK"),
+ MTK_FUNCTION(5, "O_I2SO1_WS"),
+ MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR5"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B14")
+ ),
+
+ MTK_PIN(
+ 113, "GPIO113",
+ MTK_EINT_FUNCTION(0, 113),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO113"),
+ MTK_FUNCTION(1, "I0_I2SIN_D3"),
+ MTK_FUNCTION(2, "I0_SPLIN_D3"),
+ MTK_FUNCTION(3, "I0_DMIC3_DAT_R"),
+ MTK_FUNCTION(4, "B0_TDMIN_BCK"),
+ MTK_FUNCTION(5, "O_I2SO1_D0"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B15")
+ ),
+
+ MTK_PIN(
+ 114, "GPIO114",
+ MTK_EINT_FUNCTION(0, 114),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO114"),
+ MTK_FUNCTION(1, "O_I2SO2_MCK"),
+ MTK_FUNCTION(2, "B0_I2SIN_MCK"),
+ MTK_FUNCTION(3, "I1_MCUPM_JTAG_TMS"),
+ MTK_FUNCTION(4, "B1_APU_JTAG_TMS"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG1_TMS"),
+ MTK_FUNCTION(6, "I1_SPM_JTAG_TMS"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B16")
+ ),
+
+ MTK_PIN(
+ 115, "GPIO115",
+ MTK_EINT_FUNCTION(0, 115),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO115"),
+ MTK_FUNCTION(1, "B0_I2SO2_BCK"),
+ MTK_FUNCTION(2, "B0_I2SIN_BCK"),
+ MTK_FUNCTION(3, "I1_MCUPM_JTAG_TCK"),
+ MTK_FUNCTION(4, "I0_APU_JTAG_TCK"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG1_TCK"),
+ MTK_FUNCTION(6, "I1_SPM_JTAG_TCK"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B17")
+ ),
+
+ MTK_PIN(
+ 116, "GPIO116",
+ MTK_EINT_FUNCTION(0, 116),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO116"),
+ MTK_FUNCTION(1, "B0_I2SO2_WS"),
+ MTK_FUNCTION(2, "B0_I2SIN_WS"),
+ MTK_FUNCTION(3, "I1_MCUPM_JTAG_TDI"),
+ MTK_FUNCTION(4, "I1_APU_JTAG_TDI"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG1_TDI"),
+ MTK_FUNCTION(6, "I1_SPM_JTAG_TDI"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B18")
+ ),
+
+ MTK_PIN(
+ 117, "GPIO117",
+ MTK_EINT_FUNCTION(0, 117),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO117"),
+ MTK_FUNCTION(1, "O_I2SO2_D0"),
+ MTK_FUNCTION(2, "I0_I2SIN_D0"),
+ MTK_FUNCTION(3, "O_MCUPM_JTAG_TDO"),
+ MTK_FUNCTION(4, "O_APU_JTAG_TDO"),
+ MTK_FUNCTION(5, "O_SCP_JTAG1_TDO"),
+ MTK_FUNCTION(6, "O_SPM_JTAG_TDO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B19")
+ ),
+
+ MTK_PIN(
+ 118, "GPIO118",
+ MTK_EINT_FUNCTION(0, 118),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO118"),
+ MTK_FUNCTION(1, "O_I2SO2_D1"),
+ MTK_FUNCTION(2, "I0_I2SIN_D1"),
+ MTK_FUNCTION(3, "I0_MCUPM_JTAG_TRSTN"),
+ MTK_FUNCTION(4, "I0_APU_JTAG_TRST"),
+ MTK_FUNCTION(5, "I0_SCP_JTAG1_TRSTN"),
+ MTK_FUNCTION(6, "I0_SPM_JTAG_TRSTN"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B20")
+ ),
+
+ MTK_PIN(
+ 119, "GPIO119",
+ MTK_EINT_FUNCTION(0, 119),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO119"),
+ MTK_FUNCTION(1, "O_I2SO2_D2"),
+ MTK_FUNCTION(2, "I0_I2SIN_D2"),
+ MTK_FUNCTION(3, "O_UTXD3"),
+ MTK_FUNCTION(4, "B0_TDMIN_LRCK"),
+ MTK_FUNCTION(5, "O_I2SO1_MCK"),
+ MTK_FUNCTION(6, "O_SSPM_UTXD_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B21")
+ ),
+
+ MTK_PIN(
+ 120, "GPIO120",
+ MTK_EINT_FUNCTION(0, 120),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO120"),
+ MTK_FUNCTION(1, "O_I2SO2_D3"),
+ MTK_FUNCTION(2, "I0_I2SIN_D3"),
+ MTK_FUNCTION(3, "I1_URXD3"),
+ MTK_FUNCTION(4, "I0_TDMIN_DI"),
+ MTK_FUNCTION(5, "O_I2SO1_BCK"),
+ MTK_FUNCTION(6, "I1_SSPM_URXD_AO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B22")
+ ),
+
+ MTK_PIN(
+ 121, "GPIO121",
+ MTK_EINT_FUNCTION(0, 121),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO121"),
+ MTK_FUNCTION(1, "B0_PCM_CLK"),
+ MTK_FUNCTION(2, "O_SPIM4_CSB"),
+ MTK_FUNCTION(3, "O_SCP_SPI1_B_CS"),
+ MTK_FUNCTION(4, "O_TP_UTXD2_AO"),
+ MTK_FUNCTION(5, "O_AUXIF_ST0"),
+ MTK_FUNCTION(6, "O_PGD_DA_EFUSE_RDY"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B23")
+ ),
+
+ MTK_PIN(
+ 122, "GPIO122",
+ MTK_EINT_FUNCTION(0, 122),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO122"),
+ MTK_FUNCTION(1, "B0_PCM_SYNC"),
+ MTK_FUNCTION(2, "O_SPIM4_CLK"),
+ MTK_FUNCTION(3, "O_SCP_SPI1_B_CK"),
+ MTK_FUNCTION(4, "I1_TP_URXD2_AO"),
+ MTK_FUNCTION(5, "O_AUXIF_CLK0"),
+ MTK_FUNCTION(6, "O_PGD_DA_EFUSE_RDY_PRE"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B24")
+ ),
+
+ MTK_PIN(
+ 123, "GPIO123",
+ MTK_EINT_FUNCTION(0, 123),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO123"),
+ MTK_FUNCTION(1, "O_PCM_DO"),
+ MTK_FUNCTION(2, "B0_SPIM4_MOSI"),
+ MTK_FUNCTION(3, "O_SCP_SPI1_B_MO"),
+ MTK_FUNCTION(4, "O_TP_URTS2_AO"),
+ MTK_FUNCTION(5, "O_AUXIF_ST1"),
+ MTK_FUNCTION(6, "O_PGD_DA_PWRGD_RESET"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B25")
+ ),
+
+ MTK_PIN(
+ 124, "GPIO124",
+ MTK_EINT_FUNCTION(0, 124),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO124"),
+ MTK_FUNCTION(1, "I0_PCM_DI"),
+ MTK_FUNCTION(2, "B0_SPIM4_MISO"),
+ MTK_FUNCTION(3, "I0_SCP_SPI1_B_MI"),
+ MTK_FUNCTION(4, "I1_TP_UCTS2_AO"),
+ MTK_FUNCTION(5, "O_AUXIF_CLK1"),
+ MTK_FUNCTION(6, "O_PGD_DA_PWRGD_ENB"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B26")
+ ),
+
+ MTK_PIN(
+ 125, "GPIO125",
+ MTK_EINT_FUNCTION(0, 125),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO125"),
+ MTK_FUNCTION(1, "O_DMIC1_CLK"),
+ MTK_FUNCTION(2, "O_SPINOR_CK"),
+ MTK_FUNCTION(3, "B0_TDMIN_MCK"),
+ MTK_FUNCTION(6, "O_LVTS_FOUT"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B27")
+ ),
+
+ MTK_PIN(
+ 126, "GPIO126",
+ MTK_EINT_FUNCTION(0, 126),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO126"),
+ MTK_FUNCTION(1, "I0_DMIC1_DAT"),
+ MTK_FUNCTION(2, "O_SPINOR_CS"),
+ MTK_FUNCTION(3, "B0_TDMIN_BCK"),
+ MTK_FUNCTION(6, "O_LVTS_SDO"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B28")
+ ),
+
+ MTK_PIN(
+ 127, "GPIO127",
+ MTK_EINT_FUNCTION(0, 127),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO127"),
+ MTK_FUNCTION(1, "I0_DMIC1_DAT_R"),
+ MTK_FUNCTION(2, "B0_SPINOR_IO0"),
+ MTK_FUNCTION(3, "B0_TDMIN_LRCK"),
+ MTK_FUNCTION(6, "I0_LVTS_26M"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B29")
+ ),
+
+ MTK_PIN(
+ 128, "GPIO128",
+ MTK_EINT_FUNCTION(0, 128),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO128"),
+ MTK_FUNCTION(1, "O_DMIC2_CLK"),
+ MTK_FUNCTION(2, "B0_SPINOR_IO1"),
+ MTK_FUNCTION(3, "I0_TDMIN_DI"),
+ MTK_FUNCTION(6, "I0_LVTS_SCF"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B30")
+ ),
+
+ MTK_PIN(
+ 129, "GPIO129",
+ MTK_EINT_FUNCTION(0, 129),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO129"),
+ MTK_FUNCTION(1, "I0_DMIC2_DAT"),
+ MTK_FUNCTION(2, "B0_SPINOR_IO2"),
+ MTK_FUNCTION(3, "I0_SPDIF_IN1"),
+ MTK_FUNCTION(6, "I0_LVTS_SCK"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B31")
+ ),
+
+ MTK_PIN(
+ 130, "GPIO130",
+ MTK_EINT_FUNCTION(0, 130),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO130"),
+ MTK_FUNCTION(1, "I0_DMIC2_DAT_R"),
+ MTK_FUNCTION(2, "B0_SPINOR_IO3"),
+ MTK_FUNCTION(3, "I0_SPDIF_IN2"),
+ MTK_FUNCTION(6, "I0_LVTS_SDI"),
+ MTK_FUNCTION(7, "B0_DBG_MON_B32")
+ ),
+
+ MTK_PIN(
+ 131, "GPIO131",
+ MTK_EINT_FUNCTION(0, 131),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO131"),
+ MTK_FUNCTION(1, "O_DPI_D0"),
+ MTK_FUNCTION(2, "O_GBE_TXD3"),
+ MTK_FUNCTION(3, "O_DMIC1_CLK"),
+ MTK_FUNCTION(4, "O_I2SO2_MCK"),
+ MTK_FUNCTION(5, "B0_TP_GPIO0_AO"),
+ MTK_FUNCTION(6, "O_SPIM5_CSB"),
+ MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR0")
+ ),
+
+ MTK_PIN(
+ 132, "GPIO132",
+ MTK_EINT_FUNCTION(0, 132),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO132"),
+ MTK_FUNCTION(1, "O_DPI_D1"),
+ MTK_FUNCTION(2, "O_GBE_TXD2"),
+ MTK_FUNCTION(3, "I0_DMIC1_DAT"),
+ MTK_FUNCTION(4, "B0_I2SO2_BCK"),
+ MTK_FUNCTION(5, "B0_TP_GPIO1_AO"),
+ MTK_FUNCTION(6, "O_SPIM5_CLK"),
+ MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR1")
+ ),
+
+ MTK_PIN(
+ 133, "GPIO133",
+ MTK_EINT_FUNCTION(0, 133),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO133"),
+ MTK_FUNCTION(1, "O_DPI_D2"),
+ MTK_FUNCTION(2, "O_GBE_TXD1"),
+ MTK_FUNCTION(3, "I0_DMIC1_DAT_R"),
+ MTK_FUNCTION(4, "B0_I2SO2_WS"),
+ MTK_FUNCTION(5, "B0_TP_GPIO2_AO"),
+ MTK_FUNCTION(6, "B0_SPIM5_MOSI"),
+ MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR2")
+ ),
+
+ MTK_PIN(
+ 134, "GPIO134",
+ MTK_EINT_FUNCTION(0, 134),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO134"),
+ MTK_FUNCTION(1, "O_DPI_D3"),
+ MTK_FUNCTION(2, "O_GBE_TXD0"),
+ MTK_FUNCTION(3, "O_DMIC2_CLK"),
+ MTK_FUNCTION(4, "O_I2SO2_D0"),
+ MTK_FUNCTION(5, "B0_TP_GPIO3_AO"),
+ MTK_FUNCTION(6, "B0_SPIM5_MISO"),
+ MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR3")
+ ),
+
+ MTK_PIN(
+ 135, "GPIO135",
+ MTK_EINT_FUNCTION(0, 135),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO135"),
+ MTK_FUNCTION(1, "O_DPI_D4"),
+ MTK_FUNCTION(2, "I0_GBE_RXD3"),
+ MTK_FUNCTION(3, "I0_DMIC2_DAT"),
+ MTK_FUNCTION(4, "O_I2SO2_D1"),
+ MTK_FUNCTION(5, "B0_TP_GPIO4_AO"),
+ MTK_FUNCTION(6, "I1_WAKEN"),
+ MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR4")
+ ),
+
+ MTK_PIN(
+ 136, "GPIO136",
+ MTK_EINT_FUNCTION(0, 136),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO136"),
+ MTK_FUNCTION(1, "O_DPI_D5"),
+ MTK_FUNCTION(2, "I0_GBE_RXD2"),
+ MTK_FUNCTION(3, "I0_DMIC2_DAT_R"),
+ MTK_FUNCTION(4, "O_I2SO2_D2"),
+ MTK_FUNCTION(5, "B0_TP_GPIO5_AO"),
+ MTK_FUNCTION(6, "O_PERSTN"),
+ MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR5")
+ ),
+
+ MTK_PIN(
+ 137, "GPIO137",
+ MTK_EINT_FUNCTION(0, 137),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO137"),
+ MTK_FUNCTION(1, "O_DPI_D6"),
+ MTK_FUNCTION(2, "I0_GBE_RXD1"),
+ MTK_FUNCTION(3, "O_DMIC3_CLK"),
+ MTK_FUNCTION(4, "O_I2SO2_D3"),
+ MTK_FUNCTION(5, "B0_TP_GPIO6_AO"),
+ MTK_FUNCTION(6, "B1_CLKREQN"),
+ MTK_FUNCTION(7, "O_PWM_0")
+ ),
+
+ MTK_PIN(
+ 138, "GPIO138",
+ MTK_EINT_FUNCTION(0, 138),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO138"),
+ MTK_FUNCTION(1, "O_DPI_D7"),
+ MTK_FUNCTION(2, "I0_GBE_RXD0"),
+ MTK_FUNCTION(3, "I0_DMIC3_DAT"),
+ MTK_FUNCTION(4, "O_CLKM2"),
+ MTK_FUNCTION(5, "B0_TP_GPIO7_AO"),
+ MTK_FUNCTION(7, "B0_MD32_0_GPIO0")
+ ),
+
+ MTK_PIN(
+ 139, "GPIO139",
+ MTK_EINT_FUNCTION(0, 139),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO139"),
+ MTK_FUNCTION(1, "O_DPI_D8"),
+ MTK_FUNCTION(2, "B0_GBE_TXC"),
+ MTK_FUNCTION(3, "I0_DMIC3_DAT_R"),
+ MTK_FUNCTION(4, "O_CLKM3"),
+ MTK_FUNCTION(5, "O_TP_UTXD2_AO"),
+ MTK_FUNCTION(6, "O_UTXD2"),
+ MTK_FUNCTION(7, "B0_MD32_0_GPIO1")
+ ),
+
+ MTK_PIN(
+ 140, "GPIO140",
+ MTK_EINT_FUNCTION(0, 140),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO140"),
+ MTK_FUNCTION(1, "O_DPI_D9"),
+ MTK_FUNCTION(2, "I0_GBE_RXC"),
+ MTK_FUNCTION(3, "O_DMIC4_CLK"),
+ MTK_FUNCTION(4, "O_PWM_2"),
+ MTK_FUNCTION(5, "I1_TP_URXD2_AO"),
+ MTK_FUNCTION(6, "I1_URXD2"),
+ MTK_FUNCTION(7, "B0_MD32_0_GPIO2")
+ ),
+
+ MTK_PIN(
+ 141, "GPIO141",
+ MTK_EINT_FUNCTION(0, 141),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO141"),
+ MTK_FUNCTION(1, "O_DPI_D10"),
+ MTK_FUNCTION(2, "I0_GBE_RXDV"),
+ MTK_FUNCTION(3, "I0_DMIC4_DAT"),
+ MTK_FUNCTION(4, "O_PWM_3"),
+ MTK_FUNCTION(5, "O_TP_URTS2_AO"),
+ MTK_FUNCTION(6, "O_URTS2"),
+ MTK_FUNCTION(7, "B0_MD32_1_GPIO0")
+ ),
+
+ MTK_PIN(
+ 142, "GPIO142",
+ MTK_EINT_FUNCTION(0, 142),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO142"),
+ MTK_FUNCTION(1, "O_DPI_D11"),
+ MTK_FUNCTION(2, "O_GBE_TXEN"),
+ MTK_FUNCTION(3, "I0_DMIC4_DAT_R"),
+ MTK_FUNCTION(4, "O_PWM_1"),
+ MTK_FUNCTION(5, "I1_TP_UCTS2_AO"),
+ MTK_FUNCTION(6, "I1_UCTS2"),
+ MTK_FUNCTION(7, "B0_MD32_1_GPIO1")
+ ),
+
+ MTK_PIN(
+ 143, "GPIO143",
+ MTK_EINT_FUNCTION(0, 143),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO143"),
+ MTK_FUNCTION(1, "O_DPI_D12"),
+ MTK_FUNCTION(2, "O_GBE_MDC"),
+ MTK_FUNCTION(3, "B0_MD32_0_GPIO0"),
+ MTK_FUNCTION(4, "O_CLKM0"),
+ MTK_FUNCTION(5, "O_SPIM3_CSB"),
+ MTK_FUNCTION(6, "O_UTXD1"),
+ MTK_FUNCTION(7, "B0_MD32_1_GPIO2")
+ ),
+
+ MTK_PIN(
+ 144, "GPIO144",
+ MTK_EINT_FUNCTION(0, 144),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO144"),
+ MTK_FUNCTION(1, "O_DPI_D13"),
+ MTK_FUNCTION(2, "B1_GBE_MDIO"),
+ MTK_FUNCTION(3, "B0_MD32_0_GPIO1"),
+ MTK_FUNCTION(4, "O_CLKM1"),
+ MTK_FUNCTION(5, "O_SPIM3_CLK"),
+ MTK_FUNCTION(6, "I1_URXD1"),
+ MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR0")
+ ),
+
+ MTK_PIN(
+ 145, "GPIO145",
+ MTK_EINT_FUNCTION(0, 145),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO145"),
+ MTK_FUNCTION(1, "O_DPI_D14"),
+ MTK_FUNCTION(2, "O_GBE_TXER"),
+ MTK_FUNCTION(3, "B0_MD32_1_GPIO0"),
+ MTK_FUNCTION(4, "O_CMFLASH0"),
+ MTK_FUNCTION(5, "B0_SPIM3_MOSI"),
+ MTK_FUNCTION(6, "B0_GBE_AUX_PPS2"),
+ MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR1")
+ ),
+
+ MTK_PIN(
+ 146, "GPIO146",
+ MTK_EINT_FUNCTION(0, 146),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO146"),
+ MTK_FUNCTION(1, "O_DPI_D15"),
+ MTK_FUNCTION(2, "I0_GBE_RXER"),
+ MTK_FUNCTION(3, "B0_MD32_1_GPIO1"),
+ MTK_FUNCTION(4, "O_CMFLASH1"),
+ MTK_FUNCTION(5, "B0_SPIM3_MISO"),
+ MTK_FUNCTION(6, "B0_GBE_AUX_PPS3"),
+ MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR2")
+ ),
+
+ MTK_PIN(
+ 147, "GPIO147",
+ MTK_EINT_FUNCTION(0, 147),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO147"),
+ MTK_FUNCTION(1, "O_DPI_HSYNC"),
+ MTK_FUNCTION(2, "I0_GBE_COL"),
+ MTK_FUNCTION(3, "O_I2SO1_MCK"),
+ MTK_FUNCTION(4, "O_CMVREF0"),
+ MTK_FUNCTION(5, "O_SPDIF_OUT"),
+ MTK_FUNCTION(6, "O_URTS1"),
+ MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR3")
+ ),
+
+ MTK_PIN(
+ 148, "GPIO148",
+ MTK_EINT_FUNCTION(0, 148),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO148"),
+ MTK_FUNCTION(1, "O_DPI_VSYNC"),
+ MTK_FUNCTION(2, "I0_GBE_INTR"),
+ MTK_FUNCTION(3, "O_I2SO1_BCK"),
+ MTK_FUNCTION(4, "O_CMVREF1"),
+ MTK_FUNCTION(5, "I0_SPDIF_IN0"),
+ MTK_FUNCTION(6, "I1_UCTS1"),
+ MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR4")
+ ),
+
+ MTK_PIN(
+ 149, "GPIO149",
+ MTK_EINT_FUNCTION(0, 149),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO149"),
+ MTK_FUNCTION(1, "O_DPI_DE"),
+ MTK_FUNCTION(2, "B0_GBE_AUX_PPS0"),
+ MTK_FUNCTION(3, "O_I2SO1_WS"),
+ MTK_FUNCTION(4, "O_CMVREF2"),
+ MTK_FUNCTION(5, "I0_SPDIF_IN1"),
+ MTK_FUNCTION(6, "O_UTXD3"),
+ MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR5")
+ ),
+
+ MTK_PIN(
+ 150, "GPIO150",
+ MTK_EINT_FUNCTION(0, 150),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO150"),
+ MTK_FUNCTION(1, "O_DPI_CK"),
+ MTK_FUNCTION(2, "B0_GBE_AUX_PPS1"),
+ MTK_FUNCTION(3, "O_I2SO1_D0"),
+ MTK_FUNCTION(4, "O_CMVREF3"),
+ MTK_FUNCTION(5, "I0_SPDIF_IN2"),
+ MTK_FUNCTION(6, "I1_URXD3")
+ ),
+
+ MTK_PIN(
+ 151, "GPIO151",
+ MTK_EINT_FUNCTION(0, 151),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO151"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT7")
+ ),
+
+ MTK_PIN(
+ 152, "GPIO152",
+ MTK_EINT_FUNCTION(0, 152),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO152"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT6")
+ ),
+
+ MTK_PIN(
+ 153, "GPIO153",
+ MTK_EINT_FUNCTION(0, 153),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO153"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT5")
+ ),
+
+ MTK_PIN(
+ 154, "GPIO154",
+ MTK_EINT_FUNCTION(0, 154),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO154"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT4")
+ ),
+
+ MTK_PIN(
+ 155, "GPIO155",
+ MTK_EINT_FUNCTION(0, 155),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO155"),
+ MTK_FUNCTION(1, "O_MSDC0_RSTB")
+ ),
+
+ MTK_PIN(
+ 156, "GPIO156",
+ MTK_EINT_FUNCTION(0, 156),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO156"),
+ MTK_FUNCTION(1, "B1_MSDC0_CMD")
+ ),
+
+ MTK_PIN(
+ 157, "GPIO157",
+ MTK_EINT_FUNCTION(0, 157),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO157"),
+ MTK_FUNCTION(1, "B1_MSDC0_CLK")
+ ),
+
+ MTK_PIN(
+ 158, "GPIO158",
+ MTK_EINT_FUNCTION(0, 158),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO158"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT3")
+ ),
+
+ MTK_PIN(
+ 159, "GPIO159",
+ MTK_EINT_FUNCTION(0, 159),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO159"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT2")
+ ),
+
+ MTK_PIN(
+ 160, "GPIO160",
+ MTK_EINT_FUNCTION(0, 160),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO160"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT1")
+ ),
+
+ MTK_PIN(
+ 161, "GPIO161",
+ MTK_EINT_FUNCTION(0, 161),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO161"),
+ MTK_FUNCTION(1, "B1_MSDC0_DAT0")
+ ),
+
+ MTK_PIN(
+ 162, "GPIO162",
+ MTK_EINT_FUNCTION(0, 162),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO162"),
+ MTK_FUNCTION(1, "B0_MSDC0_DSL")
+ ),
+
+ MTK_PIN(
+ 163, "GPIO163",
+ MTK_EINT_FUNCTION(0, 163),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO163"),
+ MTK_FUNCTION(1, "B1_MSDC1_CMD"),
+ MTK_FUNCTION(2, "O_SPDIF_OUT"),
+ MTK_FUNCTION(3, "I1_MD32_0_JTAG_TMS"),
+ MTK_FUNCTION(4, "I1_ADSP_JTAG0_TMS"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG0_TMS"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TMS"),
+ MTK_FUNCTION(7, "I0_IPU_JTAG_TMS")
+ ),
+
+ MTK_PIN(
+ 164, "GPIO164",
+ MTK_EINT_FUNCTION(0, 164),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO164"),
+ MTK_FUNCTION(1, "B1_MSDC1_CLK"),
+ MTK_FUNCTION(2, "I0_SPDIF_IN0"),
+ MTK_FUNCTION(3, "I1_MD32_0_JTAG_TCK"),
+ MTK_FUNCTION(4, "I0_ADSP_JTAG0_TCK"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG0_TCK"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TCK"),
+ MTK_FUNCTION(7, "I0_IPU_JTAG_TCK")
+ ),
+
+ MTK_PIN(
+ 165, "GPIO165",
+ MTK_EINT_FUNCTION(0, 165),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO165"),
+ MTK_FUNCTION(1, "B1_MSDC1_DAT0"),
+ MTK_FUNCTION(2, "I0_SPDIF_IN1"),
+ MTK_FUNCTION(3, "I1_MD32_0_JTAG_TDI"),
+ MTK_FUNCTION(4, "I1_ADSP_JTAG0_TDI"),
+ MTK_FUNCTION(5, "I1_SCP_JTAG0_TDI"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TDI"),
+ MTK_FUNCTION(7, "I0_IPU_JTAG_TDI")
+ ),
+
+ MTK_PIN(
+ 166, "GPIO166",
+ MTK_EINT_FUNCTION(0, 166),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO166"),
+ MTK_FUNCTION(1, "B1_MSDC1_DAT1"),
+ MTK_FUNCTION(2, "I0_SPDIF_IN2"),
+ MTK_FUNCTION(3, "O_MD32_0_JTAG_TDO"),
+ MTK_FUNCTION(4, "O_ADSP_JTAG0_TDO"),
+ MTK_FUNCTION(5, "O_SCP_JTAG0_TDO"),
+ MTK_FUNCTION(6, "O_CCU0_JTAG_TDO"),
+ MTK_FUNCTION(7, "O_IPU_JTAG_TDO")
+ ),
+
+ MTK_PIN(
+ 167, "GPIO167",
+ MTK_EINT_FUNCTION(0, 167),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO167"),
+ MTK_FUNCTION(1, "B1_MSDC1_DAT2"),
+ MTK_FUNCTION(2, "O_PWM_0"),
+ MTK_FUNCTION(3, "I1_MD32_0_JTAG_TRST"),
+ MTK_FUNCTION(4, "I1_ADSP_JTAG0_TRSTN"),
+ MTK_FUNCTION(5, "I0_SCP_JTAG0_TRSTN"),
+ MTK_FUNCTION(6, "I1_CCU0_JTAG_TRST"),
+ MTK_FUNCTION(7, "I0_IPU_JTAG_TRST")
+ ),
+
+ MTK_PIN(
+ 168, "GPIO168",
+ MTK_EINT_FUNCTION(0, 168),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO168"),
+ MTK_FUNCTION(1, "B1_MSDC1_DAT3"),
+ MTK_FUNCTION(2, "O_PWM_1"),
+ MTK_FUNCTION(3, "O_CLKM0")
+ ),
+
+ MTK_PIN(
+ 169, "GPIO169",
+ MTK_EINT_FUNCTION(0, 169),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO169"),
+ MTK_FUNCTION(1, "B1_MSDC2_CMD"),
+ MTK_FUNCTION(2, "O_LVTS_FOUT"),
+ MTK_FUNCTION(3, "I1_MD32_1_JTAG_TMS"),
+ MTK_FUNCTION(4, "I0_UDI_TMS"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_TMS"),
+ MTK_FUNCTION(6, "B0_TDMIN_MCK"),
+ MTK_FUNCTION(7, "I1_SSPM_JTAG_TMS")
+ ),
+
+ MTK_PIN(
+ 170, "GPIO170",
+ MTK_EINT_FUNCTION(0, 170),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO170"),
+ MTK_FUNCTION(1, "B1_MSDC2_CLK"),
+ MTK_FUNCTION(2, "O_LVTS_SDO"),
+ MTK_FUNCTION(3, "I1_MD32_1_JTAG_TCK"),
+ MTK_FUNCTION(4, "I0_UDI_TCK"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_TCK"),
+ MTK_FUNCTION(6, "B0_TDMIN_BCK"),
+ MTK_FUNCTION(7, "I1_SSPM_JTAG_TCK")
+ ),
+
+ MTK_PIN(
+ 171, "GPIO171",
+ MTK_EINT_FUNCTION(0, 171),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO171"),
+ MTK_FUNCTION(1, "B1_MSDC2_DAT0"),
+ MTK_FUNCTION(2, "I0_LVTS_26M"),
+ MTK_FUNCTION(3, "I1_MD32_1_JTAG_TDI"),
+ MTK_FUNCTION(4, "I0_UDI_TDI"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_TDI"),
+ MTK_FUNCTION(6, "B0_TDMIN_LRCK"),
+ MTK_FUNCTION(7, "I1_SSPM_JTAG_TDI")
+ ),
+
+ MTK_PIN(
+ 172, "GPIO172",
+ MTK_EINT_FUNCTION(0, 172),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO172"),
+ MTK_FUNCTION(1, "B1_MSDC2_DAT1"),
+ MTK_FUNCTION(2, "I0_LVTS_SCF"),
+ MTK_FUNCTION(3, "O_MD32_1_JTAG_TDO"),
+ MTK_FUNCTION(4, "O_UDI_TDO"),
+ MTK_FUNCTION(5, "O_VPU_UDI_TDO"),
+ MTK_FUNCTION(6, "I0_TDMIN_DI"),
+ MTK_FUNCTION(7, "O_SSPM_JTAG_TDO")
+ ),
+
+ MTK_PIN(
+ 173, "GPIO173",
+ MTK_EINT_FUNCTION(0, 173),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO173"),
+ MTK_FUNCTION(1, "B1_MSDC2_DAT2"),
+ MTK_FUNCTION(2, "I0_LVTS_SCK"),
+ MTK_FUNCTION(3, "I1_MD32_1_JTAG_TRST"),
+ MTK_FUNCTION(4, "I0_UDI_NTRST"),
+ MTK_FUNCTION(5, "I0_VPU_UDI_NTRST"),
+ MTK_FUNCTION(7, "I0_SSPM_JTAG_TRSTN")
+ ),
+
+ MTK_PIN(
+ 174, "GPIO174",
+ MTK_EINT_FUNCTION(0, 174),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO174"),
+ MTK_FUNCTION(1, "B1_MSDC2_DAT3"),
+ MTK_FUNCTION(2, "I0_LVTS_SDI")
+ ),
+
+ MTK_PIN(
+ 175, "GPIO175",
+ MTK_EINT_FUNCTION(0, 175),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO175"),
+ MTK_FUNCTION(1, "B0_SPMI_M_SCL")
+ ),
+
+ MTK_PIN(
+ 176, "GPIO176",
+ MTK_EINT_FUNCTION(0, 176),
+ DRV_GRP4,
+ MTK_FUNCTION(0, "B_GPIO176"),
+ MTK_FUNCTION(1, "B0_SPMI_M_SDA")
+ ),
+
+ MTK_PIN(
+ 177, "GPIO177",
+ MTK_EINT_FUNCTION(0, 212),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 178, "GPIO178",
+ MTK_EINT_FUNCTION(0, 213),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 179, "GPIO179",
+ MTK_EINT_FUNCTION(0, 214),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 180, "GPIO180",
+ MTK_EINT_FUNCTION(0, 215),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 181, "GPIO181",
+ MTK_EINT_FUNCTION(0, 216),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 182, "GPIO182",
+ MTK_EINT_FUNCTION(0, 217),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 183, "GPIO183",
+ MTK_EINT_FUNCTION(0, 218),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 184, "GPIO184",
+ MTK_EINT_FUNCTION(0, 219),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 185, "GPIO185",
+ MTK_EINT_FUNCTION(0, 220),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 186, "GPIO186",
+ MTK_EINT_FUNCTION(0, 221),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 187, "GPIO187",
+ MTK_EINT_FUNCTION(0, 222),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 188, "GPIO188",
+ MTK_EINT_FUNCTION(0, 223),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ ),
+
+ MTK_PIN(
+ 189, "GPIO189",
+ MTK_EINT_FUNCTION(0, 224),
+ DRV_FIXED,
+ MTK_FUNCTION(0, NULL)
+ )
+};
+
+#endif /* __PINCTRL__MTK_MT8188_H */
pc->chip.label = pc->data->name;
pc->chip.parent = pc->dev;
+ pc->chip.fwnode = pc->fwnode;
pc->chip.request = gpiochip_generic_request;
pc->chip.free = gpiochip_generic_free;
pc->chip.set_config = gpiochip_generic_config;
pc->chip.base = -1;
pc->chip.ngpio = pc->data->num_pins;
pc->chip.can_sleep = false;
- pc->chip.of_node = pc->of_node;
- pc->chip.of_gpio_n_cells = 2;
ret = gpiochip_add_data(&pc->chip, pc);
if (ret) {
return -EINVAL;
}
- gpio_np = to_of_node(gpiochip_node_get_first(pc->dev));
- pc->of_node = gpio_np;
+ pc->fwnode = gpiochip_node_get_first(pc->dev);
+ gpio_np = to_of_node(pc->fwnode);
pc->reg_mux = meson_map_resource(pc, gpio_np, "mux");
if (IS_ERR_OR_NULL(pc->reg_mux)) {
#include <linux/types.h>
#include <linux/module.h>
+struct fwnode_handle;
+
struct meson_pinctrl;
/**
struct regmap *reg_gpio;
struct regmap *reg_ds;
struct gpio_chip chip;
- struct device_node *of_node;
+ struct fwnode_handle *fwnode;
};
#define FUNCTION(fn) \
struct armada_37xx_pm_state pm;
};
-#define PIN_GRP(_name, _start, _nr, _mask, _func1, _func2) \
+#define PIN_GRP_GPIO_0(_name, _start, _nr) \
{ \
.name = _name, \
.start_pin = _start, \
.npins = _nr, \
- .reg_mask = _mask, \
- .val = {0, _mask}, \
- .funcs = {_func1, _func2} \
+ .reg_mask = 0, \
+ .val = {0}, \
+ .funcs = {"gpio"} \
}
#define PIN_GRP_GPIO(_name, _start, _nr, _mask, _func1) \
"pwm", "led"),
PIN_GRP_GPIO("pmic1", 7, 1, BIT(7), "pmic"),
PIN_GRP_GPIO("pmic0", 6, 1, BIT(8), "pmic"),
+ PIN_GRP_GPIO_0("gpio1_5", 5, 1),
PIN_GRP_GPIO("i2c2", 2, 2, BIT(9), "i2c"),
PIN_GRP_GPIO("i2c1", 0, 2, BIT(10), "i2c"),
PIN_GRP_GPIO("spi_cs1", 17, 1, BIT(12), "spi"),
static struct armada_37xx_pin_group armada_37xx_sb_groups[] = {
PIN_GRP_GPIO("usb32_drvvbus0", 0, 1, BIT(0), "drvbus"),
PIN_GRP_GPIO("usb2_drvvbus1", 1, 1, BIT(1), "drvbus"),
+ PIN_GRP_GPIO_0("gpio2_2", 2, 1),
PIN_GRP_GPIO("sdio_sb", 24, 6, BIT(2), "sdio"),
PIN_GRP_GPIO("rgmii", 6, 12, BIT(3), "mii"),
PIN_GRP_GPIO("smi", 18, 2, BIT(4), "smi"),
PIN_GRP_GPIO("pcie1", 3, 1, BIT(5), "pcie"), /* this actually controls "pcie1_reset" */
PIN_GRP_GPIO("pcie1_clkreq", 4, 1, BIT(9), "pcie"),
PIN_GRP_GPIO("pcie1_wakeup", 5, 1, BIT(10), "pcie"),
- PIN_GRP_GPIO("ptp", 20, 3, BIT(11) | BIT(12) | BIT(13), "ptp"),
- PIN_GRP("ptp_clk", 21, 1, BIT(6), "ptp", "mii"),
- PIN_GRP("ptp_trig", 22, 1, BIT(7), "ptp", "mii"),
+ PIN_GRP_GPIO("ptp", 20, 1, BIT(11), "ptp"),
+ PIN_GRP_GPIO_3("ptp_clk", 21, 1, BIT(6) | BIT(12), 0, BIT(6), BIT(12),
+ "ptp", "mii"),
+ PIN_GRP_GPIO_3("ptp_trig", 22, 1, BIT(7) | BIT(13), 0, BIT(7), BIT(13),
+ "ptp", "mii"),
PIN_GRP_GPIO_3("mii_col", 23, 1, BIT(8) | BIT(14), 0, BIT(8), BIT(14),
"mii", "mii_err"),
};
struct armada_37xx_pinctrl *info = pinctrl_dev_get_drvdata(pctldev);
struct armada_37xx_pin_group *group;
int grp = 0;
+ int ret;
dev_dbg(info->dev, "requesting gpio %d\n", offset);
- while ((group = armada_37xx_find_next_grp_by_pin(info, offset, &grp)))
- armada_37xx_pmx_set_by_name(pctldev, "gpio", group);
+ while ((group = armada_37xx_find_next_grp_by_pin(info, offset, &grp))) {
+ ret = armada_37xx_pmx_set_by_name(pctldev, "gpio", group);
+ if (ret)
+ return ret;
+ }
return 0;
}
DB8500_PIN_D21, DB8500_PIN_D20, DB8500_PIN_C20, DB8500_PIN_B21,
DB8500_PIN_C21, DB8500_PIN_A22, DB8500_PIN_B24, DB8500_PIN_C22 };
-#define DB8500_PIN_GROUP(a, b) { .name = #a, .pins = a##_pins, \
- .npins = ARRAY_SIZE(a##_pins), .altsetting = b }
-
static const struct nmk_pingroup nmk_db8500_groups[] = {
/* Altfunction A column */
- DB8500_PIN_GROUP(u0_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(u1rxtx_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(u1ctsrts_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(ipi2c_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(ipi2c_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp0txrx_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp0tfstck_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp0rfsrck_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc0_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc0_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc0_dat47_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc0dat31dir_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp1txrx_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp1_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcdb_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcdvsi0_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcdvsi1_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcd_d0_d7_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcd_d8_d11_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcd_d12_d15_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(lcd_d12_d23_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(kp_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(kpskaskb_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc2_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc2_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(ssp1_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(ssp0_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(ipgpio0_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(ipgpio1_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(modem_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(kp_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp2sck_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(msp2_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc4_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc1_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc1_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(mc1dir_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(hsir_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(hsit_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(hsit_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(clkout1_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(clkout1_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(clkout2_a_1, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(clkout2_a_2, NMK_GPIO_ALT_A),
- DB8500_PIN_GROUP(usb_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(u0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(u1rxtx_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(u1ctsrts_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(ipi2c_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(ipi2c_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp0txrx_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp0tfstck_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp0rfsrck_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc0_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc0_dat47_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc0dat31dir_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp1txrx_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcdb_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcdvsi0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcdvsi1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcd_d0_d7_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcd_d8_d11_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcd_d12_d15_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(lcd_d12_d23_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(kp_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(kpskaskb_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc2_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc2_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(ssp1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(ssp0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(ipgpio0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(ipgpio1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(modem_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(kp_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp2sck_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(msp2_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc4_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc1_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mc1dir_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(hsir_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(hsit_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(hsit_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(clkout1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(clkout1_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(clkout2_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(clkout2_a_2, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(usb_a_1, NMK_GPIO_ALT_A),
/* Altfunction B column */
- DB8500_PIN_GROUP(trig_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(i2c4_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(i2c1_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(i2c2_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(i2c2_b_2, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(msp0txrx_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(i2c1_b_2, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(u2rxtx_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(uartmodtx_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(msp0sck_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(uartmodrx_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(stmmod_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(uartmodrx_b_2, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(spi3_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(msp1txrx_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(kp_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(kp_b_2, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(sm_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(smcs0_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(smcs1_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(ipgpio7_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(ipgpio2_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(ipgpio3_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(lcdaclk_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(lcda_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(lcd_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(lcd_d16_d23_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(ddrtrig_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(pwl_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(spi1_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(mc3_b_1, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(pwl_b_2, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(pwl_b_3, NMK_GPIO_ALT_B),
- DB8500_PIN_GROUP(pwl_b_4, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(trig_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(i2c4_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(i2c1_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(i2c2_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(i2c2_b_2, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(msp0txrx_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(i2c1_b_2, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(u2rxtx_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(uartmodtx_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(msp0sck_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(uartmodrx_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(stmmod_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(uartmodrx_b_2, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(spi3_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(msp1txrx_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(kp_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(kp_b_2, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(sm_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(smcs0_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(smcs1_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(ipgpio7_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(ipgpio2_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(ipgpio3_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(lcdaclk_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(lcda_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(lcd_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(lcd_d16_d23_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(ddrtrig_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(pwl_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(spi1_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(mc3_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(pwl_b_2, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(pwl_b_3, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(pwl_b_4, NMK_GPIO_ALT_B),
/* Altfunction C column */
- DB8500_PIN_GROUP(ipjtag_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio6_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio0_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio1_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio3_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio2_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(slim0_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ms_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(iptrigout_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(u2rxtx_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(u2ctsrts_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(u0_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio4_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio5_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio6_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio7_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(smcleale_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(stmape_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(u2rxtx_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio2_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio3_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio4_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(ipgpio5_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(mc5_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(mc2rstn_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(kp_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(smps0_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(smps1_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(u2rxtx_c_3, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(stmape_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(uartmodrx_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(uartmodtx_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(stmmod_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(usbsim_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(mc4rstn_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(clkout1_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(clkout2_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(i2c3_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(spi0_c_1, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(usbsim_c_2, NMK_GPIO_ALT_C),
- DB8500_PIN_GROUP(i2c3_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipjtag_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio6_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio0_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio1_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio3_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio2_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(slim0_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ms_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(iptrigout_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(u2rxtx_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(u2ctsrts_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(u0_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio4_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio5_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio6_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio7_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(smcleale_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(stmape_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(u2rxtx_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio2_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio3_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio4_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(ipgpio5_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(mc5_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(mc2rstn_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(kp_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(smps0_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(smps1_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(u2rxtx_c_3, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(stmape_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(uartmodrx_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(uartmodtx_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(stmmod_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(usbsim_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(mc4rstn_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(clkout1_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(clkout2_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(i2c3_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(spi0_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(usbsim_c_2, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(i2c3_c_2, NMK_GPIO_ALT_C),
/* Other alt C1 column */
- DB8500_PIN_GROUP(u2rx_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(stmape_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(remap0_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(remap1_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(ptma9_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(kp_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(rf_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(hxclk_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(uartmodrx_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(uartmodtx_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(stmmod_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(hxgpio_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(rf_oc1_2, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(spi2_oc1_1, NMK_GPIO_ALT_C1),
- DB8500_PIN_GROUP(spi2_oc1_2, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(u2rx_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(stmape_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(remap0_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(remap1_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(ptma9_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(kp_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(rf_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(hxclk_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(uartmodrx_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(uartmodtx_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(stmmod_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(hxgpio_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(rf_oc1_2, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(spi2_oc1_1, NMK_GPIO_ALT_C1),
+ NMK_PIN_GROUP(spi2_oc1_2, NMK_GPIO_ALT_C1),
/* Other alt C2 column */
- DB8500_PIN_GROUP(sbag_oc2_1, NMK_GPIO_ALT_C2),
- DB8500_PIN_GROUP(etmr4_oc2_1, NMK_GPIO_ALT_C2),
- DB8500_PIN_GROUP(ptma9_oc2_1, NMK_GPIO_ALT_C2),
+ NMK_PIN_GROUP(sbag_oc2_1, NMK_GPIO_ALT_C2),
+ NMK_PIN_GROUP(etmr4_oc2_1, NMK_GPIO_ALT_C2),
+ NMK_PIN_GROUP(ptma9_oc2_1, NMK_GPIO_ALT_C2),
/* Other alt C3 column */
- DB8500_PIN_GROUP(stmmod_oc3_1, NMK_GPIO_ALT_C3),
- DB8500_PIN_GROUP(stmmod_oc3_2, NMK_GPIO_ALT_C3),
- DB8500_PIN_GROUP(uartmodrx_oc3_1, NMK_GPIO_ALT_C3),
- DB8500_PIN_GROUP(uartmodtx_oc3_1, NMK_GPIO_ALT_C3),
- DB8500_PIN_GROUP(etmr4_oc3_1, NMK_GPIO_ALT_C3),
+ NMK_PIN_GROUP(stmmod_oc3_1, NMK_GPIO_ALT_C3),
+ NMK_PIN_GROUP(stmmod_oc3_2, NMK_GPIO_ALT_C3),
+ NMK_PIN_GROUP(uartmodrx_oc3_1, NMK_GPIO_ALT_C3),
+ NMK_PIN_GROUP(uartmodtx_oc3_1, NMK_GPIO_ALT_C3),
+ NMK_PIN_GROUP(etmr4_oc3_1, NMK_GPIO_ALT_C3),
/* Other alt C4 column */
- DB8500_PIN_GROUP(sbag_oc4_1, NMK_GPIO_ALT_C4),
- DB8500_PIN_GROUP(hwobs_oc4_1, NMK_GPIO_ALT_C4),
+ NMK_PIN_GROUP(sbag_oc4_1, NMK_GPIO_ALT_C4),
+ NMK_PIN_GROUP(hwobs_oc4_1, NMK_GPIO_ALT_C4),
};
/* We use this macro to define the groups applicable to a function */
STN8815_PIN_C16, STN8815_PIN_A15,
STN8815_PIN_D17, STN8815_PIN_C17 };
-#define STN8815_PIN_GROUP(a, b) { .name = #a, .pins = a##_pins, \
- .npins = ARRAY_SIZE(a##_pins), .altsetting = b }
-
static const struct nmk_pingroup nmk_stn8815_groups[] = {
- STN8815_PIN_GROUP(u0txrx_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(u0ctsrts_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(u0modem_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(mmcsd_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(mmcsd_b_1, NMK_GPIO_ALT_B),
- STN8815_PIN_GROUP(u1_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(i2c1_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A),
- STN8815_PIN_GROUP(u1_b_1, NMK_GPIO_ALT_B),
- STN8815_PIN_GROUP(i2cusb_b_1, NMK_GPIO_ALT_B),
- STN8815_PIN_GROUP(clcd_16_23_b_1, NMK_GPIO_ALT_B),
- STN8815_PIN_GROUP(usbfs_b_1, NMK_GPIO_ALT_B),
- STN8815_PIN_GROUP(usbhs_c_1, NMK_GPIO_ALT_C),
+ NMK_PIN_GROUP(u0txrx_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(u0ctsrts_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(u0modem_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mmcsd_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(mmcsd_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(u1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(i2c1_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A),
+ NMK_PIN_GROUP(u1_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(i2cusb_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(clcd_16_23_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(usbfs_b_1, NMK_GPIO_ALT_B),
+ NMK_PIN_GROUP(usbhs_c_1, NMK_GPIO_ALT_C),
};
/* We use this macro to define the groups applicable to a function */
struct nmk_gpio_chip {
struct gpio_chip chip;
- struct irq_chip irqchip;
void __iomem *addr;
struct clk *clk;
unsigned int bank;
static void nmk_gpio_irq_ack(struct irq_data *d)
{
- struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
- struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
clk_enable(nmk_chip->clk);
writel(BIT(d->hwirq), nmk_chip->addr + NMK_GPIO_IC);
__nmk_gpio_irq_modify(nmk_chip, offset, WAKE, on);
}
-static int nmk_gpio_irq_maskunmask(struct irq_data *d, bool enable)
+static void nmk_gpio_irq_maskunmask(struct nmk_gpio_chip *nmk_chip,
+ struct irq_data *d, bool enable)
{
- struct nmk_gpio_chip *nmk_chip;
unsigned long flags;
- nmk_chip = irq_data_get_irq_chip_data(d);
- if (!nmk_chip)
- return -EINVAL;
-
clk_enable(nmk_chip->clk);
spin_lock_irqsave(&nmk_gpio_slpm_lock, flags);
spin_lock(&nmk_chip->lock);
spin_unlock(&nmk_chip->lock);
spin_unlock_irqrestore(&nmk_gpio_slpm_lock, flags);
clk_disable(nmk_chip->clk);
-
- return 0;
}
static void nmk_gpio_irq_mask(struct irq_data *d)
{
- nmk_gpio_irq_maskunmask(d, false);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
+
+ nmk_gpio_irq_maskunmask(nmk_chip, d, false);
+ gpiochip_disable_irq(gc, irqd_to_hwirq(d));
}
static void nmk_gpio_irq_unmask(struct irq_data *d)
{
- nmk_gpio_irq_maskunmask(d, true);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
+
+ gpiochip_enable_irq(gc, irqd_to_hwirq(d));
+ nmk_gpio_irq_maskunmask(nmk_chip, d, true);
}
static int nmk_gpio_irq_set_wake(struct irq_data *d, unsigned int on)
{
- struct nmk_gpio_chip *nmk_chip;
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
unsigned long flags;
- nmk_chip = irq_data_get_irq_chip_data(d);
- if (!nmk_chip)
- return -EINVAL;
-
clk_enable(nmk_chip->clk);
spin_lock_irqsave(&nmk_gpio_slpm_lock, flags);
spin_lock(&nmk_chip->lock);
static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type)
{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
bool enabled = !irqd_irq_disabled(d);
bool wake = irqd_is_wakeup_set(d);
- struct nmk_gpio_chip *nmk_chip;
unsigned long flags;
- nmk_chip = irq_data_get_irq_chip_data(d);
- if (!nmk_chip)
- return -EINVAL;
if (type & IRQ_TYPE_LEVEL_HIGH)
return -EINVAL;
if (type & IRQ_TYPE_LEVEL_LOW)
static unsigned int nmk_gpio_irq_startup(struct irq_data *d)
{
- struct nmk_gpio_chip *nmk_chip = irq_data_get_irq_chip_data(d);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
clk_enable(nmk_chip->clk);
nmk_gpio_irq_unmask(d);
static void nmk_gpio_irq_shutdown(struct irq_data *d)
{
- struct nmk_gpio_chip *nmk_chip = irq_data_get_irq_chip_data(d);
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
nmk_gpio_irq_mask(d);
clk_disable(nmk_chip->clk);
return nmk_chip;
}
+static void nmk_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc);
+
+ seq_printf(p, "nmk%u-%u-%u", nmk_chip->bank,
+ gc->base, gc->base + gc->ngpio - 1);
+}
+
+static const struct irq_chip nmk_irq_chip = {
+ .irq_ack = nmk_gpio_irq_ack,
+ .irq_mask = nmk_gpio_irq_mask,
+ .irq_unmask = nmk_gpio_irq_unmask,
+ .irq_set_type = nmk_gpio_irq_set_type,
+ .irq_set_wake = nmk_gpio_irq_set_wake,
+ .irq_startup = nmk_gpio_irq_startup,
+ .irq_shutdown = nmk_gpio_irq_shutdown,
+ .irq_print_chip = nmk_gpio_irq_print_chip,
+ .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_IMMUTABLE,
+ GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
static int nmk_gpio_probe(struct platform_device *dev)
{
struct device_node *np = dev->dev.of_node;
struct nmk_gpio_chip *nmk_chip;
struct gpio_chip *chip;
struct gpio_irq_chip *girq;
- struct irq_chip *irqchip;
bool supports_sleepmode;
int irq;
int ret;
chip->can_sleep = false;
chip->owner = THIS_MODULE;
- irqchip = &nmk_chip->irqchip;
- irqchip->irq_ack = nmk_gpio_irq_ack;
- irqchip->irq_mask = nmk_gpio_irq_mask;
- irqchip->irq_unmask = nmk_gpio_irq_unmask;
- irqchip->irq_set_type = nmk_gpio_irq_set_type;
- irqchip->irq_set_wake = nmk_gpio_irq_set_wake;
- irqchip->irq_startup = nmk_gpio_irq_startup;
- irqchip->irq_shutdown = nmk_gpio_irq_shutdown;
- irqchip->flags = IRQCHIP_MASK_ON_SUSPEND;
- irqchip->name = kasprintf(GFP_KERNEL, "nmk%u-%u-%u",
- dev->id,
- chip->base,
- chip->base + chip->ngpio - 1);
-
girq = &chip->irq;
- girq->chip = irqchip;
+ gpio_irq_chip_set_chip(girq, &nmk_irq_chip);
girq->parent_handler = nmk_gpio_irq_handler;
girq->num_parents = 1;
girq->parents = devm_kcalloc(&dev->dev, 1,
{
struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev);
- return npct->soc->groups[selector].name;
+ return npct->soc->groups[selector].grp.name;
}
static int nmk_get_group_pins(struct pinctrl_dev *pctldev, unsigned selector,
const unsigned **pins,
- unsigned *num_pins)
+ unsigned *npins)
{
struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev);
- *pins = npct->soc->groups[selector].pins;
- *num_pins = npct->soc->groups[selector].npins;
+ *pins = npct->soc->groups[selector].grp.pins;
+ *npins = npct->soc->groups[selector].grp.npins;
return 0;
}
if (g->altsetting < 0)
return -EINVAL;
- dev_dbg(npct->dev, "enable group %s, %u pins\n", g->name, g->npins);
+ dev_dbg(npct->dev, "enable group %s, %u pins\n", g->grp.name, g->grp.npins);
/*
* If we're setting altfunc C by setting both AFSLA and AFSLB to 1,
* Then mask the pins that need to be sleeping now when we're
* switching to the ALT C function.
*/
- for (i = 0; i < g->npins; i++)
- slpm[g->pins[i] / NMK_GPIO_PER_CHIP] &= ~BIT(g->pins[i]);
+ for (i = 0; i < g->grp.npins; i++)
+ slpm[g->grp.pins[i] / NMK_GPIO_PER_CHIP] &= ~BIT(g->grp.pins[i]);
nmk_gpio_glitch_slpm_init(slpm);
}
- for (i = 0; i < g->npins; i++) {
+ for (i = 0; i < g->grp.npins; i++) {
struct nmk_gpio_chip *nmk_chip;
unsigned bit;
- nmk_chip = find_nmk_gpio_from_pin(g->pins[i]);
+ nmk_chip = find_nmk_gpio_from_pin(g->grp.pins[i]);
if (!nmk_chip) {
dev_err(npct->dev,
"invalid pin offset %d in group %s at index %d\n",
- g->pins[i], g->name, i);
+ g->grp.pins[i], g->grp.name, i);
goto out_glitch;
}
- dev_dbg(npct->dev, "setting pin %d to altsetting %d\n", g->pins[i], g->altsetting);
+ dev_dbg(npct->dev, "setting pin %d to altsetting %d\n", g->grp.pins[i], g->altsetting);
clk_enable(nmk_chip->clk);
- bit = g->pins[i] % NMK_GPIO_PER_CHIP;
+ bit = g->grp.pins[i] % NMK_GPIO_PER_CHIP;
/*
* If the pin is switching to altfunc, and there was an
* interrupt installed on it which has been lazy disabled,
* then some bits in PRCM GPIOCR registers must be cleared.
*/
if ((g->altsetting & NMK_GPIO_ALT_C) == NMK_GPIO_ALT_C)
- nmk_prcm_altcx_set_mode(npct, g->pins[i],
+ nmk_prcm_altcx_set_mode(npct, g->grp.pins[i],
g->altsetting >> NMK_GPIO_ALT_CX_SHIFT);
}
.compatible = "stericsson,db8500-pinctrl",
.data = (void *)PINCTRL_NMK_DB8500,
},
- {
- .compatible = "stericsson,db8540-pinctrl",
- .data = (void *)PINCTRL_NMK_DB8540,
- },
{},
};
nmk_pinctrl_stn8815_init(&npct->soc);
if (version == PINCTRL_NMK_DB8500)
nmk_pinctrl_db8500_init(&npct->soc);
- if (version == PINCTRL_NMK_DB8540)
- nmk_pinctrl_db8540_init(&npct->soc);
/*
* Since we depend on the GPIO chips to provide clock and register base
/* Package definitions */
#define PINCTRL_NMK_STN8815 0
#define PINCTRL_NMK_DB8500 1
-#define PINCTRL_NMK_DB8540 2
/* Alternate functions: function C is set in hw by setting both A and B */
#define NMK_GPIO_ALT_GPIO 0
/**
* struct nmk_pingroup - describes a Nomadik pin group
- * @name: the name of this specific pin group
- * @pins: an array of discrete physical pins used in this group, taken
- * from the driver-local pin enumeration space
- * @num_pins: the number of pins in this group array, i.e. the number of
- * elements in .pins so we can iterate over that array
+ * @grp: Generic data of the pin group (name and pins)
* @altsetting: the altsetting to apply to all pins in this group to
* configure them to be used by a function
*/
struct nmk_pingroup {
- const char *name;
- const unsigned int *pins;
- const unsigned npins;
+ struct pingroup grp;
int altsetting;
};
+#define NMK_PIN_GROUP(a, b) \
+ { \
+ .grp = PINCTRL_PINGROUP(#a, a##_pins, ARRAY_SIZE(a##_pins)), \
+ .altsetting = b, \
+ }
+
/**
* struct nmk_pinctrl_soc_data - Nomadik pin controller per-SoC configuration
* @pins: An array describing all pins the pin controller affects.
#endif
-#ifdef CONFIG_PINCTRL_DB8540
-
-void nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc);
-
-#else
-
-static inline void
-nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc)
-{
-}
-
-#endif
-
#endif /* PINCTRL_PINCTRL_NOMADIK_H */
int irq;
struct irq_chip irq_chip;
u32 pinctrl_id;
- int (*direction_input)(struct gpio_chip *chip, unsigned offset);
- int (*direction_output)(struct gpio_chip *chip, unsigned offset,
+ int (*direction_input)(struct gpio_chip *chip, unsigned int offset);
+ int (*direction_output)(struct gpio_chip *chip, unsigned int offset,
int value);
- int (*request)(struct gpio_chip *chip, unsigned offset);
- void (*free)(struct gpio_chip *chip, unsigned offset);
+ int (*request)(struct gpio_chip *chip, unsigned int offset);
+ void (*free)(struct gpio_chip *chip, unsigned int offset);
};
struct npcm7xx_pinctrl {
girq->num_parents = 0;
for (i = 0; i < WPCM450_NUM_GPIO_IRQS; i++) {
- int irq = fwnode_irq_get(child, i);
+ int irq;
+ irq = fwnode_irq_get(child, i);
if (irq < 0)
break;
+ if (!irq)
+ continue;
girq->parents[i] = irq;
girq->num_parents++;
}
seq_printf(s, "GPIO bank%d\n", bank);
for (; i < pin_num; i++) {
- seq_printf(s, "📌%d\t", i);
+ seq_printf(s, "#%d\t", i);
raw_spin_lock_irqsave(&gpio_dev->lock, flags);
pin_reg = readl(gpio_dev->base + i * 4);
raw_spin_unlock_irqrestore(&gpio_dev->lock, flags);
}
if (pin_reg & BIT(INTERRUPT_MASK_OFF))
- interrupt_mask = "-";
+ interrupt_mask = "😛";
else
- interrupt_mask = "+";
- seq_printf(s, "int %s (🎭 %s)| active-%s| %s-🔫| ",
+ interrupt_mask = "😷";
+ seq_printf(s, "int %s (%s)| active-%s| %s-⚡| ",
interrupt_enable,
interrupt_mask,
active_level,
level_trig);
if (pin_reg & BIT(WAKE_CNTRL_OFF_S0I3))
- wake_cntrl0 = "+";
+ wake_cntrl0 = "⏰";
else
- wake_cntrl0 = "∅";
- seq_printf(s, "S0i3 🌅 %s| ", wake_cntrl0);
+ wake_cntrl0 = " ∅";
+ seq_printf(s, "S0i3 %s| ", wake_cntrl0);
if (pin_reg & BIT(WAKE_CNTRL_OFF_S3))
- wake_cntrl1 = "+";
+ wake_cntrl1 = "⏰";
else
- wake_cntrl1 = "∅";
- seq_printf(s, "S3 🌅 %s| ", wake_cntrl1);
+ wake_cntrl1 = " ∅";
+ seq_printf(s, "S3 %s| ", wake_cntrl1);
if (pin_reg & BIT(WAKE_CNTRL_OFF_S4))
- wake_cntrl2 = "+";
+ wake_cntrl2 = "⏰";
else
- wake_cntrl2 = "∅";
- seq_printf(s, "S4/S5 🌅 %s| ", wake_cntrl2);
+ wake_cntrl2 = " ∅";
+ seq_printf(s, "S4/S5 %s| ", wake_cntrl2);
if (pin_reg & BIT(PULL_UP_ENABLE_OFF)) {
pull_up_enable = "+";
debounce_enable = " ∅";
}
snprintf(debounce_value, sizeof(debounce_value), "%u", time * unit);
- seq_printf(s, "debounce %s (⏰ %sus)| ", debounce_enable, debounce_value);
+ seq_printf(s, "debounce %s (🕑 %sus)| ", debounce_enable, debounce_value);
seq_printf(s, " 0x%x\n", pin_reg);
}
}
if (!(regval & PIN_IRQ_PENDING) ||
!(regval & BIT(INTERRUPT_MASK_OFF)))
continue;
- generic_handle_domain_irq(gc->irq.domain, irqnr + i);
+ generic_handle_domain_irq_safe(gc->irq.domain, irqnr + i);
/* Clear interrupt.
* We must read the pin register again, in case the
index = device_property_match_string(dev, "pinctrl-resource-names", "iomux");
if (index < 0) {
- dev_warn(dev, "failed to get iomux index\n");
+ dev_dbg(dev, "iomux not supported\n");
goto out_no_pinmux;
}
gpio_dev->iomux_base = devm_platform_ioremap_resource(gpio_dev->pdev, index);
if (IS_ERR(gpio_dev->iomux_base)) {
- dev_warn(dev, "Failed to get iomux %d io resource\n", index);
+ dev_dbg(dev, "iomux not supported %d io resource\n", index);
goto out_no_pinmux;
}
#include <linux/pinctrl/pinmux.h>
/* Since we request GPIOs from ourself */
#include <linux/pinctrl/consumer.h>
-
-#include <soc/at91/pm.h>
+#include <linux/pm.h>
#include "pinctrl-at91.h"
#include "core.h"
struct at91_pinctrl_mux_ops;
+/**
+ * struct at91_gpio_chip: at91 gpio chip
+ * @chip: gpio chip
+ * @range: gpio range
+ * @next: bank sharing same clock
+ * @pioc_hwirq: PIO bank interrupt identifier on AIC
+ * @pioc_virq: PIO bank Linux virtual interrupt
+ * @pioc_idx: PIO bank index
+ * @regbase: PIO bank virtual address
+ * @clock: associated clock
+ * @ops: at91 pinctrl mux ops
+ * @wakeups: wakeup interrupts
+ * @backups: interrupts disabled in suspend
+ * @id: gpio chip identifier
+ */
struct at91_gpio_chip {
struct gpio_chip chip;
struct pinctrl_gpio_range range;
- struct at91_gpio_chip *next; /* Bank sharing same clock */
- int pioc_hwirq; /* PIO bank interrupt identifier on AIC */
- int pioc_virq; /* PIO bank Linux virtual interrupt */
- int pioc_idx; /* PIO bank index */
- void __iomem *regbase; /* PIO bank virtual address */
- struct clk *clock; /* associated clock */
- const struct at91_pinctrl_mux_ops *ops; /* ops */
+ struct at91_gpio_chip *next;
+ int pioc_hwirq;
+ int pioc_virq;
+ int pioc_idx;
+ void __iomem *regbase;
+ struct clk *clock;
+ const struct at91_pinctrl_mux_ops *ops;
+ u32 wakeups;
+ u32 backups;
+ u32 id;
};
static struct at91_gpio_chip *gpio_chips[MAX_GPIO_BANKS];
/* the interrupt is already cleared before by reading ISR */
}
-static u32 wakeups[MAX_GPIO_BANKS];
-static u32 backups[MAX_GPIO_BANKS];
-
static int gpio_irq_set_wake(struct irq_data *d, unsigned state)
{
struct at91_gpio_chip *at91_gpio = irq_data_get_irq_chip_data(d);
- unsigned bank = at91_gpio->pioc_idx;
unsigned mask = 1 << d->hwirq;
- if (unlikely(bank >= MAX_GPIO_BANKS))
- return -EINVAL;
-
if (state)
- wakeups[bank] |= mask;
+ at91_gpio->wakeups |= mask;
else
- wakeups[bank] &= ~mask;
+ at91_gpio->wakeups &= ~mask;
irq_set_irq_wake(at91_gpio->pioc_virq, state);
return 0;
}
-void at91_pinctrl_gpio_suspend(void)
+static int at91_gpio_suspend(struct device *dev)
{
- int i;
-
- for (i = 0; i < gpio_banks; i++) {
- void __iomem *pio;
+ struct at91_gpio_chip *at91_chip = dev_get_drvdata(dev);
+ void __iomem *pio = at91_chip->regbase;
- if (!gpio_chips[i])
- continue;
-
- pio = gpio_chips[i]->regbase;
+ at91_chip->backups = readl_relaxed(pio + PIO_IMR);
+ writel_relaxed(at91_chip->backups, pio + PIO_IDR);
+ writel_relaxed(at91_chip->wakeups, pio + PIO_IER);
- backups[i] = readl_relaxed(pio + PIO_IMR);
- writel_relaxed(backups[i], pio + PIO_IDR);
- writel_relaxed(wakeups[i], pio + PIO_IER);
+ if (!at91_chip->wakeups)
+ clk_disable_unprepare(at91_chip->clock);
+ else
+ dev_dbg(dev, "GPIO-%c may wake for %08x\n",
+ 'A' + at91_chip->id, at91_chip->wakeups);
- if (!wakeups[i])
- clk_disable_unprepare(gpio_chips[i]->clock);
- else
- printk(KERN_DEBUG "GPIO-%c may wake for %08x\n",
- 'A'+i, wakeups[i]);
- }
+ return 0;
}
-void at91_pinctrl_gpio_resume(void)
+static int at91_gpio_resume(struct device *dev)
{
- int i;
-
- for (i = 0; i < gpio_banks; i++) {
- void __iomem *pio;
+ struct at91_gpio_chip *at91_chip = dev_get_drvdata(dev);
+ void __iomem *pio = at91_chip->regbase;
- if (!gpio_chips[i])
- continue;
-
- pio = gpio_chips[i]->regbase;
+ if (!at91_chip->wakeups)
+ clk_prepare_enable(at91_chip->clock);
- if (!wakeups[i])
- clk_prepare_enable(gpio_chips[i]->clock);
+ writel_relaxed(at91_chip->wakeups, pio + PIO_IDR);
+ writel_relaxed(at91_chip->backups, pio + PIO_IER);
- writel_relaxed(wakeups[i], pio + PIO_IDR);
- writel_relaxed(backups[i], pio + PIO_IER);
- }
+ return 0;
}
static void gpio_irq_handler(struct irq_desc *desc)
}
at91_chip->chip = at91_gpio_template;
+ at91_chip->id = alias_idx;
chip = &at91_chip->chip;
chip->label = dev_name(&pdev->dev);
goto gpiochip_add_err;
gpio_chips[alias_idx] = at91_chip;
+ platform_set_drvdata(pdev, at91_chip);
gpio_banks = max(gpio_banks, alias_idx + 1);
dev_info(&pdev->dev, "at address %p\n", at91_chip->regbase);
return ret;
}
+static const struct dev_pm_ops at91_gpio_pm_ops = {
+ SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(at91_gpio_suspend, at91_gpio_resume)
+};
+
static struct platform_driver at91_gpio_driver = {
.driver = {
.name = "gpio-at91",
.of_match_table = at91_gpio_of_match,
+ .pm = pm_ptr(&at91_gpio_pm_ops),
},
.probe = at91_gpio_probe,
};
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * CY8C95X0 20/40/60 pin I2C GPIO port expander with interrupt support
+ *
+ * Copyright (C) 2022 9elements GmbH
+ * Authors: Patrick Rudolph <patrick.rudolph@9elements.com>
+ * Naresh Solanki <Naresh.Solanki@9elements.com>
+ */
+
+#include <linux/acpi.h>
+#include <linux/bitmap.h>
+#include <linux/dmi.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinmux.h>
+
+/* Fast access registers */
+#define CY8C95X0_INPUT 0x00
+#define CY8C95X0_OUTPUT 0x08
+#define CY8C95X0_INTSTATUS 0x10
+
+#define CY8C95X0_INPUT_(x) (CY8C95X0_INPUT + (x))
+#define CY8C95X0_OUTPUT_(x) (CY8C95X0_OUTPUT + (x))
+#define CY8C95X0_INTSTATUS_(x) (CY8C95X0_INTSTATUS + (x))
+
+/* Port Select configures the port */
+#define CY8C95X0_PORTSEL 0x18
+/* Port settings, write PORTSEL first */
+#define CY8C95X0_INTMASK 0x19
+#define CY8C95X0_PWMSEL 0x1A
+#define CY8C95X0_INVERT 0x1B
+#define CY8C95X0_DIRECTION 0x1C
+/* Drive mode register change state on writing '1' */
+#define CY8C95X0_DRV_PU 0x1D
+#define CY8C95X0_DRV_PD 0x1E
+#define CY8C95X0_DRV_ODH 0x1F
+#define CY8C95X0_DRV_ODL 0x20
+#define CY8C95X0_DRV_PP_FAST 0x21
+#define CY8C95X0_DRV_PP_SLOW 0x22
+#define CY8C95X0_DRV_HIZ 0x23
+#define CY8C95X0_DEVID 0x2E
+#define CY8C95X0_WATCHDOG 0x2F
+#define CY8C95X0_COMMAND 0x30
+
+#define CY8C95X0_PIN_TO_OFFSET(x) (((x) >= 20) ? ((x) + 4) : (x))
+
+static const struct i2c_device_id cy8c95x0_id[] = {
+ { "cy8c9520", 20, },
+ { "cy8c9540", 40, },
+ { "cy8c9560", 60, },
+ { }
+};
+MODULE_DEVICE_TABLE(i2c, cy8c95x0_id);
+
+#define OF_CY8C95X(__nrgpio) ((void *)(__nrgpio))
+
+static const struct of_device_id cy8c95x0_dt_ids[] = {
+ { .compatible = "cypress,cy8c9520", .data = OF_CY8C95X(20), },
+ { .compatible = "cypress,cy8c9540", .data = OF_CY8C95X(40), },
+ { .compatible = "cypress,cy8c9560", .data = OF_CY8C95X(60), },
+ { }
+};
+MODULE_DEVICE_TABLE(of, cy8c95x0_dt_ids);
+
+static const struct acpi_gpio_params cy8c95x0_irq_gpios = { 0, 0, true };
+
+static const struct acpi_gpio_mapping cy8c95x0_acpi_irq_gpios[] = {
+ { "irq-gpios", &cy8c95x0_irq_gpios, 1, ACPI_GPIO_QUIRK_ABSOLUTE_NUMBER },
+ { }
+};
+
+static int cy8c95x0_acpi_get_irq(struct device *dev)
+{
+ int ret;
+
+ ret = devm_acpi_dev_add_driver_gpios(dev, cy8c95x0_acpi_irq_gpios);
+ if (ret)
+ dev_warn(dev, "can't add GPIO ACPI mapping\n");
+
+ ret = acpi_dev_gpio_irq_get_by(ACPI_COMPANION(dev), "irq-gpios", 0);
+ if (ret < 0)
+ return ret;
+
+ dev_info(dev, "ACPI interrupt quirk (IRQ %d)\n", ret);
+ return ret;
+}
+
+static const struct dmi_system_id cy8c95x0_dmi_acpi_irq_info[] = {
+ {
+ /*
+ * On Intel Galileo Gen 1 board the IRQ pin is provided
+ * as an absolute number instead of being relative.
+ * Since first controller (gpio-sch.c) and second
+ * (gpio-dwapb.c) are at the fixed bases, we may safely
+ * refer to the number in the global space to get an IRQ
+ * out of it.
+ */
+ .matches = {
+ DMI_EXACT_MATCH(DMI_BOARD_NAME, "Galileo"),
+ },
+ },
+ {}
+};
+
+#define MAX_BANK 8
+#define BANK_SZ 8
+#define MAX_LINE (MAX_BANK * BANK_SZ)
+
+#define CY8C95X0_GPIO_MASK GENMASK(7, 0)
+
+/**
+ * struct cy8c95x0_pinctrl - driver data
+ * @regmap: Device's regmap
+ * @irq_lock: IRQ bus lock
+ * @i2c_lock: Mutex for the device internal mux register
+ * @irq_mask: I/O bits affected by interrupts
+ * @irq_trig_raise: I/O bits affected by raising voltage level
+ * @irq_trig_fall: I/O bits affected by falling voltage level
+ * @irq_trig_low: I/O bits affected by a low voltage level
+ * @irq_trig_high: I/O bits affected by a high voltage level
+ * @push_pull: I/O bits configured as push pull driver
+ * @shiftmask: Mask used to compensate for Gport2 width
+ * @nport: Number of Gports in this chip
+ * @gpio_chip: gpiolib chip
+ * @driver_data: private driver data
+ * @regulator: Pointer to the regulator for the IC
+ * @dev: struct device
+ * @pctldev: pin controller device
+ * @pinctrl_desc: pin controller description
+ * @name: Chip controller name
+ * @tpin: Total number of pins
+ */
+struct cy8c95x0_pinctrl {
+ struct regmap *regmap;
+ struct mutex irq_lock;
+ struct mutex i2c_lock;
+ DECLARE_BITMAP(irq_mask, MAX_LINE);
+ DECLARE_BITMAP(irq_trig_raise, MAX_LINE);
+ DECLARE_BITMAP(irq_trig_fall, MAX_LINE);
+ DECLARE_BITMAP(irq_trig_low, MAX_LINE);
+ DECLARE_BITMAP(irq_trig_high, MAX_LINE);
+ DECLARE_BITMAP(push_pull, MAX_LINE);
+ DECLARE_BITMAP(shiftmask, MAX_LINE);
+ int nport;
+ struct gpio_chip gpio_chip;
+ unsigned long driver_data;
+ struct regulator *regulator;
+ struct device *dev;
+ struct pinctrl_dev *pctldev;
+ struct pinctrl_desc pinctrl_desc;
+ char name[32];
+ unsigned int tpin;
+};
+
+static const struct pinctrl_pin_desc cy8c9560_pins[] = {
+ PINCTRL_PIN(0, "gp00"),
+ PINCTRL_PIN(1, "gp01"),
+ PINCTRL_PIN(2, "gp02"),
+ PINCTRL_PIN(3, "gp03"),
+ PINCTRL_PIN(4, "gp04"),
+ PINCTRL_PIN(5, "gp05"),
+ PINCTRL_PIN(6, "gp06"),
+ PINCTRL_PIN(7, "gp07"),
+
+ PINCTRL_PIN(8, "gp10"),
+ PINCTRL_PIN(9, "gp11"),
+ PINCTRL_PIN(10, "gp12"),
+ PINCTRL_PIN(11, "gp13"),
+ PINCTRL_PIN(12, "gp14"),
+ PINCTRL_PIN(13, "gp15"),
+ PINCTRL_PIN(14, "gp16"),
+ PINCTRL_PIN(15, "gp17"),
+
+ PINCTRL_PIN(16, "gp20"),
+ PINCTRL_PIN(17, "gp21"),
+ PINCTRL_PIN(18, "gp22"),
+ PINCTRL_PIN(19, "gp23"),
+
+ PINCTRL_PIN(20, "gp30"),
+ PINCTRL_PIN(21, "gp31"),
+ PINCTRL_PIN(22, "gp32"),
+ PINCTRL_PIN(23, "gp33"),
+ PINCTRL_PIN(24, "gp34"),
+ PINCTRL_PIN(25, "gp35"),
+ PINCTRL_PIN(26, "gp36"),
+ PINCTRL_PIN(27, "gp37"),
+
+ PINCTRL_PIN(28, "gp40"),
+ PINCTRL_PIN(29, "gp41"),
+ PINCTRL_PIN(30, "gp42"),
+ PINCTRL_PIN(31, "gp43"),
+ PINCTRL_PIN(32, "gp44"),
+ PINCTRL_PIN(33, "gp45"),
+ PINCTRL_PIN(34, "gp46"),
+ PINCTRL_PIN(35, "gp47"),
+
+ PINCTRL_PIN(36, "gp50"),
+ PINCTRL_PIN(37, "gp51"),
+ PINCTRL_PIN(38, "gp52"),
+ PINCTRL_PIN(39, "gp53"),
+ PINCTRL_PIN(40, "gp54"),
+ PINCTRL_PIN(41, "gp55"),
+ PINCTRL_PIN(42, "gp56"),
+ PINCTRL_PIN(43, "gp57"),
+
+ PINCTRL_PIN(44, "gp60"),
+ PINCTRL_PIN(45, "gp61"),
+ PINCTRL_PIN(46, "gp62"),
+ PINCTRL_PIN(47, "gp63"),
+ PINCTRL_PIN(48, "gp64"),
+ PINCTRL_PIN(49, "gp65"),
+ PINCTRL_PIN(50, "gp66"),
+ PINCTRL_PIN(51, "gp67"),
+
+ PINCTRL_PIN(52, "gp70"),
+ PINCTRL_PIN(53, "gp71"),
+ PINCTRL_PIN(54, "gp72"),
+ PINCTRL_PIN(55, "gp73"),
+ PINCTRL_PIN(56, "gp74"),
+ PINCTRL_PIN(57, "gp75"),
+ PINCTRL_PIN(58, "gp76"),
+ PINCTRL_PIN(59, "gp77"),
+};
+
+static const char * const cy8c95x0_groups[] = {
+ "gp00",
+ "gp01",
+ "gp02",
+ "gp03",
+ "gp04",
+ "gp05",
+ "gp06",
+ "gp07",
+
+ "gp10",
+ "gp11",
+ "gp12",
+ "gp13",
+ "gp14",
+ "gp15",
+ "gp16",
+ "gp17",
+
+ "gp20",
+ "gp21",
+ "gp22",
+ "gp23",
+
+ "gp30",
+ "gp31",
+ "gp32",
+ "gp33",
+ "gp34",
+ "gp35",
+ "gp36",
+ "gp37",
+
+ "gp40",
+ "gp41",
+ "gp42",
+ "gp43",
+ "gp44",
+ "gp45",
+ "gp46",
+ "gp47",
+
+ "gp50",
+ "gp51",
+ "gp52",
+ "gp53",
+ "gp54",
+ "gp55",
+ "gp56",
+ "gp57",
+
+ "gp60",
+ "gp61",
+ "gp62",
+ "gp63",
+ "gp64",
+ "gp65",
+ "gp66",
+ "gp67",
+
+ "gp70",
+ "gp71",
+ "gp72",
+ "gp73",
+ "gp74",
+ "gp75",
+ "gp76",
+ "gp77",
+};
+
+static inline u8 cypress_get_port(struct cy8c95x0_pinctrl *chip, unsigned int pin)
+{
+ /* Account for GPORT2 which only has 4 bits */
+ return CY8C95X0_PIN_TO_OFFSET(pin) / BANK_SZ;
+}
+
+static int cypress_get_pin_mask(struct cy8c95x0_pinctrl *chip, unsigned int pin)
+{
+ /* Account for GPORT2 which only has 4 bits */
+ return BIT(CY8C95X0_PIN_TO_OFFSET(pin) % BANK_SZ);
+}
+
+static bool cy8c95x0_readable_register(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case 0x24 ... 0x27:
+ return false;
+ default:
+ return true;
+ }
+}
+
+static bool cy8c95x0_writeable_register(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case CY8C95X0_INPUT_(0) ... CY8C95X0_INPUT_(7):
+ return false;
+ case CY8C95X0_DEVID:
+ return false;
+ case 0x24 ... 0x27:
+ return false;
+ default:
+ return true;
+ }
+}
+
+static bool cy8c95x0_volatile_register(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case CY8C95X0_INPUT_(0) ... CY8C95X0_INPUT_(7):
+ case CY8C95X0_INTSTATUS_(0) ... CY8C95X0_INTSTATUS_(7):
+ case CY8C95X0_INTMASK:
+ case CY8C95X0_INVERT:
+ case CY8C95X0_PWMSEL:
+ case CY8C95X0_DIRECTION:
+ case CY8C95X0_DRV_PU:
+ case CY8C95X0_DRV_PD:
+ case CY8C95X0_DRV_ODH:
+ case CY8C95X0_DRV_ODL:
+ case CY8C95X0_DRV_PP_FAST:
+ case CY8C95X0_DRV_PP_SLOW:
+ case CY8C95X0_DRV_HIZ:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool cy8c95x0_precious_register(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case CY8C95X0_INTSTATUS_(0) ... CY8C95X0_INTSTATUS_(7):
+ return true;
+ default:
+ return false;
+ }
+}
+
+static const struct reg_default cy8c95x0_reg_defaults[] = {
+ { CY8C95X0_OUTPUT_(0), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(1), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(2), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(3), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(4), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(5), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(6), GENMASK(7, 0) },
+ { CY8C95X0_OUTPUT_(7), GENMASK(7, 0) },
+ { CY8C95X0_PORTSEL, 0 },
+ { CY8C95X0_PWMSEL, 0 },
+};
+
+static const struct regmap_config cy8c95x0_i2c_regmap = {
+ .reg_bits = 8,
+ .val_bits = 8,
+
+ .reg_defaults = cy8c95x0_reg_defaults,
+ .num_reg_defaults = ARRAY_SIZE(cy8c95x0_reg_defaults),
+
+ .readable_reg = cy8c95x0_readable_register,
+ .writeable_reg = cy8c95x0_writeable_register,
+ .volatile_reg = cy8c95x0_volatile_register,
+ .precious_reg = cy8c95x0_precious_register,
+
+ .cache_type = REGCACHE_FLAT,
+ .max_register = CY8C95X0_COMMAND,
+};
+
+static int cy8c95x0_write_regs_mask(struct cy8c95x0_pinctrl *chip, int reg,
+ unsigned long *val, unsigned long *mask)
+{
+ DECLARE_BITMAP(tmask, MAX_LINE);
+ DECLARE_BITMAP(tval, MAX_LINE);
+ int write_val;
+ int ret = 0;
+ int i, off = 0;
+ u8 bits;
+
+ /* Add the 4 bit gap of Gport2 */
+ bitmap_andnot(tmask, mask, chip->shiftmask, MAX_LINE);
+ bitmap_shift_left(tmask, tmask, 4, MAX_LINE);
+ bitmap_replace(tmask, tmask, mask, chip->shiftmask, BANK_SZ * 3);
+
+ bitmap_andnot(tval, val, chip->shiftmask, MAX_LINE);
+ bitmap_shift_left(tval, tval, 4, MAX_LINE);
+ bitmap_replace(tval, tval, val, chip->shiftmask, BANK_SZ * 3);
+
+ mutex_lock(&chip->i2c_lock);
+ for (i = 0; i < chip->nport; i++) {
+ /* Skip over unused banks */
+ bits = bitmap_get_value8(tmask, i * BANK_SZ);
+ if (!bits)
+ continue;
+
+ switch (reg) {
+ /* Muxed registers */
+ case CY8C95X0_INTMASK:
+ case CY8C95X0_PWMSEL:
+ case CY8C95X0_INVERT:
+ case CY8C95X0_DIRECTION:
+ case CY8C95X0_DRV_PU:
+ case CY8C95X0_DRV_PD:
+ case CY8C95X0_DRV_ODH:
+ case CY8C95X0_DRV_ODL:
+ case CY8C95X0_DRV_PP_FAST:
+ case CY8C95X0_DRV_PP_SLOW:
+ case CY8C95X0_DRV_HIZ:
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, i);
+ if (ret < 0)
+ goto out;
+ off = reg;
+ break;
+ /* Direct access registers */
+ case CY8C95X0_INPUT:
+ case CY8C95X0_OUTPUT:
+ case CY8C95X0_INTSTATUS:
+ off = reg + i;
+ break;
+ default:
+ ret = -EINVAL;
+ goto out;
+ }
+
+ write_val = bitmap_get_value8(tval, i * BANK_SZ);
+
+ ret = regmap_update_bits(chip->regmap, off, bits, write_val);
+ if (ret < 0)
+ goto out;
+ }
+out:
+ mutex_unlock(&chip->i2c_lock);
+
+ if (ret < 0)
+ dev_err(chip->dev, "failed writing register %d: err %d\n", off, ret);
+
+ return ret;
+}
+
+static int cy8c95x0_read_regs_mask(struct cy8c95x0_pinctrl *chip, int reg,
+ unsigned long *val, unsigned long *mask)
+{
+ DECLARE_BITMAP(tmask, MAX_LINE);
+ DECLARE_BITMAP(tval, MAX_LINE);
+ DECLARE_BITMAP(tmp, MAX_LINE);
+ int read_val;
+ int ret = 0;
+ int i, off = 0;
+ u8 bits;
+
+ /* Add the 4 bit gap of Gport2 */
+ bitmap_andnot(tmask, mask, chip->shiftmask, MAX_LINE);
+ bitmap_shift_left(tmask, tmask, 4, MAX_LINE);
+ bitmap_replace(tmask, tmask, mask, chip->shiftmask, BANK_SZ * 3);
+
+ bitmap_andnot(tval, val, chip->shiftmask, MAX_LINE);
+ bitmap_shift_left(tval, tval, 4, MAX_LINE);
+ bitmap_replace(tval, tval, val, chip->shiftmask, BANK_SZ * 3);
+
+ mutex_lock(&chip->i2c_lock);
+ for (i = 0; i < chip->nport; i++) {
+ /* Skip over unused banks */
+ bits = bitmap_get_value8(tmask, i * BANK_SZ);
+ if (!bits)
+ continue;
+
+ switch (reg) {
+ /* Muxed registers */
+ case CY8C95X0_INTMASK:
+ case CY8C95X0_PWMSEL:
+ case CY8C95X0_INVERT:
+ case CY8C95X0_DIRECTION:
+ case CY8C95X0_DRV_PU:
+ case CY8C95X0_DRV_PD:
+ case CY8C95X0_DRV_ODH:
+ case CY8C95X0_DRV_ODL:
+ case CY8C95X0_DRV_PP_FAST:
+ case CY8C95X0_DRV_PP_SLOW:
+ case CY8C95X0_DRV_HIZ:
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, i);
+ if (ret < 0)
+ goto out;
+ off = reg;
+ break;
+ /* Direct access registers */
+ case CY8C95X0_INPUT:
+ case CY8C95X0_OUTPUT:
+ case CY8C95X0_INTSTATUS:
+ off = reg + i;
+ break;
+ default:
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = regmap_read(chip->regmap, off, &read_val);
+ if (ret < 0)
+ goto out;
+
+ read_val &= bits;
+ read_val |= bitmap_get_value8(tval, i * BANK_SZ) & ~bits;
+ bitmap_set_value8(tval, read_val, i * BANK_SZ);
+ }
+
+ /* Fill the 4 bit gap of Gport2 */
+ bitmap_shift_right(tmp, tval, 4, MAX_LINE);
+ bitmap_replace(val, tmp, tval, chip->shiftmask, MAX_LINE);
+
+out:
+ mutex_unlock(&chip->i2c_lock);
+
+ if (ret < 0)
+ dev_err(chip->dev, "failed reading register %d: err %d\n", off, ret);
+
+ return ret;
+}
+
+static int cy8c95x0_gpio_direction_input(struct gpio_chip *gc, unsigned int off)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ u8 port = cypress_get_port(chip, off);
+ u8 bit = cypress_get_pin_mask(chip, off);
+ int ret;
+
+ mutex_lock(&chip->i2c_lock);
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port);
+ if (ret)
+ goto out;
+
+ ret = regmap_write_bits(chip->regmap, CY8C95X0_DIRECTION, bit, bit);
+ if (ret)
+ goto out;
+
+ if (test_bit(off, chip->push_pull)) {
+ /*
+ * Disable driving the pin by forcing it to HighZ. Only setting the
+ * direction register isn't sufficient in Push-Pull mode.
+ */
+ ret = regmap_write_bits(chip->regmap, CY8C95X0_DRV_HIZ, bit, bit);
+ if (ret)
+ goto out;
+
+ __clear_bit(off, chip->push_pull);
+ }
+
+out:
+ mutex_unlock(&chip->i2c_lock);
+
+ return ret;
+}
+
+static int cy8c95x0_gpio_direction_output(struct gpio_chip *gc,
+ unsigned int off, int val)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ u8 port = cypress_get_port(chip, off);
+ u8 outreg = CY8C95X0_OUTPUT_(port);
+ u8 bit = cypress_get_pin_mask(chip, off);
+ int ret;
+
+ /* Set output level */
+ ret = regmap_write_bits(chip->regmap, outreg, bit, val ? bit : 0);
+ if (ret)
+ return ret;
+
+ mutex_lock(&chip->i2c_lock);
+ /* Select port... */
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port);
+ if (ret)
+ goto out;
+
+ /* ...then direction */
+ ret = regmap_write_bits(chip->regmap, CY8C95X0_DIRECTION, bit, 0);
+
+out:
+ mutex_unlock(&chip->i2c_lock);
+
+ return ret;
+}
+
+static int cy8c95x0_gpio_get_value(struct gpio_chip *gc, unsigned int off)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ u8 inreg = CY8C95X0_INPUT_(cypress_get_port(chip, off));
+ u8 bit = cypress_get_pin_mask(chip, off);
+ u32 reg_val;
+ int ret;
+
+ ret = regmap_read(chip->regmap, inreg, ®_val);
+ if (ret < 0) {
+ /*
+ * NOTE:
+ * Diagnostic already emitted; that's all we should
+ * do unless gpio_*_value_cansleep() calls become different
+ * from their nonsleeping siblings (and report faults).
+ */
+ return 0;
+ }
+
+ return !!(reg_val & bit);
+}
+
+static void cy8c95x0_gpio_set_value(struct gpio_chip *gc, unsigned int off,
+ int val)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ u8 outreg = CY8C95X0_OUTPUT_(cypress_get_port(chip, off));
+ u8 bit = cypress_get_pin_mask(chip, off);
+
+ regmap_write_bits(chip->regmap, outreg, bit, val ? bit : 0);
+}
+
+static int cy8c95x0_gpio_get_direction(struct gpio_chip *gc, unsigned int off)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ u8 port = cypress_get_port(chip, off);
+ u8 bit = cypress_get_pin_mask(chip, off);
+ u32 reg_val;
+ int ret;
+
+ mutex_lock(&chip->i2c_lock);
+
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port);
+ if (ret < 0)
+ goto out;
+
+ ret = regmap_read(chip->regmap, CY8C95X0_DIRECTION, ®_val);
+ if (ret < 0)
+ goto out;
+
+ mutex_unlock(&chip->i2c_lock);
+
+ if (reg_val & bit)
+ return GPIO_LINE_DIRECTION_IN;
+
+ return GPIO_LINE_DIRECTION_OUT;
+out:
+ mutex_unlock(&chip->i2c_lock);
+ return ret;
+}
+
+static int cy8c95x0_gpio_get_pincfg(struct cy8c95x0_pinctrl *chip,
+ unsigned int off,
+ unsigned long *config)
+{
+ enum pin_config_param param = pinconf_to_config_param(*config);
+ u8 port = cypress_get_port(chip, off);
+ u8 bit = cypress_get_pin_mask(chip, off);
+ unsigned int reg;
+ u32 reg_val;
+ u16 arg = 0;
+ int ret;
+
+ mutex_lock(&chip->i2c_lock);
+
+ /* Select port */
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port);
+ if (ret < 0)
+ goto out;
+
+ switch (param) {
+ case PIN_CONFIG_BIAS_PULL_UP:
+ reg = CY8C95X0_DRV_PU;
+ break;
+ case PIN_CONFIG_BIAS_PULL_DOWN:
+ reg = CY8C95X0_DRV_PD;
+ break;
+ case PIN_CONFIG_BIAS_DISABLE:
+ reg = CY8C95X0_DRV_HIZ;
+ break;
+ case PIN_CONFIG_DRIVE_OPEN_DRAIN:
+ reg = CY8C95X0_DRV_ODL;
+ break;
+ case PIN_CONFIG_DRIVE_OPEN_SOURCE:
+ reg = CY8C95X0_DRV_ODH;
+ break;
+ case PIN_CONFIG_DRIVE_PUSH_PULL:
+ reg = CY8C95X0_DRV_PP_FAST;
+ break;
+ case PIN_CONFIG_INPUT_ENABLE:
+ reg = CY8C95X0_DIRECTION;
+ break;
+ case PIN_CONFIG_MODE_PWM:
+ reg = CY8C95X0_PWMSEL;
+ break;
+ case PIN_CONFIG_OUTPUT:
+ reg = CY8C95X0_OUTPUT_(port);
+ break;
+ case PIN_CONFIG_OUTPUT_ENABLE:
+ reg = CY8C95X0_DIRECTION;
+ break;
+
+ case PIN_CONFIG_BIAS_HIGH_IMPEDANCE:
+ case PIN_CONFIG_BIAS_BUS_HOLD:
+ case PIN_CONFIG_BIAS_PULL_PIN_DEFAULT:
+ case PIN_CONFIG_DRIVE_STRENGTH:
+ case PIN_CONFIG_DRIVE_STRENGTH_UA:
+ case PIN_CONFIG_INPUT_DEBOUNCE:
+ case PIN_CONFIG_INPUT_SCHMITT:
+ case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
+ case PIN_CONFIG_MODE_LOW_POWER:
+ case PIN_CONFIG_PERSIST_STATE:
+ case PIN_CONFIG_POWER_SOURCE:
+ case PIN_CONFIG_SKEW_DELAY:
+ case PIN_CONFIG_SLEEP_HARDWARE_STATE:
+ case PIN_CONFIG_SLEW_RATE:
+ default:
+ ret = -ENOTSUPP;
+ goto out;
+ }
+ /*
+ * Writing 1 to one of the drive mode registers will automatically
+ * clear conflicting set bits in the other drive mode registers.
+ */
+ ret = regmap_read(chip->regmap, reg, ®_val);
+ if (reg_val & bit)
+ arg = 1;
+
+ *config = pinconf_to_config_packed(param, (u16)arg);
+out:
+ mutex_unlock(&chip->i2c_lock);
+
+ return ret;
+}
+
+static int cy8c95x0_gpio_set_pincfg(struct cy8c95x0_pinctrl *chip,
+ unsigned int off,
+ unsigned long config)
+{
+ u8 port = cypress_get_port(chip, off);
+ u8 bit = cypress_get_pin_mask(chip, off);
+ unsigned long param = pinconf_to_config_param(config);
+ unsigned int reg;
+ int ret;
+
+ mutex_lock(&chip->i2c_lock);
+
+ /* Select port */
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port);
+ if (ret < 0)
+ goto out;
+
+ switch (param) {
+ case PIN_CONFIG_BIAS_PULL_UP:
+ __clear_bit(off, chip->push_pull);
+ reg = CY8C95X0_DRV_PU;
+ break;
+ case PIN_CONFIG_BIAS_PULL_DOWN:
+ __clear_bit(off, chip->push_pull);
+ reg = CY8C95X0_DRV_PD;
+ break;
+ case PIN_CONFIG_BIAS_DISABLE:
+ __clear_bit(off, chip->push_pull);
+ reg = CY8C95X0_DRV_HIZ;
+ break;
+ case PIN_CONFIG_DRIVE_OPEN_DRAIN:
+ __clear_bit(off, chip->push_pull);
+ reg = CY8C95X0_DRV_ODL;
+ break;
+ case PIN_CONFIG_DRIVE_OPEN_SOURCE:
+ __clear_bit(off, chip->push_pull);
+ reg = CY8C95X0_DRV_ODH;
+ break;
+ case PIN_CONFIG_DRIVE_PUSH_PULL:
+ __set_bit(off, chip->push_pull);
+ reg = CY8C95X0_DRV_PP_FAST;
+ break;
+ case PIN_CONFIG_MODE_PWM:
+ reg = CY8C95X0_PWMSEL;
+ break;
+ default:
+ ret = -ENOTSUPP;
+ goto out;
+ }
+ /*
+ * Writing 1 to one of the drive mode registers will automatically
+ * clear conflicting set bits in the other drive mode registers.
+ */
+ ret = regmap_write_bits(chip->regmap, reg, bit, bit);
+
+out:
+ mutex_unlock(&chip->i2c_lock);
+ return ret;
+}
+
+static int cy8c95x0_gpio_get_multiple(struct gpio_chip *gc,
+ unsigned long *mask, unsigned long *bits)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+
+ return cy8c95x0_read_regs_mask(chip, CY8C95X0_INPUT, bits, mask);
+}
+
+static void cy8c95x0_gpio_set_multiple(struct gpio_chip *gc,
+ unsigned long *mask, unsigned long *bits)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+
+ cy8c95x0_write_regs_mask(chip, CY8C95X0_OUTPUT, bits, mask);
+}
+
+static int cy8c95x0_add_pin_ranges(struct gpio_chip *gc)
+{
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ struct device *dev = chip->dev;
+ int ret;
+
+ ret = gpiochip_add_pin_range(gc, dev_name(dev), 0, 0, chip->tpin);
+ if (ret)
+ dev_err(dev, "failed to add GPIO pin range\n");
+
+ return ret;
+}
+
+static int cy8c95x0_setup_gpiochip(struct cy8c95x0_pinctrl *chip)
+{
+ struct gpio_chip *gc = &chip->gpio_chip;
+
+ gc->direction_input = cy8c95x0_gpio_direction_input;
+ gc->direction_output = cy8c95x0_gpio_direction_output;
+ gc->get = cy8c95x0_gpio_get_value;
+ gc->set = cy8c95x0_gpio_set_value;
+ gc->get_direction = cy8c95x0_gpio_get_direction;
+ gc->get_multiple = cy8c95x0_gpio_get_multiple;
+ gc->set_multiple = cy8c95x0_gpio_set_multiple;
+ gc->set_config = gpiochip_generic_config,
+ gc->can_sleep = true;
+ gc->add_pin_ranges = cy8c95x0_add_pin_ranges;
+
+ gc->base = -1;
+ gc->ngpio = chip->tpin;
+
+ gc->parent = chip->dev;
+ gc->owner = THIS_MODULE;
+ gc->names = NULL;
+
+ gc->label = dev_name(chip->dev);
+
+ return devm_gpiochip_add_data(chip->dev, gc, chip);
+}
+
+static void cy8c95x0_irq_mask(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+ set_bit(hwirq, chip->irq_mask);
+ gpiochip_disable_irq(gc, hwirq);
+}
+
+static void cy8c95x0_irq_unmask(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+ gpiochip_enable_irq(gc, hwirq);
+ clear_bit(hwirq, chip->irq_mask);
+}
+
+static void cy8c95x0_irq_bus_lock(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+
+ mutex_lock(&chip->irq_lock);
+}
+
+static void cy8c95x0_irq_bus_sync_unlock(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ DECLARE_BITMAP(ones, MAX_LINE);
+ DECLARE_BITMAP(irq_mask, MAX_LINE);
+ DECLARE_BITMAP(reg_direction, MAX_LINE);
+
+ bitmap_fill(ones, MAX_LINE);
+
+ cy8c95x0_write_regs_mask(chip, CY8C95X0_INTMASK, chip->irq_mask, ones);
+
+ /* Switch direction to input if needed */
+ cy8c95x0_read_regs_mask(chip, CY8C95X0_DIRECTION, reg_direction, chip->irq_mask);
+ bitmap_or(irq_mask, chip->irq_mask, reg_direction, MAX_LINE);
+ bitmap_complement(irq_mask, irq_mask, MAX_LINE);
+
+ /* Look for any newly setup interrupt */
+ cy8c95x0_write_regs_mask(chip, CY8C95X0_DIRECTION, ones, irq_mask);
+
+ mutex_unlock(&chip->irq_lock);
+}
+
+static int cy8c95x0_irq_set_type(struct irq_data *d, unsigned int type)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ unsigned int trig_type;
+
+ switch (type) {
+ case IRQ_TYPE_EDGE_RISING:
+ case IRQ_TYPE_EDGE_FALLING:
+ case IRQ_TYPE_EDGE_BOTH:
+ trig_type = type;
+ break;
+ case IRQ_TYPE_LEVEL_HIGH:
+ trig_type = IRQ_TYPE_EDGE_RISING;
+ break;
+ case IRQ_TYPE_LEVEL_LOW:
+ trig_type = IRQ_TYPE_EDGE_FALLING;
+ break;
+ default:
+ dev_err(chip->dev, "irq %d: unsupported type %d\n", d->irq, type);
+ return -EINVAL;
+ }
+
+ assign_bit(hwirq, chip->irq_trig_fall, trig_type & IRQ_TYPE_EDGE_FALLING);
+ assign_bit(hwirq, chip->irq_trig_raise, trig_type & IRQ_TYPE_EDGE_RISING);
+ assign_bit(hwirq, chip->irq_trig_low, type == IRQ_TYPE_LEVEL_LOW);
+ assign_bit(hwirq, chip->irq_trig_high, type == IRQ_TYPE_LEVEL_HIGH);
+
+ return 0;
+}
+
+static void cy8c95x0_irq_shutdown(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc);
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+ clear_bit(hwirq, chip->irq_trig_raise);
+ clear_bit(hwirq, chip->irq_trig_fall);
+ clear_bit(hwirq, chip->irq_trig_low);
+ clear_bit(hwirq, chip->irq_trig_high);
+}
+
+static const struct irq_chip cy8c95x0_irqchip = {
+ .name = "cy8c95x0-irq",
+ .irq_mask = cy8c95x0_irq_mask,
+ .irq_unmask = cy8c95x0_irq_unmask,
+ .irq_bus_lock = cy8c95x0_irq_bus_lock,
+ .irq_bus_sync_unlock = cy8c95x0_irq_bus_sync_unlock,
+ .irq_set_type = cy8c95x0_irq_set_type,
+ .irq_shutdown = cy8c95x0_irq_shutdown,
+ .flags = IRQCHIP_IMMUTABLE,
+ GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
+static bool cy8c95x0_irq_pending(struct cy8c95x0_pinctrl *chip, unsigned long *pending)
+{
+ DECLARE_BITMAP(ones, MAX_LINE);
+ DECLARE_BITMAP(cur_stat, MAX_LINE);
+ DECLARE_BITMAP(new_stat, MAX_LINE);
+ DECLARE_BITMAP(trigger, MAX_LINE);
+
+ bitmap_fill(ones, MAX_LINE);
+
+ /* Read the current interrupt status from the device */
+ if (cy8c95x0_read_regs_mask(chip, CY8C95X0_INTSTATUS, trigger, ones))
+ return false;
+
+ /* Check latched inputs */
+ if (cy8c95x0_read_regs_mask(chip, CY8C95X0_INPUT, cur_stat, trigger))
+ return false;
+
+ /* Apply filter for rising/falling edge selection */
+ bitmap_replace(new_stat, chip->irq_trig_fall, chip->irq_trig_raise,
+ cur_stat, MAX_LINE);
+
+ bitmap_and(pending, new_stat, trigger, MAX_LINE);
+
+ return !bitmap_empty(pending, MAX_LINE);
+}
+
+static irqreturn_t cy8c95x0_irq_handler(int irq, void *devid)
+{
+ struct cy8c95x0_pinctrl *chip = devid;
+ struct gpio_chip *gc = &chip->gpio_chip;
+ DECLARE_BITMAP(pending, MAX_LINE);
+ int nested_irq, level;
+ bool ret;
+
+ ret = cy8c95x0_irq_pending(chip, pending);
+ if (!ret)
+ return IRQ_RETVAL(0);
+
+ ret = 0;
+ for_each_set_bit(level, pending, MAX_LINE) {
+ /* Already accounted for 4bit gap in GPort2 */
+ nested_irq = irq_find_mapping(gc->irq.domain, level);
+
+ if (unlikely(nested_irq <= 0)) {
+ dev_warn_ratelimited(gc->parent, "unmapped interrupt %d\n", level);
+ continue;
+ }
+
+ if (test_bit(level, chip->irq_trig_low))
+ while (!cy8c95x0_gpio_get_value(gc, level))
+ handle_nested_irq(nested_irq);
+ else if (test_bit(level, chip->irq_trig_high))
+ while (cy8c95x0_gpio_get_value(gc, level))
+ handle_nested_irq(nested_irq);
+ else
+ handle_nested_irq(nested_irq);
+
+ ret = 1;
+ }
+
+ return IRQ_RETVAL(ret);
+}
+
+static int cy8c95x0_pinctrl_get_groups_count(struct pinctrl_dev *pctldev)
+{
+ struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev);
+
+ return chip->tpin;
+}
+
+static const char *cy8c95x0_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
+ unsigned int group)
+{
+ return cy8c95x0_groups[group];
+}
+
+static int cy8c95x0_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
+ unsigned int group,
+ const unsigned int **pins,
+ unsigned int *num_pins)
+{
+ *pins = &cy8c9560_pins[group].number;
+ *num_pins = 1;
+ return 0;
+}
+
+static const char *cy8c95x0_get_fname(unsigned int selector)
+{
+ if (selector == 0)
+ return "gpio";
+ else
+ return "pwm";
+}
+
+static void cy8c95x0_pin_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s,
+ unsigned int pin)
+{
+ struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev);
+ DECLARE_BITMAP(mask, MAX_LINE);
+ DECLARE_BITMAP(pwm, MAX_LINE);
+
+ bitmap_zero(mask, MAX_LINE);
+ __set_bit(pin, mask);
+
+ if (cy8c95x0_read_regs_mask(chip, CY8C95X0_PWMSEL, pwm, mask)) {
+ seq_puts(s, "not available");
+ return;
+ }
+
+ seq_printf(s, "MODE:%s", cy8c95x0_get_fname(test_bit(pin, pwm)));
+}
+
+static const struct pinctrl_ops cy8c95x0_pinctrl_ops = {
+ .get_groups_count = cy8c95x0_pinctrl_get_groups_count,
+ .get_group_name = cy8c95x0_pinctrl_get_group_name,
+ .get_group_pins = cy8c95x0_pinctrl_get_group_pins,
+#ifdef CONFIG_OF
+ .dt_node_to_map = pinconf_generic_dt_node_to_map_pin,
+ .dt_free_map = pinconf_generic_dt_free_map,
+#endif
+ .pin_dbg_show = cy8c95x0_pin_dbg_show,
+};
+
+static const char *cy8c95x0_get_function_name(struct pinctrl_dev *pctldev, unsigned int selector)
+{
+ return cy8c95x0_get_fname(selector);
+}
+
+static int cy8c95x0_get_functions_count(struct pinctrl_dev *pctldev)
+{
+ return 2;
+}
+
+static int cy8c95x0_get_function_groups(struct pinctrl_dev *pctldev, unsigned int selector,
+ const char * const **groups,
+ unsigned int * const num_groups)
+{
+ struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev);
+
+ *groups = cy8c95x0_groups;
+ *num_groups = chip->tpin;
+ return 0;
+}
+
+static int cy8c95x0_pinmux_cfg(struct cy8c95x0_pinctrl *chip,
+ unsigned int val,
+ unsigned long off)
+{
+ u8 port = cypress_get_port(chip, off);
+ u8 bit = cypress_get_pin_mask(chip, off);
+ int ret;
+
+ /* Select port */
+ ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port);
+ if (ret < 0)
+ return ret;
+
+ ret = regmap_write_bits(chip->regmap, CY8C95X0_PWMSEL, bit, val ? bit : 0);
+ if (ret < 0)
+ return ret;
+
+ /* Set direction to output & set output to 1 so that PWM can work */
+ ret = regmap_write_bits(chip->regmap, CY8C95X0_DIRECTION, bit, bit);
+ if (ret < 0)
+ return ret;
+
+ return regmap_write_bits(chip->regmap, CY8C95X0_OUTPUT_(port), bit, bit);
+}
+
+static int cy8c95x0_set_mux(struct pinctrl_dev *pctldev, unsigned int selector,
+ unsigned int group)
+{
+ struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev);
+ int ret;
+
+ mutex_lock(&chip->i2c_lock);
+ ret = cy8c95x0_pinmux_cfg(chip, selector, group);
+ mutex_unlock(&chip->i2c_lock);
+
+ return ret;
+}
+
+static const struct pinmux_ops cy8c95x0_pmxops = {
+ .get_functions_count = cy8c95x0_get_functions_count,
+ .get_function_name = cy8c95x0_get_function_name,
+ .get_function_groups = cy8c95x0_get_function_groups,
+ .set_mux = cy8c95x0_set_mux,
+ .strict = true,
+};
+
+static int cy8c95x0_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
+ unsigned long *config)
+{
+ struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev);
+
+ return cy8c95x0_gpio_get_pincfg(chip, pin, config);
+}
+
+static int cy8c95x0_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
+ unsigned long *configs, unsigned int num_configs)
+{
+ struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev);
+ int ret = 0;
+ int i;
+
+ for (i = 0; i < num_configs; i++) {
+ ret = cy8c95x0_gpio_set_pincfg(chip, pin, configs[i]);
+ if (ret)
+ return ret;
+ }
+
+ return ret;
+}
+
+static const struct pinconf_ops cy8c95x0_pinconf_ops = {
+ .pin_config_get = cy8c95x0_pinconf_get,
+ .pin_config_set = cy8c95x0_pinconf_set,
+ .is_generic = true,
+};
+
+static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq)
+{
+ struct gpio_irq_chip *girq = &chip->gpio_chip.irq;
+ DECLARE_BITMAP(pending_irqs, MAX_LINE);
+ int ret;
+
+ mutex_init(&chip->irq_lock);
+
+ bitmap_zero(pending_irqs, MAX_LINE);
+
+ /* Read IRQ status register to clear all pending interrupts */
+ ret = cy8c95x0_irq_pending(chip, pending_irqs);
+ if (ret) {
+ dev_err(chip->dev, "failed to clear irq status register\n");
+ return ret;
+ }
+
+ /* Mask all interrupts */
+ bitmap_fill(chip->irq_mask, MAX_LINE);
+
+ gpio_irq_chip_set_chip(girq, &cy8c95x0_irqchip);
+
+ /* This will let us handle the parent IRQ in the driver */
+ girq->parent_handler = NULL;
+ girq->num_parents = 0;
+ girq->parents = NULL;
+ girq->default_type = IRQ_TYPE_NONE;
+ girq->handler = handle_simple_irq;
+ girq->threaded = true;
+
+ ret = devm_request_threaded_irq(chip->dev, irq,
+ NULL, cy8c95x0_irq_handler,
+ IRQF_ONESHOT | IRQF_SHARED | IRQF_TRIGGER_HIGH,
+ dev_name(chip->dev), chip);
+ if (ret) {
+ dev_err(chip->dev, "failed to request irq %d\n", irq);
+ return ret;
+ }
+ dev_info(chip->dev, "Registered threaded IRQ\n");
+
+ return 0;
+}
+
+static int cy8c95x0_setup_pinctrl(struct cy8c95x0_pinctrl *chip)
+{
+ struct pinctrl_desc *pd = &chip->pinctrl_desc;
+
+ pd->pctlops = &cy8c95x0_pinctrl_ops;
+ pd->confops = &cy8c95x0_pinconf_ops;
+ pd->pmxops = &cy8c95x0_pmxops;
+ pd->name = dev_name(chip->dev);
+ pd->pins = cy8c9560_pins;
+ pd->npins = chip->tpin;
+ pd->owner = THIS_MODULE;
+
+ chip->pctldev = devm_pinctrl_register(chip->dev, pd, chip);
+ if (IS_ERR(chip->pctldev))
+ return dev_err_probe(chip->dev, PTR_ERR(chip->pctldev),
+ "can't register controller\n");
+
+ return 0;
+}
+
+static int cy8c95x0_detect(struct i2c_client *client,
+ struct i2c_board_info *info)
+{
+ struct i2c_adapter *adapter = client->adapter;
+ int ret;
+ const char *name;
+
+ if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA))
+ return -ENODEV;
+
+ ret = i2c_smbus_read_byte_data(client, CY8C95X0_DEVID);
+ if (ret < 0)
+ return ret;
+ switch (ret & GENMASK(7, 4)) {
+ case 0x20:
+ name = cy8c95x0_id[0].name;
+ break;
+ case 0x40:
+ name = cy8c95x0_id[1].name;
+ break;
+ case 0x60:
+ name = cy8c95x0_id[2].name;
+ break;
+ default:
+ return -ENODEV;
+ }
+
+ dev_info(&client->dev, "Found a %s chip at 0x%02x.\n", name, client->addr);
+ strscpy(info->type, name, I2C_NAME_SIZE);
+
+ return 0;
+}
+
+static int cy8c95x0_probe(struct i2c_client *client)
+{
+ struct cy8c95x0_pinctrl *chip;
+ struct regulator *reg;
+ int ret;
+
+ chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL);
+ if (!chip)
+ return -ENOMEM;
+
+ chip->dev = &client->dev;
+
+ /* Set the device type */
+ chip->driver_data = (unsigned long)device_get_match_data(&client->dev);
+ if (!chip->driver_data)
+ chip->driver_data = i2c_match_id(cy8c95x0_id, client)->driver_data;
+ if (!chip->driver_data)
+ return -ENODEV;
+
+ i2c_set_clientdata(client, chip);
+
+ chip->tpin = chip->driver_data & CY8C95X0_GPIO_MASK;
+ chip->nport = DIV_ROUND_UP(CY8C95X0_PIN_TO_OFFSET(chip->tpin), BANK_SZ);
+
+ switch (chip->tpin) {
+ case 20:
+ strscpy(chip->name, cy8c95x0_id[0].name, I2C_NAME_SIZE);
+ break;
+ case 40:
+ strscpy(chip->name, cy8c95x0_id[1].name, I2C_NAME_SIZE);
+ break;
+ case 60:
+ strscpy(chip->name, cy8c95x0_id[2].name, I2C_NAME_SIZE);
+ break;
+ default:
+ return -ENODEV;
+ }
+
+ reg = devm_regulator_get(&client->dev, "vdd");
+ if (IS_ERR(reg)) {
+ if (PTR_ERR(reg) == -EPROBE_DEFER)
+ return -EPROBE_DEFER;
+ } else {
+ ret = regulator_enable(reg);
+ if (ret) {
+ dev_err(&client->dev, "failed to enable regulator vdd: %d\n", ret);
+ return ret;
+ }
+ chip->regulator = reg;
+ }
+
+ chip->regmap = devm_regmap_init_i2c(client, &cy8c95x0_i2c_regmap);
+ if (IS_ERR(chip->regmap)) {
+ ret = PTR_ERR(chip->regmap);
+ goto err_exit;
+ }
+
+ bitmap_zero(chip->push_pull, MAX_LINE);
+ bitmap_zero(chip->shiftmask, MAX_LINE);
+ bitmap_set(chip->shiftmask, 0, 20);
+ mutex_init(&chip->i2c_lock);
+
+ if (dmi_first_match(cy8c95x0_dmi_acpi_irq_info)) {
+ ret = cy8c95x0_acpi_get_irq(&client->dev);
+ if (ret > 0)
+ client->irq = ret;
+ }
+
+ if (client->irq) {
+ ret = cy8c95x0_irq_setup(chip, client->irq);
+ if (ret)
+ goto err_exit;
+ }
+
+ ret = cy8c95x0_setup_pinctrl(chip);
+ if (ret)
+ goto err_exit;
+
+ ret = cy8c95x0_setup_gpiochip(chip);
+ if (ret)
+ goto err_exit;
+
+ return 0;
+
+err_exit:
+ if (!IS_ERR_OR_NULL(chip->regulator))
+ regulator_disable(chip->regulator);
+ return ret;
+}
+
+static void cy8c95x0_remove(struct i2c_client *client)
+{
+ struct cy8c95x0_pinctrl *chip = i2c_get_clientdata(client);
+
+ if (!IS_ERR_OR_NULL(chip->regulator))
+ regulator_disable(chip->regulator);
+}
+
+static const struct acpi_device_id cy8c95x0_acpi_ids[] = {
+ { "INT3490", 40, },
+ { }
+};
+MODULE_DEVICE_TABLE(acpi, cy8c95x0_acpi_ids);
+
+static struct i2c_driver cy8c95x0_driver = {
+ .driver = {
+ .name = "cy8c95x0-pinctrl",
+ .of_match_table = cy8c95x0_dt_ids,
+ .acpi_match_table = cy8c95x0_acpi_ids,
+ },
+ .probe_new = cy8c95x0_probe,
+ .remove = cy8c95x0_remove,
+ .id_table = cy8c95x0_id,
+ .detect = cy8c95x0_detect,
+};
+module_i2c_driver(cy8c95x0_driver);
+
+MODULE_AUTHOR("Patrick Rudolph <patrick.rudolph@9elements.com>");
+MODULE_AUTHOR("Naresh Solanki <naresh.solanki@9elements.com>");
+MODULE_DESCRIPTION("Pinctrl driver for CY8C95X0");
+MODULE_LICENSE("GPL");
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kernel.h>
-#include <linux/of_device.h>
-#include <linux/of_irq.h>
-#include <linux/of_platform.h>
+#include <linux/mod_devicetable.h>
+#include <linux/of.h>
#include <linux/pinctrl/pinctrl.h>
#include <linux/pinctrl/pinmux.h>
#include <linux/pinctrl/pinconf.h>
#include <linux/pinctrl/pinconf-generic.h>
#include <linux/platform_device.h>
+#include <linux/property.h>
#include <linux/regmap.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
};
static int __init ingenic_gpio_probe(struct ingenic_pinctrl *jzpc,
- struct device_node *node)
+ struct fwnode_handle *fwnode)
{
struct ingenic_gpio_chip *jzgc;
struct device *dev = jzpc->dev;
unsigned int bank;
int err;
- err = of_property_read_u32(node, "reg", &bank);
+ err = fwnode_property_read_u32(fwnode, "reg", &bank);
if (err) {
dev_err(dev, "Cannot read \"reg\" property: %i\n", err);
return err;
jzgc->gc.ngpio = 32;
jzgc->gc.parent = dev;
- jzgc->gc.of_node = node;
+ jzgc->gc.fwnode = fwnode;
jzgc->gc.owner = THIS_MODULE;
jzgc->gc.set = ingenic_gpio_set;
jzgc->gc.request = gpiochip_generic_request;
jzgc->gc.free = gpiochip_generic_free;
- jzgc->irq = irq_of_parse_and_map(node, 0);
- if (!jzgc->irq)
+ err = fwnode_irq_get(fwnode, 0);
+ if (err < 0)
+ return err;
+ if (!err)
return -EINVAL;
+ jzgc->irq = err;
girq = &jzgc->gc.irq;
gpio_irq_chip_set_chip(girq, &ingenic_gpio_irqchip);
struct pinctrl_desc *pctl_desc;
void __iomem *base;
const struct ingenic_chip_info *chip_info;
- struct device_node *node;
struct regmap_config regmap_config;
+ struct fwnode_handle *fwnode;
unsigned int i;
int err;
- chip_info = of_device_get_match_data(dev);
+ chip_info = device_get_match_data(dev);
if (!chip_info) {
dev_err(dev, "Unsupported SoC\n");
return -EINVAL;
dev_set_drvdata(dev, jzpc->map);
- for_each_child_of_node(dev->of_node, node) {
- if (of_match_node(ingenic_gpio_of_matches, node)) {
- err = ingenic_gpio_probe(jzpc, node);
+ device_for_each_child_node(dev, fwnode) {
+ if (of_match_node(ingenic_gpio_of_matches, to_of_node(fwnode))) {
+ err = ingenic_gpio_probe(jzpc, fwnode);
if (err) {
- of_node_put(node);
+ fwnode_handle_put(fwnode);
return err;
}
}
mcp->chip.get = mcp23s08_get;
mcp->chip.direction_output = mcp23s08_direction_output;
mcp->chip.set = mcp23s08_set;
-#ifdef CONFIG_OF_GPIO
- mcp->chip.of_gpio_n_cells = 2;
-#endif
mcp->chip.base = base;
mcp->chip.can_sleep = true;
gc->can_sleep = !bank->is_input;
if (bank->is_input && priv->properties->flags & SGPIO_FLAGS_HAS_IRQ) {
- int irq = fwnode_irq_get(fwnode, 0);
+ int irq;
- if (irq) {
+ irq = fwnode_irq_get(fwnode, 0);
+ if (irq > 0) {
struct gpio_irq_chip *girq = &gc->irq;
gpio_irq_chip_set_chip(girq, µchip_sgpio_irqchip);
.remove = ocelot_pinctrl_remove,
};
module_platform_driver(ocelot_pinctrl_driver);
+
+MODULE_DESCRIPTION("Ocelot Chip Pinctrl Driver");
MODULE_LICENSE("Dual MIT/GPL");
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irq.h>
-#include <linux/of.h>
-#include <linux/of_irq.h>
+#include <linux/mod_devicetable.h>
#include <linux/pinctrl/pinconf.h>
#include <linux/pinctrl/pinconf-generic.h>
#include <linux/pinctrl/pinctrl.h>
#include <linux/pinctrl/pinmux.h>
#include <linux/platform_device.h>
+#include <linux/property.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
static int pistachio_gpio_register(struct pistachio_pinctrl *pctl)
{
- struct device_node *node = pctl->dev->of_node;
struct pistachio_gpio_bank *bank;
unsigned int i;
int irq, ret = 0;
for (i = 0; i < pctl->nbanks; i++) {
char child_name[sizeof("gpioXX")];
- struct device_node *child;
+ struct fwnode_handle *child;
struct gpio_irq_chip *girq;
snprintf(child_name, sizeof(child_name), "gpio%d", i);
- child = of_get_child_by_name(node, child_name);
+ child = device_get_named_child_node(pctl->dev, child_name);
if (!child) {
dev_err(pctl->dev, "No node for bank %u\n", i);
ret = -ENODEV;
goto err;
}
- if (!of_find_property(child, "gpio-controller", NULL)) {
+ if (!fwnode_property_present(child, "gpio-controller")) {
+ fwnode_handle_put(child);
dev_err(pctl->dev,
"No gpio-controller property for bank %u\n", i);
- of_node_put(child);
ret = -ENODEV;
goto err;
}
- irq = irq_of_parse_and_map(child, 0);
- if (!irq) {
+ ret = fwnode_irq_get(child, 0);
+ if (ret < 0) {
+ fwnode_handle_put(child);
+ dev_err(pctl->dev, "Failed to retrieve IRQ for bank %u\n", i);
+ goto err;
+ }
+ if (!ret) {
+ fwnode_handle_put(child);
dev_err(pctl->dev, "No IRQ for bank %u\n", i);
- of_node_put(child);
ret = -EINVAL;
goto err;
}
+ irq = ret;
bank = &pctl->gpio_banks[i];
bank->pctl = pctl;
bank->base = pctl->base + GPIO_BANK_BASE(i);
bank->gpio_chip.parent = pctl->dev;
- bank->gpio_chip.of_node = child;
+ bank->gpio_chip.fwnode = child;
girq = &bank->gpio_chip.irq;
girq->chip = &bank->irq_chip;
#define IOMUX_UNROUTED BIT(3)
#define IOMUX_WIDTH_3BIT BIT(4)
#define IOMUX_WIDTH_2BIT BIT(5)
+#define IOMUX_L_SOURCE_PMU BIT(6)
#define PIN_BANK(id, pins, label) \
{ \
.pull_type[3] = pull3, \
}
+#define PIN_BANK_IOMUX_FLAGS_OFFSET(id, pins, label, iom0, iom1, iom2, \
+ iom3, offset0, offset1, offset2, \
+ offset3) \
+ { \
+ .bank_num = id, \
+ .nr_pins = pins, \
+ .name = label, \
+ .iomux = { \
+ { .type = iom0, .offset = offset0 }, \
+ { .type = iom1, .offset = offset1 }, \
+ { .type = iom2, .offset = offset2 }, \
+ { .type = iom3, .offset = offset3 }, \
+ }, \
+ }
+
#define PIN_BANK_IOMUX_DRV_FLAGS_OFFSET(id, pins, label, iom0, iom1, \
iom2, iom3, drv0, drv1, drv2, \
drv3, offset0, offset1, \
},
};
+static struct rockchip_mux_recalced_data rv1126_mux_recalced_data[] = {
+ {
+ .num = 0,
+ .pin = 20,
+ .reg = 0x10000,
+ .bit = 0,
+ .mask = 0xf
+ },
+ {
+ .num = 0,
+ .pin = 21,
+ .reg = 0x10000,
+ .bit = 4,
+ .mask = 0xf
+ },
+ {
+ .num = 0,
+ .pin = 22,
+ .reg = 0x10000,
+ .bit = 8,
+ .mask = 0xf
+ },
+ {
+ .num = 0,
+ .pin = 23,
+ .reg = 0x10000,
+ .bit = 12,
+ .mask = 0xf
+ },
+};
+
static struct rockchip_mux_recalced_data rk3128_mux_recalced_data[] = {
{
.num = 2,
RK_MUXROUTE_SAME(1, RK_PB7, 2, 0x184, BIT(16 + 9) | BIT(9)), /* uart3-rxm1 */
};
+static struct rockchip_mux_route_data rv1126_mux_route_data[] = {
+ RK_MUXROUTE_GRF(3, RK_PD2, 1, 0x10260, WRITE_MASK_VAL(0, 0, 0)), /* I2S0_MCLK_M0 */
+ RK_MUXROUTE_GRF(3, RK_PB0, 3, 0x10260, WRITE_MASK_VAL(0, 0, 1)), /* I2S0_MCLK_M1 */
+
+ RK_MUXROUTE_GRF(0, RK_PD4, 4, 0x10260, WRITE_MASK_VAL(3, 2, 0)), /* I2S1_MCLK_M0 */
+ RK_MUXROUTE_GRF(1, RK_PD5, 2, 0x10260, WRITE_MASK_VAL(3, 2, 1)), /* I2S1_MCLK_M1 */
+ RK_MUXROUTE_GRF(2, RK_PC7, 6, 0x10260, WRITE_MASK_VAL(3, 2, 2)), /* I2S1_MCLK_M2 */
+
+ RK_MUXROUTE_GRF(1, RK_PD0, 1, 0x10260, WRITE_MASK_VAL(4, 4, 0)), /* I2S2_MCLK_M0 */
+ RK_MUXROUTE_GRF(2, RK_PB3, 2, 0x10260, WRITE_MASK_VAL(4, 4, 1)), /* I2S2_MCLK_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PD4, 2, 0x10260, WRITE_MASK_VAL(12, 12, 0)), /* PDM_CLK0_M0 */
+ RK_MUXROUTE_GRF(3, RK_PC0, 3, 0x10260, WRITE_MASK_VAL(12, 12, 1)), /* PDM_CLK0_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PC6, 1, 0x10264, WRITE_MASK_VAL(0, 0, 0)), /* CIF_CLKOUT_M0 */
+ RK_MUXROUTE_GRF(2, RK_PD1, 3, 0x10264, WRITE_MASK_VAL(0, 0, 1)), /* CIF_CLKOUT_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PA4, 5, 0x10264, WRITE_MASK_VAL(5, 4, 0)), /* I2C3_SCL_M0 */
+ RK_MUXROUTE_GRF(2, RK_PD4, 7, 0x10264, WRITE_MASK_VAL(5, 4, 1)), /* I2C3_SCL_M1 */
+ RK_MUXROUTE_GRF(1, RK_PD6, 3, 0x10264, WRITE_MASK_VAL(5, 4, 2)), /* I2C3_SCL_M2 */
+
+ RK_MUXROUTE_GRF(3, RK_PA0, 7, 0x10264, WRITE_MASK_VAL(6, 6, 0)), /* I2C4_SCL_M0 */
+ RK_MUXROUTE_GRF(4, RK_PA0, 4, 0x10264, WRITE_MASK_VAL(6, 6, 1)), /* I2C4_SCL_M1 */
+
+ RK_MUXROUTE_GRF(2, RK_PA5, 7, 0x10264, WRITE_MASK_VAL(9, 8, 0)), /* I2C5_SCL_M0 */
+ RK_MUXROUTE_GRF(3, RK_PB0, 5, 0x10264, WRITE_MASK_VAL(9, 8, 1)), /* I2C5_SCL_M1 */
+ RK_MUXROUTE_GRF(1, RK_PD0, 4, 0x10264, WRITE_MASK_VAL(9, 8, 2)), /* I2C5_SCL_M2 */
+
+ RK_MUXROUTE_GRF(3, RK_PC0, 5, 0x10264, WRITE_MASK_VAL(11, 10, 0)), /* SPI1_CLK_M0 */
+ RK_MUXROUTE_GRF(1, RK_PC6, 3, 0x10264, WRITE_MASK_VAL(11, 10, 1)), /* SPI1_CLK_M1 */
+ RK_MUXROUTE_GRF(2, RK_PD5, 6, 0x10264, WRITE_MASK_VAL(11, 10, 2)), /* SPI1_CLK_M2 */
+
+ RK_MUXROUTE_GRF(3, RK_PC0, 2, 0x10264, WRITE_MASK_VAL(12, 12, 0)), /* RGMII_CLK_M0 */
+ RK_MUXROUTE_GRF(2, RK_PB7, 2, 0x10264, WRITE_MASK_VAL(12, 12, 1)), /* RGMII_CLK_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PA1, 3, 0x10264, WRITE_MASK_VAL(13, 13, 0)), /* CAN_TXD_M0 */
+ RK_MUXROUTE_GRF(3, RK_PA7, 5, 0x10264, WRITE_MASK_VAL(13, 13, 1)), /* CAN_TXD_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PA4, 6, 0x10268, WRITE_MASK_VAL(0, 0, 0)), /* PWM8_M0 */
+ RK_MUXROUTE_GRF(2, RK_PD7, 5, 0x10268, WRITE_MASK_VAL(0, 0, 1)), /* PWM8_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PA5, 6, 0x10268, WRITE_MASK_VAL(2, 2, 0)), /* PWM9_M0 */
+ RK_MUXROUTE_GRF(2, RK_PD6, 5, 0x10268, WRITE_MASK_VAL(2, 2, 1)), /* PWM9_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PA6, 6, 0x10268, WRITE_MASK_VAL(4, 4, 0)), /* PWM10_M0 */
+ RK_MUXROUTE_GRF(2, RK_PD5, 5, 0x10268, WRITE_MASK_VAL(4, 4, 1)), /* PWM10_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PA7, 6, 0x10268, WRITE_MASK_VAL(6, 6, 0)), /* PWM11_IR_M0 */
+ RK_MUXROUTE_GRF(3, RK_PA1, 5, 0x10268, WRITE_MASK_VAL(6, 6, 1)), /* PWM11_IR_M1 */
+
+ RK_MUXROUTE_GRF(1, RK_PA5, 3, 0x10268, WRITE_MASK_VAL(8, 8, 0)), /* UART2_TX_M0 */
+ RK_MUXROUTE_GRF(3, RK_PA2, 1, 0x10268, WRITE_MASK_VAL(8, 8, 1)), /* UART2_TX_M1 */
+
+ RK_MUXROUTE_GRF(3, RK_PC6, 3, 0x10268, WRITE_MASK_VAL(11, 10, 0)), /* UART3_TX_M0 */
+ RK_MUXROUTE_GRF(1, RK_PA7, 2, 0x10268, WRITE_MASK_VAL(11, 10, 1)), /* UART3_TX_M1 */
+ RK_MUXROUTE_GRF(3, RK_PA0, 4, 0x10268, WRITE_MASK_VAL(11, 10, 2)), /* UART3_TX_M2 */
+
+ RK_MUXROUTE_GRF(3, RK_PA4, 4, 0x10268, WRITE_MASK_VAL(13, 12, 0)), /* UART4_TX_M0 */
+ RK_MUXROUTE_GRF(2, RK_PA6, 4, 0x10268, WRITE_MASK_VAL(13, 12, 1)), /* UART4_TX_M1 */
+ RK_MUXROUTE_GRF(1, RK_PD5, 3, 0x10268, WRITE_MASK_VAL(13, 12, 2)), /* UART4_TX_M2 */
+
+ RK_MUXROUTE_GRF(3, RK_PA6, 4, 0x10268, WRITE_MASK_VAL(15, 14, 0)), /* UART5_TX_M0 */
+ RK_MUXROUTE_GRF(2, RK_PB0, 4, 0x10268, WRITE_MASK_VAL(15, 14, 1)), /* UART5_TX_M1 */
+ RK_MUXROUTE_GRF(2, RK_PA0, 3, 0x10268, WRITE_MASK_VAL(15, 14, 2)), /* UART5_TX_M2 */
+
+ RK_MUXROUTE_PMU(0, RK_PB6, 3, 0x0114, WRITE_MASK_VAL(0, 0, 0)), /* PWM0_M0 */
+ RK_MUXROUTE_PMU(2, RK_PB3, 5, 0x0114, WRITE_MASK_VAL(0, 0, 1)), /* PWM0_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PB7, 3, 0x0114, WRITE_MASK_VAL(2, 2, 0)), /* PWM1_M0 */
+ RK_MUXROUTE_PMU(2, RK_PB2, 5, 0x0114, WRITE_MASK_VAL(2, 2, 1)), /* PWM1_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PC0, 3, 0x0114, WRITE_MASK_VAL(4, 4, 0)), /* PWM2_M0 */
+ RK_MUXROUTE_PMU(2, RK_PB1, 5, 0x0114, WRITE_MASK_VAL(4, 4, 1)), /* PWM2_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PC1, 3, 0x0114, WRITE_MASK_VAL(6, 6, 0)), /* PWM3_IR_M0 */
+ RK_MUXROUTE_PMU(2, RK_PB0, 5, 0x0114, WRITE_MASK_VAL(6, 6, 1)), /* PWM3_IR_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PC2, 3, 0x0114, WRITE_MASK_VAL(8, 8, 0)), /* PWM4_M0 */
+ RK_MUXROUTE_PMU(2, RK_PA7, 5, 0x0114, WRITE_MASK_VAL(8, 8, 1)), /* PWM4_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PC3, 3, 0x0114, WRITE_MASK_VAL(10, 10, 0)), /* PWM5_M0 */
+ RK_MUXROUTE_PMU(2, RK_PA6, 5, 0x0114, WRITE_MASK_VAL(10, 10, 1)), /* PWM5_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PB2, 3, 0x0114, WRITE_MASK_VAL(12, 12, 0)), /* PWM6_M0 */
+ RK_MUXROUTE_PMU(2, RK_PD4, 5, 0x0114, WRITE_MASK_VAL(12, 12, 1)), /* PWM6_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PB1, 3, 0x0114, WRITE_MASK_VAL(14, 14, 0)), /* PWM7_IR_M0 */
+ RK_MUXROUTE_PMU(3, RK_PA0, 5, 0x0114, WRITE_MASK_VAL(14, 14, 1)), /* PWM7_IR_M1 */
+
+ RK_MUXROUTE_PMU(0, RK_PB0, 1, 0x0118, WRITE_MASK_VAL(1, 0, 0)), /* SPI0_CLK_M0 */
+ RK_MUXROUTE_PMU(2, RK_PA1, 1, 0x0118, WRITE_MASK_VAL(1, 0, 1)), /* SPI0_CLK_M1 */
+ RK_MUXROUTE_PMU(2, RK_PB2, 6, 0x0118, WRITE_MASK_VAL(1, 0, 2)), /* SPI0_CLK_M2 */
+
+ RK_MUXROUTE_PMU(0, RK_PB6, 2, 0x0118, WRITE_MASK_VAL(2, 2, 0)), /* UART1_TX_M0 */
+ RK_MUXROUTE_PMU(1, RK_PD0, 5, 0x0118, WRITE_MASK_VAL(2, 2, 1)), /* UART1_TX_M1 */
+};
+
static struct rockchip_mux_route_data rk3128_mux_route_data[] = {
RK_MUXROUTE_SAME(1, RK_PB2, 1, 0x144, BIT(16 + 3) | BIT(16 + 4)), /* spi-0 */
RK_MUXROUTE_SAME(1, RK_PD3, 3, 0x144, BIT(16 + 3) | BIT(16 + 4) | BIT(3)), /* spi-1 */
if (bank->iomux[iomux_num].type & IOMUX_GPIO_ONLY)
return RK_FUNC_GPIO;
- regmap = (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU)
- ? info->regmap_pmu : info->regmap_base;
+ if (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU)
+ regmap = info->regmap_pmu;
+ else if (bank->iomux[iomux_num].type & IOMUX_L_SOURCE_PMU)
+ regmap = (pin % 8 < 4) ? info->regmap_pmu : info->regmap_base;
+ else
+ regmap = info->regmap_base;
/* get basic quadrupel of mux registers and the correct reg inside */
mux_type = bank->iomux[iomux_num].type;
dev_dbg(dev, "setting mux of GPIO%d-%d to %d\n", bank->bank_num, pin, mux);
- regmap = (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU)
- ? info->regmap_pmu : info->regmap_base;
+ if (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU)
+ regmap = info->regmap_pmu;
+ else if (bank->iomux[iomux_num].type & IOMUX_L_SOURCE_PMU)
+ regmap = (pin % 8 < 4) ? info->regmap_pmu : info->regmap_base;
+ else
+ regmap = info->regmap_base;
/* get basic quadrupel of mux registers and the correct reg inside */
mux_type = bank->iomux[iomux_num].type;
return 0;
}
+#define RV1126_PULL_PMU_OFFSET 0x40
+#define RV1126_PULL_GRF_GPIO1A0_OFFSET 0x10108
+#define RV1126_PULL_PINS_PER_REG 8
+#define RV1126_PULL_BITS_PER_PIN 2
+#define RV1126_PULL_BANK_STRIDE 16
+#define RV1126_GPIO_C4_D7(p) (p >= 20 && p <= 31) /* GPIO0_C4 ~ GPIO0_D7 */
+
+static int rv1126_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank,
+ int pin_num, struct regmap **regmap,
+ int *reg, u8 *bit)
+{
+ struct rockchip_pinctrl *info = bank->drvdata;
+
+ /* The first 24 pins of the first bank are located in PMU */
+ if (bank->bank_num == 0) {
+ if (RV1126_GPIO_C4_D7(pin_num)) {
+ *regmap = info->regmap_base;
+ *reg = RV1126_PULL_GRF_GPIO1A0_OFFSET;
+ *reg -= (((31 - pin_num) / RV1126_PULL_PINS_PER_REG + 1) * 4);
+ *bit = pin_num % RV1126_PULL_PINS_PER_REG;
+ *bit *= RV1126_PULL_BITS_PER_PIN;
+ return 0;
+ }
+ *regmap = info->regmap_pmu;
+ *reg = RV1126_PULL_PMU_OFFSET;
+ } else {
+ *reg = RV1126_PULL_GRF_GPIO1A0_OFFSET;
+ *regmap = info->regmap_base;
+ *reg += (bank->bank_num - 1) * RV1126_PULL_BANK_STRIDE;
+ }
+
+ *reg += ((pin_num / RV1126_PULL_PINS_PER_REG) * 4);
+ *bit = (pin_num % RV1126_PULL_PINS_PER_REG);
+ *bit *= RV1126_PULL_BITS_PER_PIN;
+
+ return 0;
+}
+
+#define RV1126_DRV_PMU_OFFSET 0x20
+#define RV1126_DRV_GRF_GPIO1A0_OFFSET 0x10090
+#define RV1126_DRV_BITS_PER_PIN 4
+#define RV1126_DRV_PINS_PER_REG 4
+#define RV1126_DRV_BANK_STRIDE 32
+
+static int rv1126_calc_drv_reg_and_bit(struct rockchip_pin_bank *bank,
+ int pin_num, struct regmap **regmap,
+ int *reg, u8 *bit)
+{
+ struct rockchip_pinctrl *info = bank->drvdata;
+
+ /* The first 24 pins of the first bank are located in PMU */
+ if (bank->bank_num == 0) {
+ if (RV1126_GPIO_C4_D7(pin_num)) {
+ *regmap = info->regmap_base;
+ *reg = RV1126_DRV_GRF_GPIO1A0_OFFSET;
+ *reg -= (((31 - pin_num) / RV1126_DRV_PINS_PER_REG + 1) * 4);
+ *reg -= 0x4;
+ *bit = pin_num % RV1126_DRV_PINS_PER_REG;
+ *bit *= RV1126_DRV_BITS_PER_PIN;
+ return 0;
+ }
+ *regmap = info->regmap_pmu;
+ *reg = RV1126_DRV_PMU_OFFSET;
+ } else {
+ *regmap = info->regmap_base;
+ *reg = RV1126_DRV_GRF_GPIO1A0_OFFSET;
+ *reg += (bank->bank_num - 1) * RV1126_DRV_BANK_STRIDE;
+ }
+
+ *reg += ((pin_num / RV1126_DRV_PINS_PER_REG) * 4);
+ *bit = pin_num % RV1126_DRV_PINS_PER_REG;
+ *bit *= RV1126_DRV_BITS_PER_PIN;
+
+ return 0;
+}
+
+#define RV1126_SCHMITT_PMU_OFFSET 0x60
+#define RV1126_SCHMITT_GRF_GPIO1A0_OFFSET 0x10188
+#define RV1126_SCHMITT_BANK_STRIDE 16
+#define RV1126_SCHMITT_PINS_PER_GRF_REG 8
+#define RV1126_SCHMITT_PINS_PER_PMU_REG 8
+
+static int rv1126_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank,
+ int pin_num,
+ struct regmap **regmap,
+ int *reg, u8 *bit)
+{
+ struct rockchip_pinctrl *info = bank->drvdata;
+ int pins_per_reg;
+
+ if (bank->bank_num == 0) {
+ if (RV1126_GPIO_C4_D7(pin_num)) {
+ *regmap = info->regmap_base;
+ *reg = RV1126_SCHMITT_GRF_GPIO1A0_OFFSET;
+ *reg -= (((31 - pin_num) / RV1126_SCHMITT_PINS_PER_GRF_REG + 1) * 4);
+ *bit = pin_num % RV1126_SCHMITT_PINS_PER_GRF_REG;
+ return 0;
+ }
+ *regmap = info->regmap_pmu;
+ *reg = RV1126_SCHMITT_PMU_OFFSET;
+ pins_per_reg = RV1126_SCHMITT_PINS_PER_PMU_REG;
+ } else {
+ *regmap = info->regmap_base;
+ *reg = RV1126_SCHMITT_GRF_GPIO1A0_OFFSET;
+ pins_per_reg = RV1126_SCHMITT_PINS_PER_GRF_REG;
+ *reg += (bank->bank_num - 1) * RV1126_SCHMITT_BANK_STRIDE;
+ }
+ *reg += ((pin_num / pins_per_reg) * 4);
+ *bit = pin_num % pins_per_reg;
+
+ return 0;
+}
+
#define RK3308_SCHMITT_PINS_PER_REG 8
#define RK3308_SCHMITT_BANK_STRIDE 16
#define RK3308_SCHMITT_GRF_OFFSET 0x1a0
goto config;
}
+ if (ctrl->type == RV1126) {
+ rmask_bits = RV1126_DRV_BITS_PER_PIN;
+ ret = strength;
+ goto config;
+ }
+
ret = -EINVAL;
for (i = 0; i < ARRAY_SIZE(rockchip_perpin_drv_list[drv_type]); i++) {
if (rockchip_perpin_drv_list[drv_type][i] == strength) {
break;
case PX30:
case RV1108:
+ case RV1126:
case RK3188:
case RK3288:
case RK3308:
return 0;
}
+static int rockchip_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
+ struct pinctrl_gpio_range *range,
+ unsigned offset,
+ bool input)
+{
+ struct rockchip_pinctrl *info = pinctrl_dev_get_drvdata(pctldev);
+ struct rockchip_pin_bank *bank;
+
+ bank = pin_to_bank(info, offset);
+ return rockchip_set_mux(bank, offset - bank->pin_base, RK_FUNC_GPIO);
+}
+
static const struct pinmux_ops rockchip_pmx_ops = {
.get_functions_count = rockchip_pmx_get_funcs_count,
.get_function_name = rockchip_pmx_get_func_name,
.get_function_groups = rockchip_pmx_get_groups,
.set_mux = rockchip_pmx_set,
+ .gpio_set_direction = rockchip_pmx_gpio_set_direction,
};
/*
return pull ? false : true;
case PX30:
case RV1108:
+ case RV1126:
case RK3188:
case RK3288:
case RK3308:
/* preset iomux offset value, set new start value */
if (iom->offset >= 0) {
- if (iom->type & IOMUX_SOURCE_PMU)
+ if ((iom->type & IOMUX_SOURCE_PMU) ||
+ (iom->type & IOMUX_L_SOURCE_PMU))
pmu_offs = iom->offset;
else
grf_offs = iom->offset;
} else { /* set current iomux offset */
- iom->offset = (iom->type & IOMUX_SOURCE_PMU) ?
+ iom->offset = ((iom->type & IOMUX_SOURCE_PMU) ||
+ (iom->type & IOMUX_L_SOURCE_PMU)) ?
pmu_offs : grf_offs;
}
inc = (iom->type & (IOMUX_WIDTH_4BIT |
IOMUX_WIDTH_3BIT |
IOMUX_WIDTH_2BIT)) ? 8 : 4;
- if (iom->type & IOMUX_SOURCE_PMU)
+ if ((iom->type & IOMUX_SOURCE_PMU) || (iom->type & IOMUX_L_SOURCE_PMU))
pmu_offs += inc;
else
grf_offs += inc;
.schmitt_calc_reg = rv1108_calc_schmitt_reg_and_bit,
};
+static struct rockchip_pin_bank rv1126_pin_banks[] = {
+ PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0",
+ IOMUX_WIDTH_4BIT | IOMUX_SOURCE_PMU,
+ IOMUX_WIDTH_4BIT | IOMUX_SOURCE_PMU,
+ IOMUX_WIDTH_4BIT | IOMUX_L_SOURCE_PMU,
+ IOMUX_WIDTH_4BIT),
+ PIN_BANK_IOMUX_FLAGS_OFFSET(1, 32, "gpio1",
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ 0x10010, 0x10018, 0x10020, 0x10028),
+ PIN_BANK_IOMUX_FLAGS(2, 32, "gpio2",
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT),
+ PIN_BANK_IOMUX_FLAGS(3, 32, "gpio3",
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT,
+ IOMUX_WIDTH_4BIT),
+ PIN_BANK_IOMUX_FLAGS(4, 2, "gpio4",
+ IOMUX_WIDTH_4BIT, 0, 0, 0),
+};
+
+static struct rockchip_pin_ctrl rv1126_pin_ctrl = {
+ .pin_banks = rv1126_pin_banks,
+ .nr_banks = ARRAY_SIZE(rv1126_pin_banks),
+ .label = "RV1126-GPIO",
+ .type = RV1126,
+ .grf_mux_offset = 0x10004, /* mux offset from GPIO0_D0 */
+ .pmu_mux_offset = 0x0,
+ .iomux_routes = rv1126_mux_route_data,
+ .niomux_routes = ARRAY_SIZE(rv1126_mux_route_data),
+ .iomux_recalced = rv1126_mux_recalced_data,
+ .niomux_recalced = ARRAY_SIZE(rv1126_mux_recalced_data),
+ .pull_calc_reg = rv1126_calc_pull_reg_and_bit,
+ .drv_calc_reg = rv1126_calc_drv_reg_and_bit,
+ .schmitt_calc_reg = rv1126_calc_schmitt_reg_and_bit,
+};
+
static struct rockchip_pin_bank rk2928_pin_banks[] = {
PIN_BANK(0, 32, "gpio0"),
PIN_BANK(1, 32, "gpio1"),
.data = &px30_pin_ctrl },
{ .compatible = "rockchip,rv1108-pinctrl",
.data = &rv1108_pin_ctrl },
+ { .compatible = "rockchip,rv1126-pinctrl",
+ .data = &rv1126_pin_ctrl },
{ .compatible = "rockchip,rk2928-pinctrl",
.data = &rk2928_pin_ctrl },
{ .compatible = "rockchip,rk3036-pinctrl",
enum rockchip_pinctrl_type {
PX30,
RV1108,
+ RV1126,
RK2928,
RK3066B,
RK3128,
#include <linux/io.h>
#include <linux/of.h>
#include <linux/of_irq.h>
-#include <linux/of_gpio.h> /* of_get_named_gpio() */
#include <linux/of_address.h>
#include <linux/gpio/driver.h>
#include <linux/regmap.h>
return;
}
+static int st_pctl_dt_calculate_pin(struct st_pinctrl *info,
+ phandle bank, unsigned int offset)
+{
+ struct device_node *np;
+ struct gpio_chip *chip;
+ int retval = -EINVAL;
+ int i;
+
+ np = of_find_node_by_phandle(bank);
+ if (!np)
+ return -EINVAL;
+
+ for (i = 0; i < info->nbanks; i++) {
+ chip = &info->banks[i].gpio_chip;
+ if (chip->of_node == np) {
+ if (offset < chip->ngpio)
+ retval = chip->base + offset;
+ break;
+ }
+ }
+
+ of_node_put(np);
+ return retval;
+}
+
/*
* Each pin is represented in of the below forms.
* <bank offset mux direction rt_type rt_delay rt_clk>
struct device *dev = info->dev;
struct st_pinconf *conf;
struct device_node *pins;
+ phandle bank;
+ unsigned int offset;
int i = 0, npins = 0, nr_props, ret = 0;
pins = of_get_child_by_name(np, "st,pins");
conf = &grp->pin_conf[i];
/* bank & offset */
- be32_to_cpup(list++);
- be32_to_cpup(list++);
- conf->pin = of_get_named_gpio(pins, pp->name, 0);
+ bank = be32_to_cpup(list++);
+ offset = be32_to_cpup(list++);
+ conf->pin = st_pctl_dt_calculate_pin(info, bank, offset);
conf->name = pp->name;
grp->pins[i] = conf->pin;
/* mux */
config PINCTRL_APQ8064
tristate "Qualcomm APQ8064 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_APQ8084
tristate "Qualcomm APQ8084 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_IPQ4019
tristate "Qualcomm IPQ4019 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_IPQ8064
tristate "Qualcomm IPQ8064 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_IPQ8074
tristate "Qualcomm Technologies, Inc. IPQ8074 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for
config PINCTRL_IPQ6018
tristate "Qualcomm Technologies, Inc. IPQ6018 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for
config PINCTRL_MSM8226
tristate "Qualcomm 8226 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8660
tristate "Qualcomm 8660 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8960
tristate "Qualcomm 8960 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MDM9615
tristate "Qualcomm 9615 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8X74
tristate "Qualcomm 8x74 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8909
tristate "Qualcomm 8909 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8953
tristate "Qualcomm 8953 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8976
tristate "Qualcomm 8976 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8994
tristate "Qualcomm 8994 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8996
tristate "Qualcomm MSM8996 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_MSM8998
tristate "Qualcomm MSM8998 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_QCM2290
tristate "Qualcomm QCM2290 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_QCS404
tristate "Qualcomm QCS404 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SC7180
tristate "Qualcomm Technologies Inc SC7180 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SC7280
tristate "Qualcomm Technologies Inc SC7280 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SC7280_LPASS_LPI
tristate "Qualcomm Technologies Inc SC7280 LPASS LPI pin controller driver"
depends on GPIOLIB
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_LPASS_LPI
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SC8180X
tristate "Qualcomm Technologies Inc SC8180x pin controller driver"
depends on (OF || ACPI)
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SC8280XP
tristate "Qualcomm Technologies Inc SC8280xp pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SDM660
tristate "Qualcomm Technologies Inc SDM660 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SDM845
tristate "Qualcomm Technologies Inc SDM845 pin controller driver"
depends on (OF || ACPI)
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SDX55
tristate "Qualcomm Technologies Inc SDX55 pin controller driver"
depends on OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM6115
tristate "Qualcomm Technologies Inc SM6115,SM4250 pin controller driver"
depends on GPIOLIB && OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM6125
tristate "Qualcomm Technologies Inc SM6125 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM6350
tristate "Qualcomm Technologies Inc SM6350 pin controller driver"
depends on GPIOLIB && OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM6375
tristate "Qualcomm Technologies Inc SM6375 pin controller driver"
depends on GPIOLIB && OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SDX65
tristate "Qualcomm Technologies Inc SDX65 pin controller driver"
depends on GPIOLIB && OF
+ depends on ARM || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM8150
tristate "Qualcomm Technologies Inc SM8150 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM8250
tristate "Qualcomm Technologies Inc SM8250 pin controller driver"
depends on OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM8250_LPASS_LPI
tristate "Qualcomm Technologies Inc SM8250 LPASS LPI pin controller driver"
depends on GPIOLIB
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_LPASS_LPI
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM8350
tristate "Qualcomm Technologies Inc SM8350 pin controller driver"
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
config PINCTRL_SM8450
tristate "Qualcomm Technologies Inc SM8450 pin controller driver"
depends on GPIOLIB && OF
+ depends on ARM64 || COMPILE_TEST
depends on PINCTRL_MSM
help
This is the pinctrl, pinmux, pinconf and gpiolib driver for the
Qualcomm Technologies Inc TLMM block found on the Qualcomm
Technologies Inc SM8450 platform.
+config PINCTRL_SM8450_LPASS_LPI
+ tristate "Qualcomm Technologies Inc SM8450 LPASS LPI pin controller driver"
+ depends on GPIOLIB
+ depends on ARM64 || COMPILE_TEST
+ depends on PINCTRL_LPASS_LPI
+ help
+ This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+ Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
+ (Low Power Island) found on the Qualcomm Technologies Inc SM8450 platform.
+
+config PINCTRL_SC8280XP_LPASS_LPI
+ tristate "Qualcomm Technologies Inc SC8280XP LPASS LPI pin controller driver"
+ depends on GPIOLIB
+ depends on ARM64 || COMPILE_TEST
+ depends on PINCTRL_LPASS_LPI
+ help
+ This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+ Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
+ (Low Power Island) found on the Qualcomm Technologies Inc SC8280XP platform.
+
config PINCTRL_LPASS_LPI
tristate "Qualcomm Technologies Inc LPASS LPI pin controller driver"
select PINMUX
obj-$(CONFIG_PINCTRL_SM8250_LPASS_LPI) += pinctrl-sm8250-lpass-lpi.o
obj-$(CONFIG_PINCTRL_SM8350) += pinctrl-sm8350.o
obj-$(CONFIG_PINCTRL_SM8450) += pinctrl-sm8450.o
+obj-$(CONFIG_PINCTRL_SM8450_LPASS_LPI) += pinctrl-sm8450-lpass-lpi.o
+obj-$(CONFIG_PINCTRL_SC8280XP_LPASS_LPI) += pinctrl-sc8280xp-lpass-lpi.o
obj-$(CONFIG_PINCTRL_LPASS_LPI) += pinctrl-lpass-lpi.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022 Linaro Ltd.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-lpass-lpi.h"
+
+enum lpass_lpi_functions {
+ LPI_MUX_dmic1_clk,
+ LPI_MUX_dmic1_data,
+ LPI_MUX_dmic2_clk,
+ LPI_MUX_dmic2_data,
+ LPI_MUX_dmic3_clk,
+ LPI_MUX_dmic3_data,
+ LPI_MUX_dmic4_clk,
+ LPI_MUX_dmic4_data,
+ LPI_MUX_i2s1_clk,
+ LPI_MUX_i2s1_data,
+ LPI_MUX_i2s1_ws,
+ LPI_MUX_i2s2_clk,
+ LPI_MUX_i2s2_data,
+ LPI_MUX_i2s2_ws,
+ LPI_MUX_i2s3_clk,
+ LPI_MUX_i2s3_data,
+ LPI_MUX_i2s3_ws,
+ LPI_MUX_qua_mi2s_data,
+ LPI_MUX_qua_mi2s_sclk,
+ LPI_MUX_qua_mi2s_ws,
+ LPI_MUX_swr_rx_clk,
+ LPI_MUX_swr_rx_data,
+ LPI_MUX_swr_tx_clk,
+ LPI_MUX_swr_tx_data,
+ LPI_MUX_wsa_swr_clk,
+ LPI_MUX_wsa_swr_data,
+ LPI_MUX_wsa2_swr_clk,
+ LPI_MUX_wsa2_swr_data,
+ LPI_MUX_ext_mclk1_a,
+ LPI_MUX_ext_mclk1_b,
+ LPI_MUX_ext_mclk1_c,
+ LPI_MUX_gpio,
+ LPI_MUX__,
+};
+
+static int gpio0_pins[] = { 0 };
+static int gpio1_pins[] = { 1 };
+static int gpio2_pins[] = { 2 };
+static int gpio3_pins[] = { 3 };
+static int gpio4_pins[] = { 4 };
+static int gpio5_pins[] = { 5 };
+static int gpio6_pins[] = { 6 };
+static int gpio7_pins[] = { 7 };
+static int gpio8_pins[] = { 8 };
+static int gpio9_pins[] = { 9 };
+static int gpio10_pins[] = { 10 };
+static int gpio11_pins[] = { 11 };
+static int gpio12_pins[] = { 12 };
+static int gpio13_pins[] = { 13 };
+static int gpio14_pins[] = { 14 };
+static int gpio15_pins[] = { 15 };
+static int gpio16_pins[] = { 16 };
+static int gpio17_pins[] = { 17 };
+static int gpio18_pins[] = { 18 };
+
+static const struct pinctrl_pin_desc sc8280xp_lpi_pins[] = {
+ PINCTRL_PIN(0, "gpio0"),
+ PINCTRL_PIN(1, "gpio1"),
+ PINCTRL_PIN(2, "gpio2"),
+ PINCTRL_PIN(3, "gpio3"),
+ PINCTRL_PIN(4, "gpio4"),
+ PINCTRL_PIN(5, "gpio5"),
+ PINCTRL_PIN(6, "gpio6"),
+ PINCTRL_PIN(7, "gpio7"),
+ PINCTRL_PIN(8, "gpio8"),
+ PINCTRL_PIN(9, "gpio9"),
+ PINCTRL_PIN(10, "gpio10"),
+ PINCTRL_PIN(11, "gpio11"),
+ PINCTRL_PIN(12, "gpio12"),
+ PINCTRL_PIN(13, "gpio13"),
+ PINCTRL_PIN(14, "gpio14"),
+ PINCTRL_PIN(15, "gpio15"),
+ PINCTRL_PIN(16, "gpio16"),
+ PINCTRL_PIN(17, "gpio17"),
+ PINCTRL_PIN(18, "gpio18"),
+};
+
+static const char * const swr_tx_clk_groups[] = { "gpio0" };
+static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" };
+static const char * const swr_rx_clk_groups[] = { "gpio3" };
+static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5" };
+static const char * const dmic1_clk_groups[] = { "gpio6" };
+static const char * const dmic1_data_groups[] = { "gpio7" };
+static const char * const dmic2_clk_groups[] = { "gpio8" };
+static const char * const dmic2_data_groups[] = { "gpio9" };
+static const char * const dmic4_clk_groups[] = { "gpio17" };
+static const char * const dmic4_data_groups[] = { "gpio18" };
+static const char * const i2s2_clk_groups[] = { "gpio10" };
+static const char * const i2s2_ws_groups[] = { "gpio11" };
+static const char * const dmic3_clk_groups[] = { "gpio12" };
+static const char * const dmic3_data_groups[] = { "gpio13" };
+static const char * const qua_mi2s_sclk_groups[] = { "gpio0" };
+static const char * const qua_mi2s_ws_groups[] = { "gpio1" };
+static const char * const qua_mi2s_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" };
+static const char * const i2s1_clk_groups[] = { "gpio6" };
+static const char * const i2s1_ws_groups[] = { "gpio7" };
+static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" };
+static const char * const wsa_swr_clk_groups[] = { "gpio10" };
+static const char * const wsa_swr_data_groups[] = { "gpio11" };
+static const char * const wsa2_swr_clk_groups[] = { "gpio15" };
+static const char * const wsa2_swr_data_groups[] = { "gpio16" };
+static const char * const i2s2_data_groups[] = { "gpio15", "gpio16" };
+static const char * const i2s3_clk_groups[] = { "gpio12"};
+static const char * const i2s3_ws_groups[] = { "gpio13"};
+static const char * const i2s3_data_groups[] = { "gpio17", "gpio18"};
+static const char * const ext_mclk1_c_groups[] = { "gpio5" };
+static const char * const ext_mclk1_b_groups[] = { "gpio9" };
+static const char * const ext_mclk1_a_groups[] = { "gpio13" };
+
+static const struct lpi_pingroup sc8280xp_groups[] = {
+ LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _),
+ LPI_PINGROUP(1, 2, swr_tx_data, qua_mi2s_ws, _, _),
+ LPI_PINGROUP(2, 4, swr_tx_data, qua_mi2s_data, _, _),
+ LPI_PINGROUP(3, 8, swr_rx_clk, qua_mi2s_data, _, _),
+ LPI_PINGROUP(4, 10, swr_rx_data, qua_mi2s_data, _, _),
+ LPI_PINGROUP(5, 12, swr_rx_data, ext_mclk1_c, qua_mi2s_data, _),
+ LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _),
+ LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _),
+ LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _),
+ LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _),
+ LPI_PINGROUP(10, 16, i2s2_clk, wsa_swr_clk, _, _),
+ LPI_PINGROUP(11, 18, i2s2_ws, wsa_swr_data, _, _),
+ LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s3_clk, _, _),
+ LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s3_ws, ext_mclk1_a, _),
+ LPI_PINGROUP(14, 6, swr_tx_data, _, _, _),
+ LPI_PINGROUP(15, 20, i2s2_data, wsa2_swr_clk, _, _),
+ LPI_PINGROUP(16, 22, i2s2_data, wsa2_swr_data, _, _),
+ LPI_PINGROUP(17, LPI_NO_SLEW, dmic4_clk, i2s3_data, _, _),
+ LPI_PINGROUP(18, LPI_NO_SLEW, dmic4_data, i2s3_data, _, _),
+};
+
+static const struct lpi_function sc8280xp_functions[] = {
+ LPI_FUNCTION(dmic1_clk),
+ LPI_FUNCTION(dmic1_data),
+ LPI_FUNCTION(dmic2_clk),
+ LPI_FUNCTION(dmic2_data),
+ LPI_FUNCTION(dmic3_clk),
+ LPI_FUNCTION(dmic3_data),
+ LPI_FUNCTION(dmic4_clk),
+ LPI_FUNCTION(dmic4_data),
+ LPI_FUNCTION(i2s1_clk),
+ LPI_FUNCTION(i2s1_data),
+ LPI_FUNCTION(i2s1_ws),
+ LPI_FUNCTION(i2s2_clk),
+ LPI_FUNCTION(i2s2_data),
+ LPI_FUNCTION(i2s2_ws),
+ LPI_FUNCTION(i2s3_clk),
+ LPI_FUNCTION(i2s3_data),
+ LPI_FUNCTION(i2s3_ws),
+ LPI_FUNCTION(qua_mi2s_data),
+ LPI_FUNCTION(qua_mi2s_sclk),
+ LPI_FUNCTION(qua_mi2s_ws),
+ LPI_FUNCTION(swr_rx_clk),
+ LPI_FUNCTION(swr_rx_data),
+ LPI_FUNCTION(swr_tx_clk),
+ LPI_FUNCTION(swr_tx_data),
+ LPI_FUNCTION(wsa_swr_clk),
+ LPI_FUNCTION(wsa_swr_data),
+ LPI_FUNCTION(wsa2_swr_clk),
+ LPI_FUNCTION(wsa2_swr_data),
+ LPI_FUNCTION(ext_mclk1_a),
+ LPI_FUNCTION(ext_mclk1_b),
+ LPI_FUNCTION(ext_mclk1_c),
+};
+
+static const struct lpi_pinctrl_variant_data sc8280xp_lpi_data = {
+ .pins = sc8280xp_lpi_pins,
+ .npins = ARRAY_SIZE(sc8280xp_lpi_pins),
+ .groups = sc8280xp_groups,
+ .ngroups = ARRAY_SIZE(sc8280xp_groups),
+ .functions = sc8280xp_functions,
+ .nfunctions = ARRAY_SIZE(sc8280xp_functions),
+};
+
+static const struct of_device_id lpi_pinctrl_of_match[] = {
+ {
+ .compatible = "qcom,sc8280xp-lpass-lpi-pinctrl",
+ .data = &sc8280xp_lpi_data,
+ },
+ { }
+};
+MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match);
+
+static struct platform_driver lpi_pinctrl_driver = {
+ .driver = {
+ .name = "qcom-sc8280xp-lpass-lpi-pinctrl",
+ .of_match_table = lpi_pinctrl_of_match,
+ },
+ .probe = lpi_pinctrl_probe,
+ .remove = lpi_pinctrl_remove,
+};
+
+module_platform_driver(lpi_pinctrl_driver);
+MODULE_DESCRIPTION("QTI SC8280XP LPI GPIO pin control driver");
+MODULE_LICENSE("GPL");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022 Linaro Ltd.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-lpass-lpi.h"
+
+enum lpass_lpi_functions {
+ LPI_MUX_dmic1_clk,
+ LPI_MUX_dmic1_data,
+ LPI_MUX_dmic2_clk,
+ LPI_MUX_dmic2_data,
+ LPI_MUX_dmic3_clk,
+ LPI_MUX_dmic3_data,
+ LPI_MUX_dmic4_clk,
+ LPI_MUX_dmic4_data,
+ LPI_MUX_i2s1_clk,
+ LPI_MUX_i2s1_data,
+ LPI_MUX_i2s1_ws,
+ LPI_MUX_i2s2_clk,
+ LPI_MUX_i2s2_data,
+ LPI_MUX_i2s2_ws,
+ LPI_MUX_i2s3_clk,
+ LPI_MUX_i2s3_data,
+ LPI_MUX_i2s3_ws,
+ LPI_MUX_i2s4_clk,
+ LPI_MUX_i2s4_data,
+ LPI_MUX_i2s4_ws,
+ LPI_MUX_qua_mi2s_data,
+ LPI_MUX_qua_mi2s_sclk,
+ LPI_MUX_qua_mi2s_ws,
+ LPI_MUX_swr_rx_clk,
+ LPI_MUX_swr_rx_data,
+ LPI_MUX_swr_tx_clk,
+ LPI_MUX_swr_tx_data,
+ LPI_MUX_wsa_swr_clk,
+ LPI_MUX_wsa_swr_data,
+ LPI_MUX_wsa2_swr_clk,
+ LPI_MUX_wsa2_swr_data,
+ LPI_MUX_slimbus_clk,
+ LPI_MUX_slimbus_data,
+ LPI_MUX_ext_mclk1_a,
+ LPI_MUX_ext_mclk1_b,
+ LPI_MUX_ext_mclk1_c,
+ LPI_MUX_ext_mclk1_d,
+ LPI_MUX_ext_mclk1_e,
+ LPI_MUX_gpio,
+ LPI_MUX__,
+};
+
+static int gpio0_pins[] = { 0 };
+static int gpio1_pins[] = { 1 };
+static int gpio2_pins[] = { 2 };
+static int gpio3_pins[] = { 3 };
+static int gpio4_pins[] = { 4 };
+static int gpio5_pins[] = { 5 };
+static int gpio6_pins[] = { 6 };
+static int gpio7_pins[] = { 7 };
+static int gpio8_pins[] = { 8 };
+static int gpio9_pins[] = { 9 };
+static int gpio10_pins[] = { 10 };
+static int gpio11_pins[] = { 11 };
+static int gpio12_pins[] = { 12 };
+static int gpio13_pins[] = { 13 };
+static int gpio14_pins[] = { 14 };
+static int gpio15_pins[] = { 15 };
+static int gpio16_pins[] = { 16 };
+static int gpio17_pins[] = { 17 };
+static int gpio18_pins[] = { 18 };
+static int gpio19_pins[] = { 19 };
+static int gpio20_pins[] = { 20 };
+static int gpio21_pins[] = { 21 };
+static int gpio22_pins[] = { 22 };
+
+static const struct pinctrl_pin_desc sm8450_lpi_pins[] = {
+ PINCTRL_PIN(0, "gpio0"),
+ PINCTRL_PIN(1, "gpio1"),
+ PINCTRL_PIN(2, "gpio2"),
+ PINCTRL_PIN(3, "gpio3"),
+ PINCTRL_PIN(4, "gpio4"),
+ PINCTRL_PIN(5, "gpio5"),
+ PINCTRL_PIN(6, "gpio6"),
+ PINCTRL_PIN(7, "gpio7"),
+ PINCTRL_PIN(8, "gpio8"),
+ PINCTRL_PIN(9, "gpio9"),
+ PINCTRL_PIN(10, "gpio10"),
+ PINCTRL_PIN(11, "gpio11"),
+ PINCTRL_PIN(12, "gpio12"),
+ PINCTRL_PIN(13, "gpio13"),
+ PINCTRL_PIN(14, "gpio14"),
+ PINCTRL_PIN(15, "gpio15"),
+ PINCTRL_PIN(16, "gpio16"),
+ PINCTRL_PIN(17, "gpio17"),
+ PINCTRL_PIN(18, "gpio18"),
+ PINCTRL_PIN(19, "gpio19"),
+ PINCTRL_PIN(20, "gpio20"),
+ PINCTRL_PIN(21, "gpio21"),
+ PINCTRL_PIN(22, "gpio22"),
+};
+
+static const char * const swr_tx_clk_groups[] = { "gpio0" };
+static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" };
+static const char * const swr_rx_clk_groups[] = { "gpio3" };
+static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5", "gpio15" };
+static const char * const dmic1_clk_groups[] = { "gpio6" };
+static const char * const dmic1_data_groups[] = { "gpio7" };
+static const char * const dmic2_clk_groups[] = { "gpio8" };
+static const char * const dmic2_data_groups[] = { "gpio9" };
+static const char * const dmic4_clk_groups[] = { "gpio17" };
+static const char * const dmic4_data_groups[] = { "gpio18" };
+static const char * const i2s2_clk_groups[] = { "gpio10" };
+static const char * const i2s2_ws_groups[] = { "gpio11" };
+static const char * const dmic3_clk_groups[] = { "gpio12" };
+static const char * const dmic3_data_groups[] = { "gpio13" };
+static const char * const qua_mi2s_sclk_groups[] = { "gpio0" };
+static const char * const qua_mi2s_ws_groups[] = { "gpio1" };
+static const char * const qua_mi2s_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" };
+static const char * const i2s1_clk_groups[] = { "gpio6" };
+static const char * const i2s1_ws_groups[] = { "gpio7" };
+static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" };
+static const char * const wsa_swr_clk_groups[] = { "gpio10" };
+static const char * const wsa_swr_data_groups[] = { "gpio11" };
+static const char * const wsa2_swr_clk_groups[] = { "gpio15" };
+static const char * const wsa2_swr_data_groups[] = { "gpio16" };
+static const char * const i2s2_data_groups[] = { "gpio15", "gpio16" };
+static const char * const i2s4_ws_groups[] = { "gpio13" };
+static const char * const i2s4_clk_groups[] = { "gpio12" };
+static const char * const i2s4_data_groups[] = { "gpio17", "gpio18" };
+static const char * const slimbus_clk_groups[] = { "gpio19"};
+static const char * const i2s3_clk_groups[] = { "gpio19"};
+static const char * const i2s3_ws_groups[] = { "gpio20"};
+static const char * const i2s3_data_groups[] = { "gpio21", "gpio22"};
+static const char * const slimbus_data_groups[] = { "gpio20"};
+static const char * const ext_mclk1_c_groups[] = { "gpio5" };
+static const char * const ext_mclk1_b_groups[] = { "gpio9" };
+static const char * const ext_mclk1_a_groups[] = { "gpio13" };
+static const char * const ext_mclk1_d_groups[] = { "gpio14" };
+static const char * const ext_mclk1_e_groups[] = { "gpio22" };
+
+static const struct lpi_pingroup sm8450_groups[] = {
+ LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _),
+ LPI_PINGROUP(1, 2, swr_tx_data, qua_mi2s_ws, _, _),
+ LPI_PINGROUP(2, 4, swr_tx_data, qua_mi2s_data, _, _),
+ LPI_PINGROUP(3, 8, swr_rx_clk, qua_mi2s_data, _, _),
+ LPI_PINGROUP(4, 10, swr_rx_data, qua_mi2s_data, _, _),
+ LPI_PINGROUP(5, 12, swr_rx_data, ext_mclk1_c, qua_mi2s_data, _),
+ LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _),
+ LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _),
+ LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _),
+ LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _),
+ LPI_PINGROUP(10, 16, i2s2_clk, wsa_swr_clk, _, _),
+ LPI_PINGROUP(11, 18, i2s2_ws, wsa_swr_data, _, _),
+ LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s4_clk, _, _),
+ LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s4_ws, ext_mclk1_a, _),
+ LPI_PINGROUP(14, 6, swr_tx_data, ext_mclk1_d, _, _),
+ LPI_PINGROUP(15, 20, i2s2_data, wsa2_swr_clk, _, _),
+ LPI_PINGROUP(16, 22, i2s2_data, wsa2_swr_data, _, _),
+ LPI_PINGROUP(17, LPI_NO_SLEW, dmic4_clk, i2s4_data, _, _),
+ LPI_PINGROUP(18, LPI_NO_SLEW, dmic4_data, i2s4_data, _, _),
+ LPI_PINGROUP(19, LPI_NO_SLEW, i2s3_clk, slimbus_clk, _, _),
+ LPI_PINGROUP(20, LPI_NO_SLEW, i2s3_ws, slimbus_data, _, _),
+ LPI_PINGROUP(21, LPI_NO_SLEW, i2s3_data, _, _, _),
+ LPI_PINGROUP(22, LPI_NO_SLEW, i2s3_data, ext_mclk1_e, _, _),
+};
+
+static const struct lpi_function sm8450_functions[] = {
+ LPI_FUNCTION(dmic1_clk),
+ LPI_FUNCTION(dmic1_data),
+ LPI_FUNCTION(dmic2_clk),
+ LPI_FUNCTION(dmic2_data),
+ LPI_FUNCTION(dmic3_clk),
+ LPI_FUNCTION(dmic3_data),
+ LPI_FUNCTION(dmic4_clk),
+ LPI_FUNCTION(dmic4_data),
+ LPI_FUNCTION(i2s1_clk),
+ LPI_FUNCTION(i2s1_data),
+ LPI_FUNCTION(i2s1_ws),
+ LPI_FUNCTION(i2s2_clk),
+ LPI_FUNCTION(i2s2_data),
+ LPI_FUNCTION(i2s2_ws),
+ LPI_FUNCTION(i2s3_clk),
+ LPI_FUNCTION(i2s3_data),
+ LPI_FUNCTION(i2s3_ws),
+ LPI_FUNCTION(i2s4_clk),
+ LPI_FUNCTION(i2s4_data),
+ LPI_FUNCTION(i2s4_ws),
+ LPI_FUNCTION(qua_mi2s_data),
+ LPI_FUNCTION(qua_mi2s_sclk),
+ LPI_FUNCTION(qua_mi2s_ws),
+ LPI_FUNCTION(swr_rx_clk),
+ LPI_FUNCTION(swr_rx_data),
+ LPI_FUNCTION(swr_tx_clk),
+ LPI_FUNCTION(swr_tx_data),
+ LPI_FUNCTION(slimbus_clk),
+ LPI_FUNCTION(slimbus_data),
+ LPI_FUNCTION(wsa_swr_clk),
+ LPI_FUNCTION(wsa_swr_data),
+ LPI_FUNCTION(wsa2_swr_clk),
+ LPI_FUNCTION(wsa2_swr_data),
+ LPI_FUNCTION(ext_mclk1_a),
+ LPI_FUNCTION(ext_mclk1_b),
+ LPI_FUNCTION(ext_mclk1_c),
+ LPI_FUNCTION(ext_mclk1_d),
+ LPI_FUNCTION(ext_mclk1_e),
+};
+
+static const struct lpi_pinctrl_variant_data sm8450_lpi_data = {
+ .pins = sm8450_lpi_pins,
+ .npins = ARRAY_SIZE(sm8450_lpi_pins),
+ .groups = sm8450_groups,
+ .ngroups = ARRAY_SIZE(sm8450_groups),
+ .functions = sm8450_functions,
+ .nfunctions = ARRAY_SIZE(sm8450_functions),
+};
+
+static const struct of_device_id lpi_pinctrl_of_match[] = {
+ {
+ .compatible = "qcom,sm8450-lpass-lpi-pinctrl",
+ .data = &sm8450_lpi_data,
+ },
+ { }
+};
+MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match);
+
+static struct platform_driver lpi_pinctrl_driver = {
+ .driver = {
+ .name = "qcom-sm8450-lpass-lpi-pinctrl",
+ .of_match_table = lpi_pinctrl_of_match,
+ },
+ .probe = lpi_pinctrl_probe,
+ .remove = lpi_pinctrl_remove,
+};
+
+module_platform_driver(lpi_pinctrl_driver);
+MODULE_DESCRIPTION("QTI SM8450 LPI GPIO pin control driver");
+MODULE_LICENSE("GPL");
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2012-2014, 2016-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*/
#include <linux/gpio/driver.h>
#define PMIC_GPIO_SUBTYPE_GPIOC_8CH 0xd
#define PMIC_GPIO_SUBTYPE_GPIO_LV 0x10
#define PMIC_GPIO_SUBTYPE_GPIO_MV 0x11
+#define PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2 0x12
+#define PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3 0x13
#define PMIC_MPP_REG_RT_STS 0x10
#define PMIC_MPP_REG_RT_STS_VAL_MASK 0x1
#define PMIC_GPIO_OUT_BUF_OPEN_DRAIN_NMOS 1
#define PMIC_GPIO_OUT_BUF_OPEN_DRAIN_PMOS 2
+#define PMIC_GPIO_OUT_STRENGTH_LOW 1
+#define PMIC_GPIO_OUT_STRENGTH_HIGH 3
+
/* PMIC_GPIO_REG_EN_CTL */
#define PMIC_GPIO_REG_MASTER_EN_SHIFT 7
struct regmap *map;
struct pinctrl_dev *ctrl;
struct gpio_chip chip;
- struct irq_chip irq;
u8 usid;
u8 pid_base;
};
arg = pad->pullup;
break;
case PMIC_GPIO_CONF_STRENGTH:
- arg = pad->strength;
+ switch (pad->strength) {
+ case PMIC_GPIO_OUT_STRENGTH_HIGH:
+ arg = PMIC_GPIO_STRENGTH_HIGH;
+ break;
+ case PMIC_GPIO_OUT_STRENGTH_LOW:
+ arg = PMIC_GPIO_STRENGTH_LOW;
+ break;
+ default:
+ arg = pad->strength;
+ break;
+ }
break;
case PMIC_GPIO_CONF_ATEST:
arg = pad->atest;
case PMIC_GPIO_CONF_STRENGTH:
if (arg > PMIC_GPIO_STRENGTH_LOW)
return -EINVAL;
- pad->strength = arg;
+ switch (arg) {
+ case PMIC_GPIO_STRENGTH_HIGH:
+ pad->strength = PMIC_GPIO_OUT_STRENGTH_HIGH;
+ break;
+ case PMIC_GPIO_STRENGTH_LOW:
+ pad->strength = PMIC_GPIO_OUT_STRENGTH_LOW;
+ break;
+ default:
+ pad->strength = arg;
+ break;
+ }
break;
case PMIC_GPIO_CONF_ATEST:
if (!pad->lv_mv_type || arg > 4)
pad->have_buffer = true;
pad->lv_mv_type = true;
break;
+ case PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2:
+ pad->num_sources = 2;
+ pad->have_buffer = true;
+ pad->lv_mv_type = true;
+ break;
+ case PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3:
+ pad->num_sources = 3;
+ pad->have_buffer = true;
+ pad->lv_mv_type = true;
+ break;
default:
dev_err(state->dev, "unknown GPIO type 0x%x\n", subtype);
return -ENODEV;
return 0;
}
+static void pmic_gpio_irq_mask(struct irq_data *data)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
+
+ irq_chip_mask_parent(data);
+ gpiochip_disable_irq(gc, data->hwirq);
+}
+
+static void pmic_gpio_irq_unmask(struct irq_data *data)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
+
+ gpiochip_enable_irq(gc, data->hwirq);
+ irq_chip_unmask_parent(data);
+}
+
+static const struct irq_chip spmi_gpio_irq_chip = {
+ .name = "spmi-gpio",
+ .irq_ack = irq_chip_ack_parent,
+ .irq_mask = pmic_gpio_irq_mask,
+ .irq_unmask = pmic_gpio_irq_unmask,
+ .irq_set_type = irq_chip_set_type_parent,
+ .irq_set_wake = irq_chip_set_wake_parent,
+ .flags = IRQCHIP_IMMUTABLE | IRQCHIP_MASK_ON_SUSPEND,
+ GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
static int pmic_gpio_probe(struct platform_device *pdev)
{
struct irq_domain *parent_domain;
if (!parent_domain)
return -ENXIO;
- state->irq.name = "spmi-gpio",
- state->irq.irq_ack = irq_chip_ack_parent,
- state->irq.irq_mask = irq_chip_mask_parent,
- state->irq.irq_unmask = irq_chip_unmask_parent,
- state->irq.irq_set_type = irq_chip_set_type_parent,
- state->irq.irq_set_wake = irq_chip_set_wake_parent,
- state->irq.flags = IRQCHIP_MASK_ON_SUSPEND,
-
girq = &state->chip.irq;
- girq->chip = &state->irq;
+ gpio_irq_chip_set_chip(girq, &spmi_gpio_irq_chip);
girq->default_type = IRQ_TYPE_NONE;
girq->handler = handle_level_irq;
girq->fwnode = of_node_to_fwnode(state->dev->of_node);
{ .compatible = "qcom,pm6150-gpio", .data = (void *) 10 },
{ .compatible = "qcom,pm6150l-gpio", .data = (void *) 12 },
{ .compatible = "qcom,pm6350-gpio", .data = (void *) 9 },
+ { .compatible = "qcom,pm7250b-gpio", .data = (void *) 12 },
{ .compatible = "qcom,pm7325-gpio", .data = (void *) 10 },
{ .compatible = "qcom,pm8005-gpio", .data = (void *) 4 },
{ .compatible = "qcom,pm8008-gpio", .data = (void *) 2 },
if (ret)
goto err_put_banks;
- ret = samsung_gpiolib_register(pdev, drvdata);
- if (ret)
- goto err_unregister;
-
if (ctrl->eint_gpio_init)
ctrl->eint_gpio_init(drvdata);
if (ctrl->eint_wkup_init)
ctrl->eint_wkup_init(drvdata);
+ ret = samsung_gpiolib_register(pdev, drvdata);
+ if (ret)
+ goto err_unregister;
+
platform_set_drvdata(pdev, drvdata);
return 0;
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+
+config PINCTRL_STARFIVE_JH7100
+ tristate "Pinctrl and GPIO driver for the StarFive JH7100 SoC"
+ depends on SOC_STARFIVE || COMPILE_TEST
+ depends on OF
+ select GENERIC_PINCTRL_GROUPS
+ select GENERIC_PINMUX_FUNCTIONS
+ select GENERIC_PINCONF
+ select GPIOLIB
+ select GPIOLIB_IRQCHIP
+ select OF_GPIO
+ default SOC_STARFIVE
+ help
+ Say yes here to support pin control on the StarFive JH7100 SoC.
+ This also provides an interface to the GPIO pins not used by other
+ peripherals supporting inputs, outputs, configuring pull-up/pull-down
+ and interrupts on input changes.
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_PINCTRL_STARFIVE_JH7100) += pinctrl-starfive-jh7100.o
#include <linux/pinctrl/pinctrl.h>
#include <linux/pinctrl/pinmux.h>
-#include <dt-bindings/pinctrl/pinctrl-starfive.h>
+#include <dt-bindings/pinctrl/pinctrl-starfive-jh7100.h>
-#include "core.h"
-#include "pinctrl-utils.h"
-#include "pinmux.h"
-#include "pinconf.h"
+#include "../core.h"
+#include "../pinctrl-utils.h"
+#include "../pinmux.h"
+#include "../pinconf.h"
#define DRIVER_NAME "pinctrl-starfive"
bank->clk = of_clk_get_by_name(np, NULL);
if (IS_ERR(bank->clk)) {
- if (PTR_ERR(bank->clk) != -EPROBE_DEFER)
- dev_err(dev, "failed to get clk (%ld)\n", PTR_ERR(bank->clk));
fwnode_handle_put(child);
- return PTR_ERR(bank->clk);
+ return dev_err_probe(dev, PTR_ERR(bank->clk),
+ "failed to get clk\n");
}
i++;
}
int ret;
ret = platform_irq_count(pdev);
- if (ret < 0) {
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev, "Couldn't determine irq count: %pe\n",
- ERR_PTR(ret));
- return ret;
- }
+ if (ret < 0)
+ return dev_err_probe(&pdev->dev, ret,
+ "Couldn't determine irq count\n");
switch (ret) {
case 2:
source "drivers/platform/mips/Kconfig"
endif
+source "drivers/platform/loongarch/Kconfig"
+
source "drivers/platform/goldfish/Kconfig"
source "drivers/platform/chrome/Kconfig"
#
obj-$(CONFIG_X86) += x86/
+obj-$(CONFIG_LOONGARCH) += loongarch/
obj-$(CONFIG_MELLANOX_PLATFORM) += mellanox/
obj-$(CONFIG_MIPS) += mips/
obj-$(CONFIG_OLPC_EC) += olpc/
--- /dev/null
+#
+# LoongArch Platform Specific Drivers
+#
+
+menuconfig LOONGARCH_PLATFORM_DEVICES
+ bool "LoongArch Platform Specific Device Drivers"
+ default y
+ depends on LOONGARCH
+ help
+ Say Y here to get to see options for device drivers of various
+ LoongArch platforms, including vendor-specific laptop/desktop
+ extension and hardware monitor drivers. This option itself does
+ not add any kernel code.
+
+ If you say N, all options in this submenu will be skipped and disabled.
+
+if LOONGARCH_PLATFORM_DEVICES
+
+config LOONGSON_LAPTOP
+ tristate "Generic Loongson-3 Laptop Driver"
+ depends on ACPI
+ depends on BACKLIGHT_CLASS_DEVICE
+ depends on INPUT
+ depends on MACH_LOONGSON64
+ select ACPI_VIDEO
+ select INPUT_SPARSEKMAP
+ default y
+ help
+ ACPI-based Loongson-3 family laptops generic driver.
+
+endif # LOONGARCH_PLATFORM_DEVICES
--- /dev/null
+obj-$(CONFIG_LOONGSON_LAPTOP) += loongson-laptop.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generic Loongson processor based LAPTOP/ALL-IN-ONE driver
+ *
+ * Jianmin Lv <lvjianmin@loongson.cn>
+ * Huacai Chen <chenhuacai@loongson.cn>
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/acpi.h>
+#include <linux/backlight.h>
+#include <linux/device.h>
+#include <linux/input.h>
+#include <linux/input/sparse-keymap.h>
+#include <linux/platform_device.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <acpi/video.h>
+
+/* 1. Driver-wide structs and misc. variables */
+
+/* ACPI HIDs */
+#define LOONGSON_ACPI_EC_HID "PNP0C09"
+#define LOONGSON_ACPI_HKEY_HID "LOON0000"
+
+#define ACPI_LAPTOP_NAME "loongson-laptop"
+#define ACPI_LAPTOP_ACPI_EVENT_PREFIX "loongson"
+
+#define MAX_ACPI_ARGS 3
+#define GENERIC_HOTKEY_MAP_MAX 64
+
+#define GENERIC_EVENT_TYPE_OFF 12
+#define GENERIC_EVENT_TYPE_MASK 0xF000
+#define GENERIC_EVENT_CODE_MASK 0x0FFF
+
+struct generic_sub_driver {
+ u32 type;
+ char *name;
+ acpi_handle *handle;
+ struct acpi_device *device;
+ struct platform_driver *driver;
+ int (*init)(struct generic_sub_driver *sub_driver);
+ void (*notify)(struct generic_sub_driver *sub_driver, u32 event);
+ u8 acpi_notify_installed;
+};
+
+static u32 input_device_registered;
+static struct input_dev *generic_inputdev;
+
+static acpi_handle hotkey_handle;
+static struct key_entry hotkey_keycode_map[GENERIC_HOTKEY_MAP_MAX];
+
+int loongson_laptop_turn_on_backlight(void);
+int loongson_laptop_turn_off_backlight(void);
+static int loongson_laptop_backlight_update(struct backlight_device *bd);
+
+/* 2. ACPI Helpers and device model */
+
+static int acpi_evalf(acpi_handle handle, int *res, char *method, char *fmt, ...)
+{
+ char res_type;
+ char *fmt0 = fmt;
+ va_list ap;
+ int success, quiet;
+ acpi_status status;
+ struct acpi_object_list params;
+ struct acpi_buffer result, *resultp;
+ union acpi_object in_objs[MAX_ACPI_ARGS], out_obj;
+
+ if (!*fmt) {
+ pr_err("acpi_evalf() called with empty format\n");
+ return 0;
+ }
+
+ if (*fmt == 'q') {
+ quiet = 1;
+ fmt++;
+ } else
+ quiet = 0;
+
+ res_type = *(fmt++);
+
+ params.count = 0;
+ params.pointer = &in_objs[0];
+
+ va_start(ap, fmt);
+ while (*fmt) {
+ char c = *(fmt++);
+ switch (c) {
+ case 'd': /* int */
+ in_objs[params.count].integer.value = va_arg(ap, int);
+ in_objs[params.count++].type = ACPI_TYPE_INTEGER;
+ break;
+ /* add more types as needed */
+ default:
+ pr_err("acpi_evalf() called with invalid format character '%c'\n", c);
+ va_end(ap);
+ return 0;
+ }
+ }
+ va_end(ap);
+
+ if (res_type != 'v') {
+ result.length = sizeof(out_obj);
+ result.pointer = &out_obj;
+ resultp = &result;
+ } else
+ resultp = NULL;
+
+ status = acpi_evaluate_object(handle, method, ¶ms, resultp);
+
+ switch (res_type) {
+ case 'd': /* int */
+ success = (status == AE_OK && out_obj.type == ACPI_TYPE_INTEGER);
+ if (success && res)
+ *res = out_obj.integer.value;
+ break;
+ case 'v': /* void */
+ success = status == AE_OK;
+ break;
+ /* add more types as needed */
+ default:
+ pr_err("acpi_evalf() called with invalid format character '%c'\n", res_type);
+ return 0;
+ }
+
+ if (!success && !quiet)
+ pr_err("acpi_evalf(%s, %s, ...) failed: %s\n",
+ method, fmt0, acpi_format_exception(status));
+
+ return success;
+}
+
+static int hotkey_status_get(int *status)
+{
+ if (!acpi_evalf(hotkey_handle, status, "GSWS", "d"))
+ return -EIO;
+
+ return 0;
+}
+
+static void dispatch_acpi_notify(acpi_handle handle, u32 event, void *data)
+{
+ struct generic_sub_driver *sub_driver = data;
+
+ if (!sub_driver || !sub_driver->notify)
+ return;
+ sub_driver->notify(sub_driver, event);
+}
+
+static int __init setup_acpi_notify(struct generic_sub_driver *sub_driver)
+{
+ acpi_status status;
+
+ if (!*sub_driver->handle)
+ return 0;
+
+ sub_driver->device = acpi_fetch_acpi_dev(*sub_driver->handle);
+ if (!sub_driver->device) {
+ pr_err("acpi_fetch_acpi_dev(%s) failed\n", sub_driver->name);
+ return -ENODEV;
+ }
+
+ sub_driver->device->driver_data = sub_driver;
+ sprintf(acpi_device_class(sub_driver->device), "%s/%s",
+ ACPI_LAPTOP_ACPI_EVENT_PREFIX, sub_driver->name);
+
+ status = acpi_install_notify_handler(*sub_driver->handle,
+ sub_driver->type, dispatch_acpi_notify, sub_driver);
+ if (ACPI_FAILURE(status)) {
+ if (status == AE_ALREADY_EXISTS) {
+ pr_notice("Another device driver is already "
+ "handling %s events\n", sub_driver->name);
+ } else {
+ pr_err("acpi_install_notify_handler(%s) failed: %s\n",
+ sub_driver->name, acpi_format_exception(status));
+ }
+ return -ENODEV;
+ }
+ sub_driver->acpi_notify_installed = 1;
+
+ return 0;
+}
+
+static int loongson_hotkey_suspend(struct device *dev)
+{
+ return 0;
+}
+
+static int loongson_hotkey_resume(struct device *dev)
+{
+ int status = 0;
+ struct key_entry ke;
+ struct backlight_device *bd;
+
+ /*
+ * Only if the firmware supports SW_LID event model, we can handle the
+ * event. This is for the consideration of development board without EC.
+ */
+ if (test_bit(SW_LID, generic_inputdev->swbit)) {
+ if (hotkey_status_get(&status) < 0)
+ return -EIO;
+ /*
+ * The input device sw element records the last lid status.
+ * When the system is awakened by other wake-up sources,
+ * the lid event will also be reported. The judgment of
+ * adding SW_LID bit which in sw element can avoid this
+ * case.
+ *
+ * Input system will drop lid event when current lid event
+ * value and last lid status in the same. So laptop driver
+ * doesn't report repeated events.
+ *
+ * Lid status is generally 0, but hardware exception is
+ * considered. So add lid status confirmation.
+ */
+ if (test_bit(SW_LID, generic_inputdev->sw) && !(status & (1 << SW_LID))) {
+ ke.type = KE_SW;
+ ke.sw.value = (u8)status;
+ ke.sw.code = SW_LID;
+ sparse_keymap_report_entry(generic_inputdev, &ke, 1, true);
+ }
+ }
+
+ bd = backlight_device_get_by_type(BACKLIGHT_PLATFORM);
+ if (bd) {
+ loongson_laptop_backlight_update(bd) ?
+ pr_warn("Loongson_backlight: resume brightness failed") :
+ pr_info("Loongson_backlight: resume brightness %d\n", bd->props.brightness);
+ }
+
+ return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(loongson_hotkey_pm,
+ loongson_hotkey_suspend, loongson_hotkey_resume);
+
+static int loongson_hotkey_probe(struct platform_device *pdev)
+{
+ hotkey_handle = ACPI_HANDLE(&pdev->dev);
+
+ if (!hotkey_handle)
+ return -ENODEV;
+
+ return 0;
+}
+
+static const struct acpi_device_id loongson_device_ids[] = {
+ {LOONGSON_ACPI_HKEY_HID, 0},
+ {"", 0},
+};
+MODULE_DEVICE_TABLE(acpi, loongson_device_ids);
+
+static struct platform_driver loongson_hotkey_driver = {
+ .probe = loongson_hotkey_probe,
+ .driver = {
+ .name = "loongson-hotkey",
+ .owner = THIS_MODULE,
+ .pm = pm_ptr(&loongson_hotkey_pm),
+ .acpi_match_table = loongson_device_ids,
+ },
+};
+
+static int hotkey_map(void)
+{
+ u32 index;
+ acpi_status status;
+ struct acpi_buffer buf;
+ union acpi_object *pack;
+
+ buf.length = ACPI_ALLOCATE_BUFFER;
+ status = acpi_evaluate_object_typed(hotkey_handle, "KMAP", NULL, &buf, ACPI_TYPE_PACKAGE);
+ if (status != AE_OK) {
+ pr_err("ACPI exception: %s\n", acpi_format_exception(status));
+ return -1;
+ }
+ pack = buf.pointer;
+ for (index = 0; index < pack->package.count; index++) {
+ union acpi_object *element, *sub_pack;
+
+ sub_pack = &pack->package.elements[index];
+
+ element = &sub_pack->package.elements[0];
+ hotkey_keycode_map[index].type = element->integer.value;
+ element = &sub_pack->package.elements[1];
+ hotkey_keycode_map[index].code = element->integer.value;
+ element = &sub_pack->package.elements[2];
+ hotkey_keycode_map[index].keycode = element->integer.value;
+ }
+
+ return 0;
+}
+
+static int hotkey_backlight_set(bool enable)
+{
+ if (!acpi_evalf(hotkey_handle, NULL, "VCBL", "vd", enable ? 1 : 0))
+ return -EIO;
+
+ return 0;
+}
+
+static int ec_get_brightness(void)
+{
+ int status = 0;
+
+ if (!hotkey_handle)
+ return -ENXIO;
+
+ if (!acpi_evalf(hotkey_handle, &status, "ECBG", "d"))
+ return -EIO;
+
+ return status;
+}
+
+static int ec_set_brightness(int level)
+{
+
+ int ret = 0;
+
+ if (!hotkey_handle)
+ return -ENXIO;
+
+ if (!acpi_evalf(hotkey_handle, NULL, "ECBS", "vd", level))
+ ret = -EIO;
+
+ return ret;
+}
+
+static int ec_backlight_level(u8 level)
+{
+ int status = 0;
+
+ if (!hotkey_handle)
+ return -ENXIO;
+
+ if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d"))
+ return -EIO;
+
+ if ((status < 0) || (level > status))
+ return status;
+
+ if (!acpi_evalf(hotkey_handle, &status, "ECSL", "d"))
+ return -EIO;
+
+ if ((status < 0) || (level < status))
+ return status;
+
+ return level;
+}
+
+static int loongson_laptop_backlight_update(struct backlight_device *bd)
+{
+ int lvl = ec_backlight_level(bd->props.brightness);
+
+ if (lvl < 0)
+ return -EIO;
+ if (ec_set_brightness(lvl))
+ return -EIO;
+
+ return 0;
+}
+
+static int loongson_laptop_get_brightness(struct backlight_device *bd)
+{
+ int level;
+
+ level = ec_get_brightness();
+ if (level < 0)
+ return -EIO;
+
+ return level;
+}
+
+static const struct backlight_ops backlight_laptop_ops = {
+ .update_status = loongson_laptop_backlight_update,
+ .get_brightness = loongson_laptop_get_brightness,
+};
+
+static int laptop_backlight_register(void)
+{
+ int status = 0;
+ struct backlight_properties props;
+
+ memset(&props, 0, sizeof(props));
+
+ if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d"))
+ return -EIO;
+
+ props.brightness = 1;
+ props.max_brightness = status;
+ props.type = BACKLIGHT_PLATFORM;
+
+ backlight_device_register("loongson_laptop",
+ NULL, NULL, &backlight_laptop_ops, &props);
+
+ return 0;
+}
+
+int loongson_laptop_turn_on_backlight(void)
+{
+ int status;
+ union acpi_object arg0 = { ACPI_TYPE_INTEGER };
+ struct acpi_object_list args = { 1, &arg0 };
+
+ arg0.integer.value = 1;
+ status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL);
+ if (ACPI_FAILURE(status)) {
+ pr_info("Loongson lvds error: 0x%x\n", status);
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
+int loongson_laptop_turn_off_backlight(void)
+{
+ int status;
+ union acpi_object arg0 = { ACPI_TYPE_INTEGER };
+ struct acpi_object_list args = { 1, &arg0 };
+
+ arg0.integer.value = 0;
+ status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL);
+ if (ACPI_FAILURE(status)) {
+ pr_info("Loongson lvds error: 0x%x\n", status);
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
+static int __init event_init(struct generic_sub_driver *sub_driver)
+{
+ int ret;
+
+ ret = hotkey_map();
+ if (ret < 0) {
+ pr_err("Failed to parse keymap from DSDT\n");
+ return ret;
+ }
+
+ ret = sparse_keymap_setup(generic_inputdev, hotkey_keycode_map, NULL);
+ if (ret < 0) {
+ pr_err("Failed to setup input device keymap\n");
+ input_free_device(generic_inputdev);
+
+ return ret;
+ }
+
+ /*
+ * This hotkey driver handle backlight event when
+ * acpi_video_get_backlight_type() gets acpi_backlight_vendor
+ */
+ if (acpi_video_get_backlight_type() == acpi_backlight_vendor)
+ hotkey_backlight_set(true);
+ else
+ hotkey_backlight_set(false);
+
+ pr_info("ACPI: enabling firmware HKEY event interface...\n");
+
+ return ret;
+}
+
+static void event_notify(struct generic_sub_driver *sub_driver, u32 event)
+{
+ int type, scan_code;
+ struct key_entry *ke = NULL;
+
+ scan_code = event & GENERIC_EVENT_CODE_MASK;
+ type = (event & GENERIC_EVENT_TYPE_MASK) >> GENERIC_EVENT_TYPE_OFF;
+ ke = sparse_keymap_entry_from_scancode(generic_inputdev, scan_code);
+ if (ke) {
+ if (type == KE_SW) {
+ int status = 0;
+
+ if (hotkey_status_get(&status) < 0)
+ return;
+
+ ke->sw.value = !!(status & (1 << ke->sw.code));
+ }
+ sparse_keymap_report_entry(generic_inputdev, ke, 1, true);
+ }
+}
+
+/* 3. Infrastructure */
+
+static void generic_subdriver_exit(struct generic_sub_driver *sub_driver);
+
+static int __init generic_subdriver_init(struct generic_sub_driver *sub_driver)
+{
+ int ret;
+
+ if (!sub_driver || !sub_driver->driver)
+ return -EINVAL;
+
+ ret = platform_driver_register(sub_driver->driver);
+ if (ret)
+ return -EINVAL;
+
+ if (sub_driver->init)
+ sub_driver->init(sub_driver);
+
+ if (sub_driver->notify) {
+ ret = setup_acpi_notify(sub_driver);
+ if (ret == -ENODEV) {
+ ret = 0;
+ goto err_out;
+ }
+ if (ret < 0)
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ generic_subdriver_exit(sub_driver);
+ return (ret < 0) ? ret : 0;
+}
+
+static void generic_subdriver_exit(struct generic_sub_driver *sub_driver)
+{
+
+ if (sub_driver->acpi_notify_installed) {
+ acpi_remove_notify_handler(*sub_driver->handle,
+ sub_driver->type, dispatch_acpi_notify);
+ sub_driver->acpi_notify_installed = 0;
+ }
+ platform_driver_unregister(sub_driver->driver);
+}
+
+static struct generic_sub_driver generic_sub_drivers[] __refdata = {
+ {
+ .name = "hotkey",
+ .init = event_init,
+ .notify = event_notify,
+ .handle = &hotkey_handle,
+ .type = ACPI_DEVICE_NOTIFY,
+ .driver = &loongson_hotkey_driver,
+ },
+};
+
+static int __init generic_acpi_laptop_init(void)
+{
+ bool ec_found;
+ int i, ret, status;
+
+ if (acpi_disabled)
+ return -ENODEV;
+
+ /* The EC device is required */
+ ec_found = acpi_dev_found(LOONGSON_ACPI_EC_HID);
+ if (!ec_found)
+ return -ENODEV;
+
+ /* Enable SCI for EC */
+ acpi_write_bit_register(ACPI_BITREG_SCI_ENABLE, 1);
+
+ generic_inputdev = input_allocate_device();
+ if (!generic_inputdev) {
+ pr_err("Unable to allocate input device\n");
+ return -ENOMEM;
+ }
+
+ /* Prepare input device, but don't register */
+ generic_inputdev->name =
+ "Loongson Generic Laptop/All-in-One Extra Buttons";
+ generic_inputdev->phys = ACPI_LAPTOP_NAME "/input0";
+ generic_inputdev->id.bustype = BUS_HOST;
+ generic_inputdev->dev.parent = NULL;
+
+ /* Init subdrivers */
+ for (i = 0; i < ARRAY_SIZE(generic_sub_drivers); i++) {
+ ret = generic_subdriver_init(&generic_sub_drivers[i]);
+ if (ret < 0) {
+ input_free_device(generic_inputdev);
+ while (--i >= 0)
+ generic_subdriver_exit(&generic_sub_drivers[i]);
+ return ret;
+ }
+ }
+
+ ret = input_register_device(generic_inputdev);
+ if (ret < 0) {
+ input_free_device(generic_inputdev);
+ while (--i >= 0)
+ generic_subdriver_exit(&generic_sub_drivers[i]);
+ pr_err("Unable to register input device\n");
+ return ret;
+ }
+
+ input_device_registered = 1;
+
+ if (acpi_evalf(hotkey_handle, &status, "ECBG", "d")) {
+ pr_info("Loongson Laptop used, init brightness is 0x%x\n", status);
+ ret = laptop_backlight_register();
+ if (ret < 0)
+ pr_err("Loongson Laptop: laptop-backlight device register failed\n");
+ }
+
+ return 0;
+}
+
+static void __exit generic_acpi_laptop_exit(void)
+{
+ if (generic_inputdev) {
+ if (input_device_registered)
+ input_unregister_device(generic_inputdev);
+ else
+ input_free_device(generic_inputdev);
+ }
+}
+
+module_init(generic_acpi_laptop_init);
+module_exit(generic_acpi_laptop_exit);
+
+MODULE_AUTHOR("Jianmin Lv <lvjianmin@loongson.cn>");
+MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
+MODULE_DESCRIPTION("Loongson Laptop/All-in-One ACPI Driver");
+MODULE_LICENSE("GPL");
if (!(gpe_sts_reg & GPE0A_PME_B0_STS_BIT))
return IRQ_NONE;
- generic_handle_irq(irq_find_mapping(chip->irq.domain,
- GPE0A_PME_B0_VIRT_GPIO_PIN));
+ generic_handle_domain_irq_safe(chip->irq.domain, GPE0A_PME_B0_VIRT_GPIO_PIN);
pm_wakeup_hard_event(chip->parent);
struct device *dev = &bp->dev;
sysfs_remove_link(&dev->kobj, "ttyGNSS");
+ sysfs_remove_link(&dev->kobj, "ttyGNSS2");
sysfs_remove_link(&dev->kobj, "ttyMAC");
sysfs_remove_link(&dev->kobj, "ptp");
sysfs_remove_link(&dev->kobj, "pps");
config RTC_DRV_ISL12022
tristate "Intersil ISL12022"
+ select REGMAP_I2C
help
If you say yes here you get support for the
Intersil ISL12022 RTC chip.
static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
{
- cmos_wake_setup(&pnp->dev);
+ int irq, ret;
if (pnp_port_start(pnp, 0) == 0x70 && !pnp_irq_valid(pnp, 0)) {
- unsigned int irq = 0;
+ irq = 0;
#ifdef CONFIG_X86
/* Some machines contain a PNP entry for the RTC, but
* don't define the IRQ. It should always be safe to
if (nr_legacy_irqs())
irq = RTC_IRQ;
#endif
- return cmos_do_probe(&pnp->dev,
- pnp_get_resource(pnp, IORESOURCE_IO, 0), irq);
} else {
- return cmos_do_probe(&pnp->dev,
- pnp_get_resource(pnp, IORESOURCE_IO, 0),
- pnp_irq(pnp, 0));
+ irq = pnp_irq(pnp, 0);
}
+
+ ret = cmos_do_probe(&pnp->dev, pnp_get_resource(pnp, IORESOURCE_IO, 0), irq);
+ if (ret)
+ return ret;
+
+ cmos_wake_setup(&pnp->dev);
+
+ return 0;
}
static void cmos_pnp_remove(struct pnp_dev *pnp)
static int __init cmos_platform_probe(struct platform_device *pdev)
{
struct resource *resource;
- int irq;
+ int irq, ret;
cmos_of_init(pdev);
- cmos_wake_setup(&pdev->dev);
if (RTC_IOMAPPED)
resource = platform_get_resource(pdev, IORESOURCE_IO, 0);
if (irq < 0)
irq = -1;
- return cmos_do_probe(&pdev->dev, resource, irq);
+ ret = cmos_do_probe(&pdev->dev, resource, irq);
+ if (ret)
+ return ret;
+
+ cmos_wake_setup(&pdev->dev);
+
+ return 0;
}
static int cmos_platform_remove(struct platform_device *pdev)
}
/**
- * s1685_rtc_check_mday - check validity of the day of month.
+ * ds1685_rtc_check_mday - check validity of the day of month.
* @rtc: pointer to the ds1685 rtc structure.
* @mday: day of month.
*
* SRAM address as on previous consoles.
*/
ret = regmap_read(d->regmap, RTC_SRAM_BIAS, &d->rtc_bias);
- if (ret) {
- pr_err("failed to get the RTC bias\n");
- iounmap(hw_srnprot);
- return -1;
- }
/* Reset SRAM access to how it was before, our job here is done. */
if (old != 0x7bf)
iowrite32be(old, hw_srnprot);
+
iounmap(hw_srnprot);
- return 0;
+ if (ret)
+ pr_err("failed to get the RTC bias\n");
+
+ return ret;
}
static const struct regmap_range rtc_rd_ranges[] = {
#include <linux/err.h>
#include <linux/of.h>
#include <linux/of_device.h>
+#include <linux/regmap.h>
/* ISL register offsets */
#define ISL12022_REG_SC 0x00
struct isl12022 {
struct rtc_device *rtc;
-
- bool write_enabled; /* true if write enable is set */
+ struct regmap *regmap;
};
-
-static int isl12022_read_regs(struct i2c_client *client, uint8_t reg,
- uint8_t *data, size_t n)
-{
- struct i2c_msg msgs[] = {
- {
- .addr = client->addr,
- .flags = 0,
- .len = 1,
- .buf = data
- }, /* setup read ptr */
- {
- .addr = client->addr,
- .flags = I2C_M_RD,
- .len = n,
- .buf = data
- }
- };
-
- int ret;
-
- data[0] = reg;
- ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
- if (ret != ARRAY_SIZE(msgs)) {
- dev_err(&client->dev, "%s: read error, ret=%d\n",
- __func__, ret);
- return -EIO;
- }
-
- return 0;
-}
-
-
-static int isl12022_write_reg(struct i2c_client *client,
- uint8_t reg, uint8_t val)
-{
- uint8_t data[2] = { reg, val };
- int err;
-
- err = i2c_master_send(client, data, sizeof(data));
- if (err != sizeof(data)) {
- dev_err(&client->dev,
- "%s: err=%d addr=%02x, data=%02x\n",
- __func__, err, data[0], data[1]);
- return -EIO;
- }
-
- return 0;
-}
-
-
/*
* In the routines that deal directly with the isl12022 hardware, we use
* rtc_time -- month 0-11, hour 0-23, yr = calendar year-epoch.
*/
static int isl12022_rtc_read_time(struct device *dev, struct rtc_time *tm)
{
- struct i2c_client *client = to_i2c_client(dev);
+ struct isl12022 *isl12022 = dev_get_drvdata(dev);
+ struct regmap *regmap = isl12022->regmap;
uint8_t buf[ISL12022_REG_INT + 1];
int ret;
- ret = isl12022_read_regs(client, ISL12022_REG_SC, buf, sizeof(buf));
+ ret = regmap_bulk_read(regmap, ISL12022_REG_SC, buf, sizeof(buf));
if (ret)
return ret;
if (buf[ISL12022_REG_SR] & (ISL12022_SR_LBAT85 | ISL12022_SR_LBAT75)) {
- dev_warn(&client->dev,
+ dev_warn(dev,
"voltage dropped below %u%%, "
"date and time is not reliable.\n",
buf[ISL12022_REG_SR] & ISL12022_SR_LBAT85 ? 85 : 75);
}
- dev_dbg(&client->dev,
+ dev_dbg(dev,
"%s: raw data is sec=%02x, min=%02x, hr=%02x, "
"mday=%02x, mon=%02x, year=%02x, wday=%02x, "
"sr=%02x, int=%02x",
tm->tm_mon = bcd2bin(buf[ISL12022_REG_MO] & 0x1F) - 1;
tm->tm_year = bcd2bin(buf[ISL12022_REG_YR]) + 100;
- dev_dbg(&client->dev, "%s: secs=%d, mins=%d, hours=%d, "
- "mday=%d, mon=%d, year=%d, wday=%d\n",
- __func__,
- tm->tm_sec, tm->tm_min, tm->tm_hour,
- tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday);
+ dev_dbg(dev, "%s: %ptR\n", __func__, tm);
return 0;
}
static int isl12022_rtc_set_time(struct device *dev, struct rtc_time *tm)
{
- struct i2c_client *client = to_i2c_client(dev);
- struct isl12022 *isl12022 = i2c_get_clientdata(client);
- size_t i;
+ struct isl12022 *isl12022 = dev_get_drvdata(dev);
+ struct regmap *regmap = isl12022->regmap;
int ret;
uint8_t buf[ISL12022_REG_DW + 1];
- dev_dbg(&client->dev, "%s: secs=%d, mins=%d, hours=%d, "
- "mday=%d, mon=%d, year=%d, wday=%d\n",
- __func__,
- tm->tm_sec, tm->tm_min, tm->tm_hour,
- tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday);
-
- if (!isl12022->write_enabled) {
-
- ret = isl12022_read_regs(client, ISL12022_REG_INT, buf, 1);
- if (ret)
- return ret;
-
- /* Check if WRTC (write rtc enable) is set factory default is
- * 0 (not set) */
- if (!(buf[0] & ISL12022_INT_WRTC)) {
- dev_info(&client->dev,
- "init write enable and 24 hour format\n");
-
- /* Set the write enable bit. */
- ret = isl12022_write_reg(client,
- ISL12022_REG_INT,
- buf[0] | ISL12022_INT_WRTC);
- if (ret)
- return ret;
-
- /* Write to any RTC register to start RTC, we use the
- * HR register, setting the MIL bit to use the 24 hour
- * format. */
- ret = isl12022_read_regs(client, ISL12022_REG_HR,
- buf, 1);
- if (ret)
- return ret;
-
- ret = isl12022_write_reg(client,
- ISL12022_REG_HR,
- buf[0] | ISL12022_HR_MIL);
- if (ret)
- return ret;
- }
-
- isl12022->write_enabled = true;
- }
+ dev_dbg(dev, "%s: %ptR\n", __func__, tm);
+
+ /* Ensure the write enable bit is set. */
+ ret = regmap_update_bits(regmap, ISL12022_REG_INT,
+ ISL12022_INT_WRTC, ISL12022_INT_WRTC);
+ if (ret)
+ return ret;
/* hours, minutes and seconds */
buf[ISL12022_REG_SC] = bin2bcd(tm->tm_sec);
buf[ISL12022_REG_DW] = tm->tm_wday & 0x07;
- /* write register's data */
- for (i = 0; i < ARRAY_SIZE(buf); i++) {
- ret = isl12022_write_reg(client, ISL12022_REG_SC + i,
- buf[ISL12022_REG_SC + i]);
- if (ret)
- return -EIO;
- }
-
- return 0;
+ return regmap_bulk_write(isl12022->regmap, ISL12022_REG_SC,
+ buf, sizeof(buf));
}
static const struct rtc_class_ops isl12022_rtc_ops = {
.set_time = isl12022_rtc_set_time,
};
+static const struct regmap_config regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .use_single_write = true,
+};
+
static int isl12022_probe(struct i2c_client *client)
{
struct isl12022 *isl12022;
GFP_KERNEL);
if (!isl12022)
return -ENOMEM;
+ dev_set_drvdata(&client->dev, isl12022);
+
+ isl12022->regmap = devm_regmap_init_i2c(client, ®map_config);
+ if (IS_ERR(isl12022->regmap)) {
+ dev_err(&client->dev, "regmap allocation failed\n");
+ return PTR_ERR(isl12022->regmap);
+ }
+
+ isl12022->rtc = devm_rtc_allocate_device(&client->dev);
+ if (IS_ERR(isl12022->rtc))
+ return PTR_ERR(isl12022->rtc);
- i2c_set_clientdata(client, isl12022);
+ isl12022->rtc->ops = &isl12022_rtc_ops;
+ isl12022->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+ isl12022->rtc->range_max = RTC_TIMESTAMP_END_2099;
- isl12022->rtc = devm_rtc_device_register(&client->dev,
- isl12022_driver.driver.name,
- &isl12022_rtc_ops, THIS_MODULE);
- return PTR_ERR_OR_ZERO(isl12022->rtc);
+ return devm_rtc_register_device(isl12022->rtc);
}
#ifdef CONFIG_OF
kernel_halt();
}
-static void jz4740_rtc_clk_disable(void *data)
-{
- clk_disable_unprepare(data);
-}
-
static const struct of_device_id jz4740_rtc_of_match[] = {
{ .compatible = "ingenic,jz4740-rtc", .data = (void *)ID_JZ4740 },
{ .compatible = "ingenic,jz4760-rtc", .data = (void *)ID_JZ4760 },
if (IS_ERR(rtc->base))
return PTR_ERR(rtc->base);
- clk = devm_clk_get(dev, "rtc");
- if (IS_ERR(clk)) {
- dev_err(dev, "Failed to get RTC clock\n");
- return PTR_ERR(clk);
- }
-
- ret = clk_prepare_enable(clk);
- if (ret) {
- dev_err(dev, "Failed to enable clock\n");
- return ret;
- }
-
- ret = devm_add_action_or_reset(dev, jz4740_rtc_clk_disable, clk);
- if (ret) {
- dev_err(dev, "Failed to register devm action\n");
- return ret;
- }
+ clk = devm_clk_get_enabled(dev, "rtc");
+ if (IS_ERR(clk))
+ return dev_err_probe(dev, PTR_ERR(clk), "Failed to get RTC clock\n");
spin_lock_init(&rtc->lock);
return 0;
}
-static inline struct clk *mpfs_rtc_init_clk(struct device *dev)
-{
- struct clk *clk;
- int ret;
-
- clk = devm_clk_get(dev, "rtc");
- if (IS_ERR(clk))
- return clk;
-
- ret = clk_prepare_enable(clk);
- if (ret)
- return ERR_PTR(ret);
-
- devm_add_action_or_reset(dev, (void (*) (void *))clk_disable_unprepare, clk);
- return clk;
-}
-
static irqreturn_t mpfs_rtc_wakeup_irq_handler(int irq, void *dev)
{
struct mpfs_rtc_dev *rtcdev = dev;
{
struct mpfs_rtc_dev *rtcdev;
struct clk *clk;
- u32 prescaler;
+ unsigned long prescaler;
int wakeup_irq, ret;
rtcdev = devm_kzalloc(&pdev->dev, sizeof(struct mpfs_rtc_dev), GFP_KERNEL);
/* range is capped by alarm max, lower reg is 31:0 & upper is 10:0 */
rtcdev->rtc->range_max = GENMASK_ULL(42, 0);
- clk = mpfs_rtc_init_clk(&pdev->dev);
+ clk = devm_clk_get_enabled(&pdev->dev, "rtc");
if (IS_ERR(clk))
return PTR_ERR(clk);
/* prescaler hardware adds 1 to reg value */
prescaler = clk_get_rate(devm_clk_get(&pdev->dev, "rtcref")) - 1;
-
if (prescaler > MAX_PRESCALER_COUNT) {
- dev_dbg(&pdev->dev, "invalid prescaler %d\n", prescaler);
+ dev_dbg(&pdev->dev, "invalid prescaler %lu\n", prescaler);
return -EINVAL;
}
writel(prescaler, rtcdev->base + PRESCALER_REG);
- dev_info(&pdev->dev, "prescaler set to: 0x%X \r\n", prescaler);
+ dev_info(&pdev->dev, "prescaler set to: %lu\n", prescaler);
device_init_wakeup(&pdev->dev, true);
ret = dev_pm_set_wake_irq(&pdev->dev, wakeup_irq);
.alarm_irq_enable = mxc_rtc_alarm_irq_enable,
};
-static void mxc_rtc_action(void *p)
-{
- struct rtc_plat_data *pdata = p;
-
- clk_disable_unprepare(pdata->clk_ref);
- clk_disable_unprepare(pdata->clk_ipg);
-}
-
static int mxc_rtc_probe(struct platform_device *pdev)
{
struct rtc_device *rtc;
rtc->range_max = (1 << 16) * 86400ULL - 1;
}
- pdata->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
+ pdata->clk_ipg = devm_clk_get_enabled(&pdev->dev, "ipg");
if (IS_ERR(pdata->clk_ipg)) {
dev_err(&pdev->dev, "unable to get ipg clock!\n");
return PTR_ERR(pdata->clk_ipg);
}
- ret = clk_prepare_enable(pdata->clk_ipg);
- if (ret)
- return ret;
-
- pdata->clk_ref = devm_clk_get(&pdev->dev, "ref");
+ pdata->clk_ref = devm_clk_get_enabled(&pdev->dev, "ref");
if (IS_ERR(pdata->clk_ref)) {
- clk_disable_unprepare(pdata->clk_ipg);
dev_err(&pdev->dev, "unable to get ref clock!\n");
return PTR_ERR(pdata->clk_ref);
}
- ret = clk_prepare_enable(pdata->clk_ref);
- if (ret) {
- clk_disable_unprepare(pdata->clk_ipg);
- return ret;
- }
-
- ret = devm_add_action_or_reset(&pdev->dev, mxc_rtc_action, pdata);
- if (ret)
- return ret;
-
rate = clk_get_rate(pdata->clk_ref);
if (rate == 32768)
{
struct rv3028_data *rv3028 = dev_get_drvdata(dev);
int ret;
+ u32 value;
switch(param->param) {
- u32 value;
-
case RTC_PARAM_BACKUP_SWITCH_MODE:
ret = regmap_read(rv3028->regmap, RV3028_BACKUP, &value);
if (ret < 0)
static int rv3028_param_set(struct device *dev, struct rtc_param *param)
{
struct rv3028_data *rv3028 = dev_get_drvdata(dev);
+ u8 mode;
switch(param->param) {
- u8 mode;
case RTC_PARAM_BACKUP_SWITCH_MODE:
switch (param->uvalue) {
case RTC_BSM_DISABLED:
wdt_pdev->dev.parent = &rtc_pdev->dev;
wdt_pdev->dev.platform_data = &wdt_pdata;
rc = platform_device_add(wdt_pdev);
+ if (rc)
+ platform_device_put(wdt_pdev);
}
if (rc)
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
+#include <linux/sys_soc.h>
#include <linux/property.h>
#include <linux/regmap.h>
#include <linux/rtc.h>
#define K3RTC_MIN_OFFSET (-277761)
#define K3RTC_MAX_OFFSET (277778)
-/**
- * struct ti_k3_rtc_soc_data - Private of compatible data for ti-k3-rtc
- * @unlock_irq_erratum: Has erratum for unlock infinite IRQs (erratum i2327)
- */
-struct ti_k3_rtc_soc_data {
- const bool unlock_irq_erratum;
-};
-
static const struct regmap_config ti_k3_rtc_regmap_config = {
.name = "peripheral-registers",
.reg_bits = 32,
* @rtc_dev: rtc device
* @regmap: rtc mmio regmap
* @r_fields: rtc register fields
- * @soc: SoC compatible match data
*/
struct ti_k3_rtc {
unsigned int irq;
struct rtc_device *rtc_dev;
struct regmap *regmap;
struct regmap_field *r_fields[K3_RTC_MAX_FIELDS];
- const struct ti_k3_rtc_soc_data *soc;
};
static int k3rtc_field_read(struct ti_k3_rtc *priv, enum ti_k3_rtc_fields f)
/* Skip fence since we are going to check the unlock bit as fence */
ret = regmap_field_read_poll_timeout(priv->r_fields[K3RTC_UNLOCK], ret,
- !ret, 2, priv->sync_timeout_us);
+ ret, 2, priv->sync_timeout_us);
return ret;
}
+/*
+ * This is the list of SoCs affected by TI's i2327 errata causing the RTC
+ * state-machine to break if not unlocked fast enough during boot. These
+ * SoCs must have the bootloader unlock this device very early in the
+ * boot-flow before we (Linux) can use this device.
+ */
+static const struct soc_device_attribute has_erratum_i2327[] = {
+ { .family = "AM62X", .revision = "SR1.0" },
+ { /* sentinel */ }
+};
+
static int k3rtc_configure(struct device *dev)
{
int ret;
*
* In such occurrence, it is assumed that the RTC module is unusable
*/
- if (priv->soc->unlock_irq_erratum) {
+ if (soc_device_match(has_erratum_i2327)) {
ret = k3rtc_check_unlocked(priv);
/* If there is an error OR if we are locked, return error */
if (ret) {
static int k3rtc_get_32kclk(struct device *dev, struct ti_k3_rtc *priv)
{
- int ret;
struct clk *clk;
- clk = devm_clk_get(dev, "osc32k");
+ clk = devm_clk_get_enabled(dev, "osc32k");
if (IS_ERR(clk))
return PTR_ERR(clk);
- ret = clk_prepare_enable(clk);
- if (ret)
- return ret;
-
- ret = devm_add_action_or_reset(dev, (void (*)(void *))clk_disable_unprepare, clk);
- if (ret)
- return ret;
-
priv->rate_32k = clk_get_rate(clk);
/* Make sure we are exact 32k clock. Else, try to compensate delay */
*/
priv->sync_timeout_us = (u32)(DIV_ROUND_UP_ULL(1000000, priv->rate_32k) * 4);
- return ret;
+ return 0;
}
static int k3rtc_get_vbusclk(struct device *dev, struct ti_k3_rtc *priv)
{
- int ret;
struct clk *clk;
/* Note: VBUS isn't a context clock, it is needed for hardware operation */
- clk = devm_clk_get(dev, "vbus");
+ clk = devm_clk_get_enabled(dev, "vbus");
if (IS_ERR(clk))
return PTR_ERR(clk);
- ret = clk_prepare_enable(clk);
- if (ret)
- return ret;
-
- return devm_add_action_or_reset(dev, (void (*)(void *))clk_disable_unprepare, clk);
+ return 0;
}
static int ti_k3_rtc_probe(struct platform_device *pdev)
if (IS_ERR(priv->rtc_dev))
return PTR_ERR(priv->rtc_dev);
- priv->soc = of_device_get_match_data(dev);
-
priv->rtc_dev->ops = &ti_k3_rtc_ops;
priv->rtc_dev->range_max = (1ULL << 48) - 1; /* 48Bit seconds */
ti_k3_rtc_nvmem_config.priv = priv;
return devm_rtc_nvmem_register(priv->rtc_dev, &ti_k3_rtc_nvmem_config);
}
-static const struct ti_k3_rtc_soc_data ti_k3_am62_data = {
- .unlock_irq_erratum = true,
-};
-
static const struct of_device_id ti_k3_rtc_of_match_table[] = {
- {.compatible = "ti,am62-rtc", .data = &ti_k3_am62_data},
+ {.compatible = "ti,am62-rtc" },
{}
};
MODULE_DEVICE_TABLE(of, ti_k3_rtc_of_match_table);
#include <linux/cdev.h>
#include <linux/slab.h>
#include <linux/module.h>
+#include <linux/kobject.h>
#include <linux/uaccess.h>
#include <asm/cio.h>
#include <asm/ccwdev.h>
#include <asm/debug.h>
#include <asm/diag.h>
+#include <asm/scsw.h>
#include "vmur.h"
static DEFINE_MUTEX(vmur_mutex);
+static void ur_uevent(struct work_struct *ws);
+
/*
* Allocation, freeing, getting and putting of urdev structures
*
ccw_device_get_id(cdev, &urd->dev_id);
mutex_init(&urd->io_mutex);
init_waitqueue_head(&urd->wait);
+ INIT_WORK(&urd->uevent_work, ur_uevent);
spin_lock_init(&urd->open_lock);
refcount_set(&urd->ref_count, 1);
urd->cdev = cdev;
return rc;
}
+static void ur_uevent(struct work_struct *ws)
+{
+ struct urdev *urd = container_of(ws, struct urdev, uevent_work);
+ char *envp[] = {
+ "EVENT=unsol_de", /* Unsolicited device-end interrupt */
+ NULL
+ };
+
+ kobject_uevent_env(&urd->cdev->dev.kobj, KOBJ_CHANGE, envp);
+ urdev_put(urd);
+}
+
/*
* ur interrupt handler, called from the ccw_device layer
*/
intparm, irb->scsw.cmd.cstat, irb->scsw.cmd.dstat,
irb->scsw.cmd.count);
}
+ urd = dev_get_drvdata(&cdev->dev);
if (!intparm) {
TRACE("ur_int_handler: unsolicited interrupt\n");
+
+ if (scsw_dstat(&irb->scsw) & DEV_STAT_DEV_END) {
+ /*
+ * Userspace might be interested in a transition to
+ * device-ready state.
+ */
+ urdev_get(urd);
+ schedule_work(&urd->uevent_work);
+ }
+
return;
}
- urd = dev_get_drvdata(&cdev->dev);
- BUG_ON(!urd);
/* On special conditions irb is an error pointer */
if (IS_ERR(irb))
urd->io_request_rc = PTR_ERR(irb);
rc = -ENOMEM;
goto fail_urdev_put;
}
- cdev->handler = ur_int_handler;
/* validate virtual unit record device */
urd->class = get_urd_class(urd);
}
spin_lock_irq(get_ccwdev_lock(cdev));
dev_set_drvdata(&cdev->dev, urd);
+ cdev->handler = ur_int_handler;
spin_unlock_irq(get_ccwdev_lock(cdev));
mutex_unlock(&vmur_mutex);
rc = -EBUSY;
goto fail_urdev_put;
}
+ if (cancel_work_sync(&urd->uevent_work)) {
+ /* Work not run yet - need to release reference here */
+ urdev_put(urd);
+ }
device_destroy(vmur_class, urd->char_device->dev);
cdev_del(urd->char_device);
urd->char_device = NULL;
spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
urdev_put(dev_get_drvdata(&cdev->dev));
dev_set_drvdata(&cdev->dev, NULL);
+ cdev->handler = NULL;
spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
mutex_unlock(&vmur_mutex);
#define _VMUR_H_
#include <linux/refcount.h>
+#include <linux/workqueue.h>
#define DEV_CLASS_UR_I 0x20 /* diag210 unit record input device class */
#define DEV_CLASS_UR_O 0x10 /* diag210 unit record output device class */
wait_queue_head_t wait; /* wait queue to serialize open */
int open_flag; /* "urdev is open" flag */
spinlock_t open_lock; /* serialize critical sections */
+ struct work_struct uevent_work; /* work to send uevent */
};
/*
#include <linux/module.h>
#include <linux/init.h>
-#include <linux/device.h>
#include <linux/slab.h>
#include <linux/mdev.h>
INIT_LIST_HEAD(&private->crw);
INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
- atomic_set(&private->avail, 1);
private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
GFP_KERNEL);
mutex_destroy(&private->io_mutex);
kfree(private);
}
-
static int vfio_ccw_sch_probe(struct subchannel *sch)
{
struct pmcw *pmcw = &sch->schib.pmcw;
dev_set_drvdata(&sch->dev, private);
- ret = mdev_register_device(&sch->dev, &vfio_ccw_mdev_driver);
+ private->mdev_type.sysfs_name = "io";
+ private->mdev_type.pretty_name = "I/O subchannel (Non-QDIO)";
+ private->mdev_types[0] = &private->mdev_type;
+ ret = mdev_register_parent(&private->parent, &sch->dev,
+ &vfio_ccw_mdev_driver,
+ private->mdev_types, 1);
if (ret)
goto out_free;
{
struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
- mdev_unregister_device(&sch->dev);
+ mdev_unregister_parent(&private->parent);
dev_set_drvdata(&sch->dev, NULL);
*/
#include <linux/vfio.h>
-#include <linux/mdev.h>
#include <linux/nospec.h>
#include <linux/slab.h>
vfio_ccw_mdev_reset(private);
}
-static ssize_t name_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "I/O subchannel (Non-QDIO)\n");
-}
-static MDEV_TYPE_ATTR_RO(name);
-
-static ssize_t device_api_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING);
-}
-static MDEV_TYPE_ATTR_RO(device_api);
-
-static ssize_t available_instances_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr,
- char *buf)
+static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev)
{
struct vfio_ccw_private *private =
- dev_get_drvdata(mtype_get_parent_dev(mtype));
+ container_of(vdev, struct vfio_ccw_private, vdev);
- return sprintf(buf, "%d\n", atomic_read(&private->avail));
+ init_completion(&private->release_comp);
+ return 0;
}
-static MDEV_TYPE_ATTR_RO(available_instances);
-
-static struct attribute *mdev_types_attrs[] = {
- &mdev_type_attr_name.attr,
- &mdev_type_attr_device_api.attr,
- &mdev_type_attr_available_instances.attr,
- NULL,
-};
-
-static struct attribute_group mdev_type_group = {
- .name = "io",
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group *mdev_type_groups[] = {
- &mdev_type_group,
- NULL,
-};
static int vfio_ccw_mdev_probe(struct mdev_device *mdev)
{
if (private->state == VFIO_CCW_STATE_NOT_OPER)
return -ENODEV;
- if (atomic_dec_if_positive(&private->avail) < 0)
- return -EPERM;
-
- memset(&private->vdev, 0, sizeof(private->vdev));
- vfio_init_group_dev(&private->vdev, &mdev->dev,
- &vfio_ccw_dev_ops);
+ ret = vfio_init_device(&private->vdev, &mdev->dev, &vfio_ccw_dev_ops);
+ if (ret)
+ return ret;
VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n",
private->sch->schid.cssid,
ret = vfio_register_emulated_iommu_dev(&private->vdev);
if (ret)
- goto err_atomic;
+ goto err_put_vdev;
dev_set_drvdata(&mdev->dev, private);
return 0;
-err_atomic:
- vfio_uninit_group_dev(&private->vdev);
- atomic_inc(&private->avail);
+err_put_vdev:
+ vfio_put_device(&private->vdev);
return ret;
}
+static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev)
+{
+ struct vfio_ccw_private *private =
+ container_of(vdev, struct vfio_ccw_private, vdev);
+
+ /*
+ * We cannot free vfio_ccw_private here because it includes
+ * parent info which must be free'ed by css driver.
+ *
+ * Use a workaround by memset'ing the core device part and
+ * then notifying the remove path that all active references
+ * to this device have been released.
+ */
+ memset(vdev, 0, sizeof(*vdev));
+ complete(&private->release_comp);
+}
+
static void vfio_ccw_mdev_remove(struct mdev_device *mdev)
{
struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent);
vfio_unregister_group_dev(&private->vdev);
- vfio_uninit_group_dev(&private->vdev);
- atomic_inc(&private->avail);
+ vfio_put_device(&private->vdev);
+ /*
+ * Wait for all active references on mdev are released so it
+ * is safe to defer kfree() to a later point.
+ *
+ * TODO: the clean fix is to split parent/mdev info from ccw
+ * private structure so each can be managed in its own life
+ * cycle.
+ */
+ wait_for_completion(&private->release_comp);
}
static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)
}
static const struct vfio_device_ops vfio_ccw_dev_ops = {
+ .init = vfio_ccw_mdev_init_dev,
+ .release = vfio_ccw_mdev_release_dev,
.open_device = vfio_ccw_mdev_open_device,
.close_device = vfio_ccw_mdev_close_device,
.read = vfio_ccw_mdev_read,
};
struct mdev_driver vfio_ccw_mdev_driver = {
+ .device_api = VFIO_DEVICE_API_CCW_STRING,
+ .max_instances = 1,
.driver = {
.name = "vfio_ccw_mdev",
.owner = THIS_MODULE,
},
.probe = vfio_ccw_mdev_probe,
.remove = vfio_ccw_mdev_remove,
- .supported_type_groups = mdev_type_groups,
};
#include <linux/workqueue.h>
#include <linux/vfio_ccw.h>
#include <linux/vfio.h>
+#include <linux/mdev.h>
#include <asm/crw.h>
#include <asm/debug.h>
* @sch: pointer to the subchannel
* @state: internal state of the device
* @completion: synchronization helper of the I/O completion
- * @avail: available for creating a mediated device
* @io_region: MMIO region to input/output I/O arguments/results
* @io_mutex: protect against concurrent update of I/O regions
* @region: additional regions for other subchannel operations
* @req_trigger: eventfd ctx for signaling userspace to return device
* @io_work: work for deferral process of I/O handling
* @crw_work: work for deferral process of CRW handling
+ * @release_comp: synchronization helper for vfio device release
+ * @parent: parent data structures for mdevs created
*/
struct vfio_ccw_private {
struct vfio_device vdev;
struct subchannel *sch;
int state;
struct completion *completion;
- atomic_t avail;
struct ccw_io_region *io_region;
struct mutex io_mutex;
struct vfio_ccw_region *region;
struct eventfd_ctx *req_trigger;
struct work_struct io_work;
struct work_struct crw_work;
+
+ struct completion release_comp;
+
+ struct mdev_parent parent;
+ struct mdev_type mdev_type;
+ struct mdev_type *mdev_types[1];
} __aligned(8);
int vfio_ccw_sch_quiesce(struct subchannel *sch);
AP_DOMAINS);
}
-static int vfio_ap_mdev_probe(struct mdev_device *mdev)
+static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
{
- struct ap_matrix_mdev *matrix_mdev;
- int ret;
-
- if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))
- return -EPERM;
-
- matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
- if (!matrix_mdev) {
- ret = -ENOMEM;
- goto err_dec_available;
- }
- vfio_init_group_dev(&matrix_mdev->vdev, &mdev->dev,
- &vfio_ap_matrix_dev_ops);
+ struct ap_matrix_mdev *matrix_mdev =
+ container_of(vdev, struct ap_matrix_mdev, vdev);
- matrix_mdev->mdev = mdev;
+ matrix_mdev->mdev = to_mdev_device(vdev->dev);
vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
matrix_mdev->pqap_hook = handle_pqap;
vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
hash_init(matrix_mdev->qtable.queues);
+ return 0;
+}
+
+static int vfio_ap_mdev_probe(struct mdev_device *mdev)
+{
+ struct ap_matrix_mdev *matrix_mdev;
+ int ret;
+
+ matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
+ &vfio_ap_matrix_dev_ops);
+ if (IS_ERR(matrix_mdev))
+ return PTR_ERR(matrix_mdev);
+
ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
if (ret)
- goto err_list;
+ goto err_put_vdev;
dev_set_drvdata(&mdev->dev, matrix_mdev);
mutex_lock(&matrix_dev->mdevs_lock);
list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
mutex_unlock(&matrix_dev->mdevs_lock);
return 0;
-err_list:
- vfio_uninit_group_dev(&matrix_mdev->vdev);
- kfree(matrix_mdev);
-err_dec_available:
- atomic_inc(&matrix_dev->available_instances);
+err_put_vdev:
+ vfio_put_device(&matrix_mdev->vdev);
return ret;
}
}
}
+static void vfio_ap_mdev_release_dev(struct vfio_device *vdev)
+{
+ vfio_free_device(vdev);
+}
+
static void vfio_ap_mdev_remove(struct mdev_device *mdev)
{
struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
list_del(&matrix_mdev->node);
mutex_unlock(&matrix_dev->mdevs_lock);
mutex_unlock(&matrix_dev->guests_lock);
- vfio_uninit_group_dev(&matrix_mdev->vdev);
- kfree(matrix_mdev);
- atomic_inc(&matrix_dev->available_instances);
+ vfio_put_device(&matrix_mdev->vdev);
}
-static ssize_t name_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT);
-}
-
-static MDEV_TYPE_ATTR_RO(name);
-
-static ssize_t available_instances_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "%d\n",
- atomic_read(&matrix_dev->available_instances));
-}
-
-static MDEV_TYPE_ATTR_RO(available_instances);
-
-static ssize_t device_api_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING);
-}
-
-static MDEV_TYPE_ATTR_RO(device_api);
-
-static struct attribute *vfio_ap_mdev_type_attrs[] = {
- &mdev_type_attr_name.attr,
- &mdev_type_attr_device_api.attr,
- &mdev_type_attr_available_instances.attr,
- NULL,
-};
-
-static struct attribute_group vfio_ap_mdev_hwvirt_type_group = {
- .name = VFIO_AP_MDEV_TYPE_HWVIRT,
- .attrs = vfio_ap_mdev_type_attrs,
-};
-
-static struct attribute_group *vfio_ap_mdev_type_groups[] = {
- &vfio_ap_mdev_hwvirt_type_group,
- NULL,
-};
-
#define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \
"already assigned to %s"
};
static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
+ .init = vfio_ap_mdev_init_dev,
+ .release = vfio_ap_mdev_release_dev,
.open_device = vfio_ap_mdev_open_device,
.close_device = vfio_ap_mdev_close_device,
.ioctl = vfio_ap_mdev_ioctl,
};
static struct mdev_driver vfio_ap_matrix_driver = {
+ .device_api = VFIO_DEVICE_API_AP_STRING,
+ .max_instances = MAX_ZDEV_ENTRIES_EXT,
.driver = {
.name = "vfio_ap_mdev",
.owner = THIS_MODULE,
},
.probe = vfio_ap_mdev_probe,
.remove = vfio_ap_mdev_remove,
- .supported_type_groups = vfio_ap_mdev_type_groups,
};
int vfio_ap_mdev_register(void)
{
int ret;
- atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
-
ret = mdev_register_driver(&vfio_ap_matrix_driver);
if (ret)
return ret;
- ret = mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_driver);
+ matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
+ matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
+ matrix_dev->mdev_types[0] = &matrix_dev->mdev_type;
+ ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
+ &vfio_ap_matrix_driver,
+ matrix_dev->mdev_types, 1);
if (ret)
goto err_driver;
return 0;
void vfio_ap_mdev_unregister(void)
{
- mdev_unregister_device(&matrix_dev->device);
+ mdev_unregister_parent(&matrix_dev->parent);
mdev_unregister_driver(&vfio_ap_matrix_driver);
}
#define _VFIO_AP_PRIVATE_H_
#include <linux/types.h>
-#include <linux/device.h>
#include <linux/mdev.h>
#include <linux/delay.h>
#include <linux/mutex.h>
* struct ap_matrix_dev - Contains the data for the matrix device.
*
* @device: generic device structure associated with the AP matrix device
- * @available_instances: number of mediated matrix devices that can be created
* @info: the struct containing the output from the PQAP(QCI) instruction
* @mdev_list: the list of mediated matrix devices created
* @mdevs_lock: mutex for locking the AP matrix device. This lock will be
*/
struct ap_matrix_dev {
struct device device;
- atomic_t available_instances;
struct ap_config_info info;
struct list_head mdev_list;
struct mutex mdevs_lock; /* serializes access to each ap_matrix_mdev */
struct ap_driver *vfio_ap_drv;
struct mutex guests_lock; /* serializes access to each KVM guest */
+ struct mdev_parent parent;
+ struct mdev_type mdev_type;
+ struct mdev_type *mdev_types[];
};
extern struct ap_matrix_dev *matrix_dev;
} else if (is_t5(lldi->adapter_type)) {
struct cpl_t5_act_open_req *req =
(struct cpl_t5_act_open_req *)skb->head;
- u32 isn = (prandom_u32() & ~7UL) - 1;
+ u32 isn = (get_random_u32() & ~7UL) - 1;
INIT_TP_WR(req, 0);
OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
} else {
struct cpl_t6_act_open_req *req =
(struct cpl_t6_act_open_req *)skb->head;
- u32 isn = (prandom_u32() & ~7UL) - 1;
+ u32 isn = (get_random_u32() & ~7UL) - 1;
INIT_TP_WR(req, 0);
OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
if (fip->probe_tries < FIP_VN_RLIM_COUNT) {
fip->probe_tries++;
- wait = prandom_u32() % FIP_VN_PROBE_WAIT;
+ wait = prandom_u32_max(FIP_VN_PROBE_WAIT);
} else
wait = FIP_VN_RLIM_INT;
mod_timer(&fip->timer, jiffies + msecs_to_jiffies(wait));
fcoe_all_vn2vn, 0);
fip->port_ka_time = jiffies +
msecs_to_jiffies(FIP_VN_BEACON_INT +
- (prandom_u32() % FIP_VN_BEACON_FUZZ));
+ prandom_u32_max(FIP_VN_BEACON_FUZZ));
}
if (time_before(fip->port_ka_time, next_time))
next_time = fip->port_ka_time;
* This function makes an running random selection decision on FCF record to
* use through a sequence of @fcf_cnt eligible FCF records with equal
* probability. To perform integer manunipulation of random numbers with
- * size unit32_t, the lower 16 bits of the 32-bit random number returned
- * from prandom_u32() are taken as the random random number generated.
+ * size unit32_t, a 16-bit random number returned from get_random_u16() is
+ * taken as the random random number generated.
*
* Returns true when outcome is for the newly read FCF record should be
* chosen; otherwise, return false when outcome is for keeping the previously
uint32_t rand_num;
/* Get 16-bit uniform random number */
- rand_num = 0xFFFF & prandom_u32();
+ rand_num = get_random_u16();
/* Decision with probability 1/fcf_cnt */
if ((fcf_cnt * rand_num) < 0xFFFF)
sizeof(struct qedi_endpoint *)), GFP_KERNEL);
if (!qedi->ep_tbl)
return -ENOMEM;
- port_id = prandom_u32() % QEDI_LOCAL_PORT_RANGE;
+ port_id = prandom_u32_max(QEDI_LOCAL_PORT_RANGE);
if (qedi_init_id_tbl(&qedi->lcl_port_tbl, QEDI_LOCAL_PORT_RANGE,
QEDI_LOCAL_PORT_MIN, port_id)) {
qedi_cm_free_mem(qedi);
#define VMSTOR_PROTO_VERSION_WIN8_1 VMSTOR_PROTO_VERSION(6, 0)
#define VMSTOR_PROTO_VERSION_WIN10 VMSTOR_PROTO_VERSION(6, 2)
+/* channel callback timeout in ms */
+#define CALLBACK_TIMEOUT 2
+
/* Packet structure describing virtual storage requests. */
enum vstor_packet_operation {
VSTOR_OPERATION_COMPLETE_IO = 1,
struct hv_device *device;
struct storvsc_device *stor_device;
struct Scsi_Host *shost;
+ unsigned long time_limit = jiffies + msecs_to_jiffies(CALLBACK_TIMEOUT);
if (channel->primary_channel != NULL)
device = channel->primary_channel->device_obj;
u32 minlen = rqst_id ? sizeof(struct vstor_packet) :
sizeof(enum vstor_packet_operation);
+ if (unlikely(time_after(jiffies, time_limit))) {
+ hv_pkt_iter_close(channel);
+ return;
+ }
+
if (pktlen < minlen) {
dev_err(&device->device,
"Invalid pkt: id=%llu, len=%u, minlen=%u\n",
err_out2:
/*
* Once we have connected with the host, we would need to
- * to invoke storvsc_dev_remove() to rollback this state and
+ * invoke storvsc_dev_remove() to rollback this state and
* this call also frees up the stor_device; hence the jump around
* err_out1 label.
*/
if SOC_SIFIVE
-config SIFIVE_L2
- bool "Sifive L2 Cache controller"
+config SIFIVE_CCACHE
+ bool "Sifive Composable Cache controller"
help
- Support for the L2 cache controller on SiFive platforms.
+ Support for the composable cache controller on SiFive platforms.
endif
# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_SIFIVE_L2) += sifive_l2_cache.o
+obj-$(CONFIG_SIFIVE_CCACHE) += sifive_ccache.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SiFive composable cache controller Driver
+ *
+ * Copyright (C) 2018-2022 SiFive, Inc.
+ *
+ */
+
+#define pr_fmt(fmt) "CCACHE: " fmt
+
+#include <linux/debugfs.h>
+#include <linux/interrupt.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/device.h>
+#include <linux/bitfield.h>
+#include <asm/cacheinfo.h>
+#include <soc/sifive/sifive_ccache.h>
+
+#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
+#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104
+#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108
+
+#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120
+#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124
+#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128
+
+#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140
+#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144
+#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148
+
+#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160
+#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164
+#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168
+
+#define SIFIVE_CCACHE_CONFIG 0x00
+#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0)
+#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8)
+#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)
+#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)
+
+#define SIFIVE_CCACHE_WAYENABLE 0x08
+#define SIFIVE_CCACHE_ECCINJECTERR 0x40
+
+#define SIFIVE_CCACHE_MAX_ECCINTR 4
+
+static void __iomem *ccache_base;
+static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
+static struct riscv_cacheinfo_ops ccache_cache_ops;
+static int level;
+
+enum {
+ DIR_CORR = 0,
+ DATA_CORR,
+ DATA_UNCORR,
+ DIR_UNCORR,
+};
+
+#ifdef CONFIG_DEBUG_FS
+static struct dentry *sifive_test;
+
+static ssize_t ccache_write(struct file *file, const char __user *data,
+ size_t count, loff_t *ppos)
+{
+ unsigned int val;
+
+ if (kstrtouint_from_user(data, count, 0, &val))
+ return -EINVAL;
+ if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
+ writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR);
+ else
+ return -EINVAL;
+ return count;
+}
+
+static const struct file_operations ccache_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .write = ccache_write
+};
+
+static void setup_sifive_debug(void)
+{
+ sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL);
+
+ debugfs_create_file("sifive_debug_inject_error", 0200,
+ sifive_test, NULL, &ccache_fops);
+}
+#endif
+
+static void ccache_config_read(void)
+{
+ u32 cfg;
+
+ cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG);
+ pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n",
+ FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg),
+ FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg),
+ BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)),
+ BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg)));
+
+ cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE);
+ pr_info("Index of the largest way enabled: %u\n", cfg);
+}
+
+static const struct of_device_id sifive_ccache_ids[] = {
+ { .compatible = "sifive,fu540-c000-ccache" },
+ { .compatible = "sifive,fu740-c000-ccache" },
+ { .compatible = "sifive,ccache0" },
+ { /* end of table */ }
+};
+
+static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);
+
+int register_sifive_ccache_error_notifier(struct notifier_block *nb)
+{
+ return atomic_notifier_chain_register(&ccache_err_chain, nb);
+}
+EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier);
+
+int unregister_sifive_ccache_error_notifier(struct notifier_block *nb)
+{
+ return atomic_notifier_chain_unregister(&ccache_err_chain, nb);
+}
+EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);
+
+static int ccache_largest_wayenabled(void)
+{
+ return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;
+}
+
+static ssize_t number_of_ways_enabled_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ccache_largest_wayenabled());
+}
+
+static DEVICE_ATTR_RO(number_of_ways_enabled);
+
+static struct attribute *priv_attrs[] = {
+ &dev_attr_number_of_ways_enabled.attr,
+ NULL,
+};
+
+static const struct attribute_group priv_attr_group = {
+ .attrs = priv_attrs,
+};
+
+static const struct attribute_group *ccache_get_priv_group(struct cacheinfo
+ *this_leaf)
+{
+ /* We want to use private group for composable cache only */
+ if (this_leaf->level == level)
+ return &priv_attr_group;
+ else
+ return NULL;
+}
+
+static irqreturn_t ccache_int_handler(int irq, void *device)
+{
+ unsigned int add_h, add_l;
+
+ if (irq == g_irq[DIR_CORR]) {
+ add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH);
+ add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW);
+ pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l);
+ /* Reading this register clears the DirError interrupt sig */
+ readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT);
+ atomic_notifier_call_chain(&ccache_err_chain,
+ SIFIVE_CCACHE_ERR_TYPE_CE,
+ "DirECCFix");
+ }
+ if (irq == g_irq[DIR_UNCORR]) {
+ add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH);
+ add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW);
+ /* Reading this register clears the DirFail interrupt sig */
+ readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT);
+ atomic_notifier_call_chain(&ccache_err_chain,
+ SIFIVE_CCACHE_ERR_TYPE_UE,
+ "DirECCFail");
+ panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);
+ }
+ if (irq == g_irq[DATA_CORR]) {
+ add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH);
+ add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW);
+ pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l);
+ /* Reading this register clears the DataError interrupt sig */
+ readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT);
+ atomic_notifier_call_chain(&ccache_err_chain,
+ SIFIVE_CCACHE_ERR_TYPE_CE,
+ "DatECCFix");
+ }
+ if (irq == g_irq[DATA_UNCORR]) {
+ add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH);
+ add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW);
+ pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l);
+ /* Reading this register clears the DataFail interrupt sig */
+ readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT);
+ atomic_notifier_call_chain(&ccache_err_chain,
+ SIFIVE_CCACHE_ERR_TYPE_UE,
+ "DatECCFail");
+ }
+
+ return IRQ_HANDLED;
+}
+
+static int __init sifive_ccache_init(void)
+{
+ struct device_node *np;
+ struct resource res;
+ int i, rc, intr_num;
+
+ np = of_find_matching_node(NULL, sifive_ccache_ids);
+ if (!np)
+ return -ENODEV;
+
+ if (of_address_to_resource(np, 0, &res))
+ return -ENODEV;
+
+ ccache_base = ioremap(res.start, resource_size(&res));
+ if (!ccache_base)
+ return -ENOMEM;
+
+ if (of_property_read_u32(np, "cache-level", &level))
+ return -ENOENT;
+
+ intr_num = of_property_count_u32_elems(np, "interrupts");
+ if (!intr_num) {
+ pr_err("No interrupts property\n");
+ return -ENODEV;
+ }
+
+ for (i = 0; i < intr_num; i++) {
+ g_irq[i] = irq_of_parse_and_map(np, i);
+ rc = request_irq(g_irq[i], ccache_int_handler, 0, "ccache_ecc",
+ NULL);
+ if (rc) {
+ pr_err("Could not request IRQ %d\n", g_irq[i]);
+ return rc;
+ }
+ }
+
+ ccache_config_read();
+
+ ccache_cache_ops.get_priv_group = ccache_get_priv_group;
+ riscv_set_cacheinfo_ops(&ccache_cache_ops);
+
+#ifdef CONFIG_DEBUG_FS
+ setup_sifive_debug();
+#endif
+ return 0;
+}
+
+device_initcall(sifive_ccache_init);
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * SiFive L2 cache controller Driver
- *
- * Copyright (C) 2018-2019 SiFive, Inc.
- *
- */
-#include <linux/debugfs.h>
-#include <linux/interrupt.h>
-#include <linux/of_irq.h>
-#include <linux/of_address.h>
-#include <linux/device.h>
-#include <asm/cacheinfo.h>
-#include <soc/sifive/sifive_l2_cache.h>
-
-#define SIFIVE_L2_DIRECCFIX_LOW 0x100
-#define SIFIVE_L2_DIRECCFIX_HIGH 0x104
-#define SIFIVE_L2_DIRECCFIX_COUNT 0x108
-
-#define SIFIVE_L2_DIRECCFAIL_LOW 0x120
-#define SIFIVE_L2_DIRECCFAIL_HIGH 0x124
-#define SIFIVE_L2_DIRECCFAIL_COUNT 0x128
-
-#define SIFIVE_L2_DATECCFIX_LOW 0x140
-#define SIFIVE_L2_DATECCFIX_HIGH 0x144
-#define SIFIVE_L2_DATECCFIX_COUNT 0x148
-
-#define SIFIVE_L2_DATECCFAIL_LOW 0x160
-#define SIFIVE_L2_DATECCFAIL_HIGH 0x164
-#define SIFIVE_L2_DATECCFAIL_COUNT 0x168
-
-#define SIFIVE_L2_CONFIG 0x00
-#define SIFIVE_L2_WAYENABLE 0x08
-#define SIFIVE_L2_ECCINJECTERR 0x40
-
-#define SIFIVE_L2_MAX_ECCINTR 4
-
-static void __iomem *l2_base;
-static int g_irq[SIFIVE_L2_MAX_ECCINTR];
-static struct riscv_cacheinfo_ops l2_cache_ops;
-
-enum {
- DIR_CORR = 0,
- DATA_CORR,
- DATA_UNCORR,
- DIR_UNCORR,
-};
-
-#ifdef CONFIG_DEBUG_FS
-static struct dentry *sifive_test;
-
-static ssize_t l2_write(struct file *file, const char __user *data,
- size_t count, loff_t *ppos)
-{
- unsigned int val;
-
- if (kstrtouint_from_user(data, count, 0, &val))
- return -EINVAL;
- if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
- writel(val, l2_base + SIFIVE_L2_ECCINJECTERR);
- else
- return -EINVAL;
- return count;
-}
-
-static const struct file_operations l2_fops = {
- .owner = THIS_MODULE,
- .open = simple_open,
- .write = l2_write
-};
-
-static void setup_sifive_debug(void)
-{
- sifive_test = debugfs_create_dir("sifive_l2_cache", NULL);
-
- debugfs_create_file("sifive_debug_inject_error", 0200,
- sifive_test, NULL, &l2_fops);
-}
-#endif
-
-static void l2_config_read(void)
-{
- u32 regval, val;
-
- regval = readl(l2_base + SIFIVE_L2_CONFIG);
- val = regval & 0xFF;
- pr_info("L2CACHE: No. of Banks in the cache: %d\n", val);
- val = (regval & 0xFF00) >> 8;
- pr_info("L2CACHE: No. of ways per bank: %d\n", val);
- val = (regval & 0xFF0000) >> 16;
- pr_info("L2CACHE: Sets per bank: %llu\n", (uint64_t)1 << val);
- val = (regval & 0xFF000000) >> 24;
- pr_info("L2CACHE: Bytes per cache block: %llu\n", (uint64_t)1 << val);
-
- regval = readl(l2_base + SIFIVE_L2_WAYENABLE);
- pr_info("L2CACHE: Index of the largest way enabled: %d\n", regval);
-}
-
-static const struct of_device_id sifive_l2_ids[] = {
- { .compatible = "sifive,fu540-c000-ccache" },
- { .compatible = "sifive,fu740-c000-ccache" },
- { /* end of table */ },
-};
-
-static ATOMIC_NOTIFIER_HEAD(l2_err_chain);
-
-int register_sifive_l2_error_notifier(struct notifier_block *nb)
-{
- return atomic_notifier_chain_register(&l2_err_chain, nb);
-}
-EXPORT_SYMBOL_GPL(register_sifive_l2_error_notifier);
-
-int unregister_sifive_l2_error_notifier(struct notifier_block *nb)
-{
- return atomic_notifier_chain_unregister(&l2_err_chain, nb);
-}
-EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
-
-static int l2_largest_wayenabled(void)
-{
- return readl(l2_base + SIFIVE_L2_WAYENABLE) & 0xFF;
-}
-
-static ssize_t number_of_ways_enabled_show(struct device *dev,
- struct device_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "%u\n", l2_largest_wayenabled());
-}
-
-static DEVICE_ATTR_RO(number_of_ways_enabled);
-
-static struct attribute *priv_attrs[] = {
- &dev_attr_number_of_ways_enabled.attr,
- NULL,
-};
-
-static const struct attribute_group priv_attr_group = {
- .attrs = priv_attrs,
-};
-
-static const struct attribute_group *l2_get_priv_group(struct cacheinfo *this_leaf)
-{
- /* We want to use private group for L2 cache only */
- if (this_leaf->level == 2)
- return &priv_attr_group;
- else
- return NULL;
-}
-
-static irqreturn_t l2_int_handler(int irq, void *device)
-{
- unsigned int add_h, add_l;
-
- if (irq == g_irq[DIR_CORR]) {
- add_h = readl(l2_base + SIFIVE_L2_DIRECCFIX_HIGH);
- add_l = readl(l2_base + SIFIVE_L2_DIRECCFIX_LOW);
- pr_err("L2CACHE: DirError @ 0x%08X.%08X\n", add_h, add_l);
- /* Reading this register clears the DirError interrupt sig */
- readl(l2_base + SIFIVE_L2_DIRECCFIX_COUNT);
- atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE,
- "DirECCFix");
- }
- if (irq == g_irq[DIR_UNCORR]) {
- add_h = readl(l2_base + SIFIVE_L2_DIRECCFAIL_HIGH);
- add_l = readl(l2_base + SIFIVE_L2_DIRECCFAIL_LOW);
- /* Reading this register clears the DirFail interrupt sig */
- readl(l2_base + SIFIVE_L2_DIRECCFAIL_COUNT);
- atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE,
- "DirECCFail");
- panic("L2CACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);
- }
- if (irq == g_irq[DATA_CORR]) {
- add_h = readl(l2_base + SIFIVE_L2_DATECCFIX_HIGH);
- add_l = readl(l2_base + SIFIVE_L2_DATECCFIX_LOW);
- pr_err("L2CACHE: DataError @ 0x%08X.%08X\n", add_h, add_l);
- /* Reading this register clears the DataError interrupt sig */
- readl(l2_base + SIFIVE_L2_DATECCFIX_COUNT);
- atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE,
- "DatECCFix");
- }
- if (irq == g_irq[DATA_UNCORR]) {
- add_h = readl(l2_base + SIFIVE_L2_DATECCFAIL_HIGH);
- add_l = readl(l2_base + SIFIVE_L2_DATECCFAIL_LOW);
- pr_err("L2CACHE: DataFail @ 0x%08X.%08X\n", add_h, add_l);
- /* Reading this register clears the DataFail interrupt sig */
- readl(l2_base + SIFIVE_L2_DATECCFAIL_COUNT);
- atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE,
- "DatECCFail");
- }
-
- return IRQ_HANDLED;
-}
-
-static int __init sifive_l2_init(void)
-{
- struct device_node *np;
- struct resource res;
- int i, rc, intr_num;
-
- np = of_find_matching_node(NULL, sifive_l2_ids);
- if (!np)
- return -ENODEV;
-
- if (of_address_to_resource(np, 0, &res))
- return -ENODEV;
-
- l2_base = ioremap(res.start, resource_size(&res));
- if (!l2_base)
- return -ENOMEM;
-
- intr_num = of_property_count_u32_elems(np, "interrupts");
- if (!intr_num) {
- pr_err("L2CACHE: no interrupts property\n");
- return -ENODEV;
- }
-
- for (i = 0; i < intr_num; i++) {
- g_irq[i] = irq_of_parse_and_map(np, i);
- rc = request_irq(g_irq[i], l2_int_handler, 0, "l2_ecc", NULL);
- if (rc) {
- pr_err("L2CACHE: Could not request IRQ %d\n", g_irq[i]);
- return rc;
- }
- }
-
- l2_config_read();
-
- l2_cache_ops.get_priv_group = l2_get_priv_group;
- riscv_set_cacheinfo_ops(&l2_cache_ops);
-
-#ifdef CONFIG_DEBUG_FS
- setup_sifive_debug();
-#endif
- return 0;
-}
-device_initcall(sifive_l2_init);
return IRQ_NONE;
for_each_set_bit(gpio, &irqs, bus->gpio.ngpio)
- generic_handle_irq(ssb_gpio_to_irq(&bus->gpio, gpio));
+ generic_handle_domain_irq_safe(bus->irq_domain, gpio);
+
ssb_chipco_gpio_polarity(chipco, irqs, val & irqs);
return IRQ_HANDLED;
return IRQ_NONE;
for_each_set_bit(gpio, &irqs, bus->gpio.ngpio)
- generic_handle_irq(ssb_gpio_to_irq(&bus->gpio, gpio));
+ generic_handle_domain_irq_safe(bus->irq_domain, gpio);
+
ssb_extif_gpio_polarity(extif, irqs, val & irqs);
return IRQ_HANDLED;
opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
opt2 |= T5_ISS_F;
- rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
+ rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);
opt2 |= T5_OPT_2_VALID_F;
#endif
}
-static int __check_mem_type(struct vm_area_struct *vma, unsigned long end)
+static int __check_mem_type(struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
- while (vma && is_normal_memory(vma->vm_page_prot)) {
- if (vma->vm_end >= end)
- return 0;
- vma = vma->vm_next;
+ struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, start);
+
+ for_each_vma_range(vmi, vma, end) {
+ if (!is_normal_memory(vma->vm_page_prot))
+ return -EINVAL;
}
- return -EINVAL;
+ return 0;
}
int optee_check_mem_type(unsigned long start, size_t num_pages)
return 0;
mmap_read_lock(mm);
- rc = __check_mem_type(find_vma(mm, start),
- start + num_pages * PAGE_SIZE);
+ rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE);
mmap_read_unlock(mm);
return rc;
obj-y += intel/
obj-$(CONFIG_TI_SOC_THERMAL) += ti-soc-thermal/
obj-y += st/
-obj-$(CONFIG_QCOM_TSENS) += qcom/
+obj-y += qcom/
obj-y += tegra/
obj-$(CONFIG_HISI_THERMAL) += hisi_thermal.o
obj-$(CONFIG_MTK_THERMAL) += mtk_thermal.o
static int imx_sc_thermal_probe(struct platform_device *pdev)
{
- struct device_node *np, *child, *sensor_np;
struct imx_sc_sensor *sensor;
- int ret;
+ const int *resource_id;
+ int i, ret;
ret = imx_scu_get_handle(&thermal_ipc_handle);
if (ret)
return ret;
- np = of_find_node_by_name(NULL, "thermal-zones");
- if (!np)
- return -ENODEV;
+ resource_id = of_device_get_match_data(&pdev->dev);
+ if (!resource_id)
+ return -EINVAL;
- sensor_np = of_node_get(pdev->dev.of_node);
+ for (i = 0; resource_id[i] > 0; i++) {
- for_each_available_child_of_node(np, child) {
sensor = devm_kzalloc(&pdev->dev, sizeof(*sensor), GFP_KERNEL);
- if (!sensor) {
- of_node_put(child);
- ret = -ENOMEM;
- goto put_node;
- }
+ if (!sensor)
+ return -ENOMEM;
- ret = thermal_zone_of_get_sensor_id(child,
- sensor_np,
- &sensor->resource_id);
- if (ret < 0) {
- dev_err(&pdev->dev,
- "failed to get valid sensor resource id: %d\n",
- ret);
- of_node_put(child);
- break;
- }
+ sensor->resource_id = resource_id[i];
- sensor->tzd = devm_thermal_of_zone_register(&pdev->dev,
- sensor->resource_id,
- sensor,
- &imx_sc_thermal_ops);
+ sensor->tzd = devm_thermal_of_zone_register(&pdev->dev, sensor->resource_id,
+ sensor, &imx_sc_thermal_ops);
if (IS_ERR(sensor->tzd)) {
- dev_err(&pdev->dev, "failed to register thermal zone\n");
+ /*
+ * Save the error value before freeing the
+ * sensor pointer, otherwise we endup with a
+ * use-after-free error
+ */
ret = PTR_ERR(sensor->tzd);
- of_node_put(child);
- break;
+
+ devm_kfree(&pdev->dev, sensor);
+
+ /*
+ * The thermal framework notifies us there is
+ * no thermal zone description for such a
+ * sensor id
+ */
+ if (ret == -ENODEV)
+ continue;
+
+ dev_err(&pdev->dev, "failed to register thermal zone\n");
+ return ret;
}
if (devm_thermal_add_hwmon_sysfs(sensor->tzd))
dev_warn(&pdev->dev, "failed to add hwmon sysfs attributes\n");
}
-put_node:
- of_node_put(sensor_np);
- of_node_put(np);
-
- return ret;
+ return 0;
}
static int imx_sc_thermal_remove(struct platform_device *pdev)
return 0;
}
+static int imx_sc_sensors[] = { IMX_SC_R_SYSTEM, IMX_SC_R_PMIC_0, -1 };
+
static const struct of_device_id imx_sc_thermal_table[] = {
- { .compatible = "fsl,imx-sc-thermal", },
+ { .compatible = "fsl,imx-sc-thermal", .data = imx_sc_sensors },
{}
};
MODULE_DEVICE_TABLE(of, imx_sc_thermal_table);
struct tsens_plat_data data_8939 = {
.num_sensors = 10,
.ops = &ops_8939,
- .hw_ids = (unsigned int []){ 0, 1, 2, 4, 5, 6, 7, 8, 9, 10 },
+ .hw_ids = (unsigned int []){ 0, 1, 2, 3, 5, 6, 7, 8, 9, 10 },
.feat = &tsens_v0_1_feat,
.fields = tsens_v0_1_regfields,
return 0;
}
-static struct thermal_zone_device_ops rcar_thermal_zone_of_ops = {
+static const struct thermal_zone_device_ops rcar_thermal_zone_of_ops = {
.get_temp = rcar_thermal_get_temp,
};
return ERR_PTR(-EINVAL);
}
- if (type && strlen(type) >= THERMAL_NAME_LENGTH) {
+ if (strlen(type) >= THERMAL_NAME_LENGTH) {
pr_err("Thermal zone name (%s) too long, should be under %d chars\n",
type, THERMAL_NAME_LENGTH);
return ERR_PTR(-EINVAL);
return -EINVAL;
}
-/**
- * thermal_zone_of_get_sensor_id - get sensor ID from a DT thermal zone
- * @tz_np: a valid thermal zone device node.
- * @sensor_np: a sensor node of a valid sensor device.
- * @id: the sensor ID returned if success.
- *
- * This function will get sensor ID from a given thermal zone node and
- * the sensor node must match the temperature provider @sensor_np.
- *
- * Return: 0 on success, proper error code otherwise.
- */
-
-int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
- struct device_node *sensor_np,
- u32 *id)
-{
- struct of_phandle_args sensor_specs;
- int ret;
-
- ret = of_parse_phandle_with_args(tz_np,
- "thermal-sensors",
- "#thermal-sensor-cells",
- 0,
- &sensor_specs);
- if (ret)
- return ret;
-
- if (sensor_specs.np != sensor_np) {
- of_node_put(sensor_specs.np);
- return -ENODEV;
- }
-
- if (sensor_specs.args_count > 1)
- pr_warn("%pOFn: too many cells in sensor specifier %d\n",
- sensor_specs.np, sensor_specs.args_count);
-
- *id = sensor_specs.args_count ? sensor_specs.args[0] : 0;
-
- of_node_put(sensor_specs.np);
-
- return 0;
-}
-EXPORT_SYMBOL_GPL(thermal_zone_of_get_sensor_id);
-
/*** functions parsing device tree nodes ***/
static int of_find_trip_id(struct device_node *np, struct device_node *trip)
if (kstrtoint(buf, 10, &temperature))
return -EINVAL;
- ret = tz->ops->set_trip_temp(tz, trip, temperature);
- if (ret)
- return ret;
+ if (tz->ops->set_trip_temp) {
+ ret = tz->ops->set_trip_temp(tz, trip, temperature);
+ if (ret)
+ return ret;
+ }
if (tz->trips)
tz->trips[trip].temperature = temperature;
tb_property_add_immediate(xdomain_property_dir, "deviceid", 0x1);
tb_property_add_immediate(xdomain_property_dir, "devicerv", 0x80000100);
- xdomain_property_block_gen = prandom_u32();
+ xdomain_property_block_gen = get_random_u32();
return 0;
}
select SERIAL_CORE_CONSOLE
default y
-config PDC_CONSOLE
- bool "PDC software console support"
- depends on PARISC && !SERIAL_MUX && VT
- help
- Saying Y here will enable the software based PDC console to be
- used as the system console. This is useful for machines in
- which the hardware based console has not been written yet. The
- following steps must be completed to use the PDC console:
-
- 1. create the device entry (mknod /dev/ttyB0 c 11 0)
- 2. Edit the /etc/inittab to start a getty listening on /dev/ttyB0
- 3. Add device ttyB0 to /etc/securetty (if you want to log on as
- root on this console.)
- 4. Change the kernel command console parameter to: console=ttyB0
-
config SERIAL_SUNSAB
tristate "Sun Siemens SAB82532 serial support"
depends on SPARC && PCI
#include <linux/bitops.h>
#include <linux/slab.h>
#include <linux/log2.h>
+#include <linux/kmsan.h>
#include <linux/usb.h>
#include <linux/wait.h>
#include <linux/usb/hcd.h>
URB_SETUP_MAP_SINGLE | URB_SETUP_MAP_LOCAL |
URB_DMA_SG_COMBINED);
urb->transfer_flags |= (is_out ? URB_DIR_OUT : URB_DIR_IN);
+ kmsan_handle_urb(urb, is_out);
if (xfertype != USB_ENDPOINT_XFER_CONTROL &&
dev->state < USB_STATE_CONFIGURED)
}
config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
}
+ if (nl_attrs[VDPA_ATTR_DEV_FEATURES]) {
+ config.device_features =
+ nla_get_u64(nl_attrs[VDPA_ATTR_DEV_FEATURES]);
+ config.mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
+ }
/* Skip checking capability if user didn't prefer to configure any
* device networking attributes. It is likely that user might have used
return msg->len;
}
-static int vdpa_dev_net_mq_config_fill(struct vdpa_device *vdev,
- struct sk_buff *msg, u64 features,
+static int vdpa_dev_net_mq_config_fill(struct sk_buff *msg, u64 features,
const struct virtio_net_config *config)
{
u16 val_u16;
- if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0)
+ if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0 &&
+ (features & BIT_ULL(VIRTIO_NET_F_RSS)) == 0)
return 0;
- val_u16 = le16_to_cpu(config->max_virtqueue_pairs);
+ val_u16 = __virtio16_to_cpu(true, config->max_virtqueue_pairs);
+
return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, val_u16);
}
+static int vdpa_dev_net_mtu_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_net_config *config)
+{
+ u16 val_u16;
+
+ if ((features & BIT_ULL(VIRTIO_NET_F_MTU)) == 0)
+ return 0;
+
+ val_u16 = __virtio16_to_cpu(true, config->mtu);
+
+ return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16);
+}
+
+static int vdpa_dev_net_mac_config_fill(struct sk_buff *msg, u64 features,
+ const struct virtio_net_config *config)
+{
+ if ((features & BIT_ULL(VIRTIO_NET_F_MAC)) == 0)
+ return 0;
+
+ return nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR,
+ sizeof(config->mac), config->mac);
+}
+
static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *msg)
{
struct virtio_net_config config = {};
- u64 features;
+ u64 features_device;
u16 val_u16;
- vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config));
-
- if (nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR, sizeof(config.mac),
- config.mac))
- return -EMSGSIZE;
+ vdev->config->get_config(vdev, 0, &config, sizeof(config));
val_u16 = __virtio16_to_cpu(true, config.status);
if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16))
return -EMSGSIZE;
- val_u16 = __virtio16_to_cpu(true, config.mtu);
- if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16))
- return -EMSGSIZE;
+ features_device = vdev->config->get_device_features(vdev);
- features = vdev->config->get_driver_features(vdev);
- if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features,
+ if (nla_put_u64_64bit(msg, VDPA_ATTR_VDPA_DEV_SUPPORTED_FEATURES, features_device,
VDPA_ATTR_PAD))
return -EMSGSIZE;
- return vdpa_dev_net_mq_config_fill(vdev, msg, features, &config);
+ if (vdpa_dev_net_mtu_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ if (vdpa_dev_net_mac_config_fill(msg, features_device, &config))
+ return -EMSGSIZE;
+
+ return vdpa_dev_net_mq_config_fill(msg, features_device, &config);
}
static int
vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq,
int flags, struct netlink_ext_ack *extack)
{
+ u64 features_driver;
+ u8 status = 0;
u32 device_id;
void *hdr;
int err;
goto msg_err;
}
+ /* only read driver features after the feature negotiation is done */
+ status = vdev->config->get_status(vdev);
+ if (status & VIRTIO_CONFIG_S_FEATURES_OK) {
+ features_driver = vdev->config->get_driver_features(vdev);
+ if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features_driver,
+ VDPA_ATTR_PAD)) {
+ err = -EMSGSIZE;
+ goto msg_err;
+ }
+ }
+
switch (device_id) {
case VIRTIO_ID_NET:
err = vdpa_dev_net_config_fill(vdev, msg);
#include <linux/vdpa.h>
#include <linux/vhost_iotlb.h>
#include <linux/iova.h>
+#include <uapi/linux/vdpa.h>
#include "vdpa_sim.h"
static const struct vdpa_config_ops vdpasim_config_ops;
static const struct vdpa_config_ops vdpasim_batch_config_ops;
-struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr)
+struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
+ const struct vdpa_dev_set_config *config)
{
const struct vdpa_config_ops *ops;
struct vdpasim *vdpasim;
struct device *dev;
int i, ret = -ENOMEM;
+ if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
+ if (config->device_features &
+ ~dev_attr->supported_features)
+ return ERR_PTR(-EINVAL);
+ dev_attr->supported_features =
+ config->device_features;
+ }
+
if (batch_mapping)
ops = &vdpasim_batch_config_ops;
else
spinlock_t iommu_lock;
};
-struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr);
+struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr,
+ const struct vdpa_dev_set_config *config);
/* TODO: cross-endian support */
static inline bool vdpasim_is_little_endian(struct vdpasim *vdpasim)
dev_attr.work_fn = vdpasim_blk_work;
dev_attr.buffer_size = VDPASIM_BLK_CAPACITY << SECTOR_SHIFT;
- simdev = vdpasim_create(&dev_attr);
+ simdev = vdpasim_create(&dev_attr, config);
if (IS_ERR(simdev))
return PTR_ERR(simdev);
dev_attr.work_fn = vdpasim_net_work;
dev_attr.buffer_size = PAGE_SIZE;
- simdev = vdpasim_create(&dev_attr);
+ simdev = vdpasim_create(&dev_attr, config);
if (IS_ERR(simdev))
return PTR_ERR(simdev);
.id_table = id_table,
.ops = &vdpasim_net_mgmtdev_ops,
.config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR |
- 1 << VDPA_ATTR_DEV_NET_CFG_MTU),
+ 1 << VDPA_ATTR_DEV_NET_CFG_MTU |
+ 1 << VDPA_ATTR_DEV_FEATURES),
.max_supported_vqs = VDPASIM_NET_VQ_NUM,
.supported_features = VDPASIM_NET_FEATURES,
};
#include <linux/virtio_ring.h>
#include <linux/virtio_pci.h>
#include <linux/virtio_pci_modern.h>
+#include <uapi/linux/vdpa.h>
#define VP_VDPA_QUEUE_MAX 256
#define VP_VDPA_DRIVER_NAME "vp_vdpa"
struct virtio_pci_modern_device *mdev;
struct vp_vring *vring;
struct vdpa_callback config_cb;
+ u64 device_features;
char msix_name[VP_VDPA_NAME_SIZE];
int config_irq;
int queues;
static u64 vp_vdpa_get_device_features(struct vdpa_device *vdpa)
{
- struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
+ struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa);
- return vp_modern_get_features(mdev);
+ return vp_vdpa->device_features;
}
static int vp_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
struct pci_dev *pdev = mdev->pci_dev;
struct device *dev = &pdev->dev;
struct vp_vdpa *vp_vdpa = NULL;
+ u64 device_features;
int ret, i;
vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa,
vp_vdpa->queues = vp_modern_get_num_queues(mdev);
vp_vdpa->mdev = mdev;
+ device_features = vp_modern_get_features(mdev);
+ if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
+ if (add_config->device_features & ~device_features) {
+ ret = -EINVAL;
+ dev_err(&pdev->dev, "Try to provision features "
+ "that are not supported by the device: "
+ "device_features 0x%llx provisioned 0x%llx\n",
+ device_features, add_config->device_features);
+ goto err;
+ }
+ device_features = add_config->device_features;
+ }
+ vp_vdpa->device_features = device_features;
+
ret = devm_add_action_or_reset(dev, vp_vdpa_free_irq_vectors, pdev);
if (ret) {
dev_err(&pdev->dev,
mgtdev->id_table = mdev_id;
mgtdev->max_supported_vqs = vp_modern_get_num_queues(mdev);
mgtdev->supported_features = vp_modern_get_features(mdev);
+ mgtdev->config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);
pci_set_master(pdev);
pci_set_drvdata(pdev, vp_vdpa_mgtdev);
tristate "VFIO Non-Privileged userspace driver framework"
select IOMMU_API
select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
+ select INTERVAL_TREE
help
VFIO provides a framework for secure userspace device drivers.
See Documentation/driver-api/vfio.rst for more details.
# SPDX-License-Identifier: GPL-2.0
vfio_virqfd-y := virqfd.o
-vfio-y += vfio_main.o
-
obj-$(CONFIG_VFIO) += vfio.o
+
+vfio-y += vfio_main.o \
+ iova_bitmap.o \
+ container.o
+
obj-$(CONFIG_VFIO_VIRQFD) += vfio_virqfd.o
obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
+ *
+ * VFIO container (/dev/vfio/vfio)
+ */
+#include <linux/file.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/capability.h>
+#include <linux/iommu.h>
+#include <linux/miscdevice.h>
+#include <linux/vfio.h>
+#include <uapi/linux/vfio.h>
+
+#include "vfio.h"
+
+struct vfio_container {
+ struct kref kref;
+ struct list_head group_list;
+ struct rw_semaphore group_lock;
+ struct vfio_iommu_driver *iommu_driver;
+ void *iommu_data;
+ bool noiommu;
+};
+
+static struct vfio {
+ struct list_head iommu_drivers_list;
+ struct mutex iommu_drivers_lock;
+} vfio;
+
+#ifdef CONFIG_VFIO_NOIOMMU
+bool vfio_noiommu __read_mostly;
+module_param_named(enable_unsafe_noiommu_mode,
+ vfio_noiommu, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)");
+#endif
+
+static void *vfio_noiommu_open(unsigned long arg)
+{
+ if (arg != VFIO_NOIOMMU_IOMMU)
+ return ERR_PTR(-EINVAL);
+ if (!capable(CAP_SYS_RAWIO))
+ return ERR_PTR(-EPERM);
+
+ return NULL;
+}
+
+static void vfio_noiommu_release(void *iommu_data)
+{
+}
+
+static long vfio_noiommu_ioctl(void *iommu_data,
+ unsigned int cmd, unsigned long arg)
+{
+ if (cmd == VFIO_CHECK_EXTENSION)
+ return vfio_noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;
+
+ return -ENOTTY;
+}
+
+static int vfio_noiommu_attach_group(void *iommu_data,
+ struct iommu_group *iommu_group, enum vfio_group_type type)
+{
+ return 0;
+}
+
+static void vfio_noiommu_detach_group(void *iommu_data,
+ struct iommu_group *iommu_group)
+{
+}
+
+static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {
+ .name = "vfio-noiommu",
+ .owner = THIS_MODULE,
+ .open = vfio_noiommu_open,
+ .release = vfio_noiommu_release,
+ .ioctl = vfio_noiommu_ioctl,
+ .attach_group = vfio_noiommu_attach_group,
+ .detach_group = vfio_noiommu_detach_group,
+};
+
+/*
+ * Only noiommu containers can use vfio-noiommu and noiommu containers can only
+ * use vfio-noiommu.
+ */
+static bool vfio_iommu_driver_allowed(struct vfio_container *container,
+ const struct vfio_iommu_driver *driver)
+{
+ if (!IS_ENABLED(CONFIG_VFIO_NOIOMMU))
+ return true;
+ return container->noiommu == (driver->ops == &vfio_noiommu_ops);
+}
+
+/*
+ * IOMMU driver registration
+ */
+int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
+{
+ struct vfio_iommu_driver *driver, *tmp;
+
+ if (WARN_ON(!ops->register_device != !ops->unregister_device))
+ return -EINVAL;
+
+ driver = kzalloc(sizeof(*driver), GFP_KERNEL);
+ if (!driver)
+ return -ENOMEM;
+
+ driver->ops = ops;
+
+ mutex_lock(&vfio.iommu_drivers_lock);
+
+ /* Check for duplicates */
+ list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
+ if (tmp->ops == ops) {
+ mutex_unlock(&vfio.iommu_drivers_lock);
+ kfree(driver);
+ return -EINVAL;
+ }
+ }
+
+ list_add(&driver->vfio_next, &vfio.iommu_drivers_list);
+
+ mutex_unlock(&vfio.iommu_drivers_lock);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);
+
+void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
+{
+ struct vfio_iommu_driver *driver;
+
+ mutex_lock(&vfio.iommu_drivers_lock);
+ list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
+ if (driver->ops == ops) {
+ list_del(&driver->vfio_next);
+ mutex_unlock(&vfio.iommu_drivers_lock);
+ kfree(driver);
+ return;
+ }
+ }
+ mutex_unlock(&vfio.iommu_drivers_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);
+
+/*
+ * Container objects - containers are created when /dev/vfio/vfio is
+ * opened, but their lifecycle extends until the last user is done, so
+ * it's freed via kref. Must support container/group/device being
+ * closed in any order.
+ */
+static void vfio_container_release(struct kref *kref)
+{
+ struct vfio_container *container;
+ container = container_of(kref, struct vfio_container, kref);
+
+ kfree(container);
+}
+
+static void vfio_container_get(struct vfio_container *container)
+{
+ kref_get(&container->kref);
+}
+
+static void vfio_container_put(struct vfio_container *container)
+{
+ kref_put(&container->kref, vfio_container_release);
+}
+
+void vfio_device_container_register(struct vfio_device *device)
+{
+ struct vfio_iommu_driver *iommu_driver =
+ device->group->container->iommu_driver;
+
+ if (iommu_driver && iommu_driver->ops->register_device)
+ iommu_driver->ops->register_device(
+ device->group->container->iommu_data, device);
+}
+
+void vfio_device_container_unregister(struct vfio_device *device)
+{
+ struct vfio_iommu_driver *iommu_driver =
+ device->group->container->iommu_driver;
+
+ if (iommu_driver && iommu_driver->ops->unregister_device)
+ iommu_driver->ops->unregister_device(
+ device->group->container->iommu_data, device);
+}
+
+long vfio_container_ioctl_check_extension(struct vfio_container *container,
+ unsigned long arg)
+{
+ struct vfio_iommu_driver *driver;
+ long ret = 0;
+
+ down_read(&container->group_lock);
+
+ driver = container->iommu_driver;
+
+ switch (arg) {
+ /* No base extensions yet */
+ default:
+ /*
+ * If no driver is set, poll all registered drivers for
+ * extensions and return the first positive result. If
+ * a driver is already set, further queries will be passed
+ * only to that driver.
+ */
+ if (!driver) {
+ mutex_lock(&vfio.iommu_drivers_lock);
+ list_for_each_entry(driver, &vfio.iommu_drivers_list,
+ vfio_next) {
+
+ if (!list_empty(&container->group_list) &&
+ !vfio_iommu_driver_allowed(container,
+ driver))
+ continue;
+ if (!try_module_get(driver->ops->owner))
+ continue;
+
+ ret = driver->ops->ioctl(NULL,
+ VFIO_CHECK_EXTENSION,
+ arg);
+ module_put(driver->ops->owner);
+ if (ret > 0)
+ break;
+ }
+ mutex_unlock(&vfio.iommu_drivers_lock);
+ } else
+ ret = driver->ops->ioctl(container->iommu_data,
+ VFIO_CHECK_EXTENSION, arg);
+ }
+
+ up_read(&container->group_lock);
+
+ return ret;
+}
+
+/* hold write lock on container->group_lock */
+static int __vfio_container_attach_groups(struct vfio_container *container,
+ struct vfio_iommu_driver *driver,
+ void *data)
+{
+ struct vfio_group *group;
+ int ret = -ENODEV;
+
+ list_for_each_entry(group, &container->group_list, container_next) {
+ ret = driver->ops->attach_group(data, group->iommu_group,
+ group->type);
+ if (ret)
+ goto unwind;
+ }
+
+ return ret;
+
+unwind:
+ list_for_each_entry_continue_reverse(group, &container->group_list,
+ container_next) {
+ driver->ops->detach_group(data, group->iommu_group);
+ }
+
+ return ret;
+}
+
+static long vfio_ioctl_set_iommu(struct vfio_container *container,
+ unsigned long arg)
+{
+ struct vfio_iommu_driver *driver;
+ long ret = -ENODEV;
+
+ down_write(&container->group_lock);
+
+ /*
+ * The container is designed to be an unprivileged interface while
+ * the group can be assigned to specific users. Therefore, only by
+ * adding a group to a container does the user get the privilege of
+ * enabling the iommu, which may allocate finite resources. There
+ * is no unset_iommu, but by removing all the groups from a container,
+ * the container is deprivileged and returns to an unset state.
+ */
+ if (list_empty(&container->group_list) || container->iommu_driver) {
+ up_write(&container->group_lock);
+ return -EINVAL;
+ }
+
+ mutex_lock(&vfio.iommu_drivers_lock);
+ list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
+ void *data;
+
+ if (!vfio_iommu_driver_allowed(container, driver))
+ continue;
+ if (!try_module_get(driver->ops->owner))
+ continue;
+
+ /*
+ * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
+ * so test which iommu driver reported support for this
+ * extension and call open on them. We also pass them the
+ * magic, allowing a single driver to support multiple
+ * interfaces if they'd like.
+ */
+ if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
+ module_put(driver->ops->owner);
+ continue;
+ }
+
+ data = driver->ops->open(arg);
+ if (IS_ERR(data)) {
+ ret = PTR_ERR(data);
+ module_put(driver->ops->owner);
+ continue;
+ }
+
+ ret = __vfio_container_attach_groups(container, driver, data);
+ if (ret) {
+ driver->ops->release(data);
+ module_put(driver->ops->owner);
+ continue;
+ }
+
+ container->iommu_driver = driver;
+ container->iommu_data = data;
+ break;
+ }
+
+ mutex_unlock(&vfio.iommu_drivers_lock);
+ up_write(&container->group_lock);
+
+ return ret;
+}
+
+static long vfio_fops_unl_ioctl(struct file *filep,
+ unsigned int cmd, unsigned long arg)
+{
+ struct vfio_container *container = filep->private_data;
+ struct vfio_iommu_driver *driver;
+ void *data;
+ long ret = -EINVAL;
+
+ if (!container)
+ return ret;
+
+ switch (cmd) {
+ case VFIO_GET_API_VERSION:
+ ret = VFIO_API_VERSION;
+ break;
+ case VFIO_CHECK_EXTENSION:
+ ret = vfio_container_ioctl_check_extension(container, arg);
+ break;
+ case VFIO_SET_IOMMU:
+ ret = vfio_ioctl_set_iommu(container, arg);
+ break;
+ default:
+ driver = container->iommu_driver;
+ data = container->iommu_data;
+
+ if (driver) /* passthrough all unrecognized ioctls */
+ ret = driver->ops->ioctl(data, cmd, arg);
+ }
+
+ return ret;
+}
+
+static int vfio_fops_open(struct inode *inode, struct file *filep)
+{
+ struct vfio_container *container;
+
+ container = kzalloc(sizeof(*container), GFP_KERNEL);
+ if (!container)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&container->group_list);
+ init_rwsem(&container->group_lock);
+ kref_init(&container->kref);
+
+ filep->private_data = container;
+
+ return 0;
+}
+
+static int vfio_fops_release(struct inode *inode, struct file *filep)
+{
+ struct vfio_container *container = filep->private_data;
+ struct vfio_iommu_driver *driver = container->iommu_driver;
+
+ if (driver && driver->ops->notify)
+ driver->ops->notify(container->iommu_data,
+ VFIO_IOMMU_CONTAINER_CLOSE);
+
+ filep->private_data = NULL;
+
+ vfio_container_put(container);
+
+ return 0;
+}
+
+static const struct file_operations vfio_fops = {
+ .owner = THIS_MODULE,
+ .open = vfio_fops_open,
+ .release = vfio_fops_release,
+ .unlocked_ioctl = vfio_fops_unl_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+};
+
+struct vfio_container *vfio_container_from_file(struct file *file)
+{
+ struct vfio_container *container;
+
+ /* Sanity check, is this really our fd? */
+ if (file->f_op != &vfio_fops)
+ return NULL;
+
+ container = file->private_data;
+ WARN_ON(!container); /* fget ensures we don't race vfio_release */
+ return container;
+}
+
+static struct miscdevice vfio_dev = {
+ .minor = VFIO_MINOR,
+ .name = "vfio",
+ .fops = &vfio_fops,
+ .nodename = "vfio/vfio",
+ .mode = S_IRUGO | S_IWUGO,
+};
+
+int vfio_container_attach_group(struct vfio_container *container,
+ struct vfio_group *group)
+{
+ struct vfio_iommu_driver *driver;
+ int ret = 0;
+
+ lockdep_assert_held(&group->group_lock);
+
+ if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
+ return -EPERM;
+
+ down_write(&container->group_lock);
+
+ /* Real groups and fake groups cannot mix */
+ if (!list_empty(&container->group_list) &&
+ container->noiommu != (group->type == VFIO_NO_IOMMU)) {
+ ret = -EPERM;
+ goto out_unlock_container;
+ }
+
+ if (group->type == VFIO_IOMMU) {
+ ret = iommu_group_claim_dma_owner(group->iommu_group, group);
+ if (ret)
+ goto out_unlock_container;
+ }
+
+ driver = container->iommu_driver;
+ if (driver) {
+ ret = driver->ops->attach_group(container->iommu_data,
+ group->iommu_group,
+ group->type);
+ if (ret) {
+ if (group->type == VFIO_IOMMU)
+ iommu_group_release_dma_owner(
+ group->iommu_group);
+ goto out_unlock_container;
+ }
+ }
+
+ group->container = container;
+ group->container_users = 1;
+ container->noiommu = (group->type == VFIO_NO_IOMMU);
+ list_add(&group->container_next, &container->group_list);
+
+ /* Get a reference on the container and mark a user within the group */
+ vfio_container_get(container);
+
+out_unlock_container:
+ up_write(&container->group_lock);
+ return ret;
+}
+
+void vfio_group_detach_container(struct vfio_group *group)
+{
+ struct vfio_container *container = group->container;
+ struct vfio_iommu_driver *driver;
+
+ lockdep_assert_held(&group->group_lock);
+ WARN_ON(group->container_users != 1);
+
+ down_write(&container->group_lock);
+
+ driver = container->iommu_driver;
+ if (driver)
+ driver->ops->detach_group(container->iommu_data,
+ group->iommu_group);
+
+ if (group->type == VFIO_IOMMU)
+ iommu_group_release_dma_owner(group->iommu_group);
+
+ group->container = NULL;
+ group->container_users = 0;
+ list_del(&group->container_next);
+
+ /* Detaching the last group deprivileges a container, remove iommu */
+ if (driver && list_empty(&container->group_list)) {
+ driver->ops->release(container->iommu_data);
+ module_put(driver->ops->owner);
+ container->iommu_driver = NULL;
+ container->iommu_data = NULL;
+ }
+
+ up_write(&container->group_lock);
+
+ vfio_container_put(container);
+}
+
+int vfio_device_assign_container(struct vfio_device *device)
+{
+ struct vfio_group *group = device->group;
+
+ lockdep_assert_held(&group->group_lock);
+
+ if (!group->container || !group->container->iommu_driver ||
+ WARN_ON(!group->container_users))
+ return -EINVAL;
+
+ if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
+ return -EPERM;
+
+ get_file(group->opened_file);
+ group->container_users++;
+ return 0;
+}
+
+void vfio_device_unassign_container(struct vfio_device *device)
+{
+ mutex_lock(&device->group->group_lock);
+ WARN_ON(device->group->container_users <= 1);
+ device->group->container_users--;
+ fput(device->group->opened_file);
+ mutex_unlock(&device->group->group_lock);
+}
+
+/*
+ * Pin contiguous user pages and return their associated host pages for local
+ * domain only.
+ * @device [in] : device
+ * @iova [in] : starting IOVA of user pages to be pinned.
+ * @npage [in] : count of pages to be pinned. This count should not
+ * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
+ * @prot [in] : protection flags
+ * @pages[out] : array of host pages
+ * Return error or number of pages pinned.
+ *
+ * A driver may only call this function if the vfio_device was created
+ * by vfio_register_emulated_iommu_dev().
+ */
+int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
+ int npage, int prot, struct page **pages)
+{
+ struct vfio_container *container;
+ struct vfio_group *group = device->group;
+ struct vfio_iommu_driver *driver;
+ int ret;
+
+ if (!pages || !npage || !vfio_assert_device_open(device))
+ return -EINVAL;
+
+ if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
+ return -E2BIG;
+
+ /* group->container cannot change while a vfio device is open */
+ container = group->container;
+ driver = container->iommu_driver;
+ if (likely(driver && driver->ops->pin_pages))
+ ret = driver->ops->pin_pages(container->iommu_data,
+ group->iommu_group, iova,
+ npage, prot, pages);
+ else
+ ret = -ENOTTY;
+
+ return ret;
+}
+EXPORT_SYMBOL(vfio_pin_pages);
+
+/*
+ * Unpin contiguous host pages for local domain only.
+ * @device [in] : device
+ * @iova [in] : starting address of user pages to be unpinned.
+ * @npage [in] : count of pages to be unpinned. This count should not
+ * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
+ */
+void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
+{
+ struct vfio_container *container;
+ struct vfio_iommu_driver *driver;
+
+ if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES))
+ return;
+
+ if (WARN_ON(!vfio_assert_device_open(device)))
+ return;
+
+ /* group->container cannot change while a vfio device is open */
+ container = device->group->container;
+ driver = container->iommu_driver;
+
+ driver->ops->unpin_pages(container->iommu_data, iova, npage);
+}
+EXPORT_SYMBOL(vfio_unpin_pages);
+
+/*
+ * This interface allows the CPUs to perform some sort of virtual DMA on
+ * behalf of the device.
+ *
+ * CPUs read/write from/into a range of IOVAs pointing to user space memory
+ * into/from a kernel buffer.
+ *
+ * As the read/write of user space memory is conducted via the CPUs and is
+ * not a real device DMA, it is not necessary to pin the user space memory.
+ *
+ * @device [in] : VFIO device
+ * @iova [in] : base IOVA of a user space buffer
+ * @data [in] : pointer to kernel buffer
+ * @len [in] : kernel buffer length
+ * @write : indicate read or write
+ * Return error code on failure or 0 on success.
+ */
+int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
+ size_t len, bool write)
+{
+ struct vfio_container *container;
+ struct vfio_iommu_driver *driver;
+ int ret = 0;
+
+ if (!data || len <= 0 || !vfio_assert_device_open(device))
+ return -EINVAL;
+
+ /* group->container cannot change while a vfio device is open */
+ container = device->group->container;
+ driver = container->iommu_driver;
+
+ if (likely(driver && driver->ops->dma_rw))
+ ret = driver->ops->dma_rw(container->iommu_data,
+ iova, data, len, write);
+ else
+ ret = -ENOTTY;
+ return ret;
+}
+EXPORT_SYMBOL(vfio_dma_rw);
+
+int __init vfio_container_init(void)
+{
+ int ret;
+
+ mutex_init(&vfio.iommu_drivers_lock);
+ INIT_LIST_HEAD(&vfio.iommu_drivers_list);
+
+ ret = misc_register(&vfio_dev);
+ if (ret) {
+ pr_err("vfio: misc device register failed\n");
+ return ret;
+ }
+
+ if (IS_ENABLED(CONFIG_VFIO_NOIOMMU)) {
+ ret = vfio_register_iommu_driver(&vfio_noiommu_ops);
+ if (ret)
+ goto err_misc;
+ }
+ return 0;
+
+err_misc:
+ misc_deregister(&vfio_dev);
+ return ret;
+}
+
+void vfio_container_cleanup(void)
+{
+ if (IS_ENABLED(CONFIG_VFIO_NOIOMMU))
+ vfio_unregister_iommu_driver(&vfio_noiommu_ops);
+ misc_deregister(&vfio_dev);
+ mutex_destroy(&vfio.iommu_drivers_lock);
+}
/* reset the device before cleaning up the interrupts */
ret = vfio_fsl_mc_reset_device(vdev);
- if (WARN_ON(ret))
+ if (ret)
dev_warn(&mc_cont->dev,
- "VFIO_FLS_MC: reset device has failed (%d)\n", ret);
+ "VFIO_FSL_MC: reset device has failed (%d)\n", ret);
vfio_fsl_mc_irqs_cleanup(vdev);
return vfio_fsl_mc_mmap_mmio(vdev->regions[index], vma);
}
-static const struct vfio_device_ops vfio_fsl_mc_ops = {
- .name = "vfio-fsl-mc",
- .open_device = vfio_fsl_mc_open_device,
- .close_device = vfio_fsl_mc_close_device,
- .ioctl = vfio_fsl_mc_ioctl,
- .read = vfio_fsl_mc_read,
- .write = vfio_fsl_mc_write,
- .mmap = vfio_fsl_mc_mmap,
-};
-
+static const struct vfio_device_ops vfio_fsl_mc_ops;
static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb);
}
-static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev)
+static int vfio_fsl_mc_init_dev(struct vfio_device *core_vdev)
{
- struct vfio_fsl_mc_device *vdev;
- struct device *dev = &mc_dev->dev;
+ struct vfio_fsl_mc_device *vdev =
+ container_of(core_vdev, struct vfio_fsl_mc_device, vdev);
+ struct fsl_mc_device *mc_dev = to_fsl_mc_device(core_vdev->dev);
int ret;
- vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
- if (!vdev)
- return -ENOMEM;
-
- vfio_init_group_dev(&vdev->vdev, dev, &vfio_fsl_mc_ops);
vdev->mc_dev = mc_dev;
mutex_init(&vdev->igate);
if (is_fsl_mc_bus_dprc(mc_dev))
- ret = vfio_assign_device_set(&vdev->vdev, &mc_dev->dev);
+ ret = vfio_assign_device_set(core_vdev, &mc_dev->dev);
else
- ret = vfio_assign_device_set(&vdev->vdev, mc_dev->dev.parent);
- if (ret)
- goto out_uninit;
+ ret = vfio_assign_device_set(core_vdev, mc_dev->dev.parent);
- ret = vfio_fsl_mc_init_device(vdev);
if (ret)
- goto out_uninit;
+ return ret;
+
+ /* device_set is released by vfio core if @init fails */
+ return vfio_fsl_mc_init_device(vdev);
+}
+
+static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev)
+{
+ struct vfio_fsl_mc_device *vdev;
+ struct device *dev = &mc_dev->dev;
+ int ret;
+
+ vdev = vfio_alloc_device(vfio_fsl_mc_device, vdev, dev,
+ &vfio_fsl_mc_ops);
+ if (IS_ERR(vdev))
+ return PTR_ERR(vdev);
ret = vfio_register_group_dev(&vdev->vdev);
if (ret) {
dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n");
- goto out_device;
+ goto out_put_vdev;
}
ret = vfio_fsl_mc_scan_container(mc_dev);
out_group_dev:
vfio_unregister_group_dev(&vdev->vdev);
-out_device:
- vfio_fsl_uninit_device(vdev);
-out_uninit:
- vfio_uninit_group_dev(&vdev->vdev);
- kfree(vdev);
+out_put_vdev:
+ vfio_put_device(&vdev->vdev);
return ret;
}
+static void vfio_fsl_mc_release_dev(struct vfio_device *core_vdev)
+{
+ struct vfio_fsl_mc_device *vdev =
+ container_of(core_vdev, struct vfio_fsl_mc_device, vdev);
+
+ vfio_fsl_uninit_device(vdev);
+ mutex_destroy(&vdev->igate);
+ vfio_free_device(core_vdev);
+}
+
static int vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev)
{
struct device *dev = &mc_dev->dev;
struct vfio_fsl_mc_device *vdev = dev_get_drvdata(dev);
vfio_unregister_group_dev(&vdev->vdev);
- mutex_destroy(&vdev->igate);
-
dprc_remove_devices(mc_dev, NULL, 0);
- vfio_fsl_uninit_device(vdev);
-
- vfio_uninit_group_dev(&vdev->vdev);
- kfree(vdev);
+ vfio_put_device(&vdev->vdev);
return 0;
}
+static const struct vfio_device_ops vfio_fsl_mc_ops = {
+ .name = "vfio-fsl-mc",
+ .init = vfio_fsl_mc_init_dev,
+ .release = vfio_fsl_mc_release_dev,
+ .open_device = vfio_fsl_mc_open_device,
+ .close_device = vfio_fsl_mc_close_device,
+ .ioctl = vfio_fsl_mc_ioctl,
+ .read = vfio_fsl_mc_read,
+ .write = vfio_fsl_mc_write,
+ .mmap = vfio_fsl_mc_mmap,
+};
+
static struct fsl_mc_driver vfio_fsl_mc_driver = {
.probe = vfio_fsl_mc_probe,
.remove = vfio_fsl_mc_remove,
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+#include <linux/iova_bitmap.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+
+#define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE)
+
+/*
+ * struct iova_bitmap_map - A bitmap representing an IOVA range
+ *
+ * Main data structure for tracking mapped user pages of bitmap data.
+ *
+ * For example, for something recording dirty IOVAs, it will be provided a
+ * struct iova_bitmap structure, as a general structure for iterating the
+ * total IOVA range. The struct iova_bitmap_map, though, represents the
+ * subset of said IOVA space that is pinned by its parent structure (struct
+ * iova_bitmap).
+ *
+ * The user does not need to exact location of the bits in the bitmap.
+ * From user perspective the only API available is iova_bitmap_set() which
+ * records the IOVA *range* in the bitmap by setting the corresponding
+ * bits.
+ *
+ * The bitmap is an array of u64 whereas each bit represents an IOVA of
+ * range of (1 << pgshift). Thus formula for the bitmap data to be set is:
+ *
+ * data[(iova / page_size) / 64] & (1ULL << (iova % 64))
+ */
+struct iova_bitmap_map {
+ /* base IOVA representing bit 0 of the first page */
+ unsigned long iova;
+
+ /* page size order that each bit granules to */
+ unsigned long pgshift;
+
+ /* page offset of the first user page pinned */
+ unsigned long pgoff;
+
+ /* number of pages pinned */
+ unsigned long npages;
+
+ /* pinned pages representing the bitmap data */
+ struct page **pages;
+};
+
+/*
+ * struct iova_bitmap - The IOVA bitmap object
+ *
+ * Main data structure for iterating over the bitmap data.
+ *
+ * Abstracts the pinning work and iterates in IOVA ranges.
+ * It uses a windowing scheme and pins the bitmap in relatively
+ * big ranges e.g.
+ *
+ * The bitmap object uses one base page to store all the pinned pages
+ * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores
+ * 512 struct page pointers which, if the base page size is 4K, it means
+ * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is
+ * also 4K then the range window to iterate is 64G.
+ *
+ * For example iterating on a total IOVA range of 4G..128G, it will walk
+ * through this set of ranges:
+ *
+ * 4G - 68G-1 (64G)
+ * 68G - 128G-1 (64G)
+ *
+ * An example of the APIs on how to use/iterate over the IOVA bitmap:
+ *
+ * bitmap = iova_bitmap_alloc(iova, length, page_size, data);
+ * if (IS_ERR(bitmap))
+ * return PTR_ERR(bitmap);
+ *
+ * ret = iova_bitmap_for_each(bitmap, arg, dirty_reporter_fn);
+ *
+ * iova_bitmap_free(bitmap);
+ *
+ * Each iteration of the @dirty_reporter_fn is called with a unique @iova
+ * and @length argument, indicating the current range available through the
+ * iova_bitmap. The @dirty_reporter_fn uses iova_bitmap_set() to mark dirty
+ * areas (@iova_length) within that provided range, as following:
+ *
+ * iova_bitmap_set(bitmap, iova, iova_length);
+ *
+ * The internals of the object uses an index @mapped_base_index that indexes
+ * which u64 word of the bitmap is mapped, up to @mapped_total_index.
+ * Those keep being incremented until @mapped_total_index is reached while
+ * mapping up to PAGE_SIZE / sizeof(struct page*) maximum of pages.
+ *
+ * The IOVA bitmap is usually located on what tracks DMA mapped ranges or
+ * some form of IOVA range tracking that co-relates to the user passed
+ * bitmap.
+ */
+struct iova_bitmap {
+ /* IOVA range representing the currently mapped bitmap data */
+ struct iova_bitmap_map mapped;
+
+ /* userspace address of the bitmap */
+ u64 __user *bitmap;
+
+ /* u64 index that @mapped points to */
+ unsigned long mapped_base_index;
+
+ /* how many u64 can we walk in total */
+ unsigned long mapped_total_index;
+
+ /* base IOVA of the whole bitmap */
+ unsigned long iova;
+
+ /* length of the IOVA range for the whole bitmap */
+ size_t length;
+};
+
+/*
+ * Converts a relative IOVA to a bitmap index.
+ * This function provides the index into the u64 array (bitmap::bitmap)
+ * for a given IOVA offset.
+ * Relative IOVA means relative to the bitmap::mapped base IOVA
+ * (stored in mapped::iova). All computations in this file are done using
+ * relative IOVAs and thus avoid an extra subtraction against mapped::iova.
+ * The user API iova_bitmap_set() always uses a regular absolute IOVAs.
+ */
+static unsigned long iova_bitmap_offset_to_index(struct iova_bitmap *bitmap,
+ unsigned long iova)
+{
+ unsigned long pgsize = 1 << bitmap->mapped.pgshift;
+
+ return iova / (BITS_PER_TYPE(*bitmap->bitmap) * pgsize);
+}
+
+/*
+ * Converts a bitmap index to a *relative* IOVA.
+ */
+static unsigned long iova_bitmap_index_to_offset(struct iova_bitmap *bitmap,
+ unsigned long index)
+{
+ unsigned long pgshift = bitmap->mapped.pgshift;
+
+ return (index * BITS_PER_TYPE(*bitmap->bitmap)) << pgshift;
+}
+
+/*
+ * Returns the base IOVA of the mapped range.
+ */
+static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap)
+{
+ unsigned long skip = bitmap->mapped_base_index;
+
+ return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip);
+}
+
+/*
+ * Pins the bitmap user pages for the current range window.
+ * This is internal to IOVA bitmap and called when advancing the
+ * index (@mapped_base_index) or allocating the bitmap.
+ */
+static int iova_bitmap_get(struct iova_bitmap *bitmap)
+{
+ struct iova_bitmap_map *mapped = &bitmap->mapped;
+ unsigned long npages;
+ u64 __user *addr;
+ long ret;
+
+ /*
+ * @mapped_base_index is the index of the currently mapped u64 words
+ * that we have access. Anything before @mapped_base_index is not
+ * mapped. The range @mapped_base_index .. @mapped_total_index-1 is
+ * mapped but capped at a maximum number of pages.
+ */
+ npages = DIV_ROUND_UP((bitmap->mapped_total_index -
+ bitmap->mapped_base_index) *
+ sizeof(*bitmap->bitmap), PAGE_SIZE);
+
+ /*
+ * We always cap at max number of 'struct page' a base page can fit.
+ * This is, for example, on x86 means 2M of bitmap data max.
+ */
+ npages = min(npages, PAGE_SIZE / sizeof(struct page *));
+
+ /*
+ * Bitmap address to be pinned is calculated via pointer arithmetic
+ * with bitmap u64 word index.
+ */
+ addr = bitmap->bitmap + bitmap->mapped_base_index;
+
+ ret = pin_user_pages_fast((unsigned long)addr, npages,
+ FOLL_WRITE, mapped->pages);
+ if (ret <= 0)
+ return -EFAULT;
+
+ mapped->npages = (unsigned long)ret;
+ /* Base IOVA where @pages point to i.e. bit 0 of the first page */
+ mapped->iova = iova_bitmap_mapped_iova(bitmap);
+
+ /*
+ * offset of the page where pinned pages bit 0 is located.
+ * This handles the case where the bitmap is not PAGE_SIZE
+ * aligned.
+ */
+ mapped->pgoff = offset_in_page(addr);
+ return 0;
+}
+
+/*
+ * Unpins the bitmap user pages and clears @npages
+ * (un)pinning is abstracted from API user and it's done when advancing
+ * the index or freeing the bitmap.
+ */
+static void iova_bitmap_put(struct iova_bitmap *bitmap)
+{
+ struct iova_bitmap_map *mapped = &bitmap->mapped;
+
+ if (mapped->npages) {
+ unpin_user_pages(mapped->pages, mapped->npages);
+ mapped->npages = 0;
+ }
+}
+
+/**
+ * iova_bitmap_alloc() - Allocates an IOVA bitmap object
+ * @iova: Start address of the IOVA range
+ * @length: Length of the IOVA range
+ * @page_size: Page size of the IOVA bitmap. It defines what each bit
+ * granularity represents
+ * @data: Userspace address of the bitmap
+ *
+ * Allocates an IOVA object and initializes all its fields including the
+ * first user pages of @data.
+ *
+ * Return: A pointer to a newly allocated struct iova_bitmap
+ * or ERR_PTR() on error.
+ */
+struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length,
+ unsigned long page_size, u64 __user *data)
+{
+ struct iova_bitmap_map *mapped;
+ struct iova_bitmap *bitmap;
+ int rc;
+
+ bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
+ if (!bitmap)
+ return ERR_PTR(-ENOMEM);
+
+ mapped = &bitmap->mapped;
+ mapped->pgshift = __ffs(page_size);
+ bitmap->bitmap = data;
+ bitmap->mapped_total_index =
+ iova_bitmap_offset_to_index(bitmap, length - 1) + 1;
+ bitmap->iova = iova;
+ bitmap->length = length;
+ mapped->iova = iova;
+ mapped->pages = (struct page **)__get_free_page(GFP_KERNEL);
+ if (!mapped->pages) {
+ rc = -ENOMEM;
+ goto err;
+ }
+
+ rc = iova_bitmap_get(bitmap);
+ if (rc)
+ goto err;
+ return bitmap;
+
+err:
+ iova_bitmap_free(bitmap);
+ return ERR_PTR(rc);
+}
+
+/**
+ * iova_bitmap_free() - Frees an IOVA bitmap object
+ * @bitmap: IOVA bitmap to free
+ *
+ * It unpins and releases pages array memory and clears any leftover
+ * state.
+ */
+void iova_bitmap_free(struct iova_bitmap *bitmap)
+{
+ struct iova_bitmap_map *mapped = &bitmap->mapped;
+
+ iova_bitmap_put(bitmap);
+
+ if (mapped->pages) {
+ free_page((unsigned long)mapped->pages);
+ mapped->pages = NULL;
+ }
+
+ kfree(bitmap);
+}
+
+/*
+ * Returns the remaining bitmap indexes from mapped_total_index to process for
+ * the currently pinned bitmap pages.
+ */
+static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap)
+{
+ unsigned long remaining;
+
+ remaining = bitmap->mapped_total_index - bitmap->mapped_base_index;
+ remaining = min_t(unsigned long, remaining,
+ (bitmap->mapped.npages << PAGE_SHIFT) / sizeof(*bitmap->bitmap));
+
+ return remaining;
+}
+
+/*
+ * Returns the length of the mapped IOVA range.
+ */
+static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap)
+{
+ unsigned long max_iova = bitmap->iova + bitmap->length - 1;
+ unsigned long iova = iova_bitmap_mapped_iova(bitmap);
+ unsigned long remaining;
+
+ /*
+ * iova_bitmap_mapped_remaining() returns a number of indexes which
+ * when converted to IOVA gives us a max length that the bitmap
+ * pinned data can cover. Afterwards, that is capped to
+ * only cover the IOVA range in @bitmap::iova .. @bitmap::length.
+ */
+ remaining = iova_bitmap_index_to_offset(bitmap,
+ iova_bitmap_mapped_remaining(bitmap));
+
+ if (iova + remaining - 1 > max_iova)
+ remaining -= ((iova + remaining - 1) - max_iova);
+
+ return remaining;
+}
+
+/*
+ * Returns true if there's not more data to iterate.
+ */
+static bool iova_bitmap_done(struct iova_bitmap *bitmap)
+{
+ return bitmap->mapped_base_index >= bitmap->mapped_total_index;
+}
+
+/*
+ * Advances to the next range, releases the current pinned
+ * pages and pins the next set of bitmap pages.
+ * Returns 0 on success or otherwise errno.
+ */
+static int iova_bitmap_advance(struct iova_bitmap *bitmap)
+{
+ unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1;
+ unsigned long count = iova_bitmap_offset_to_index(bitmap, iova) + 1;
+
+ bitmap->mapped_base_index += count;
+
+ iova_bitmap_put(bitmap);
+ if (iova_bitmap_done(bitmap))
+ return 0;
+
+ /* When advancing the index we pin the next set of bitmap pages */
+ return iova_bitmap_get(bitmap);
+}
+
+/**
+ * iova_bitmap_for_each() - Iterates over the bitmap
+ * @bitmap: IOVA bitmap to iterate
+ * @opaque: Additional argument to pass to the callback
+ * @fn: Function that gets called for each IOVA range
+ *
+ * Helper function to iterate over bitmap data representing a portion of IOVA
+ * space. It hides the complexity of iterating bitmaps and translating the
+ * mapped bitmap user pages into IOVA ranges to process.
+ *
+ * Return: 0 on success, and an error on failure either upon
+ * iteration or when the callback returns an error.
+ */
+int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque,
+ iova_bitmap_fn_t fn)
+{
+ int ret = 0;
+
+ for (; !iova_bitmap_done(bitmap) && !ret;
+ ret = iova_bitmap_advance(bitmap)) {
+ ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap),
+ iova_bitmap_mapped_length(bitmap), opaque);
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
+
+/**
+ * iova_bitmap_set() - Records an IOVA range in bitmap
+ * @bitmap: IOVA bitmap
+ * @iova: IOVA to start
+ * @length: IOVA range length
+ *
+ * Set the bits corresponding to the range [iova .. iova+length-1] in
+ * the user bitmap.
+ *
+ * Return: The number of bits set.
+ */
+void iova_bitmap_set(struct iova_bitmap *bitmap,
+ unsigned long iova, size_t length)
+{
+ struct iova_bitmap_map *mapped = &bitmap->mapped;
+ unsigned long offset = (iova - mapped->iova) >> mapped->pgshift;
+ unsigned long nbits = max_t(unsigned long, 1, length >> mapped->pgshift);
+ unsigned long page_idx = offset / BITS_PER_PAGE;
+ unsigned long page_offset = mapped->pgoff;
+ void *kaddr;
+
+ offset = offset % BITS_PER_PAGE;
+
+ do {
+ unsigned long size = min(BITS_PER_PAGE - offset, nbits);
+
+ kaddr = kmap_local_page(mapped->pages[page_idx]);
+ bitmap_set(kaddr + page_offset, offset, size);
+ kunmap_local(kaddr);
+ page_offset = offset = 0;
+ nbits -= size;
+ page_idx++;
+ } while (nbits > 0);
+}
+EXPORT_SYMBOL_GPL(iova_bitmap_set);
*/
#include <linux/module.h>
-#include <linux/device.h>
#include <linux/slab.h>
-#include <linux/uuid.h>
#include <linux/sysfs.h>
#include <linux/mdev.h>
#define DRIVER_AUTHOR "NVIDIA Corporation"
#define DRIVER_DESC "Mediated device Core Driver"
-static LIST_HEAD(parent_list);
-static DEFINE_MUTEX(parent_list_lock);
static struct class_compat *mdev_bus_compat_class;
static LIST_HEAD(mdev_list);
static DEFINE_MUTEX(mdev_list_lock);
-struct device *mdev_parent_dev(struct mdev_device *mdev)
-{
- return mdev->type->parent->dev;
-}
-EXPORT_SYMBOL(mdev_parent_dev);
-
-/*
- * Return the index in supported_type_groups that this mdev_device was created
- * from.
- */
-unsigned int mdev_get_type_group_id(struct mdev_device *mdev)
-{
- return mdev->type->type_group_id;
-}
-EXPORT_SYMBOL(mdev_get_type_group_id);
-
-/*
- * Used in mdev_type_attribute sysfs functions to return the index in the
- * supported_type_groups that the sysfs is called from.
- */
-unsigned int mtype_get_type_group_id(struct mdev_type *mtype)
-{
- return mtype->type_group_id;
-}
-EXPORT_SYMBOL(mtype_get_type_group_id);
-
-/*
- * Used in mdev_type_attribute sysfs functions to return the parent struct
- * device
- */
-struct device *mtype_get_parent_dev(struct mdev_type *mtype)
-{
- return mtype->parent->dev;
-}
-EXPORT_SYMBOL(mtype_get_parent_dev);
-
-/* Should be called holding parent_list_lock */
-static struct mdev_parent *__find_parent_device(struct device *dev)
-{
- struct mdev_parent *parent;
-
- list_for_each_entry(parent, &parent_list, next) {
- if (parent->dev == dev)
- return parent;
- }
- return NULL;
-}
-
-void mdev_release_parent(struct kref *kref)
-{
- struct mdev_parent *parent = container_of(kref, struct mdev_parent,
- ref);
- struct device *dev = parent->dev;
-
- kfree(parent);
- put_device(dev);
-}
-
/* Caller must hold parent unreg_sem read or write lock */
static void mdev_device_remove_common(struct mdev_device *mdev)
{
static int mdev_device_remove_cb(struct device *dev, void *data)
{
- struct mdev_device *mdev = mdev_from_dev(dev);
-
- if (mdev)
- mdev_device_remove_common(mdev);
+ if (dev->bus == &mdev_bus_type)
+ mdev_device_remove_common(to_mdev_device(dev));
return 0;
}
/*
- * mdev_register_device : Register a device
+ * mdev_register_parent: Register a device as parent for mdevs
+ * @parent: parent structure registered
* @dev: device structure representing parent device.
* @mdev_driver: Device driver to bind to the newly created mdev
+ * @types: Array of supported mdev types
+ * @nr_types: Number of entries in @types
+ *
+ * Registers the @parent stucture as a parent for mdev types and thus mdev
+ * devices. The caller needs to hold a reference on @dev that must not be
+ * released until after the call to mdev_unregister_parent().
*
- * Add device to list of registered parent devices.
* Returns a negative value on error, otherwise 0.
*/
-int mdev_register_device(struct device *dev, struct mdev_driver *mdev_driver)
+int mdev_register_parent(struct mdev_parent *parent, struct device *dev,
+ struct mdev_driver *mdev_driver, struct mdev_type **types,
+ unsigned int nr_types)
{
- int ret;
- struct mdev_parent *parent;
char *env_string = "MDEV_STATE=registered";
char *envp[] = { env_string, NULL };
+ int ret;
- /* check for mandatory ops */
- if (!mdev_driver->supported_type_groups)
- return -EINVAL;
-
- dev = get_device(dev);
- if (!dev)
- return -EINVAL;
-
- mutex_lock(&parent_list_lock);
-
- /* Check for duplicate */
- parent = __find_parent_device(dev);
- if (parent) {
- parent = NULL;
- ret = -EEXIST;
- goto add_dev_err;
- }
-
- parent = kzalloc(sizeof(*parent), GFP_KERNEL);
- if (!parent) {
- ret = -ENOMEM;
- goto add_dev_err;
- }
-
- kref_init(&parent->ref);
+ memset(parent, 0, sizeof(*parent));
init_rwsem(&parent->unreg_sem);
-
parent->dev = dev;
parent->mdev_driver = mdev_driver;
+ parent->types = types;
+ parent->nr_types = nr_types;
+ atomic_set(&parent->available_instances, mdev_driver->max_instances);
if (!mdev_bus_compat_class) {
mdev_bus_compat_class = class_compat_register("mdev_bus");
- if (!mdev_bus_compat_class) {
- ret = -ENOMEM;
- goto add_dev_err;
- }
+ if (!mdev_bus_compat_class)
+ return -ENOMEM;
}
ret = parent_create_sysfs_files(parent);
if (ret)
- goto add_dev_err;
+ return ret;
ret = class_compat_create_link(mdev_bus_compat_class, dev, NULL);
if (ret)
dev_warn(dev, "Failed to create compatibility class link\n");
- list_add(&parent->next, &parent_list);
- mutex_unlock(&parent_list_lock);
-
dev_info(dev, "MDEV: Registered\n");
kobject_uevent_env(&dev->kobj, KOBJ_CHANGE, envp);
-
return 0;
-
-add_dev_err:
- mutex_unlock(&parent_list_lock);
- if (parent)
- mdev_put_parent(parent);
- else
- put_device(dev);
- return ret;
}
-EXPORT_SYMBOL(mdev_register_device);
+EXPORT_SYMBOL(mdev_register_parent);
/*
- * mdev_unregister_device : Unregister a parent device
- * @dev: device structure representing parent device.
- *
- * Remove device from list of registered parent devices. Give a chance to free
- * existing mediated devices for given device.
+ * mdev_unregister_parent : Unregister a parent device
+ * @parent: parent structure to unregister
*/
-
-void mdev_unregister_device(struct device *dev)
+void mdev_unregister_parent(struct mdev_parent *parent)
{
- struct mdev_parent *parent;
char *env_string = "MDEV_STATE=unregistered";
char *envp[] = { env_string, NULL };
- mutex_lock(&parent_list_lock);
- parent = __find_parent_device(dev);
-
- if (!parent) {
- mutex_unlock(&parent_list_lock);
- return;
- }
- dev_info(dev, "MDEV: Unregistering\n");
-
- list_del(&parent->next);
- mutex_unlock(&parent_list_lock);
+ dev_info(parent->dev, "MDEV: Unregistering\n");
down_write(&parent->unreg_sem);
-
- class_compat_remove_link(mdev_bus_compat_class, dev, NULL);
-
- device_for_each_child(dev, NULL, mdev_device_remove_cb);
-
+ class_compat_remove_link(mdev_bus_compat_class, parent->dev, NULL);
+ device_for_each_child(parent->dev, NULL, mdev_device_remove_cb);
parent_remove_sysfs_files(parent);
up_write(&parent->unreg_sem);
- mdev_put_parent(parent);
-
- /* We still have the caller's reference to use for the uevent */
- kobject_uevent_env(&dev->kobj, KOBJ_CHANGE, envp);
+ kobject_uevent_env(&parent->dev->kobj, KOBJ_CHANGE, envp);
}
-EXPORT_SYMBOL(mdev_unregister_device);
+EXPORT_SYMBOL(mdev_unregister_parent);
static void mdev_device_release(struct device *dev)
{
struct mdev_device *mdev = to_mdev_device(dev);
-
- /* Pairs with the get in mdev_device_create() */
- kobject_put(&mdev->type->kobj);
+ struct mdev_parent *parent = mdev->type->parent;
mutex_lock(&mdev_list_lock);
list_del(&mdev->next);
+ if (!parent->mdev_driver->get_available)
+ atomic_inc(&parent->available_instances);
mutex_unlock(&mdev_list_lock);
+ /* Pairs with the get in mdev_device_create() */
+ kobject_put(&mdev->type->kobj);
+
dev_dbg(&mdev->dev, "MDEV: destroying\n");
kfree(mdev);
}
}
}
+ if (!drv->get_available) {
+ /*
+ * Note: that non-atomic read and dec is fine here because
+ * all modifications are under mdev_list_lock.
+ */
+ if (!atomic_read(&parent->available_instances)) {
+ mutex_unlock(&mdev_list_lock);
+ return -EUSERS;
+ }
+ atomic_dec(&parent->available_instances);
+ }
+
mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
if (!mdev) {
mutex_unlock(&mdev_list_lock);
* Kirti Wankhede <kwankhede@nvidia.com>
*/
-#include <linux/device.h>
#include <linux/iommu.h>
#include <linux/mdev.h>
.remove = mdev_remove,
.match = mdev_match,
};
-EXPORT_SYMBOL_GPL(mdev_bus_type);
/**
* mdev_register_driver - register a new MDEV driver
**/
int mdev_register_driver(struct mdev_driver *drv)
{
+ if (!drv->device_api)
+ return -EINVAL;
+
/* initialize common driver fields */
drv->driver.bus = &mdev_bus_type;
-
- /* register with core */
return driver_register(&drv->driver);
}
EXPORT_SYMBOL(mdev_register_driver);
int mdev_bus_register(void);
void mdev_bus_unregister(void);
-struct mdev_parent {
- struct device *dev;
- struct mdev_driver *mdev_driver;
- struct kref ref;
- struct list_head next;
- struct kset *mdev_types_kset;
- struct list_head type_list;
- /* Synchronize device creation/removal with parent unregistration */
- struct rw_semaphore unreg_sem;
-};
-
-struct mdev_type {
- struct kobject kobj;
- struct kobject *devices_kobj;
- struct mdev_parent *parent;
- struct list_head next;
- unsigned int type_group_id;
-};
-
+extern struct bus_type mdev_bus_type;
extern const struct attribute_group *mdev_device_groups[];
#define to_mdev_type_attr(_attr) \
int mdev_device_create(struct mdev_type *kobj, const guid_t *uuid);
int mdev_device_remove(struct mdev_device *dev);
-void mdev_release_parent(struct kref *kref);
-
-static inline void mdev_get_parent(struct mdev_parent *parent)
-{
- kref_get(&parent->ref);
-}
-
-static inline void mdev_put_parent(struct mdev_parent *parent)
-{
- kref_put(&parent->ref, mdev_release_parent);
-}
-
#endif /* MDEV_PRIVATE_H */
#include <linux/sysfs.h>
#include <linux/ctype.h>
-#include <linux/device.h>
#include <linux/slab.h>
-#include <linux/uuid.h>
#include <linux/mdev.h>
#include "mdev_private.h"
-/* Static functions */
+struct mdev_type_attribute {
+ struct attribute attr;
+ ssize_t (*show)(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf);
+ ssize_t (*store)(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, const char *buf,
+ size_t count);
+};
+
+#define MDEV_TYPE_ATTR_RO(_name) \
+ struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RO(_name)
+#define MDEV_TYPE_ATTR_WO(_name) \
+ struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_WO(_name)
static ssize_t mdev_type_attr_show(struct kobject *kobj,
struct attribute *__attr, char *buf)
return count;
}
-
static MDEV_TYPE_ATTR_WO(create);
+static ssize_t device_api_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf)
+{
+ return sysfs_emit(buf, "%s\n", mtype->parent->mdev_driver->device_api);
+}
+static MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t name_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf)
+{
+ return sprintf(buf, "%s\n",
+ mtype->pretty_name ? mtype->pretty_name : mtype->sysfs_name);
+}
+
+static MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t available_instances_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr,
+ char *buf)
+{
+ struct mdev_driver *drv = mtype->parent->mdev_driver;
+
+ if (drv->get_available)
+ return sysfs_emit(buf, "%u\n", drv->get_available(mtype));
+ return sysfs_emit(buf, "%u\n",
+ atomic_read(&mtype->parent->available_instances));
+}
+static MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t description_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr,
+ char *buf)
+{
+ return mtype->parent->mdev_driver->show_description(mtype, buf);
+}
+static MDEV_TYPE_ATTR_RO(description);
+
+static struct attribute *mdev_types_core_attrs[] = {
+ &mdev_type_attr_create.attr,
+ &mdev_type_attr_device_api.attr,
+ &mdev_type_attr_name.attr,
+ &mdev_type_attr_available_instances.attr,
+ &mdev_type_attr_description.attr,
+ NULL,
+};
+
+static umode_t mdev_types_core_is_visible(struct kobject *kobj,
+ struct attribute *attr, int n)
+{
+ if (attr == &mdev_type_attr_description.attr &&
+ !to_mdev_type(kobj)->parent->mdev_driver->show_description)
+ return 0;
+ return attr->mode;
+}
+
+static struct attribute_group mdev_type_core_group = {
+ .attrs = mdev_types_core_attrs,
+ .is_visible = mdev_types_core_is_visible,
+};
+
+static const struct attribute_group *mdev_type_groups[] = {
+ &mdev_type_core_group,
+ NULL,
+};
+
static void mdev_type_release(struct kobject *kobj)
{
struct mdev_type *type = to_mdev_type(kobj);
pr_debug("Releasing group %s\n", kobj->name);
/* Pairs with the get in add_mdev_supported_type() */
- mdev_put_parent(type->parent);
- kfree(type);
+ put_device(type->parent->dev);
}
static struct kobj_type mdev_type_ktype = {
- .sysfs_ops = &mdev_type_sysfs_ops,
- .release = mdev_type_release,
+ .sysfs_ops = &mdev_type_sysfs_ops,
+ .release = mdev_type_release,
+ .default_groups = mdev_type_groups,
};
-static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent,
- unsigned int type_group_id)
+static int mdev_type_add(struct mdev_parent *parent, struct mdev_type *type)
{
- struct mdev_type *type;
- struct attribute_group *group =
- parent->mdev_driver->supported_type_groups[type_group_id];
int ret;
- if (!group->name) {
- pr_err("%s: Type name empty!\n", __func__);
- return ERR_PTR(-EINVAL);
- }
-
- type = kzalloc(sizeof(*type), GFP_KERNEL);
- if (!type)
- return ERR_PTR(-ENOMEM);
-
type->kobj.kset = parent->mdev_types_kset;
type->parent = parent;
/* Pairs with the put in mdev_type_release() */
- mdev_get_parent(parent);
- type->type_group_id = type_group_id;
+ get_device(parent->dev);
ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL,
"%s-%s", dev_driver_string(parent->dev),
- group->name);
+ type->sysfs_name);
if (ret) {
kobject_put(&type->kobj);
- return ERR_PTR(ret);
+ return ret;
}
- ret = sysfs_create_file(&type->kobj, &mdev_type_attr_create.attr);
- if (ret)
- goto attr_create_failed;
-
type->devices_kobj = kobject_create_and_add("devices", &type->kobj);
if (!type->devices_kobj) {
ret = -ENOMEM;
goto attr_devices_failed;
}
- ret = sysfs_create_files(&type->kobj,
- (const struct attribute **)group->attrs);
- if (ret) {
- ret = -ENOMEM;
- goto attrs_failed;
- }
- return type;
+ return 0;
-attrs_failed:
- kobject_put(type->devices_kobj);
attr_devices_failed:
- sysfs_remove_file(&type->kobj, &mdev_type_attr_create.attr);
-attr_create_failed:
kobject_del(&type->kobj);
kobject_put(&type->kobj);
- return ERR_PTR(ret);
+ return ret;
}
-static void remove_mdev_supported_type(struct mdev_type *type)
+static void mdev_type_remove(struct mdev_type *type)
{
- struct attribute_group *group =
- type->parent->mdev_driver->supported_type_groups[type->type_group_id];
-
- sysfs_remove_files(&type->kobj,
- (const struct attribute **)group->attrs);
kobject_put(type->devices_kobj);
- sysfs_remove_file(&type->kobj, &mdev_type_attr_create.attr);
kobject_del(&type->kobj);
kobject_put(&type->kobj);
}
-static int add_mdev_supported_type_groups(struct mdev_parent *parent)
-{
- int i;
-
- for (i = 0; parent->mdev_driver->supported_type_groups[i]; i++) {
- struct mdev_type *type;
-
- type = add_mdev_supported_type(parent, i);
- if (IS_ERR(type)) {
- struct mdev_type *ltype, *tmp;
-
- list_for_each_entry_safe(ltype, tmp, &parent->type_list,
- next) {
- list_del(<ype->next);
- remove_mdev_supported_type(ltype);
- }
- return PTR_ERR(type);
- }
- list_add(&type->next, &parent->type_list);
- }
- return 0;
-}
-
/* mdev sysfs functions */
void parent_remove_sysfs_files(struct mdev_parent *parent)
{
- struct mdev_type *type, *tmp;
-
- list_for_each_entry_safe(type, tmp, &parent->type_list, next) {
- list_del(&type->next);
- remove_mdev_supported_type(type);
- }
+ int i;
+ for (i = 0; i < parent->nr_types; i++)
+ mdev_type_remove(parent->types[i]);
kset_unregister(parent->mdev_types_kset);
}
int parent_create_sysfs_files(struct mdev_parent *parent)
{
- int ret;
+ int ret, i;
parent->mdev_types_kset = kset_create_and_add("mdev_supported_types",
NULL, &parent->dev->kobj);
-
if (!parent->mdev_types_kset)
return -ENOMEM;
- INIT_LIST_HEAD(&parent->type_list);
-
- ret = add_mdev_supported_type_groups(parent);
- if (ret)
- goto create_err;
+ for (i = 0; i < parent->nr_types; i++) {
+ ret = mdev_type_add(parent, parent->types[i]);
+ if (ret)
+ goto out_err;
+ }
return 0;
-create_err:
- kset_unregister(parent->mdev_types_kset);
- return ret;
+out_err:
+ while (--i >= 0)
+ mdev_type_remove(parent->types[i]);
+ return 0;
}
static ssize_t remove_store(struct device *dev, struct device_attribute *attr,
#include "hisi_acc_vfio_pci.h"
-/* return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */
+/* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */
static int qm_wait_dev_not_ready(struct hisi_qm *qm)
{
u32 val;
struct device *dev = &qm->pdev->dev;
int ret;
- /* check VF state */
+ /* Check VF state */
if (unlikely(hisi_qm_wait_mb_ready(qm))) {
dev_err(&qm->pdev->dev, "QM device is not ready to write\n");
return -EBUSY;
return 0;
}
-static struct hisi_acc_vf_core_device *hssi_acc_drvdata(struct pci_dev *pdev)
-{
- struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
-
- return container_of(core_device, struct hisi_acc_vf_core_device,
- core_device);
-}
-
-static void vf_qm_fun_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev,
- struct hisi_qm *qm)
+static void vf_qm_fun_reset(struct hisi_qm *qm)
{
int i;
return -EINVAL;
}
- /* vf qp num check */
+ /* VF qp num check */
ret = qm_get_vft(vf_qm, &vf_qm->qp_base);
if (ret <= 0) {
dev_err(dev, "failed to get vft qp nums\n");
vf_qm->qp_num = ret;
- /* vf isolation state check */
+ /* VF isolation state check */
ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1);
if (ret) {
dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n");
if (vf_data->que_iso_cfg != que_iso_state) {
dev_err(dev, "failed to match isolation state\n");
- return ret;
+ return -EINVAL;
}
ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1);
int ret;
vf_data->acc_magic = ACC_DEV_MAGIC;
- /* save device id */
+ /* Save device id */
vf_data->dev_id = hisi_acc_vdev->vf_dev->device;
- /* vf qp num save from PF */
+ /* VF qp num save from PF */
ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base);
if (ret <= 0) {
dev_err(dev, "failed to get vft qp nums!\n");
ret = qm_set_regs(qm, vf_data);
if (ret) {
- dev_err(dev, "Set VF regs failed\n");
+ dev_err(dev, "set VF regs failed\n");
return ret;
}
ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
if (ret) {
- dev_err(dev, "Set sqc failed\n");
+ dev_err(dev, "set sqc failed\n");
return ret;
}
ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
if (ret) {
- dev_err(dev, "Set cqc failed\n");
+ dev_err(dev, "set cqc failed\n");
return ret;
}
return -EINVAL;
/* Every reg is 32 bit, the dma address is 64 bit. */
- vf_data->eqe_dma = vf_data->qm_eqc_dw[2];
+ vf_data->eqe_dma = vf_data->qm_eqc_dw[1];
vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;
- vf_data->eqe_dma |= vf_data->qm_eqc_dw[1];
- vf_data->aeqe_dma = vf_data->qm_aeqc_dw[2];
+ vf_data->eqe_dma |= vf_data->qm_eqc_dw[0];
+ vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1];
vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;
- vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[1];
+ vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0];
/* Through SQC_BT/CQC_BT to get sqc and cqc address */
ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma);
return 0;
}
+static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev)
+{
+ struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);
+
+ return container_of(core_device, struct hisi_acc_vf_core_device,
+ core_device);
+}
+
/* Check the PF's RAS state and Function INT state */
static int
hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
if (hisi_acc_vdev->vf_qm_state != QM_READY)
return;
- vf_qm_fun_reset(hisi_acc_vdev, vf_qm);
+ /* Make sure the device is enabled */
+ qm_dev_cmd_init(vf_qm);
+
+ vf_qm_fun_reset(vf_qm);
}
static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
{
- struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev);
+ struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
if (hisi_acc_vdev->core_device.vdev.migration_flags !=
VFIO_MIGRATION_STOP_COPY)
.migration_get_state = hisi_acc_vfio_pci_get_device_state,
};
+static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev)
+{
+ struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
+ struct hisi_acc_vf_core_device, core_device.vdev);
+ struct pci_dev *pdev = to_pci_dev(core_vdev->dev);
+ struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev);
+
+ hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1;
+ hisi_acc_vdev->pf_qm = pf_qm;
+ hisi_acc_vdev->vf_dev = pdev;
+ mutex_init(&hisi_acc_vdev->state_mutex);
+
+ core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY;
+ core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops;
+
+ return vfio_pci_core_init_dev(core_vdev);
+}
+
static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = {
.name = "hisi-acc-vfio-pci-migration",
+ .init = hisi_acc_vfio_pci_migrn_init_dev,
+ .release = vfio_pci_core_release_dev,
.open_device = hisi_acc_vfio_pci_open_device,
.close_device = hisi_acc_vfio_pci_close_device,
.ioctl = hisi_acc_vfio_pci_ioctl,
static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
.name = "hisi-acc-vfio-pci",
+ .init = vfio_pci_core_init_dev,
+ .release = vfio_pci_core_release_dev,
.open_device = hisi_acc_vfio_pci_open_device,
.close_device = vfio_pci_core_close_device,
.ioctl = vfio_pci_core_ioctl,
.match = vfio_pci_core_match,
};
-static int
-hisi_acc_vfio_pci_migrn_init(struct hisi_acc_vf_core_device *hisi_acc_vdev,
- struct pci_dev *pdev, struct hisi_qm *pf_qm)
-{
- int vf_id;
-
- vf_id = pci_iov_vf_id(pdev);
- if (vf_id < 0)
- return vf_id;
-
- hisi_acc_vdev->vf_id = vf_id + 1;
- hisi_acc_vdev->core_device.vdev.migration_flags =
- VFIO_MIGRATION_STOP_COPY;
- hisi_acc_vdev->pf_qm = pf_qm;
- hisi_acc_vdev->vf_dev = pdev;
- mutex_init(&hisi_acc_vdev->state_mutex);
-
- return 0;
-}
-
static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct hisi_acc_vf_core_device *hisi_acc_vdev;
+ const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops;
struct hisi_qm *pf_qm;
+ int vf_id;
int ret;
- hisi_acc_vdev = kzalloc(sizeof(*hisi_acc_vdev), GFP_KERNEL);
- if (!hisi_acc_vdev)
- return -ENOMEM;
-
pf_qm = hisi_acc_get_pf_qm(pdev);
if (pf_qm && pf_qm->ver >= QM_HW_V3) {
- ret = hisi_acc_vfio_pci_migrn_init(hisi_acc_vdev, pdev, pf_qm);
- if (!ret) {
- vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev,
- &hisi_acc_vfio_pci_migrn_ops);
- hisi_acc_vdev->core_device.vdev.mig_ops =
- &hisi_acc_vfio_pci_migrn_state_ops;
- } else {
+ vf_id = pci_iov_vf_id(pdev);
+ if (vf_id >= 0)
+ ops = &hisi_acc_vfio_pci_migrn_ops;
+ else
pci_warn(pdev, "migration support failed, continue with generic interface\n");
- vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev,
- &hisi_acc_vfio_pci_ops);
- }
- } else {
- vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev,
- &hisi_acc_vfio_pci_ops);
}
+ hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device,
+ core_device.vdev, &pdev->dev, ops);
+ if (IS_ERR(hisi_acc_vdev))
+ return PTR_ERR(hisi_acc_vdev);
+
dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device);
ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device);
if (ret)
- goto out_free;
+ goto out_put_vdev;
return 0;
-out_free:
- vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device);
- kfree(hisi_acc_vdev);
+out_put_vdev:
+ vfio_put_device(&hisi_acc_vdev->core_device.vdev);
return ret;
}
static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)
{
- struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev);
+ struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device);
- vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device);
- kfree(hisi_acc_vdev);
+ vfio_put_device(&hisi_acc_vdev->core_device.vdev);
}
static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
#define SEC_CORE_INT_STATUS 0x301008
#define HPRE_HAC_INT_STATUS 0x301800
#define HZIP_CORE_INT_STATUS 0x3010AC
-#define QM_QUE_ISO_CFG 0x301154
#define QM_VFT_CFG_RDY 0x10006c
#define QM_VFT_CFG_OP_WR 0x100058
/* QM reserved 5 regs */
u32 qm_rsv_regs[5];
u32 padding;
- /* qm memory init information */
+ /* QM memory init information */
u64 eqe_dma;
u64 aeqe_dma;
u64 sqc_dma;
struct hisi_acc_vf_core_device {
struct vfio_pci_core_device core_device;
u8 deferred_reset:1;
- /* for migration state */
+ /* For migration state */
struct mutex state_mutex;
enum vfio_device_mig_state mig_state;
struct pci_dev *pf_dev;
struct hisi_qm vf_qm;
u32 vf_qm_state;
int vf_id;
- /* for reset handler */
+ /* For reset handler */
spinlock_t reset_lock;
struct hisi_acc_vf_migration_file *resuming_migf;
struct hisi_acc_vf_migration_file *saving_migf;
#include "cmd.h"
+enum { CQ_OK = 0, CQ_EMPTY = -1, CQ_POLL_ERR = -2 };
+
static int mlx5vf_cmd_get_vhca_id(struct mlx5_core_dev *mdev, u16 function_id,
u16 *vhca_id);
+static void
+_mlx5vf_free_page_tracker_resources(struct mlx5vf_pci_core_device *mvdev);
int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod)
{
return 0;
}
+static void set_tracker_error(struct mlx5vf_pci_core_device *mvdev)
+{
+ /* Mark the tracker under an error and wake it up if it's running */
+ mvdev->tracker.is_err = true;
+ complete(&mvdev->tracker_comp);
+}
+
static int mlx5fv_vf_event(struct notifier_block *nb,
unsigned long event, void *data)
{
struct mlx5vf_pci_core_device *mvdev =
container_of(nb, struct mlx5vf_pci_core_device, nb);
- mutex_lock(&mvdev->state_mutex);
switch (event) {
case MLX5_PF_NOTIFY_ENABLE_VF:
+ mutex_lock(&mvdev->state_mutex);
mvdev->mdev_detach = false;
+ mlx5vf_state_mutex_unlock(mvdev);
break;
case MLX5_PF_NOTIFY_DISABLE_VF:
- mlx5vf_disable_fds(mvdev);
+ mlx5vf_cmd_close_migratable(mvdev);
+ mutex_lock(&mvdev->state_mutex);
mvdev->mdev_detach = true;
+ mlx5vf_state_mutex_unlock(mvdev);
break;
default:
break;
}
- mlx5vf_state_mutex_unlock(mvdev);
+
return 0;
}
if (!mvdev->migrate_cap)
return;
+ /* Must be done outside the lock to let it progress */
+ set_tracker_error(mvdev);
mutex_lock(&mvdev->state_mutex);
mlx5vf_disable_fds(mvdev);
+ _mlx5vf_free_page_tracker_resources(mvdev);
mlx5vf_state_mutex_unlock(mvdev);
}
}
void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
- const struct vfio_migration_ops *mig_ops)
+ const struct vfio_migration_ops *mig_ops,
+ const struct vfio_log_ops *log_ops)
{
struct pci_dev *pdev = mvdev->core_device.pdev;
int ret;
VFIO_MIGRATION_STOP_COPY |
VFIO_MIGRATION_P2P;
mvdev->core_device.vdev.mig_ops = mig_ops;
+ init_completion(&mvdev->tracker_comp);
+ if (MLX5_CAP_GEN(mvdev->mdev, adv_virtualization))
+ mvdev->core_device.vdev.log_ops = log_ops;
end:
mlx5_vf_put_core_dev(mvdev->mdev);
return ret;
}
-static int _create_state_mkey(struct mlx5_core_dev *mdev, u32 pdn,
- struct mlx5_vf_migration_file *migf, u32 *mkey)
+static int _create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
+ struct mlx5_vf_migration_file *migf,
+ struct mlx5_vhca_recv_buf *recv_buf,
+ u32 *mkey)
{
- size_t npages = DIV_ROUND_UP(migf->total_length, PAGE_SIZE);
- struct sg_dma_page_iter dma_iter;
+ size_t npages = migf ? DIV_ROUND_UP(migf->total_length, PAGE_SIZE) :
+ recv_buf->npages;
int err = 0, inlen;
__be64 *mtt;
void *mkc;
DIV_ROUND_UP(npages, 2));
mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
- for_each_sgtable_dma_page(&migf->table.sgt, &dma_iter, 0)
- *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter));
+ if (migf) {
+ struct sg_dma_page_iter dma_iter;
+
+ for_each_sgtable_dma_page(&migf->table.sgt, &dma_iter, 0)
+ *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter));
+ } else {
+ int i;
+
+ for (i = 0; i < npages; i++)
+ *mtt++ = cpu_to_be64(recv_buf->dma_addrs[i]);
+ }
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
MLX5_SET(mkc, mkc, translations_octword_size, DIV_ROUND_UP(npages, 2));
- MLX5_SET64(mkc, mkc, len, migf->total_length);
+ MLX5_SET64(mkc, mkc, len,
+ migf ? migf->total_length : (npages * PAGE_SIZE));
err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
kvfree(in);
return err;
if (err)
goto err_dma_map;
- err = _create_state_mkey(mdev, pdn, migf, &mkey);
+ err = _create_mkey(mdev, pdn, migf, NULL, &mkey);
if (err)
goto err_create_mkey;
if (err)
goto err_reg;
- err = _create_state_mkey(mdev, pdn, migf, &mkey);
+ err = _create_mkey(mdev, pdn, migf, NULL, &mkey);
if (err)
goto err_mkey;
mutex_unlock(&migf->lock);
return err;
}
+
+static void combine_ranges(struct rb_root_cached *root, u32 cur_nodes,
+ u32 req_nodes)
+{
+ struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
+ unsigned long min_gap;
+ unsigned long curr_gap;
+
+ /* Special shortcut when a single range is required */
+ if (req_nodes == 1) {
+ unsigned long last;
+
+ curr = comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
+ while (curr) {
+ last = curr->last;
+ prev = curr;
+ curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
+ if (prev != comb_start)
+ interval_tree_remove(prev, root);
+ }
+ comb_start->last = last;
+ return;
+ }
+
+ /* Combine ranges which have the smallest gap */
+ while (cur_nodes > req_nodes) {
+ prev = NULL;
+ min_gap = ULONG_MAX;
+ curr = interval_tree_iter_first(root, 0, ULONG_MAX);
+ while (curr) {
+ if (prev) {
+ curr_gap = curr->start - prev->last;
+ if (curr_gap < min_gap) {
+ min_gap = curr_gap;
+ comb_start = prev;
+ comb_end = curr;
+ }
+ }
+ prev = curr;
+ curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
+ }
+ comb_start->last = comb_end->last;
+ interval_tree_remove(comb_end, root);
+ cur_nodes--;
+ }
+}
+
+static int mlx5vf_create_tracker(struct mlx5_core_dev *mdev,
+ struct mlx5vf_pci_core_device *mvdev,
+ struct rb_root_cached *ranges, u32 nnodes)
+{
+ int max_num_range =
+ MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_max_num_range);
+ struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker;
+ int record_size = MLX5_ST_SZ_BYTES(page_track_range);
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+ struct interval_tree_node *node = NULL;
+ u64 total_ranges_len = 0;
+ u32 num_ranges = nnodes;
+ u8 log_addr_space_size;
+ void *range_list_ptr;
+ void *obj_context;
+ void *cmd_hdr;
+ int inlen;
+ void *in;
+ int err;
+ int i;
+
+ if (num_ranges > max_num_range) {
+ combine_ranges(ranges, nnodes, max_num_range);
+ num_ranges = max_num_range;
+ }
+
+ inlen = MLX5_ST_SZ_BYTES(create_page_track_obj_in) +
+ record_size * num_ranges;
+ in = kzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ cmd_hdr = MLX5_ADDR_OF(create_page_track_obj_in, in,
+ general_obj_in_cmd_hdr);
+ MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode,
+ MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type,
+ MLX5_OBJ_TYPE_PAGE_TRACK);
+ obj_context = MLX5_ADDR_OF(create_page_track_obj_in, in, obj_context);
+ MLX5_SET(page_track, obj_context, vhca_id, mvdev->vhca_id);
+ MLX5_SET(page_track, obj_context, track_type, 1);
+ MLX5_SET(page_track, obj_context, log_page_size,
+ ilog2(tracker->host_qp->tracked_page_size));
+ MLX5_SET(page_track, obj_context, log_msg_size,
+ ilog2(tracker->host_qp->max_msg_size));
+ MLX5_SET(page_track, obj_context, reporting_qpn, tracker->fw_qp->qpn);
+ MLX5_SET(page_track, obj_context, num_ranges, num_ranges);
+
+ range_list_ptr = MLX5_ADDR_OF(page_track, obj_context, track_range);
+ node = interval_tree_iter_first(ranges, 0, ULONG_MAX);
+ for (i = 0; i < num_ranges; i++) {
+ void *addr_range_i_base = range_list_ptr + record_size * i;
+ unsigned long length = node->last - node->start;
+
+ MLX5_SET64(page_track_range, addr_range_i_base, start_address,
+ node->start);
+ MLX5_SET64(page_track_range, addr_range_i_base, length, length);
+ total_ranges_len += length;
+ node = interval_tree_iter_next(node, 0, ULONG_MAX);
+ }
+
+ WARN_ON(node);
+ log_addr_space_size = ilog2(total_ranges_len);
+ if (log_addr_space_size <
+ (MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_log_min_addr_space)) ||
+ log_addr_space_size >
+ (MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_log_max_addr_space))) {
+ err = -EOPNOTSUPP;
+ goto out;
+ }
+
+ MLX5_SET(page_track, obj_context, log_addr_space_size,
+ log_addr_space_size);
+ err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
+ if (err)
+ goto out;
+
+ tracker->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+out:
+ kfree(in);
+ return err;
+}
+
+static int mlx5vf_cmd_destroy_tracker(struct mlx5_core_dev *mdev,
+ u32 tracker_id)
+{
+ u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_PAGE_TRACK);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, tracker_id);
+
+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5vf_cmd_modify_tracker(struct mlx5_core_dev *mdev,
+ u32 tracker_id, unsigned long iova,
+ unsigned long length, u32 tracker_state)
+{
+ u32 in[MLX5_ST_SZ_DW(modify_page_track_obj_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+ void *obj_context;
+ void *cmd_hdr;
+
+ cmd_hdr = MLX5_ADDR_OF(modify_page_track_obj_in, in, general_obj_in_cmd_hdr);
+ MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
+ MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_PAGE_TRACK);
+ MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, tracker_id);
+
+ obj_context = MLX5_ADDR_OF(modify_page_track_obj_in, in, obj_context);
+ MLX5_SET64(page_track, obj_context, modify_field_select, 0x3);
+ MLX5_SET64(page_track, obj_context, range_start_address, iova);
+ MLX5_SET64(page_track, obj_context, length, length);
+ MLX5_SET(page_track, obj_context, state, tracker_state);
+
+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static int alloc_cq_frag_buf(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_cq_buf *buf, int nent,
+ int cqe_size)
+{
+ struct mlx5_frag_buf *frag_buf = &buf->frag_buf;
+ u8 log_wq_stride = 6 + (cqe_size == 128 ? 1 : 0);
+ u8 log_wq_sz = ilog2(cqe_size);
+ int err;
+
+ err = mlx5_frag_buf_alloc_node(mdev, nent * cqe_size, frag_buf,
+ mdev->priv.numa_node);
+ if (err)
+ return err;
+
+ mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc);
+ buf->cqe_size = cqe_size;
+ buf->nent = nent;
+ return 0;
+}
+
+static void init_cq_frag_buf(struct mlx5_vhca_cq_buf *buf)
+{
+ struct mlx5_cqe64 *cqe64;
+ void *cqe;
+ int i;
+
+ for (i = 0; i < buf->nent; i++) {
+ cqe = mlx5_frag_buf_get_wqe(&buf->fbc, i);
+ cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64;
+ cqe64->op_own = MLX5_CQE_INVALID << 4;
+ }
+}
+
+static void mlx5vf_destroy_cq(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_cq *cq)
+{
+ mlx5_core_destroy_cq(mdev, &cq->mcq);
+ mlx5_frag_buf_free(mdev, &cq->buf.frag_buf);
+ mlx5_db_free(mdev, &cq->db);
+}
+
+static void mlx5vf_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type)
+{
+ if (type != MLX5_EVENT_TYPE_CQ_ERROR)
+ return;
+
+ set_tracker_error(container_of(mcq, struct mlx5vf_pci_core_device,
+ tracker.cq.mcq));
+}
+
+static int mlx5vf_event_notifier(struct notifier_block *nb, unsigned long type,
+ void *data)
+{
+ struct mlx5_vhca_page_tracker *tracker =
+ mlx5_nb_cof(nb, struct mlx5_vhca_page_tracker, nb);
+ struct mlx5vf_pci_core_device *mvdev = container_of(
+ tracker, struct mlx5vf_pci_core_device, tracker);
+ struct mlx5_eqe *eqe = data;
+ u8 event_type = (u8)type;
+ u8 queue_type;
+ int qp_num;
+
+ switch (event_type) {
+ case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+ case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+ queue_type = eqe->data.qp_srq.type;
+ if (queue_type != MLX5_EVENT_QUEUE_TYPE_QP)
+ break;
+ qp_num = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+ if (qp_num != tracker->host_qp->qpn &&
+ qp_num != tracker->fw_qp->qpn)
+ break;
+ set_tracker_error(mvdev);
+ break;
+ default:
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
+static void mlx5vf_cq_complete(struct mlx5_core_cq *mcq,
+ struct mlx5_eqe *eqe)
+{
+ struct mlx5vf_pci_core_device *mvdev =
+ container_of(mcq, struct mlx5vf_pci_core_device,
+ tracker.cq.mcq);
+
+ complete(&mvdev->tracker_comp);
+}
+
+static int mlx5vf_create_cq(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_page_tracker *tracker,
+ size_t ncqe)
+{
+ int cqe_size = cache_line_size() == 128 ? 128 : 64;
+ u32 out[MLX5_ST_SZ_DW(create_cq_out)];
+ struct mlx5_vhca_cq *cq;
+ int inlen, err, eqn;
+ void *cqc, *in;
+ __be64 *pas;
+ int vector;
+
+ cq = &tracker->cq;
+ ncqe = roundup_pow_of_two(ncqe);
+ err = mlx5_db_alloc_node(mdev, &cq->db, mdev->priv.numa_node);
+ if (err)
+ return err;
+
+ cq->ncqe = ncqe;
+ cq->mcq.set_ci_db = cq->db.db;
+ cq->mcq.arm_db = cq->db.db + 1;
+ cq->mcq.cqe_sz = cqe_size;
+ err = alloc_cq_frag_buf(mdev, &cq->buf, ncqe, cqe_size);
+ if (err)
+ goto err_db_free;
+
+ init_cq_frag_buf(&cq->buf);
+ inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
+ MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) *
+ cq->buf.frag_buf.npages;
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in) {
+ err = -ENOMEM;
+ goto err_buff;
+ }
+
+ vector = raw_smp_processor_id() % mlx5_comp_vectors_count(mdev);
+ err = mlx5_vector2eqn(mdev, vector, &eqn);
+ if (err)
+ goto err_vec;
+
+ cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
+ MLX5_SET(cqc, cqc, log_cq_size, ilog2(ncqe));
+ MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
+ MLX5_SET(cqc, cqc, uar_page, tracker->uar->index);
+ MLX5_SET(cqc, cqc, log_page_size, cq->buf.frag_buf.page_shift -
+ MLX5_ADAPTER_PAGE_SHIFT);
+ MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
+ pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
+ mlx5_fill_page_frag_array(&cq->buf.frag_buf, pas);
+ cq->mcq.comp = mlx5vf_cq_complete;
+ cq->mcq.event = mlx5vf_cq_event;
+ err = mlx5_core_create_cq(mdev, &cq->mcq, in, inlen, out, sizeof(out));
+ if (err)
+ goto err_vec;
+
+ mlx5_cq_arm(&cq->mcq, MLX5_CQ_DB_REQ_NOT, tracker->uar->map,
+ cq->mcq.cons_index);
+ kvfree(in);
+ return 0;
+
+err_vec:
+ kvfree(in);
+err_buff:
+ mlx5_frag_buf_free(mdev, &cq->buf.frag_buf);
+err_db_free:
+ mlx5_db_free(mdev, &cq->db);
+ return err;
+}
+
+static struct mlx5_vhca_qp *
+mlx5vf_create_rc_qp(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_page_tracker *tracker, u32 max_recv_wr)
+{
+ u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
+ struct mlx5_vhca_qp *qp;
+ u8 log_rq_stride;
+ u8 log_rq_sz;
+ void *qpc;
+ int inlen;
+ void *in;
+ int err;
+
+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+ if (!qp)
+ return ERR_PTR(-ENOMEM);
+
+ qp->rq.wqe_cnt = roundup_pow_of_two(max_recv_wr);
+ log_rq_stride = ilog2(MLX5_SEND_WQE_DS);
+ log_rq_sz = ilog2(qp->rq.wqe_cnt);
+ err = mlx5_db_alloc_node(mdev, &qp->db, mdev->priv.numa_node);
+ if (err)
+ goto err_free;
+
+ if (max_recv_wr) {
+ err = mlx5_frag_buf_alloc_node(mdev,
+ wq_get_byte_sz(log_rq_sz, log_rq_stride),
+ &qp->buf, mdev->priv.numa_node);
+ if (err)
+ goto err_db_free;
+ mlx5_init_fbc(qp->buf.frags, log_rq_stride, log_rq_sz, &qp->rq.fbc);
+ }
+
+ qp->rq.db = &qp->db.db[MLX5_RCV_DBR];
+ inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
+ MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) *
+ qp->buf.npages;
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in) {
+ err = -ENOMEM;
+ goto err_in;
+ }
+
+ qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
+ MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
+ MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
+ MLX5_SET(qpc, qpc, pd, tracker->pdn);
+ MLX5_SET(qpc, qpc, uar_page, tracker->uar->index);
+ MLX5_SET(qpc, qpc, log_page_size,
+ qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
+ MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(mdev));
+ if (MLX5_CAP_GEN(mdev, cqe_version) == 1)
+ MLX5_SET(qpc, qpc, user_index, 0xFFFFFF);
+ MLX5_SET(qpc, qpc, no_sq, 1);
+ if (max_recv_wr) {
+ MLX5_SET(qpc, qpc, cqn_rcv, tracker->cq.mcq.cqn);
+ MLX5_SET(qpc, qpc, log_rq_stride, log_rq_stride - 4);
+ MLX5_SET(qpc, qpc, log_rq_size, log_rq_sz);
+ MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
+ MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
+ mlx5_fill_page_frag_array(&qp->buf,
+ (__be64 *)MLX5_ADDR_OF(create_qp_in,
+ in, pas));
+ } else {
+ MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
+ }
+
+ MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
+ err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
+ kvfree(in);
+ if (err)
+ goto err_in;
+
+ qp->qpn = MLX5_GET(create_qp_out, out, qpn);
+ return qp;
+
+err_in:
+ if (max_recv_wr)
+ mlx5_frag_buf_free(mdev, &qp->buf);
+err_db_free:
+ mlx5_db_free(mdev, &qp->db);
+err_free:
+ kfree(qp);
+ return ERR_PTR(err);
+}
+
+static void mlx5vf_post_recv(struct mlx5_vhca_qp *qp)
+{
+ struct mlx5_wqe_data_seg *data;
+ unsigned int ix;
+
+ WARN_ON(qp->rq.pc - qp->rq.cc >= qp->rq.wqe_cnt);
+ ix = qp->rq.pc & (qp->rq.wqe_cnt - 1);
+ data = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ix);
+ data->byte_count = cpu_to_be32(qp->max_msg_size);
+ data->lkey = cpu_to_be32(qp->recv_buf.mkey);
+ data->addr = cpu_to_be64(qp->recv_buf.next_rq_offset);
+ qp->rq.pc++;
+ /* Make sure that descriptors are written before doorbell record. */
+ dma_wmb();
+ *qp->rq.db = cpu_to_be32(qp->rq.pc & 0xffff);
+}
+
+static int mlx5vf_activate_qp(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_qp *qp, u32 remote_qpn,
+ bool host_qp)
+{
+ u32 init_in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {};
+ u32 rtr_in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {};
+ u32 rts_in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {};
+ void *qpc;
+ int ret;
+
+ /* Init */
+ qpc = MLX5_ADDR_OF(rst2init_qp_in, init_in, qpc);
+ MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
+ MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED);
+ MLX5_SET(qpc, qpc, rre, 1);
+ MLX5_SET(qpc, qpc, rwe, 1);
+ MLX5_SET(rst2init_qp_in, init_in, opcode, MLX5_CMD_OP_RST2INIT_QP);
+ MLX5_SET(rst2init_qp_in, init_in, qpn, qp->qpn);
+ ret = mlx5_cmd_exec_in(mdev, rst2init_qp, init_in);
+ if (ret)
+ return ret;
+
+ if (host_qp) {
+ struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf;
+ int i;
+
+ for (i = 0; i < qp->rq.wqe_cnt; i++) {
+ mlx5vf_post_recv(qp);
+ recv_buf->next_rq_offset += qp->max_msg_size;
+ }
+ }
+
+ /* RTR */
+ qpc = MLX5_ADDR_OF(init2rtr_qp_in, rtr_in, qpc);
+ MLX5_SET(init2rtr_qp_in, rtr_in, qpn, qp->qpn);
+ MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
+ MLX5_SET(qpc, qpc, log_msg_max, MLX5_CAP_GEN(mdev, log_max_msg));
+ MLX5_SET(qpc, qpc, remote_qpn, remote_qpn);
+ MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
+ MLX5_SET(qpc, qpc, primary_address_path.fl, 1);
+ MLX5_SET(qpc, qpc, min_rnr_nak, 1);
+ MLX5_SET(init2rtr_qp_in, rtr_in, opcode, MLX5_CMD_OP_INIT2RTR_QP);
+ MLX5_SET(init2rtr_qp_in, rtr_in, qpn, qp->qpn);
+ ret = mlx5_cmd_exec_in(mdev, init2rtr_qp, rtr_in);
+ if (ret || host_qp)
+ return ret;
+
+ /* RTS */
+ qpc = MLX5_ADDR_OF(rtr2rts_qp_in, rts_in, qpc);
+ MLX5_SET(rtr2rts_qp_in, rts_in, qpn, qp->qpn);
+ MLX5_SET(qpc, qpc, retry_count, 7);
+ MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */
+ MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x8); /* ~1ms */
+ MLX5_SET(rtr2rts_qp_in, rts_in, opcode, MLX5_CMD_OP_RTR2RTS_QP);
+ MLX5_SET(rtr2rts_qp_in, rts_in, qpn, qp->qpn);
+
+ return mlx5_cmd_exec_in(mdev, rtr2rts_qp, rts_in);
+}
+
+static void mlx5vf_destroy_qp(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_qp *qp)
+{
+ u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
+
+ MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
+ MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);
+ mlx5_cmd_exec_in(mdev, destroy_qp, in);
+
+ mlx5_frag_buf_free(mdev, &qp->buf);
+ mlx5_db_free(mdev, &qp->db);
+ kfree(qp);
+}
+
+static void free_recv_pages(struct mlx5_vhca_recv_buf *recv_buf)
+{
+ int i;
+
+ /* Undo alloc_pages_bulk_array() */
+ for (i = 0; i < recv_buf->npages; i++)
+ __free_page(recv_buf->page_list[i]);
+
+ kvfree(recv_buf->page_list);
+}
+
+static int alloc_recv_pages(struct mlx5_vhca_recv_buf *recv_buf,
+ unsigned int npages)
+{
+ unsigned int filled = 0, done = 0;
+ int i;
+
+ recv_buf->page_list = kvcalloc(npages, sizeof(*recv_buf->page_list),
+ GFP_KERNEL);
+ if (!recv_buf->page_list)
+ return -ENOMEM;
+
+ for (;;) {
+ filled = alloc_pages_bulk_array(GFP_KERNEL, npages - done,
+ recv_buf->page_list + done);
+ if (!filled)
+ goto err;
+
+ done += filled;
+ if (done == npages)
+ break;
+ }
+
+ recv_buf->npages = npages;
+ return 0;
+
+err:
+ for (i = 0; i < npages; i++) {
+ if (recv_buf->page_list[i])
+ __free_page(recv_buf->page_list[i]);
+ }
+
+ kvfree(recv_buf->page_list);
+ return -ENOMEM;
+}
+
+static int register_dma_recv_pages(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_recv_buf *recv_buf)
+{
+ int i, j;
+
+ recv_buf->dma_addrs = kvcalloc(recv_buf->npages,
+ sizeof(*recv_buf->dma_addrs),
+ GFP_KERNEL);
+ if (!recv_buf->dma_addrs)
+ return -ENOMEM;
+
+ for (i = 0; i < recv_buf->npages; i++) {
+ recv_buf->dma_addrs[i] = dma_map_page(mdev->device,
+ recv_buf->page_list[i],
+ 0, PAGE_SIZE,
+ DMA_FROM_DEVICE);
+ if (dma_mapping_error(mdev->device, recv_buf->dma_addrs[i]))
+ goto error;
+ }
+ return 0;
+
+error:
+ for (j = 0; j < i; j++)
+ dma_unmap_single(mdev->device, recv_buf->dma_addrs[j],
+ PAGE_SIZE, DMA_FROM_DEVICE);
+
+ kvfree(recv_buf->dma_addrs);
+ return -ENOMEM;
+}
+
+static void unregister_dma_recv_pages(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_recv_buf *recv_buf)
+{
+ int i;
+
+ for (i = 0; i < recv_buf->npages; i++)
+ dma_unmap_single(mdev->device, recv_buf->dma_addrs[i],
+ PAGE_SIZE, DMA_FROM_DEVICE);
+
+ kvfree(recv_buf->dma_addrs);
+}
+
+static void mlx5vf_free_qp_recv_resources(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_qp *qp)
+{
+ struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf;
+
+ mlx5_core_destroy_mkey(mdev, recv_buf->mkey);
+ unregister_dma_recv_pages(mdev, recv_buf);
+ free_recv_pages(&qp->recv_buf);
+}
+
+static int mlx5vf_alloc_qp_recv_resources(struct mlx5_core_dev *mdev,
+ struct mlx5_vhca_qp *qp, u32 pdn,
+ u64 rq_size)
+{
+ unsigned int npages = DIV_ROUND_UP_ULL(rq_size, PAGE_SIZE);
+ struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf;
+ int err;
+
+ err = alloc_recv_pages(recv_buf, npages);
+ if (err < 0)
+ return err;
+
+ err = register_dma_recv_pages(mdev, recv_buf);
+ if (err)
+ goto end;
+
+ err = _create_mkey(mdev, pdn, NULL, recv_buf, &recv_buf->mkey);
+ if (err)
+ goto err_create_mkey;
+
+ return 0;
+
+err_create_mkey:
+ unregister_dma_recv_pages(mdev, recv_buf);
+end:
+ free_recv_pages(recv_buf);
+ return err;
+}
+
+static void
+_mlx5vf_free_page_tracker_resources(struct mlx5vf_pci_core_device *mvdev)
+{
+ struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker;
+ struct mlx5_core_dev *mdev = mvdev->mdev;
+
+ lockdep_assert_held(&mvdev->state_mutex);
+
+ if (!mvdev->log_active)
+ return;
+
+ WARN_ON(mvdev->mdev_detach);
+
+ mlx5_eq_notifier_unregister(mdev, &tracker->nb);
+ mlx5vf_cmd_destroy_tracker(mdev, tracker->id);
+ mlx5vf_destroy_qp(mdev, tracker->fw_qp);
+ mlx5vf_free_qp_recv_resources(mdev, tracker->host_qp);
+ mlx5vf_destroy_qp(mdev, tracker->host_qp);
+ mlx5vf_destroy_cq(mdev, &tracker->cq);
+ mlx5_core_dealloc_pd(mdev, tracker->pdn);
+ mlx5_put_uars_page(mdev, tracker->uar);
+ mvdev->log_active = false;
+}
+
+int mlx5vf_stop_page_tracker(struct vfio_device *vdev)
+{
+ struct mlx5vf_pci_core_device *mvdev = container_of(
+ vdev, struct mlx5vf_pci_core_device, core_device.vdev);
+
+ mutex_lock(&mvdev->state_mutex);
+ if (!mvdev->log_active)
+ goto end;
+
+ _mlx5vf_free_page_tracker_resources(mvdev);
+ mvdev->log_active = false;
+end:
+ mlx5vf_state_mutex_unlock(mvdev);
+ return 0;
+}
+
+int mlx5vf_start_page_tracker(struct vfio_device *vdev,
+ struct rb_root_cached *ranges, u32 nnodes,
+ u64 *page_size)
+{
+ struct mlx5vf_pci_core_device *mvdev = container_of(
+ vdev, struct mlx5vf_pci_core_device, core_device.vdev);
+ struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker;
+ u8 log_tracked_page = ilog2(*page_size);
+ struct mlx5_vhca_qp *host_qp;
+ struct mlx5_vhca_qp *fw_qp;
+ struct mlx5_core_dev *mdev;
+ u32 max_msg_size = PAGE_SIZE;
+ u64 rq_size = SZ_2M;
+ u32 max_recv_wr;
+ int err;
+
+ mutex_lock(&mvdev->state_mutex);
+ if (mvdev->mdev_detach) {
+ err = -ENOTCONN;
+ goto end;
+ }
+
+ if (mvdev->log_active) {
+ err = -EINVAL;
+ goto end;
+ }
+
+ mdev = mvdev->mdev;
+ memset(tracker, 0, sizeof(*tracker));
+ tracker->uar = mlx5_get_uars_page(mdev);
+ if (IS_ERR(tracker->uar)) {
+ err = PTR_ERR(tracker->uar);
+ goto end;
+ }
+
+ err = mlx5_core_alloc_pd(mdev, &tracker->pdn);
+ if (err)
+ goto err_uar;
+
+ max_recv_wr = DIV_ROUND_UP_ULL(rq_size, max_msg_size);
+ err = mlx5vf_create_cq(mdev, tracker, max_recv_wr);
+ if (err)
+ goto err_dealloc_pd;
+
+ host_qp = mlx5vf_create_rc_qp(mdev, tracker, max_recv_wr);
+ if (IS_ERR(host_qp)) {
+ err = PTR_ERR(host_qp);
+ goto err_cq;
+ }
+
+ host_qp->max_msg_size = max_msg_size;
+ if (log_tracked_page < MLX5_CAP_ADV_VIRTUALIZATION(mdev,
+ pg_track_log_min_page_size)) {
+ log_tracked_page = MLX5_CAP_ADV_VIRTUALIZATION(mdev,
+ pg_track_log_min_page_size);
+ } else if (log_tracked_page > MLX5_CAP_ADV_VIRTUALIZATION(mdev,
+ pg_track_log_max_page_size)) {
+ log_tracked_page = MLX5_CAP_ADV_VIRTUALIZATION(mdev,
+ pg_track_log_max_page_size);
+ }
+
+ host_qp->tracked_page_size = (1ULL << log_tracked_page);
+ err = mlx5vf_alloc_qp_recv_resources(mdev, host_qp, tracker->pdn,
+ rq_size);
+ if (err)
+ goto err_host_qp;
+
+ fw_qp = mlx5vf_create_rc_qp(mdev, tracker, 0);
+ if (IS_ERR(fw_qp)) {
+ err = PTR_ERR(fw_qp);
+ goto err_recv_resources;
+ }
+
+ err = mlx5vf_activate_qp(mdev, host_qp, fw_qp->qpn, true);
+ if (err)
+ goto err_activate;
+
+ err = mlx5vf_activate_qp(mdev, fw_qp, host_qp->qpn, false);
+ if (err)
+ goto err_activate;
+
+ tracker->host_qp = host_qp;
+ tracker->fw_qp = fw_qp;
+ err = mlx5vf_create_tracker(mdev, mvdev, ranges, nnodes);
+ if (err)
+ goto err_activate;
+
+ MLX5_NB_INIT(&tracker->nb, mlx5vf_event_notifier, NOTIFY_ANY);
+ mlx5_eq_notifier_register(mdev, &tracker->nb);
+ *page_size = host_qp->tracked_page_size;
+ mvdev->log_active = true;
+ mlx5vf_state_mutex_unlock(mvdev);
+ return 0;
+
+err_activate:
+ mlx5vf_destroy_qp(mdev, fw_qp);
+err_recv_resources:
+ mlx5vf_free_qp_recv_resources(mdev, host_qp);
+err_host_qp:
+ mlx5vf_destroy_qp(mdev, host_qp);
+err_cq:
+ mlx5vf_destroy_cq(mdev, &tracker->cq);
+err_dealloc_pd:
+ mlx5_core_dealloc_pd(mdev, tracker->pdn);
+err_uar:
+ mlx5_put_uars_page(mdev, tracker->uar);
+end:
+ mlx5vf_state_mutex_unlock(mvdev);
+ return err;
+}
+
+static void
+set_report_output(u32 size, int index, struct mlx5_vhca_qp *qp,
+ struct iova_bitmap *dirty)
+{
+ u32 entry_size = MLX5_ST_SZ_BYTES(page_track_report_entry);
+ u32 nent = size / entry_size;
+ struct page *page;
+ u64 addr;
+ u64 *buf;
+ int i;
+
+ if (WARN_ON(index >= qp->recv_buf.npages ||
+ (nent > qp->max_msg_size / entry_size)))
+ return;
+
+ page = qp->recv_buf.page_list[index];
+ buf = kmap_local_page(page);
+ for (i = 0; i < nent; i++) {
+ addr = MLX5_GET(page_track_report_entry, buf + i,
+ dirty_address_low);
+ addr |= (u64)MLX5_GET(page_track_report_entry, buf + i,
+ dirty_address_high) << 32;
+ iova_bitmap_set(dirty, addr, qp->tracked_page_size);
+ }
+ kunmap_local(buf);
+}
+
+static void
+mlx5vf_rq_cqe(struct mlx5_vhca_qp *qp, struct mlx5_cqe64 *cqe,
+ struct iova_bitmap *dirty, int *tracker_status)
+{
+ u32 size;
+ int ix;
+
+ qp->rq.cc++;
+ *tracker_status = be32_to_cpu(cqe->immediate) >> 28;
+ size = be32_to_cpu(cqe->byte_cnt);
+ ix = be16_to_cpu(cqe->wqe_counter) & (qp->rq.wqe_cnt - 1);
+
+ /* zero length CQE, no data */
+ WARN_ON(!size && *tracker_status == MLX5_PAGE_TRACK_STATE_REPORTING);
+ if (size)
+ set_report_output(size, ix, qp, dirty);
+
+ qp->recv_buf.next_rq_offset = ix * qp->max_msg_size;
+ mlx5vf_post_recv(qp);
+}
+
+static void *get_cqe(struct mlx5_vhca_cq *cq, int n)
+{
+ return mlx5_frag_buf_get_wqe(&cq->buf.fbc, n);
+}
+
+static struct mlx5_cqe64 *get_sw_cqe(struct mlx5_vhca_cq *cq, int n)
+{
+ void *cqe = get_cqe(cq, n & (cq->ncqe - 1));
+ struct mlx5_cqe64 *cqe64;
+
+ cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
+
+ if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) &&
+ !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ncqe)))) {
+ return cqe64;
+ } else {
+ return NULL;
+ }
+}
+
+static int
+mlx5vf_cq_poll_one(struct mlx5_vhca_cq *cq, struct mlx5_vhca_qp *qp,
+ struct iova_bitmap *dirty, int *tracker_status)
+{
+ struct mlx5_cqe64 *cqe;
+ u8 opcode;
+
+ cqe = get_sw_cqe(cq, cq->mcq.cons_index);
+ if (!cqe)
+ return CQ_EMPTY;
+
+ ++cq->mcq.cons_index;
+ /*
+ * Make sure we read CQ entry contents after we've checked the
+ * ownership bit.
+ */
+ rmb();
+ opcode = get_cqe_opcode(cqe);
+ switch (opcode) {
+ case MLX5_CQE_RESP_SEND_IMM:
+ mlx5vf_rq_cqe(qp, cqe, dirty, tracker_status);
+ return CQ_OK;
+ default:
+ return CQ_POLL_ERR;
+ }
+}
+
+int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
+ unsigned long length,
+ struct iova_bitmap *dirty)
+{
+ struct mlx5vf_pci_core_device *mvdev = container_of(
+ vdev, struct mlx5vf_pci_core_device, core_device.vdev);
+ struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker;
+ struct mlx5_vhca_cq *cq = &tracker->cq;
+ struct mlx5_core_dev *mdev;
+ int poll_err, err;
+
+ mutex_lock(&mvdev->state_mutex);
+ if (!mvdev->log_active) {
+ err = -EINVAL;
+ goto end;
+ }
+
+ if (mvdev->mdev_detach) {
+ err = -ENOTCONN;
+ goto end;
+ }
+
+ mdev = mvdev->mdev;
+ err = mlx5vf_cmd_modify_tracker(mdev, tracker->id, iova, length,
+ MLX5_PAGE_TRACK_STATE_REPORTING);
+ if (err)
+ goto end;
+
+ tracker->status = MLX5_PAGE_TRACK_STATE_REPORTING;
+ while (tracker->status == MLX5_PAGE_TRACK_STATE_REPORTING &&
+ !tracker->is_err) {
+ poll_err = mlx5vf_cq_poll_one(cq, tracker->host_qp, dirty,
+ &tracker->status);
+ if (poll_err == CQ_EMPTY) {
+ mlx5_cq_arm(&cq->mcq, MLX5_CQ_DB_REQ_NOT, tracker->uar->map,
+ cq->mcq.cons_index);
+ poll_err = mlx5vf_cq_poll_one(cq, tracker->host_qp,
+ dirty, &tracker->status);
+ if (poll_err == CQ_EMPTY) {
+ wait_for_completion(&mvdev->tracker_comp);
+ continue;
+ }
+ }
+ if (poll_err == CQ_POLL_ERR) {
+ err = -EIO;
+ goto end;
+ }
+ mlx5_cq_set_ci(&cq->mcq);
+ }
+
+ if (tracker->status == MLX5_PAGE_TRACK_STATE_ERROR)
+ tracker->is_err = true;
+
+ if (tracker->is_err)
+ err = -EIO;
+end:
+ mlx5vf_state_mutex_unlock(mvdev);
+ return err;
+}
#include <linux/kernel.h>
#include <linux/vfio_pci_core.h>
#include <linux/mlx5/driver.h>
+#include <linux/mlx5/cq.h>
+#include <linux/mlx5/qp.h>
struct mlx5vf_async_data {
struct mlx5_async_work cb_work;
struct mlx5vf_async_data async_data;
};
+struct mlx5_vhca_cq_buf {
+ struct mlx5_frag_buf_ctrl fbc;
+ struct mlx5_frag_buf frag_buf;
+ int cqe_size;
+ int nent;
+};
+
+struct mlx5_vhca_cq {
+ struct mlx5_vhca_cq_buf buf;
+ struct mlx5_db db;
+ struct mlx5_core_cq mcq;
+ size_t ncqe;
+};
+
+struct mlx5_vhca_recv_buf {
+ u32 npages;
+ struct page **page_list;
+ dma_addr_t *dma_addrs;
+ u32 next_rq_offset;
+ u32 mkey;
+};
+
+struct mlx5_vhca_qp {
+ struct mlx5_frag_buf buf;
+ struct mlx5_db db;
+ struct mlx5_vhca_recv_buf recv_buf;
+ u32 tracked_page_size;
+ u32 max_msg_size;
+ u32 qpn;
+ struct {
+ unsigned int pc;
+ unsigned int cc;
+ unsigned int wqe_cnt;
+ __be32 *db;
+ struct mlx5_frag_buf_ctrl fbc;
+ } rq;
+};
+
+struct mlx5_vhca_page_tracker {
+ u32 id;
+ u32 pdn;
+ u8 is_err:1;
+ struct mlx5_uars_page *uar;
+ struct mlx5_vhca_cq cq;
+ struct mlx5_vhca_qp *host_qp;
+ struct mlx5_vhca_qp *fw_qp;
+ struct mlx5_nb nb;
+ int status;
+};
+
struct mlx5vf_pci_core_device {
struct vfio_pci_core_device core_device;
int vf_id;
u8 migrate_cap:1;
u8 deferred_reset:1;
u8 mdev_detach:1;
+ u8 log_active:1;
+ struct completion tracker_comp;
/* protect migration state */
struct mutex state_mutex;
enum vfio_device_mig_state mig_state;
spinlock_t reset_lock;
struct mlx5_vf_migration_file *resuming_migf;
struct mlx5_vf_migration_file *saving_migf;
+ struct mlx5_vhca_page_tracker tracker;
struct workqueue_struct *cb_wq;
struct notifier_block nb;
struct mlx5_core_dev *mdev;
int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
size_t *state_size);
void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
- const struct vfio_migration_ops *mig_ops);
+ const struct vfio_migration_ops *mig_ops,
+ const struct vfio_log_ops *log_ops);
void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);
void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);
int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev);
void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
+int mlx5vf_start_page_tracker(struct vfio_device *vdev,
+ struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
+int mlx5vf_stop_page_tracker(struct vfio_device *vdev);
+int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
+ unsigned long length, struct iova_bitmap *dirty);
#endif /* MLX5_VFIO_CMD_H */
.migration_get_state = mlx5vf_pci_get_device_state,
};
+static const struct vfio_log_ops mlx5vf_pci_log_ops = {
+ .log_start = mlx5vf_start_page_tracker,
+ .log_stop = mlx5vf_stop_page_tracker,
+ .log_read_and_clear = mlx5vf_tracker_read_and_clear,
+};
+
+static int mlx5vf_pci_init_dev(struct vfio_device *core_vdev)
+{
+ struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev,
+ struct mlx5vf_pci_core_device, core_device.vdev);
+ int ret;
+
+ ret = vfio_pci_core_init_dev(core_vdev);
+ if (ret)
+ return ret;
+
+ mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops,
+ &mlx5vf_pci_log_ops);
+
+ return 0;
+}
+
+static void mlx5vf_pci_release_dev(struct vfio_device *core_vdev)
+{
+ struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev,
+ struct mlx5vf_pci_core_device, core_device.vdev);
+
+ mlx5vf_cmd_remove_migratable(mvdev);
+ vfio_pci_core_release_dev(core_vdev);
+}
+
static const struct vfio_device_ops mlx5vf_pci_ops = {
.name = "mlx5-vfio-pci",
+ .init = mlx5vf_pci_init_dev,
+ .release = mlx5vf_pci_release_dev,
.open_device = mlx5vf_pci_open_device,
.close_device = mlx5vf_pci_close_device,
.ioctl = vfio_pci_core_ioctl,
struct mlx5vf_pci_core_device *mvdev;
int ret;
- mvdev = kzalloc(sizeof(*mvdev), GFP_KERNEL);
- if (!mvdev)
- return -ENOMEM;
- vfio_pci_core_init_device(&mvdev->core_device, pdev, &mlx5vf_pci_ops);
- mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops);
+ mvdev = vfio_alloc_device(mlx5vf_pci_core_device, core_device.vdev,
+ &pdev->dev, &mlx5vf_pci_ops);
+ if (IS_ERR(mvdev))
+ return PTR_ERR(mvdev);
+
dev_set_drvdata(&pdev->dev, &mvdev->core_device);
ret = vfio_pci_core_register_device(&mvdev->core_device);
if (ret)
- goto out_free;
+ goto out_put_vdev;
return 0;
-out_free:
- mlx5vf_cmd_remove_migratable(mvdev);
- vfio_pci_core_uninit_device(&mvdev->core_device);
- kfree(mvdev);
+out_put_vdev:
+ vfio_put_device(&mvdev->core_device.vdev);
return ret;
}
struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);
vfio_pci_core_unregister_device(&mvdev->core_device);
- mlx5vf_cmd_remove_migratable(mvdev);
- vfio_pci_core_uninit_device(&mvdev->core_device);
- kfree(mvdev);
+ vfio_put_device(&mvdev->core_device.vdev);
}
static const struct pci_device_id mlx5vf_pci_table[] = {
#include <linux/types.h>
#include <linux/uaccess.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
#define DRIVER_DESC "VFIO PCI - User Level meta-driver"
static const struct vfio_device_ops vfio_pci_ops = {
.name = "vfio-pci",
+ .init = vfio_pci_core_init_dev,
+ .release = vfio_pci_core_release_dev,
.open_device = vfio_pci_open_device,
.close_device = vfio_pci_core_close_device,
.ioctl = vfio_pci_core_ioctl,
if (vfio_pci_is_denylisted(pdev))
return -EINVAL;
- vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
- if (!vdev)
- return -ENOMEM;
- vfio_pci_core_init_device(vdev, pdev, &vfio_pci_ops);
+ vdev = vfio_alloc_device(vfio_pci_core_device, vdev, &pdev->dev,
+ &vfio_pci_ops);
+ if (IS_ERR(vdev))
+ return PTR_ERR(vdev);
dev_set_drvdata(&pdev->dev, vdev);
ret = vfio_pci_core_register_device(vdev);
if (ret)
- goto out_free;
+ goto out_put_vdev;
return 0;
-out_free:
- vfio_pci_core_uninit_device(vdev);
- kfree(vdev);
+out_put_vdev:
+ vfio_put_device(&vdev->vdev);
return ret;
}
struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
vfio_pci_core_unregister_device(vdev);
- vfio_pci_core_uninit_device(vdev);
- kfree(vdev);
+ vfio_put_device(&vdev->vdev);
}
static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
#include <linux/vfio.h>
#include <linux/slab.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
/* Fake capability ID for standard config space */
#define PCI_CAP_ID_BASIC 0
flags = le16_to_cpu(*pflags);
/* MSI is enabled via ioctl */
- if (!is_msi(vdev))
+ if (vdev->irq_type != VFIO_PCI_MSI_IRQ_INDEX)
flags &= ~PCI_MSI_FLAGS_ENABLE;
/* Check queue size */
#include <linux/nospec.h>
#include <linux/sched/mm.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
#define DRIVER_DESC "core driver for VFIO based PCI devices"
static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex);
static LIST_HEAD(vfio_pci_sriov_pfs);
+struct vfio_pci_dummy_resource {
+ struct resource resource;
+ int index;
+ struct list_head res_next;
+};
+
+struct vfio_pci_vf_token {
+ struct mutex lock;
+ uuid_t uuid;
+ int users;
+};
+
+struct vfio_pci_mmap_vma {
+ struct vm_area_struct *vma;
+ struct list_head vma_next;
+};
+
static inline bool vfio_vga_disabled(void)
{
#ifdef CONFIG_VFIO_PCI_VGA
return ret;
}
+static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,
+ struct eventfd_ctx *efdctx)
+{
+ /*
+ * The vdev power related flags are protected with 'memory_lock'
+ * semaphore.
+ */
+ vfio_pci_zap_and_down_write_memory_lock(vdev);
+ if (vdev->pm_runtime_engaged) {
+ up_write(&vdev->memory_lock);
+ return -EINVAL;
+ }
+
+ vdev->pm_runtime_engaged = true;
+ vdev->pm_wake_eventfd_ctx = efdctx;
+ pm_runtime_put_noidle(&vdev->pdev->dev);
+ up_write(&vdev->memory_lock);
+
+ return 0;
+}
+
+static int vfio_pci_core_pm_entry(struct vfio_device *device, u32 flags,
+ void __user *arg, size_t argsz)
+{
+ struct vfio_pci_core_device *vdev =
+ container_of(device, struct vfio_pci_core_device, vdev);
+ int ret;
+
+ ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
+ if (ret != 1)
+ return ret;
+
+ /*
+ * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count
+ * will be decremented. The pm_runtime_put() will be invoked again
+ * while returning from the ioctl and then the device can go into
+ * runtime suspended state.
+ */
+ return vfio_pci_runtime_pm_entry(vdev, NULL);
+}
+
+static int vfio_pci_core_pm_entry_with_wakeup(
+ struct vfio_device *device, u32 flags,
+ struct vfio_device_low_power_entry_with_wakeup __user *arg,
+ size_t argsz)
+{
+ struct vfio_pci_core_device *vdev =
+ container_of(device, struct vfio_pci_core_device, vdev);
+ struct vfio_device_low_power_entry_with_wakeup entry;
+ struct eventfd_ctx *efdctx;
+ int ret;
+
+ ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
+ sizeof(entry));
+ if (ret != 1)
+ return ret;
+
+ if (copy_from_user(&entry, arg, sizeof(entry)))
+ return -EFAULT;
+
+ if (entry.wakeup_eventfd < 0)
+ return -EINVAL;
+
+ efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd);
+ if (IS_ERR(efdctx))
+ return PTR_ERR(efdctx);
+
+ ret = vfio_pci_runtime_pm_entry(vdev, efdctx);
+ if (ret)
+ eventfd_ctx_put(efdctx);
+
+ return ret;
+}
+
+static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
+{
+ if (vdev->pm_runtime_engaged) {
+ vdev->pm_runtime_engaged = false;
+ pm_runtime_get_noresume(&vdev->pdev->dev);
+
+ if (vdev->pm_wake_eventfd_ctx) {
+ eventfd_ctx_put(vdev->pm_wake_eventfd_ctx);
+ vdev->pm_wake_eventfd_ctx = NULL;
+ }
+ }
+}
+
+static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
+{
+ /*
+ * The vdev power related flags are protected with 'memory_lock'
+ * semaphore.
+ */
+ down_write(&vdev->memory_lock);
+ __vfio_pci_runtime_pm_exit(vdev);
+ up_write(&vdev->memory_lock);
+}
+
+static int vfio_pci_core_pm_exit(struct vfio_device *device, u32 flags,
+ void __user *arg, size_t argsz)
+{
+ struct vfio_pci_core_device *vdev =
+ container_of(device, struct vfio_pci_core_device, vdev);
+ int ret;
+
+ ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
+ if (ret != 1)
+ return ret;
+
+ /*
+ * The device is always in the active state here due to pm wrappers
+ * around ioctls. If the device had entered a low power state and
+ * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has
+ * already signaled the eventfd and exited low power mode itself.
+ * pm_runtime_engaged protects the redundant call here.
+ */
+ vfio_pci_runtime_pm_exit(vdev);
+ return 0;
+}
+
+#ifdef CONFIG_PM
+static int vfio_pci_core_runtime_suspend(struct device *dev)
+{
+ struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
+
+ down_write(&vdev->memory_lock);
+ /*
+ * The user can move the device into D3hot state before invoking
+ * power management IOCTL. Move the device into D0 state here and then
+ * the pci-driver core runtime PM suspend function will move the device
+ * into the low power state. Also, for the devices which have
+ * NoSoftRst-, it will help in restoring the original state
+ * (saved locally in 'vdev->pm_save').
+ */
+ vfio_pci_set_power_state(vdev, PCI_D0);
+ up_write(&vdev->memory_lock);
+
+ /*
+ * If INTx is enabled, then mask INTx before going into the runtime
+ * suspended state and unmask the same in the runtime resume.
+ * If INTx has already been masked by the user, then
+ * vfio_pci_intx_mask() will return false and in that case, INTx
+ * should not be unmasked in the runtime resume.
+ */
+ vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) &&
+ vfio_pci_intx_mask(vdev));
+
+ return 0;
+}
+
+static int vfio_pci_core_runtime_resume(struct device *dev)
+{
+ struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
+
+ /*
+ * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit
+ * low power mode.
+ */
+ down_write(&vdev->memory_lock);
+ if (vdev->pm_wake_eventfd_ctx) {
+ eventfd_signal(vdev->pm_wake_eventfd_ctx, 1);
+ __vfio_pci_runtime_pm_exit(vdev);
+ }
+ up_write(&vdev->memory_lock);
+
+ if (vdev->pm_intx_masked)
+ vfio_pci_intx_unmask(vdev);
+
+ return 0;
+}
+#endif /* CONFIG_PM */
+
/*
- * The dev_pm_ops needs to be provided to make pci-driver runtime PM working,
- * so use structure without any callbacks.
- *
* The pci-driver core runtime PM routines always save the device state
* before going into suspended state. If the device is going into low power
* state with only with runtime PM ops, then no explicit handling is needed
* for the devices which have NoSoftRst-.
*/
-static const struct dev_pm_ops vfio_pci_core_pm_ops = { };
+static const struct dev_pm_ops vfio_pci_core_pm_ops = {
+ SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend,
+ vfio_pci_core_runtime_resume,
+ NULL)
+};
int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
{
/*
* This function can be invoked while the power state is non-D0.
+ * This non-D0 power state can be with or without runtime PM.
+ * vfio_pci_runtime_pm_exit() will internally increment the usage
+ * count corresponding to pm_runtime_put() called during low power
+ * feature entry and then pm_runtime_resume() will wake up the device,
+ * if the device has already gone into the suspended state. Otherwise,
+ * the vfio_pci_set_power_state() will change the device power state
+ * to D0.
+ */
+ vfio_pci_runtime_pm_exit(vdev);
+ pm_runtime_resume(&pdev->dev);
+
+ /*
* This function calls __pci_reset_function_locked() which internally
* can use pci_pm_reset() for the function reset. pci_pm_reset() will
* fail if the power state is non-D0. Also, for the devices which
return vfio_info_add_capability(caps, &header, sizeof(header));
}
-int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev,
- unsigned int type, unsigned int subtype,
- const struct vfio_pci_regops *ops,
- size_t size, u32 flags, void *data)
+int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
+ unsigned int type, unsigned int subtype,
+ const struct vfio_pci_regops *ops,
+ size_t size, u32 flags, void *data)
{
struct vfio_pci_region *region;
return 0;
}
-EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region);
+EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region);
-long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
- unsigned long arg)
+static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
+ struct vfio_device_info __user *arg)
{
- struct vfio_pci_core_device *vdev =
- container_of(core_vdev, struct vfio_pci_core_device, vdev);
- unsigned long minsz;
-
- if (cmd == VFIO_DEVICE_GET_INFO) {
- struct vfio_device_info info;
- struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
- unsigned long capsz;
- int ret;
-
- minsz = offsetofend(struct vfio_device_info, num_irqs);
+ unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs);
+ struct vfio_device_info info;
+ struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+ unsigned long capsz;
+ int ret;
- /* For backward compatibility, cannot require this */
- capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset);
+ /* For backward compatibility, cannot require this */
+ capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset);
- if (copy_from_user(&info, (void __user *)arg, minsz))
- return -EFAULT;
+ if (copy_from_user(&info, arg, minsz))
+ return -EFAULT;
- if (info.argsz < minsz)
- return -EINVAL;
+ if (info.argsz < minsz)
+ return -EINVAL;
- if (info.argsz >= capsz) {
- minsz = capsz;
- info.cap_offset = 0;
- }
+ if (info.argsz >= capsz) {
+ minsz = capsz;
+ info.cap_offset = 0;
+ }
- info.flags = VFIO_DEVICE_FLAGS_PCI;
+ info.flags = VFIO_DEVICE_FLAGS_PCI;
- if (vdev->reset_works)
- info.flags |= VFIO_DEVICE_FLAGS_RESET;
+ if (vdev->reset_works)
+ info.flags |= VFIO_DEVICE_FLAGS_RESET;
- info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
- info.num_irqs = VFIO_PCI_NUM_IRQS;
+ info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
+ info.num_irqs = VFIO_PCI_NUM_IRQS;
- ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
- if (ret && ret != -ENODEV) {
- pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
- return ret;
- }
+ ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
+ if (ret && ret != -ENODEV) {
+ pci_warn(vdev->pdev,
+ "Failed to setup zPCI info capabilities\n");
+ return ret;
+ }
- if (caps.size) {
- info.flags |= VFIO_DEVICE_FLAGS_CAPS;
- if (info.argsz < sizeof(info) + caps.size) {
- info.argsz = sizeof(info) + caps.size;
- } else {
- vfio_info_cap_shift(&caps, sizeof(info));
- if (copy_to_user((void __user *)arg +
- sizeof(info), caps.buf,
- caps.size)) {
- kfree(caps.buf);
- return -EFAULT;
- }
- info.cap_offset = sizeof(info);
+ if (caps.size) {
+ info.flags |= VFIO_DEVICE_FLAGS_CAPS;
+ if (info.argsz < sizeof(info) + caps.size) {
+ info.argsz = sizeof(info) + caps.size;
+ } else {
+ vfio_info_cap_shift(&caps, sizeof(info));
+ if (copy_to_user(arg + 1, caps.buf, caps.size)) {
+ kfree(caps.buf);
+ return -EFAULT;
}
-
- kfree(caps.buf);
+ info.cap_offset = sizeof(*arg);
}
- return copy_to_user((void __user *)arg, &info, minsz) ?
- -EFAULT : 0;
+ kfree(caps.buf);
+ }
- } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
- struct pci_dev *pdev = vdev->pdev;
- struct vfio_region_info info;
- struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
- int i, ret;
+ return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
+}
- minsz = offsetofend(struct vfio_region_info, offset);
+static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev,
+ struct vfio_region_info __user *arg)
+{
+ unsigned long minsz = offsetofend(struct vfio_region_info, offset);
+ struct pci_dev *pdev = vdev->pdev;
+ struct vfio_region_info info;
+ struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+ int i, ret;
- if (copy_from_user(&info, (void __user *)arg, minsz))
- return -EFAULT;
+ if (copy_from_user(&info, arg, minsz))
+ return -EFAULT;
- if (info.argsz < minsz)
- return -EINVAL;
+ if (info.argsz < minsz)
+ return -EINVAL;
- switch (info.index) {
- case VFIO_PCI_CONFIG_REGION_INDEX:
- info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.size = pdev->cfg_size;
- info.flags = VFIO_REGION_INFO_FLAG_READ |
- VFIO_REGION_INFO_FLAG_WRITE;
+ switch (info.index) {
+ case VFIO_PCI_CONFIG_REGION_INDEX:
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = pdev->cfg_size;
+ info.flags = VFIO_REGION_INFO_FLAG_READ |
+ VFIO_REGION_INFO_FLAG_WRITE;
+ break;
+ case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = pci_resource_len(pdev, info.index);
+ if (!info.size) {
+ info.flags = 0;
break;
- case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
- info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.size = pci_resource_len(pdev, info.index);
- if (!info.size) {
- info.flags = 0;
- break;
- }
+ }
- info.flags = VFIO_REGION_INFO_FLAG_READ |
- VFIO_REGION_INFO_FLAG_WRITE;
- if (vdev->bar_mmap_supported[info.index]) {
- info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
- if (info.index == vdev->msix_bar) {
- ret = msix_mmappable_cap(vdev, &caps);
- if (ret)
- return ret;
- }
+ info.flags = VFIO_REGION_INFO_FLAG_READ |
+ VFIO_REGION_INFO_FLAG_WRITE;
+ if (vdev->bar_mmap_supported[info.index]) {
+ info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
+ if (info.index == vdev->msix_bar) {
+ ret = msix_mmappable_cap(vdev, &caps);
+ if (ret)
+ return ret;
}
+ }
- break;
- case VFIO_PCI_ROM_REGION_INDEX:
- {
- void __iomem *io;
- size_t size;
- u16 cmd;
-
- info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.flags = 0;
+ break;
+ case VFIO_PCI_ROM_REGION_INDEX: {
+ void __iomem *io;
+ size_t size;
+ u16 cmd;
+
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.flags = 0;
+
+ /* Report the BAR size, not the ROM size */
+ info.size = pci_resource_len(pdev, info.index);
+ if (!info.size) {
+ /* Shadow ROMs appear as PCI option ROMs */
+ if (pdev->resource[PCI_ROM_RESOURCE].flags &
+ IORESOURCE_ROM_SHADOW)
+ info.size = 0x20000;
+ else
+ break;
+ }
- /* Report the BAR size, not the ROM size */
- info.size = pci_resource_len(pdev, info.index);
- if (!info.size) {
- /* Shadow ROMs appear as PCI option ROMs */
- if (pdev->resource[PCI_ROM_RESOURCE].flags &
- IORESOURCE_ROM_SHADOW)
- info.size = 0x20000;
- else
- break;
- }
+ /*
+ * Is it really there? Enable memory decode for implicit access
+ * in pci_map_rom().
+ */
+ cmd = vfio_pci_memory_lock_and_enable(vdev);
+ io = pci_map_rom(pdev, &size);
+ if (io) {
+ info.flags = VFIO_REGION_INFO_FLAG_READ;
+ pci_unmap_rom(pdev, io);
+ } else {
+ info.size = 0;
+ }
+ vfio_pci_memory_unlock_and_restore(vdev, cmd);
- /*
- * Is it really there? Enable memory decode for
- * implicit access in pci_map_rom().
- */
- cmd = vfio_pci_memory_lock_and_enable(vdev);
- io = pci_map_rom(pdev, &size);
- if (io) {
- info.flags = VFIO_REGION_INFO_FLAG_READ;
- pci_unmap_rom(pdev, io);
- } else {
- info.size = 0;
- }
- vfio_pci_memory_unlock_and_restore(vdev, cmd);
+ break;
+ }
+ case VFIO_PCI_VGA_REGION_INDEX:
+ if (!vdev->has_vga)
+ return -EINVAL;
- break;
- }
- case VFIO_PCI_VGA_REGION_INDEX:
- if (!vdev->has_vga)
- return -EINVAL;
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = 0xc0000;
+ info.flags = VFIO_REGION_INFO_FLAG_READ |
+ VFIO_REGION_INFO_FLAG_WRITE;
- info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.size = 0xc0000;
- info.flags = VFIO_REGION_INFO_FLAG_READ |
- VFIO_REGION_INFO_FLAG_WRITE;
+ break;
+ default: {
+ struct vfio_region_info_cap_type cap_type = {
+ .header.id = VFIO_REGION_INFO_CAP_TYPE,
+ .header.version = 1
+ };
- break;
- default:
- {
- struct vfio_region_info_cap_type cap_type = {
- .header.id = VFIO_REGION_INFO_CAP_TYPE,
- .header.version = 1 };
+ if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
+ return -EINVAL;
+ info.index = array_index_nospec(
+ info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions);
- if (info.index >=
- VFIO_PCI_NUM_REGIONS + vdev->num_regions)
- return -EINVAL;
- info.index = array_index_nospec(info.index,
- VFIO_PCI_NUM_REGIONS +
- vdev->num_regions);
+ i = info.index - VFIO_PCI_NUM_REGIONS;
- i = info.index - VFIO_PCI_NUM_REGIONS;
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vdev->region[i].size;
+ info.flags = vdev->region[i].flags;
- info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
- info.size = vdev->region[i].size;
- info.flags = vdev->region[i].flags;
+ cap_type.type = vdev->region[i].type;
+ cap_type.subtype = vdev->region[i].subtype;
- cap_type.type = vdev->region[i].type;
- cap_type.subtype = vdev->region[i].subtype;
+ ret = vfio_info_add_capability(&caps, &cap_type.header,
+ sizeof(cap_type));
+ if (ret)
+ return ret;
- ret = vfio_info_add_capability(&caps, &cap_type.header,
- sizeof(cap_type));
+ if (vdev->region[i].ops->add_capability) {
+ ret = vdev->region[i].ops->add_capability(
+ vdev, &vdev->region[i], &caps);
if (ret)
return ret;
-
- if (vdev->region[i].ops->add_capability) {
- ret = vdev->region[i].ops->add_capability(vdev,
- &vdev->region[i], &caps);
- if (ret)
- return ret;
- }
- }
}
+ }
+ }
- if (caps.size) {
- info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
- if (info.argsz < sizeof(info) + caps.size) {
- info.argsz = sizeof(info) + caps.size;
- info.cap_offset = 0;
- } else {
- vfio_info_cap_shift(&caps, sizeof(info));
- if (copy_to_user((void __user *)arg +
- sizeof(info), caps.buf,
- caps.size)) {
- kfree(caps.buf);
- return -EFAULT;
- }
- info.cap_offset = sizeof(info);
+ if (caps.size) {
+ info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
+ if (info.argsz < sizeof(info) + caps.size) {
+ info.argsz = sizeof(info) + caps.size;
+ info.cap_offset = 0;
+ } else {
+ vfio_info_cap_shift(&caps, sizeof(info));
+ if (copy_to_user(arg + 1, caps.buf, caps.size)) {
+ kfree(caps.buf);
+ return -EFAULT;
}
-
- kfree(caps.buf);
+ info.cap_offset = sizeof(*arg);
}
- return copy_to_user((void __user *)arg, &info, minsz) ?
- -EFAULT : 0;
+ kfree(caps.buf);
+ }
- } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
- struct vfio_irq_info info;
+ return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
+}
- minsz = offsetofend(struct vfio_irq_info, count);
+static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev,
+ struct vfio_irq_info __user *arg)
+{
+ unsigned long minsz = offsetofend(struct vfio_irq_info, count);
+ struct vfio_irq_info info;
- if (copy_from_user(&info, (void __user *)arg, minsz))
- return -EFAULT;
+ if (copy_from_user(&info, arg, minsz))
+ return -EFAULT;
- if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
- return -EINVAL;
+ if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
+ return -EINVAL;
- switch (info.index) {
- case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
- case VFIO_PCI_REQ_IRQ_INDEX:
+ switch (info.index) {
+ case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
+ case VFIO_PCI_REQ_IRQ_INDEX:
+ break;
+ case VFIO_PCI_ERR_IRQ_INDEX:
+ if (pci_is_pcie(vdev->pdev))
break;
- case VFIO_PCI_ERR_IRQ_INDEX:
- if (pci_is_pcie(vdev->pdev))
- break;
- fallthrough;
- default:
- return -EINVAL;
- }
-
- info.flags = VFIO_IRQ_INFO_EVENTFD;
-
- info.count = vfio_pci_get_irq_count(vdev, info.index);
+ fallthrough;
+ default:
+ return -EINVAL;
+ }
- if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
- info.flags |= (VFIO_IRQ_INFO_MASKABLE |
- VFIO_IRQ_INFO_AUTOMASKED);
- else
- info.flags |= VFIO_IRQ_INFO_NORESIZE;
+ info.flags = VFIO_IRQ_INFO_EVENTFD;
- return copy_to_user((void __user *)arg, &info, minsz) ?
- -EFAULT : 0;
+ info.count = vfio_pci_get_irq_count(vdev, info.index);
- } else if (cmd == VFIO_DEVICE_SET_IRQS) {
- struct vfio_irq_set hdr;
- u8 *data = NULL;
- int max, ret = 0;
- size_t data_size = 0;
+ if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
+ info.flags |=
+ (VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED);
+ else
+ info.flags |= VFIO_IRQ_INFO_NORESIZE;
- minsz = offsetofend(struct vfio_irq_set, count);
+ return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
+}
- if (copy_from_user(&hdr, (void __user *)arg, minsz))
- return -EFAULT;
+static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,
+ struct vfio_irq_set __user *arg)
+{
+ unsigned long minsz = offsetofend(struct vfio_irq_set, count);
+ struct vfio_irq_set hdr;
+ u8 *data = NULL;
+ int max, ret = 0;
+ size_t data_size = 0;
- max = vfio_pci_get_irq_count(vdev, hdr.index);
+ if (copy_from_user(&hdr, arg, minsz))
+ return -EFAULT;
- ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
- VFIO_PCI_NUM_IRQS, &data_size);
- if (ret)
- return ret;
+ max = vfio_pci_get_irq_count(vdev, hdr.index);
- if (data_size) {
- data = memdup_user((void __user *)(arg + minsz),
- data_size);
- if (IS_ERR(data))
- return PTR_ERR(data);
- }
+ ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS,
+ &data_size);
+ if (ret)
+ return ret;
- mutex_lock(&vdev->igate);
+ if (data_size) {
+ data = memdup_user(&arg->data, data_size);
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+ }
- ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
- hdr.start, hdr.count, data);
+ mutex_lock(&vdev->igate);
- mutex_unlock(&vdev->igate);
- kfree(data);
+ ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start,
+ hdr.count, data);
- return ret;
+ mutex_unlock(&vdev->igate);
+ kfree(data);
- } else if (cmd == VFIO_DEVICE_RESET) {
- int ret;
+ return ret;
+}
- if (!vdev->reset_works)
- return -EINVAL;
+static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
+ void __user *arg)
+{
+ int ret;
- vfio_pci_zap_and_down_write_memory_lock(vdev);
+ if (!vdev->reset_works)
+ return -EINVAL;
- /*
- * This function can be invoked while the power state is non-D0.
- * If pci_try_reset_function() has been called while the power
- * state is non-D0, then pci_try_reset_function() will
- * internally set the power state to D0 without vfio driver
- * involvement. For the devices which have NoSoftRst-, the
- * reset function can cause the PCI config space reset without
- * restoring the original state (saved locally in
- * 'vdev->pm_save').
- */
- vfio_pci_set_power_state(vdev, PCI_D0);
+ vfio_pci_zap_and_down_write_memory_lock(vdev);
- ret = pci_try_reset_function(vdev->pdev);
- up_write(&vdev->memory_lock);
+ /*
+ * This function can be invoked while the power state is non-D0. If
+ * pci_try_reset_function() has been called while the power state is
+ * non-D0, then pci_try_reset_function() will internally set the power
+ * state to D0 without vfio driver involvement. For the devices which
+ * have NoSoftRst-, the reset function can cause the PCI config space
+ * reset without restoring the original state (saved locally in
+ * 'vdev->pm_save').
+ */
+ vfio_pci_set_power_state(vdev, PCI_D0);
- return ret;
+ ret = pci_try_reset_function(vdev->pdev);
+ up_write(&vdev->memory_lock);
- } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
- struct vfio_pci_hot_reset_info hdr;
- struct vfio_pci_fill_info fill = { 0 };
- struct vfio_pci_dependent_device *devices = NULL;
- bool slot = false;
- int ret = 0;
+ return ret;
+}
- minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
+static int vfio_pci_ioctl_get_pci_hot_reset_info(
+ struct vfio_pci_core_device *vdev,
+ struct vfio_pci_hot_reset_info __user *arg)
+{
+ unsigned long minsz =
+ offsetofend(struct vfio_pci_hot_reset_info, count);
+ struct vfio_pci_hot_reset_info hdr;
+ struct vfio_pci_fill_info fill = { 0 };
+ struct vfio_pci_dependent_device *devices = NULL;
+ bool slot = false;
+ int ret = 0;
- if (copy_from_user(&hdr, (void __user *)arg, minsz))
- return -EFAULT;
+ if (copy_from_user(&hdr, arg, minsz))
+ return -EFAULT;
- if (hdr.argsz < minsz)
- return -EINVAL;
+ if (hdr.argsz < minsz)
+ return -EINVAL;
- hdr.flags = 0;
+ hdr.flags = 0;
- /* Can we do a slot or bus reset or neither? */
- if (!pci_probe_reset_slot(vdev->pdev->slot))
- slot = true;
- else if (pci_probe_reset_bus(vdev->pdev->bus))
- return -ENODEV;
+ /* Can we do a slot or bus reset or neither? */
+ if (!pci_probe_reset_slot(vdev->pdev->slot))
+ slot = true;
+ else if (pci_probe_reset_bus(vdev->pdev->bus))
+ return -ENODEV;
- /* How many devices are affected? */
- ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
- vfio_pci_count_devs,
- &fill.max, slot);
- if (ret)
- return ret;
+ /* How many devices are affected? */
+ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
+ &fill.max, slot);
+ if (ret)
+ return ret;
- WARN_ON(!fill.max); /* Should always be at least one */
+ WARN_ON(!fill.max); /* Should always be at least one */
- /*
- * If there's enough space, fill it now, otherwise return
- * -ENOSPC and the number of devices affected.
- */
- if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
- ret = -ENOSPC;
- hdr.count = fill.max;
- goto reset_info_exit;
- }
+ /*
+ * If there's enough space, fill it now, otherwise return -ENOSPC and
+ * the number of devices affected.
+ */
+ if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
+ ret = -ENOSPC;
+ hdr.count = fill.max;
+ goto reset_info_exit;
+ }
- devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
- if (!devices)
- return -ENOMEM;
+ devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
+ if (!devices)
+ return -ENOMEM;
- fill.devices = devices;
+ fill.devices = devices;
- ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
- vfio_pci_fill_devs,
- &fill, slot);
+ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs,
+ &fill, slot);
- /*
- * If a device was removed between counting and filling,
- * we may come up short of fill.max. If a device was
- * added, we'll have a return of -EAGAIN above.
- */
- if (!ret)
- hdr.count = fill.cur;
+ /*
+ * If a device was removed between counting and filling, we may come up
+ * short of fill.max. If a device was added, we'll have a return of
+ * -EAGAIN above.
+ */
+ if (!ret)
+ hdr.count = fill.cur;
reset_info_exit:
- if (copy_to_user((void __user *)arg, &hdr, minsz))
+ if (copy_to_user(arg, &hdr, minsz))
+ ret = -EFAULT;
+
+ if (!ret) {
+ if (copy_to_user(&arg->devices, devices,
+ hdr.count * sizeof(*devices)))
ret = -EFAULT;
+ }
- if (!ret) {
- if (copy_to_user((void __user *)(arg + minsz), devices,
- hdr.count * sizeof(*devices)))
- ret = -EFAULT;
- }
+ kfree(devices);
+ return ret;
+}
- kfree(devices);
- return ret;
+static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev,
+ struct vfio_pci_hot_reset __user *arg)
+{
+ unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count);
+ struct vfio_pci_hot_reset hdr;
+ int32_t *group_fds;
+ struct file **files;
+ struct vfio_pci_group_info info;
+ bool slot = false;
+ int file_idx, count = 0, ret = 0;
- } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
- struct vfio_pci_hot_reset hdr;
- int32_t *group_fds;
- struct file **files;
- struct vfio_pci_group_info info;
- bool slot = false;
- int file_idx, count = 0, ret = 0;
+ if (copy_from_user(&hdr, arg, minsz))
+ return -EFAULT;
- minsz = offsetofend(struct vfio_pci_hot_reset, count);
+ if (hdr.argsz < minsz || hdr.flags)
+ return -EINVAL;
- if (copy_from_user(&hdr, (void __user *)arg, minsz))
- return -EFAULT;
+ /* Can we do a slot or bus reset or neither? */
+ if (!pci_probe_reset_slot(vdev->pdev->slot))
+ slot = true;
+ else if (pci_probe_reset_bus(vdev->pdev->bus))
+ return -ENODEV;
- if (hdr.argsz < minsz || hdr.flags)
- return -EINVAL;
+ /*
+ * We can't let userspace give us an arbitrarily large buffer to copy,
+ * so verify how many we think there could be. Note groups can have
+ * multiple devices so one group per device is the max.
+ */
+ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
+ &count, slot);
+ if (ret)
+ return ret;
- /* Can we do a slot or bus reset or neither? */
- if (!pci_probe_reset_slot(vdev->pdev->slot))
- slot = true;
- else if (pci_probe_reset_bus(vdev->pdev->bus))
- return -ENODEV;
+ /* Somewhere between 1 and count is OK */
+ if (!hdr.count || hdr.count > count)
+ return -EINVAL;
- /*
- * We can't let userspace give us an arbitrarily large
- * buffer to copy, so verify how many we think there
- * could be. Note groups can have multiple devices so
- * one group per device is the max.
- */
- ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
- vfio_pci_count_devs,
- &count, slot);
- if (ret)
- return ret;
+ group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
+ files = kcalloc(hdr.count, sizeof(*files), GFP_KERNEL);
+ if (!group_fds || !files) {
+ kfree(group_fds);
+ kfree(files);
+ return -ENOMEM;
+ }
- /* Somewhere between 1 and count is OK */
- if (!hdr.count || hdr.count > count)
- return -EINVAL;
+ if (copy_from_user(group_fds, arg->group_fds,
+ hdr.count * sizeof(*group_fds))) {
+ kfree(group_fds);
+ kfree(files);
+ return -EFAULT;
+ }
- group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
- files = kcalloc(hdr.count, sizeof(*files), GFP_KERNEL);
- if (!group_fds || !files) {
- kfree(group_fds);
- kfree(files);
- return -ENOMEM;
- }
+ /*
+ * For each group_fd, get the group through the vfio external user
+ * interface and store the group and iommu ID. This ensures the group
+ * is held across the reset.
+ */
+ for (file_idx = 0; file_idx < hdr.count; file_idx++) {
+ struct file *file = fget(group_fds[file_idx]);
- if (copy_from_user(group_fds, (void __user *)(arg + minsz),
- hdr.count * sizeof(*group_fds))) {
- kfree(group_fds);
- kfree(files);
- return -EFAULT;
+ if (!file) {
+ ret = -EBADF;
+ break;
}
- /*
- * For each group_fd, get the group through the vfio external
- * user interface and store the group and iommu ID. This
- * ensures the group is held across the reset.
- */
- for (file_idx = 0; file_idx < hdr.count; file_idx++) {
- struct file *file = fget(group_fds[file_idx]);
-
- if (!file) {
- ret = -EBADF;
- break;
- }
-
- /* Ensure the FD is a vfio group FD.*/
- if (!vfio_file_iommu_group(file)) {
- fput(file);
- ret = -EINVAL;
- break;
- }
-
- files[file_idx] = file;
+ /* Ensure the FD is a vfio group FD.*/
+ if (!vfio_file_is_group(file)) {
+ fput(file);
+ ret = -EINVAL;
+ break;
}
- kfree(group_fds);
+ files[file_idx] = file;
+ }
- /* release reference to groups on error */
- if (ret)
- goto hot_reset_release;
+ kfree(group_fds);
+
+ /* release reference to groups on error */
+ if (ret)
+ goto hot_reset_release;
- info.count = hdr.count;
- info.files = files;
+ info.count = hdr.count;
+ info.files = files;
- ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info);
+ ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info);
hot_reset_release:
- for (file_idx--; file_idx >= 0; file_idx--)
- fput(files[file_idx]);
+ for (file_idx--; file_idx >= 0; file_idx--)
+ fput(files[file_idx]);
- kfree(files);
- return ret;
- } else if (cmd == VFIO_DEVICE_IOEVENTFD) {
- struct vfio_device_ioeventfd ioeventfd;
- int count;
+ kfree(files);
+ return ret;
+}
- minsz = offsetofend(struct vfio_device_ioeventfd, fd);
+static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev,
+ struct vfio_device_ioeventfd __user *arg)
+{
+ unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd);
+ struct vfio_device_ioeventfd ioeventfd;
+ int count;
- if (copy_from_user(&ioeventfd, (void __user *)arg, minsz))
- return -EFAULT;
+ if (copy_from_user(&ioeventfd, arg, minsz))
+ return -EFAULT;
- if (ioeventfd.argsz < minsz)
- return -EINVAL;
+ if (ioeventfd.argsz < minsz)
+ return -EINVAL;
- if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
- return -EINVAL;
+ if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
+ return -EINVAL;
- count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
+ count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
- if (hweight8(count) != 1 || ioeventfd.fd < -1)
- return -EINVAL;
+ if (hweight8(count) != 1 || ioeventfd.fd < -1)
+ return -EINVAL;
- return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
- ioeventfd.data, count, ioeventfd.fd);
+ return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count,
+ ioeventfd.fd);
+}
+
+long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
+ unsigned long arg)
+{
+ struct vfio_pci_core_device *vdev =
+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
+ void __user *uarg = (void __user *)arg;
+
+ switch (cmd) {
+ case VFIO_DEVICE_GET_INFO:
+ return vfio_pci_ioctl_get_info(vdev, uarg);
+ case VFIO_DEVICE_GET_IRQ_INFO:
+ return vfio_pci_ioctl_get_irq_info(vdev, uarg);
+ case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO:
+ return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg);
+ case VFIO_DEVICE_GET_REGION_INFO:
+ return vfio_pci_ioctl_get_region_info(vdev, uarg);
+ case VFIO_DEVICE_IOEVENTFD:
+ return vfio_pci_ioctl_ioeventfd(vdev, uarg);
+ case VFIO_DEVICE_PCI_HOT_RESET:
+ return vfio_pci_ioctl_pci_hot_reset(vdev, uarg);
+ case VFIO_DEVICE_RESET:
+ return vfio_pci_ioctl_reset(vdev, uarg);
+ case VFIO_DEVICE_SET_IRQS:
+ return vfio_pci_ioctl_set_irqs(vdev, uarg);
+ default:
+ return -ENOTTY;
}
- return -ENOTTY;
}
EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);
static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags,
- void __user *arg, size_t argsz)
+ uuid_t __user *arg, size_t argsz)
{
struct vfio_pci_core_device *vdev =
container_of(device, struct vfio_pci_core_device, vdev);
void __user *arg, size_t argsz)
{
switch (flags & VFIO_DEVICE_FEATURE_MASK) {
+ case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY:
+ return vfio_pci_core_pm_entry(device, flags, arg, argsz);
+ case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP:
+ return vfio_pci_core_pm_entry_with_wakeup(device, flags,
+ arg, argsz);
+ case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT:
+ return vfio_pci_core_pm_exit(device, flags, arg, argsz);
case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
return vfio_pci_core_feature_token(device, flags, arg, argsz);
default:
size_t count, loff_t *ppos, bool iswrite)
{
unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
+ int ret;
if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
return -EINVAL;
+ ret = pm_runtime_resume_and_get(&vdev->pdev->dev);
+ if (ret) {
+ pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n",
+ ret);
+ return -EIO;
+ }
+
switch (index) {
case VFIO_PCI_CONFIG_REGION_INDEX:
- return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
+ ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
+ break;
case VFIO_PCI_ROM_REGION_INDEX:
if (iswrite)
- return -EINVAL;
- return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
+ ret = -EINVAL;
+ else
+ ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false);
+ break;
case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
- return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
+ ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
+ break;
case VFIO_PCI_VGA_REGION_INDEX:
- return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
+ ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
+ break;
+
default:
index -= VFIO_PCI_NUM_REGIONS;
- return vdev->region[index].ops->rw(vdev, buf,
+ ret = vdev->region[index].ops->rw(vdev, buf,
count, ppos, iswrite);
+ break;
}
- return -EINVAL;
+ pm_runtime_put(&vdev->pdev->dev);
+ return ret;
}
ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
mutex_lock(&vdev->vma_lock);
down_read(&vdev->memory_lock);
- if (!__vfio_pci_memory_enabled(vdev)) {
+ /*
+ * Memory region cannot be accessed if the low power feature is engaged
+ * or memory access is disabled.
+ */
+ if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) {
ret = VM_FAULT_SIGBUS;
goto up_out;
}
VGA_RSRC_LEGACY_MEM);
}
-void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
- struct pci_dev *pdev,
- const struct vfio_device_ops *vfio_pci_ops)
+int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
{
- vfio_init_group_dev(&vdev->vdev, &pdev->dev, vfio_pci_ops);
- vdev->pdev = pdev;
+ struct vfio_pci_core_device *vdev =
+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
+
+ vdev->pdev = to_pci_dev(core_vdev->dev);
vdev->irq_type = VFIO_PCI_NUM_IRQS;
mutex_init(&vdev->igate);
spin_lock_init(&vdev->irqlock);
INIT_LIST_HEAD(&vdev->vma_list);
INIT_LIST_HEAD(&vdev->sriov_pfs_item);
init_rwsem(&vdev->memory_lock);
+
+ return 0;
}
-EXPORT_SYMBOL_GPL(vfio_pci_core_init_device);
+EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev);
-void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev)
+void vfio_pci_core_release_dev(struct vfio_device *core_vdev)
{
+ struct vfio_pci_core_device *vdev =
+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
+
mutex_destroy(&vdev->igate);
mutex_destroy(&vdev->ioeventfds_lock);
mutex_destroy(&vdev->vma_lock);
- vfio_uninit_group_dev(&vdev->vdev);
kfree(vdev->region);
kfree(vdev->pm_save);
+ vfio_free_device(core_vdev);
}
-EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device);
+EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev);
int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
{
return -EINVAL;
}
+ if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start &&
+ vdev->vdev.log_ops->log_stop &&
+ vdev->vdev.log_ops->log_read_and_clear))
+ return -EINVAL;
+
/*
* Prevent binding to PFs with VFs enabled, the VFs might be in use
* by the host or other users. We cannot capture the VFs if they
goto err_unlock;
}
+ /*
+ * Some of the devices in the dev_set can be in the runtime suspended
+ * state. Increment the usage count for all the devices in the dev_set
+ * before reset and decrement the same after reset.
+ */
+ ret = vfio_pci_dev_set_pm_runtime_get(dev_set);
+ if (ret)
+ goto err_unlock;
+
list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
/*
* Test whether all the affected devices are contained by the
else
mutex_unlock(&cur->vma_lock);
}
+
+ list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
+ pm_runtime_put(&cur->pdev->dev);
err_unlock:
mutex_unlock(&dev_set->lock);
return ret;
#include <linux/uaccess.h>
#include <linux/vfio.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
#define OPREGION_SIGNATURE "IntelGraphicsMem"
#define OPREGION_SIZE (8 * 1024)
}
}
- ret = vfio_pci_register_dev_region(vdev,
+ ret = vfio_pci_core_register_dev_region(vdev,
PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &vfio_pci_igd_regops,
size, VFIO_REGION_INFO_FLAG_READ, opregionvbt);
return -EINVAL;
}
- ret = vfio_pci_register_dev_region(vdev,
+ ret = vfio_pci_core_register_dev_region(vdev,
PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG,
&vfio_pci_igd_cfg_regops, host_bridge->cfg_size,
return -EINVAL;
}
- ret = vfio_pci_register_dev_region(vdev,
+ ret = vfio_pci_core_register_dev_region(vdev,
PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG,
&vfio_pci_igd_cfg_regops, lpc_bridge->cfg_size,
#include <linux/wait.h>
#include <linux/slab.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
+
+struct vfio_pci_irq_ctx {
+ struct eventfd_ctx *trigger;
+ struct virqfd *unmask;
+ struct virqfd *mask;
+ char *name;
+ bool masked;
+ struct irq_bypass_producer producer;
+};
+
+static bool irq_is(struct vfio_pci_core_device *vdev, int type)
+{
+ return vdev->irq_type == type;
+}
+
+static bool is_intx(struct vfio_pci_core_device *vdev)
+{
+ return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX;
+}
+
+static bool is_irq_none(struct vfio_pci_core_device *vdev)
+{
+ return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX ||
+ vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX ||
+ vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX);
+}
/*
* INTx
eventfd_signal(vdev->ctx[0].trigger, 1);
}
-void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
+/* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */
+bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
{
struct pci_dev *pdev = vdev->pdev;
unsigned long flags;
+ bool masked_changed = false;
spin_lock_irqsave(&vdev->irqlock, flags);
disable_irq_nosync(pdev->irq);
vdev->ctx[0].masked = true;
+ masked_changed = true;
}
spin_unlock_irqrestore(&vdev->irqlock, flags);
+ return masked_changed;
}
/*
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef VFIO_PCI_PRIV_H
+#define VFIO_PCI_PRIV_H
+
+#include <linux/vfio_pci_core.h>
+
+/* Special capability IDs predefined access */
+#define PCI_CAP_ID_INVALID 0xFF /* default raw access */
+#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */
+
+/* Cap maximum number of ioeventfds per device (arbitrary) */
+#define VFIO_PCI_IOEVENTFD_MAX 1000
+
+struct vfio_pci_ioeventfd {
+ struct list_head next;
+ struct vfio_pci_core_device *vdev;
+ struct virqfd *virqfd;
+ void __iomem *addr;
+ uint64_t data;
+ loff_t pos;
+ int bar;
+ int count;
+ bool test_mem;
+};
+
+bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);
+void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);
+
+int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags,
+ unsigned index, unsigned start, unsigned count,
+ void *data);
+
+ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos, bool iswrite);
+
+ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos, bool iswrite);
+
+#ifdef CONFIG_VFIO_PCI_VGA
+ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos, bool iswrite);
+#else
+static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev,
+ char __user *buf, size_t count,
+ loff_t *ppos, bool iswrite)
+{
+ return -EINVAL;
+}
+#endif
+
+int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
+ uint64_t data, int count, int fd);
+
+int vfio_pci_init_perm_bits(void);
+void vfio_pci_uninit_perm_bits(void);
+
+int vfio_config_init(struct vfio_pci_core_device *vdev);
+void vfio_config_free(struct vfio_pci_core_device *vdev);
+
+int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,
+ pci_power_t state);
+
+bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev);
+void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev);
+u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);
+void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,
+ u16 cmd);
+
+#ifdef CONFIG_VFIO_PCI_IGD
+int vfio_pci_igd_init(struct vfio_pci_core_device *vdev);
+#else
+static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
+{
+ return -ENODEV;
+}
+#endif
+
+#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
+int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
+ struct vfio_info_cap *caps);
+int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev);
+void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev);
+#else
+static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
+ struct vfio_info_cap *caps)
+{
+ return -ENODEV;
+}
+
+static inline int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev)
+{
+ return 0;
+}
+
+static inline void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
+{}
+#endif
+
+static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
+{
+ return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
+}
+
+#endif
#include <linux/vfio.h>
#include <linux/vgaarb.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
#ifdef __LITTLE_ENDIAN
#define vfio_ioread64 ioread64
vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
}
-long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
- uint64_t data, int count, int fd)
+int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
+ uint64_t data, int count, int fd)
{
struct pci_dev *pdev = vdev->pdev;
loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
#include <asm/pci_clp.h>
#include <asm/pci_io.h>
-#include <linux/vfio_pci_core.h>
+#include "vfio_pci_priv.h"
/*
* Add the Base PCI Function information to the device info region.
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vfio.h>
+#include <linux/pm_runtime.h>
#include <linux/amba/bus.h>
#include "vfio_platform_private.h"
return ret ? ret : -ENXIO;
}
-static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id)
+static int vfio_amba_init_dev(struct vfio_device *core_vdev)
{
- struct vfio_platform_device *vdev;
+ struct vfio_platform_device *vdev =
+ container_of(core_vdev, struct vfio_platform_device, vdev);
+ struct amba_device *adev = to_amba_device(core_vdev->dev);
int ret;
- vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
- if (!vdev)
- return -ENOMEM;
-
vdev->name = kasprintf(GFP_KERNEL, "vfio-amba-%08x", adev->periphid);
- if (!vdev->name) {
- kfree(vdev);
+ if (!vdev->name)
return -ENOMEM;
- }
vdev->opaque = (void *) adev;
vdev->flags = VFIO_DEVICE_FLAGS_AMBA;
vdev->get_irq = get_amba_irq;
vdev->reset_required = false;
- ret = vfio_platform_probe_common(vdev, &adev->dev);
- if (ret) {
+ ret = vfio_platform_init_common(vdev);
+ if (ret)
kfree(vdev->name);
- kfree(vdev);
- return ret;
- }
+ return ret;
+}
+
+static const struct vfio_device_ops vfio_amba_ops;
+static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id)
+{
+ struct vfio_platform_device *vdev;
+ int ret;
+
+ vdev = vfio_alloc_device(vfio_platform_device, vdev, &adev->dev,
+ &vfio_amba_ops);
+ if (IS_ERR(vdev))
+ return PTR_ERR(vdev);
+ ret = vfio_register_group_dev(&vdev->vdev);
+ if (ret)
+ goto out_put_vdev;
+
+ pm_runtime_enable(&adev->dev);
dev_set_drvdata(&adev->dev, vdev);
return 0;
+
+out_put_vdev:
+ vfio_put_device(&vdev->vdev);
+ return ret;
+}
+
+static void vfio_amba_release_dev(struct vfio_device *core_vdev)
+{
+ struct vfio_platform_device *vdev =
+ container_of(core_vdev, struct vfio_platform_device, vdev);
+
+ vfio_platform_release_common(vdev);
+ kfree(vdev->name);
+ vfio_free_device(core_vdev);
}
static void vfio_amba_remove(struct amba_device *adev)
{
struct vfio_platform_device *vdev = dev_get_drvdata(&adev->dev);
- vfio_platform_remove_common(vdev);
- kfree(vdev->name);
- kfree(vdev);
+ vfio_unregister_group_dev(&vdev->vdev);
+ pm_runtime_disable(vdev->device);
+ vfio_put_device(&vdev->vdev);
}
+static const struct vfio_device_ops vfio_amba_ops = {
+ .name = "vfio-amba",
+ .init = vfio_amba_init_dev,
+ .release = vfio_amba_release_dev,
+ .open_device = vfio_platform_open_device,
+ .close_device = vfio_platform_close_device,
+ .ioctl = vfio_platform_ioctl,
+ .read = vfio_platform_read,
+ .write = vfio_platform_write,
+ .mmap = vfio_platform_mmap,
+};
+
static const struct amba_id pl330_ids[] = {
{ 0, 0 },
};
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vfio.h>
+#include <linux/pm_runtime.h>
#include <linux/platform_device.h>
#include "vfio_platform_private.h"
return platform_get_irq_optional(pdev, i);
}
-static int vfio_platform_probe(struct platform_device *pdev)
+static int vfio_platform_init_dev(struct vfio_device *core_vdev)
{
- struct vfio_platform_device *vdev;
- int ret;
-
- vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
- if (!vdev)
- return -ENOMEM;
+ struct vfio_platform_device *vdev =
+ container_of(core_vdev, struct vfio_platform_device, vdev);
+ struct platform_device *pdev = to_platform_device(core_vdev->dev);
vdev->opaque = (void *) pdev;
vdev->name = pdev->name;
vdev->get_irq = get_platform_irq;
vdev->reset_required = reset_required;
- ret = vfio_platform_probe_common(vdev, &pdev->dev);
- if (ret) {
- kfree(vdev);
- return ret;
- }
+ return vfio_platform_init_common(vdev);
+}
+
+static const struct vfio_device_ops vfio_platform_ops;
+static int vfio_platform_probe(struct platform_device *pdev)
+{
+ struct vfio_platform_device *vdev;
+ int ret;
+
+ vdev = vfio_alloc_device(vfio_platform_device, vdev, &pdev->dev,
+ &vfio_platform_ops);
+ if (IS_ERR(vdev))
+ return PTR_ERR(vdev);
+
+ ret = vfio_register_group_dev(&vdev->vdev);
+ if (ret)
+ goto out_put_vdev;
+
+ pm_runtime_enable(&pdev->dev);
dev_set_drvdata(&pdev->dev, vdev);
return 0;
+
+out_put_vdev:
+ vfio_put_device(&vdev->vdev);
+ return ret;
+}
+
+static void vfio_platform_release_dev(struct vfio_device *core_vdev)
+{
+ struct vfio_platform_device *vdev =
+ container_of(core_vdev, struct vfio_platform_device, vdev);
+
+ vfio_platform_release_common(vdev);
+ vfio_free_device(core_vdev);
}
static int vfio_platform_remove(struct platform_device *pdev)
{
struct vfio_platform_device *vdev = dev_get_drvdata(&pdev->dev);
- vfio_platform_remove_common(vdev);
- kfree(vdev);
+ vfio_unregister_group_dev(&vdev->vdev);
+ pm_runtime_disable(vdev->device);
+ vfio_put_device(&vdev->vdev);
return 0;
}
+static const struct vfio_device_ops vfio_platform_ops = {
+ .name = "vfio-platform",
+ .init = vfio_platform_init_dev,
+ .release = vfio_platform_release_dev,
+ .open_device = vfio_platform_open_device,
+ .close_device = vfio_platform_close_device,
+ .ioctl = vfio_platform_ioctl,
+ .read = vfio_platform_read,
+ .write = vfio_platform_write,
+ .mmap = vfio_platform_mmap,
+};
+
static struct platform_driver vfio_platform_driver = {
.probe = vfio_platform_probe,
.remove = vfio_platform_remove,
return -EINVAL;
}
-static void vfio_platform_close_device(struct vfio_device *core_vdev)
+void vfio_platform_close_device(struct vfio_device *core_vdev)
{
struct vfio_platform_device *vdev =
container_of(core_vdev, struct vfio_platform_device, vdev);
vfio_platform_regions_cleanup(vdev);
vfio_platform_irq_cleanup(vdev);
}
+EXPORT_SYMBOL_GPL(vfio_platform_close_device);
-static int vfio_platform_open_device(struct vfio_device *core_vdev)
+int vfio_platform_open_device(struct vfio_device *core_vdev)
{
struct vfio_platform_device *vdev =
container_of(core_vdev, struct vfio_platform_device, vdev);
vfio_platform_regions_cleanup(vdev);
return ret;
}
+EXPORT_SYMBOL_GPL(vfio_platform_open_device);
-static long vfio_platform_ioctl(struct vfio_device *core_vdev,
- unsigned int cmd, unsigned long arg)
+long vfio_platform_ioctl(struct vfio_device *core_vdev,
+ unsigned int cmd, unsigned long arg)
{
struct vfio_platform_device *vdev =
container_of(core_vdev, struct vfio_platform_device, vdev);
return -ENOTTY;
}
+EXPORT_SYMBOL_GPL(vfio_platform_ioctl);
static ssize_t vfio_platform_read_mmio(struct vfio_platform_region *reg,
char __user *buf, size_t count,
return -EFAULT;
}
-static ssize_t vfio_platform_read(struct vfio_device *core_vdev,
- char __user *buf, size_t count, loff_t *ppos)
+ssize_t vfio_platform_read(struct vfio_device *core_vdev,
+ char __user *buf, size_t count, loff_t *ppos)
{
struct vfio_platform_device *vdev =
container_of(core_vdev, struct vfio_platform_device, vdev);
return -EINVAL;
}
+EXPORT_SYMBOL_GPL(vfio_platform_read);
static ssize_t vfio_platform_write_mmio(struct vfio_platform_region *reg,
const char __user *buf, size_t count,
return -EFAULT;
}
-static ssize_t vfio_platform_write(struct vfio_device *core_vdev, const char __user *buf,
- size_t count, loff_t *ppos)
+ssize_t vfio_platform_write(struct vfio_device *core_vdev, const char __user *buf,
+ size_t count, loff_t *ppos)
{
struct vfio_platform_device *vdev =
container_of(core_vdev, struct vfio_platform_device, vdev);
return -EINVAL;
}
+EXPORT_SYMBOL_GPL(vfio_platform_write);
static int vfio_platform_mmap_mmio(struct vfio_platform_region region,
struct vm_area_struct *vma)
req_len, vma->vm_page_prot);
}
-static int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
+int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
{
struct vfio_platform_device *vdev =
container_of(core_vdev, struct vfio_platform_device, vdev);
return -EINVAL;
}
-
-static const struct vfio_device_ops vfio_platform_ops = {
- .name = "vfio-platform",
- .open_device = vfio_platform_open_device,
- .close_device = vfio_platform_close_device,
- .ioctl = vfio_platform_ioctl,
- .read = vfio_platform_read,
- .write = vfio_platform_write,
- .mmap = vfio_platform_mmap,
-};
+EXPORT_SYMBOL_GPL(vfio_platform_mmap);
static int vfio_platform_of_probe(struct vfio_platform_device *vdev,
struct device *dev)
* If the firmware is ACPI type, then acpi_disabled is 0. All other checks are
* valid checks. We cannot claim that this system is DT.
*/
-int vfio_platform_probe_common(struct vfio_platform_device *vdev,
- struct device *dev)
+int vfio_platform_init_common(struct vfio_platform_device *vdev)
{
int ret;
-
- vfio_init_group_dev(&vdev->vdev, dev, &vfio_platform_ops);
+ struct device *dev = vdev->vdev.dev;
ret = vfio_platform_acpi_probe(vdev, dev);
if (ret)
ret = vfio_platform_of_probe(vdev, dev);
if (ret)
- goto out_uninit;
+ return ret;
vdev->device = dev;
+ mutex_init(&vdev->igate);
ret = vfio_platform_get_reset(vdev);
- if (ret && vdev->reset_required) {
+ if (ret && vdev->reset_required)
dev_err(dev, "No reset function found for device %s\n",
vdev->name);
- goto out_uninit;
- }
-
- ret = vfio_register_group_dev(&vdev->vdev);
- if (ret)
- goto put_reset;
-
- mutex_init(&vdev->igate);
-
- pm_runtime_enable(dev);
- return 0;
-
-put_reset:
- vfio_platform_put_reset(vdev);
-out_uninit:
- vfio_uninit_group_dev(&vdev->vdev);
return ret;
}
-EXPORT_SYMBOL_GPL(vfio_platform_probe_common);
+EXPORT_SYMBOL_GPL(vfio_platform_init_common);
-void vfio_platform_remove_common(struct vfio_platform_device *vdev)
+void vfio_platform_release_common(struct vfio_platform_device *vdev)
{
- vfio_unregister_group_dev(&vdev->vdev);
-
- pm_runtime_disable(vdev->device);
vfio_platform_put_reset(vdev);
- vfio_uninit_group_dev(&vdev->vdev);
}
-EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
+EXPORT_SYMBOL_GPL(vfio_platform_release_common);
void __vfio_platform_register_reset(struct vfio_platform_reset_node *node)
{
vfio_platform_reset_fn_t of_reset;
};
-int vfio_platform_probe_common(struct vfio_platform_device *vdev,
- struct device *dev);
-void vfio_platform_remove_common(struct vfio_platform_device *vdev);
+int vfio_platform_init_common(struct vfio_platform_device *vdev);
+void vfio_platform_release_common(struct vfio_platform_device *vdev);
+
+int vfio_platform_open_device(struct vfio_device *core_vdev);
+void vfio_platform_close_device(struct vfio_device *core_vdev);
+long vfio_platform_ioctl(struct vfio_device *core_vdev,
+ unsigned int cmd, unsigned long arg);
+ssize_t vfio_platform_read(struct vfio_device *core_vdev,
+ char __user *buf, size_t count,
+ loff_t *ppos);
+ssize_t vfio_platform_write(struct vfio_device *core_vdev,
+ const char __user *buf,
+ size_t count, loff_t *ppos);
+int vfio_platform_mmap(struct vfio_device *core_vdev,
+ struct vm_area_struct *vma);
int vfio_platform_irq_init(struct vfio_platform_device *vdev);
void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev);
* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
* Author: Alex Williamson <alex.williamson@redhat.com>
*/
+#ifndef __VFIO_VFIO_H__
+#define __VFIO_VFIO_H__
+
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/module.h>
+
+struct iommu_group;
+struct vfio_device;
+struct vfio_container;
enum vfio_group_type {
/*
VFIO_NO_IOMMU,
};
+struct vfio_group {
+ struct device dev;
+ struct cdev cdev;
+ /*
+ * When drivers is non-zero a driver is attached to the struct device
+ * that provided the iommu_group and thus the iommu_group is a valid
+ * pointer. When drivers is 0 the driver is being detached. Once users
+ * reaches 0 then the iommu_group is invalid.
+ */
+ refcount_t drivers;
+ unsigned int container_users;
+ struct iommu_group *iommu_group;
+ struct vfio_container *container;
+ struct list_head device_list;
+ struct mutex device_lock;
+ struct list_head vfio_next;
+ struct list_head container_next;
+ enum vfio_group_type type;
+ struct mutex group_lock;
+ struct kvm *kvm;
+ struct file *opened_file;
+ struct blocking_notifier_head notifier;
+};
+
/* events for the backend driver notify callback */
enum vfio_iommu_notify_type {
VFIO_IOMMU_CONTAINER_CLOSE = 0,
enum vfio_iommu_notify_type event);
};
+struct vfio_iommu_driver {
+ const struct vfio_iommu_driver_ops *ops;
+ struct list_head vfio_next;
+};
+
int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops);
+
+bool vfio_assert_device_open(struct vfio_device *device);
+
+struct vfio_container *vfio_container_from_file(struct file *filep);
+int vfio_device_assign_container(struct vfio_device *device);
+void vfio_device_unassign_container(struct vfio_device *device);
+int vfio_container_attach_group(struct vfio_container *container,
+ struct vfio_group *group);
+void vfio_group_detach_container(struct vfio_group *group);
+void vfio_device_container_register(struct vfio_device *device);
+void vfio_device_container_unregister(struct vfio_device *device);
+long vfio_container_ioctl_check_extension(struct vfio_container *container,
+ unsigned long arg);
+int __init vfio_container_init(void);
+void vfio_container_cleanup(void);
+
+#ifdef CONFIG_VFIO_NOIOMMU
+extern bool vfio_noiommu __read_mostly;
+#else
+enum { vfio_noiommu = false };
+#endif
+
+#endif
#include <linux/vfio.h>
#include <linux/wait.h>
#include <linux/sched/signal.h>
+#include <linux/pm_runtime.h>
+#include <linux/interval_tree.h>
+#include <linux/iova_bitmap.h>
#include "vfio.h"
#define DRIVER_VERSION "0.3"
static struct vfio {
struct class *class;
- struct list_head iommu_drivers_list;
- struct mutex iommu_drivers_lock;
struct list_head group_list;
struct mutex group_lock; /* locks group_list */
struct ida group_ida;
dev_t group_devt;
+ struct class *device_class;
+ struct ida device_ida;
} vfio;
-struct vfio_iommu_driver {
- const struct vfio_iommu_driver_ops *ops;
- struct list_head vfio_next;
-};
-
-struct vfio_container {
- struct kref kref;
- struct list_head group_list;
- struct rw_semaphore group_lock;
- struct vfio_iommu_driver *iommu_driver;
- void *iommu_data;
- bool noiommu;
-};
-
-struct vfio_group {
- struct device dev;
- struct cdev cdev;
- refcount_t users;
- unsigned int container_users;
- struct iommu_group *iommu_group;
- struct vfio_container *container;
- struct list_head device_list;
- struct mutex device_lock;
- struct list_head vfio_next;
- struct list_head container_next;
- enum vfio_group_type type;
- unsigned int dev_counter;
- struct rw_semaphore group_rwsem;
- struct kvm *kvm;
- struct file *opened_file;
- struct blocking_notifier_head notifier;
-};
-
-#ifdef CONFIG_VFIO_NOIOMMU
-static bool noiommu __read_mostly;
-module_param_named(enable_unsafe_noiommu_mode,
- noiommu, bool, S_IRUGO | S_IWUSR);
-MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)");
-#endif
-
static DEFINE_XARRAY(vfio_device_set_xa);
static const struct file_operations vfio_group_fops;
xa_unlock(&vfio_device_set_xa);
}
-#ifdef CONFIG_VFIO_NOIOMMU
-static void *vfio_noiommu_open(unsigned long arg)
-{
- if (arg != VFIO_NOIOMMU_IOMMU)
- return ERR_PTR(-EINVAL);
- if (!capable(CAP_SYS_RAWIO))
- return ERR_PTR(-EPERM);
-
- return NULL;
-}
-
-static void vfio_noiommu_release(void *iommu_data)
-{
-}
-
-static long vfio_noiommu_ioctl(void *iommu_data,
- unsigned int cmd, unsigned long arg)
-{
- if (cmd == VFIO_CHECK_EXTENSION)
- return noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;
-
- return -ENOTTY;
-}
-
-static int vfio_noiommu_attach_group(void *iommu_data,
- struct iommu_group *iommu_group, enum vfio_group_type type)
-{
- return 0;
-}
-
-static void vfio_noiommu_detach_group(void *iommu_data,
- struct iommu_group *iommu_group)
-{
-}
-
-static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {
- .name = "vfio-noiommu",
- .owner = THIS_MODULE,
- .open = vfio_noiommu_open,
- .release = vfio_noiommu_release,
- .ioctl = vfio_noiommu_ioctl,
- .attach_group = vfio_noiommu_attach_group,
- .detach_group = vfio_noiommu_detach_group,
-};
-
-/*
- * Only noiommu containers can use vfio-noiommu and noiommu containers can only
- * use vfio-noiommu.
- */
-static inline bool vfio_iommu_driver_allowed(struct vfio_container *container,
- const struct vfio_iommu_driver *driver)
-{
- return container->noiommu == (driver->ops == &vfio_noiommu_ops);
-}
-#else
-static inline bool vfio_iommu_driver_allowed(struct vfio_container *container,
- const struct vfio_iommu_driver *driver)
-{
- return true;
-}
-#endif /* CONFIG_VFIO_NOIOMMU */
-
-/*
- * IOMMU driver registration
- */
-int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
-{
- struct vfio_iommu_driver *driver, *tmp;
-
- if (WARN_ON(!ops->register_device != !ops->unregister_device))
- return -EINVAL;
-
- driver = kzalloc(sizeof(*driver), GFP_KERNEL);
- if (!driver)
- return -ENOMEM;
-
- driver->ops = ops;
-
- mutex_lock(&vfio.iommu_drivers_lock);
-
- /* Check for duplicates */
- list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
- if (tmp->ops == ops) {
- mutex_unlock(&vfio.iommu_drivers_lock);
- kfree(driver);
- return -EINVAL;
- }
- }
-
- list_add(&driver->vfio_next, &vfio.iommu_drivers_list);
-
- mutex_unlock(&vfio.iommu_drivers_lock);
-
- return 0;
-}
-EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);
-
-void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
-{
- struct vfio_iommu_driver *driver;
-
- mutex_lock(&vfio.iommu_drivers_lock);
- list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
- if (driver->ops == ops) {
- list_del(&driver->vfio_next);
- mutex_unlock(&vfio.iommu_drivers_lock);
- kfree(driver);
- return;
- }
- }
- mutex_unlock(&vfio.iommu_drivers_lock);
-}
-EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);
-
-static void vfio_group_get(struct vfio_group *group);
-
-/*
- * Container objects - containers are created when /dev/vfio/vfio is
- * opened, but their lifecycle extends until the last user is done, so
- * it's freed via kref. Must support container/group/device being
- * closed in any order.
- */
-static void vfio_container_get(struct vfio_container *container)
-{
- kref_get(&container->kref);
-}
-
-static void vfio_container_release(struct kref *kref)
-{
- struct vfio_container *container;
- container = container_of(kref, struct vfio_container, kref);
-
- kfree(container);
-}
-
-static void vfio_container_put(struct vfio_container *container)
-{
- kref_put(&container->kref, vfio_container_release);
-}
-
/*
* Group objects - create, release, get, put, search
*/
{
struct vfio_group *group;
+ /*
+ * group->iommu_group from the vfio.group_list cannot be NULL
+ * under the vfio.group_lock.
+ */
list_for_each_entry(group, &vfio.group_list, vfio_next) {
if (group->iommu_group == iommu_group) {
- vfio_group_get(group);
+ refcount_inc(&group->drivers);
return group;
}
}
struct vfio_group *group = container_of(dev, struct vfio_group, dev);
mutex_destroy(&group->device_lock);
- iommu_group_put(group->iommu_group);
+ mutex_destroy(&group->group_lock);
+ WARN_ON(group->iommu_group);
ida_free(&vfio.group_ida, MINOR(group->dev.devt));
kfree(group);
}
cdev_init(&group->cdev, &vfio_group_fops);
group->cdev.owner = THIS_MODULE;
- refcount_set(&group->users, 1);
- init_rwsem(&group->group_rwsem);
+ refcount_set(&group->drivers, 1);
+ mutex_init(&group->group_lock);
INIT_LIST_HEAD(&group->device_list);
mutex_init(&group->device_lock);
group->iommu_group = iommu_group;
return ret;
}
-static void vfio_group_put(struct vfio_group *group)
+static void vfio_device_remove_group(struct vfio_device *device)
{
- if (!refcount_dec_and_mutex_lock(&group->users, &vfio.group_lock))
+ struct vfio_group *group = device->group;
+ struct iommu_group *iommu_group;
+
+ if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU)
+ iommu_group_remove_device(device->dev);
+
+ /* Pairs with vfio_create_group() / vfio_group_get_from_iommu() */
+ if (!refcount_dec_and_mutex_lock(&group->drivers, &vfio.group_lock))
return;
+ list_del(&group->vfio_next);
+ /*
+ * We could concurrently probe another driver in the group that might
+ * race vfio_device_remove_group() with vfio_get_group(), so we have to
+ * ensure that the sysfs is all cleaned up under lock otherwise the
+ * cdev_device_add() will fail due to the name aready existing.
+ */
+ cdev_device_del(&group->cdev, &group->dev);
+
+ mutex_lock(&group->group_lock);
/*
* These data structures all have paired operations that can only be
- * undone when the caller holds a live reference on the group. Since all
- * pairs must be undone these WARN_ON's indicate some caller did not
+ * undone when the caller holds a live reference on the device. Since
+ * all pairs must be undone these WARN_ON's indicate some caller did not
* properly hold the group reference.
*/
WARN_ON(!list_empty(&group->device_list));
- WARN_ON(group->container || group->container_users);
WARN_ON(group->notifier.head);
- list_del(&group->vfio_next);
- cdev_device_del(&group->cdev, &group->dev);
+ /*
+ * Revoke all users of group->iommu_group. At this point we know there
+ * are no devices active because we are unplugging the last one. Setting
+ * iommu_group to NULL blocks all new users.
+ */
+ if (group->container)
+ vfio_group_detach_container(group);
+ iommu_group = group->iommu_group;
+ group->iommu_group = NULL;
+ mutex_unlock(&group->group_lock);
mutex_unlock(&vfio.group_lock);
+ iommu_group_put(iommu_group);
put_device(&group->dev);
}
-static void vfio_group_get(struct vfio_group *group)
-{
- refcount_inc(&group->users);
-}
-
/*
* Device objects - create, release, get, put, search
*/
/* Device reference always implies a group reference */
-static void vfio_device_put(struct vfio_device *device)
+static void vfio_device_put_registration(struct vfio_device *device)
{
if (refcount_dec_and_test(&device->refcount))
complete(&device->comp);
}
-static bool vfio_device_try_get(struct vfio_device *device)
+static bool vfio_device_try_get_registration(struct vfio_device *device)
{
return refcount_inc_not_zero(&device->refcount);
}
mutex_lock(&group->device_lock);
list_for_each_entry(device, &group->device_list, group_next) {
- if (device->dev == dev && vfio_device_try_get(device)) {
+ if (device->dev == dev &&
+ vfio_device_try_get_registration(device)) {
mutex_unlock(&group->device_lock);
return device;
}
/*
* VFIO driver API
*/
-void vfio_init_group_dev(struct vfio_device *device, struct device *dev,
- const struct vfio_device_ops *ops)
+/* Release helper called by vfio_put_device() */
+static void vfio_device_release(struct device *dev)
{
+ struct vfio_device *device =
+ container_of(dev, struct vfio_device, device);
+
+ vfio_release_device_set(device);
+ ida_free(&vfio.device_ida, device->index);
+
+ /*
+ * kvfree() cannot be done here due to a life cycle mess in
+ * vfio-ccw. Before the ccw part is fixed all drivers are
+ * required to support @release and call vfio_free_device()
+ * from there.
+ */
+ device->ops->release(device);
+}
+
+/*
+ * Allocate and initialize vfio_device so it can be registered to vfio
+ * core.
+ *
+ * Drivers should use the wrapper vfio_alloc_device() for allocation.
+ * @size is the size of the structure to be allocated, including any
+ * private data used by the driver.
+ *
+ * Driver may provide an @init callback to cover device private data.
+ *
+ * Use vfio_put_device() to release the structure after success return.
+ */
+struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
+ const struct vfio_device_ops *ops)
+{
+ struct vfio_device *device;
+ int ret;
+
+ if (WARN_ON(size < sizeof(struct vfio_device)))
+ return ERR_PTR(-EINVAL);
+
+ device = kvzalloc(size, GFP_KERNEL);
+ if (!device)
+ return ERR_PTR(-ENOMEM);
+
+ ret = vfio_init_device(device, dev, ops);
+ if (ret)
+ goto out_free;
+ return device;
+
+out_free:
+ kvfree(device);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(_vfio_alloc_device);
+
+/*
+ * Initialize a vfio_device so it can be registered to vfio core.
+ *
+ * Only vfio-ccw driver should call this interface.
+ */
+int vfio_init_device(struct vfio_device *device, struct device *dev,
+ const struct vfio_device_ops *ops)
+{
+ int ret;
+
+ ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
+ if (ret < 0) {
+ dev_dbg(dev, "Error to alloc index\n");
+ return ret;
+ }
+
+ device->index = ret;
init_completion(&device->comp);
device->dev = dev;
device->ops = ops;
+
+ if (ops->init) {
+ ret = ops->init(device);
+ if (ret)
+ goto out_uninit;
+ }
+
+ device_initialize(&device->device);
+ device->device.release = vfio_device_release;
+ device->device.class = vfio.device_class;
+ device->device.parent = device->dev;
+ return 0;
+
+out_uninit:
+ vfio_release_device_set(device);
+ ida_free(&vfio.device_ida, device->index);
+ return ret;
}
-EXPORT_SYMBOL_GPL(vfio_init_group_dev);
+EXPORT_SYMBOL_GPL(vfio_init_device);
-void vfio_uninit_group_dev(struct vfio_device *device)
+/*
+ * The helper called by driver @release callback to free the device
+ * structure. Drivers which don't have private data to clean can
+ * simply use this helper as its @release.
+ */
+void vfio_free_device(struct vfio_device *device)
{
- vfio_release_device_set(device);
+ kvfree(device);
}
-EXPORT_SYMBOL_GPL(vfio_uninit_group_dev);
+EXPORT_SYMBOL_GPL(vfio_free_device);
static struct vfio_group *vfio_noiommu_group_alloc(struct device *dev,
enum vfio_group_type type)
struct vfio_group *group;
iommu_group = iommu_group_get(dev);
-#ifdef CONFIG_VFIO_NOIOMMU
- if (!iommu_group && noiommu) {
+ if (!iommu_group && vfio_noiommu) {
/*
* With noiommu enabled, create an IOMMU group for devices that
* don't already have one, implying no IOMMU hardware/driver
}
return group;
}
-#endif
+
if (!iommu_group)
return ERR_PTR(-EINVAL);
struct vfio_group *group)
{
struct vfio_device *existing_device;
+ int ret;
+ /*
+ * In all cases group is the output of one of the group allocation
+ * functions and we have group->drivers incremented for us.
+ */
if (IS_ERR(group))
return PTR_ERR(group);
existing_device = vfio_group_get_device(group, device->dev);
if (existing_device) {
+ /*
+ * group->iommu_group is non-NULL because we hold the drivers
+ * refcount.
+ */
dev_WARN(device->dev, "Device already exists on group %d\n",
iommu_group_id(group->iommu_group));
- vfio_device_put(existing_device);
- if (group->type == VFIO_NO_IOMMU ||
- group->type == VFIO_EMULATED_IOMMU)
- iommu_group_remove_device(device->dev);
- vfio_group_put(group);
- return -EBUSY;
+ vfio_device_put_registration(existing_device);
+ ret = -EBUSY;
+ goto err_out;
}
/* Our reference on group is moved to the device */
device->group = group;
+ ret = dev_set_name(&device->device, "vfio%d", device->index);
+ if (ret)
+ goto err_out;
+
+ ret = device_add(&device->device);
+ if (ret)
+ goto err_out;
+
/* Refcounting can't start until the driver calls register */
refcount_set(&device->refcount, 1);
mutex_lock(&group->device_lock);
list_add(&device->group_next, &group->device_list);
- group->dev_counter++;
mutex_unlock(&group->device_lock);
return 0;
+err_out:
+ vfio_device_remove_group(device);
+ return ret;
}
int vfio_register_group_dev(struct vfio_device *device)
ret = !strcmp(dev_name(it->dev), buf);
}
- if (ret && vfio_device_try_get(it)) {
+ if (ret && vfio_device_try_get_registration(it)) {
device = it;
break;
}
bool interrupted = false;
long rc;
- vfio_device_put(device);
+ vfio_device_put_registration(device);
rc = try_wait_for_completion(&device->comp);
while (rc <= 0) {
if (device->ops->request)
mutex_lock(&group->device_lock);
list_del(&device->group_next);
- group->dev_counter--;
mutex_unlock(&group->device_lock);
- if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU)
- iommu_group_remove_device(device->dev);
+ /* Balances device_add in register path */
+ device_del(&device->device);
- /* Matches the get in vfio_register_group_dev() */
- vfio_group_put(group);
+ vfio_device_remove_group(device);
}
EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
-/*
- * VFIO base fd, /dev/vfio/vfio
- */
-static long vfio_ioctl_check_extension(struct vfio_container *container,
- unsigned long arg)
-{
- struct vfio_iommu_driver *driver;
- long ret = 0;
-
- down_read(&container->group_lock);
-
- driver = container->iommu_driver;
-
- switch (arg) {
- /* No base extensions yet */
- default:
- /*
- * If no driver is set, poll all registered drivers for
- * extensions and return the first positive result. If
- * a driver is already set, further queries will be passed
- * only to that driver.
- */
- if (!driver) {
- mutex_lock(&vfio.iommu_drivers_lock);
- list_for_each_entry(driver, &vfio.iommu_drivers_list,
- vfio_next) {
-
- if (!list_empty(&container->group_list) &&
- !vfio_iommu_driver_allowed(container,
- driver))
- continue;
- if (!try_module_get(driver->ops->owner))
- continue;
-
- ret = driver->ops->ioctl(NULL,
- VFIO_CHECK_EXTENSION,
- arg);
- module_put(driver->ops->owner);
- if (ret > 0)
- break;
- }
- mutex_unlock(&vfio.iommu_drivers_lock);
- } else
- ret = driver->ops->ioctl(container->iommu_data,
- VFIO_CHECK_EXTENSION, arg);
- }
-
- up_read(&container->group_lock);
-
- return ret;
-}
-
-/* hold write lock on container->group_lock */
-static int __vfio_container_attach_groups(struct vfio_container *container,
- struct vfio_iommu_driver *driver,
- void *data)
-{
- struct vfio_group *group;
- int ret = -ENODEV;
-
- list_for_each_entry(group, &container->group_list, container_next) {
- ret = driver->ops->attach_group(data, group->iommu_group,
- group->type);
- if (ret)
- goto unwind;
- }
-
- return ret;
-
-unwind:
- list_for_each_entry_continue_reverse(group, &container->group_list,
- container_next) {
- driver->ops->detach_group(data, group->iommu_group);
- }
-
- return ret;
-}
-
-static long vfio_ioctl_set_iommu(struct vfio_container *container,
- unsigned long arg)
-{
- struct vfio_iommu_driver *driver;
- long ret = -ENODEV;
-
- down_write(&container->group_lock);
-
- /*
- * The container is designed to be an unprivileged interface while
- * the group can be assigned to specific users. Therefore, only by
- * adding a group to a container does the user get the privilege of
- * enabling the iommu, which may allocate finite resources. There
- * is no unset_iommu, but by removing all the groups from a container,
- * the container is deprivileged and returns to an unset state.
- */
- if (list_empty(&container->group_list) || container->iommu_driver) {
- up_write(&container->group_lock);
- return -EINVAL;
- }
-
- mutex_lock(&vfio.iommu_drivers_lock);
- list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
- void *data;
-
- if (!vfio_iommu_driver_allowed(container, driver))
- continue;
- if (!try_module_get(driver->ops->owner))
- continue;
-
- /*
- * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
- * so test which iommu driver reported support for this
- * extension and call open on them. We also pass them the
- * magic, allowing a single driver to support multiple
- * interfaces if they'd like.
- */
- if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
- module_put(driver->ops->owner);
- continue;
- }
-
- data = driver->ops->open(arg);
- if (IS_ERR(data)) {
- ret = PTR_ERR(data);
- module_put(driver->ops->owner);
- continue;
- }
-
- ret = __vfio_container_attach_groups(container, driver, data);
- if (ret) {
- driver->ops->release(data);
- module_put(driver->ops->owner);
- continue;
- }
-
- container->iommu_driver = driver;
- container->iommu_data = data;
- break;
- }
-
- mutex_unlock(&vfio.iommu_drivers_lock);
- up_write(&container->group_lock);
-
- return ret;
-}
-
-static long vfio_fops_unl_ioctl(struct file *filep,
- unsigned int cmd, unsigned long arg)
-{
- struct vfio_container *container = filep->private_data;
- struct vfio_iommu_driver *driver;
- void *data;
- long ret = -EINVAL;
-
- if (!container)
- return ret;
-
- switch (cmd) {
- case VFIO_GET_API_VERSION:
- ret = VFIO_API_VERSION;
- break;
- case VFIO_CHECK_EXTENSION:
- ret = vfio_ioctl_check_extension(container, arg);
- break;
- case VFIO_SET_IOMMU:
- ret = vfio_ioctl_set_iommu(container, arg);
- break;
- default:
- driver = container->iommu_driver;
- data = container->iommu_data;
-
- if (driver) /* passthrough all unrecognized ioctls */
- ret = driver->ops->ioctl(data, cmd, arg);
- }
-
- return ret;
-}
-
-static int vfio_fops_open(struct inode *inode, struct file *filep)
-{
- struct vfio_container *container;
-
- container = kzalloc(sizeof(*container), GFP_KERNEL);
- if (!container)
- return -ENOMEM;
-
- INIT_LIST_HEAD(&container->group_list);
- init_rwsem(&container->group_lock);
- kref_init(&container->kref);
-
- filep->private_data = container;
-
- return 0;
-}
-
-static int vfio_fops_release(struct inode *inode, struct file *filep)
-{
- struct vfio_container *container = filep->private_data;
- struct vfio_iommu_driver *driver = container->iommu_driver;
-
- if (driver && driver->ops->notify)
- driver->ops->notify(container->iommu_data,
- VFIO_IOMMU_CONTAINER_CLOSE);
-
- filep->private_data = NULL;
-
- vfio_container_put(container);
-
- return 0;
-}
-
-static const struct file_operations vfio_fops = {
- .owner = THIS_MODULE,
- .open = vfio_fops_open,
- .release = vfio_fops_release,
- .unlocked_ioctl = vfio_fops_unl_ioctl,
- .compat_ioctl = compat_ptr_ioctl,
-};
-
/*
* VFIO Group fd, /dev/vfio/$GROUP
*/
-static void __vfio_group_unset_container(struct vfio_group *group)
-{
- struct vfio_container *container = group->container;
- struct vfio_iommu_driver *driver;
-
- lockdep_assert_held_write(&group->group_rwsem);
-
- down_write(&container->group_lock);
-
- driver = container->iommu_driver;
- if (driver)
- driver->ops->detach_group(container->iommu_data,
- group->iommu_group);
-
- if (group->type == VFIO_IOMMU)
- iommu_group_release_dma_owner(group->iommu_group);
-
- group->container = NULL;
- group->container_users = 0;
- list_del(&group->container_next);
-
- /* Detaching the last group deprivileges a container, remove iommu */
- if (driver && list_empty(&container->group_list)) {
- driver->ops->release(container->iommu_data);
- module_put(driver->ops->owner);
- container->iommu_driver = NULL;
- container->iommu_data = NULL;
- }
-
- up_write(&container->group_lock);
-
- vfio_container_put(container);
-}
-
/*
* VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
* if there was no container to unset. Since the ioctl is called on
* the group, we know that still exists, therefore the only valid
* transition here is 1->0.
*/
-static int vfio_group_unset_container(struct vfio_group *group)
+static int vfio_group_ioctl_unset_container(struct vfio_group *group)
{
- lockdep_assert_held_write(&group->group_rwsem);
+ int ret = 0;
- if (!group->container)
- return -EINVAL;
- if (group->container_users != 1)
- return -EBUSY;
- __vfio_group_unset_container(group);
- return 0;
+ mutex_lock(&group->group_lock);
+ if (!group->container) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+ if (group->container_users != 1) {
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+ vfio_group_detach_container(group);
+
+out_unlock:
+ mutex_unlock(&group->group_lock);
+ return ret;
}
-static int vfio_group_set_container(struct vfio_group *group, int container_fd)
+static int vfio_group_ioctl_set_container(struct vfio_group *group,
+ int __user *arg)
{
- struct fd f;
- struct vfio_container *container;
- struct vfio_iommu_driver *driver;
- int ret = 0;
-
- lockdep_assert_held_write(&group->group_rwsem);
-
- if (group->container || WARN_ON(group->container_users))
- return -EINVAL;
+ struct vfio_container *container;
+ struct fd f;
+ int ret;
+ int fd;
- if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
- return -EPERM;
+ if (get_user(fd, arg))
+ return -EFAULT;
- f = fdget(container_fd);
+ f = fdget(fd);
if (!f.file)
return -EBADF;
- /* Sanity check, is this really our fd? */
- if (f.file->f_op != &vfio_fops) {
- fdput(f);
- return -EINVAL;
- }
-
- container = f.file->private_data;
- WARN_ON(!container); /* fget ensures we don't race vfio_release */
-
- down_write(&container->group_lock);
-
- /* Real groups and fake groups cannot mix */
- if (!list_empty(&container->group_list) &&
- container->noiommu != (group->type == VFIO_NO_IOMMU)) {
- ret = -EPERM;
- goto unlock_out;
+ mutex_lock(&group->group_lock);
+ if (group->container || WARN_ON(group->container_users)) {
+ ret = -EINVAL;
+ goto out_unlock;
}
-
- if (group->type == VFIO_IOMMU) {
- ret = iommu_group_claim_dma_owner(group->iommu_group, f.file);
- if (ret)
- goto unlock_out;
+ if (!group->iommu_group) {
+ ret = -ENODEV;
+ goto out_unlock;
}
- driver = container->iommu_driver;
- if (driver) {
- ret = driver->ops->attach_group(container->iommu_data,
- group->iommu_group,
- group->type);
- if (ret) {
- if (group->type == VFIO_IOMMU)
- iommu_group_release_dma_owner(
- group->iommu_group);
- goto unlock_out;
- }
+ container = vfio_container_from_file(f.file);
+ ret = -EINVAL;
+ if (container) {
+ ret = vfio_container_attach_group(container, group);
+ goto out_unlock;
}
- group->container = container;
- group->container_users = 1;
- container->noiommu = (group->type == VFIO_NO_IOMMU);
- list_add(&group->container_next, &container->group_list);
-
- /* Get a reference on the container and mark a user within the group */
- vfio_container_get(container);
-
-unlock_out:
- up_write(&container->group_lock);
+out_unlock:
+ mutex_unlock(&group->group_lock);
fdput(f);
return ret;
}
static const struct file_operations vfio_device_fops;
/* true if the vfio_device has open_device() called but not close_device() */
-static bool vfio_assert_device_open(struct vfio_device *device)
+bool vfio_assert_device_open(struct vfio_device *device)
{
return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
}
-static int vfio_device_assign_container(struct vfio_device *device)
-{
- struct vfio_group *group = device->group;
-
- lockdep_assert_held_write(&group->group_rwsem);
-
- if (!group->container || !group->container->iommu_driver ||
- WARN_ON(!group->container_users))
- return -EINVAL;
-
- if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
- return -EPERM;
-
- get_file(group->opened_file);
- group->container_users++;
- return 0;
-}
-
-static void vfio_device_unassign_container(struct vfio_device *device)
-{
- down_write(&device->group->group_rwsem);
- WARN_ON(device->group->container_users <= 1);
- device->group->container_users--;
- fput(device->group->opened_file);
- up_write(&device->group->group_rwsem);
-}
-
static struct file *vfio_device_open(struct vfio_device *device)
{
- struct vfio_iommu_driver *iommu_driver;
struct file *filep;
int ret;
- down_write(&device->group->group_rwsem);
+ mutex_lock(&device->group->group_lock);
ret = vfio_device_assign_container(device);
- up_write(&device->group->group_rwsem);
+ mutex_unlock(&device->group->group_lock);
if (ret)
return ERR_PTR(ret);
* lock. If the device driver will use it, it must obtain a
* reference and release it during close_device.
*/
- down_read(&device->group->group_rwsem);
+ mutex_lock(&device->group->group_lock);
device->kvm = device->group->kvm;
if (device->ops->open_device) {
if (ret)
goto err_undo_count;
}
-
- iommu_driver = device->group->container->iommu_driver;
- if (iommu_driver && iommu_driver->ops->register_device)
- iommu_driver->ops->register_device(
- device->group->container->iommu_data, device);
-
- up_read(&device->group->group_rwsem);
+ vfio_device_container_register(device);
+ mutex_unlock(&device->group->group_lock);
}
mutex_unlock(&device->dev_set->lock);
err_close_device:
mutex_lock(&device->dev_set->lock);
- down_read(&device->group->group_rwsem);
+ mutex_lock(&device->group->group_lock);
if (device->open_count == 1 && device->ops->close_device) {
device->ops->close_device(device);
- iommu_driver = device->group->container->iommu_driver;
- if (iommu_driver && iommu_driver->ops->unregister_device)
- iommu_driver->ops->unregister_device(
- device->group->container->iommu_data, device);
+ vfio_device_container_unregister(device);
}
err_undo_count:
- up_read(&device->group->group_rwsem);
+ mutex_unlock(&device->group->group_lock);
device->open_count--;
if (device->open_count == 0 && device->kvm)
device->kvm = NULL;
return ERR_PTR(ret);
}
-static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
+static int vfio_group_ioctl_get_device_fd(struct vfio_group *group,
+ char __user *arg)
{
struct vfio_device *device;
struct file *filep;
+ char *buf;
int fdno;
int ret;
+ buf = strndup_user(arg, PAGE_SIZE);
+ if (IS_ERR(buf))
+ return PTR_ERR(buf);
+
device = vfio_device_get_from_name(group, buf);
+ kfree(buf);
if (IS_ERR(device))
return PTR_ERR(device);
err_put_fdno:
put_unused_fd(fdno);
err_put_device:
- vfio_device_put(device);
+ vfio_device_put_registration(device);
return ret;
}
-static long vfio_group_fops_unl_ioctl(struct file *filep,
- unsigned int cmd, unsigned long arg)
+static int vfio_group_ioctl_get_status(struct vfio_group *group,
+ struct vfio_group_status __user *arg)
{
- struct vfio_group *group = filep->private_data;
- long ret = -ENOTTY;
+ unsigned long minsz = offsetofend(struct vfio_group_status, flags);
+ struct vfio_group_status status;
- switch (cmd) {
- case VFIO_GROUP_GET_STATUS:
- {
- struct vfio_group_status status;
- unsigned long minsz;
+ if (copy_from_user(&status, arg, minsz))
+ return -EFAULT;
- minsz = offsetofend(struct vfio_group_status, flags);
+ if (status.argsz < minsz)
+ return -EINVAL;
- if (copy_from_user(&status, (void __user *)arg, minsz))
- return -EFAULT;
+ status.flags = 0;
- if (status.argsz < minsz)
- return -EINVAL;
+ mutex_lock(&group->group_lock);
+ if (!group->iommu_group) {
+ mutex_unlock(&group->group_lock);
+ return -ENODEV;
+ }
- status.flags = 0;
+ if (group->container)
+ status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET |
+ VFIO_GROUP_FLAGS_VIABLE;
+ else if (!iommu_group_dma_owner_claimed(group->iommu_group))
+ status.flags |= VFIO_GROUP_FLAGS_VIABLE;
+ mutex_unlock(&group->group_lock);
- down_read(&group->group_rwsem);
- if (group->container)
- status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET |
- VFIO_GROUP_FLAGS_VIABLE;
- else if (!iommu_group_dma_owner_claimed(group->iommu_group))
- status.flags |= VFIO_GROUP_FLAGS_VIABLE;
- up_read(&group->group_rwsem);
+ if (copy_to_user(arg, &status, minsz))
+ return -EFAULT;
+ return 0;
+}
- if (copy_to_user((void __user *)arg, &status, minsz))
- return -EFAULT;
+static long vfio_group_fops_unl_ioctl(struct file *filep,
+ unsigned int cmd, unsigned long arg)
+{
+ struct vfio_group *group = filep->private_data;
+ void __user *uarg = (void __user *)arg;
- ret = 0;
- break;
- }
+ switch (cmd) {
+ case VFIO_GROUP_GET_DEVICE_FD:
+ return vfio_group_ioctl_get_device_fd(group, uarg);
+ case VFIO_GROUP_GET_STATUS:
+ return vfio_group_ioctl_get_status(group, uarg);
case VFIO_GROUP_SET_CONTAINER:
- {
- int fd;
-
- if (get_user(fd, (int __user *)arg))
- return -EFAULT;
-
- if (fd < 0)
- return -EINVAL;
-
- down_write(&group->group_rwsem);
- ret = vfio_group_set_container(group, fd);
- up_write(&group->group_rwsem);
- break;
- }
+ return vfio_group_ioctl_set_container(group, uarg);
case VFIO_GROUP_UNSET_CONTAINER:
- down_write(&group->group_rwsem);
- ret = vfio_group_unset_container(group);
- up_write(&group->group_rwsem);
- break;
- case VFIO_GROUP_GET_DEVICE_FD:
- {
- char *buf;
-
- buf = strndup_user((const char __user *)arg, PAGE_SIZE);
- if (IS_ERR(buf))
- return PTR_ERR(buf);
-
- ret = vfio_group_get_device_fd(group, buf);
- kfree(buf);
- break;
- }
+ return vfio_group_ioctl_unset_container(group);
+ default:
+ return -ENOTTY;
}
-
- return ret;
}
static int vfio_group_fops_open(struct inode *inode, struct file *filep)
container_of(inode->i_cdev, struct vfio_group, cdev);
int ret;
- down_write(&group->group_rwsem);
+ mutex_lock(&group->group_lock);
- /* users can be zero if this races with vfio_group_put() */
- if (!refcount_inc_not_zero(&group->users)) {
+ /*
+ * drivers can be zero if this races with vfio_device_remove_group(), it
+ * will be stable at 0 under the group rwsem
+ */
+ if (refcount_read(&group->drivers) == 0) {
ret = -ENODEV;
- goto err_unlock;
+ goto out_unlock;
}
if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) {
ret = -EPERM;
- goto err_put;
+ goto out_unlock;
}
/*
*/
if (group->opened_file) {
ret = -EBUSY;
- goto err_put;
+ goto out_unlock;
}
group->opened_file = filep;
filep->private_data = group;
-
- up_write(&group->group_rwsem);
- return 0;
-err_put:
- vfio_group_put(group);
-err_unlock:
- up_write(&group->group_rwsem);
+ ret = 0;
+out_unlock:
+ mutex_unlock(&group->group_lock);
return ret;
}
filep->private_data = NULL;
- down_write(&group->group_rwsem);
+ mutex_lock(&group->group_lock);
/*
* Device FDs hold a group file reference, therefore the group release
* is only called when there are no open devices.
*/
WARN_ON(group->notifier.head);
- if (group->container) {
- WARN_ON(group->container_users != 1);
- __vfio_group_unset_container(group);
- }
+ if (group->container)
+ vfio_group_detach_container(group);
group->opened_file = NULL;
- up_write(&group->group_rwsem);
-
- vfio_group_put(group);
-
+ mutex_unlock(&group->group_lock);
return 0;
}
.release = vfio_group_fops_release,
};
+/*
+ * Wrapper around pm_runtime_resume_and_get().
+ * Return error code on failure or 0 on success.
+ */
+static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
+{
+ struct device *dev = device->dev;
+
+ if (dev->driver && dev->driver->pm) {
+ int ret;
+
+ ret = pm_runtime_resume_and_get(dev);
+ if (ret) {
+ dev_info_ratelimited(dev,
+ "vfio: runtime resume failed %d\n", ret);
+ return -EIO;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Wrapper around pm_runtime_put().
+ */
+static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
+{
+ struct device *dev = device->dev;
+
+ if (dev->driver && dev->driver->pm)
+ pm_runtime_put(dev);
+}
+
/*
* VFIO Device fd
*/
static int vfio_device_fops_release(struct inode *inode, struct file *filep)
{
struct vfio_device *device = filep->private_data;
- struct vfio_iommu_driver *iommu_driver;
mutex_lock(&device->dev_set->lock);
vfio_assert_device_open(device);
- down_read(&device->group->group_rwsem);
+ mutex_lock(&device->group->group_lock);
if (device->open_count == 1 && device->ops->close_device)
device->ops->close_device(device);
- iommu_driver = device->group->container->iommu_driver;
- if (iommu_driver && iommu_driver->ops->unregister_device)
- iommu_driver->ops->unregister_device(
- device->group->container->iommu_data, device);
- up_read(&device->group->group_rwsem);
+ vfio_device_container_unregister(device);
+ mutex_unlock(&device->group->group_lock);
device->open_count--;
if (device->open_count == 0)
device->kvm = NULL;
vfio_device_unassign_container(device);
- vfio_device_put(device);
+ vfio_device_put_registration(device);
return 0;
}
return 0;
}
+/* Ranges should fit into a single kernel page */
+#define LOG_MAX_RANGES \
+ (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
+
+static int
+vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
+ u32 flags, void __user *arg,
+ size_t argsz)
+{
+ size_t minsz =
+ offsetofend(struct vfio_device_feature_dma_logging_control,
+ ranges);
+ struct vfio_device_feature_dma_logging_range __user *ranges;
+ struct vfio_device_feature_dma_logging_control control;
+ struct vfio_device_feature_dma_logging_range range;
+ struct rb_root_cached root = RB_ROOT_CACHED;
+ struct interval_tree_node *nodes;
+ u64 iova_end;
+ u32 nnodes;
+ int i, ret;
+
+ if (!device->log_ops)
+ return -ENOTTY;
+
+ ret = vfio_check_feature(flags, argsz,
+ VFIO_DEVICE_FEATURE_SET,
+ sizeof(control));
+ if (ret != 1)
+ return ret;
+
+ if (copy_from_user(&control, arg, minsz))
+ return -EFAULT;
+
+ nnodes = control.num_ranges;
+ if (!nnodes)
+ return -EINVAL;
+
+ if (nnodes > LOG_MAX_RANGES)
+ return -E2BIG;
+
+ ranges = u64_to_user_ptr(control.ranges);
+ nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
+ GFP_KERNEL);
+ if (!nodes)
+ return -ENOMEM;
+
+ for (i = 0; i < nnodes; i++) {
+ if (copy_from_user(&range, &ranges[i], sizeof(range))) {
+ ret = -EFAULT;
+ goto end;
+ }
+ if (!IS_ALIGNED(range.iova, control.page_size) ||
+ !IS_ALIGNED(range.length, control.page_size)) {
+ ret = -EINVAL;
+ goto end;
+ }
+
+ if (check_add_overflow(range.iova, range.length, &iova_end) ||
+ iova_end > ULONG_MAX) {
+ ret = -EOVERFLOW;
+ goto end;
+ }
+
+ nodes[i].start = range.iova;
+ nodes[i].last = range.iova + range.length - 1;
+ if (interval_tree_iter_first(&root, nodes[i].start,
+ nodes[i].last)) {
+ /* Range overlapping */
+ ret = -EINVAL;
+ goto end;
+ }
+ interval_tree_insert(nodes + i, &root);
+ }
+
+ ret = device->log_ops->log_start(device, &root, nnodes,
+ &control.page_size);
+ if (ret)
+ goto end;
+
+ if (copy_to_user(arg, &control, sizeof(control))) {
+ ret = -EFAULT;
+ device->log_ops->log_stop(device);
+ }
+
+end:
+ kfree(nodes);
+ return ret;
+}
+
+static int
+vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
+ u32 flags, void __user *arg,
+ size_t argsz)
+{
+ int ret;
+
+ if (!device->log_ops)
+ return -ENOTTY;
+
+ ret = vfio_check_feature(flags, argsz,
+ VFIO_DEVICE_FEATURE_SET, 0);
+ if (ret != 1)
+ return ret;
+
+ return device->log_ops->log_stop(device);
+}
+
+static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
+ unsigned long iova, size_t length,
+ void *opaque)
+{
+ struct vfio_device *device = opaque;
+
+ return device->log_ops->log_read_and_clear(device, iova, length, iter);
+}
+
+static int
+vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
+ u32 flags, void __user *arg,
+ size_t argsz)
+{
+ size_t minsz =
+ offsetofend(struct vfio_device_feature_dma_logging_report,
+ bitmap);
+ struct vfio_device_feature_dma_logging_report report;
+ struct iova_bitmap *iter;
+ u64 iova_end;
+ int ret;
+
+ if (!device->log_ops)
+ return -ENOTTY;
+
+ ret = vfio_check_feature(flags, argsz,
+ VFIO_DEVICE_FEATURE_GET,
+ sizeof(report));
+ if (ret != 1)
+ return ret;
+
+ if (copy_from_user(&report, arg, minsz))
+ return -EFAULT;
+
+ if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
+ return -EINVAL;
+
+ if (check_add_overflow(report.iova, report.length, &iova_end) ||
+ iova_end > ULONG_MAX)
+ return -EOVERFLOW;
+
+ iter = iova_bitmap_alloc(report.iova, report.length,
+ report.page_size,
+ u64_to_user_ptr(report.bitmap));
+ if (IS_ERR(iter))
+ return PTR_ERR(iter);
+
+ ret = iova_bitmap_for_each(iter, device,
+ vfio_device_log_read_and_clear);
+
+ iova_bitmap_free(iter);
+ return ret;
+}
+
static int vfio_ioctl_device_feature(struct vfio_device *device,
struct vfio_device_feature __user *arg)
{
return vfio_ioctl_device_feature_mig_device_state(
device, feature.flags, arg->data,
feature.argsz - minsz);
+ case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
+ return vfio_ioctl_device_feature_logging_start(
+ device, feature.flags, arg->data,
+ feature.argsz - minsz);
+ case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
+ return vfio_ioctl_device_feature_logging_stop(
+ device, feature.flags, arg->data,
+ feature.argsz - minsz);
+ case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
+ return vfio_ioctl_device_feature_logging_report(
+ device, feature.flags, arg->data,
+ feature.argsz - minsz);
default:
if (unlikely(!device->ops->device_feature))
return -EINVAL;
unsigned int cmd, unsigned long arg)
{
struct vfio_device *device = filep->private_data;
+ int ret;
+
+ ret = vfio_device_pm_runtime_get(device);
+ if (ret)
+ return ret;
switch (cmd) {
case VFIO_DEVICE_FEATURE:
- return vfio_ioctl_device_feature(device, (void __user *)arg);
+ ret = vfio_ioctl_device_feature(device, (void __user *)arg);
+ break;
+
default:
if (unlikely(!device->ops->ioctl))
- return -EINVAL;
- return device->ops->ioctl(device, cmd, arg);
+ ret = -EINVAL;
+ else
+ ret = device->ops->ioctl(device, cmd, arg);
+ break;
}
+
+ vfio_device_pm_runtime_put(device);
+ return ret;
}
static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
* vfio_file_iommu_group - Return the struct iommu_group for the vfio group file
* @file: VFIO group file
*
- * The returned iommu_group is valid as long as a ref is held on the file.
+ * The returned iommu_group is valid as long as a ref is held on the file. This
+ * returns a reference on the group. This function is deprecated, only the SPAPR
+ * path in kvm should call it.
*/
struct iommu_group *vfio_file_iommu_group(struct file *file)
{
struct vfio_group *group = file->private_data;
+ struct iommu_group *iommu_group = NULL;
- if (file->f_op != &vfio_group_fops)
+ if (!IS_ENABLED(CONFIG_SPAPR_TCE_IOMMU))
return NULL;
- return group->iommu_group;
+
+ if (!vfio_file_is_group(file))
+ return NULL;
+
+ mutex_lock(&group->group_lock);
+ if (group->iommu_group) {
+ iommu_group = group->iommu_group;
+ iommu_group_ref_get(iommu_group);
+ }
+ mutex_unlock(&group->group_lock);
+ return iommu_group;
}
EXPORT_SYMBOL_GPL(vfio_file_iommu_group);
+/**
+ * vfio_file_is_group - True if the file is usable with VFIO aPIS
+ * @file: VFIO group file
+ */
+bool vfio_file_is_group(struct file *file)
+{
+ return file->f_op == &vfio_group_fops;
+}
+EXPORT_SYMBOL_GPL(vfio_file_is_group);
+
/**
* vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
* is always CPU cache coherent
struct vfio_group *group = file->private_data;
bool ret;
- if (file->f_op != &vfio_group_fops)
+ if (!vfio_file_is_group(file))
return true;
- down_read(&group->group_rwsem);
+ mutex_lock(&group->group_lock);
if (group->container) {
- ret = vfio_ioctl_check_extension(group->container,
- VFIO_DMA_CC_IOMMU);
+ ret = vfio_container_ioctl_check_extension(group->container,
+ VFIO_DMA_CC_IOMMU);
} else {
/*
* Since the coherency state is determined only once a container
*/
ret = true;
}
- up_read(&group->group_rwsem);
+ mutex_unlock(&group->group_lock);
return ret;
}
EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
{
struct vfio_group *group = file->private_data;
- if (file->f_op != &vfio_group_fops)
+ if (!vfio_file_is_group(file))
return;
- down_write(&group->group_rwsem);
+ mutex_lock(&group->group_lock);
group->kvm = kvm;
- up_write(&group->group_rwsem);
+ mutex_unlock(&group->group_lock);
}
EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
{
struct vfio_group *group = file->private_data;
- if (file->f_op != &vfio_group_fops)
+ if (!vfio_file_is_group(file))
return false;
return group == device->group;
}
EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
-/*
- * Pin contiguous user pages and return their associated host pages for local
- * domain only.
- * @device [in] : device
- * @iova [in] : starting IOVA of user pages to be pinned.
- * @npage [in] : count of pages to be pinned. This count should not
- * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
- * @prot [in] : protection flags
- * @pages[out] : array of host pages
- * Return error or number of pages pinned.
- */
-int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
- int npage, int prot, struct page **pages)
-{
- struct vfio_container *container;
- struct vfio_group *group = device->group;
- struct vfio_iommu_driver *driver;
- int ret;
-
- if (!pages || !npage || !vfio_assert_device_open(device))
- return -EINVAL;
-
- if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
- return -E2BIG;
-
- if (group->dev_counter > 1)
- return -EINVAL;
-
- /* group->container cannot change while a vfio device is open */
- container = group->container;
- driver = container->iommu_driver;
- if (likely(driver && driver->ops->pin_pages))
- ret = driver->ops->pin_pages(container->iommu_data,
- group->iommu_group, iova,
- npage, prot, pages);
- else
- ret = -ENOTTY;
-
- return ret;
-}
-EXPORT_SYMBOL(vfio_pin_pages);
-
-/*
- * Unpin contiguous host pages for local domain only.
- * @device [in] : device
- * @iova [in] : starting address of user pages to be unpinned.
- * @npage [in] : count of pages to be unpinned. This count should not
- * be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
- */
-void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
-{
- struct vfio_container *container;
- struct vfio_iommu_driver *driver;
-
- if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES))
- return;
-
- if (WARN_ON(!vfio_assert_device_open(device)))
- return;
-
- /* group->container cannot change while a vfio device is open */
- container = device->group->container;
- driver = container->iommu_driver;
-
- driver->ops->unpin_pages(container->iommu_data, iova, npage);
-}
-EXPORT_SYMBOL(vfio_unpin_pages);
-
-/*
- * This interface allows the CPUs to perform some sort of virtual DMA on
- * behalf of the device.
- *
- * CPUs read/write from/into a range of IOVAs pointing to user space memory
- * into/from a kernel buffer.
- *
- * As the read/write of user space memory is conducted via the CPUs and is
- * not a real device DMA, it is not necessary to pin the user space memory.
- *
- * @device [in] : VFIO device
- * @iova [in] : base IOVA of a user space buffer
- * @data [in] : pointer to kernel buffer
- * @len [in] : kernel buffer length
- * @write : indicate read or write
- * Return error code on failure or 0 on success.
- */
-int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
- size_t len, bool write)
-{
- struct vfio_container *container;
- struct vfio_iommu_driver *driver;
- int ret = 0;
-
- if (!data || len <= 0 || !vfio_assert_device_open(device))
- return -EINVAL;
-
- /* group->container cannot change while a vfio device is open */
- container = device->group->container;
- driver = container->iommu_driver;
-
- if (likely(driver && driver->ops->dma_rw))
- ret = driver->ops->dma_rw(container->iommu_data,
- iova, data, len, write);
- else
- ret = -ENOTTY;
- return ret;
-}
-EXPORT_SYMBOL(vfio_dma_rw);
-
/*
* Module/class support
*/
return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev));
}
-static struct miscdevice vfio_dev = {
- .minor = VFIO_MINOR,
- .name = "vfio",
- .fops = &vfio_fops,
- .nodename = "vfio/vfio",
- .mode = S_IRUGO | S_IWUGO,
-};
-
static int __init vfio_init(void)
{
int ret;
ida_init(&vfio.group_ida);
+ ida_init(&vfio.device_ida);
mutex_init(&vfio.group_lock);
- mutex_init(&vfio.iommu_drivers_lock);
INIT_LIST_HEAD(&vfio.group_list);
- INIT_LIST_HEAD(&vfio.iommu_drivers_list);
- ret = misc_register(&vfio_dev);
- if (ret) {
- pr_err("vfio: misc device register failed\n");
+ ret = vfio_container_init();
+ if (ret)
return ret;
- }
/* /dev/vfio/$GROUP */
vfio.class = class_create(THIS_MODULE, "vfio");
if (IS_ERR(vfio.class)) {
ret = PTR_ERR(vfio.class);
- goto err_class;
+ goto err_group_class;
}
vfio.class->devnode = vfio_devnode;
+ /* /sys/class/vfio-dev/vfioX */
+ vfio.device_class = class_create(THIS_MODULE, "vfio-dev");
+ if (IS_ERR(vfio.device_class)) {
+ ret = PTR_ERR(vfio.device_class);
+ goto err_dev_class;
+ }
+
ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio");
if (ret)
goto err_alloc_chrdev;
-#ifdef CONFIG_VFIO_NOIOMMU
- ret = vfio_register_iommu_driver(&vfio_noiommu_ops);
-#endif
- if (ret)
- goto err_driver_register;
-
pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
return 0;
-err_driver_register:
- unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
err_alloc_chrdev:
+ class_destroy(vfio.device_class);
+ vfio.device_class = NULL;
+err_dev_class:
class_destroy(vfio.class);
vfio.class = NULL;
-err_class:
- misc_deregister(&vfio_dev);
+err_group_class:
+ vfio_container_cleanup();
return ret;
}
{
WARN_ON(!list_empty(&vfio.group_list));
-#ifdef CONFIG_VFIO_NOIOMMU
- vfio_unregister_iommu_driver(&vfio_noiommu_ops);
-#endif
+ ida_destroy(&vfio.device_ida);
ida_destroy(&vfio.group_ida);
unregister_chrdev_region(vfio.group_devt, MINORMASK + 1);
+ class_destroy(vfio.device_class);
+ vfio.device_class = NULL;
class_destroy(vfio.class);
+ vfio_container_cleanup();
vfio.class = NULL;
- misc_deregister(&vfio_dev);
xa_destroy(&vfio_device_set_xa);
}
.fops = &vhost_net_fops,
};
-static int vhost_net_init(void)
+static int __init vhost_net_init(void)
{
if (experimental_zcopytx)
vhost_net_enable_zcopy(VHOST_NET_VQ_TX);
}
module_init(vhost_net_init);
-static void vhost_net_exit(void)
+static void __exit vhost_net_exit(void)
{
misc_deregister(&vhost_net_misc);
}
void *data;
};
-
-static inline u8 dac_read_reg(struct dac_info *info, u8 reg)
-{
- u8 code[2] = {reg, 0};
- info->dac_read_regs(info->data, code, 1);
- return code[1];
-}
-
static inline void dac_read_regs(struct dac_info *info, u8 *code, int count)
{
info->dac_read_regs(info->data, code, count);
return (!DIRTY(cmode) && !DIRTY(xres) && !DIRTY(yres)
&& !DIRTY(vxres) && !DIRTY(vyres));
}
-static inline int VAR_MATCH(struct fb_var_screeninfo *x, struct fb_var_screeninfo *y)
-{
- return (!DIRTY(bits_per_pixel) && !DIRTY(xres)
- && !DIRTY(yres) && !DIRTY(xres_virtual)
- && !DIRTY(yres_virtual)
- && !DIRTY_CMAP(red) && !DIRTY_CMAP(green) && !DIRTY_CMAP(blue));
-}
struct fb_info_control {
struct fb_info info;
static DEVICE_ATTR(revision, S_IRUGO, gbefb_show_rev, NULL);
-static void gbefb_remove_sysfs(struct device *dev)
-{
- device_remove_file(dev, &dev_attr_size);
- device_remove_file(dev, &dev_attr_revision);
-}
-
-static void gbefb_create_sysfs(struct device *dev)
-{
- device_create_file(dev, &dev_attr_size);
- device_create_file(dev, &dev_attr_revision);
-}
+static struct attribute *gbefb_attrs[] = {
+ &dev_attr_size.attr,
+ &dev_attr_revision.attr,
+ NULL,
+};
+ATTRIBUTE_GROUPS(gbefb);
/*
* Initialization
}
platform_set_drvdata(p_dev, info);
- gbefb_create_sysfs(&p_dev->dev);
fb_info(info, "%s rev %d @ 0x%08x using %dkB memory\n",
info->fix.id, gbe_revision, (unsigned)GBE_BASE,
gbe_turn_off();
arch_phys_wc_del(par->wc_cookie);
release_mem_region(GBE_BASE, sizeof(struct sgi_gbe));
- gbefb_remove_sysfs(&p_dev->dev);
framebuffer_release(info);
return 0;
.remove = gbefb_remove,
.driver = {
.name = "gbefb",
+ .dev_groups = gbefb_groups,
},
};
fbi->regs = devm_ioremap_resource(&pdev->dev, res);
if (IS_ERR(fbi->regs)) {
- dev_err(&pdev->dev, "Cannot map frame buffer registers\n");
ret = PTR_ERR(fbi->regs);
goto failed_ioremap;
}
par->dev = dev;
par->irq = irq_of_parse_and_map(np, 0);
- if (par->irq == NO_IRQ) {
+ if (!par->irq) {
dev_err(dev, "failed to map irq\n");
ret = -ENODEV;
goto fbrel;
DSSDBG("dispc_runtime_get\n");
- r = pm_runtime_get_sync(&dispc.pdev->dev);
- if (WARN_ON(r < 0)) {
- pm_runtime_put_sync(&dispc.pdev->dev);
+ r = pm_runtime_resume_and_get(&dispc.pdev->dev);
+ if (WARN_ON(r < 0))
return r;
- }
return 0;
}
EXPORT_SYMBOL(dispc_runtime_get);
DSSDBG("dsi_runtime_get\n");
- r = pm_runtime_get_sync(&dsi->pdev->dev);
- if (WARN_ON(r < 0)) {
- pm_runtime_put_sync(&dsi->pdev->dev);
+ r = pm_runtime_resume_and_get(&dsi->pdev->dev);
+ if (WARN_ON(r < 0))
return r;
- }
return 0;
}
DSSDBG("dss_runtime_get\n");
- r = pm_runtime_get_sync(&dss.pdev->dev);
- if (WARN_ON(r < 0)) {
- pm_runtime_put_sync(&dss.pdev->dev);
+ r = pm_runtime_resume_and_get(&dss.pdev->dev);
+ if (WARN_ON(r < 0))
return r;
- }
return 0;
}
DSSDBG("hdmi_runtime_get\n");
- r = pm_runtime_get_sync(&hdmi.pdev->dev);
- if (WARN_ON(r < 0)) {
- pm_runtime_put_sync(&hdmi.pdev->dev);
+ r = pm_runtime_resume_and_get(&hdmi.pdev->dev);
+ if (WARN_ON(r < 0))
return r;
- }
return 0;
}
DSSDBG("hdmi_runtime_get\n");
- r = pm_runtime_get_sync(&hdmi.pdev->dev);
- if (WARN_ON(r < 0)) {
- pm_runtime_put_sync(&hdmi.pdev->dev);
+ r = pm_runtime_resume_and_get(&hdmi.pdev->dev);
+ if (WARN_ON(r < 0))
return r;
- }
return 0;
}
DSSDBG("venc_runtime_get\n");
- r = pm_runtime_get_sync(&venc.pdev->dev);
- if (WARN_ON(r < 0)) {
- pm_runtime_put_sync(&venc.pdev->dev);
+ r = pm_runtime_resume_and_get(&venc.pdev->dev);
+ if (WARN_ON(r < 0))
return r;
- }
return 0;
}
static int ufx_alloc_urb_list(struct ufx_data *dev, int count, size_t size);
static void ufx_free_urb_list(struct ufx_data *dev);
+static DEFINE_MUTEX(disconnect_mutex);
+
/* reads a control register */
static int ufx_reg_read(struct ufx_data *dev, u32 index, u32 *data)
{
if (user == 0 && !console)
return -EBUSY;
+ mutex_lock(&disconnect_mutex);
+
/* If the USB device is gone, we don't accept new opens */
- if (dev->virtualized)
+ if (dev->virtualized) {
+ mutex_unlock(&disconnect_mutex);
return -ENODEV;
+ }
dev->fb_count++;
pr_debug("open /dev/fb%d user=%d fb_info=%p count=%d",
info->node, user, info, dev->fb_count);
+ mutex_unlock(&disconnect_mutex);
+
return 0;
}
{
struct ufx_data *dev;
+ mutex_lock(&disconnect_mutex);
+
dev = usb_get_intfdata(interface);
pr_debug("USB disconnect starting\n");
kref_put(&dev->kref, ufx_free);
/* consider ufx_data freed */
+
+ mutex_unlock(&disconnect_mutex);
}
static struct usb_driver ufx_driver = {
/* limit fbsize to max visible screen size */
if (fix->smem_len > yres*fix->line_length)
- fix->smem_len = yres*fix->line_length;
+ fix->smem_len = ALIGN(yres*fix->line_length, 4*1024*1024);
fix->accel = FB_ACCEL_NONE;
write3CE(par, CyberControl, read3CE(par, CyberControl) | 0x81);
}
-static inline void shadowmode_off(struct tridentfb_par *par)
-{
- write3CE(par, CyberControl, read3CE(par, CyberControl) & 0x7E);
-}
-
/* Set the hardware to the requested video mode */
static int tridentfb_set_par(struct fb_info *info)
{
if (err)
return err;
- err = pci_enable_device(dev);
+ err = pcim_enable_device(dev);
if (err)
return err;
kfree(info->pixmap.addr);
if (info->screen_base)
iounmap(info->screen_base);
- release_mem_region(tridentfb_fix.smem_start, tridentfb_fix.smem_len);
disable_mmio(info->par);
out_unmap1:
if (default_par->io_virt)
iounmap(default_par->io_virt);
- release_mem_region(tridentfb_fix.mmio_start, tridentfb_fix.mmio_len);
framebuffer_release(info);
return err;
}
i2c_del_adapter(&par->ddc_adapter);
iounmap(par->io_virt);
iounmap(info->screen_base);
- release_mem_region(tridentfb_fix.smem_start, tridentfb_fix.smem_len);
- release_mem_region(tridentfb_fix.mmio_start, tridentfb_fix.mmio_len);
kfree(info->pixmap.addr);
fb_dealloc_cmap(&info->cmap);
framebuffer_release(info);
const unsigned long *back = (const unsigned long *) bback;
const unsigned long *front = (const unsigned long *) *bfront;
const int width = *width_bytes / sizeof(unsigned long);
- int identical = width;
+ int identical;
int start = width;
int end = width;
memcpy(&m->id, &uvesafb_cn_id, sizeof(m->id));
m->seq = seq;
m->len = len;
- m->ack = prandom_u32();
+ m->ack = get_random_u32();
/* uvesafb_task structure */
memcpy(m + 1, &task->t, sizeof(task->t));
struct uvesafb_par *par = info->par;
if (par->vbe_ib.oem_vendor_name_ptr)
- return snprintf(buf, PAGE_SIZE, "%s\n", (char *)
+ return scnprintf(buf, PAGE_SIZE, "%s\n", (char *)
(&par->vbe_ib) + par->vbe_ib.oem_vendor_name_ptr);
else
return 0;
struct uvesafb_par *par = info->par;
if (par->vbe_ib.oem_product_name_ptr)
- return snprintf(buf, PAGE_SIZE, "%s\n", (char *)
+ return scnprintf(buf, PAGE_SIZE, "%s\n", (char *)
(&par->vbe_ib) + par->vbe_ib.oem_product_name_ptr);
else
return 0;
struct uvesafb_par *par = info->par;
if (par->vbe_ib.oem_product_rev_ptr)
- return snprintf(buf, PAGE_SIZE, "%s\n", (char *)
+ return scnprintf(buf, PAGE_SIZE, "%s\n", (char *)
(&par->vbe_ib) + par->vbe_ib.oem_product_rev_ptr);
else
return 0;
struct uvesafb_par *par = info->par;
if (par->vbe_ib.oem_string_ptr)
- return snprintf(buf, PAGE_SIZE, "%s\n",
+ return scnprintf(buf, PAGE_SIZE, "%s\n",
(char *)(&par->vbe_ib) + par->vbe_ib.oem_string_ptr);
else
return 0;
struct fb_info *info = dev_get_drvdata(dev);
struct uvesafb_par *par = info->par;
- return snprintf(buf, PAGE_SIZE, "%d\n", par->nocrtc);
+ return scnprintf(buf, PAGE_SIZE, "%d\n", par->nocrtc);
}
static ssize_t uvesafb_store_nocrtc(struct device *dev,
{"vga-framebuffer", 0},
{ }
};
+MODULE_DEVICE_TABLE(platform, vga16fb_driver_id_table);
static struct platform_driver vga16fb_driver = {
.probe = vga16fb_probe,
err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc);
if (!err)
return 0;
+ /* Is there an interrupt? If not give up. */
+ if (!(to_vp_device(vdev)->pci_dev->irq))
+ return err;
/* Finally fall back to regular interrupts. */
return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx);
}
#include <linux/module.h>
#include <linux/hrtimer.h>
#include <linux/dma-mapping.h>
+#include <linux/kmsan.h>
#include <linux/spinlock.h>
#include <xen/xen.h>
struct scatterlist *sg,
enum dma_data_direction direction)
{
- if (!vq->use_dma_api)
+ if (!vq->use_dma_api) {
+ /*
+ * If DMA is not used, KMSAN doesn't know that the scatterlist
+ * is initialized by the hardware. Explicitly check/unpoison it
+ * depending on the direction.
+ */
+ kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction);
return (dma_addr_t)sg_phys(sg);
+ }
/*
* We can't use dma_map_sg, because we don't use scatterlists in
if (!queue) {
/* Try to get a single page. You are my only hope! */
queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
- &dma_addr, GFP_KERNEL|__GFP_ZERO);
+ &dma_addr, GFP_KERNEL | __GFP_ZERO);
}
if (!queue)
return -ENOMEM;
ring = vring_alloc_queue(vdev, ring_size_in_bytes,
&ring_dma_addr,
- GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
+ GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
if (!ring)
goto err;
driver = vring_alloc_queue(vdev, event_size_in_bytes,
&driver_event_dma_addr,
- GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
+ GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
if (!driver)
goto err;
device = vring_alloc_queue(vdev, event_size_in_bytes,
&device_event_dma_addr,
- GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
+ GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
if (!device)
goto err;
WinSystems EBC-C384 motherboard. The timeout may be configured via
the timeout module parameter.
+config EXAR_WDT
+ tristate "Exar Watchdog Timer"
+ depends on X86
+ select WATCHDOG_CORE
+ help
+ Enables watchdog timer support for the watchdog timer present
+ in some Exar/MaxLinear UART chips like the XR28V38x.
+
+ To compile this driver as a module, choose M here: the
+ module will be called exar_wdt.
+
config F71808E_WDT
tristate "Fintek F718xx, F818xx Super I/O Watchdog"
depends on X86
config HP_WATCHDOG
tristate "HP ProLiant iLO2+ Hardware Watchdog Timer"
select WATCHDOG_CORE
- depends on X86 && PCI
+ depends on (ARM64 || X86) && PCI
help
A software monitoring watchdog and NMI handling driver. This driver
will detect lockups and provide a stack trace. This is a driver that
config HPWDT_NMI_DECODING
bool "NMI support for the HP ProLiant iLO2+ Hardware Watchdog Timer"
- depends on HP_WATCHDOG
+ depends on X86 && HP_WATCHDOG
default y
help
Enables the NMI handler for the watchdog pretimeout NMI and the iLO
obj-$(CONFIG_ALIM1535_WDT) += alim1535_wdt.o
obj-$(CONFIG_ALIM7101_WDT) += alim7101_wdt.o
obj-$(CONFIG_EBC_C384_WDT) += ebc-c384_wdt.o
+obj-$(CONFIG_EXAR_WDT) += exar_wdt.o
obj-$(CONFIG_F71808E_WDT) += f71808e_wdt.o
obj-$(CONFIG_SP5100_TCO) += sp5100_tco.o
obj-$(CONFIG_GEODE_WDT) += geodewdt.o
dev->timeout = (u64)dev->clk_rate * timeout;
do_div(dev->timeout, CNTR_CTRL_PRESCALE_MIN);
+ set_counter_value(dev, CNTR_ID_WDOG, dev->timeout);
+
return 0;
}
u32 reg = readl(wdt->base + WDT_RESET_WIDTH);
reg &= config->ext_pulse_width_mask;
- if (of_property_read_bool(np, "aspeed,ext-push-pull"))
- reg |= WDT_PUSH_PULL_MAGIC;
+ if (of_property_read_bool(np, "aspeed,ext-active-high"))
+ reg |= WDT_ACTIVE_HIGH_MAGIC;
else
- reg |= WDT_OPEN_DRAIN_MAGIC;
+ reg |= WDT_ACTIVE_LOW_MAGIC;
writel(reg, wdt->base + WDT_RESET_WIDTH);
reg &= config->ext_pulse_width_mask;
- if (of_property_read_bool(np, "aspeed,ext-active-high"))
- reg |= WDT_ACTIVE_HIGH_MAGIC;
+ if (of_property_read_bool(np, "aspeed,ext-push-pull"))
+ reg |= WDT_PUSH_PULL_MAGIC;
else
- reg |= WDT_ACTIVE_LOW_MAGIC;
+ reg |= WDT_OPEN_DRAIN_MAGIC;
writel(reg, wdt->base + WDT_RESET_WIDTH);
}
#include <linux/gpio/consumer.h>
#include <linux/mfd/rohm-bd957x.h>
#include <linux/module.h>
-#include <linux/of.h>
#include <linux/platform_device.h>
+#include <linux/property.h>
#include <linux/regmap.h>
#include <linux/watchdog.h>
static int bd9576_wdt_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
- struct device_node *np = dev->parent->of_node;
struct bd9576_wdt_priv *priv;
u32 hw_margin[2];
u32 hw_margin_max = BD957X_WDT_DEFAULT_MARGIN, hw_margin_min = 0;
+ int count;
int ret;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
return -ENODEV;
}
- priv->gpiod_en = devm_gpiod_get_from_of_node(dev, dev->parent->of_node,
- "rohm,watchdog-enable-gpios",
- 0, GPIOD_OUT_LOW,
- "watchdog-enable");
+ priv->gpiod_en = devm_fwnode_gpiod_get(dev, dev_fwnode(dev->parent),
+ "rohm,watchdog-enable",
+ GPIOD_OUT_LOW,
+ "watchdog-enable");
if (IS_ERR(priv->gpiod_en))
return dev_err_probe(dev, PTR_ERR(priv->gpiod_en),
"getting watchdog-enable GPIO failed\n");
- priv->gpiod_ping = devm_gpiod_get_from_of_node(dev, dev->parent->of_node,
- "rohm,watchdog-ping-gpios",
- 0, GPIOD_OUT_LOW,
- "watchdog-ping");
+ priv->gpiod_ping = devm_fwnode_gpiod_get(dev, dev_fwnode(dev->parent),
+ "rohm,watchdog-ping",
+ GPIOD_OUT_LOW,
+ "watchdog-ping");
if (IS_ERR(priv->gpiod_ping))
return dev_err_probe(dev, PTR_ERR(priv->gpiod_ping),
"getting watchdog-ping GPIO failed\n");
- ret = of_property_read_variable_u32_array(np, "rohm,hw-timeout-ms",
- &hw_margin[0], 1, 2);
- if (ret < 0 && ret != -EINVAL)
- return ret;
+ count = device_property_count_u32(dev->parent, "rohm,hw-timeout-ms");
+ if (count < 0 && count != -EINVAL)
+ return count;
+
+ if (count > 0) {
+ if (count > ARRAY_SIZE(hw_margin))
+ return -EINVAL;
- if (ret == 1)
- hw_margin_max = hw_margin[0];
+ ret = device_property_read_u32_array(dev->parent,
+ "rohm,hw-timeout-ms",
+ hw_margin, count);
+ if (ret < 0)
+ return ret;
- if (ret == 2) {
- hw_margin_max = hw_margin[1];
- hw_margin_min = hw_margin[0];
+ if (count == 1)
+ hw_margin_max = hw_margin[0];
+
+ if (count == 2) {
+ hw_margin_max = hw_margin[1];
+ hw_margin_min = hw_margin[0];
+ }
}
ret = bd957x_set_wdt_mode(priv, hw_margin_max, hw_margin_min);
if (ret)
return ret;
- priv->always_running = of_property_read_bool(np, "always-running");
+ priv->always_running = device_property_read_bool(dev->parent,
+ "always-running");
watchdog_set_drvdata(&priv->wdd, priv);
* @ppos: pointer to the position to write. No seeks allowed
*
* A write to a watchdog device is defined as a keepalive signal. Any
- * write of data will do, as we we don't define content meaning.
+ * write of data will do, as we don't define content meaning.
*/
static ssize_t eurwdt_write(struct file *file, const char __user *buf,
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * exar_wdt.c - Driver for the watchdog present in some
+ * Exar/MaxLinear UART chips like the XR28V38x.
+ *
+ * (c) Copyright 2022 D. Müller <d.mueller@elsoft.ch>.
+ *
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/io.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/watchdog.h>
+
+#define DRV_NAME "exar_wdt"
+
+static const unsigned short sio_config_ports[] = { 0x2e, 0x4e };
+static const unsigned char sio_enter_keys[] = { 0x67, 0x77, 0x87, 0xA0 };
+#define EXAR_EXIT_KEY 0xAA
+
+#define EXAR_LDN 0x07
+#define EXAR_DID 0x20
+#define EXAR_VID 0x23
+#define EXAR_WDT 0x26
+#define EXAR_ACT 0x30
+#define EXAR_RTBASE 0x60
+
+#define EXAR_WDT_LDEV 0x08
+
+#define EXAR_VEN_ID 0x13A8
+#define EXAR_DEV_382 0x0382
+#define EXAR_DEV_384 0x0384
+
+/* WDT runtime registers */
+#define WDT_CTRL 0x00
+#define WDT_VAL 0x01
+
+#define WDT_UNITS_10MS 0x0 /* the 10 millisec unit of the HW is not used */
+#define WDT_UNITS_SEC 0x2
+#define WDT_UNITS_MIN 0x4
+
+/* default WDT control for WDTOUT signal activ / rearm by read */
+#define EXAR_WDT_DEF_CONF 0
+
+struct wdt_pdev_node {
+ struct list_head list;
+ struct platform_device *pdev;
+ const char name[16];
+};
+
+struct wdt_priv {
+ /* the lock for WDT io operations */
+ spinlock_t io_lock;
+ struct resource wdt_res;
+ struct watchdog_device wdt_dev;
+ unsigned short did;
+ unsigned short config_port;
+ unsigned char enter_key;
+ unsigned char unit;
+ unsigned char timeout;
+};
+
+#define WATCHDOG_TIMEOUT 60
+
+static int timeout = WATCHDOG_TIMEOUT;
+module_param(timeout, int, 0);
+MODULE_PARM_DESC(timeout,
+ "Watchdog timeout in seconds. 1<=timeout<=15300, default="
+ __MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout,
+ "Watchdog cannot be stopped once started (default="
+ __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+static int exar_sio_enter(const unsigned short config_port,
+ const unsigned char key)
+{
+ if (!request_muxed_region(config_port, 2, DRV_NAME))
+ return -EBUSY;
+
+ /* write the ENTER-KEY twice */
+ outb(key, config_port);
+ outb(key, config_port);
+
+ return 0;
+}
+
+static void exar_sio_exit(const unsigned short config_port)
+{
+ outb(EXAR_EXIT_KEY, config_port);
+ release_region(config_port, 2);
+}
+
+static unsigned char exar_sio_read(const unsigned short config_port,
+ const unsigned char reg)
+{
+ outb(reg, config_port);
+ return inb(config_port + 1);
+}
+
+static void exar_sio_write(const unsigned short config_port,
+ const unsigned char reg, const unsigned char val)
+{
+ outb(reg, config_port);
+ outb(val, config_port + 1);
+}
+
+static unsigned short exar_sio_read16(const unsigned short config_port,
+ const unsigned char reg)
+{
+ unsigned char msb, lsb;
+
+ msb = exar_sio_read(config_port, reg);
+ lsb = exar_sio_read(config_port, reg + 1);
+
+ return (msb << 8) | lsb;
+}
+
+static void exar_sio_select_wdt(const unsigned short config_port)
+{
+ exar_sio_write(config_port, EXAR_LDN, EXAR_WDT_LDEV);
+}
+
+static void exar_wdt_arm(const struct wdt_priv *priv)
+{
+ unsigned short rt_base = priv->wdt_res.start;
+
+ /* write timeout value twice to arm watchdog */
+ outb(priv->timeout, rt_base + WDT_VAL);
+ outb(priv->timeout, rt_base + WDT_VAL);
+}
+
+static void exar_wdt_disarm(const struct wdt_priv *priv)
+{
+ unsigned short rt_base = priv->wdt_res.start;
+
+ /*
+ * use two accesses with different values to make sure
+ * that a combination of a previous single access and
+ * the ones below with the same value are not falsely
+ * interpreted as "arm watchdog"
+ */
+ outb(0xFF, rt_base + WDT_VAL);
+ outb(0, rt_base + WDT_VAL);
+}
+
+static int exar_wdt_start(struct watchdog_device *wdog)
+{
+ struct wdt_priv *priv = watchdog_get_drvdata(wdog);
+ unsigned short rt_base = priv->wdt_res.start;
+
+ spin_lock(&priv->io_lock);
+
+ exar_wdt_disarm(priv);
+ outb(priv->unit, rt_base + WDT_CTRL);
+ exar_wdt_arm(priv);
+
+ spin_unlock(&priv->io_lock);
+ return 0;
+}
+
+static int exar_wdt_stop(struct watchdog_device *wdog)
+{
+ struct wdt_priv *priv = watchdog_get_drvdata(wdog);
+
+ spin_lock(&priv->io_lock);
+
+ exar_wdt_disarm(priv);
+
+ spin_unlock(&priv->io_lock);
+ return 0;
+}
+
+static int exar_wdt_keepalive(struct watchdog_device *wdog)
+{
+ struct wdt_priv *priv = watchdog_get_drvdata(wdog);
+ unsigned short rt_base = priv->wdt_res.start;
+
+ spin_lock(&priv->io_lock);
+
+ /* reading the WDT_VAL reg will feed the watchdog */
+ inb(rt_base + WDT_VAL);
+
+ spin_unlock(&priv->io_lock);
+ return 0;
+}
+
+static int exar_wdt_set_timeout(struct watchdog_device *wdog, unsigned int t)
+{
+ struct wdt_priv *priv = watchdog_get_drvdata(wdog);
+ bool unit_min = false;
+
+ /*
+ * if new timeout is bigger then 255 seconds, change the
+ * unit to minutes and round the timeout up to the next whole minute
+ */
+ if (t > 255) {
+ unit_min = true;
+ t = DIV_ROUND_UP(t, 60);
+ }
+
+ /* save for later use in exar_wdt_start() */
+ priv->unit = unit_min ? WDT_UNITS_MIN : WDT_UNITS_SEC;
+ priv->timeout = t;
+
+ wdog->timeout = unit_min ? t * 60 : t;
+
+ if (watchdog_hw_running(wdog))
+ exar_wdt_start(wdog);
+
+ return 0;
+}
+
+static const struct watchdog_info exar_wdt_info = {
+ .options = WDIOF_KEEPALIVEPING |
+ WDIOF_SETTIMEOUT |
+ WDIOF_MAGICCLOSE,
+ .identity = "Exar/MaxLinear XR28V38x Watchdog",
+};
+
+static const struct watchdog_ops exar_wdt_ops = {
+ .owner = THIS_MODULE,
+ .start = exar_wdt_start,
+ .stop = exar_wdt_stop,
+ .ping = exar_wdt_keepalive,
+ .set_timeout = exar_wdt_set_timeout,
+};
+
+static int exar_wdt_config(struct watchdog_device *wdog,
+ const unsigned char conf)
+{
+ struct wdt_priv *priv = watchdog_get_drvdata(wdog);
+ int ret;
+
+ ret = exar_sio_enter(priv->config_port, priv->enter_key);
+ if (ret)
+ return ret;
+
+ exar_sio_select_wdt(priv->config_port);
+ exar_sio_write(priv->config_port, EXAR_WDT, conf);
+
+ exar_sio_exit(priv->config_port);
+
+ return 0;
+}
+
+static int __init exar_wdt_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct wdt_priv *priv = dev->platform_data;
+ struct watchdog_device *wdt_dev = &priv->wdt_dev;
+ struct resource *res;
+ int ret;
+
+ res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+ if (!res)
+ return -ENXIO;
+
+ spin_lock_init(&priv->io_lock);
+
+ wdt_dev->info = &exar_wdt_info;
+ wdt_dev->ops = &exar_wdt_ops;
+ wdt_dev->min_timeout = 1;
+ wdt_dev->max_timeout = 255 * 60;
+
+ watchdog_init_timeout(wdt_dev, timeout, NULL);
+ watchdog_set_nowayout(wdt_dev, nowayout);
+ watchdog_stop_on_reboot(wdt_dev);
+ watchdog_stop_on_unregister(wdt_dev);
+ watchdog_set_drvdata(wdt_dev, priv);
+
+ ret = exar_wdt_config(wdt_dev, EXAR_WDT_DEF_CONF);
+ if (ret)
+ return ret;
+
+ exar_wdt_set_timeout(wdt_dev, timeout);
+ /* Make sure that the watchdog is not running */
+ exar_wdt_stop(wdt_dev);
+
+ ret = devm_watchdog_register_device(dev, wdt_dev);
+ if (ret)
+ return ret;
+
+ dev_info(dev, "XR28V%X WDT initialized. timeout=%d sec (nowayout=%d)\n",
+ priv->did, timeout, nowayout);
+
+ return 0;
+}
+
+static unsigned short __init exar_detect(const unsigned short config_port,
+ const unsigned char key,
+ unsigned short *rt_base)
+{
+ int ret;
+ unsigned short base = 0;
+ unsigned short vid, did;
+
+ ret = exar_sio_enter(config_port, key);
+ if (ret)
+ return 0;
+
+ vid = exar_sio_read16(config_port, EXAR_VID);
+ did = exar_sio_read16(config_port, EXAR_DID);
+
+ /* check for the vendor and device IDs we currently know about */
+ if (vid == EXAR_VEN_ID &&
+ (did == EXAR_DEV_382 ||
+ did == EXAR_DEV_384)) {
+ exar_sio_select_wdt(config_port);
+ /* is device active? */
+ if (exar_sio_read(config_port, EXAR_ACT) == 0x01)
+ base = exar_sio_read16(config_port, EXAR_RTBASE);
+ }
+
+ exar_sio_exit(config_port);
+
+ if (base) {
+ pr_debug("Found a XR28V%X WDT (conf: 0x%x / rt: 0x%04x)\n",
+ did, config_port, base);
+ *rt_base = base;
+ return did;
+ }
+
+ return 0;
+}
+
+static struct platform_driver exar_wdt_driver = {
+ .driver = {
+ .name = DRV_NAME,
+ },
+};
+
+static LIST_HEAD(pdev_list);
+
+static int __init exar_wdt_register(struct wdt_priv *priv, const int idx)
+{
+ struct wdt_pdev_node *n;
+
+ n = kzalloc(sizeof(*n), GFP_KERNEL);
+ if (!n)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&n->list);
+
+ scnprintf((char *)n->name, sizeof(n->name), DRV_NAME ".%d", idx);
+ priv->wdt_res.name = n->name;
+
+ n->pdev = platform_device_register_resndata(NULL, DRV_NAME, idx,
+ &priv->wdt_res, 1,
+ priv, sizeof(*priv));
+ if (IS_ERR(n->pdev)) {
+ kfree(n);
+ return PTR_ERR(n->pdev);
+ }
+
+ list_add_tail(&n->list, &pdev_list);
+
+ return 0;
+}
+
+static void exar_wdt_unregister(void)
+{
+ struct wdt_pdev_node *n, *t;
+
+ list_for_each_entry_safe(n, t, &pdev_list, list) {
+ platform_device_unregister(n->pdev);
+ list_del(&n->list);
+ kfree(n);
+ }
+}
+
+static int __init exar_wdt_init(void)
+{
+ int ret, i, j, idx = 0;
+
+ /* search for active Exar watchdogs on all possible locations */
+ for (i = 0; i < ARRAY_SIZE(sio_config_ports); i++) {
+ for (j = 0; j < ARRAY_SIZE(sio_enter_keys); j++) {
+ unsigned short did, rt_base = 0;
+
+ did = exar_detect(sio_config_ports[i],
+ sio_enter_keys[j],
+ &rt_base);
+
+ if (did) {
+ struct wdt_priv priv = {
+ .wdt_res = DEFINE_RES_IO(rt_base, 2),
+ .did = did,
+ .config_port = sio_config_ports[i],
+ .enter_key = sio_enter_keys[j],
+ };
+
+ ret = exar_wdt_register(&priv, idx);
+ if (!ret)
+ idx++;
+ }
+ }
+ }
+
+ if (!idx)
+ return -ENODEV;
+
+ ret = platform_driver_probe(&exar_wdt_driver, exar_wdt_probe);
+ if (ret)
+ exar_wdt_unregister();
+
+ return ret;
+}
+
+static void __exit exar_wdt_exit(void)
+{
+ exar_wdt_unregister();
+ platform_driver_unregister(&exar_wdt_driver);
+}
+
+module_init(exar_wdt_init);
+module_exit(exar_wdt_exit);
+
+MODULE_AUTHOR("David Müller <d.mueller@elsoft.ch>");
+MODULE_DESCRIPTION("Exar/MaxLinear Watchdog Driver");
+MODULE_LICENSE("GPL");
return container_of(wdd, struct ftwdt010_wdt, wdd);
}
-static int ftwdt010_wdt_start(struct watchdog_device *wdd)
+static void ftwdt010_enable(struct ftwdt010_wdt *gwdt,
+ unsigned int timeout,
+ bool need_irq)
{
- struct ftwdt010_wdt *gwdt = to_ftwdt010_wdt(wdd);
u32 enable;
- writel(wdd->timeout * WDT_CLOCK, gwdt->base + FTWDT010_WDLOAD);
+ writel(timeout * WDT_CLOCK, gwdt->base + FTWDT010_WDLOAD);
writel(WDRESTART_MAGIC, gwdt->base + FTWDT010_WDRESTART);
/* set clock before enabling */
enable = WDCR_CLOCK_5MHZ | WDCR_SYS_RST;
writel(enable, gwdt->base + FTWDT010_WDCR);
- if (gwdt->has_irq)
+ if (need_irq)
enable |= WDCR_WDINTR;
enable |= WDCR_ENABLE;
writel(enable, gwdt->base + FTWDT010_WDCR);
+}
+static int ftwdt010_wdt_start(struct watchdog_device *wdd)
+{
+ struct ftwdt010_wdt *gwdt = to_ftwdt010_wdt(wdd);
+
+ ftwdt010_enable(gwdt, wdd->timeout, gwdt->has_irq);
return 0;
}
return 0;
}
+static int ftwdt010_wdt_restart(struct watchdog_device *wdd,
+ unsigned long action, void *data)
+{
+ ftwdt010_enable(to_ftwdt010_wdt(wdd), 0, false);
+ return 0;
+}
+
static irqreturn_t ftwdt010_wdt_interrupt(int irq, void *data)
{
struct ftwdt010_wdt *gwdt = data;
.stop = ftwdt010_wdt_stop,
.ping = ftwdt010_wdt_ping,
.set_timeout = ftwdt010_wdt_set_timeout,
+ .restart = ftwdt010_wdt_restart,
.owner = THIS_MODULE,
};
}
irq = platform_get_irq(pdev, 0);
- if (irq) {
+ if (irq > 0) {
ret = devm_request_irq(dev, irq, ftwdt010_wdt_interrupt, 0,
"watchdog bark", gwdt);
if (ret)
#include <linux/pci_ids.h>
#include <linux/types.h>
#include <linux/watchdog.h>
+#ifdef CONFIG_HPWDT_NMI_DECODING
#include <asm/nmi.h>
+#endif
#include <linux/crash_dump.h>
#define HPWDT_VERSION "2.0.4"
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
+#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/reboot.h>
#include <linux/watchdog.h>
#define WDOG_CS 0x0
+#define WDOG_CS_FLG BIT(14)
#define WDOG_CS_CMD32EN BIT(13)
+#define WDOG_CS_PRES BIT(12)
#define WDOG_CS_ULK BIT(11)
#define WDOG_CS_RCS BIT(10)
#define LPO_CLK 0x1
#define DEFAULT_TIMEOUT 60
#define MAX_TIMEOUT 128
#define WDOG_CLOCK_RATE 1000
-#define WDOG_WAIT_TIMEOUT 20
+#define WDOG_ULK_WAIT_TIMEOUT 1000
+#define WDOG_RCS_WAIT_TIMEOUT 10000
+#define WDOG_RCS_POST_WAIT 3000
+
+#define RETRY_MAX 5
static bool nowayout = WATCHDOG_NOWAYOUT;
module_param(nowayout, bool, 0000);
MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+struct imx_wdt_hw_feature {
+ bool prescaler_enable;
+ u32 wdog_clock_rate;
+};
+
struct imx7ulp_wdt_device {
struct watchdog_device wdd;
void __iomem *base;
struct clk *clk;
+ bool post_rcs_wait;
+ const struct imx_wdt_hw_feature *hw;
};
-static int imx7ulp_wdt_wait(void __iomem *base, u32 mask)
+static int imx7ulp_wdt_wait_ulk(void __iomem *base)
{
u32 val = readl(base + WDOG_CS);
- if (!(val & mask) && readl_poll_timeout_atomic(base + WDOG_CS, val,
- val & mask, 0,
- WDOG_WAIT_TIMEOUT))
+ if (!(val & WDOG_CS_ULK) &&
+ readl_poll_timeout_atomic(base + WDOG_CS, val,
+ val & WDOG_CS_ULK, 0,
+ WDOG_ULK_WAIT_TIMEOUT))
return -ETIMEDOUT;
return 0;
}
-static int imx7ulp_wdt_enable(struct watchdog_device *wdog, bool enable)
+static int imx7ulp_wdt_wait_rcs(struct imx7ulp_wdt_device *wdt)
{
- struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog);
+ int ret = 0;
+ u32 val = readl(wdt->base + WDOG_CS);
+ u64 timeout = (val & WDOG_CS_PRES) ?
+ WDOG_RCS_WAIT_TIMEOUT * 256 : WDOG_RCS_WAIT_TIMEOUT;
+ unsigned long wait_min = (val & WDOG_CS_PRES) ?
+ WDOG_RCS_POST_WAIT * 256 : WDOG_RCS_POST_WAIT;
+
+ if (!(val & WDOG_CS_RCS) &&
+ readl_poll_timeout(wdt->base + WDOG_CS, val, val & WDOG_CS_RCS, 100,
+ timeout))
+ ret = -ETIMEDOUT;
+
+ /* Wait 2.5 clocks after RCS done */
+ if (wdt->post_rcs_wait)
+ usleep_range(wait_min, wait_min + 2000);
+
+ return ret;
+}
+static int _imx7ulp_wdt_enable(struct imx7ulp_wdt_device *wdt, bool enable)
+{
u32 val = readl(wdt->base + WDOG_CS);
int ret;
local_irq_disable();
writel(UNLOCK, wdt->base + WDOG_CNT);
- ret = imx7ulp_wdt_wait(wdt->base, WDOG_CS_ULK);
+ ret = imx7ulp_wdt_wait_ulk(wdt->base);
if (ret)
goto enable_out;
if (enable)
writel(val | WDOG_CS_EN, wdt->base + WDOG_CS);
else
writel(val & ~WDOG_CS_EN, wdt->base + WDOG_CS);
- imx7ulp_wdt_wait(wdt->base, WDOG_CS_RCS);
-enable_out:
local_irq_enable();
+ ret = imx7ulp_wdt_wait_rcs(wdt);
return ret;
+
+enable_out:
+ local_irq_enable();
+ return ret;
}
-static bool imx7ulp_wdt_is_enabled(void __iomem *base)
+static int imx7ulp_wdt_enable(struct watchdog_device *wdog, bool enable)
{
- u32 val = readl(base + WDOG_CS);
+ struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog);
+ int ret;
+ u32 val;
+ u32 loop = RETRY_MAX;
+
+ do {
+ ret = _imx7ulp_wdt_enable(wdt, enable);
+ val = readl(wdt->base + WDOG_CS);
+ } while (--loop > 0 && ((!!(val & WDOG_CS_EN)) != enable || ret));
- return val & WDOG_CS_EN;
+ if (loop == 0)
+ return -EBUSY;
+
+ return ret;
}
static int imx7ulp_wdt_ping(struct watchdog_device *wdog)
return imx7ulp_wdt_enable(wdog, false);
}
-static int imx7ulp_wdt_set_timeout(struct watchdog_device *wdog,
- unsigned int timeout)
+static int _imx7ulp_wdt_set_timeout(struct imx7ulp_wdt_device *wdt,
+ unsigned int toval)
{
- struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog);
- u32 val = WDOG_CLOCK_RATE * timeout;
int ret;
local_irq_disable();
writel(UNLOCK, wdt->base + WDOG_CNT);
- ret = imx7ulp_wdt_wait(wdt->base, WDOG_CS_ULK);
+ ret = imx7ulp_wdt_wait_ulk(wdt->base);
if (ret)
goto timeout_out;
- writel(val, wdt->base + WDOG_TOVAL);
- imx7ulp_wdt_wait(wdt->base, WDOG_CS_RCS);
-
- wdog->timeout = timeout;
+ writel(toval, wdt->base + WDOG_TOVAL);
+ local_irq_enable();
+ ret = imx7ulp_wdt_wait_rcs(wdt);
+ return ret;
timeout_out:
local_irq_enable();
+ return ret;
+}
+static int imx7ulp_wdt_set_timeout(struct watchdog_device *wdog,
+ unsigned int timeout)
+{
+ struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog);
+ u32 toval = wdt->hw->wdog_clock_rate * timeout;
+ u32 val;
+ int ret;
+ u32 loop = RETRY_MAX;
+
+ do {
+ ret = _imx7ulp_wdt_set_timeout(wdt, toval);
+ val = readl(wdt->base + WDOG_TOVAL);
+ } while (--loop > 0 && (val != toval || ret));
+
+ if (loop == 0)
+ return -EBUSY;
+
+ wdog->timeout = timeout;
return ret;
}
WDIOF_MAGICCLOSE,
};
-static int imx7ulp_wdt_init(void __iomem *base, unsigned int timeout)
+static int _imx7ulp_wdt_init(struct imx7ulp_wdt_device *wdt, unsigned int timeout, unsigned int cs)
{
u32 val;
int ret;
local_irq_disable();
- /* unlock the wdog for reconfiguration */
- writel_relaxed(UNLOCK_SEQ0, base + WDOG_CNT);
- writel_relaxed(UNLOCK_SEQ1, base + WDOG_CNT);
- ret = imx7ulp_wdt_wait(base, WDOG_CS_ULK);
+
+ val = readl(wdt->base + WDOG_CS);
+ if (val & WDOG_CS_CMD32EN) {
+ writel(UNLOCK, wdt->base + WDOG_CNT);
+ } else {
+ mb();
+ /* unlock the wdog for reconfiguration */
+ writel_relaxed(UNLOCK_SEQ0, wdt->base + WDOG_CNT);
+ writel_relaxed(UNLOCK_SEQ1, wdt->base + WDOG_CNT);
+ mb();
+ }
+
+ ret = imx7ulp_wdt_wait_ulk(wdt->base);
if (ret)
goto init_out;
/* set an initial timeout value in TOVAL */
- writel(timeout, base + WDOG_TOVAL);
- /* enable 32bit command sequence and reconfigure */
- val = WDOG_CS_CMD32EN | WDOG_CS_CLK | WDOG_CS_UPDATE |
- WDOG_CS_WAIT | WDOG_CS_STOP;
- writel(val, base + WDOG_CS);
- imx7ulp_wdt_wait(base, WDOG_CS_RCS);
+ writel(timeout, wdt->base + WDOG_TOVAL);
+ writel(cs, wdt->base + WDOG_CS);
+ local_irq_enable();
+ ret = imx7ulp_wdt_wait_rcs(wdt);
+
+ return ret;
init_out:
local_irq_enable();
+ return ret;
+}
+
+static int imx7ulp_wdt_init(struct imx7ulp_wdt_device *wdt, unsigned int timeout)
+{
+ /* enable 32bit command sequence and reconfigure */
+ u32 val = WDOG_CS_CMD32EN | WDOG_CS_CLK | WDOG_CS_UPDATE |
+ WDOG_CS_WAIT | WDOG_CS_STOP;
+ u32 cs, toval;
+ int ret;
+ u32 loop = RETRY_MAX;
+
+ if (wdt->hw->prescaler_enable)
+ val |= WDOG_CS_PRES;
+
+ do {
+ ret = _imx7ulp_wdt_init(wdt, timeout, val);
+ toval = readl(wdt->base + WDOG_TOVAL);
+ cs = readl(wdt->base + WDOG_CS);
+ cs &= ~(WDOG_CS_FLG | WDOG_CS_ULK | WDOG_CS_RCS);
+ } while (--loop > 0 && (cs != val || toval != timeout || ret));
+
+ if (loop == 0)
+ return -EBUSY;
return ret;
}
return PTR_ERR(imx7ulp_wdt->clk);
}
+ imx7ulp_wdt->post_rcs_wait = true;
+ if (of_device_is_compatible(dev->of_node,
+ "fsl,imx8ulp-wdt")) {
+ dev_info(dev, "imx8ulp wdt probe\n");
+ imx7ulp_wdt->post_rcs_wait = false;
+ } else {
+ dev_info(dev, "imx7ulp wdt probe\n");
+ }
+
ret = clk_prepare_enable(imx7ulp_wdt->clk);
if (ret)
return ret;
watchdog_stop_on_reboot(wdog);
watchdog_stop_on_unregister(wdog);
watchdog_set_drvdata(wdog, imx7ulp_wdt);
- ret = imx7ulp_wdt_init(imx7ulp_wdt->base, wdog->timeout * WDOG_CLOCK_RATE);
+
+ imx7ulp_wdt->hw = of_device_get_match_data(dev);
+ ret = imx7ulp_wdt_init(imx7ulp_wdt, wdog->timeout * imx7ulp_wdt->hw->wdog_clock_rate);
if (ret)
return ret;
return devm_watchdog_register_device(dev, wdog);
}
-static int __maybe_unused imx7ulp_wdt_suspend(struct device *dev)
+static int __maybe_unused imx7ulp_wdt_suspend_noirq(struct device *dev)
{
struct imx7ulp_wdt_device *imx7ulp_wdt = dev_get_drvdata(dev);
return 0;
}
-static int __maybe_unused imx7ulp_wdt_resume(struct device *dev)
+static int __maybe_unused imx7ulp_wdt_resume_noirq(struct device *dev)
{
struct imx7ulp_wdt_device *imx7ulp_wdt = dev_get_drvdata(dev);
- u32 timeout = imx7ulp_wdt->wdd.timeout * WDOG_CLOCK_RATE;
+ u32 timeout = imx7ulp_wdt->wdd.timeout * imx7ulp_wdt->hw->wdog_clock_rate;
int ret;
ret = clk_prepare_enable(imx7ulp_wdt->clk);
if (ret)
return ret;
- if (imx7ulp_wdt_is_enabled(imx7ulp_wdt->base))
- imx7ulp_wdt_init(imx7ulp_wdt->base, timeout);
-
- if (watchdog_active(&imx7ulp_wdt->wdd))
+ if (watchdog_active(&imx7ulp_wdt->wdd)) {
+ imx7ulp_wdt_init(imx7ulp_wdt, timeout);
imx7ulp_wdt_start(&imx7ulp_wdt->wdd);
+ imx7ulp_wdt_ping(&imx7ulp_wdt->wdd);
+ }
return 0;
}
-static SIMPLE_DEV_PM_OPS(imx7ulp_wdt_pm_ops, imx7ulp_wdt_suspend,
- imx7ulp_wdt_resume);
+static const struct dev_pm_ops imx7ulp_wdt_pm_ops = {
+ SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(imx7ulp_wdt_suspend_noirq,
+ imx7ulp_wdt_resume_noirq)
+};
+
+static const struct imx_wdt_hw_feature imx7ulp_wdt_hw = {
+ .prescaler_enable = false,
+ .wdog_clock_rate = 1000,
+};
+
+static const struct imx_wdt_hw_feature imx93_wdt_hw = {
+ .prescaler_enable = true,
+ .wdog_clock_rate = 125,
+};
static const struct of_device_id imx7ulp_wdt_dt_ids[] = {
- { .compatible = "fsl,imx7ulp-wdt", },
+ { .compatible = "fsl,imx8ulp-wdt", .data = &imx7ulp_wdt_hw, },
+ { .compatible = "fsl,imx7ulp-wdt", .data = &imx7ulp_wdt_hw, },
+ { .compatible = "fsl,imx93-wdt", .data = &imx93_wdt_hw, },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, imx7ulp_wdt_dt_ids);
struct device *dev = &pdev->dev;
struct meson_gxbb_wdt *data;
int ret;
+ u32 ctrl_reg;
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
watchdog_set_nowayout(&data->wdt_dev, nowayout);
watchdog_set_drvdata(&data->wdt_dev, data);
+ ctrl_reg = readl(data->reg_base + GXBB_WDT_CTRL_REG) &
+ GXBB_WDT_CTRL_EN;
+
+ if (ctrl_reg) {
+ /* Watchdog is running - keep it running but extend timeout
+ * to the maximum while setting the timebase
+ */
+ set_bit(WDOG_HW_RUNNING, &data->wdt_dev.status);
+ meson_gxbb_wdt_set_timeout(&data->wdt_dev,
+ GXBB_WDT_TCNT_SETUP_MASK / 1000);
+ }
+
/* Setup with 1ms timebase */
- writel(((clk_get_rate(data->clk) / 1000) & GXBB_WDT_CTRL_DIV_MASK) |
- GXBB_WDT_CTRL_EE_RESET |
- GXBB_WDT_CTRL_CLK_EN |
- GXBB_WDT_CTRL_CLKDIV_EN,
- data->reg_base + GXBB_WDT_CTRL_REG);
+ ctrl_reg |= ((clk_get_rate(data->clk) / 1000) &
+ GXBB_WDT_CTRL_DIV_MASK) |
+ GXBB_WDT_CTRL_EE_RESET |
+ GXBB_WDT_CTRL_CLK_EN |
+ GXBB_WDT_CTRL_CLKDIV_EN;
+ writel(ctrl_reg, data->reg_base + GXBB_WDT_CTRL_REG);
meson_gxbb_wdt_set_timeout(&data->wdt_dev, data->wdt_dev.timeout);
return devm_watchdog_register_device(dev, &data->wdt_dev);
// Copyright (c) 2018 IBM Corp.
#include <linux/bitops.h>
+#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
struct npcm_wdt {
struct watchdog_device wdd;
void __iomem *reg;
+ struct clk *clk;
};
static inline struct npcm_wdt *to_npcm_wdt(struct watchdog_device *wdd)
struct npcm_wdt *wdt = to_npcm_wdt(wdd);
u32 val;
+ if (wdt->clk)
+ clk_prepare_enable(wdt->clk);
+
if (wdd->timeout < 2)
val = 0x800;
else if (wdd->timeout < 3)
writel(0, wdt->reg);
+ if (wdt->clk)
+ clk_disable_unprepare(wdt->clk);
+
return 0;
}
{
struct npcm_wdt *wdt = to_npcm_wdt(wdd);
+ /* For reset, we start the WDT clock and leave it running. */
+ if (wdt->clk)
+ clk_prepare_enable(wdt->clk);
+
writel(NPCM_WTR | NPCM_WTRE | NPCM_WTE, wdt->reg);
udelay(1000);
if (IS_ERR(wdt->reg))
return PTR_ERR(wdt->reg);
+ wdt->clk = devm_clk_get_optional(&pdev->dev, NULL);
+ if (IS_ERR(wdt->clk))
+ return PTR_ERR(wdt->clk);
+
irq = platform_get_irq(pdev, 0);
if (irq < 0)
return irq;
wdt->freq = wdt->freq * 9 / 10;
pm_runtime_enable(dev);
- ret = pm_runtime_get_sync(dev);
+ ret = pm_runtime_resume_and_get(dev);
if (ret < 0) {
- pm_runtime_put_noidle(dev);
pm_runtime_disable(&pdev->dev);
return dev_err_probe(dev, ret, "runtime pm failed\n");
}
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/module.h>
-#include <linux/of.h>
+#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+enum rz_wdt_type {
+ WDT_RZG2L,
+ WDT_RZV2M,
+};
+
struct rzg2l_wdt_priv {
void __iomem *base;
struct watchdog_device wdev;
unsigned long delay;
struct clk *pclk;
struct clk *osc_clk;
+ enum rz_wdt_type devtype;
};
static void rzg2l_wdt_wait_delay(struct rzg2l_wdt_priv *priv)
clk_prepare_enable(priv->pclk);
clk_prepare_enable(priv->osc_clk);
- /* Generate Reset (WDTRSTB) Signal on parity error */
- rzg2l_wdt_write(priv, 0, PECR);
+ if (priv->devtype == WDT_RZG2L) {
+ /* Generate Reset (WDTRSTB) Signal on parity error */
+ rzg2l_wdt_write(priv, 0, PECR);
+
+ /* Force parity error */
+ rzg2l_wdt_write(priv, PEEN_FORCE, PEEN);
+ } else {
+ /* RZ/V2M doesn't have parity error registers */
+
+ wdev->timeout = 0;
+
+ /* Initialize time out */
+ rzg2l_wdt_init_timeout(wdev);
- /* Force parity error */
- rzg2l_wdt_write(priv, PEEN_FORCE, PEEN);
+ /* Initialize watchdog counter register */
+ rzg2l_wdt_write(priv, 0, WDTTIM);
+
+ /* Enable watchdog timer*/
+ rzg2l_wdt_write(priv, WDTCNT_WDTEN, WDTCNT);
+
+ /* Wait 2 consecutive overflow cycles for reset */
+ mdelay(DIV_ROUND_UP(2 * 0xFFFFF * 1000, priv->osc_clk_rate));
+ }
return 0;
}
if (ret)
return dev_err_probe(dev, ret, "failed to deassert");
+ priv->devtype = (uintptr_t)of_device_get_match_data(dev);
+
pm_runtime_enable(&pdev->dev);
priv->wdev.info = &rzg2l_wdt_ident;
}
static const struct of_device_id rzg2l_wdt_ids[] = {
- { .compatible = "renesas,rzg2l-wdt", },
+ { .compatible = "renesas,rzg2l-wdt", .data = (void *)WDT_RZG2L },
+ { .compatible = "renesas,rzv2m-wdt", .data = (void *)WDT_RZV2M },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, rzg2l_wdt_ids);
#define EXYNOS850_CLUSTER0_NONCPU_INT_EN 0x1244
#define EXYNOS850_CLUSTER1_NONCPU_OUT 0x1620
#define EXYNOS850_CLUSTER1_NONCPU_INT_EN 0x1644
+#define EXYNOSAUTOV9_CLUSTER1_NONCPU_OUT 0x1520
+#define EXYNOSAUTOV9_CLUSTER1_NONCPU_INT_EN 0x1544
#define EXYNOS850_CLUSTER0_WDTRESET_BIT 24
#define EXYNOS850_CLUSTER1_WDTRESET_BIT 23
+#define EXYNOSAUTOV9_CLUSTER0_WDTRESET_BIT 25
+#define EXYNOSAUTOV9_CLUSTER1_WDTRESET_BIT 24
/**
* DOC: Quirk flags for different Samsung watchdog IP-cores
QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_CNT_EN,
};
+static const struct s3c2410_wdt_variant drv_data_exynosautov9_cl0 = {
+ .mask_reset_reg = EXYNOS850_CLUSTER0_NONCPU_INT_EN,
+ .mask_bit = 2,
+ .mask_reset_inv = true,
+ .rst_stat_reg = EXYNOS5_RST_STAT_REG_OFFSET,
+ .rst_stat_bit = EXYNOSAUTOV9_CLUSTER0_WDTRESET_BIT,
+ .cnt_en_reg = EXYNOS850_CLUSTER0_NONCPU_OUT,
+ .cnt_en_bit = 7,
+ .quirks = QUIRK_HAS_WTCLRINT_REG | QUIRK_HAS_PMU_MASK_RESET |
+ QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_CNT_EN,
+};
+
+static const struct s3c2410_wdt_variant drv_data_exynosautov9_cl1 = {
+ .mask_reset_reg = EXYNOSAUTOV9_CLUSTER1_NONCPU_INT_EN,
+ .mask_bit = 2,
+ .mask_reset_inv = true,
+ .rst_stat_reg = EXYNOS5_RST_STAT_REG_OFFSET,
+ .rst_stat_bit = EXYNOSAUTOV9_CLUSTER1_WDTRESET_BIT,
+ .cnt_en_reg = EXYNOSAUTOV9_CLUSTER1_NONCPU_OUT,
+ .cnt_en_bit = 7,
+ .quirks = QUIRK_HAS_WTCLRINT_REG | QUIRK_HAS_PMU_MASK_RESET |
+ QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_CNT_EN,
+};
+
static const struct of_device_id s3c2410_wdt_match[] = {
{ .compatible = "samsung,s3c2410-wdt",
.data = &drv_data_s3c2410 },
.data = &drv_data_exynos7 },
{ .compatible = "samsung,exynos850-wdt",
.data = &drv_data_exynos850_cl0 },
+ { .compatible = "samsung,exynosautov9-wdt",
+ .data = &drv_data_exynosautov9_cl0 },
{},
};
MODULE_DEVICE_TABLE(of, s3c2410_wdt_match);
}
#ifdef CONFIG_OF
- /* Choose Exynos850 driver data w.r.t. cluster index */
- if (variant == &drv_data_exynos850_cl0) {
+ /* Choose Exynos850/ExynosAutov9 driver data w.r.t. cluster index */
+ if (variant == &drv_data_exynos850_cl0 ||
+ variant == &drv_data_exynosautov9_cl0) {
u32 index;
int err;
switch (index) {
case 0:
- return &drv_data_exynos850_cl0;
+ return variant;
case 1:
- return &drv_data_exynos850_cl1;
+ return (variant == &drv_data_exynos850_cl0) ?
+ &drv_data_exynos850_cl1 :
+ &drv_data_exynosautov9_cl1;
default:
dev_err(dev, "wrong cluster index: %u\n", index);
return NULL;
return 0;
}
-struct platform_driver sa1100dog_driver = {
+static struct platform_driver sa1100dog_driver = {
.driver.name = "sa1100_wdt",
.probe = sa1100dog_probe,
.remove = sa1100dog_remove,
/* module parameters */
+#define WATCHDOG_ACTION 0
+static bool action = WATCHDOG_ACTION;
+module_param(action, bool, 0);
+MODULE_PARM_DESC(action, "Action taken when watchdog expires, 0 to reset, 1 to poweroff (default="
+ __MODULE_STRING(WATCHDOG_ACTION) ")");
+
#define WATCHDOG_HEARTBEAT 60 /* 60 sec default heartbeat. */
static int heartbeat = WATCHDOG_HEARTBEAT; /* in seconds */
module_param(heartbeat, int, 0);
if (val & SP5100_WDT_FIRED)
wdd->bootstatus = WDIOF_CARDRESET;
- /* Set watchdog action to reset the system */
- val &= ~SP5100_WDT_ACTION_RESET;
+ /* Set watchdog action */
+ if (action)
+ val |= SP5100_WDT_ACTION_RESET;
+ else
+ val &= ~SP5100_WDT_ACTION_RESET;
writel(val, SP5100_WDT_CONTROL(tco->tcobase));
/* Set a reasonable heartbeat before we stop the timer */
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
#include <linux/watchdog.h>
#include <linux/platform_device.h>
#include <linux/mfd/twl.h>
#define W836X7HF_WDT_CSR 0xf7
#define NCT6102D_WDT_CSR 0xf2
+#define WDT_CSR_STATUS 0x10
+#define WDT_CSR_KBD 0x40
+#define WDT_CSR_MOUSE 0x80
+
static void superio_outb(int reg, int val)
{
outb(reg, WDT_EFER);
t = superio_inb(cr_wdt_control) & ~0x0C;
superio_outb(cr_wdt_control, t);
- /* reset trigger, disable keyboard & mouse turning off watchdog */
- t = superio_inb(cr_wdt_csr) & ~0xD0;
+ t = superio_inb(cr_wdt_csr);
+ if (t & WDT_CSR_STATUS)
+ wdog->bootstatus |= WDIOF_CARDRESET;
+
+ /* reset status, disable keyboard & mouse turning off watchdog */
+ t &= ~(WDT_CSR_STATUS | WDT_CSR_KBD | WDT_CSR_MOUSE);
superio_outb(cr_wdt_csr, t);
superio_exit();
* @ppos: pointer to the position to write. No seeks allowed
*
* A write to a watchdog device is defined as a keepalive signal. Any
- * write of data will do, as we we don't define content meaning.
+ * write of data will do, as we don't define content meaning.
*/
static ssize_t wdt_write(struct file *file, const char __user *buf,
wd_data->dev.groups = wdd->groups;
wd_data->dev.release = watchdog_core_data_release;
dev_set_drvdata(&wd_data->dev, wdd);
- dev_set_name(&wd_data->dev, "watchdog%d", wdd->id);
+ err = dev_set_name(&wd_data->dev, "watchdog%d", wdd->id);
+ if (err) {
+ put_device(&wd_data->dev);
+ return err;
+ }
kthread_init_work(&wd_data->work, watchdog_ping_work);
hrtimer_init(&wd_data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
return -EINVAL;
wdat->period = tbl->timer_period;
- wdat->wdd.min_hw_heartbeat_ms = wdat->period * tbl->min_count;
- wdat->wdd.max_hw_heartbeat_ms = wdat->period * tbl->max_count;
- wdat->wdd.min_timeout = 1;
+ wdat->wdd.min_timeout = DIV_ROUND_UP(wdat->period * tbl->min_count, 1000);
+ wdat->wdd.max_timeout = wdat->period * tbl->max_count / 1000;
wdat->stopped_in_sleep = tbl->flags & ACPI_WDAT_STOPPED;
wdat->wdd.info = &wdat_wdt_info;
wdat->wdd.ops = &wdat_wdt_ops;
depends on XEN_HAVE_PVMMU
depends on MEMORY_HOTPLUG
help
- Maxmium amount of memory (in GiB) that a PV guest can be
+ Maximum amount of memory (in GiB) that a PV guest can be
expanded to when using memory hotplug.
A PV guest can have more memory than this limit if is
};
struct gntdev_grant_map {
+ atomic_t in_use;
struct mmu_interval_notifier notifier;
+ bool notifier_init;
struct list_head next;
- struct vm_area_struct *vma;
int index;
int count;
int flags;
*/
}
+ if (use_ptemod && map->notifier_init)
+ mmu_interval_notifier_remove(&map->notifier);
+
if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {
notify_remote_via_evtchn(map->notify.event);
evtchn_put(map->notify.event);
static int find_grant_ptes(pte_t *pte, unsigned long addr, void *data)
{
struct gntdev_grant_map *map = data;
- unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT;
+ unsigned int pgnr = (addr - map->pages_vm_start) >> PAGE_SHIFT;
int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte |
(1 << _GNTMAP_guest_avail0);
u64 pte_maddr;
for (i = 0; i < map->count; i++) {
if (map->map_ops[i].status == GNTST_okay) {
map->unmap_ops[i].handle = map->map_ops[i].handle;
- if (!use_ptemod)
- alloced++;
+ alloced++;
} else if (!err)
err = -EINVAL;
if (use_ptemod) {
if (map->kmap_ops[i].status == GNTST_okay) {
- if (map->map_ops[i].status == GNTST_okay)
- alloced++;
+ alloced++;
map->kunmap_ops[i].handle = map->kmap_ops[i].handle;
} else if (!err)
err = -EINVAL;
unsigned int i;
struct gntdev_grant_map *map = data->data;
unsigned int offset = data->unmap_ops - map->unmap_ops;
+ int successful_unmaps = 0;
+ int live_grants;
for (i = 0; i < data->count; i++) {
+ if (map->unmap_ops[offset + i].status == GNTST_okay &&
+ map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE)
+ successful_unmaps++;
+
WARN_ON(map->unmap_ops[offset + i].status != GNTST_okay &&
map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE);
pr_debug("unmap handle=%d st=%d\n",
map->unmap_ops[offset+i].status);
map->unmap_ops[offset+i].handle = INVALID_GRANT_HANDLE;
if (use_ptemod) {
+ if (map->kunmap_ops[offset + i].status == GNTST_okay &&
+ map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE)
+ successful_unmaps++;
+
WARN_ON(map->kunmap_ops[offset + i].status != GNTST_okay &&
map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE);
pr_debug("kunmap handle=%u st=%d\n",
map->kunmap_ops[offset+i].handle = INVALID_GRANT_HANDLE;
}
}
+
/*
* Decrease the live-grant counter. This must happen after the loop to
* prevent premature reuse of the grants by gnttab_mmap().
*/
- atomic_sub(data->count, &map->live_grants);
+ live_grants = atomic_sub_return(successful_unmaps, &map->live_grants);
+ if (WARN_ON(live_grants < 0))
+ pr_err("%s: live_grants became negative (%d) after unmapping %d pages!\n",
+ __func__, live_grants, successful_unmaps);
/* Release reference taken by __unmap_grant_pages */
gntdev_put_map(NULL, map);
struct gntdev_priv *priv = file->private_data;
pr_debug("gntdev_vma_close %p\n", vma);
- if (use_ptemod) {
- WARN_ON(map->vma != vma);
- mmu_interval_notifier_remove(&map->notifier);
- map->vma = NULL;
- }
+
vma->vm_private_data = NULL;
gntdev_put_map(priv, map);
}
struct gntdev_grant_map *map =
container_of(mn, struct gntdev_grant_map, notifier);
unsigned long mstart, mend;
+ unsigned long map_start, map_end;
if (!mmu_notifier_range_blockable(range))
return false;
+ map_start = map->pages_vm_start;
+ map_end = map->pages_vm_start + (map->count << PAGE_SHIFT);
+
/*
* If the VMA is split or otherwise changed the notifier is not
* updated, but we don't want to process VA's outside the modified
* VMA. FIXME: It would be much more understandable to just prevent
* modifying the VMA in the first place.
*/
- if (map->vma->vm_start >= range->end ||
- map->vma->vm_end <= range->start)
+ if (map_start >= range->end || map_end <= range->start)
return true;
- mstart = max(range->start, map->vma->vm_start);
- mend = min(range->end, map->vma->vm_end);
+ mstart = max(range->start, map_start);
+ mend = min(range->end, map_end);
pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
- map->index, map->count,
- map->vma->vm_start, map->vma->vm_end,
- range->start, range->end, mstart, mend);
- unmap_grant_pages(map,
- (mstart - map->vma->vm_start) >> PAGE_SHIFT,
- (mend - mstart) >> PAGE_SHIFT);
+ map->index, map->count, map_start, map_end,
+ range->start, range->end, mstart, mend);
+ unmap_grant_pages(map, (mstart - map_start) >> PAGE_SHIFT,
+ (mend - mstart) >> PAGE_SHIFT);
return true;
}
return -EINVAL;
pr_debug("map %d+%d at %lx (pgoff %lx)\n",
- index, count, vma->vm_start, vma->vm_pgoff);
+ index, count, vma->vm_start, vma->vm_pgoff);
mutex_lock(&priv->lock);
map = gntdev_find_map_index(priv, index, count);
if (!map)
goto unlock_out;
- if (use_ptemod && map->vma)
+ if (!atomic_add_unless(&map->in_use, 1, 1))
goto unlock_out;
- if (atomic_read(&map->live_grants)) {
- err = -EAGAIN;
- goto unlock_out;
- }
+
refcount_inc(&map->users);
vma->vm_ops = &gntdev_vmops;
map->flags |= GNTMAP_readonly;
}
+ map->pages_vm_start = vma->vm_start;
+
if (use_ptemod) {
- map->vma = vma;
err = mmu_interval_notifier_insert_locked(
&map->notifier, vma->vm_mm, vma->vm_start,
vma->vm_end - vma->vm_start, &gntdev_mmu_ops);
- if (err) {
- map->vma = NULL;
+ if (err)
goto out_unlock_put;
- }
+
+ map->notifier_init = true;
}
mutex_unlock(&priv->lock);
*/
mmu_interval_read_begin(&map->notifier);
- map->pages_vm_start = vma->vm_start;
err = apply_to_page_range(vma->vm_mm, vma->vm_start,
vma->vm_end - vma->vm_start,
find_grant_ptes, map);
out_unlock_put:
mutex_unlock(&priv->lock);
out_put_map:
- if (use_ptemod) {
+ if (use_ptemod)
unmap_grant_pages(map, 0, map->count);
- if (map->vma) {
- mmu_interval_notifier_remove(&map->notifier);
- map->vma = NULL;
- }
- }
gntdev_put_map(priv, map);
return err;
}
bool broken;
};
-static DEFINE_XARRAY(xen_grant_dma_devices);
+static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);
#define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63)
static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)
{
struct xen_grant_dma_data *data;
+ unsigned long flags;
- xa_lock(&xen_grant_dma_devices);
+ xa_lock_irqsave(&xen_grant_dma_devices, flags);
data = xa_load(&xen_grant_dma_devices, (unsigned long)dev);
- xa_unlock(&xen_grant_dma_devices);
+ xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
return data;
}
+static int store_xen_grant_dma_data(struct device *dev,
+ struct xen_grant_dma_data *data)
+{
+ unsigned long flags;
+ int ret;
+
+ xa_lock_irqsave(&xen_grant_dma_devices, flags);
+ ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,
+ GFP_ATOMIC));
+ xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
+
+ return ret;
+}
+
/*
* DMA ops for Xen frontends (e.g. virtio).
*
unsigned long attrs)
{
struct xen_grant_dma_data *data;
- unsigned int i, n_pages = PFN_UP(size);
+ unsigned int i, n_pages = PFN_UP(offset + size);
grant_ref_t grant;
dma_addr_t dma_handle;
unsigned long attrs)
{
struct xen_grant_dma_data *data;
- unsigned int i, n_pages = PFN_UP(size);
+ unsigned long offset = dma_handle & (PAGE_SIZE - 1);
+ unsigned int i, n_pages = PFN_UP(offset + size);
grant_ref_t grant;
if (WARN_ON(dir == DMA_NONE))
.dma_supported = xen_grant_dma_supported,
};
-bool xen_is_grant_dma_device(struct device *dev)
+static bool xen_is_dt_grant_dma_device(struct device *dev)
{
struct device_node *iommu_np;
bool has_iommu;
- /* XXX Handle only DT devices for now */
- if (!dev->of_node)
- return false;
-
iommu_np = of_parse_phandle(dev->of_node, "iommus", 0);
- has_iommu = iommu_np && of_device_is_compatible(iommu_np, "xen,grant-dma");
+ has_iommu = iommu_np &&
+ of_device_is_compatible(iommu_np, "xen,grant-dma");
of_node_put(iommu_np);
return has_iommu;
}
+bool xen_is_grant_dma_device(struct device *dev)
+{
+ /* XXX Handle only DT devices for now */
+ if (dev->of_node)
+ return xen_is_dt_grant_dma_device(dev);
+
+ return false;
+}
+
bool xen_virtio_mem_acc(struct virtio_device *dev)
{
- if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT))
+ if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())
return true;
return xen_is_grant_dma_device(dev->dev.parent);
}
-void xen_grant_setup_dma_ops(struct device *dev)
+static int xen_dt_grant_init_backend_domid(struct device *dev,
+ struct xen_grant_dma_data *data)
{
- struct xen_grant_dma_data *data;
struct of_phandle_args iommu_spec;
- data = find_xen_grant_dma_data(dev);
- if (data) {
- dev_err(dev, "Xen grant DMA data is already created\n");
- return;
- }
-
- /* XXX ACPI device unsupported for now */
- if (!dev->of_node)
- goto err;
-
if (of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells",
0, &iommu_spec)) {
dev_err(dev, "Cannot parse iommus property\n");
- goto err;
+ return -ESRCH;
}
if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") ||
iommu_spec.args_count != 1) {
dev_err(dev, "Incompatible IOMMU node\n");
of_node_put(iommu_spec.np);
- goto err;
+ return -ESRCH;
}
of_node_put(iommu_spec.np);
+ /*
+ * The endpoint ID here means the ID of the domain where the
+ * corresponding backend is running
+ */
+ data->backend_domid = iommu_spec.args[0];
+
+ return 0;
+}
+
+void xen_grant_setup_dma_ops(struct device *dev)
+{
+ struct xen_grant_dma_data *data;
+
+ data = find_xen_grant_dma_data(dev);
+ if (data) {
+ dev_err(dev, "Xen grant DMA data is already created\n");
+ return;
+ }
+
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
goto err;
- /*
- * The endpoint ID here means the ID of the domain where the corresponding
- * backend is running
- */
- data->backend_domid = iommu_spec.args[0];
+ if (dev->of_node) {
+ if (xen_dt_grant_init_backend_domid(dev, data))
+ goto err;
+ } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT)) {
+ dev_info(dev, "Using dom0 as backend\n");
+ data->backend_domid = 0;
+ } else {
+ /* XXX ACPI device unsupported for now */
+ goto err;
+ }
- if (xa_err(xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,
- GFP_KERNEL))) {
+ if (store_xen_grant_dma_data(dev, data)) {
dev_err(dev, "Cannot store Xen grant DMA data\n");
goto err;
}
return;
err:
+ devm_kfree(dev, data);
dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");
}
+bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
+{
+ bool ret = xen_virtio_mem_acc(dev);
+
+ if (ret)
+ xen_grant_setup_dma_ops(dev->dev.parent);
+
+ return ret;
+}
+
MODULE_DESCRIPTION("Xen grant DMA-mapping layer");
MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");
MODULE_LICENSE("GPL");
struct page, lru);
struct privcmd_mmap_entry *msg = page_address(page);
- vma = find_vma(mm, msg->va);
+ vma = vma_lookup(mm, msg->va);
rc = -EINVAL;
if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data)
" frontend (for example, a device at 06:01.b will still appear at\n"\
" 06:01.b to the frontend). This is similar to how Xen 2.0.x\n"\
" exposed PCI devices to its driver domains. This may be required\n"\
- " for drivers which depend on finding their hardward in certain\n"\
+ " for drivers which depend on finding their hardware in certain\n"\
" bus/slot locations.");
static struct xen_pcibk_device *alloc_pdev(struct xenbus_device *xdev)
config HUGETLBFS
bool "HugeTLB file system support"
depends on X86 || IA64 || SPARC64 || ARCH_SUPPORTS_HUGETLBFS || BROKEN
+ depends on (SYSFS || SYSCTL)
help
hugetlbfs is a filesystem backing for HugeTLB pages, based on
ramfs. For architectures that support it, say Y here and read
config BINFMT_ELF_FDPIC
bool "Kernel support for FDPIC ELF binaries"
default y if !BINFMT_ELF
- depends on ARM || ((M68K || SUPERH) && !MMU)
+ depends on ARM || ((M68K || SUPERH || XTENSA) && !MMU)
select ELFCORE
help
ELF FDPIC binaries are based on ELF, but allow the individual load
local_irq_save(flags);
kcpu = this_cpu_ptr(ctx->cpu);
if (!kcpu->reqs_available) {
- int old, avail = atomic_read(&ctx->reqs_available);
+ int avail = atomic_read(&ctx->reqs_available);
do {
if (avail < ctx->req_batch)
goto out;
-
- old = avail;
- avail = atomic_cmpxchg(&ctx->reqs_available,
- avail, avail - ctx->req_batch);
- } while (avail != old);
+ } while (!atomic_try_cmpxchg(&ctx->reqs_available,
+ &avail, avail - ctx->req_batch));
kcpu->reqs_available += ctx->req_batch;
}
}
static int bad_inode_tmpfile(struct user_namespace *mnt_userns,
- struct inode *inode, struct dentry *dentry,
+ struct inode *inode, struct file *file,
umode_t mode)
{
return -EIO;
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
+#include <linux/pagevec.h>
#include <linux/highmem.h>
#include <linux/kthread.h>
#include <linux/time.h>
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
unsigned long index = cb->start >> PAGE_SHIFT;
unsigned long end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT;
- struct page *pages[16];
- unsigned long nr_pages = end_index - index + 1;
+ struct folio_batch fbatch;
const int errno = blk_status_to_errno(cb->status);
int i;
int ret;
if (errno)
mapping_set_error(inode->i_mapping, errno);
- while (nr_pages > 0) {
- ret = find_get_pages_contig(inode->i_mapping, index,
- min_t(unsigned long,
- nr_pages, ARRAY_SIZE(pages)), pages);
- if (ret == 0) {
- nr_pages -= 1;
- index += 1;
- continue;
- }
+ folio_batch_init(&fbatch);
+ while (index <= end_index) {
+ ret = filemap_get_folios(inode->i_mapping, &index, end_index,
+ &fbatch);
+
+ if (ret == 0)
+ return;
+
for (i = 0; i < ret; i++) {
+ struct folio *folio = fbatch.folios[i];
+
if (errno)
- SetPageError(pages[i]);
- btrfs_page_clamp_clear_writeback(fs_info, pages[i],
+ folio_set_error(folio);
+ btrfs_page_clamp_clear_writeback(fs_info, &folio->page,
cb->start, cb->len);
- put_page(pages[i]);
}
- nr_pages -= ret;
- index += ret;
+ folio_batch_release(&fbatch);
}
/* the inode may be gone now */
}
pgoff_t start_index = start >> PAGE_SHIFT;
pgoff_t end_index = end >> PAGE_SHIFT;
pgoff_t index = start_index;
- unsigned long nr_pages = end_index - start_index + 1;
unsigned long pages_processed = 0;
- struct page *pages[16];
+ struct folio_batch fbatch;
int err = 0;
int i;
ASSERT(processed_end && *processed_end == start);
}
- if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
+ if ((page_ops & PAGE_SET_ERROR) && start_index <= end_index)
mapping_set_error(mapping, -EIO);
- while (nr_pages > 0) {
- int found_pages;
+ folio_batch_init(&fbatch);
+ while (index <= end_index) {
+ int found_folios;
+
+ found_folios = filemap_get_folios_contig(mapping, &index,
+ end_index, &fbatch);
- found_pages = find_get_pages_contig(mapping, index,
- min_t(unsigned long,
- nr_pages, ARRAY_SIZE(pages)), pages);
- if (found_pages == 0) {
+ if (found_folios == 0) {
/*
* Only if we're going to lock these pages, we can find
* nothing at @index.
goto out;
}
- for (i = 0; i < found_pages; i++) {
+ for (i = 0; i < found_folios; i++) {
int process_ret;
-
+ struct folio *folio = fbatch.folios[i];
process_ret = process_one_page(fs_info, mapping,
- pages[i], locked_page, page_ops,
+ &folio->page, locked_page, page_ops,
start, end);
if (process_ret < 0) {
- for (; i < found_pages; i++)
- put_page(pages[i]);
err = -EAGAIN;
+ folio_batch_release(&fbatch);
goto out;
}
- put_page(pages[i]);
- pages_processed++;
+ pages_processed += folio_nr_pages(folio);
}
- nr_pages -= found_pages;
- index += found_pages;
+ folio_batch_release(&fbatch);
cond_resched();
}
out:
}
static int btrfs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
struct btrfs_trans_handle *trans;
struct inode *inode;
struct btrfs_new_inode_args new_inode_args = {
.dir = dir,
- .dentry = dentry,
+ .dentry = file->f_path.dentry,
.orphan = true,
};
unsigned int trans_num_items;
set_nlink(inode, 1);
if (!ret) {
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
unlock_new_inode(inode);
mark_inode_dirty(inode);
}
out_inode:
if (ret)
iput(inode);
- return ret;
+ return finish_open_simple(file, ret);
}
void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end)
*
* Even with 0 returned, the page still need extra check to make sure
* it's really the correct page, as the caller is using
- * find_get_pages_contig(), which can race with page invalidating.
+ * filemap_get_folios_contig(), which can race with page invalidating.
*/
int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
*/
#include <linux/pagemap.h>
+#include <linux/pagevec.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/sizes.h>
unsigned long flags)
{
int ret;
- struct page *pages[16];
+ struct folio_batch fbatch;
unsigned long index = start >> PAGE_SHIFT;
unsigned long end_index = end >> PAGE_SHIFT;
- unsigned long nr_pages = end_index - index + 1;
int i;
int count = 0;
int loops = 0;
- while (nr_pages > 0) {
- ret = find_get_pages_contig(inode->i_mapping, index,
- min_t(unsigned long, nr_pages,
- ARRAY_SIZE(pages)), pages);
+ folio_batch_init(&fbatch);
+
+ while (index <= end_index) {
+ ret = filemap_get_folios_contig(inode->i_mapping, &index,
+ end_index, &fbatch);
for (i = 0; i < ret; i++) {
+ struct folio *folio = fbatch.folios[i];
+
if (flags & PROCESS_TEST_LOCKED &&
- !PageLocked(pages[i]))
+ !folio_test_locked(folio))
count++;
- if (flags & PROCESS_UNLOCK && PageLocked(pages[i]))
- unlock_page(pages[i]);
- put_page(pages[i]);
+ if (flags & PROCESS_UNLOCK && folio_test_locked(folio))
+ folio_unlock(folio);
if (flags & PROCESS_RELEASE)
- put_page(pages[i]);
+ folio_put(folio);
}
- nr_pages -= ret;
- index += ret;
+ folio_batch_release(&fbatch);
cond_resched();
loops++;
if (loops > 100000) {
printk(KERN_ERR
- "stuck in a loop, start %llu, end %llu, nr_pages %lu, ret %d\n",
- start, end, nr_pages, ret);
+ "stuck in a loop, start %llu, end %llu, ret %d\n",
+ start, end, ret);
break;
}
}
+
return count;
}
/*
* Default synchronous end-of-IO handler.. Just mark it up-to-date and
- * unlock the buffer. This is what ll_rw_block uses too.
+ * unlock the buffer.
*/
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
{
* all already-submitted IO to complete, but does not queue any new
* writes to the disk.
*
- * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
- * you dirty the buffers, and then use osync_inode_buffers to wait for
+ * To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
+ * as you dirty the buffers, and then use osync_inode_buffers to wait for
* completion. Any other dirty buffers which are not yet queued for
* write will not be flushed to disk by the osync.
*/
struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
if (bh) {
if (buffer_dirty(bh))
- ll_rw_block(REQ_OP_WRITE, 1, &bh);
+ write_dirty_buffer(bh, 0);
put_bh(bh);
}
}
{
struct buffer_head *bh = __getblk(bdev, block, size);
if (likely(bh)) {
- ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh);
+ bh_readahead(bh, REQ_RAHEAD);
brelse(bh);
}
}
EXPORT_SYMBOL(__breadahead);
-void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size,
- gfp_t gfp)
-{
- struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
- if (likely(bh)) {
- ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh);
- brelse(bh);
- }
-}
-EXPORT_SYMBOL(__breadahead_gfp);
-
/**
* __bread_gfp() - reads a specified block and returns the bh
* @bdev: the block_device to read from
static void discard_buffer(struct buffer_head * bh)
{
- unsigned long b_state, b_state_old;
+ unsigned long b_state;
lock_buffer(bh);
clear_buffer_dirty(bh);
bh->b_bdev = NULL;
- b_state = bh->b_state;
- for (;;) {
- b_state_old = cmpxchg(&bh->b_state, b_state,
- (b_state & ~BUFFER_FLAGS_DISCARD));
- if (b_state_old == b_state)
- break;
- b_state = b_state_old;
- }
+ b_state = READ_ONCE(bh->b_state);
+ do {
+ } while (!try_cmpxchg(&bh->b_state, &b_state,
+ b_state & ~BUFFER_FLAGS_DISCARD));
unlock_buffer(bh);
}
/*
* The page was marked dirty, but the buffers were
* clean. Someone wrote them back by hand with
- * ll_rw_block/submit_bh. A rare case.
+ * write_dirty_buffer/submit_bh. A rare case.
*/
end_page_writeback(page);
if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
!buffer_unwritten(bh) &&
(block_start < from || block_end > to)) {
- ll_rw_block(REQ_OP_READ, 1, &bh);
+ bh_read_nowait(bh, 0);
*wait_bh++=bh;
}
}
struct address_space *mapping = inode->i_mapping;
const struct address_space_operations *aops = mapping->a_ops;
struct page *page;
- void *fsdata;
+ void *fsdata = NULL;
int err;
err = inode_newsize_ok(inode, size);
const struct address_space_operations *aops = mapping->a_ops;
unsigned int blocksize = i_blocksize(inode);
struct page *page;
- void *fsdata;
+ void *fsdata = NULL;
pgoff_t index, curidx;
loff_t curpos;
unsigned zerofrom, offset, len;
set_buffer_uptodate(bh);
if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
- err = -EIO;
- ll_rw_block(REQ_OP_READ, 1, &bh);
- wait_on_buffer(bh);
+ err = bh_read(bh, 0);
/* Uhhuh. Read error. Complain and punt. */
- if (!buffer_uptodate(bh))
+ if (err < 0)
goto unlock;
}
}
EXPORT_SYMBOL(submit_bh);
-/**
- * ll_rw_block: low-level access to block devices (DEPRECATED)
- * @opf: block layer request operation and flags.
- * @nr: number of &struct buffer_heads in the array
- * @bhs: array of pointers to &struct buffer_head
- *
- * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
- * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
- * @opf contains flags modifying the detailed I/O behavior, most notably
- * %REQ_RAHEAD.
- *
- * This function drops any buffer that it cannot get a lock on (with the
- * BH_Lock state bit), any buffer that appears to be clean when doing a write
- * request, and any buffer that appears to be up-to-date when doing read
- * request. Further it marks as clean buffers that are processed for
- * writing (the buffer cache won't assume that they are actually clean
- * until the buffer gets unlocked).
- *
- * ll_rw_block sets b_end_io to simple completion handler that marks
- * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
- * any waiters.
- *
- * All of the buffers must be for the same device, and must also be a
- * multiple of the current approved size for the device.
- */
-void ll_rw_block(const blk_opf_t opf, int nr, struct buffer_head *bhs[])
-{
- const enum req_op op = opf & REQ_OP_MASK;
- int i;
-
- for (i = 0; i < nr; i++) {
- struct buffer_head *bh = bhs[i];
-
- if (!trylock_buffer(bh))
- continue;
- if (op == REQ_OP_WRITE) {
- if (test_clear_buffer_dirty(bh)) {
- bh->b_end_io = end_buffer_write_sync;
- get_bh(bh);
- submit_bh(opf, bh);
- continue;
- }
- } else {
- if (!buffer_uptodate(bh)) {
- bh->b_end_io = end_buffer_read_sync;
- get_bh(bh);
- submit_bh(opf, bh);
- continue;
- }
- }
- unlock_buffer(bh);
- }
-}
-EXPORT_SYMBOL(ll_rw_block);
-
void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
{
lock_buffer(bh);
EXPORT_SYMBOL(bh_uptodate_or_lock);
/**
- * bh_submit_read - Submit a locked buffer for reading
+ * __bh_read - Submit read for a locked buffer
* @bh: struct buffer_head
+ * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
+ * @wait: wait until reading finish
*
- * Returns zero on success and -EIO on error.
+ * Returns zero on success or don't wait, and -EIO on error.
*/
-int bh_submit_read(struct buffer_head *bh)
+int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
{
- BUG_ON(!buffer_locked(bh));
+ int ret = 0;
- if (buffer_uptodate(bh)) {
- unlock_buffer(bh);
- return 0;
- }
+ BUG_ON(!buffer_locked(bh));
get_bh(bh);
bh->b_end_io = end_buffer_read_sync;
- submit_bh(REQ_OP_READ, bh);
- wait_on_buffer(bh);
- if (buffer_uptodate(bh))
- return 0;
- return -EIO;
+ submit_bh(REQ_OP_READ | op_flags, bh);
+ if (wait) {
+ wait_on_buffer(bh);
+ if (!buffer_uptodate(bh))
+ ret = -EIO;
+ }
+ return ret;
+}
+EXPORT_SYMBOL(__bh_read);
+
+/**
+ * __bh_read_batch - Submit read for a batch of unlocked buffers
+ * @nr: entry number of the buffer batch
+ * @bhs: a batch of struct buffer_head
+ * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
+ * @force_lock: force to get a lock on the buffer if set, otherwise drops any
+ * buffer that cannot lock.
+ *
+ * Returns zero on success or don't wait, and -EIO on error.
+ */
+void __bh_read_batch(int nr, struct buffer_head *bhs[],
+ blk_opf_t op_flags, bool force_lock)
+{
+ int i;
+
+ for (i = 0; i < nr; i++) {
+ struct buffer_head *bh = bhs[i];
+
+ if (buffer_uptodate(bh))
+ continue;
+
+ if (force_lock)
+ lock_buffer(bh);
+ else
+ if (!trylock_buffer(bh))
+ continue;
+
+ if (buffer_uptodate(bh)) {
+ unlock_buffer(bh);
+ continue;
+ }
+
+ bh->b_end_io = end_buffer_read_sync;
+ get_bh(bh);
+ submit_bh(REQ_OP_READ | op_flags, bh);
+ }
}
-EXPORT_SYMBOL(bh_submit_read);
+EXPORT_SYMBOL(__bh_read_batch);
void __init buffer_init(void)
{
* file or directory. The caller must hold the inode lock.
*/
static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object,
- struct dentry *dentry)
+ struct inode *inode)
{
- struct inode *inode = d_backing_inode(dentry);
bool can_use = false;
if (!(inode->i_flags & S_KERNEL_FILE)) {
can_use = true;
} else {
trace_cachefiles_mark_failed(object, inode);
- pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n",
- dentry, inode->i_ino);
}
return can_use;
}
static bool cachefiles_mark_inode_in_use(struct cachefiles_object *object,
- struct dentry *dentry)
+ struct inode *inode)
{
- struct inode *inode = d_backing_inode(dentry);
bool can_use;
inode_lock(inode);
- can_use = __cachefiles_mark_inode_in_use(object, dentry);
+ can_use = __cachefiles_mark_inode_in_use(object, inode);
inode_unlock(inode);
return can_use;
}
* Unmark a backing inode. The caller must hold the inode lock.
*/
static void __cachefiles_unmark_inode_in_use(struct cachefiles_object *object,
- struct dentry *dentry)
+ struct inode *inode)
{
- struct inode *inode = d_backing_inode(dentry);
-
inode->i_flags &= ~S_KERNEL_FILE;
trace_cachefiles_mark_inactive(object, inode);
}
static void cachefiles_do_unmark_inode_in_use(struct cachefiles_object *object,
- struct dentry *dentry)
+ struct inode *inode)
{
- struct inode *inode = d_backing_inode(dentry);
-
inode_lock(inode);
- __cachefiles_unmark_inode_in_use(object, dentry);
+ __cachefiles_unmark_inode_in_use(object, inode);
inode_unlock(inode);
}
struct cachefiles_cache *cache = object->volume->cache;
struct inode *inode = file_inode(file);
- if (inode) {
- cachefiles_do_unmark_inode_in_use(object, file->f_path.dentry);
+ cachefiles_do_unmark_inode_in_use(object, inode);
- if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) {
- atomic_long_add(inode->i_blocks, &cache->b_released);
- if (atomic_inc_return(&cache->f_released))
- cachefiles_state_changed(cache);
- }
+ if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) {
+ atomic_long_add(inode->i_blocks, &cache->b_released);
+ if (atomic_inc_return(&cache->f_released))
+ cachefiles_state_changed(cache);
}
}
inode_lock(d_inode(subdir));
inode_unlock(d_inode(dir));
- if (!__cachefiles_mark_inode_in_use(NULL, subdir))
+ if (!__cachefiles_mark_inode_in_use(NULL, d_inode(subdir))) {
+ pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n",
+ subdir, d_inode(subdir)->i_ino);
goto mark_error;
+ }
inode_unlock(d_inode(subdir));
void cachefiles_put_directory(struct dentry *dir)
{
if (dir) {
- inode_lock(dir->d_inode);
- __cachefiles_unmark_inode_in_use(NULL, dir);
- inode_unlock(dir->d_inode);
+ cachefiles_do_unmark_inode_in_use(NULL, d_inode(dir));
dput(dir);
}
}
"Rename failed with error %d", ret);
}
- __cachefiles_unmark_inode_in_use(object, rep);
+ __cachefiles_unmark_inode_in_use(object, d_inode(rep));
unlock_rename(cache->graveyard, dir);
dput(grave);
_leave(" = 0");
const struct cred *saved_cred;
struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash];
struct file *file;
- struct path path;
+ const struct path parentpath = { .mnt = cache->mnt, .dentry = fan };
uint64_t ni_size;
long ret;
cachefiles_begin_secure(cache, &saved_cred);
- path.mnt = cache->mnt;
ret = cachefiles_inject_write_error();
- if (ret == 0)
- path.dentry = vfs_tmpfile(&init_user_ns, fan, S_IFREG, O_RDWR);
- else
- path.dentry = ERR_PTR(ret);
- if (IS_ERR(path.dentry)) {
- trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(path.dentry),
+ if (ret == 0) {
+ file = vfs_tmpfile_open(&init_user_ns, &parentpath, S_IFREG,
+ O_RDWR | O_LARGEFILE | O_DIRECT,
+ cache->cache_cred);
+ ret = PTR_ERR_OR_ZERO(file);
+ }
+ if (ret) {
+ trace_cachefiles_vfs_error(object, d_inode(fan), ret,
cachefiles_trace_tmpfile_error);
- if (PTR_ERR(path.dentry) == -EIO)
+ if (ret == -EIO)
cachefiles_io_error_obj(object, "Failed to create tmpfile");
- file = ERR_CAST(path.dentry);
- goto out;
+ goto err;
}
- trace_cachefiles_tmpfile(object, d_backing_inode(path.dentry));
+ trace_cachefiles_tmpfile(object, file_inode(file));
- if (!cachefiles_mark_inode_in_use(object, path.dentry)) {
- file = ERR_PTR(-EBUSY);
- goto out_dput;
- }
+ /* This is a newly created file with no other possible user */
+ if (!cachefiles_mark_inode_in_use(object, file_inode(file)))
+ WARN_ON(1);
ret = cachefiles_ondemand_init_object(object);
- if (ret < 0) {
- file = ERR_PTR(ret);
- goto out_unuse;
- }
+ if (ret < 0)
+ goto err_unuse;
ni_size = object->cookie->object_size;
ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE);
if (ni_size > 0) {
- trace_cachefiles_trunc(object, d_backing_inode(path.dentry), 0, ni_size,
+ trace_cachefiles_trunc(object, file_inode(file), 0, ni_size,
cachefiles_trunc_expand_tmpfile);
ret = cachefiles_inject_write_error();
if (ret == 0)
- ret = vfs_truncate(&path, ni_size);
+ ret = vfs_truncate(&file->f_path, ni_size);
if (ret < 0) {
trace_cachefiles_vfs_error(
- object, d_backing_inode(path.dentry), ret,
+ object, file_inode(file), ret,
cachefiles_trace_trunc_error);
- file = ERR_PTR(ret);
- goto out_unuse;
+ goto err_unuse;
}
}
- file = open_with_fake_path(&path, O_RDWR | O_LARGEFILE | O_DIRECT,
- d_backing_inode(path.dentry), cache->cache_cred);
- if (IS_ERR(file)) {
- trace_cachefiles_vfs_error(object, d_backing_inode(path.dentry),
- PTR_ERR(file),
- cachefiles_trace_open_error);
- goto out_unuse;
- }
+ ret = -EINVAL;
if (unlikely(!file->f_op->read_iter) ||
unlikely(!file->f_op->write_iter)) {
fput(file);
pr_notice("Cache does not support read_iter and write_iter\n");
- file = ERR_PTR(-EINVAL);
- goto out_unuse;
+ goto err_unuse;
}
-
- goto out_dput;
-
-out_unuse:
- cachefiles_do_unmark_inode_in_use(object, path.dentry);
-out_dput:
- dput(path.dentry);
out:
cachefiles_end_secure(cache, saved_cred);
return file;
+
+err_unuse:
+ cachefiles_do_unmark_inode_in_use(object, file_inode(file));
+ fput(file);
+err:
+ file = ERR_PTR(ret);
+ goto out;
}
/*
_enter("%pd", dentry);
- if (!cachefiles_mark_inode_in_use(object, dentry))
+ if (!cachefiles_mark_inode_in_use(object, d_inode(dentry))) {
+ pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n",
+ dentry, d_inode(dentry)->i_ino);
return false;
+ }
/* We need to open a file interface onto a data file now as we can't do
* it on demand because writeback called from do_exit() sees
error_fput:
fput(file);
error:
- cachefiles_do_unmark_inode_in_use(object, dentry);
+ cachefiles_do_unmark_inode_in_use(object, d_inode(dentry));
dput(dentry);
return false;
}
cap->issue_seq = seq;
cap->mseq = mseq;
cap->cap_gen = gen;
+ wake_up_all(&ci->i_cap_wq);
}
/*
struct ceph_mds_request *req;
int i;
- sessions = kzalloc(max_sessions * sizeof(s), GFP_KERNEL);
+ sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL);
if (!sessions) {
err = -ENOMEM;
goto out;
* on transition from wanted -> needed caps. This is needed
* for WRBUFFER|WR -> WR to avoid a new WR sync write from
* going before a prior buffered writeback happens.
+ *
+ * For RDCACHE|RD -> RD, there is not need to wait and we can
+ * just exclude the revoking caps and force to sync read.
*/
int not = want & ~(have & need);
int revoking = implemented & ~have;
+ int exclude = revoking & not;
dout("get_cap_refs %p have %s but not %s (revoking %s)\n",
inode, ceph_cap_string(have), ceph_cap_string(not),
ceph_cap_string(revoking));
- if ((revoking & not) == 0) {
+ if (!exclude || !(exclude & CEPH_CAP_FILE_BUFFER)) {
if (!snap_rwsem_locked &&
!ci->i_head_snapc &&
(need & CEPH_CAP_FILE_WR)) {
snap_rwsem_locked = true;
}
if ((have & want) == want)
- *got = need | want;
+ *got = need | (want & ~exclude);
else
*got = need;
ceph_take_cap_refs(ci, *got, true);
check_caps = 1; /* check auth cap only */
else
check_caps = 2; /* check all caps */
+ /* If there is new caps, try to wake up the waiters */
+ if (~cap->issued & newcaps)
+ wake = true;
cap->issued = newcaps;
cap->implemented |= newcaps;
} else if (cap->issued == newcaps) {
static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino)
{
struct inode *inode = __lookup_inode(sb, ino);
+ struct ceph_inode_info *ci = ceph_inode(inode);
int err;
if (IS_ERR(inode))
return ERR_PTR(err);
}
/* -ESTALE if inode as been unlinked and no file is open */
- if ((inode->i_nlink == 0) && (atomic_read(&inode->i_count) == 1)) {
+ if ((inode->i_nlink == 0) && !__ceph_is_file_opened(ci)) {
iput(inode);
return ERR_PTR(-ESTALE);
}
if (nsplits != ci->i_fragtree_nsplits) {
update = true;
} else if (nsplits) {
- i = prandom_u32() % nsplits;
+ i = prandom_u32_max(nsplits);
id = le32_to_cpu(fragtree->splits[i].frag);
if (!__ceph_find_frag(ci, id))
update = true;
inode_dirty_flags = __ceph_mark_dirty_caps(ci, dirtied,
&prealloc_cf);
inode->i_ctime = attr->ia_ctime;
+ inode_inc_iversion_raw(inode);
}
release &= issued;
goto out;
}
+ req->r_feature_needed = CEPHFS_FEATURE_OP_GETVXATTR;
req->r_path2 = kstrdup(name, GFP_NOFS);
if (!req->r_path2) {
err = -ENOMEM;
struct kstat *stat, u32 request_mask, unsigned int flags)
{
struct inode *inode = d_inode(path->dentry);
+ struct super_block *sb = inode->i_sb;
struct ceph_inode_info *ci = ceph_inode(inode);
u32 valid_mask = STATX_BASIC_STATS;
int err = 0;
}
if (ceph_snap(inode) == CEPH_NOSNAP)
- stat->dev = inode->i_sb->s_dev;
+ stat->dev = sb->s_dev;
else
stat->dev = ci->i_snapid_map ? ci->i_snapid_map->dev : 0;
if (S_ISDIR(inode->i_mode)) {
- if (ceph_test_mount_opt(ceph_sb_to_client(inode->i_sb),
- RBYTES))
+ if (ceph_test_mount_opt(ceph_sb_to_client(sb), RBYTES)) {
stat->size = ci->i_rbytes;
- else
+ } else if (ceph_snap(inode) == CEPH_SNAPDIR) {
+ struct ceph_inode_info *pci;
+ struct ceph_snap_realm *realm;
+ struct inode *parent;
+
+ parent = ceph_lookup_inode(sb, ceph_ino(inode));
+ if (!parent)
+ return PTR_ERR(parent);
+
+ pci = ceph_inode(parent);
+ spin_lock(&pci->i_ceph_lock);
+ realm = pci->i_snap_realm;
+ if (realm)
+ stat->size = realm->num_snaps;
+ else
+ stat->size = 0;
+ spin_unlock(&pci->i_ceph_lock);
+ iput(parent);
+ } else {
stat->size = ci->i_files + ci->i_subdirs;
+ }
stat->blocks = 0;
stat->blksize = 65536;
/*
INIT_LIST_HEAD(&req->r_unsafe_dir_item);
INIT_LIST_HEAD(&req->r_unsafe_target_item);
req->r_fmode = -1;
+ req->r_feature_needed = -1;
kref_init(&req->r_kref);
RB_CLEAR_NODE(&req->r_node);
INIT_LIST_HEAD(&req->r_wait);
dout("do_request mds%d session %p state %s\n", mds, session,
ceph_session_state_name(session->s_state));
+
+ /*
+ * The old ceph will crash the MDSs when see unknown OPs
+ */
+ if (req->r_feature_needed > 0 &&
+ !test_bit(req->r_feature_needed, &session->s_features)) {
+ err = -EOPNOTSUPP;
+ goto out_session;
+ }
+
if (session->s_state != CEPH_MDS_SESSION_OPEN &&
session->s_state != CEPH_MDS_SESSION_HUNG) {
/*
CEPHFS_FEATURE_METRIC_COLLECT,
CEPHFS_FEATURE_ALTERNATE_NAME,
CEPHFS_FEATURE_NOTIFY_SESSION_STATE,
+ CEPHFS_FEATURE_OP_GETVXATTR,
- CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_NOTIFY_SESSION_STATE,
+ CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_OP_GETVXATTR,
};
#define CEPHFS_FEATURES_CLIENT_SUPPORTED { \
CEPHFS_FEATURE_DELEG_INO, \
CEPHFS_FEATURE_METRIC_COLLECT, \
CEPHFS_FEATURE_NOTIFY_SESSION_STATE, \
+ CEPHFS_FEATURE_OP_GETVXATTR, \
}
/*
long long r_dir_ordered_cnt;
int r_readdir_cache_idx;
+ int r_feature_needed;
+
struct ceph_cap_reservation r_caps_reservation;
};
return -1;
/* pick */
- n = prandom_u32() % n;
+ n = prandom_u32_max(n);
for (j = 0, i = 0; i < m->possible_max_rank; i++) {
if (CEPH_MDS_IS_READY(i, ignore_laggy))
j++;
* Copyright (c) 2022, Ronnie Sahlberg <lsahlber@redhat.com>
*/
+#include <linux/namei.h>
#include "cifsglob.h"
#include "cifsproto.h"
#include "cifs_debug.h"
#include "smb2proto.h"
#include "cached_dir.h"
+static struct cached_fid *init_cached_dir(const char *path);
+static void free_cached_dir(struct cached_fid *cfid);
+
+static struct cached_fid *find_or_create_cached_dir(struct cached_fids *cfids,
+ const char *path,
+ bool lookup_only)
+{
+ struct cached_fid *cfid;
+
+ spin_lock(&cfids->cfid_list_lock);
+ list_for_each_entry(cfid, &cfids->entries, entry) {
+ if (!strcmp(cfid->path, path)) {
+ /*
+ * If it doesn't have a lease it is either not yet
+ * fully cached or it may be in the process of
+ * being deleted due to a lease break.
+ */
+ if (!cfid->has_lease) {
+ spin_unlock(&cfids->cfid_list_lock);
+ return NULL;
+ }
+ kref_get(&cfid->refcount);
+ spin_unlock(&cfids->cfid_list_lock);
+ return cfid;
+ }
+ }
+ if (lookup_only) {
+ spin_unlock(&cfids->cfid_list_lock);
+ return NULL;
+ }
+ if (cfids->num_entries >= MAX_CACHED_FIDS) {
+ spin_unlock(&cfids->cfid_list_lock);
+ return NULL;
+ }
+ cfid = init_cached_dir(path);
+ if (cfid == NULL) {
+ spin_unlock(&cfids->cfid_list_lock);
+ return NULL;
+ }
+ cfid->cfids = cfids;
+ cfids->num_entries++;
+ list_add(&cfid->entry, &cfids->entries);
+ cfid->on_list = true;
+ kref_get(&cfid->refcount);
+ spin_unlock(&cfids->cfid_list_lock);
+ return cfid;
+}
+
+static struct dentry *
+path_to_dentry(struct cifs_sb_info *cifs_sb, const char *path)
+{
+ struct dentry *dentry;
+ const char *s, *p;
+ char sep;
+
+ sep = CIFS_DIR_SEP(cifs_sb);
+ dentry = dget(cifs_sb->root);
+ s = path;
+
+ do {
+ struct inode *dir = d_inode(dentry);
+ struct dentry *child;
+
+ if (!S_ISDIR(dir->i_mode)) {
+ dput(dentry);
+ dentry = ERR_PTR(-ENOTDIR);
+ break;
+ }
+
+ /* skip separators */
+ while (*s == sep)
+ s++;
+ if (!*s)
+ break;
+ p = s++;
+ /* next separator */
+ while (*s && *s != sep)
+ s++;
+
+ child = lookup_positive_unlocked(p, dentry, s - p);
+ dput(dentry);
+ dentry = child;
+ } while (!IS_ERR(dentry));
+ return dentry;
+}
+
/*
* Open the and cache a directory handle.
* If error then *cfid is not initialized.
struct kvec open_iov[SMB2_CREATE_IOV_SIZE];
struct kvec qi_iov[1];
int rc, flags = 0;
- __le16 utf16_path = 0; /* Null - since an open of top of share */
+ __le16 *utf16_path = NULL;
u8 oplock = SMB2_OPLOCK_LEVEL_II;
struct cifs_fid *pfid;
- struct dentry *dentry;
+ struct dentry *dentry = NULL;
struct cached_fid *cfid;
+ struct cached_fids *cfids;
- if (tcon == NULL || tcon->nohandlecache ||
+ if (tcon == NULL || tcon->cfids == NULL || tcon->nohandlecache ||
is_smb1_server(tcon->ses->server))
return -EOPNOTSUPP;
ses = tcon->ses;
server = ses->server;
+ cfids = tcon->cfids;
- if (cifs_sb->root == NULL)
- return -ENOENT;
+ if (!server->ops->new_lease_key)
+ return -EIO;
- if (strlen(path))
+ if (cifs_sb->root == NULL)
return -ENOENT;
- dentry = cifs_sb->root;
+ utf16_path = cifs_convert_path_to_utf16(path, cifs_sb);
+ if (!utf16_path)
+ return -ENOMEM;
- cfid = tcon->cfid;
- mutex_lock(&cfid->fid_mutex);
- if (cfid->is_valid) {
- cifs_dbg(FYI, "found a cached root file handle\n");
+ cfid = find_or_create_cached_dir(cfids, path, lookup_only);
+ if (cfid == NULL) {
+ kfree(utf16_path);
+ return -ENOENT;
+ }
+ /*
+ * At this point we either have a lease already and we can just
+ * return it. If not we are guaranteed to be the only thread accessing
+ * this cfid.
+ */
+ if (cfid->has_lease) {
*ret_cfid = cfid;
- kref_get(&cfid->refcount);
- mutex_unlock(&cfid->fid_mutex);
+ kfree(utf16_path);
return 0;
}
/*
* We do not hold the lock for the open because in case
- * SMB2_open needs to reconnect, it will end up calling
- * cifs_mark_open_files_invalid() which takes the lock again
- * thus causing a deadlock
+ * SMB2_open needs to reconnect.
+ * This is safe because no other thread will be able to get a ref
+ * to the cfid until we have finished opening the file and (possibly)
+ * acquired a lease.
*/
- mutex_unlock(&cfid->fid_mutex);
-
- if (lookup_only)
- return -ENOENT;
-
if (smb3_encryption_required(tcon))
flags |= CIFS_TRANSFORM_REQ;
- if (!server->ops->new_lease_key)
- return -EIO;
-
pfid = &cfid->fid;
server->ops->new_lease_key(pfid);
oparms.reconnect = false;
rc = SMB2_open_init(tcon, server,
- &rqst[0], &oplock, &oparms, &utf16_path);
+ &rqst[0], &oplock, &oparms, utf16_path);
if (rc)
goto oshr_free;
smb2_set_next_command(tcon, &rqst[0]);
rc = compound_send_recv(xid, ses, server,
flags, 2, rqst,
resp_buftype, rsp_iov);
- mutex_lock(&cfid->fid_mutex);
-
- /*
- * Now we need to check again as the cached root might have
- * been successfully re-opened from a concurrent process
- */
-
- if (cfid->is_valid) {
- /* work was already done */
-
- /* stash fids for close() later */
- struct cifs_fid fid = {
- .persistent_fid = pfid->persistent_fid,
- .volatile_fid = pfid->volatile_fid,
- };
-
- /*
- * caller expects this func to set the fid in cfid to valid
- * cached root, so increment the refcount.
- */
- kref_get(&cfid->refcount);
-
- mutex_unlock(&cfid->fid_mutex);
-
- if (rc == 0) {
- /* close extra handle outside of crit sec */
- SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
- }
- rc = 0;
- goto oshr_free;
- }
-
- /* Cached root is still invalid, continue normaly */
-
if (rc) {
if (rc == -EREMCHG) {
tcon->need_reconnect = true;
pr_warn_once("server share %s deleted\n",
- tcon->treeName);
+ tcon->tree_name);
}
- goto oshr_exit;
+ goto oshr_free;
}
atomic_inc(&tcon->num_remote_opens);
oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId);
#endif /* CIFS_DEBUG2 */
- cfid->tcon = tcon;
- cfid->is_valid = true;
- cfid->dentry = dentry;
- dget(dentry);
- kref_init(&cfid->refcount);
+ if (o_rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE)
+ goto oshr_free;
- /* BB TBD check to see if oplock level check can be removed below */
- if (o_rsp->OplockLevel == SMB2_OPLOCK_LEVEL_LEASE) {
- /*
- * See commit 2f94a3125b87. Increment the refcount when we
- * get a lease for root, release it if lease break occurs
- */
- kref_get(&cfid->refcount);
- cfid->has_lease = true;
- smb2_parse_contexts(server, o_rsp,
- &oparms.fid->epoch,
- oparms.fid->lease_key, &oplock,
- NULL, NULL);
- } else
- goto oshr_exit;
+
+ smb2_parse_contexts(server, o_rsp,
+ &oparms.fid->epoch,
+ oparms.fid->lease_key, &oplock,
+ NULL, NULL);
qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base;
if (le32_to_cpu(qi_rsp->OutputBufferLength) < sizeof(struct smb2_file_all_info))
- goto oshr_exit;
+ goto oshr_free;
if (!smb2_validate_and_copy_iov(
le16_to_cpu(qi_rsp->OutputBufferOffset),
sizeof(struct smb2_file_all_info),
(char *)&cfid->file_all_info))
cfid->file_all_info_is_valid = true;
+ if (!path[0])
+ dentry = dget(cifs_sb->root);
+ else {
+ dentry = path_to_dentry(cifs_sb, path);
+ if (IS_ERR(dentry))
+ goto oshr_free;
+ }
+ cfid->dentry = dentry;
+ cfid->tcon = tcon;
cfid->time = jiffies;
+ cfid->is_open = true;
+ cfid->has_lease = true;
-oshr_exit:
- mutex_unlock(&cfid->fid_mutex);
oshr_free:
+ kfree(utf16_path);
SMB2_open_free(&rqst[0]);
SMB2_query_info_free(&rqst[1]);
free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
+ spin_lock(&cfids->cfid_list_lock);
+ if (!cfid->has_lease) {
+ if (cfid->on_list) {
+ list_del(&cfid->entry);
+ cfid->on_list = false;
+ cfids->num_entries--;
+ }
+ rc = -ENOENT;
+ }
+ spin_unlock(&cfids->cfid_list_lock);
+ if (rc) {
+ free_cached_dir(cfid);
+ cfid = NULL;
+ }
+
if (rc == 0)
*ret_cfid = cfid;
struct cached_fid **ret_cfid)
{
struct cached_fid *cfid;
+ struct cached_fids *cfids = tcon->cfids;
- cfid = tcon->cfid;
+ if (cfids == NULL)
+ return -ENOENT;
- mutex_lock(&cfid->fid_mutex);
- if (cfid->dentry == dentry) {
- cifs_dbg(FYI, "found a cached root file handle by dentry\n");
- *ret_cfid = cfid;
- kref_get(&cfid->refcount);
- mutex_unlock(&cfid->fid_mutex);
- return 0;
+ spin_lock(&cfids->cfid_list_lock);
+ list_for_each_entry(cfid, &cfids->entries, entry) {
+ if (dentry && cfid->dentry == dentry) {
+ cifs_dbg(FYI, "found a cached root file handle by dentry\n");
+ kref_get(&cfid->refcount);
+ *ret_cfid = cfid;
+ spin_unlock(&cfids->cfid_list_lock);
+ return 0;
+ }
}
- mutex_unlock(&cfid->fid_mutex);
+ spin_unlock(&cfids->cfid_list_lock);
return -ENOENT;
}
{
struct cached_fid *cfid = container_of(ref, struct cached_fid,
refcount);
- struct cached_dirent *dirent, *q;
- if (cfid->is_valid) {
- cifs_dbg(FYI, "clear cached root file handle\n");
- SMB2_close(0, cfid->tcon, cfid->fid.persistent_fid,
- cfid->fid.volatile_fid);
+ spin_lock(&cfid->cfids->cfid_list_lock);
+ if (cfid->on_list) {
+ list_del(&cfid->entry);
+ cfid->on_list = false;
+ cfid->cfids->num_entries--;
}
+ spin_unlock(&cfid->cfids->cfid_list_lock);
- /*
- * We only check validity above to send SMB2_close,
- * but we still need to invalidate these entries
- * when this function is called
- */
- cfid->is_valid = false;
- cfid->file_all_info_is_valid = false;
- cfid->has_lease = false;
- if (cfid->dentry) {
- dput(cfid->dentry);
- cfid->dentry = NULL;
- }
- /*
- * Delete all cached dirent names
- */
- mutex_lock(&cfid->dirents.de_mutex);
- list_for_each_entry_safe(dirent, q, &cfid->dirents.entries, entry) {
- list_del(&dirent->entry);
- kfree(dirent->name);
- kfree(dirent);
+ dput(cfid->dentry);
+ cfid->dentry = NULL;
+
+ if (cfid->is_open) {
+ SMB2_close(0, cfid->tcon, cfid->fid.persistent_fid,
+ cfid->fid.volatile_fid);
}
- cfid->dirents.is_valid = 0;
- cfid->dirents.is_failed = 0;
- cfid->dirents.ctx = NULL;
- cfid->dirents.pos = 0;
- mutex_unlock(&cfid->dirents.de_mutex);
+ free_cached_dir(cfid);
}
void close_cached_dir(struct cached_fid *cfid)
{
- mutex_lock(&cfid->fid_mutex);
kref_put(&cfid->refcount, smb2_close_cached_fid);
- mutex_unlock(&cfid->fid_mutex);
-}
-
-void close_cached_dir_lease_locked(struct cached_fid *cfid)
-{
- if (cfid->has_lease) {
- cfid->has_lease = false;
- kref_put(&cfid->refcount, smb2_close_cached_fid);
- }
-}
-
-void close_cached_dir_lease(struct cached_fid *cfid)
-{
- mutex_lock(&cfid->fid_mutex);
- close_cached_dir_lease_locked(cfid);
- mutex_unlock(&cfid->fid_mutex);
}
/*
struct cached_fid *cfid;
struct cifs_tcon *tcon;
struct tcon_link *tlink;
+ struct cached_fids *cfids;
for (node = rb_first(root); node; node = rb_next(node)) {
tlink = rb_entry(node, struct tcon_link, tl_rbnode);
tcon = tlink_tcon(tlink);
if (IS_ERR(tcon))
continue;
- cfid = tcon->cfid;
- mutex_lock(&cfid->fid_mutex);
- if (cfid->dentry) {
+ cfids = tcon->cfids;
+ if (cfids == NULL)
+ continue;
+ list_for_each_entry(cfid, &cfids->entries, entry) {
dput(cfid->dentry);
cfid->dentry = NULL;
}
- mutex_unlock(&cfid->fid_mutex);
}
}
/*
- * Invalidate and close all cached dirs when a TCON has been reset
+ * Invalidate all cached dirs when a TCON has been reset
* due to a session loss.
*/
void invalidate_all_cached_dirs(struct cifs_tcon *tcon)
{
- mutex_lock(&tcon->cfid->fid_mutex);
- tcon->cfid->is_valid = false;
- /* cached handle is not valid, so SMB2_CLOSE won't be sent below */
- close_cached_dir_lease_locked(tcon->cfid);
- memset(&tcon->cfid->fid, 0, sizeof(struct cifs_fid));
- mutex_unlock(&tcon->cfid->fid_mutex);
+ struct cached_fids *cfids = tcon->cfids;
+ struct cached_fid *cfid, *q;
+ struct list_head entry;
+
+ INIT_LIST_HEAD(&entry);
+ spin_lock(&cfids->cfid_list_lock);
+ list_for_each_entry_safe(cfid, q, &cfids->entries, entry) {
+ list_del(&cfid->entry);
+ list_add(&cfid->entry, &entry);
+ cfids->num_entries--;
+ cfid->is_open = false;
+ /* To prevent race with smb2_cached_lease_break() */
+ kref_get(&cfid->refcount);
+ }
+ spin_unlock(&cfids->cfid_list_lock);
+
+ list_for_each_entry_safe(cfid, q, &entry, entry) {
+ cfid->on_list = false;
+ list_del(&cfid->entry);
+ cancel_work_sync(&cfid->lease_break);
+ if (cfid->has_lease) {
+ /*
+ * We lease was never cancelled from the server so we
+ * need to drop the reference.
+ */
+ spin_lock(&cfids->cfid_list_lock);
+ cfid->has_lease = false;
+ spin_unlock(&cfids->cfid_list_lock);
+ kref_put(&cfid->refcount, smb2_close_cached_fid);
+ }
+ /* Drop the extra reference opened above*/
+ kref_put(&cfid->refcount, smb2_close_cached_fid);
+ }
}
static void
struct cached_fid *cfid = container_of(work,
struct cached_fid, lease_break);
- close_cached_dir_lease(cfid);
+ spin_lock(&cfid->cfids->cfid_list_lock);
+ cfid->has_lease = false;
+ spin_unlock(&cfid->cfids->cfid_list_lock);
+ kref_put(&cfid->refcount, smb2_close_cached_fid);
}
int cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16])
{
- if (tcon->cfid->is_valid &&
- !memcmp(lease_key,
- tcon->cfid->fid.lease_key,
- SMB2_LEASE_KEY_SIZE)) {
- tcon->cfid->time = 0;
- INIT_WORK(&tcon->cfid->lease_break,
- smb2_cached_lease_break);
- queue_work(cifsiod_wq,
- &tcon->cfid->lease_break);
- return true;
+ struct cached_fids *cfids = tcon->cfids;
+ struct cached_fid *cfid;
+
+ if (cfids == NULL)
+ return false;
+
+ spin_lock(&cfids->cfid_list_lock);
+ list_for_each_entry(cfid, &cfids->entries, entry) {
+ if (cfid->has_lease &&
+ !memcmp(lease_key,
+ cfid->fid.lease_key,
+ SMB2_LEASE_KEY_SIZE)) {
+ cfid->time = 0;
+ /*
+ * We found a lease remove it from the list
+ * so no threads can access it.
+ */
+ list_del(&cfid->entry);
+ cfid->on_list = false;
+ cfids->num_entries--;
+
+ queue_work(cifsiod_wq,
+ &cfid->lease_break);
+ spin_unlock(&cfids->cfid_list_lock);
+ return true;
+ }
}
+ spin_unlock(&cfids->cfid_list_lock);
return false;
}
-struct cached_fid *init_cached_dir(void)
+static struct cached_fid *init_cached_dir(const char *path)
{
struct cached_fid *cfid;
- cfid = kzalloc(sizeof(*cfid), GFP_KERNEL);
+ cfid = kzalloc(sizeof(*cfid), GFP_ATOMIC);
if (!cfid)
return NULL;
+ cfid->path = kstrdup(path, GFP_ATOMIC);
+ if (!cfid->path) {
+ kfree(cfid);
+ return NULL;
+ }
+
+ INIT_WORK(&cfid->lease_break, smb2_cached_lease_break);
+ INIT_LIST_HEAD(&cfid->entry);
INIT_LIST_HEAD(&cfid->dirents.entries);
mutex_init(&cfid->dirents.de_mutex);
- mutex_init(&cfid->fid_mutex);
+ spin_lock_init(&cfid->fid_lock);
+ kref_init(&cfid->refcount);
return cfid;
}
-void free_cached_dir(struct cifs_tcon *tcon)
+static void free_cached_dir(struct cached_fid *cfid)
+{
+ struct cached_dirent *dirent, *q;
+
+ dput(cfid->dentry);
+ cfid->dentry = NULL;
+
+ /*
+ * Delete all cached dirent names
+ */
+ list_for_each_entry_safe(dirent, q, &cfid->dirents.entries, entry) {
+ list_del(&dirent->entry);
+ kfree(dirent->name);
+ kfree(dirent);
+ }
+
+ kfree(cfid->path);
+ cfid->path = NULL;
+ kfree(cfid);
+}
+
+struct cached_fids *init_cached_dirs(void)
+{
+ struct cached_fids *cfids;
+
+ cfids = kzalloc(sizeof(*cfids), GFP_KERNEL);
+ if (!cfids)
+ return NULL;
+ spin_lock_init(&cfids->cfid_list_lock);
+ INIT_LIST_HEAD(&cfids->entries);
+ return cfids;
+}
+
+/*
+ * Called from tconInfoFree when we are tearing down the tcon.
+ * There are no active users or open files/directories at this point.
+ */
+void free_cached_dirs(struct cached_fids *cfids)
{
- kfree(tcon->cfid);
+ struct cached_fid *cfid, *q;
+ struct list_head entry;
+
+ INIT_LIST_HEAD(&entry);
+ spin_lock(&cfids->cfid_list_lock);
+ list_for_each_entry_safe(cfid, q, &cfids->entries, entry) {
+ cfid->on_list = false;
+ cfid->is_open = false;
+ list_del(&cfid->entry);
+ list_add(&cfid->entry, &entry);
+ }
+ spin_unlock(&cfids->cfid_list_lock);
+
+ list_for_each_entry_safe(cfid, q, &entry, entry) {
+ list_del(&cfid->entry);
+ free_cached_dir(cfid);
+ }
+
+ kfree(cfids);
}
};
struct cached_fid {
- bool is_valid:1; /* Do we have a useable root fid */
- bool file_all_info_is_valid:1;
+ struct list_head entry;
+ struct cached_fids *cfids;
+ const char *path;
bool has_lease:1;
+ bool is_open:1;
+ bool on_list:1;
+ bool file_all_info_is_valid:1;
unsigned long time; /* jiffies of when lease was taken */
struct kref refcount;
struct cifs_fid fid;
- struct mutex fid_mutex;
+ spinlock_t fid_lock;
struct cifs_tcon *tcon;
struct dentry *dentry;
struct work_struct lease_break;
struct cached_dirents dirents;
};
-extern struct cached_fid *init_cached_dir(void);
-extern void free_cached_dir(struct cifs_tcon *tcon);
+#define MAX_CACHED_FIDS 16
+struct cached_fids {
+ /* Must be held when:
+ * - accessing the cfids->entries list
+ */
+ spinlock_t cfid_list_lock;
+ int num_entries;
+ struct list_head entries;
+};
+
+extern struct cached_fids *init_cached_dirs(void);
+extern void free_cached_dirs(struct cached_fids *cfids);
extern int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon,
const char *path,
struct cifs_sb_info *cifs_sb,
struct dentry *dentry,
struct cached_fid **cfid);
extern void close_cached_dir(struct cached_fid *cfid);
-extern void close_cached_dir_lease(struct cached_fid *cfid);
-extern void close_cached_dir_lease_locked(struct cached_fid *cfid);
extern void close_all_cached_dirs(struct cifs_sb_info *cifs_sb);
extern void invalidate_all_cached_dirs(struct cifs_tcon *tcon);
extern int cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16]);
{
__u32 dev_type = le32_to_cpu(tcon->fsDevInfo.DeviceType);
- seq_printf(m, "%s Mounts: %d ", tcon->treeName, tcon->tc_count);
+ seq_printf(m, "%s Mounts: %d ", tcon->tree_name, tcon->tc_count);
if (tcon->nativeFileSystem)
seq_printf(m, "Type: %s ", tcon->nativeFileSystem);
seq_printf(m, "DevInfo: 0x%x Attributes: 0x%x\n\tPathComponentMax: %d Status: %d",
list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) {
list_for_each_entry(tcon, &ses->tcon_list, tcon_list) {
i++;
- seq_printf(m, "\n%d) %s", i, tcon->treeName);
+ seq_printf(m, "\n%d) %s", i, tcon->tree_name);
if (tcon->need_reconnect)
seq_puts(m, "\tDISCONNECTED ");
seq_printf(m, "\nSMBs: %d",
#define cifs_tcon_dbg_func(ratefunc, type, fmt, ...) \
do { \
const char *tn = ""; \
- if (tcon && tcon->treeName) \
- tn = tcon->treeName; \
+ if (tcon && tcon->tree_name) \
+ tn = tcon->tree_name; \
if ((type) & FYI && cifsFYI & CIFS_INFO) { \
pr_debug_ ## ratefunc("%s: %s " fmt, \
__FILE__, tn, ##__VA_ARGS__); \
#define cifs_tcon_dbg(type, fmt, ...) \
do { \
if (0) \
- pr_debug("%s " fmt, tcon->treeName, ##__VA_ARGS__); \
+ pr_debug("%s " fmt, tcon->tree_name, ##__VA_ARGS__); \
} while (0)
#define cifs_info(fmt, ...) \
bool watch_tree;
} __packed;
+struct smb3_notify_info {
+ __u32 completion_filter;
+ bool watch_tree;
+ __u32 data_len; /* size of notify data below */
+ __u8 notify_data[];
+} __packed;
+
#define CIFS_IOCTL_MAGIC 0xCF
#define CIFS_IOC_COPYCHUNK_FILE _IOW(CIFS_IOCTL_MAGIC, 3, int)
#define CIFS_IOC_SET_INTEGRITY _IO(CIFS_IOCTL_MAGIC, 4)
#define CIFS_DUMP_KEY _IOWR(CIFS_IOCTL_MAGIC, 8, struct smb3_key_debug_info)
#define CIFS_IOC_NOTIFY _IOW(CIFS_IOCTL_MAGIC, 9, struct smb3_notify)
#define CIFS_DUMP_FULL_KEY _IOWR(CIFS_IOCTL_MAGIC, 10, struct smb3_full_key_debug_info)
+#define CIFS_IOC_NOTIFY_INFO _IOWR(CIFS_IOCTL_MAGIC, 11, struct smb3_notify_info)
#define CIFS_IOC_SHUTDOWN _IOR ('X', 125, __u32)
/*
const char *share_name;
const char *net_name;
- net_name = extract_hostname(tcon->treeName);
+ net_name = extract_hostname(tcon->tree_name);
if (IS_ERR(net_name)) {
int ret;
ret = PTR_ERR(net_name);
cifs_dbg(VFS, "%s: failed to extract host name from target '%s': %d\n",
- __func__, tcon->treeName, ret);
+ __func__, tcon->tree_name, ret);
return ERR_PTR(-EINVAL);
}
- share_name = extract_sharename(tcon->treeName);
+ share_name = extract_sharename(tcon->tree_name);
if (IS_ERR(share_name)) {
int ret;
ret = PTR_ERR(share_name);
cifs_dbg(VFS, "%s: failed to extract share name from target '%s': %d\n",
- __func__, tcon->treeName, ret);
+ __func__, tcon->tree_name, ret);
kfree(net_name);
return ERR_PTR(-EINVAL);
}
goto fail;
}
- reg->net_name = extract_hostname(tcon->treeName);
+ reg->net_name = extract_hostname(tcon->tree_name);
if (IS_ERR(reg->net_name)) {
ret = PTR_ERR(reg->net_name);
cifs_dbg(VFS, "%s: failed to extract host name from target: %d\n", __func__, ret);
goto fail_idr;
}
- reg->share_name = extract_sharename(tcon->treeName);
+ reg->share_name = extract_sharename(tcon->tree_name);
if (IS_ERR(reg->share_name)) {
ret = PTR_ERR(reg->share_name);
cifs_dbg(VFS, "%s: failed to extract share name from target: %d\n", __func__, ret);
if (!rqst->rq_iov || !signature || !server)
return -EINVAL;
- rc = cifs_alloc_hash("md5", &server->secmech.md5,
- &server->secmech.sdescmd5);
+ rc = cifs_alloc_hash("md5", &server->secmech.md5);
if (rc)
return -1;
- rc = crypto_shash_init(&server->secmech.sdescmd5->shash);
+ rc = crypto_shash_init(server->secmech.md5);
if (rc) {
cifs_dbg(VFS, "%s: Could not init md5\n", __func__);
return rc;
}
- rc = crypto_shash_update(&server->secmech.sdescmd5->shash,
+ rc = crypto_shash_update(server->secmech.md5,
server->session_key.response, server->session_key.len);
if (rc) {
cifs_dbg(VFS, "%s: Could not update with response\n", __func__);
return rc;
}
- return __cifs_calc_signature(rqst, server, signature,
- &server->secmech.sdescmd5->shash);
+ return __cifs_calc_signature(rqst, server, signature, server->secmech.md5);
}
/* must be called with server->srv_mutex held */
wchar_t *domain;
wchar_t *server;
- if (!ses->server->secmech.sdeschmacmd5) {
+ if (!ses->server->secmech.hmacmd5) {
cifs_dbg(VFS, "%s: can't generate ntlmv2 hash\n", __func__);
return -1;
}
/* calculate md4 hash of password */
E_md4hash(ses->password, nt_hash, nls_cp);
- rc = crypto_shash_setkey(ses->server->secmech.hmacmd5, nt_hash,
+ rc = crypto_shash_setkey(ses->server->secmech.hmacmd5->tfm, nt_hash,
CIFS_NTHASH_SIZE);
if (rc) {
cifs_dbg(VFS, "%s: Could not set NT Hash as a key\n", __func__);
return rc;
}
- rc = crypto_shash_init(&ses->server->secmech.sdeschmacmd5->shash);
+ rc = crypto_shash_init(ses->server->secmech.hmacmd5);
if (rc) {
cifs_dbg(VFS, "%s: Could not init hmacmd5\n", __func__);
return rc;
memset(user, '\0', 2);
}
- rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash,
+ rc = crypto_shash_update(ses->server->secmech.hmacmd5,
(char *)user, 2 * len);
kfree(user);
if (rc) {
len = cifs_strtoUTF16((__le16 *)domain, ses->domainName, len,
nls_cp);
rc =
- crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash,
+ crypto_shash_update(ses->server->secmech.hmacmd5,
(char *)domain, 2 * len);
kfree(domain);
if (rc) {
len = cifs_strtoUTF16((__le16 *)server, ses->ip_addr, len,
nls_cp);
rc =
- crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash,
+ crypto_shash_update(ses->server->secmech.hmacmd5,
(char *)server, 2 * len);
kfree(server);
if (rc) {
}
}
- rc = crypto_shash_final(&ses->server->secmech.sdeschmacmd5->shash,
+ rc = crypto_shash_final(ses->server->secmech.hmacmd5,
ntlmv2_hash);
if (rc)
cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__);
hash_len = ses->auth_key.len - (CIFS_SESS_KEY_SIZE +
offsetof(struct ntlmv2_resp, challenge.key[0]));
- if (!ses->server->secmech.sdeschmacmd5) {
+ if (!ses->server->secmech.hmacmd5) {
cifs_dbg(VFS, "%s: can't generate ntlmv2 hash\n", __func__);
return -1;
}
- rc = crypto_shash_setkey(ses->server->secmech.hmacmd5,
+ rc = crypto_shash_setkey(ses->server->secmech.hmacmd5->tfm,
ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE);
if (rc) {
cifs_dbg(VFS, "%s: Could not set NTLMV2 Hash as a key\n",
return rc;
}
- rc = crypto_shash_init(&ses->server->secmech.sdeschmacmd5->shash);
+ rc = crypto_shash_init(ses->server->secmech.hmacmd5);
if (rc) {
cifs_dbg(VFS, "%s: Could not init hmacmd5\n", __func__);
return rc;
else
memcpy(ntlmv2->challenge.key,
ses->server->cryptkey, CIFS_SERVER_CHALLENGE_SIZE);
- rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash,
+ rc = crypto_shash_update(ses->server->secmech.hmacmd5,
ntlmv2->challenge.key, hash_len);
if (rc) {
cifs_dbg(VFS, "%s: Could not update with response\n", __func__);
}
/* Note that the MD5 digest over writes anon.challenge_key.key */
- rc = crypto_shash_final(&ses->server->secmech.sdeschmacmd5->shash,
+ rc = crypto_shash_final(ses->server->secmech.hmacmd5,
ntlmv2->ntlmv2_hash);
if (rc)
cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__);
cifs_server_lock(ses->server);
- rc = cifs_alloc_hash("hmac(md5)",
- &ses->server->secmech.hmacmd5,
- &ses->server->secmech.sdeschmacmd5);
+ rc = cifs_alloc_hash("hmac(md5)", &ses->server->secmech.hmacmd5);
if (rc) {
goto unlock;
}
}
/* now calculate the session key for NTLMv2 */
- rc = crypto_shash_setkey(ses->server->secmech.hmacmd5,
+ rc = crypto_shash_setkey(ses->server->secmech.hmacmd5->tfm,
ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE);
if (rc) {
cifs_dbg(VFS, "%s: Could not set NTLMV2 Hash as a key\n",
goto unlock;
}
- rc = crypto_shash_init(&ses->server->secmech.sdeschmacmd5->shash);
+ rc = crypto_shash_init(ses->server->secmech.hmacmd5);
if (rc) {
cifs_dbg(VFS, "%s: Could not init hmacmd5\n", __func__);
goto unlock;
}
- rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash,
+ rc = crypto_shash_update(ses->server->secmech.hmacmd5,
ntlmv2->ntlmv2_hash,
CIFS_HMAC_MD5_HASH_SIZE);
if (rc) {
goto unlock;
}
- rc = crypto_shash_final(&ses->server->secmech.sdeschmacmd5->shash,
+ rc = crypto_shash_final(ses->server->secmech.hmacmd5,
ses->auth_key.response);
if (rc)
cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__);
unlock:
cifs_server_unlock(ses->server);
setup_ntlmv2_rsp_ret:
- kfree(tiblob);
+ kfree_sensitive(tiblob);
return rc;
}
void
cifs_crypto_secmech_release(struct TCP_Server_Info *server)
{
- if (server->secmech.cmacaes) {
- crypto_free_shash(server->secmech.cmacaes);
- server->secmech.cmacaes = NULL;
- }
-
- if (server->secmech.hmacsha256) {
- crypto_free_shash(server->secmech.hmacsha256);
- server->secmech.hmacsha256 = NULL;
- }
-
- if (server->secmech.md5) {
- crypto_free_shash(server->secmech.md5);
- server->secmech.md5 = NULL;
- }
+ cifs_free_hash(&server->secmech.aes_cmac);
+ cifs_free_hash(&server->secmech.hmacsha256);
+ cifs_free_hash(&server->secmech.md5);
+ cifs_free_hash(&server->secmech.sha512);
+ cifs_free_hash(&server->secmech.hmacmd5);
- if (server->secmech.sha512) {
- crypto_free_shash(server->secmech.sha512);
- server->secmech.sha512 = NULL;
+ if (server->secmech.enc) {
+ crypto_free_aead(server->secmech.enc);
+ server->secmech.enc = NULL;
}
- if (server->secmech.hmacmd5) {
- crypto_free_shash(server->secmech.hmacmd5);
- server->secmech.hmacmd5 = NULL;
+ if (server->secmech.dec) {
+ crypto_free_aead(server->secmech.dec);
+ server->secmech.dec = NULL;
}
-
- if (server->secmech.ccmaesencrypt) {
- crypto_free_aead(server->secmech.ccmaesencrypt);
- server->secmech.ccmaesencrypt = NULL;
- }
-
- if (server->secmech.ccmaesdecrypt) {
- crypto_free_aead(server->secmech.ccmaesdecrypt);
- server->secmech.ccmaesdecrypt = NULL;
- }
-
- kfree(server->secmech.sdesccmacaes);
- server->secmech.sdesccmacaes = NULL;
- kfree(server->secmech.sdeschmacsha256);
- server->secmech.sdeschmacsha256 = NULL;
- kfree(server->secmech.sdeschmacmd5);
- server->secmech.sdeschmacmd5 = NULL;
- kfree(server->secmech.sdescmd5);
- server->secmech.sdescmd5 = NULL;
- kfree(server->secmech.sdescsha512);
- server->secmech.sdescsha512 = NULL;
}
cifs_inode->epoch = 0;
spin_lock_init(&cifs_inode->open_file_lock);
generate_random_uuid(cifs_inode->lease_key);
+ cifs_inode->symlink_target = NULL;
/*
* Can not set i_flags here - they get immediately overwritten to zero
static void
cifs_free_inode(struct inode *inode)
{
- kmem_cache_free(cifs_inode_cachep, CIFS_I(inode));
+ struct cifsInodeInfo *cinode = CIFS_I(inode);
+
+ if (S_ISLNK(inode->i_mode))
+ kfree(cinode->symlink_target);
+ kmem_cache_free(cifs_inode_cachep, cinode);
}
static void
};
const struct inode_operations cifs_symlink_inode_ops = {
- .get_link = cifs_get_link,
+ .get_link = simple_get_link,
.permission = cifs_permission,
.listxattr = cifs_listxattr,
};
char *response;
};
-/* crypto security descriptor definition */
-struct sdesc {
- struct shash_desc shash;
- char ctx[];
-};
-
/* crypto hashing related structure/fields, not specific to a sec mech */
struct cifs_secmech {
- struct crypto_shash *hmacmd5; /* hmac-md5 hash function */
- struct crypto_shash *md5; /* md5 hash function */
- struct crypto_shash *hmacsha256; /* hmac-sha256 hash function */
- struct crypto_shash *cmacaes; /* block-cipher based MAC function */
- struct crypto_shash *sha512; /* sha512 hash function */
- struct sdesc *sdeschmacmd5; /* ctxt to generate ntlmv2 hash, CR1 */
- struct sdesc *sdescmd5; /* ctxt to generate cifs/smb signature */
- struct sdesc *sdeschmacsha256; /* ctxt to generate smb2 signature */
- struct sdesc *sdesccmacaes; /* ctxt to generate smb3 signature */
- struct sdesc *sdescsha512; /* ctxt to generate smb3.11 signing key */
- struct crypto_aead *ccmaesencrypt; /* smb3 encryption aead */
- struct crypto_aead *ccmaesdecrypt; /* smb3 decryption aead */
+ struct shash_desc *hmacmd5; /* hmacmd5 hash function, for NTLMv2/CR1 hashes */
+ struct shash_desc *md5; /* md5 hash function, for CIFS/SMB1 signatures */
+ struct shash_desc *hmacsha256; /* hmac-sha256 hash function, for SMB2 signatures */
+ struct shash_desc *sha512; /* sha512 hash function, for SMB3.1.1 preauth hash */
+ struct shash_desc *aes_cmac; /* block-cipher based MAC function, for SMB3 signatures */
+
+ struct crypto_aead *enc; /* smb3 encryption AEAD TFM (AES-CCM and AES-GCM) */
+ struct crypto_aead *dec; /* smb3 decryption AEAD TFM (AES-CCM and AES-GCM) */
};
/* per smb session structure/fields */
struct cifs_ace *aces;
};
+struct cifs_open_info_data {
+ char *symlink_target;
+ union {
+ struct smb2_file_all_info fi;
+ struct smb311_posix_qinfo posix_fi;
+ };
+};
+
+static inline void cifs_free_open_info(struct cifs_open_info_data *data)
+{
+ kfree(data->symlink_target);
+}
+
/*
*****************************************************************
* Except the CIFS PDUs themselves all the
int (*is_path_accessible)(const unsigned int, struct cifs_tcon *,
struct cifs_sb_info *, const char *);
/* query path data from the server */
- int (*query_path_info)(const unsigned int, struct cifs_tcon *,
- struct cifs_sb_info *, const char *,
- FILE_ALL_INFO *, bool *, bool *);
+ int (*query_path_info)(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse);
/* query file data from the server */
- int (*query_file_info)(const unsigned int, struct cifs_tcon *,
- struct cifs_fid *, FILE_ALL_INFO *);
+ int (*query_file_info)(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifsFileInfo *cfile, struct cifs_open_info_data *data);
/* query reparse tag from srv to determine which type of special file */
int (*query_reparse_tag)(const unsigned int xid, struct cifs_tcon *tcon,
struct cifs_sb_info *cifs_sb, const char *path,
__u32 *reparse_tag);
/* get server index number */
- int (*get_srv_inum)(const unsigned int, struct cifs_tcon *,
- struct cifs_sb_info *, const char *,
- u64 *uniqueid, FILE_ALL_INFO *);
+ int (*get_srv_inum)(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path, u64 *uniqueid,
+ struct cifs_open_info_data *data);
/* set size by path */
int (*set_path_size)(const unsigned int, struct cifs_tcon *,
const char *, __u64, struct cifs_sb_info *, bool);
struct cifs_sb_info *, const char *,
char **, bool);
/* open a file for non-posix mounts */
- int (*open)(const unsigned int, struct cifs_open_parms *,
- __u32 *, FILE_ALL_INFO *);
+ int (*open)(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock,
+ void *buf);
/* set fid protocol-specific info */
void (*set_fid)(struct cifsFileInfo *, struct cifs_fid *, __u32);
/* close a file */
int (*enum_snapshots)(const unsigned int xid, struct cifs_tcon *tcon,
struct cifsFileInfo *src_file, void __user *);
int (*notify)(const unsigned int xid, struct file *pfile,
- void __user *pbuf);
+ void __user *pbuf, bool return_changes);
int (*query_mf_symlink)(unsigned int, struct cifs_tcon *,
struct cifs_sb_info *, const unsigned char *,
char *, unsigned int *);
struct timespec64 cf_mtime;
struct timespec64 cf_ctime;
u32 cf_cifstag;
+ char *cf_symlink_target;
};
/*
struct list_head openFileList;
spinlock_t open_file_lock; /* protects list above */
struct cifs_ses *ses; /* pointer to session associated with */
- char treeName[MAX_TREE_SIZE + 1]; /* UNC name of resource in ASCII */
+ char tree_name[MAX_TREE_SIZE + 1]; /* UNC name of resource in ASCII */
char *nativeFileSystem;
char *password; /* for share-level security */
__u32 tid; /* The 4 byte tree id */
struct fscache_volume *fscache; /* cookie for share */
#endif
struct list_head pending_opens; /* list of incomplete opens */
- struct cached_fid *cfid; /* Cached root fid */
+ struct cached_fids *cfids;
/* BB add field for back pointer to sb struct(s)? */
#ifdef CONFIG_CIFS_DFS_UPCALL
struct list_head ulist; /* cache update list */
struct work_struct put; /* work for the final part of _put */
struct delayed_work deferred;
bool deferred_close_scheduled; /* Flag to indicate close is scheduled */
+ char *symlink_target;
};
struct cifs_io_parms {
struct list_head deferred_closes; /* list of deferred closes */
spinlock_t deferred_lock; /* protection on deferred list */
bool lease_granted; /* Flag to indicate whether lease or oplock is granted. */
+ char *symlink_target;
};
static inline struct cifsInodeInfo *
return sizeof(ses->workstation_name);
}
+static inline void move_cifs_info_to_smb2(struct smb2_file_all_info *dst, const FILE_ALL_INFO *src)
+{
+ memcpy(dst, src, (size_t)((u8 *)&src->AccessFlags - (u8 *)src));
+ dst->AccessFlags = src->AccessFlags;
+ dst->CurrentByteOffset = src->CurrentByteOffset;
+ dst->Mode = src->Mode;
+ dst->AlignmentRequirement = src->AlignmentRequirement;
+ dst->FileNameLength = src->FileNameLength;
+}
+
#endif /* _CIFS_GLOB_H */
typedef struct negotiate_req {
struct smb_hdr hdr; /* wct = 0 */
__le16 ByteCount;
- unsigned char DialectsArray[1];
+ unsigned char DialectsArray[];
} __attribute__((packed)) NEGOTIATE_REQ;
#define MIN_TZ_ADJ (15 * 60) /* minimum grid for timezones in seconds */
__u8 EncryptionKeyLength;
__u16 ByteCount;
union {
- unsigned char EncryptionKey[1]; /* cap extended security off */
+ /* cap extended security off */
+ DECLARE_FLEX_ARRAY(unsigned char, EncryptionKey);
/* followed by Domain name - if extended security is off */
/* followed by 16 bytes of server GUID */
/* then security blob if cap_extended_security negotiated */
struct {
unsigned char GUID[SMB1_CLIENT_GUID_SIZE];
- unsigned char SecurityBlob[1];
+ unsigned char SecurityBlob[];
} __attribute__((packed)) extended_response;
} __attribute__((packed)) u;
} __attribute__((packed)) NEGOTIATE_RSP;
extern int cifs_push_mandatory_locks(struct cifsFileInfo *cfile);
extern void cifs_down_write(struct rw_semaphore *sem);
-extern struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid,
- struct file *file,
- struct tcon_link *tlink,
- __u32 oplock);
+struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file,
+ struct tcon_link *tlink, __u32 oplock,
+ const char *symlink_target);
extern int cifs_posix_open(const char *full_path, struct inode **inode,
struct super_block *sb, int mode,
unsigned int f_flags, __u32 *oplock, __u16 *netfid,
extern struct inode *cifs_iget(struct super_block *sb,
struct cifs_fattr *fattr);
-extern int cifs_get_inode_info(struct inode **inode, const char *full_path,
- FILE_ALL_INFO *data, struct super_block *sb,
- int xid, const struct cifs_fid *fid);
+int cifs_get_inode_info(struct inode **inode, const char *full_path,
+ struct cifs_open_info_data *data, struct super_block *sb, int xid,
+ const struct cifs_fid *fid);
extern int smb311_posix_get_inode_info(struct inode **pinode, const char *search_path,
struct super_block *sb, unsigned int xid);
extern int cifs_get_inode_info_unix(struct inode **pinode,
void cifs_aio_ctx_release(struct kref *refcount);
int setup_aio_ctx_iter(struct cifs_aio_ctx *ctx, struct iov_iter *iter, int rw);
-int cifs_alloc_hash(const char *name, struct crypto_shash **shash,
- struct sdesc **sdesc);
-void cifs_free_hash(struct crypto_shash **shash, struct sdesc **sdesc);
+int cifs_alloc_hash(const char *name, struct shash_desc **sdesc);
+void cifs_free_hash(struct shash_desc **sdesc);
extern void rqst_page_get_length(struct smb_rqst *rqst, unsigned int page,
unsigned int *len, unsigned int *offset);
int
cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server);
int
-SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon);
+SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_mount);
void extract_unc_hostname(const char *unc, const char **h, size_t *len);
int copy_path_name(char *dst, const char *src);
for (i = 0; i < CIFS_NUM_PROT; i++) {
size_t len = strlen(protocols[i].name) + 1;
- memcpy(pSMB->DialectsArray+count, protocols[i].name, len);
+ memcpy(&pSMB->DialectsArray[count], protocols[i].name, len);
count += len;
}
inc_rfc1001_len(pSMB, count);
remap);
}
rename_info->target_name_len = cpu_to_le32(2 * len_of_str);
- count = 12 /* sizeof(struct set_file_rename) */ + (2 * len_of_str);
+ count = sizeof(struct set_file_rename) + (2 * len_of_str);
byte_count += count;
pSMB->DataCount = cpu_to_le16(count);
pSMB->TotalDataCount = pSMB->DataCount;
/*
* query server network interfaces, in case they change
*/
- rc = SMB3_request_interfaces(0, tcon);
+ rc = SMB3_request_interfaces(0, tcon, false);
if (rc) {
cifs_dbg(FYI, "%s: failed to query server interfaces: %d\n",
__func__, rc);
}
server->sequence_number = 0;
server->session_estab = false;
- kfree(server->session_key.response);
+ kfree_sensitive(server->session_key.response);
server->session_key.response = NULL;
server->session_key.len = 0;
server->lstrp = jiffies;
cifs_crypto_secmech_release(server);
- kfree(server->session_key.response);
+ kfree_sensitive(server->session_key.response);
server->session_key.response = NULL;
server->session_key.len = 0;
kfree(server->hostname);
spin_unlock(&ses->ses_lock);
cifs_dbg(FYI, "%s: ses_count=%d\n", __func__, ses->ses_count);
- cifs_dbg(FYI, "%s: ses ipc: %s\n", __func__, ses->tcon_ipc ? ses->tcon_ipc->treeName : "NONE");
+ cifs_dbg(FYI,
+ "%s: ses ipc: %s\n", __func__, ses->tcon_ipc ? ses->tcon_ipc->tree_name : "NONE");
spin_lock(&cifs_tcp_ses_lock);
if (--ses->ses_count > 0) {
{
if (tcon->status == TID_EXITING)
return 0;
- if (strncmp(tcon->treeName, ctx->UNC, MAX_TREE_SIZE))
+ if (strncmp(tcon->tree_name, ctx->UNC, MAX_TREE_SIZE))
return 0;
if (tcon->seal != ctx->seal)
return 0;
* sessinit is sent but no second negprot
*/
struct rfc1002_session_packet *ses_init_buf;
+ unsigned int req_noscope_len;
struct smb_hdr *smb_buf;
+
ses_init_buf = kzalloc(sizeof(struct rfc1002_session_packet),
GFP_KERNEL);
+
if (ses_init_buf) {
ses_init_buf->trailer.session_req.called_len = 32;
ses_init_buf->trailer.session_req.scope2 = 0;
smb_buf = (struct smb_hdr *)ses_init_buf;
- /* sizeof RFC1002_SESSION_REQUEST with no scope */
- smb_buf->smb_buf_length = cpu_to_be32(0x81000044);
+ /* sizeof RFC1002_SESSION_REQUEST with no scopes */
+ req_noscope_len = sizeof(struct rfc1002_session_packet) - 2;
+
+ /* == cpu_to_be32(0x81000044) */
+ smb_buf->smb_buf_length =
+ cpu_to_be32((RFC1002_SESSION_REQUEST << 24) | req_noscope_len);
rc = smb_send(server, smb_buf, 0x44);
kfree(ses_init_buf);
/*
pSMB->AndXCommand = 0xFF;
pSMB->Flags = cpu_to_le16(TCON_EXTENDED_SECINFO);
bcc_ptr = &pSMB->Password[0];
- if (tcon->pipe || (ses->server->sec_mode & SECMODE_USER)) {
- pSMB->PasswordLength = cpu_to_le16(1); /* minimum */
- *bcc_ptr = 0; /* password is null byte */
- bcc_ptr++; /* skip password */
- /* already aligned so no need to do it below */
- }
+
+ pSMB->PasswordLength = cpu_to_le16(1); /* minimum */
+ *bcc_ptr = 0; /* password is null byte */
+ bcc_ptr++; /* skip password */
+ /* already aligned so no need to do it below */
if (ses->server->sign)
smb_buffer->Flags2 |= SMBFLG2_SECURITY_SIGNATURE;
}
bcc_ptr += length + 1;
bytes_left -= (length + 1);
- strscpy(tcon->treeName, tree, sizeof(tcon->treeName));
+ strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name));
/* mostly informational -- no need to fail on error here */
kfree(tcon->nativeFileSystem);
if (ses->auth_key.response) {
cifs_dbg(FYI, "Free previous auth_key.response = %p\n",
ses->auth_key.response);
- kfree(ses->auth_key.response);
+ kfree_sensitive(ses->auth_key.response);
ses->auth_key.response = NULL;
ses->auth_key.len = 0;
}
ctx->local_nls = cifs_sb->local_nls;
ctx->linux_uid = fsuid;
ctx->cred_uid = fsuid;
- ctx->UNC = master_tcon->treeName;
+ ctx->UNC = master_tcon->tree_name;
ctx->retry = master_tcon->retry;
ctx->nocase = master_tcon->nocase;
ctx->nohandlecache = master_tcon->nohandlecache;
/* If it is not dfs or there was no cached dfs referral, then reconnect to same share */
if (!server->current_fullpath ||
dfs_cache_noreq_find(server->current_fullpath + 1, &ref, &tl)) {
- rc = ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, cifs_sb->local_nls);
+ rc = ops->tree_connect(xid, tcon->ses, tcon->tree_name, tcon, cifs_sb->local_nls);
goto out;
}
tcon->status = TID_IN_TCON;
spin_unlock(&tcon->tc_lock);
- rc = ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, nlsc);
+ rc = ops->tree_connect(xid, tcon->ses, tcon->tree_name, tcon, nlsc);
if (rc) {
spin_lock(&tcon->tc_lock);
if (tcon->status == TID_IN_TCON)
get_ipc_unc(path, unc, sizeof(unc));
for (; *ses; ses++) {
- if (!strcasecmp(unc, (*ses)->tcon_ipc->treeName))
+ if (!strcasecmp(unc, (*ses)->tcon_ipc->tree_name))
return *ses;
}
return ERR_PTR(-ENOENT);
}
if (add_treename)
- dfsplen = strnlen(tcon->treeName, MAX_TREE_SIZE + 1);
+ dfsplen = strnlen(tcon->tree_name, MAX_TREE_SIZE + 1);
else
dfsplen = 0;
return full_path;
if (dfsplen)
- memcpy(full_path, tcon->treeName, dfsplen);
+ memcpy(full_path, tcon->tree_name, dfsplen);
full_path[dfsplen] = CIFS_DIR_SEP(cifs_sb);
memcpy(full_path + dfsplen + 1, ctx->prepath, pplen);
convert_delimiter(full_path, CIFS_DIR_SEP(cifs_sb));
return ERR_PTR(-ENOMEM);
if (prefix)
- dfsplen = strnlen(tcon->treeName, MAX_TREE_SIZE + 1);
+ dfsplen = strnlen(tcon->tree_name, MAX_TREE_SIZE + 1);
else
dfsplen = 0;
}
if (dfsplen) {
s -= dfsplen;
- memcpy(s, tcon->treeName, dfsplen);
+ memcpy(s, tcon->tree_name, dfsplen);
if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) {
int i;
for (i = 0; i < dfsplen; i++) {
/* Inode operations in similar order to how they appear in Linux file fs.h */
-static int
-cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int xid,
- struct tcon_link *tlink, unsigned oflags, umode_t mode,
- __u32 *oplock, struct cifs_fid *fid)
+static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int xid,
+ struct tcon_link *tlink, unsigned int oflags, umode_t mode, __u32 *oplock,
+ struct cifs_fid *fid, struct cifs_open_info_data *buf)
{
int rc = -ENOENT;
int create_options = CREATE_NOT_DIR;
struct cifs_tcon *tcon = tlink_tcon(tlink);
const char *full_path;
void *page = alloc_dentry_path();
- FILE_ALL_INFO *buf = NULL;
struct inode *newinode = NULL;
int disposition;
struct TCP_Server_Info *server = tcon->ses->server;
goto out;
}
- buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL);
- if (buf == NULL) {
- rc = -ENOMEM;
- goto out;
- }
-
/*
* if we're not using unix extensions, see if we need to set
* ATTR_READONLY on the create call
{
#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */
/* TODO: Add support for calling POSIX query info here, but passing in fid */
- rc = cifs_get_inode_info(&newinode, full_path, buf, inode->i_sb,
- xid, fid);
+ rc = cifs_get_inode_info(&newinode, full_path, buf, inode->i_sb, xid, fid);
if (newinode) {
if (server->ops->set_lease_key)
server->ops->set_lease_key(newinode, fid);
d_add(direntry, newinode);
out:
- kfree(buf);
free_dentry_path(page);
return rc;
struct tcon_link *tlink;
struct cifs_tcon *tcon;
struct TCP_Server_Info *server;
- struct cifs_fid fid;
+ struct cifs_fid fid = {};
struct cifs_pending_open open;
__u32 oplock;
struct cifsFileInfo *file_info;
+ struct cifs_open_info_data buf = {};
if (unlikely(cifs_forced_shutdown(CIFS_SB(inode->i_sb))))
return -EIO;
cifs_add_pending_open(&fid, tlink, &open);
rc = cifs_do_create(inode, direntry, xid, tlink, oflags, mode,
- &oplock, &fid);
-
+ &oplock, &fid, &buf);
if (rc) {
cifs_del_pending_open(&open);
goto out;
file->f_op = &cifs_file_direct_ops;
}
- file_info = cifs_new_fileinfo(&fid, file, tlink, oplock);
+ file_info = cifs_new_fileinfo(&fid, file, tlink, oplock, buf.symlink_target);
if (file_info == NULL) {
if (server->ops->close)
server->ops->close(xid, tcon, &fid);
cifs_put_tlink(tlink);
out_free_xid:
free_xid(xid);
+ cifs_free_open_info(&buf);
return rc;
}
struct TCP_Server_Info *server;
struct cifs_fid fid;
__u32 oplock;
+ struct cifs_open_info_data buf = {};
cifs_dbg(FYI, "cifs_create parent inode = 0x%p name is: %pd and dentry = 0x%p\n",
inode, direntry, direntry);
if (server->ops->new_lease_key)
server->ops->new_lease_key(&fid);
- rc = cifs_do_create(inode, direntry, xid, tlink, oflags, mode,
- &oplock, &fid);
+ rc = cifs_do_create(inode, direntry, xid, tlink, oflags, mode, &oplock, &fid, &buf);
if (!rc && server->ops->close)
server->ops->close(xid, tcon, &fid);
+ cifs_free_open_info(&buf);
cifs_put_tlink(tlink);
out_free_xid:
free_xid(xid);
}
#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */
-static int
-cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *cifs_sb,
- struct cifs_tcon *tcon, unsigned int f_flags, __u32 *oplock,
- struct cifs_fid *fid, unsigned int xid)
+static int cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *cifs_sb,
+ struct cifs_tcon *tcon, unsigned int f_flags, __u32 *oplock,
+ struct cifs_fid *fid, unsigned int xid, struct cifs_open_info_data *buf)
{
int rc;
int desired_access;
int disposition;
int create_options = CREATE_NOT_DIR;
- FILE_ALL_INFO *buf;
struct TCP_Server_Info *server = tcon->ses->server;
struct cifs_open_parms oparms;
/* BB pass O_SYNC flag through on file attributes .. BB */
- buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL);
- if (!buf)
- return -ENOMEM;
-
/* O_SYNC also has bit for O_DSYNC so following check picks up either */
if (f_flags & O_SYNC)
create_options |= CREATE_WRITE_THROUGH;
oparms.reconnect = false;
rc = server->ops->open(xid, &oparms, oplock, buf);
-
if (rc)
- goto out;
+ return rc;
/* TODO: Add support for calling posix query info but with passing in fid */
if (tcon->unix_ext)
rc = -EOPENSTALE;
}
-out:
- kfree(buf);
return rc;
}
static void cifsFileInfo_put_work(struct work_struct *work);
-struct cifsFileInfo *
-cifs_new_fileinfo(struct cifs_fid *fid, struct file *file,
- struct tcon_link *tlink, __u32 oplock)
+struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file,
+ struct tcon_link *tlink, __u32 oplock,
+ const char *symlink_target)
{
struct dentry *dentry = file_dentry(file);
struct inode *inode = d_inode(dentry);
return NULL;
}
+ if (symlink_target) {
+ cfile->symlink_target = kstrdup(symlink_target, GFP_KERNEL);
+ if (!cfile->symlink_target) {
+ kfree(fdlocks);
+ kfree(cfile);
+ return NULL;
+ }
+ }
+
INIT_LIST_HEAD(&fdlocks->locks);
fdlocks->cfile = cfile;
cfile->llist = fdlocks;
cifs_put_tlink(cifs_file->tlink);
dput(cifs_file->dentry);
cifs_sb_deactive(sb);
+ kfree(cifs_file->symlink_target);
kfree(cifs_file);
}
struct cifsInodeInfo *cifsi = CIFS_I(inode);
struct super_block *sb = inode->i_sb;
struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
- struct cifs_fid fid;
+ struct cifs_fid fid = {};
struct cifs_pending_open open;
bool oplock_break_cancelled;
void *page;
const char *full_path;
bool posix_open_ok = false;
- struct cifs_fid fid;
+ struct cifs_fid fid = {};
struct cifs_pending_open open;
+ struct cifs_open_info_data data = {};
xid = get_xid();
if (server->ops->get_lease_key)
server->ops->get_lease_key(inode, &fid);
- rc = cifs_nt_open(full_path, inode, cifs_sb, tcon,
- file->f_flags, &oplock, &fid, xid);
+ rc = cifs_nt_open(full_path, inode, cifs_sb, tcon, file->f_flags, &oplock, &fid,
+ xid, &data);
if (rc) {
cifs_del_pending_open(&open);
goto out;
}
}
- cfile = cifs_new_fileinfo(&fid, file, tlink, oplock);
+ cfile = cifs_new_fileinfo(&fid, file, tlink, oplock, data.symlink_target);
if (cfile == NULL) {
if (server->ops->close)
server->ops->close(xid, tcon, &fid);
free_dentry_path(page);
free_xid(xid);
cifs_put_tlink(tlink);
+ cifs_free_open_info(&data);
return rc;
}
len = ctx->len;
}
+ if (direct) {
+ rc = filemap_write_and_wait_range(file->f_inode->i_mapping,
+ offset, offset + len - 1);
+ if (rc) {
+ kref_put(&ctx->refcount, cifs_aio_ctx_release);
+ return -EAGAIN;
+ }
+ }
+
/* grab a lock here due to read response handlers can access ctx */
mutex_lock(&ctx->aio_mutex);
cifs_sb->ctx->field = NULL; \
} while (0)
+#define STEAL_STRING_SENSITIVE(cifs_sb, ctx, field) \
+do { \
+ kfree_sensitive(ctx->field); \
+ ctx->field = cifs_sb->ctx->field; \
+ cifs_sb->ctx->field = NULL; \
+} while (0)
+
static int smb3_reconfigure(struct fs_context *fc)
{
struct smb3_fs_context *ctx = smb3_fc2context(fc);
STEAL_STRING(cifs_sb, ctx, UNC);
STEAL_STRING(cifs_sb, ctx, source);
STEAL_STRING(cifs_sb, ctx, username);
- STEAL_STRING(cifs_sb, ctx, password);
+ STEAL_STRING_SENSITIVE(cifs_sb, ctx, password);
STEAL_STRING(cifs_sb, ctx, domainname);
STEAL_STRING(cifs_sb, ctx, nodename);
STEAL_STRING(cifs_sb, ctx, iocharset);
}
break;
case Opt_pass:
- kfree(ctx->password);
+ kfree_sensitive(ctx->password);
ctx->password = NULL;
if (strlen(param->string) == 0)
break;
return 0;
cifs_parse_mount_err:
+ kfree_sensitive(ctx->password);
return -EINVAL;
}
memset(&key, 0, sizeof(key));
- sharename = extract_sharename(tcon->treeName);
+ sharename = extract_sharename(tcon->tree_name);
if (IS_ERR(sharename)) {
cifs_dbg(FYI, "%s: couldn't extract sharename\n", __func__);
return -EINVAL;
*/
inode->i_blocks = (512 - 1 + fattr->cf_bytes) >> 9;
}
+
+ if (S_ISLNK(fattr->cf_mode)) {
+ kfree(cifs_i->symlink_target);
+ cifs_i->symlink_target = fattr->cf_symlink_target;
+ fattr->cf_symlink_target = NULL;
+
+ if (unlikely(!cifs_i->symlink_target))
+ inode->i_link = ERR_PTR(-EOPNOTSUPP);
+ else
+ inode->i_link = cifs_i->symlink_target;
+ }
spin_unlock(&inode->i_lock);
if (fattr->cf_flags & CIFS_FATTR_DFS_REFERRAL)
int rc;
unsigned int xid;
FILE_UNIX_BASIC_INFO find_data;
- struct cifs_fattr fattr;
+ struct cifs_fattr fattr = {};
struct inode *inode = file_inode(filp);
struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb);
struct cifsFileInfo *cfile = filp->private_data;
struct cifs_tcon *tcon = tlink_tcon(cfile->tlink);
xid = get_xid();
+
+ if (cfile->symlink_target) {
+ fattr.cf_symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL);
+ if (!fattr.cf_symlink_target)
+ return -ENOMEM;
+ }
+
rc = CIFSSMBUnixQFileInfo(xid, tcon, cfile->fid.netfid, &find_data);
if (!rc) {
cifs_unix_basic_to_fattr(&fattr, &find_data, cifs_sb);
FILE_UNIX_BASIC_INFO find_data;
struct cifs_fattr fattr;
struct cifs_tcon *tcon;
+ struct TCP_Server_Info *server;
struct tcon_link *tlink;
struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
if (IS_ERR(tlink))
return PTR_ERR(tlink);
tcon = tlink_tcon(tlink);
+ server = tcon->ses->server;
/* could have done a find first instead but this returns more info */
rc = CIFSSMBUnixQPathInfo(xid, tcon, full_path, &find_data,
cifs_sb->local_nls, cifs_remap(cifs_sb));
+ cifs_dbg(FYI, "%s: query path info: rc = %d\n", __func__, rc);
cifs_put_tlink(tlink);
if (!rc) {
cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc);
}
+ if (S_ISLNK(fattr.cf_mode) && !fattr.cf_symlink_target) {
+ if (!server->ops->query_symlink)
+ return -EOPNOTSUPP;
+ rc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path,
+ &fattr.cf_symlink_target, false);
+ if (rc) {
+ cifs_dbg(FYI, "%s: query_symlink: %d\n", __func__, rc);
+ goto cgiiu_exit;
+ }
+ }
+
if (*pinode == NULL) {
/* get new inode */
cifs_fill_uniqueid(sb, &fattr);
}
cgiiu_exit:
+ kfree(fattr.cf_symlink_target);
return rc;
}
#else
}
/* Fill a cifs_fattr struct with info from POSIX info struct */
-static void
-smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct smb311_posix_qinfo *info,
- struct super_block *sb, bool adjust_tz, bool symlink)
+static void smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct cifs_open_info_data *data,
+ struct super_block *sb, bool adjust_tz, bool symlink)
{
+ struct smb311_posix_qinfo *info = &data->posix_fi;
struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb);
if (symlink) {
fattr->cf_mode |= S_IFLNK;
fattr->cf_dtype = DT_LNK;
+ fattr->cf_symlink_target = data->symlink_target;
+ data->symlink_target = NULL;
} else if (fattr->cf_cifsattrs & ATTR_DIRECTORY) {
fattr->cf_mode |= S_IFDIR;
fattr->cf_dtype = DT_DIR;
fattr->cf_mode, fattr->cf_uniqueid, fattr->cf_nlink);
}
-
-/* Fill a cifs_fattr struct with info from FILE_ALL_INFO */
-static void
-cifs_all_info_to_fattr(struct cifs_fattr *fattr, FILE_ALL_INFO *info,
- struct super_block *sb, bool adjust_tz,
- bool symlink, u32 reparse_tag)
+static void cifs_open_info_to_fattr(struct cifs_fattr *fattr, struct cifs_open_info_data *data,
+ struct super_block *sb, bool adjust_tz, bool symlink,
+ u32 reparse_tag)
{
+ struct smb2_file_all_info *info = &data->fi;
struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb);
} else if (reparse_tag == IO_REPARSE_TAG_LX_BLK) {
fattr->cf_mode |= S_IFBLK | cifs_sb->ctx->file_mode;
fattr->cf_dtype = DT_BLK;
- } else if (symlink) { /* TODO add more reparse tag checks */
+ } else if (symlink || reparse_tag == IO_REPARSE_TAG_SYMLINK ||
+ reparse_tag == IO_REPARSE_TAG_NFS) {
fattr->cf_mode = S_IFLNK;
fattr->cf_dtype = DT_LNK;
} else if (fattr->cf_cifsattrs & ATTR_DIRECTORY) {
}
}
+ if (S_ISLNK(fattr->cf_mode)) {
+ fattr->cf_symlink_target = data->symlink_target;
+ data->symlink_target = NULL;
+ }
+
fattr->cf_uid = cifs_sb->ctx->linux_uid;
fattr->cf_gid = cifs_sb->ctx->linux_gid;
}
{
int rc;
unsigned int xid;
- FILE_ALL_INFO find_data;
+ struct cifs_open_info_data data = {};
struct cifs_fattr fattr;
struct inode *inode = file_inode(filp);
struct cifsFileInfo *cfile = filp->private_data;
struct cifs_tcon *tcon = tlink_tcon(cfile->tlink);
struct TCP_Server_Info *server = tcon->ses->server;
+ bool symlink = false;
+ u32 tag = 0;
if (!server->ops->query_file_info)
return -ENOSYS;
xid = get_xid();
- rc = server->ops->query_file_info(xid, tcon, &cfile->fid, &find_data);
+ rc = server->ops->query_file_info(xid, tcon, cfile, &data);
switch (rc) {
case 0:
/* TODO: add support to query reparse tag */
- cifs_all_info_to_fattr(&fattr, &find_data, inode->i_sb, false,
- false, 0 /* no reparse tag */);
+ if (data.symlink_target) {
+ symlink = true;
+ tag = IO_REPARSE_TAG_SYMLINK;
+ }
+ cifs_open_info_to_fattr(&fattr, &data, inode->i_sb, false, symlink, tag);
break;
case -EREMOTE:
cifs_create_dfs_fattr(&fattr, inode->i_sb);
/* if filetype is different, return error */
rc = cifs_fattr_to_inode(inode, &fattr);
cgfi_exit:
+ cifs_free_open_info(&data);
free_xid(xid);
return rc;
}
}
#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */
-static void
-cifs_set_fattr_ino(int xid,
- struct cifs_tcon *tcon,
- struct super_block *sb,
- struct inode **inode,
- const char *full_path,
- FILE_ALL_INFO *data,
- struct cifs_fattr *fattr)
+static void cifs_set_fattr_ino(int xid, struct cifs_tcon *tcon, struct super_block *sb,
+ struct inode **inode, const char *full_path,
+ struct cifs_open_info_data *data, struct cifs_fattr *fattr)
{
struct cifs_sb_info *cifs_sb = CIFS_SB(sb);
struct TCP_Server_Info *server = tcon->ses->server;
* If we have an inode pass a NULL tcon to ensure we don't
* make a round trip to the server. This only works for SMB2+.
*/
- rc = server->ops->get_srv_inum(xid,
- *inode ? NULL : tcon,
- cifs_sb, full_path,
- &fattr->cf_uniqueid,
- data);
+ rc = server->ops->get_srv_inum(xid, *inode ? NULL : tcon, cifs_sb, full_path,
+ &fattr->cf_uniqueid, data);
if (rc) {
/*
* If that fails reuse existing ino or generate one
} else {
/* make an ino by hashing the UNC */
fattr->cf_flags |= CIFS_FATTR_FAKE_ROOT_INO;
- fattr->cf_uniqueid = simple_hashstr(tcon->treeName);
+ fattr->cf_uniqueid = simple_hashstr(tcon->tree_name);
}
}
}
return ino && CIFS_CACHE_READ(CIFS_I(ino)) && CIFS_I(ino)->time != 0;
}
-int
-cifs_get_inode_info(struct inode **inode,
- const char *full_path,
- FILE_ALL_INFO *in_data,
- struct super_block *sb, int xid,
- const struct cifs_fid *fid)
+int cifs_get_inode_info(struct inode **inode, const char *full_path,
+ struct cifs_open_info_data *data, struct super_block *sb, int xid,
+ const struct cifs_fid *fid)
{
-
struct cifs_tcon *tcon;
struct TCP_Server_Info *server;
struct tcon_link *tlink;
bool adjust_tz = false;
struct cifs_fattr fattr = {0};
bool is_reparse_point = false;
- FILE_ALL_INFO *data = in_data;
- FILE_ALL_INFO *tmp_data = NULL;
+ struct cifs_open_info_data tmp_data = {};
void *smb1_backup_rsp_buf = NULL;
int rc = 0;
int tmprc = 0;
cifs_dbg(FYI, "No need to revalidate cached inode sizes\n");
goto out;
}
- tmp_data = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL);
- if (!tmp_data) {
- rc = -ENOMEM;
- goto out;
- }
- rc = server->ops->query_path_info(xid, tcon, cifs_sb,
- full_path, tmp_data,
- &adjust_tz, &is_reparse_point);
+ rc = server->ops->query_path_info(xid, tcon, cifs_sb, full_path, &tmp_data,
+ &adjust_tz, &is_reparse_point);
#ifdef CONFIG_CIFS_DFS_UPCALL
if (rc == -ENOENT && is_tcon_dfs(tcon))
rc = cifs_dfs_query_info_nonascii_quirk(xid, tcon,
cifs_sb,
full_path);
#endif
- data = tmp_data;
+ data = &tmp_data;
}
/*
* since we have to check if its reparse tag matches a known
* special file type e.g. symlink or fifo or char etc.
*/
- if ((le32_to_cpu(data->Attributes) & ATTR_REPARSE) &&
- server->ops->query_reparse_tag) {
- rc = server->ops->query_reparse_tag(xid, tcon, cifs_sb,
- full_path, &reparse_tag);
- cifs_dbg(FYI, "reparse tag 0x%x\n", reparse_tag);
+ if (is_reparse_point && data->symlink_target) {
+ reparse_tag = IO_REPARSE_TAG_SYMLINK;
+ } else if ((le32_to_cpu(data->fi.Attributes) & ATTR_REPARSE) &&
+ server->ops->query_reparse_tag) {
+ tmprc = server->ops->query_reparse_tag(xid, tcon, cifs_sb, full_path,
+ &reparse_tag);
+ if (tmprc)
+ cifs_dbg(FYI, "%s: query_reparse_tag: rc = %d\n", __func__, tmprc);
+ if (server->ops->query_symlink) {
+ tmprc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path,
+ &data->symlink_target,
+ is_reparse_point);
+ if (tmprc)
+ cifs_dbg(FYI, "%s: query_symlink: rc = %d\n", __func__,
+ tmprc);
+ }
}
- cifs_all_info_to_fattr(&fattr, data, sb, adjust_tz,
- is_reparse_point, reparse_tag);
+ cifs_open_info_to_fattr(&fattr, data, sb, adjust_tz, is_reparse_point, reparse_tag);
break;
case -EREMOTE:
/* DFS link, no metadata available on this server */
*/
if (backup_cred(cifs_sb) && is_smb1_server(server)) {
/* for easier reading */
+ FILE_ALL_INFO *fi;
FILE_DIRECTORY_INFO *fdi;
SEARCH_ID_FULL_DIR_INFO *si;
rc = cifs_backup_query_path_info(xid, tcon, sb,
full_path,
&smb1_backup_rsp_buf,
- &data);
+ &fi);
if (rc)
goto out;
- fdi = (FILE_DIRECTORY_INFO *)data;
- si = (SEARCH_ID_FULL_DIR_INFO *)data;
+ move_cifs_info_to_smb2(&data->fi, fi);
+ fdi = (FILE_DIRECTORY_INFO *)fi;
+ si = (SEARCH_ID_FULL_DIR_INFO *)fi;
cifs_dir_info_to_fattr(&fattr, fdi, cifs_sb);
fattr.cf_uniqueid = le64_to_cpu(si->UniqueId);
out:
cifs_buf_release(smb1_backup_rsp_buf);
cifs_put_tlink(tlink);
- kfree(tmp_data);
+ cifs_free_open_info(&tmp_data);
+ kfree(fattr.cf_symlink_target);
return rc;
}
bool adjust_tz = false;
struct cifs_fattr fattr = {0};
bool symlink = false;
- struct smb311_posix_qinfo *data = NULL;
+ struct cifs_open_info_data data = {};
int rc = 0;
int tmprc = 0;
cifs_dbg(FYI, "No need to revalidate cached inode sizes\n");
goto out;
}
- data = kmalloc(sizeof(struct smb311_posix_qinfo), GFP_KERNEL);
- if (!data) {
- rc = -ENOMEM;
- goto out;
- }
- rc = smb311_posix_query_path_info(xid, tcon, cifs_sb,
- full_path, data,
- &adjust_tz, &symlink);
+ rc = smb311_posix_query_path_info(xid, tcon, cifs_sb, full_path, &data, &adjust_tz,
+ &symlink);
/*
* 2. Convert it to internal cifs metadata (fattr)
switch (rc) {
case 0:
- smb311_posix_info_to_fattr(&fattr, data, sb, adjust_tz, symlink);
+ smb311_posix_info_to_fattr(&fattr, &data, sb, adjust_tz, symlink);
break;
case -EREMOTE:
/* DFS link, no metadata available on this server */
}
out:
cifs_put_tlink(tlink);
- kfree(data);
+ cifs_free_open_info(&data);
+ kfree(fattr.cf_symlink_target);
return rc;
}
return true;
if (!open_cached_dir_by_dentry(tcon, dentry->d_parent, &cfid)) {
- mutex_lock(&cfid->fid_mutex);
+ spin_lock(&cfid->fid_lock);
if (cfid->time && cifs_i->time > cfid->time) {
- mutex_unlock(&cfid->fid_mutex);
+ spin_unlock(&cfid->fid_lock);
close_cached_dir(cfid);
return false;
}
- mutex_unlock(&cfid->fid_mutex);
+ spin_unlock(&cfid->fid_lock);
close_cached_dir(cfid);
}
/*
tcon = tlink_tcon(tlink);
if (tcon && tcon->ses->server->ops->notify) {
rc = tcon->ses->server->ops->notify(xid,
- filep, (void __user *)arg);
+ filep, (void __user *)arg,
+ false /* no ret data */);
cifs_dbg(FYI, "ioctl notify rc %d\n", rc);
} else
rc = -EOPNOTSUPP;
cifs_put_tlink(tlink);
break;
+ case CIFS_IOC_NOTIFY_INFO:
+ if (!S_ISDIR(inode->i_mode)) {
+ /* Notify can only be done on directories */
+ rc = -EOPNOTSUPP;
+ break;
+ }
+ cifs_sb = CIFS_SB(inode->i_sb);
+ tlink = cifs_sb_tlink(cifs_sb);
+ if (IS_ERR(tlink)) {
+ rc = PTR_ERR(tlink);
+ break;
+ }
+ tcon = tlink_tcon(tlink);
+ if (tcon && tcon->ses->server->ops->notify) {
+ rc = tcon->ses->server->ops->notify(xid,
+ filep, (void __user *)arg,
+ true /* return details */);
+ cifs_dbg(FYI, "ioctl notify info rc %d\n", rc);
+ } else
+ rc = -EOPNOTSUPP;
+ cifs_put_tlink(tlink);
+ break;
case CIFS_IOC_SHUTDOWN:
rc = cifs_shutdown(inode->i_sb, arg);
break;
symlink_hash(unsigned int link_len, const char *link_str, u8 *md5_hash)
{
int rc;
- struct crypto_shash *md5 = NULL;
- struct sdesc *sdescmd5 = NULL;
+ struct shash_desc *md5 = NULL;
- rc = cifs_alloc_hash("md5", &md5, &sdescmd5);
+ rc = cifs_alloc_hash("md5", &md5);
if (rc)
goto symlink_hash_err;
- rc = crypto_shash_init(&sdescmd5->shash);
+ rc = crypto_shash_init(md5);
if (rc) {
cifs_dbg(VFS, "%s: Could not init md5 shash\n", __func__);
goto symlink_hash_err;
}
- rc = crypto_shash_update(&sdescmd5->shash, link_str, link_len);
+ rc = crypto_shash_update(md5, link_str, link_len);
if (rc) {
cifs_dbg(VFS, "%s: Could not update with link_str\n", __func__);
goto symlink_hash_err;
}
- rc = crypto_shash_final(&sdescmd5->shash, md5_hash);
+ rc = crypto_shash_final(md5, md5_hash);
if (rc)
cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__);
symlink_hash_err:
- cifs_free_hash(&md5, &sdescmd5);
+ cifs_free_hash(&md5);
return rc;
}
return rc;
}
-static int
-query_mf_symlink(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const unsigned char *path,
- char **symlinkinfo)
-{
- int rc;
- u8 *buf = NULL;
- unsigned int link_len = 0;
- unsigned int bytes_read = 0;
-
- buf = kmalloc(CIFS_MF_SYMLINK_FILE_SIZE, GFP_KERNEL);
- if (!buf)
- return -ENOMEM;
-
- if (tcon->ses->server->ops->query_mf_symlink)
- rc = tcon->ses->server->ops->query_mf_symlink(xid, tcon,
- cifs_sb, path, buf, &bytes_read);
- else
- rc = -ENOSYS;
-
- if (rc)
- goto out;
-
- if (bytes_read == 0) { /* not a symlink */
- rc = -EINVAL;
- goto out;
- }
-
- rc = parse_mf_symlink(buf, bytes_read, &link_len, symlinkinfo);
-out:
- kfree(buf);
- return rc;
-}
-
int
check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon,
struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr,
u8 *buf = NULL;
unsigned int link_len = 0;
unsigned int bytes_read = 0;
+ char *symlink = NULL;
if (!couldbe_mf_symlink(fattr))
/* it's not a symlink */
if (bytes_read == 0) /* not a symlink */
goto out;
- rc = parse_mf_symlink(buf, bytes_read, &link_len, NULL);
+ rc = parse_mf_symlink(buf, bytes_read, &link_len, &symlink);
if (rc == -EINVAL) {
/* it's not a symlink */
rc = 0;
fattr->cf_mode &= ~S_IFMT;
fattr->cf_mode |= S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO;
fattr->cf_dtype = DT_LNK;
+ fattr->cf_symlink_target = symlink;
out:
kfree(buf);
return rc;
return rc;
}
-const char *
-cifs_get_link(struct dentry *direntry, struct inode *inode,
- struct delayed_call *done)
-{
- int rc = -ENOMEM;
- unsigned int xid;
- const char *full_path;
- void *page;
- char *target_path = NULL;
- struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb);
- struct tcon_link *tlink = NULL;
- struct cifs_tcon *tcon;
- struct TCP_Server_Info *server;
-
- if (!direntry)
- return ERR_PTR(-ECHILD);
-
- xid = get_xid();
-
- tlink = cifs_sb_tlink(cifs_sb);
- if (IS_ERR(tlink)) {
- free_xid(xid);
- return ERR_CAST(tlink);
- }
- tcon = tlink_tcon(tlink);
- server = tcon->ses->server;
-
- page = alloc_dentry_path();
- full_path = build_path_from_dentry(direntry, page);
- if (IS_ERR(full_path)) {
- free_xid(xid);
- cifs_put_tlink(tlink);
- free_dentry_path(page);
- return ERR_CAST(full_path);
- }
-
- cifs_dbg(FYI, "Full path: %s inode = 0x%p\n", full_path, inode);
-
- rc = -EACCES;
- /*
- * First try Minshall+French Symlinks, if configured
- * and fallback to UNIX Extensions Symlinks.
- */
- if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS)
- rc = query_mf_symlink(xid, tcon, cifs_sb, full_path,
- &target_path);
-
- if (rc != 0 && server->ops->query_symlink) {
- struct cifsInodeInfo *cifsi = CIFS_I(inode);
- bool reparse_point = false;
-
- if (cifsi->cifsAttrs & ATTR_REPARSE)
- reparse_point = true;
-
- rc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path,
- &target_path, reparse_point);
- }
-
- free_dentry_path(page);
- free_xid(xid);
- cifs_put_tlink(tlink);
- if (rc != 0) {
- kfree(target_path);
- return ERR_PTR(rc);
- }
- set_delayed_call(done, kfree_link, target_path);
- return target_path;
-}
-
int
cifs_symlink(struct user_namespace *mnt_userns, struct inode *inode,
struct dentry *direntry, const char *symname)
ret_buf = kzalloc(sizeof(*ret_buf), GFP_KERNEL);
if (!ret_buf)
return NULL;
- ret_buf->cfid = init_cached_dir();
- if (!ret_buf->cfid) {
+ ret_buf->cfids = init_cached_dirs();
+ if (!ret_buf->cfids) {
kfree(ret_buf);
return NULL;
}
cifs_dbg(FYI, "Null buffer passed to tconInfoFree\n");
return;
}
- free_cached_dir(tcon);
+ free_cached_dirs(tcon->cfids);
atomic_dec(&tconInfoAllocCount);
kfree(tcon->nativeFileSystem);
kfree_sensitive(tcon->password);
cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SERVER_INUM;
cifs_sb->mnt_cifs_serverino_autodisabled = true;
cifs_dbg(VFS, "Autodisabling the use of server inode numbers on %s\n",
- tcon ? tcon->treeName : "new server");
+ tcon ? tcon->tree_name : "new server");
cifs_dbg(VFS, "The server doesn't seem to support them properly or the files might be on different servers (DFS)\n");
cifs_dbg(VFS, "Hardlinks will not be recognized on this mount. Consider mounting with the \"noserverino\" option to silence this message.\n");
free_dentry_path(page);
}
-/* parses DFS refferal V3 structure
+/* parses DFS referral V3 structure
* caller is responsible for freeing target_nodes
* returns:
* - on success - 0
/**
* cifs_alloc_hash - allocate hash and hash context together
* @name: The name of the crypto hash algo
- * @shash: Where to put the pointer to the hash algo
- * @sdesc: Where to put the pointer to the hash descriptor
+ * @sdesc: SHASH descriptor where to put the pointer to the hash TFM
*
* The caller has to make sure @sdesc is initialized to either NULL or
- * a valid context. Both can be freed via cifs_free_hash().
+ * a valid context. It can be freed via cifs_free_hash().
*/
int
-cifs_alloc_hash(const char *name,
- struct crypto_shash **shash, struct sdesc **sdesc)
+cifs_alloc_hash(const char *name, struct shash_desc **sdesc)
{
int rc = 0;
- size_t size;
+ struct crypto_shash *alg = NULL;
- if (*sdesc != NULL)
+ if (*sdesc)
return 0;
- *shash = crypto_alloc_shash(name, 0, 0);
- if (IS_ERR(*shash)) {
- cifs_dbg(VFS, "Could not allocate crypto %s\n", name);
- rc = PTR_ERR(*shash);
- *shash = NULL;
+ alg = crypto_alloc_shash(name, 0, 0);
+ if (IS_ERR(alg)) {
+ cifs_dbg(VFS, "Could not allocate shash TFM '%s'\n", name);
+ rc = PTR_ERR(alg);
*sdesc = NULL;
return rc;
}
- size = sizeof(struct shash_desc) + crypto_shash_descsize(*shash);
- *sdesc = kmalloc(size, GFP_KERNEL);
+ *sdesc = kmalloc(sizeof(struct shash_desc) + crypto_shash_descsize(alg), GFP_KERNEL);
if (*sdesc == NULL) {
- cifs_dbg(VFS, "no memory left to allocate crypto %s\n", name);
- crypto_free_shash(*shash);
- *shash = NULL;
+ cifs_dbg(VFS, "no memory left to allocate shash TFM '%s'\n", name);
+ crypto_free_shash(alg);
return -ENOMEM;
}
- (*sdesc)->shash.tfm = *shash;
+ (*sdesc)->tfm = alg;
return 0;
}
/**
* cifs_free_hash - free hash and hash context together
- * @shash: Where to find the pointer to the hash algo
- * @sdesc: Where to find the pointer to the hash descriptor
+ * @sdesc: Where to find the pointer to the hash TFM
*
- * Freeing a NULL hash or context is safe.
+ * Freeing a NULL descriptor is safe.
*/
void
-cifs_free_hash(struct crypto_shash **shash, struct sdesc **sdesc)
+cifs_free_hash(struct shash_desc **sdesc)
{
- kfree(*sdesc);
+ if (unlikely(!sdesc) || !*sdesc)
+ return;
+
+ if ((*sdesc)->tfm) {
+ crypto_free_shash((*sdesc)->tfm);
+ (*sdesc)->tfm = NULL;
+ }
+
+ kfree_sensitive(*sdesc);
*sdesc = NULL;
- if (*shash)
- crypto_free_shash(*shash);
- *shash = NULL;
}
/**
char *treename, *dfspath, sep;
int treenamelen, linkpathlen, rc;
- treename = tcon->treeName;
+ treename = tcon->tree_name;
/* MS-DFSC: All paths in REQ_GET_DFS_REFERRAL and RESP_GET_DFS_REFERRAL
* messages MUST be encoded with exactly one leading backslash, not two
* leading backslashes.
struct dir_context *ctx)
{
struct cached_dirent *dirent;
- int rc;
+ bool rc;
list_for_each_entry(dirent, &cde->entries, entry) {
- if (ctx->pos >= dirent->pos)
+ /*
+ * Skip all early entries prior to the current lseek()
+ * position.
+ */
+ if (ctx->pos > dirent->pos)
continue;
+ /*
+ * We recorded the current ->pos value for the dirent
+ * when we stored it in the cache.
+ * However, this sequence of ->pos values may have holes
+ * in it, for example dot-dirs returned from the server
+ * are suppressed.
+ * Handle this bu forcing ctx->pos to be the same as the
+ * ->pos of the current dirent we emit from the cache.
+ * This means that when we emit these entries from the cache
+ * we now emit them with the same ->pos value as in the
+ * initial scan.
+ */
ctx->pos = dirent->pos;
rc = dir_emit(ctx, dirent->name, dirent->namelen,
dirent->fattr.cf_uniqueid,
dirent->fattr.cf_dtype);
if (!rc)
return rc;
+ ctx->pos++;
}
return true;
}
cifs_unix_basic_to_fattr(&fattr,
&((FILE_UNIX_INFO *)find_entry)->basic,
cifs_sb);
+ if (S_ISLNK(fattr.cf_mode))
+ fattr.cf_flags |= CIFS_FATTR_NEED_REVAL;
break;
case SMB_FIND_FILE_INFO_STANDARD:
cifs_std_info_to_fattr(&fattr,
ctx->pos, tmp_buf);
cifs_save_resume_key(current_entry, cifsFile);
break;
- } else
- current_entry =
- nxt_dir_entry(current_entry, end_of_smb,
- cifsFile->srch_inf.info_level);
+ }
+ current_entry =
+ nxt_dir_entry(current_entry, end_of_smb,
+ cifsFile->srch_inf.info_level);
}
kfree(tmp_buf);
/* BB FIXME add check that strings total less
than 335 or will need to send them as arrays */
- /* unicode strings, must be word aligned before the call */
-/* if ((long) bcc_ptr % 2) {
- *bcc_ptr = 0;
- bcc_ptr++;
- } */
/* copy user */
if (ses->user_name == NULL) {
/* null user mount */
static void
sess_free_buffer(struct sess_data *sess_data)
{
+ struct kvec *iov = sess_data->iov;
+
+ /*
+ * Zero the session data before freeing, as it might contain sensitive info (keys, etc).
+ * Note that iov[1] is already freed by caller.
+ */
+ if (sess_data->buf0_type != CIFS_NO_BUFFER && iov[0].iov_base)
+ memzero_explicit(iov[0].iov_base, iov[0].iov_len);
- free_rsp_buf(sess_data->buf0_type, sess_data->iov[0].iov_base);
+ free_rsp_buf(sess_data->buf0_type, iov[0].iov_base);
sess_data->buf0_type = CIFS_NO_BUFFER;
- kfree(sess_data->iov[2].iov_base);
+ kfree_sensitive(iov[2].iov_base);
}
static int
}
if (ses->capabilities & CAP_UNICODE) {
- if (sess_data->iov[0].iov_len % 2) {
+ if (!IS_ALIGNED(sess_data->iov[0].iov_len, 2)) {
*bcc_ptr = 0;
bcc_ptr++;
}
/* no string area to decode, do nothing */
} else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
/* unicode string area must be word-aligned */
- if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+ if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) {
++bcc_ptr;
--bytes_remaining;
}
sess_data->result = rc;
sess_data->func = NULL;
sess_free_buffer(sess_data);
- kfree(ses->auth_key.response);
+ kfree_sensitive(ses->auth_key.response);
ses->auth_key.response = NULL;
}
if (ses->capabilities & CAP_UNICODE) {
/* unicode strings must be word aligned */
- if ((sess_data->iov[0].iov_len
- + sess_data->iov[1].iov_len) % 2) {
+ if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) {
*bcc_ptr = 0;
bcc_ptr++;
}
/* no string area to decode, do nothing */
} else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
/* unicode string area must be word-aligned */
- if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+ if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) {
++bcc_ptr;
--bytes_remaining;
}
sess_data->result = rc;
sess_data->func = NULL;
sess_free_buffer(sess_data);
- kfree(ses->auth_key.response);
+ kfree_sensitive(ses->auth_key.response);
ses->auth_key.response = NULL;
}
bcc_ptr = sess_data->iov[2].iov_base;
/* unicode strings must be word aligned */
- if ((sess_data->iov[0].iov_len + sess_data->iov[1].iov_len) % 2) {
+ if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) {
*bcc_ptr = 0;
bcc_ptr++;
}
rc = decode_ntlmssp_challenge(bcc_ptr, blob_len, ses);
out_free_ntlmsspblob:
- kfree(ntlmsspblob);
+ kfree_sensitive(ntlmsspblob);
out:
sess_free_buffer(sess_data);
}
/* Else error. Cleanup */
- kfree(ses->auth_key.response);
+ kfree_sensitive(ses->auth_key.response);
ses->auth_key.response = NULL;
- kfree(ses->ntlmssp);
+ kfree_sensitive(ses->ntlmssp);
ses->ntlmssp = NULL;
sess_data->func = NULL;
/* no string area to decode, do nothing */
} else if (smb_buf->Flags2 & SMBFLG2_UNICODE) {
/* unicode string area must be word-aligned */
- if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) {
+ if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) {
++bcc_ptr;
--bytes_remaining;
}
}
out_free_ntlmsspblob:
- kfree(ntlmsspblob);
+ kfree_sensitive(ntlmsspblob);
out:
sess_free_buffer(sess_data);
rc = sess_establish_session(sess_data);
/* Cleanup */
- kfree(ses->auth_key.response);
+ kfree_sensitive(ses->auth_key.response);
ses->auth_key.response = NULL;
- kfree(ses->ntlmssp);
+ kfree_sensitive(ses->ntlmssp);
ses->ntlmssp = NULL;
sess_data->func = NULL;
rc = sess_data->result;
out:
- kfree(sess_data);
+ kfree_sensitive(sess_data);
return rc;
}
#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */
return rc;
}
-static int
-cifs_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const char *full_path,
- FILE_ALL_INFO *data, bool *adjustTZ, bool *symlink)
+static int cifs_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ struct cifs_open_info_data *data, bool *adjustTZ, bool *symlink)
{
int rc;
+ FILE_ALL_INFO fi = {};
*symlink = false;
/* could do find first instead but this returns more info */
- rc = CIFSSMBQPathInfo(xid, tcon, full_path, data, 0 /* not legacy */,
- cifs_sb->local_nls, cifs_remap(cifs_sb));
+ rc = CIFSSMBQPathInfo(xid, tcon, full_path, &fi, 0 /* not legacy */, cifs_sb->local_nls,
+ cifs_remap(cifs_sb));
/*
* BB optimize code so we do not make the above call when server claims
* no NT SMB support and the above call failed at least once - set flag
* in tcon or mount.
*/
if ((rc == -EOPNOTSUPP) || (rc == -EINVAL)) {
- rc = SMBQueryInformation(xid, tcon, full_path, data,
- cifs_sb->local_nls,
+ rc = SMBQueryInformation(xid, tcon, full_path, &fi, cifs_sb->local_nls,
cifs_remap(cifs_sb));
+ if (!rc)
+ move_cifs_info_to_smb2(&data->fi, &fi);
*adjustTZ = true;
}
- if (!rc && (le32_to_cpu(data->Attributes) & ATTR_REPARSE)) {
+ if (!rc && (le32_to_cpu(fi.Attributes) & ATTR_REPARSE)) {
int tmprc;
int oplock = 0;
struct cifs_fid fid;
return rc;
}
-static int
-cifs_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const char *full_path,
- u64 *uniqueid, FILE_ALL_INFO *data)
+static int cifs_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ u64 *uniqueid, struct cifs_open_info_data *unused)
{
/*
* We can not use the IndexNumber field by default from Windows or
cifs_remap(cifs_sb));
}
-static int
-cifs_query_file_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_fid *fid, FILE_ALL_INFO *data)
+static int cifs_query_file_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifsFileInfo *cfile, struct cifs_open_info_data *data)
{
- return CIFSSMBQFileInfo(xid, tcon, fid->netfid, data);
+ int rc;
+ FILE_ALL_INFO fi = {};
+
+ if (cfile->symlink_target) {
+ data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL);
+ if (!data->symlink_target)
+ return -ENOMEM;
+ }
+
+ rc = CIFSSMBQFileInfo(xid, tcon, cfile->fid.netfid, &fi);
+ if (!rc)
+ move_cifs_info_to_smb2(&data->fi, &fi);
+ return rc;
}
static void
cifsInode->cifsAttrs = dosattrs;
}
-static int
-cifs_open_file(const unsigned int xid, struct cifs_open_parms *oparms,
- __u32 *oplock, FILE_ALL_INFO *buf)
+static int cifs_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock,
+ void *buf)
{
+ FILE_ALL_INFO *fi = buf;
+
if (!(oparms->tcon->ses->capabilities & CAP_NT_SMBS))
return SMBLegacyOpen(xid, oparms->tcon, oparms->path,
oparms->disposition,
oparms->desired_access,
oparms->create_options,
- &oparms->fid->netfid, oplock, buf,
+ &oparms->fid->netfid, oplock, fi,
oparms->cifs_sb->local_nls,
cifs_remap(oparms->cifs_sb));
- return CIFS_open(xid, oparms, oplock, buf);
+ return CIFS_open(xid, oparms, oplock, fi);
}
static void
#include "cifs_unicode.h"
#include "fscache.h"
#include "smb2proto.h"
+#include "smb2status.h"
-int
-smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms,
- __u32 *oplock, FILE_ALL_INFO *buf)
+static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov)
+{
+ struct smb2_err_rsp *err = iov->iov_base;
+ struct smb2_symlink_err_rsp *sym = ERR_PTR(-EINVAL);
+ u32 len;
+
+ if (err->ErrorContextCount) {
+ struct smb2_error_context_rsp *p, *end;
+
+ len = (u32)err->ErrorContextCount * (offsetof(struct smb2_error_context_rsp,
+ ErrorContextData) +
+ sizeof(struct smb2_symlink_err_rsp));
+ if (le32_to_cpu(err->ByteCount) < len || iov->iov_len < len + sizeof(*err))
+ return ERR_PTR(-EINVAL);
+
+ p = (struct smb2_error_context_rsp *)err->ErrorData;
+ end = (struct smb2_error_context_rsp *)((u8 *)err + iov->iov_len);
+ do {
+ if (le32_to_cpu(p->ErrorId) == SMB2_ERROR_ID_DEFAULT) {
+ sym = (struct smb2_symlink_err_rsp *)&p->ErrorContextData;
+ break;
+ }
+ cifs_dbg(FYI, "%s: skipping unhandled error context: 0x%x\n",
+ __func__, le32_to_cpu(p->ErrorId));
+
+ len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8);
+ p = (struct smb2_error_context_rsp *)((u8 *)&p->ErrorContextData + len);
+ } while (p < end);
+ } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) &&
+ iov->iov_len >= SMB2_SYMLINK_STRUCT_SIZE) {
+ sym = (struct smb2_symlink_err_rsp *)err->ErrorData;
+ }
+
+ if (!IS_ERR(sym) && (le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG ||
+ le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK))
+ sym = ERR_PTR(-EINVAL);
+
+ return sym;
+}
+
+int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec *iov, char **path)
+{
+ struct smb2_symlink_err_rsp *sym;
+ unsigned int sub_offs, sub_len;
+ unsigned int print_offs, print_len;
+ char *s;
+
+ if (!cifs_sb || !iov || !iov->iov_base || !iov->iov_len || !path)
+ return -EINVAL;
+
+ sym = symlink_data(iov);
+ if (IS_ERR(sym))
+ return PTR_ERR(sym);
+
+ sub_len = le16_to_cpu(sym->SubstituteNameLength);
+ sub_offs = le16_to_cpu(sym->SubstituteNameOffset);
+ print_len = le16_to_cpu(sym->PrintNameLength);
+ print_offs = le16_to_cpu(sym->PrintNameOffset);
+
+ if (iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offs + sub_len ||
+ iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + print_offs + print_len)
+ return -EINVAL;
+
+ s = cifs_strndup_from_utf16((char *)sym->PathBuffer + sub_offs, sub_len, true,
+ cifs_sb->local_nls);
+ if (!s)
+ return -ENOMEM;
+ convert_delimiter(s, '/');
+ cifs_dbg(FYI, "%s: symlink target: %s\n", __func__, s);
+
+ *path = s;
+ return 0;
+}
+
+int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, void *buf)
{
int rc;
__le16 *smb2_path;
- struct smb2_file_all_info *smb2_data = NULL;
__u8 smb2_oplock;
+ struct cifs_open_info_data *data = buf;
+ struct smb2_file_all_info file_info = {};
+ struct smb2_file_all_info *smb2_data = data ? &file_info : NULL;
+ struct kvec err_iov = {};
+ int err_buftype = CIFS_NO_BUFFER;
struct cifs_fid *fid = oparms->fid;
struct network_resiliency_req nr_ioctl_req;
smb2_path = cifs_convert_path_to_utf16(oparms->path, oparms->cifs_sb);
- if (smb2_path == NULL) {
- rc = -ENOMEM;
- goto out;
- }
-
- smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2,
- GFP_KERNEL);
- if (smb2_data == NULL) {
- rc = -ENOMEM;
- goto out;
- }
+ if (smb2_path == NULL)
+ return -ENOMEM;
oparms->desired_access |= FILE_READ_ATTRIBUTES;
smb2_oplock = SMB2_OPLOCK_LEVEL_BATCH;
- rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL,
- NULL, NULL);
+ rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov,
+ &err_buftype);
+ if (rc && data) {
+ struct smb2_hdr *hdr = err_iov.iov_base;
+
+ if (unlikely(!err_iov.iov_base || err_buftype == CIFS_NO_BUFFER))
+ rc = -ENOMEM;
+ else if (hdr->Status == STATUS_STOPPED_ON_SYMLINK) {
+ rc = smb2_parse_symlink_response(oparms->cifs_sb, &err_iov,
+ &data->symlink_target);
+ if (!rc) {
+ memset(smb2_data, 0, sizeof(*smb2_data));
+ oparms->create_options |= OPEN_REPARSE_POINT;
+ rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data,
+ NULL, NULL, NULL);
+ oparms->create_options &= ~OPEN_REPARSE_POINT;
+ }
+ }
+ }
+
if (rc)
goto out;
-
if (oparms->tcon->use_resilient) {
/* default timeout is 0, servers pick default (120 seconds) */
nr_ioctl_req.Timeout =
rc = 0;
}
- if (buf) {
+ if (smb2_data) {
/* if open response does not have IndexNumber field - get it */
if (smb2_data->IndexNumber == 0) {
rc = SMB2_get_srv_num(xid, oparms->tcon,
rc = 0;
}
}
- move_smb2_info_to_cifs(buf, smb2_data);
+ memcpy(&data->fi, smb2_data, sizeof(data->fi));
}
*oplock = smb2_oplock;
out:
- kfree(smb2_data);
+ free_rsp_buf(err_buftype, err_iov.iov_base);
kfree(smb2_path);
return rc;
}
#include "smb2pdu.h"
#include "smb2proto.h"
#include "cached_dir.h"
+#include "smb2status.h"
static void
free_set_inf_compound(struct smb_rqst *rqst)
/*
* note: If cfile is passed, the reference to it is dropped here.
* So make sure that you do not reuse cfile after return from this func.
+ *
+ * If passing @err_iov and @err_buftype, ensure to make them both large enough (>= 3) to hold all
+ * error responses. Caller is also responsible for freeing them up.
*/
-static int
-smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const char *full_path,
- __u32 desired_access, __u32 create_disposition,
- __u32 create_options, umode_t mode, void *ptr, int command,
- struct cifsFileInfo *cfile)
+static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ __u32 desired_access, __u32 create_disposition, __u32 create_options,
+ umode_t mode, void *ptr, int command, struct cifsFileInfo *cfile,
+ struct kvec *err_iov, int *err_buftype)
{
struct cop_vars *vars = NULL;
struct kvec *rsp_iov;
int num_rqst = 0;
int resp_buftype[3];
struct smb2_query_info_rsp *qi_rsp = NULL;
+ struct cifs_open_info_data *idata;
int flags = 0;
__u8 delete_pending[8] = {1, 0, 0, 0, 0, 0, 0, 0};
unsigned int size[2];
SMB2_open_free(&rqst[0]);
if (rc == -EREMCHG) {
- pr_warn_once("server share %s deleted\n", tcon->treeName);
+ pr_warn_once("server share %s deleted\n", tcon->tree_name);
tcon->need_reconnect = true;
}
switch (command) {
case SMB2_OP_QUERY_INFO:
+ idata = ptr;
+ if (rc == 0 && cfile && cfile->symlink_target) {
+ idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL);
+ if (!idata->symlink_target)
+ rc = -ENOMEM;
+ }
if (rc == 0) {
qi_rsp = (struct smb2_query_info_rsp *)
rsp_iov[1].iov_base;
rc = smb2_validate_and_copy_iov(
le16_to_cpu(qi_rsp->OutputBufferOffset),
le32_to_cpu(qi_rsp->OutputBufferLength),
- &rsp_iov[1], sizeof(struct smb2_file_all_info),
- ptr);
+ &rsp_iov[1], sizeof(idata->fi), (char *)&idata->fi);
}
if (rqst[1].rq_iov)
SMB2_query_info_free(&rqst[1]);
tcon->tid);
break;
case SMB2_OP_POSIX_QUERY_INFO:
+ idata = ptr;
+ if (rc == 0 && cfile && cfile->symlink_target) {
+ idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL);
+ if (!idata->symlink_target)
+ rc = -ENOMEM;
+ }
if (rc == 0) {
qi_rsp = (struct smb2_query_info_rsp *)
rsp_iov[1].iov_base;
rc = smb2_validate_and_copy_iov(
le16_to_cpu(qi_rsp->OutputBufferOffset),
le32_to_cpu(qi_rsp->OutputBufferLength),
- &rsp_iov[1], sizeof(struct smb311_posix_qinfo) /* add SIDs */, ptr);
+ &rsp_iov[1], sizeof(idata->posix_fi) /* add SIDs */,
+ (char *)&idata->posix_fi);
}
if (rqst[1].rq_iov)
SMB2_query_info_free(&rqst[1]);
free_set_inf_compound(rqst);
break;
}
- free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
- free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
- free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
+
+ if (rc && err_iov && err_buftype) {
+ memcpy(err_iov, rsp_iov, 3 * sizeof(*err_iov));
+ memcpy(err_buftype, resp_buftype, 3 * sizeof(*err_buftype));
+ } else {
+ free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
+ free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
+ free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
+ }
kfree(vars);
return rc;
}
-void
-move_smb2_info_to_cifs(FILE_ALL_INFO *dst, struct smb2_file_all_info *src)
-{
- memcpy(dst, src, (size_t)(&src->CurrentByteOffset) - (size_t)src);
- dst->CurrentByteOffset = src->CurrentByteOffset;
- dst->Mode = src->Mode;
- dst->AlignmentRequirement = src->AlignmentRequirement;
- dst->IndexNumber1 = 0; /* we don't use it */
-}
-
-int
-smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const char *full_path,
- FILE_ALL_INFO *data, bool *adjust_tz, bool *reparse)
+int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse)
{
int rc;
- struct smb2_file_all_info *smb2_data;
__u32 create_options = 0;
struct cifsFileInfo *cfile;
struct cached_fid *cfid = NULL;
+ struct kvec err_iov[3] = {};
+ int err_buftype[3] = {};
*adjust_tz = false;
*reparse = false;
- smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2,
- GFP_KERNEL);
- if (smb2_data == NULL)
- return -ENOMEM;
-
if (strcmp(full_path, ""))
rc = -ENOENT;
else
/* If it is a root and its handle is cached then use it */
if (!rc) {
if (cfid->file_all_info_is_valid) {
- move_smb2_info_to_cifs(data,
- &cfid->file_all_info);
+ memcpy(&data->fi, &cfid->file_all_info, sizeof(data->fi));
} else {
- rc = SMB2_query_info(xid, tcon,
- cfid->fid.persistent_fid,
- cfid->fid.volatile_fid, smb2_data);
- if (!rc)
- move_smb2_info_to_cifs(data, smb2_data);
+ rc = SMB2_query_info(xid, tcon, cfid->fid.persistent_fid,
+ cfid->fid.volatile_fid, &data->fi);
}
close_cached_dir(cfid);
- goto out;
+ return rc;
}
cifs_get_readable_path(tcon, full_path, &cfile);
- rc = smb2_compound_op(xid, tcon, cifs_sb, full_path,
- FILE_READ_ATTRIBUTES, FILE_OPEN, create_options,
- ACL_NO_MODE, smb2_data, SMB2_OP_QUERY_INFO, cfile);
+ rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES, FILE_OPEN,
+ create_options, ACL_NO_MODE, data, SMB2_OP_QUERY_INFO, cfile,
+ err_iov, err_buftype);
if (rc == -EOPNOTSUPP) {
+ if (err_iov[0].iov_base && err_buftype[0] != CIFS_NO_BUFFER &&
+ ((struct smb2_hdr *)err_iov[0].iov_base)->Command == SMB2_CREATE &&
+ ((struct smb2_hdr *)err_iov[0].iov_base)->Status == STATUS_STOPPED_ON_SYMLINK) {
+ rc = smb2_parse_symlink_response(cifs_sb, err_iov, &data->symlink_target);
+ if (rc)
+ goto out;
+ }
*reparse = true;
create_options |= OPEN_REPARSE_POINT;
/* Failed on a symbolic link - query a reparse point info */
cifs_get_readable_path(tcon, full_path, &cfile);
- rc = smb2_compound_op(xid, tcon, cifs_sb, full_path,
- FILE_READ_ATTRIBUTES, FILE_OPEN,
- create_options, ACL_NO_MODE,
- smb2_data, SMB2_OP_QUERY_INFO, cfile);
+ rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES,
+ FILE_OPEN, create_options, ACL_NO_MODE, data,
+ SMB2_OP_QUERY_INFO, cfile, NULL, NULL);
}
- if (rc)
- goto out;
- move_smb2_info_to_cifs(data, smb2_data);
out:
- kfree(smb2_data);
+ free_rsp_buf(err_buftype[0], err_iov[0].iov_base);
+ free_rsp_buf(err_buftype[1], err_iov[1].iov_base);
+ free_rsp_buf(err_buftype[2], err_iov[2].iov_base);
return rc;
}
-int
-smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const char *full_path,
- struct smb311_posix_qinfo *data, bool *adjust_tz, bool *reparse)
+int smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse)
{
int rc;
__u32 create_options = 0;
struct cifsFileInfo *cfile;
- struct smb311_posix_qinfo *smb2_data;
+ struct kvec err_iov[3] = {};
+ int err_buftype[3] = {};
*adjust_tz = false;
*reparse = false;
- /* BB TODO: Make struct larger when add support for parsing owner SIDs */
- smb2_data = kzalloc(sizeof(struct smb311_posix_qinfo),
- GFP_KERNEL);
- if (smb2_data == NULL)
- return -ENOMEM;
-
/*
* BB TODO: Add support for using the cached root handle.
* Create SMB2_query_posix_info worker function to do non-compounded query
*/
cifs_get_readable_path(tcon, full_path, &cfile);
- rc = smb2_compound_op(xid, tcon, cifs_sb, full_path,
- FILE_READ_ATTRIBUTES, FILE_OPEN, create_options,
- ACL_NO_MODE, smb2_data, SMB2_OP_POSIX_QUERY_INFO, cfile);
+ rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES, FILE_OPEN,
+ create_options, ACL_NO_MODE, data, SMB2_OP_POSIX_QUERY_INFO, cfile,
+ err_iov, err_buftype);
if (rc == -EOPNOTSUPP) {
/* BB TODO: When support for special files added to Samba re-verify this path */
+ if (err_iov[0].iov_base && err_buftype[0] != CIFS_NO_BUFFER &&
+ ((struct smb2_hdr *)err_iov[0].iov_base)->Command == SMB2_CREATE &&
+ ((struct smb2_hdr *)err_iov[0].iov_base)->Status == STATUS_STOPPED_ON_SYMLINK) {
+ rc = smb2_parse_symlink_response(cifs_sb, err_iov, &data->symlink_target);
+ if (rc)
+ goto out;
+ }
*reparse = true;
create_options |= OPEN_REPARSE_POINT;
/* Failed on a symbolic link - query a reparse point info */
cifs_get_readable_path(tcon, full_path, &cfile);
- rc = smb2_compound_op(xid, tcon, cifs_sb, full_path,
- FILE_READ_ATTRIBUTES, FILE_OPEN,
- create_options, ACL_NO_MODE,
- smb2_data, SMB2_OP_POSIX_QUERY_INFO, cfile);
+ rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES,
+ FILE_OPEN, create_options, ACL_NO_MODE, data,
+ SMB2_OP_POSIX_QUERY_INFO, cfile, NULL, NULL);
}
- if (rc)
- goto out;
-
- /* TODO: will need to allow for the 2 SIDs when add support for getting owner UID/GID */
- memcpy(data, smb2_data, sizeof(struct smb311_posix_qinfo));
out:
- kfree(smb2_data);
+ free_rsp_buf(err_buftype[0], err_iov[0].iov_base);
+ free_rsp_buf(err_buftype[1], err_iov[1].iov_base);
+ free_rsp_buf(err_buftype[2], err_iov[2].iov_base);
return rc;
}
return smb2_compound_op(xid, tcon, cifs_sb, name,
FILE_WRITE_ATTRIBUTES, FILE_CREATE,
CREATE_NOT_FILE, mode, NULL, SMB2_OP_MKDIR,
- NULL);
+ NULL, NULL, NULL);
}
void
tmprc = smb2_compound_op(xid, tcon, cifs_sb, name,
FILE_WRITE_ATTRIBUTES, FILE_CREATE,
CREATE_NOT_FILE, ACL_NO_MODE,
- &data, SMB2_OP_SET_INFO, cfile);
+ &data, SMB2_OP_SET_INFO, cfile, NULL, NULL);
if (tmprc == 0)
cifs_i->cifsAttrs = dosattrs;
}
{
return smb2_compound_op(xid, tcon, cifs_sb, name, DELETE, FILE_OPEN,
CREATE_NOT_FILE, ACL_NO_MODE,
- NULL, SMB2_OP_RMDIR, NULL);
+ NULL, SMB2_OP_RMDIR, NULL, NULL, NULL);
}
int
{
return smb2_compound_op(xid, tcon, cifs_sb, name, DELETE, FILE_OPEN,
CREATE_DELETE_ON_CLOSE | OPEN_REPARSE_POINT,
- ACL_NO_MODE, NULL, SMB2_OP_DELETE, NULL);
+ ACL_NO_MODE, NULL, SMB2_OP_DELETE, NULL, NULL, NULL);
}
static int
}
rc = smb2_compound_op(xid, tcon, cifs_sb, from_name, access,
FILE_OPEN, 0, ACL_NO_MODE, smb2_to_name,
- command, cfile);
+ command, cfile, NULL, NULL);
smb2_rename_path:
kfree(smb2_to_name);
return rc;
cifs_get_writable_path(tcon, full_path, FIND_WR_ANY, &cfile);
return smb2_compound_op(xid, tcon, cifs_sb, full_path,
FILE_WRITE_DATA, FILE_OPEN, 0, ACL_NO_MODE,
- &eof, SMB2_OP_SET_EOF, cfile);
+ &eof, SMB2_OP_SET_EOF, cfile, NULL, NULL);
}
int
cifs_get_writable_path(tcon, full_path, FIND_WR_ANY, &cfile);
rc = smb2_compound_op(xid, tcon, cifs_sb, full_path,
FILE_WRITE_ATTRIBUTES, FILE_OPEN,
- 0, ACL_NO_MODE, buf, SMB2_OP_SET_INFO, cfile);
+ 0, ACL_NO_MODE, buf, SMB2_OP_SET_INFO, cfile,
+ NULL, NULL);
cifs_put_tlink(tlink);
return rc;
}
* Some windows servers (win2016) will pad also the final
* PDU in a compound to 8 bytes.
*/
- if (((calc_len + 7) & ~7) == len)
+ if (ALIGN(calc_len, 8) == len)
return 0;
/*
struct kvec *iov, int nvec)
{
int i, rc;
- struct sdesc *d;
struct smb2_hdr *hdr;
+ struct shash_desc *sha512 = NULL;
hdr = (struct smb2_hdr *)iov[0].iov_base;
/* neg prot are always taken */
if (rc)
return rc;
- d = server->secmech.sdescsha512;
- rc = crypto_shash_init(&d->shash);
+ sha512 = server->secmech.sha512;
+ rc = crypto_shash_init(sha512);
if (rc) {
cifs_dbg(VFS, "%s: Could not init sha512 shash\n", __func__);
return rc;
}
- rc = crypto_shash_update(&d->shash, ses->preauth_sha_hash,
+ rc = crypto_shash_update(sha512, ses->preauth_sha_hash,
SMB2_PREAUTH_HASH_SIZE);
if (rc) {
cifs_dbg(VFS, "%s: Could not update sha512 shash\n", __func__);
}
for (i = 0; i < nvec; i++) {
- rc = crypto_shash_update(&d->shash,
- iov[i].iov_base, iov[i].iov_len);
+ rc = crypto_shash_update(sha512, iov[i].iov_base, iov[i].iov_len);
if (rc) {
cifs_dbg(VFS, "%s: Could not update sha512 shash\n",
__func__);
}
}
- rc = crypto_shash_final(&d->shash, ses->preauth_sha_hash);
+ rc = crypto_shash_final(sha512, ses->preauth_sha_hash);
if (rc) {
cifs_dbg(VFS, "%s: Could not finalize sha512 shash\n",
__func__);
static int
parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf,
- size_t buf_len,
- struct cifs_ses *ses)
+ size_t buf_len, struct cifs_ses *ses, bool in_mount)
{
struct network_interface_info_ioctl_rsp *p;
struct sockaddr_in *addr4;
}
spin_unlock(&ses->iface_lock);
+ /*
+ * Samba server e.g. can return an empty interface list in some cases,
+ * which would only be a problem if we were requesting multichannel
+ */
+ if (bytes_left == 0) {
+ /* avoid spamming logs every 10 minutes, so log only in mount */
+ if ((ses->chan_max > 1) && in_mount)
+ cifs_dbg(VFS,
+ "multichannel not available\n"
+ "Empty network interface list returned by server %s\n",
+ ses->server->hostname);
+ rc = -EINVAL;
+ goto out;
+ }
+
while (bytes_left >= sizeof(*p)) {
memset(&tmp_iface, 0, sizeof(tmp_iface));
tmp_iface.speed = le64_to_cpu(p->LinkSpeed);
}
int
-SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon)
+SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_mount)
{
int rc;
unsigned int ret_data_len = 0;
goto out;
}
- rc = parse_server_interfaces(out_buf, ret_data_len, ses);
+ rc = parse_server_interfaces(out_buf, ret_data_len, ses, in_mount);
if (rc)
goto out;
if (rc)
return;
- SMB3_request_interfaces(xid, tcon);
+ SMB3_request_interfaces(xid, tcon, true /* called during mount */);
SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid,
FS_ATTRIBUTE_INFORMATION);
rc = open_cached_dir(xid, tcon, full_path, cifs_sb, true, &cfid);
if (!rc) {
- if (cfid->is_valid) {
+ if (cfid->has_lease) {
close_cached_dir(cfid);
return 0;
}
return rc;
}
-static int
-smb2_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb, const char *full_path,
- u64 *uniqueid, FILE_ALL_INFO *data)
+static int smb2_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ u64 *uniqueid, struct cifs_open_info_data *data)
{
- *uniqueid = le64_to_cpu(data->IndexNumber);
+ *uniqueid = le64_to_cpu(data->fi.IndexNumber);
return 0;
}
-static int
-smb2_query_file_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_fid *fid, FILE_ALL_INFO *data)
+static int smb2_query_file_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifsFileInfo *cfile, struct cifs_open_info_data *data)
{
- int rc;
- struct smb2_file_all_info *smb2_data;
-
- smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2,
- GFP_KERNEL);
- if (smb2_data == NULL)
- return -ENOMEM;
+ struct cifs_fid *fid = &cfile->fid;
- rc = SMB2_query_info(xid, tcon, fid->persistent_fid, fid->volatile_fid,
- smb2_data);
- if (!rc)
- move_smb2_info_to_cifs(data, smb2_data);
- kfree(smb2_data);
- return rc;
+ if (cfile->symlink_target) {
+ data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL);
+ if (!data->symlink_target)
+ return -ENOMEM;
+ }
+ return SMB2_query_info(xid, tcon, fid->persistent_fid, fid->volatile_fid, &data->fi);
}
#ifdef CONFIG_CIFS_XATTR
CIFSMaxBufSize, (char **)&res_key, &ret_data_len);
if (rc == -EOPNOTSUPP) {
- pr_warn_once("Server share %s does not support copy range\n", tcon->treeName);
+ pr_warn_once("Server share %s does not support copy range\n", tcon->tree_name);
goto req_res_key_exit;
} else if (rc) {
cifs_tcon_dbg(VFS, "refcpy ioctl error %d getting resume key\n", rc);
static int
smb3_notify(const unsigned int xid, struct file *pfile,
- void __user *ioc_buf)
+ void __user *ioc_buf, bool return_changes)
{
- struct smb3_notify notify;
+ struct smb3_notify_info notify;
+ struct smb3_notify_info __user *pnotify_buf;
struct dentry *dentry = pfile->f_path.dentry;
struct inode *inode = file_inode(pfile);
struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb);
struct cifs_fid fid;
struct cifs_tcon *tcon;
const unsigned char *path;
+ char *returned_ioctl_info = NULL;
void *page = alloc_dentry_path();
__le16 *utf16_path = NULL;
u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
int rc = 0;
+ __u32 ret_len = 0;
path = build_path_from_dentry(dentry, page);
if (IS_ERR(path)) {
goto notify_exit;
}
- if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify))) {
- rc = -EFAULT;
- goto notify_exit;
+ if (return_changes) {
+ if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify_info))) {
+ rc = -EFAULT;
+ goto notify_exit;
+ }
+ } else {
+ if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify))) {
+ rc = -EFAULT;
+ goto notify_exit;
+ }
+ notify.data_len = 0;
}
tcon = cifs_sb_master_tcon(cifs_sb);
goto notify_exit;
rc = SMB2_change_notify(xid, tcon, fid.persistent_fid, fid.volatile_fid,
- notify.watch_tree, notify.completion_filter);
+ notify.watch_tree, notify.completion_filter,
+ notify.data_len, &returned_ioctl_info, &ret_len);
SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
cifs_dbg(FYI, "change notify for path %s rc %d\n", path, rc);
-
+ if (return_changes && (ret_len > 0) && (notify.data_len > 0)) {
+ if (ret_len > notify.data_len)
+ ret_len = notify.data_len;
+ pnotify_buf = (struct smb3_notify_info __user *)ioc_buf;
+ if (copy_to_user(pnotify_buf->notify_data, returned_ioctl_info, ret_len))
+ rc = -EFAULT;
+ else if (copy_to_user(&pnotify_buf->data_len, &ret_len, sizeof(ret_len)))
+ rc = -EFAULT;
+ }
+ kfree(returned_ioctl_info);
notify_exit:
free_dentry_path(page);
kfree(utf16_path);
spin_unlock(&tcon->tc_lock);
spin_unlock(&cifs_tcp_ses_lock);
pr_warn_once("Server share %s deleted.\n",
- tcon->treeName);
+ tcon->tree_name);
return;
}
}
if (rc == -EREMCHG) {
tcon->need_reconnect = true;
pr_warn_once("server share %s deleted\n",
- tcon->treeName);
+ tcon->tree_name);
}
goto qic_exit;
}
}
}
-#define SMB2_SYMLINK_STRUCT_SIZE \
- (sizeof(struct smb2_err_rsp) - 1 + sizeof(struct smb2_symlink_err_rsp))
-
static int
smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon,
struct cifs_sb_info *cifs_sb, const char *full_path,
struct cifs_open_parms oparms;
struct cifs_fid fid;
struct kvec err_iov = {NULL, 0};
- struct smb2_err_rsp *err_buf = NULL;
- struct smb2_symlink_err_rsp *symlink;
struct TCP_Server_Info *server = cifs_pick_channel(tcon->ses);
- unsigned int sub_len;
- unsigned int sub_offset;
- unsigned int print_len;
- unsigned int print_offset;
int flags = CIFS_CP_CREATE_CLOSE_OP;
struct smb_rqst rqst[3];
int resp_buftype[3];
goto querty_exit;
}
- err_buf = err_iov.iov_base;
- if (le32_to_cpu(err_buf->ByteCount) < sizeof(struct smb2_symlink_err_rsp) ||
- err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE) {
- rc = -EINVAL;
- goto querty_exit;
- }
-
- symlink = (struct smb2_symlink_err_rsp *)err_buf->ErrorData;
- if (le32_to_cpu(symlink->SymLinkErrorTag) != SYMLINK_ERROR_TAG ||
- le32_to_cpu(symlink->ReparseTag) != IO_REPARSE_TAG_SYMLINK) {
- rc = -EINVAL;
- goto querty_exit;
- }
-
- /* open must fail on symlink - reset rc */
- rc = 0;
- sub_len = le16_to_cpu(symlink->SubstituteNameLength);
- sub_offset = le16_to_cpu(symlink->SubstituteNameOffset);
- print_len = le16_to_cpu(symlink->PrintNameLength);
- print_offset = le16_to_cpu(symlink->PrintNameOffset);
-
- if (err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offset + sub_len) {
- rc = -EINVAL;
- goto querty_exit;
- }
-
- if (err_iov.iov_len <
- SMB2_SYMLINK_STRUCT_SIZE + print_offset + print_len) {
- rc = -EINVAL;
- goto querty_exit;
- }
-
- *target_path = cifs_strndup_from_utf16(
- (char *)symlink->PathBuffer + sub_offset,
- sub_len, true, cifs_sb->local_nls);
- if (!(*target_path)) {
- rc = -ENOMEM;
- goto querty_exit;
- }
- convert_delimiter(*target_path, '/');
- cifs_dbg(FYI, "%s: target path: %s\n", __func__, *target_path);
+ rc = smb2_parse_symlink_response(cifs_sb, &err_iov, target_path);
querty_exit:
cifs_dbg(FYI, "query symlink rc %d\n", rc);
return rc;
}
- tfm = enc ? server->secmech.ccmaesencrypt :
- server->secmech.ccmaesdecrypt;
+ tfm = enc ? server->secmech.enc : server->secmech.dec;
if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) ||
(server->cipher_type == SMB2_ENCRYPTION_AES256_GCM))
if (!rc && enc)
memcpy(&tr_hdr->Signature, sign, SMB2_SIGNATURE_SIZE);
- kfree(iv);
+ kfree_sensitive(iv);
free_sg:
- kfree(sg);
+ kfree_sensitive(sg);
free_req:
- kfree(req);
+ kfree_sensitive(req);
return rc;
}
{
struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb);
int rc = -EPERM;
- FILE_ALL_INFO *buf = NULL;
+ struct cifs_open_info_data buf = {};
struct cifs_io_parms io_parms = {0};
__u32 oplock = 0;
struct cifs_fid fid;
* and was used by default in earlier versions of Windows
*/
if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL))
- goto out;
+ return rc;
/*
* TODO: Add ability to create instead via reparse point. Windows (e.g.
*/
if (!S_ISCHR(mode) && !S_ISBLK(mode))
- goto out;
+ return rc;
cifs_dbg(FYI, "sfu compat create special file\n");
- buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL);
- if (buf == NULL) {
- rc = -ENOMEM;
- goto out;
- }
-
oparms.tcon = tcon;
oparms.cifs_sb = cifs_sb;
oparms.desired_access = GENERIC_WRITE;
oplock = REQ_OPLOCK;
else
oplock = 0;
- rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, buf);
+ rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, &buf);
if (rc)
- goto out;
+ return rc;
/*
* BB Do not bother to decode buf since no local inode yet to put
* timestamps in, but we can reuse it safely.
*/
- pdev = (struct win_dev *)buf;
+ pdev = (struct win_dev *)&buf.fi;
io_parms.pid = current->tgid;
io_parms.tcon = tcon;
io_parms.offset = 0;
io_parms.length = sizeof(struct win_dev);
- iov[1].iov_base = buf;
+ iov[1].iov_base = &buf.fi;
iov[1].iov_len = sizeof(struct win_dev);
if (S_ISCHR(mode)) {
memcpy(pdev->type, "IntxCHR", 8);
d_drop(dentry);
/* FIXME: add code here to set EAs */
-out:
- kfree(buf);
+
+ cifs_free_open_info(&buf);
return rc;
}
/*
* Context Data length must be rounded to multiple of 8 for some servers
*/
- pneg_ctxt->DataLength = cpu_to_le16(DIV_ROUND_UP(
- sizeof(struct smb2_signing_capabilities) -
- sizeof(struct smb2_neg_context) +
- (num_algs * 2 /* sizeof u16 */), 8) * 8);
+ pneg_ctxt->DataLength = cpu_to_le16(ALIGN(sizeof(struct smb2_signing_capabilities) -
+ sizeof(struct smb2_neg_context) +
+ (num_algs * sizeof(u16)), 8));
pneg_ctxt->SigningAlgorithmCount = cpu_to_le16(num_algs);
pneg_ctxt->SigningAlgorithms[0] = cpu_to_le16(SIGNING_ALG_AES_CMAC);
- ctxt_len += 2 /* sizeof le16 */ * num_algs;
- ctxt_len = DIV_ROUND_UP(ctxt_len, 8) * 8;
+ ctxt_len += sizeof(__le16) * num_algs;
+ ctxt_len = ALIGN(ctxt_len, 8);
return ctxt_len;
/* TBD add SIGNING_ALG_AES_GMAC and/or SIGNING_ALG_HMAC_SHA256 */
}
/* copy up to max of first 100 bytes of server name to NetName field */
pneg_ctxt->DataLength = cpu_to_le16(2 * cifs_strtoUTF16(pneg_ctxt->NetName, hostname, 100, cp));
/* context size is DataLength + minimal smb2_neg_context */
- return DIV_ROUND_UP(le16_to_cpu(pneg_ctxt->DataLength) +
- sizeof(struct smb2_neg_context), 8) * 8;
+ return ALIGN(le16_to_cpu(pneg_ctxt->DataLength) + sizeof(struct smb2_neg_context), 8);
}
static void
* round up total_len of fixed part of SMB3 negotiate request to 8
* byte boundary before adding negotiate contexts
*/
- *total_len = roundup(*total_len, 8);
+ *total_len = ALIGN(*total_len, 8);
pneg_ctxt = (*total_len) + (char *)req;
req->NegotiateContextOffset = cpu_to_le32(*total_len);
build_preauth_ctxt((struct smb2_preauth_neg_context *)pneg_ctxt);
- ctxt_len = DIV_ROUND_UP(sizeof(struct smb2_preauth_neg_context), 8) * 8;
+ ctxt_len = ALIGN(sizeof(struct smb2_preauth_neg_context), 8);
*total_len += ctxt_len;
pneg_ctxt += ctxt_len;
build_encrypt_ctxt((struct smb2_encryption_neg_context *)pneg_ctxt);
- ctxt_len = DIV_ROUND_UP(sizeof(struct smb2_encryption_neg_context), 8) * 8;
+ ctxt_len = ALIGN(sizeof(struct smb2_encryption_neg_context), 8);
*total_len += ctxt_len;
pneg_ctxt += ctxt_len;
if (server->compress_algorithm) {
build_compression_ctxt((struct smb2_compression_capabilities_context *)
pneg_ctxt);
- ctxt_len = DIV_ROUND_UP(
- sizeof(struct smb2_compression_capabilities_context),
- 8) * 8;
+ ctxt_len = ALIGN(sizeof(struct smb2_compression_capabilities_context), 8);
*total_len += ctxt_len;
pneg_ctxt += ctxt_len;
neg_context_count++;
if (rc)
break;
/* offsets must be 8 byte aligned */
- clen = (clen + 7) & ~0x7;
+ clen = ALIGN(clen, 8);
offset += clen + sizeof(struct smb2_neg_context);
len_of_ctxts -= clen;
}
struct smb2_negotiate_rsp *rsp;
struct kvec iov[1];
struct kvec rsp_iov;
- int rc = 0;
+ int rc;
int resp_buftype;
int blob_offset, blob_length;
char *security_blob;
pneg_inbuf->Dialects[0] =
cpu_to_le16(server->vals->protocol_id);
pneg_inbuf->DialectCount = cpu_to_le16(1);
- /* structure is big enough for 3 dialects, sending only 1 */
+ /* structure is big enough for 4 dialects, sending only 1 */
inbuflen = sizeof(*pneg_inbuf) -
- sizeof(pneg_inbuf->Dialects[0]) * 2;
+ sizeof(pneg_inbuf->Dialects[0]) * 3;
}
rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
static void
SMB2_sess_free_buffer(struct SMB2_sess_data *sess_data)
{
+ int i;
+
+ /* zero the session data before freeing, as it might contain sensitive info (keys, etc) */
+ for (i = 0; i < 2; i++)
+ if (sess_data->iov[i].iov_base)
+ memzero_explicit(sess_data->iov[i].iov_base, sess_data->iov[i].iov_len);
+
free_rsp_buf(sess_data->buf0_type, sess_data->iov[0].iov_base);
sess_data->buf0_type = CIFS_NO_BUFFER;
}
out_put_spnego_key:
key_invalidate(spnego_key);
key_put(spnego_key);
+ if (rc)
+ kfree_sensitive(ses->auth_key.response);
out:
sess_data->result = rc;
sess_data->func = NULL;
}
out:
- kfree(ntlmssp_blob);
+ memzero_explicit(ntlmssp_blob, blob_length);
SMB2_sess_free_buffer(sess_data);
if (!rc) {
sess_data->result = 0;
return;
}
out_err:
- kfree(ses->ntlmssp);
+ kfree_sensitive(ses->ntlmssp);
ses->ntlmssp = NULL;
sess_data->result = rc;
sess_data->func = NULL;
}
#endif
out:
- kfree(ntlmssp_blob);
+ memzero_explicit(ntlmssp_blob, blob_length);
SMB2_sess_free_buffer(sess_data);
- kfree(ses->ntlmssp);
+ kfree_sensitive(ses->ntlmssp);
ses->ntlmssp = NULL;
sess_data->result = rc;
sess_data->func = NULL;
cifs_server_dbg(VFS, "signing requested but authenticated as guest\n");
rc = sess_data->result;
out:
- kfree(sess_data);
+ kfree_sensitive(sess_data);
return rc;
}
tcon->capabilities = rsp->Capabilities; /* we keep caps little endian */
tcon->maximal_access = le32_to_cpu(rsp->MaximalAccess);
tcon->tid = le32_to_cpu(rsp->hdr.Id.SyncId.TreeId);
- strscpy(tcon->treeName, tree, sizeof(tcon->treeName));
+ strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name));
if ((rsp->Capabilities & SMB2_SHARE_CAP_DFS) &&
((tcon->share_flags & SHI1005_FLAGS_DFS) == 0))
if (!ses || !(ses->server))
return -EIO;
+ trace_smb3_tdis_enter(xid, tcon->tid, ses->Suid, tcon->tree_name);
spin_lock(&ses->chan_lock);
if ((tcon->need_reconnect) ||
(CIFS_ALL_CHANS_NEED_RECONNECT(tcon->ses))) {
rc = cifs_send_recv(xid, ses, ses->server,
&rqst, &resp_buf_type, flags, &rsp_iov);
cifs_small_buf_release(req);
- if (rc)
+ if (rc) {
cifs_stats_fail_inc(tcon, SMB2_TREE_DISCONNECT_HE);
+ trace_smb3_tdis_err(xid, tcon->tid, ses->Suid, rc);
+ }
+ trace_smb3_tdis_done(xid, tcon->tid, ses->Suid);
return rc;
}
unsigned int acelen, acl_size, ace_count;
unsigned int owner_offset = 0;
unsigned int group_offset = 0;
- struct smb3_acl acl;
+ struct smb3_acl acl = {};
- *len = roundup(sizeof(struct crt_sd_ctxt) + (sizeof(struct cifs_ace) * 4), 8);
+ *len = round_up(sizeof(struct crt_sd_ctxt) + (sizeof(struct cifs_ace) * 4), 8);
if (set_owner) {
/* sizeof(struct owner_group_sids) is already multiple of 8 so no need to round */
acl.AclRevision = ACL_REVISION; /* See 2.4.4.1 of MS-DTYP */
acl.AclSize = cpu_to_le16(acl_size);
acl.AceCount = cpu_to_le16(ace_count);
+ /* acl.Sbz1 and Sbz2 MBZ so are not set here, but initialized above */
memcpy(aclptr, &acl, sizeof(struct smb3_acl));
buf->ccontext.DataLength = cpu_to_le32(ptr - (__u8 *)&buf->sd);
- *len = roundup(ptr - (__u8 *)buf, 8);
+ *len = round_up((unsigned int)(ptr - (__u8 *)buf), 8);
return buf;
}
* final path needs to be 8-byte aligned as specified in
* MS-SMB2 2.2.13 SMB2 CREATE Request.
*/
- *out_size = roundup(*out_len * sizeof(__le16), 8);
+ *out_size = round_up(*out_len * sizeof(__le16), 8);
*out_path = kzalloc(*out_size + sizeof(__le16) /* null */, GFP_KERNEL);
if (!*out_path)
return -ENOMEM;
req->hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS;
rc = alloc_path_with_tree_prefix(©_path, ©_size,
&name_len,
- tcon->treeName, utf16_path);
+ tcon->tree_name, utf16_path);
if (rc)
goto err_free_req;
req->hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS;
rc = alloc_path_with_tree_prefix(©_path, ©_size,
&name_len,
- tcon->treeName, path);
+ tcon->tree_name, path);
if (rc)
return rc;
req->NameLength = cpu_to_le16(name_len * 2);
uni_path_len = (2 * UniStrnlen((wchar_t *)path, PATH_MAX)) + 2;
/* MUST set path len (NameLength) to 0 opening root of share */
req->NameLength = cpu_to_le16(uni_path_len - 2);
- copy_size = uni_path_len;
- if (copy_size % 8 != 0)
- copy_size = roundup(copy_size, 8);
+ copy_size = round_up(uni_path_len, 8);
copy_path = kzalloc(copy_size, GFP_KERNEL);
if (!copy_path)
return -ENOMEM;
oparms->create_options, oparms->desired_access, rc);
if (rc == -EREMCHG) {
pr_warn_once("server share %s deleted\n",
- tcon->treeName);
+ tcon->tree_name);
tcon->need_reconnect = true;
}
goto creat_exit;
if (rc)
return rc;
- memcpy(data, begin_of_buf, buffer_length);
+ memcpy(data, begin_of_buf, minbufsize);
return 0;
}
rc = smb2_validate_and_copy_iov(le16_to_cpu(rsp->OutputBufferOffset),
le32_to_cpu(rsp->OutputBufferLength),
- &rsp_iov, min_len, *data);
+ &rsp_iov, dlen ? *dlen : min_len, *data);
if (rc && allocated) {
kfree(*data);
*data = NULL;
int
SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon,
u64 persistent_fid, u64 volatile_fid, bool watch_tree,
- u32 completion_filter)
+ u32 completion_filter, u32 max_out_data_len, char **out_data,
+ u32 *plen /* returned data len */)
{
struct cifs_ses *ses = tcon->ses;
struct TCP_Server_Info *server = cifs_pick_channel(ses);
struct smb_rqst rqst;
+ struct smb2_change_notify_rsp *smb_rsp;
struct kvec iov[1];
struct kvec rsp_iov = {NULL, 0};
int resp_buftype = CIFS_NO_BUFFER;
memset(&rqst, 0, sizeof(struct smb_rqst));
memset(&iov, 0, sizeof(iov));
+ if (plen)
+ *plen = 0;
+
rqst.rq_iov = iov;
rqst.rq_nvec = 1;
cifs_stats_fail_inc(tcon, SMB2_CHANGE_NOTIFY_HE);
trace_smb3_notify_err(xid, persistent_fid, tcon->tid, ses->Suid,
(u8)watch_tree, completion_filter, rc);
- } else
+ } else {
trace_smb3_notify_done(xid, persistent_fid, tcon->tid,
- ses->Suid, (u8)watch_tree, completion_filter);
+ ses->Suid, (u8)watch_tree, completion_filter);
+ /* validate that notify information is plausible */
+ if ((rsp_iov.iov_base == NULL) ||
+ (rsp_iov.iov_len < sizeof(struct smb2_change_notify_rsp)))
+ goto cnotify_exit;
+
+ smb_rsp = (struct smb2_change_notify_rsp *)rsp_iov.iov_base;
+
+ smb2_validate_iov(le16_to_cpu(smb_rsp->OutputBufferOffset),
+ le32_to_cpu(smb_rsp->OutputBufferLength), &rsp_iov,
+ sizeof(struct file_notify_information));
+
+ *out_data = kmemdup((char *)smb_rsp + le16_to_cpu(smb_rsp->OutputBufferOffset),
+ le32_to_cpu(smb_rsp->OutputBufferLength), GFP_KERNEL);
+ if (*out_data == NULL) {
+ rc = -ENOMEM;
+ goto cnotify_exit;
+ } else
+ *plen = le32_to_cpu(smb_rsp->OutputBufferLength);
+ }
cnotify_exit:
if (rqst.rq_iov)
if (request_type & CHAINED_REQUEST) {
if (!(request_type & END_OF_CHAIN)) {
/* next 8-byte aligned request */
- *total_len = DIV_ROUND_UP(*total_len, 8) * 8;
+ *total_len = ALIGN(*total_len, 8);
shdr->NextCommand = cpu_to_le32(*total_len);
} else /* END_OF_CHAIN */
shdr->NextCommand = 0;
wdata->bytes, wdata->result);
if (wdata->result == -ENOSPC)
pr_warn_once("Out of space writing to %s\n",
- tcon->treeName);
+ tcon->tree_name);
} else
trace_smb3_write_done(0 /* no xid */,
wdata->cfile->fid.persistent_fid,
#define COMPOUND_FID 0xFFFFFFFFFFFFFFFFULL
+#define SMB2_SYMLINK_STRUCT_SIZE \
+ (sizeof(struct smb2_err_rsp) - 1 + sizeof(struct smb2_symlink_err_rsp))
+
#define SYMLINK_ERROR_TAG 0x4c4d5953
struct smb2_symlink_err_rsp {
struct TCP_Server_Info *srv);
extern int smb3_handle_read_data(struct TCP_Server_Info *server,
struct mid_q_entry *mid);
-
-extern void move_smb2_info_to_cifs(FILE_ALL_INFO *dst,
- struct smb2_file_all_info *src);
extern int smb2_query_reparse_tag(const unsigned int xid, struct cifs_tcon *tcon,
struct cifs_sb_info *cifs_sb, const char *path,
__u32 *reparse_tag);
-extern int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *cifs_sb,
- const char *full_path, FILE_ALL_INFO *data,
- bool *adjust_tz, bool *symlink);
+int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse);
extern int smb2_set_path_size(const unsigned int xid, struct cifs_tcon *tcon,
const char *full_path, __u64 size,
struct cifs_sb_info *cifs_sb, bool set_alloc);
struct cifs_sb_info *cifs_sb,
const unsigned char *path, char *pbuf,
unsigned int *pbytes_read);
-extern int smb2_open_file(const unsigned int xid,
- struct cifs_open_parms *oparms,
- __u32 *oplock, FILE_ALL_INFO *buf);
+int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec *iov, char **path);
+int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock,
+ void *buf);
extern int smb2_unlock_range(struct cifsFileInfo *cfile,
struct file_lock *flock, const unsigned int xid);
extern int smb2_push_mandatory_locks(struct cifsFileInfo *cfile);
extern void SMB2_ioctl_free(struct smb_rqst *rqst);
extern int SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon,
u64 persistent_fid, u64 volatile_fid, bool watch_tree,
- u32 completion_filter);
+ u32 completion_filter, u32 max_out_data_len,
+ char **out_data, u32 *plen /* returned data len */);
extern int __SMB2_close(const unsigned int xid, struct cifs_tcon *tcon,
u64 persistent_fid, u64 volatile_fid,
struct kvec *rsp, int *buftype,
struct cifs_sb_info *cifs_sb);
/* query path info from the server using SMB311 POSIX extensions*/
-extern int smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
- struct cifs_sb_info *sb, const char *path, struct smb311_posix_qinfo *qinf,
- bool *adjust_tx, bool *symlink);
+int smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon,
+ struct cifs_sb_info *cifs_sb, const char *full_path,
+ struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse);
int posix_info_parse(const void *beg, const void *end,
struct smb2_posix_info_parsed *out);
int posix_info_sid_size(const void *beg, const void *end);
struct cifs_secmech *p = &server->secmech;
int rc;
- rc = cifs_alloc_hash("hmac(sha256)",
- &p->hmacsha256,
- &p->sdeschmacsha256);
+ rc = cifs_alloc_hash("hmac(sha256)", &p->hmacsha256);
if (rc)
goto err;
- rc = cifs_alloc_hash("cmac(aes)", &p->cmacaes, &p->sdesccmacaes);
+ rc = cifs_alloc_hash("cmac(aes)", &p->aes_cmac);
if (rc)
goto err;
return 0;
err:
- cifs_free_hash(&p->hmacsha256, &p->sdeschmacsha256);
+ cifs_free_hash(&p->hmacsha256);
return rc;
}
struct cifs_secmech *p = &server->secmech;
int rc = 0;
- rc = cifs_alloc_hash("hmac(sha256)",
- &p->hmacsha256,
- &p->sdeschmacsha256);
+ rc = cifs_alloc_hash("hmac(sha256)", &p->hmacsha256);
if (rc)
return rc;
- rc = cifs_alloc_hash("cmac(aes)", &p->cmacaes, &p->sdesccmacaes);
+ rc = cifs_alloc_hash("cmac(aes)", &p->aes_cmac);
if (rc)
goto err;
- rc = cifs_alloc_hash("sha512", &p->sha512, &p->sdescsha512);
+ rc = cifs_alloc_hash("sha512", &p->sha512);
if (rc)
goto err;
return 0;
err:
- cifs_free_hash(&p->cmacaes, &p->sdesccmacaes);
- cifs_free_hash(&p->hmacsha256, &p->sdeschmacsha256);
+ cifs_free_hash(&p->aes_cmac);
+ cifs_free_hash(&p->hmacsha256);
return rc;
}
struct kvec *iov = rqst->rq_iov;
struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base;
struct cifs_ses *ses;
- struct shash_desc *shash;
- struct crypto_shash *hash;
- struct sdesc *sdesc = NULL;
+ struct shash_desc *shash = NULL;
struct smb_rqst drqst;
ses = smb2_find_smb_ses(server, le64_to_cpu(shdr->SessionId));
- if (!ses) {
+ if (unlikely(!ses)) {
cifs_server_dbg(VFS, "%s: Could not find session\n", __func__);
- return 0;
+ return -ENOENT;
}
memset(smb2_signature, 0x0, SMB2_HMACSHA256_SIZE);
memset(shdr->Signature, 0x0, SMB2_SIGNATURE_SIZE);
if (allocate_crypto) {
- rc = cifs_alloc_hash("hmac(sha256)", &hash, &sdesc);
+ rc = cifs_alloc_hash("hmac(sha256)", &shash);
if (rc) {
cifs_server_dbg(VFS,
"%s: sha256 alloc failed\n", __func__);
goto out;
}
- shash = &sdesc->shash;
} else {
- hash = server->secmech.hmacsha256;
- shash = &server->secmech.sdeschmacsha256->shash;
+ shash = server->secmech.hmacsha256;
}
- rc = crypto_shash_setkey(hash, ses->auth_key.response,
+ rc = crypto_shash_setkey(shash->tfm, ses->auth_key.response,
SMB2_NTLMV2_SESSKEY_SIZE);
if (rc) {
cifs_server_dbg(VFS,
out:
if (allocate_crypto)
- cifs_free_hash(&hash, &sdesc);
+ cifs_free_hash(&shash);
if (ses)
cifs_put_smb_ses(ses);
return rc;
goto smb3signkey_ret;
}
- rc = crypto_shash_setkey(server->secmech.hmacsha256,
+ rc = crypto_shash_setkey(server->secmech.hmacsha256->tfm,
ses->auth_key.response, SMB2_NTLMV2_SESSKEY_SIZE);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not set with session key\n", __func__);
goto smb3signkey_ret;
}
- rc = crypto_shash_init(&server->secmech.sdeschmacsha256->shash);
+ rc = crypto_shash_init(server->secmech.hmacsha256);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not init sign hmac\n", __func__);
goto smb3signkey_ret;
}
- rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash,
- i, 4);
+ rc = crypto_shash_update(server->secmech.hmacsha256, i, 4);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not update with n\n", __func__);
goto smb3signkey_ret;
}
- rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash,
- label.iov_base, label.iov_len);
+ rc = crypto_shash_update(server->secmech.hmacsha256, label.iov_base, label.iov_len);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not update with label\n", __func__);
goto smb3signkey_ret;
}
- rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash,
- &zero, 1);
+ rc = crypto_shash_update(server->secmech.hmacsha256, &zero, 1);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not update with zero\n", __func__);
goto smb3signkey_ret;
}
- rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash,
- context.iov_base, context.iov_len);
+ rc = crypto_shash_update(server->secmech.hmacsha256, context.iov_base, context.iov_len);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not update with context\n", __func__);
goto smb3signkey_ret;
if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) ||
(server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) {
- rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash,
- L256, 4);
+ rc = crypto_shash_update(server->secmech.hmacsha256, L256, 4);
} else {
- rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash,
- L128, 4);
+ rc = crypto_shash_update(server->secmech.hmacsha256, L128, 4);
}
if (rc) {
cifs_server_dbg(VFS, "%s: Could not update with L\n", __func__);
goto smb3signkey_ret;
}
- rc = crypto_shash_final(&server->secmech.sdeschmacsha256->shash,
- hashptr);
+ rc = crypto_shash_final(server->secmech.hmacsha256, hashptr);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not generate sha256 hash\n", __func__);
goto smb3signkey_ret;
unsigned char *sigptr = smb3_signature;
struct kvec *iov = rqst->rq_iov;
struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base;
- struct shash_desc *shash;
- struct crypto_shash *hash;
- struct sdesc *sdesc = NULL;
+ struct shash_desc *shash = NULL;
struct smb_rqst drqst;
u8 key[SMB3_SIGN_KEY_SIZE];
rc = smb2_get_sign_key(le64_to_cpu(shdr->SessionId), server, key);
- if (rc)
- return 0;
+ if (unlikely(rc)) {
+ cifs_server_dbg(VFS, "%s: Could not get signing key\n", __func__);
+ return rc;
+ }
if (allocate_crypto) {
- rc = cifs_alloc_hash("cmac(aes)", &hash, &sdesc);
+ rc = cifs_alloc_hash("cmac(aes)", &shash);
if (rc)
return rc;
-
- shash = &sdesc->shash;
} else {
- hash = server->secmech.cmacaes;
- shash = &server->secmech.sdesccmacaes->shash;
+ shash = server->secmech.aes_cmac;
}
memset(smb3_signature, 0x0, SMB2_CMACAES_SIZE);
memset(shdr->Signature, 0x0, SMB2_SIGNATURE_SIZE);
- rc = crypto_shash_setkey(hash, key, SMB2_CMACAES_SIZE);
+ rc = crypto_shash_setkey(shash->tfm, key, SMB2_CMACAES_SIZE);
if (rc) {
cifs_server_dbg(VFS, "%s: Could not set key for cmac aes\n", __func__);
goto out;
}
/*
- * we already allocate sdesccmacaes when we init smb3 signing key,
+ * we already allocate aes_cmac when we init smb3 signing key,
* so unlike smb2 case we do not have to check here if secmech are
* initialized
*/
out:
if (allocate_crypto)
- cifs_free_hash(&hash, &sdesc);
+ cifs_free_hash(&shash);
return rc;
}
{
struct crypto_aead *tfm;
- if (!server->secmech.ccmaesencrypt) {
+ if (!server->secmech.enc) {
if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) ||
(server->cipher_type == SMB2_ENCRYPTION_AES256_GCM))
tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
__func__);
return PTR_ERR(tfm);
}
- server->secmech.ccmaesencrypt = tfm;
+ server->secmech.enc = tfm;
}
- if (!server->secmech.ccmaesdecrypt) {
+ if (!server->secmech.dec) {
if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) ||
(server->cipher_type == SMB2_ENCRYPTION_AES256_GCM))
tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
else
tfm = crypto_alloc_aead("ccm(aes)", 0, 0);
if (IS_ERR(tfm)) {
- crypto_free_aead(server->secmech.ccmaesencrypt);
- server->secmech.ccmaesencrypt = NULL;
+ crypto_free_aead(server->secmech.enc);
+ server->secmech.enc = NULL;
cifs_server_dbg(VFS, "%s: Failed to alloc decrypt aead\n",
__func__);
return PTR_ERR(tfm);
}
- server->secmech.ccmaesdecrypt = tfm;
+ server->secmech.dec = tfm;
}
return 0;
int smbd_max_fragmented_recv_size = 1024 * 1024;
/* The maximum single-message size which can be received */
-int smbd_max_receive_size = 8192;
+int smbd_max_receive_size = 1364;
/* The timeout to initiate send of a keepalive message on idle */
int smbd_keep_alive_interval = 120;
* User configurable initial values for RDMA transport
* The actual values used may be lower and are limited to hardware capabilities
*/
-/* Default maximum number of SGEs in a RDMA write/read */
+/* Default maximum number of pages in a single RDMA write/read */
int smbd_max_frmr_depth = 2048;
/* If payload is less than this byte, use RDMA send/recv not read/write */
struct smbd_request *request =
container_of(wc->wr_cqe, struct smbd_request, cqe);
- log_rdma_send(INFO, "smbd_request %p completed wc->status=%d\n",
+ log_rdma_send(INFO, "smbd_request 0x%p completed wc->status=%d\n",
request, wc->status);
if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
struct smbd_connection *info = response->info;
int data_length = 0;
- log_rdma_recv(INFO, "response=%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%x\n",
+ log_rdma_recv(INFO, "response=0x%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%u\n",
response, response->type, wc->status, wc->opcode,
wc->byte_len, wc->pkey_index);
send_wr.opcode = IB_WR_SEND;
send_wr.send_flags = IB_SEND_SIGNALED;
- log_rdma_send(INFO, "sge addr=%llx length=%x lkey=%x\n",
+ log_rdma_send(INFO, "sge addr=0x%llx length=%u lkey=0x%x\n",
request->sge[0].addr,
request->sge[0].length, request->sge[0].lkey);
for (i = 0; i < request->num_sge; i++) {
log_rdma_send(INFO,
- "rdma_request sge[%d] addr=%llu length=%u\n",
+ "rdma_request sge[%d] addr=0x%llx length=%u\n",
i, request->sge[i].addr, request->sge[i].length);
ib_dma_sync_single_for_device(
info->id->device,
{
int i;
u32 data_length = 0;
- struct scatterlist sgl[SMBDIRECT_MAX_SGE];
+ struct scatterlist sgl[SMBDIRECT_MAX_SEND_SGE - 1];
- if (n_vec > SMBDIRECT_MAX_SGE) {
+ if (n_vec > SMBDIRECT_MAX_SEND_SGE - 1) {
cifs_dbg(VFS, "Can't fit data to SGL, n_vec=%d\n", n_vec);
return -EINVAL;
}
response->type = SMBD_NEGOTIATE_RESP;
rc = smbd_post_recv(info, response);
- log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=%llx iov.length=%x iov.lkey=%x\n",
+ log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n",
rc, response->sge.addr,
response->sge.length, response->sge.lkey);
if (rc)
if (smbd_send_credit_target > info->id->device->attrs.max_cqe ||
smbd_send_credit_target > info->id->device->attrs.max_qp_wr) {
- log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
+ log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
smbd_send_credit_target,
info->id->device->attrs.max_cqe,
info->id->device->attrs.max_qp_wr);
if (smbd_receive_credit_max > info->id->device->attrs.max_cqe ||
smbd_receive_credit_max > info->id->device->attrs.max_qp_wr) {
- log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
+ log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
smbd_receive_credit_max,
info->id->device->attrs.max_cqe,
info->id->device->attrs.max_qp_wr);
info->max_receive_size = smbd_max_receive_size;
info->keep_alive_interval = smbd_keep_alive_interval;
- if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SGE) {
+ if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SEND_SGE ||
+ info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_RECV_SGE) {
log_rdma_event(ERR,
- "warning: device max_send_sge = %d too small\n",
- info->id->device->attrs.max_send_sge);
- log_rdma_event(ERR, "Queue Pair creation may fail\n");
- }
- if (info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_SGE) {
- log_rdma_event(ERR,
- "warning: device max_recv_sge = %d too small\n",
+ "device %.*s max_send_sge/max_recv_sge = %d/%d too small\n",
+ IB_DEVICE_NAME_MAX,
+ info->id->device->name,
+ info->id->device->attrs.max_send_sge,
info->id->device->attrs.max_recv_sge);
- log_rdma_event(ERR, "Queue Pair creation may fail\n");
+ goto config_failed;
}
info->send_cq = NULL;
qp_attr.qp_context = info;
qp_attr.cap.max_send_wr = info->send_credit_target;
qp_attr.cap.max_recv_wr = info->receive_credit_max;
- qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SGE;
- qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_SGE;
+ qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SEND_SGE;
+ qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_RECV_SGE;
qp_attr.cap.max_inline_data = 0;
qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
qp_attr.qp_type = IB_QPT_RC;
int num_rqst, struct smb_rqst *rqst_array)
{
struct smbd_connection *info = server->smbd_conn;
- struct kvec vec;
+ struct kvec vecs[SMBDIRECT_MAX_SEND_SGE - 1];
int nvecs;
int size;
unsigned int buflen, remaining_data_length;
+ unsigned int offset, remaining_vec_data_length;
int start, i, j;
int max_iov_size =
info->max_send_size - sizeof(struct smbd_data_transfer);
struct smb_rqst *rqst;
int rqst_idx;
- if (info->transport_status != SMBD_CONNECTED) {
- rc = -EAGAIN;
- goto done;
- }
+ if (info->transport_status != SMBD_CONNECTED)
+ return -EAGAIN;
/*
* Add in the page array if there is one. The caller needs to set
for (i = 0; i < num_rqst; i++)
remaining_data_length += smb_rqst_len(server, &rqst_array[i]);
- if (remaining_data_length > info->max_fragmented_send_size) {
+ if (unlikely(remaining_data_length > info->max_fragmented_send_size)) {
+ /* assertion: payload never exceeds negotiated maximum */
log_write(ERR, "payload size %d > max size %d\n",
remaining_data_length, info->max_fragmented_send_size);
- rc = -EINVAL;
- goto done;
+ return -EINVAL;
}
log_write(INFO, "num_rqst=%d total length=%u\n",
num_rqst, remaining_data_length);
rqst_idx = 0;
-next_rqst:
- rqst = &rqst_array[rqst_idx];
- iov = rqst->rq_iov;
-
- cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n",
- rqst_idx, smb_rqst_len(server, rqst));
- for (i = 0; i < rqst->rq_nvec; i++)
- dump_smb(iov[i].iov_base, iov[i].iov_len);
-
-
- log_write(INFO, "rqst_idx=%d nvec=%d rqst->rq_npages=%d rq_pagesz=%d rq_tailsz=%d buflen=%lu\n",
- rqst_idx, rqst->rq_nvec, rqst->rq_npages, rqst->rq_pagesz,
- rqst->rq_tailsz, smb_rqst_len(server, rqst));
-
- start = i = 0;
- buflen = 0;
- while (true) {
- buflen += iov[i].iov_len;
- if (buflen > max_iov_size) {
- if (i > start) {
- remaining_data_length -=
- (buflen-iov[i].iov_len);
- log_write(INFO, "sending iov[] from start=%d i=%d nvecs=%d remaining_data_length=%d\n",
- start, i, i - start,
- remaining_data_length);
- rc = smbd_post_send_data(
- info, &iov[start], i-start,
- remaining_data_length);
- if (rc)
- goto done;
- } else {
- /* iov[start] is too big, break it */
- nvecs = (buflen+max_iov_size-1)/max_iov_size;
- log_write(INFO, "iov[%d] iov_base=%p buflen=%d break to %d vectors\n",
- start, iov[start].iov_base,
- buflen, nvecs);
- for (j = 0; j < nvecs; j++) {
- vec.iov_base =
- (char *)iov[start].iov_base +
- j*max_iov_size;
- vec.iov_len = max_iov_size;
- if (j == nvecs-1)
- vec.iov_len =
- buflen -
- max_iov_size*(nvecs-1);
- remaining_data_length -= vec.iov_len;
- log_write(INFO,
- "sending vec j=%d iov_base=%p iov_len=%zu remaining_data_length=%d\n",
- j, vec.iov_base, vec.iov_len,
- remaining_data_length);
- rc = smbd_post_send_data(
- info, &vec, 1,
- remaining_data_length);
- if (rc)
- goto done;
+ do {
+ rqst = &rqst_array[rqst_idx];
+ iov = rqst->rq_iov;
+
+ cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n",
+ rqst_idx, smb_rqst_len(server, rqst));
+ remaining_vec_data_length = 0;
+ for (i = 0; i < rqst->rq_nvec; i++) {
+ remaining_vec_data_length += iov[i].iov_len;
+ dump_smb(iov[i].iov_base, iov[i].iov_len);
+ }
+
+ log_write(INFO, "rqst_idx=%d nvec=%d rqst->rq_npages=%d rq_pagesz=%d rq_tailsz=%d buflen=%lu\n",
+ rqst_idx, rqst->rq_nvec,
+ rqst->rq_npages, rqst->rq_pagesz,
+ rqst->rq_tailsz, smb_rqst_len(server, rqst));
+
+ start = 0;
+ offset = 0;
+ do {
+ buflen = 0;
+ i = start;
+ j = 0;
+ while (i < rqst->rq_nvec &&
+ j < SMBDIRECT_MAX_SEND_SGE - 1 &&
+ buflen < max_iov_size) {
+
+ vecs[j].iov_base = iov[i].iov_base + offset;
+ if (buflen + iov[i].iov_len > max_iov_size) {
+ vecs[j].iov_len =
+ max_iov_size - iov[i].iov_len;
+ buflen = max_iov_size;
+ offset = vecs[j].iov_len;
+ } else {
+ vecs[j].iov_len =
+ iov[i].iov_len - offset;
+ buflen += vecs[j].iov_len;
+ offset = 0;
+ ++i;
}
- i++;
- if (i == rqst->rq_nvec)
- break;
+ ++j;
}
+
+ remaining_vec_data_length -= buflen;
+ remaining_data_length -= buflen;
+ log_write(INFO, "sending %s iov[%d] from start=%d nvecs=%d remaining_data_length=%d\n",
+ remaining_vec_data_length > 0 ?
+ "partial" : "complete",
+ rqst->rq_nvec, start, j,
+ remaining_data_length);
+
start = i;
- buflen = 0;
- } else {
- i++;
- if (i == rqst->rq_nvec) {
- /* send out all remaining vecs */
- remaining_data_length -= buflen;
- log_write(INFO, "sending iov[] from start=%d i=%d nvecs=%d remaining_data_length=%d\n",
- start, i, i - start,
+ rc = smbd_post_send_data(info, vecs, j, remaining_data_length);
+ if (rc)
+ goto done;
+ } while (remaining_vec_data_length > 0);
+
+ /* now sending pages if there are any */
+ for (i = 0; i < rqst->rq_npages; i++) {
+ rqst_page_get_length(rqst, i, &buflen, &offset);
+ nvecs = (buflen + max_iov_size - 1) / max_iov_size;
+ log_write(INFO, "sending pages buflen=%d nvecs=%d\n",
+ buflen, nvecs);
+ for (j = 0; j < nvecs; j++) {
+ size = min_t(unsigned int, max_iov_size, remaining_data_length);
+ remaining_data_length -= size;
+ log_write(INFO, "sending pages i=%d offset=%d size=%d remaining_data_length=%d\n",
+ i, j * max_iov_size + offset, size,
remaining_data_length);
- rc = smbd_post_send_data(info, &iov[start],
- i-start, remaining_data_length);
+ rc = smbd_post_send_page(
+ info, rqst->rq_pages[i],
+ j*max_iov_size + offset,
+ size, remaining_data_length);
if (rc)
goto done;
- break;
}
}
- log_write(INFO, "looping i=%d buflen=%d\n", i, buflen);
- }
-
- /* now sending pages if there are any */
- for (i = 0; i < rqst->rq_npages; i++) {
- unsigned int offset;
-
- rqst_page_get_length(rqst, i, &buflen, &offset);
- nvecs = (buflen + max_iov_size - 1) / max_iov_size;
- log_write(INFO, "sending pages buflen=%d nvecs=%d\n",
- buflen, nvecs);
- for (j = 0; j < nvecs; j++) {
- size = max_iov_size;
- if (j == nvecs-1)
- size = buflen - j*max_iov_size;
- remaining_data_length -= size;
- log_write(INFO, "sending pages i=%d offset=%d size=%d remaining_data_length=%d\n",
- i, j * max_iov_size + offset, size,
- remaining_data_length);
- rc = smbd_post_send_page(
- info, rqst->rq_pages[i],
- j*max_iov_size + offset,
- size, remaining_data_length);
- if (rc)
- goto done;
- }
- }
-
- rqst_idx++;
- if (rqst_idx < num_rqst)
- goto next_rqst;
+ } while (++rqst_idx < num_rqst);
done:
/*
/* Memory registrations */
/* Maximum number of RDMA read/write outstanding on this connection */
int responder_resources;
- /* Maximum number of SGEs in a RDMA write/read */
+ /* Maximum number of pages in a single RDMA write/read on this connection */
int max_frmr_depth;
/*
* If payload is less than or equal to the threshold,
__le32 length;
} __packed;
-/* Default maximum number of SGEs in a RDMA send/recv */
-#define SMBDIRECT_MAX_SGE 16
+/* Maximum number of SGEs used by smbdirect.c in any send work request */
+#define SMBDIRECT_MAX_SEND_SGE 6
+
/* The context for a SMBD request */
struct smbd_request {
struct smbd_connection *info;
struct ib_cqe cqe;
- /* the SGE entries for this packet */
- struct ib_sge sge[SMBDIRECT_MAX_SGE];
+ /* the SGE entries for this work request */
+ struct ib_sge sge[SMBDIRECT_MAX_SEND_SGE];
int num_sge;
/* SMBD packet header follows this structure */
u8 packet[];
};
+/* Maximum number of SGEs used by smbdirect.c in any receive work request */
+#define SMBDIRECT_MAX_RECV_SGE 1
+
/* The context for a SMBD response */
struct smbd_response {
struct smbd_connection *info;
DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_info_compound_enter);
DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(delete_enter);
DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(mkdir_enter);
+DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(tdis_enter);
DECLARE_EVENT_CLASS(smb3_inf_compound_done_class,
DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_info_compound_done);
DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(delete_done);
DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(mkdir_done);
+DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(tdis_done);
DECLARE_EVENT_CLASS(smb3_inf_compound_err_class,
DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_info_compound_err);
DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(mkdir_err);
DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(delete_err);
+DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(tdis_err);
/*
* For logging SMB3 Status code and Command for responses which return errors
return vma->vm_end - vma->vm_start;
}
-static struct vm_area_struct *first_vma(struct task_struct *tsk,
- struct vm_area_struct *gate_vma)
-{
- struct vm_area_struct *ret = tsk->mm->mmap;
-
- if (ret)
- return ret;
- return gate_vma;
-}
-
/*
* Helper function for iterating across a vma list. It ensures that the caller
* will visit `gate_vma' prior to terminating the search.
*/
-static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma,
+static struct vm_area_struct *coredump_next_vma(struct ma_state *mas,
+ struct vm_area_struct *vma,
struct vm_area_struct *gate_vma)
{
- struct vm_area_struct *ret;
-
- ret = this_vma->vm_next;
- if (ret)
- return ret;
- if (this_vma == gate_vma)
+ if (gate_vma && (vma == gate_vma))
return NULL;
+
+ vma = mas_next(mas, ULONG_MAX);
+ if (vma)
+ return vma;
return gate_vma;
}
*/
static bool dump_vma_snapshot(struct coredump_params *cprm)
{
- struct vm_area_struct *vma, *gate_vma;
+ struct vm_area_struct *gate_vma, *vma = NULL;
struct mm_struct *mm = current->mm;
- int i;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+ int i = 0;
/*
* Once the stack expansion code is fixed to not change VMA bounds
return false;
}
- for (i = 0, vma = first_vma(current, gate_vma); vma != NULL;
- vma = next_vma(vma, gate_vma), i++) {
+ while ((vma = coredump_next_vma(&mas, vma, gate_vma)) != NULL) {
struct core_vma_metadata *m = cprm->vma_meta + i;
m->start = vma->vm_start;
m->flags = vma->vm_flags;
m->dump_size = vma_dump_size(vma, cprm->mm_flags);
m->pgoff = vma->vm_pgoff;
-
m->file = vma->vm_file;
if (m->file)
get_file(m->file);
+ i++;
}
mmap_write_unlock(mm);
EXPORT_SYMBOL(d_genocide);
-void d_tmpfile(struct dentry *dentry, struct inode *inode)
+void d_tmpfile(struct file *file, struct inode *inode)
{
+ struct dentry *dentry = file->f_path.dentry;
+
inode_dec_link_count(inode);
BUG_ON(dentry->d_name.name != dentry->d_iname ||
!hlist_unhashed(&dentry->d_u.d_alias) ||
* added to the list from another CPU: the winner observes
* new->next == new.
*/
- if (cmpxchg(&new->next, new, head) != new)
+ if (!try_cmpxchg(&new->next, &new, head))
return false;
/*
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
-#include <linux/vmacache.h>
#include <linux/stat.h>
#include <linux/fcntl.h>
#include <linux/swap.h>
unsigned long length = old_end - old_start;
unsigned long new_start = old_start - shift;
unsigned long new_end = old_end - shift;
+ VMA_ITERATOR(vmi, mm, new_start);
+ struct vm_area_struct *next;
struct mmu_gather tlb;
BUG_ON(new_start > new_end);
* ensure there are no vmas between where we want to go
* and where we are
*/
- if (vma != find_vma(mm, new_start))
+ if (vma != vma_next(&vmi))
return -EFAULT;
/*
lru_add_drain();
tlb_gather_mmu(&tlb, mm);
+ next = vma_next(&vmi);
if (new_end > old_start) {
/*
* when the old and new regions overlap clear from new_end.
*/
free_pgd_range(&tlb, new_end, old_end, new_end,
- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING);
+ next ? next->vm_start : USER_PGTABLES_CEILING);
} else {
/*
* otherwise, clean from old_start; this is done to not touch
* for the others its just a little faster.
*/
free_pgd_range(&tlb, old_start, old_end, new_end,
- vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING);
+ next ? next->vm_start : USER_PGTABLES_CEILING);
}
tlb_finish_mmu(&tlb);
active_mm = tsk->active_mm;
tsk->active_mm = mm;
tsk->mm = mm;
+ lru_gen_add_mm(mm);
/*
* This prevents preemption while active_mm is being loaded and
* it and mm are being updated, which could cause problems for
activate_mm(active_mm, mm);
if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
local_irq_enable();
- tsk->mm->vmacache_seqnum = 0;
- vmacache_flush(tsk);
task_unlock(tsk);
+ lru_gen_use_mm(mm);
if (old_mm) {
mmap_read_unlock(old_mm);
BUG_ON(active_mm != old_mm);
inode->i_uid = sbi->options.fs_uid;
inode->i_gid = sbi->options.fs_gid;
inode_inc_iversion(inode);
- inode->i_generation = prandom_u32();
+ inode->i_generation = get_random_u32();
if (info->attr & ATTR_SUBDIR) { /* directory */
inode->i_generation &= ~1;
struct ext2_group_desc * desc;
struct buffer_head * bh = NULL;
ext2_fsblk_t bitmap_blk;
+ int ret;
desc = ext2_get_group_desc(sb, block_group, NULL);
if (!desc)
block_group, le32_to_cpu(desc->bg_block_bitmap));
return NULL;
}
- if (likely(bh_uptodate_or_lock(bh)))
+ ret = bh_read(bh, 0);
+ if (ret > 0)
return bh;
-
- if (bh_submit_read(bh) < 0) {
+ if (ret < 0) {
brelse(bh);
ext2_error(sb, __func__,
"Cannot read block bitmap - "
int best_ndir = inodes_per_group;
int best_group = -1;
- group = prandom_u32();
- parent_group = (unsigned)group % ngroups;
+ parent_group = prandom_u32_max(ngroups);
for (i = 0; i < ngroups; i++) {
group = (parent_group + i) % ngroups;
desc = ext2_get_group_desc (sb, group, NULL);
}
static int ext2_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
struct inode *inode = ext2_new_inode(dir, mode, NULL);
if (IS_ERR(inode))
ext2_set_file_ops(inode);
mark_inode_dirty(inode);
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
unlock_new_inode(inode);
- return 0;
+ return finish_open_simple(file, 0);
}
static int ext2_mknod (struct user_namespace * mnt_userns, struct inode * dir,
hinfo.hash_version = DX_HASH_HALF_MD4;
hinfo.seed = sbi->s_hash_seed;
ext4fs_dirhash(parent, qstr->name, qstr->len, &hinfo);
- grp = hinfo.hash;
+ parent_group = hinfo.hash % ngroups;
} else
- grp = prandom_u32();
- parent_group = (unsigned)grp % ngroups;
+ parent_group = prandom_u32_max(ngroups);
for (i = 0; i < ngroups; i++) {
g = (parent_group + i) % ngroups;
get_orlov_stats(sb, g, flex_size, &stats);
EXT4_GROUP_INFO_IBITMAP_CORRUPT);
goto out;
}
- inode->i_generation = prandom_u32();
+ inode->i_generation = get_random_u32();
/* Precompute checksum seed for inode metadata */
if (ext4_has_metadata_csum(sb)) {
inode->i_ctime = inode_bl->i_ctime = current_time(inode);
inode_inc_iversion(inode);
- inode->i_generation = prandom_u32();
- inode_bl->i_generation = prandom_u32();
+ inode->i_generation = get_random_u32();
+ inode_bl->i_generation = get_random_u32();
ext4_reset_inode_seed(inode);
ext4_reset_inode_seed(inode_bl);
u32 new_seq;
do {
- new_seq = prandom_u32();
+ new_seq = get_random_u32();
} while (new_seq > EXT4_MMP_SEQ_MAX);
return new_seq;
}
static int ext4_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
handle_t *handle;
struct inode *inode;
inode->i_op = &ext4_file_inode_operations;
inode->i_fop = &ext4_file_operations;
ext4_set_aops(inode);
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
err = ext4_orphan_add(handle, inode);
if (err)
goto err_unlock_inode;
ext4_journal_stop(handle);
if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
goto retry;
- return err;
+ return finish_open_simple(file, err);
err_unlock_inode:
ext4_journal_stop(handle);
unlock_new_inode(inode);
}
if (!progress) {
elr->lr_next_sched = jiffies +
- (prandom_u32()
- % (EXT4_DEF_LI_MAX_START_DELAY * HZ));
+ prandom_u32_max(EXT4_DEF_LI_MAX_START_DELAY * HZ);
}
if (time_before(elr->lr_next_sched, next_wakeup))
next_wakeup = elr->lr_next_sched;
* spread the inode table initialization requests
* better.
*/
- elr->lr_next_sched = jiffies + (prandom_u32() %
- (EXT4_DEF_LI_MAX_START_DELAY * HZ));
+ elr->lr_next_sched = jiffies + prandom_u32_max(
+ EXT4_DEF_LI_MAX_START_DELAY * HZ);
return elr;
}
pgoff_t index,
unsigned long num_ra_pages)
{
- DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
struct page *page;
index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT;
page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED);
if (!page || !PageUptodate(page)) {
+ DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
+
if (page)
put_page(page);
else if (num_ra_pages > 1)
return error;
if (error == 0)
*acl = NULL;
- if (!in_group_p(i_gid_into_mnt(mnt_userns, inode)) &&
+ if (!vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode)) &&
!capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID))
mode &= ~S_ISGID;
*mode_p = mode;
static struct kmem_cache *ino_entry_slab;
struct kmem_cache *f2fs_inode_entry_slab;
-void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io)
+void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
+ unsigned char reason)
{
f2fs_build_fault_attr(sbi, 0, 0);
set_ckpt_flags(sbi, CP_ERROR_FLAG);
- if (!end_io)
+ if (!end_io) {
f2fs_flush_merged_writes(sbi);
+
+ f2fs_handle_stop(sbi, reason);
+ }
}
/*
return ERR_PTR(err);
}
- f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE);
lock_page(page);
if (unlikely(page->mapping != mapping)) {
if (PTR_ERR(page) == -EIO &&
++count <= DEFAULT_RETRY_IO_COUNT)
goto retry;
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE);
}
return page;
}
unsigned int segno, offset;
bool exist;
- if (type != DATA_GENERIC_ENHANCE && type != DATA_GENERIC_ENHANCE_READ)
+ if (type == DATA_GENERIC)
return true;
segno = GET_SEGNO(sbi, blkaddr);
se = get_seg_entry(sbi, segno);
exist = f2fs_test_bit(offset, se->cur_valid_map);
+ if (exist && type == DATA_GENERIC_ENHANCE_UPDATE) {
+ f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
+ blkaddr, exist);
+ set_sbi_flag(sbi, SBI_NEED_FSCK);
+ return exist;
+ }
+
if (!exist && type == DATA_GENERIC_ENHANCE) {
f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
blkaddr, exist);
case DATA_GENERIC:
case DATA_GENERIC_ENHANCE:
case DATA_GENERIC_ENHANCE_READ:
+ case DATA_GENERIC_ENHANCE_UPDATE:
if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
blkaddr < MAIN_BLKADDR(sbi))) {
f2fs_warn(sbi, "access invalid blkaddr:%u",
f2fs_put_page(page, err ? 1 : 0);
if (!err)
- f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, NULL, FS_META_READ_IO,
+ F2FS_BLKSIZE);
}
out:
blk_finish_plug(&plug);
if (!folio_test_uptodate(folio))
folio_mark_uptodate(folio);
- if (!folio_test_dirty(folio)) {
- filemap_dirty_folio(mapping, folio);
+ if (filemap_dirty_folio(mapping, folio)) {
inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META);
set_page_private_reference(&folio->page);
return true;
spin_unlock(&sbi->inode_lock[type]);
}
-int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type)
+int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
+ bool from_cp)
{
struct list_head *head;
struct inode *inode;
if (inode) {
unsigned long cur_ino = inode->i_ino;
- F2FS_I(inode)->cp_task = current;
+ if (from_cp)
+ F2FS_I(inode)->cp_task = current;
+ F2FS_I(inode)->wb_task = current;
filemap_fdatawrite(inode->i_mapping);
- F2FS_I(inode)->cp_task = NULL;
+ F2FS_I(inode)->wb_task = NULL;
+ if (from_cp)
+ F2FS_I(inode)->cp_task = NULL;
iput(inode);
/* We need to give cpu to another writers. */
/* write all the dirty dentry pages */
if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
f2fs_unlock_all(sbi);
- err = f2fs_sync_dirty_inodes(sbi, DIR_INODE);
+ err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true);
if (err)
return err;
cond_resched();
void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi)
{
struct ckpt_req_control *cprc = &sbi->cprc_info;
+ struct task_struct *ckpt_task;
+
+ if (!cprc->f2fs_issue_ckpt)
+ return;
- if (cprc->f2fs_issue_ckpt) {
- struct task_struct *ckpt_task = cprc->f2fs_issue_ckpt;
+ ckpt_task = cprc->f2fs_issue_ckpt;
+ cprc->f2fs_issue_ckpt = NULL;
+ kthread_stop(ckpt_task);
- cprc->f2fs_issue_ckpt = NULL;
- kthread_stop(ckpt_task);
+ f2fs_flush_ckpt_thread(sbi);
+}
- flush_remained_ckpt_reqs(sbi, NULL);
- }
+void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi)
+{
+ struct ckpt_req_control *cprc = &sbi->cprc_info;
+
+ flush_remained_ckpt_reqs(sbi, NULL);
+
+ /* Let's wait for the previous dispatched checkpoint. */
+ while (atomic_read(&cprc->queued_ckpt))
+ io_schedule_timeout(DEFAULT_IO_TIMEOUT);
}
void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi)
if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
ret = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION);
goto out_release;
}
reason = "[C|*|C|*]";
goto out;
}
- if (compressed) {
- if (!__is_valid_data_blkaddr(blkaddr)) {
- if (!cluster_end)
- cluster_end = i;
- continue;
- }
- /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */
- if (cluster_end) {
- reason = "[C|N|N|V]";
- goto out;
- }
+ if (!__is_valid_data_blkaddr(blkaddr)) {
+ if (!cluster_end)
+ cluster_end = i;
+ continue;
+ }
+ /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */
+ if (cluster_end) {
+ reason = "[C|N|N|V]";
+ goto out;
}
}
return false;
if (f2fs_sanity_check_cluster(&dn)) {
ret = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_CLUSTER);
goto fail;
}
if (!dic->cbuf)
return -ENOMEM;
- if (cops->init_decompress_ctx) {
- int ret = cops->init_decompress_ctx(dic);
-
- if (ret)
- return ret;
- }
+ if (cops->init_decompress_ctx)
+ return cops->init_decompress_ctx(dic);
return 0;
}
void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino)
{
- struct address_space *mapping = sbi->compress_inode->i_mapping;
+ struct address_space *mapping = COMPRESS_MAPPING(sbi);
struct folio_batch fbatch;
pgoff_t index = 0;
pgoff_t end = MAX_BLKADDR(sbi);
mempool_free(page, sbi->write_io_dummy);
if (unlikely(bio->bi_status))
- f2fs_stop_checkpoint(sbi, true);
+ f2fs_stop_checkpoint(sbi, true,
+ STOP_CP_REASON_WRITE_FAIL);
continue;
}
if (unlikely(bio->bi_status)) {
mapping_set_error(page->mapping, -EIO);
if (type == F2FS_WB_CP_DATA)
- f2fs_stop_checkpoint(sbi, true);
+ f2fs_stop_checkpoint(sbi, true,
+ STOP_CP_REASON_WRITE_FAIL);
}
f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
fio->is_por ? META_POR : (__is_meta_io(fio) ?
- META_GENERIC : DATA_GENERIC_ENHANCE)))
+ META_GENERIC : DATA_GENERIC_ENHANCE))) {
+ f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
+ }
trace_f2fs_submit_page_bio(page, fio);
wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
inc_page_count(fio->sbi, is_read_io(fio->op) ?
- __read_io_type(page): WB_DATA_TYPE(fio->page));
+ __read_io_type(page) : WB_DATA_TYPE(fio->page));
__submit_bio(fio->sbi, bio, fio->type);
return 0;
fio->encrypted_page : fio->page;
if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
- __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
+ __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) {
+ f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
+ }
trace_f2fs_submit_page_bio(page, fio);
}
ClearPageError(page);
inc_page_count(sbi, F2FS_RD_DATA);
- f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE);
__submit_bio(sbi, bio, DATA);
return 0;
}
if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
DATA_GENERIC_ENHANCE_READ)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_INVALID_BLKADDR);
goto put_err;
}
goto got_it;
dn.data_blkaddr,
DATA_GENERIC_ENHANCE)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_INVALID_BLKADDR);
goto put_err;
}
got_it:
if (__is_valid_data_blkaddr(blkaddr) &&
!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto sync_out;
}
(flag != F2FS_GET_BLOCK_FIEMAP ||
IS_ENABLED(CONFIG_F2FS_CHECK_FS))) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi,
+ ERROR_CORRUPTED_CLUSTER);
goto sync_out;
}
if (flag == F2FS_GET_BLOCK_BMAP) {
err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
- if (err || err == 1)
+ if (err)
return err;
}
if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
DATA_GENERIC_ENHANCE_READ)) {
ret = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_INVALID_BLKADDR);
goto out;
}
} else {
goto submit_and_realloc;
inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
- f2fs_update_iostat(F2FS_I_SB(inode), FS_DATA_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO,
+ F2FS_BLKSIZE);
ClearPageError(page);
*last_block_in_bio = block_nr;
goto out;
refcount_inc(&dic->refcnt);
inc_page_count(sbi, F2FS_RD_DATA);
- f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
- f2fs_update_iostat(sbi, FS_CDATA_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE);
ClearPageError(page);
*last_block_in_bio = blkaddr;
}
return true;
/* if this is cold file, we should overwrite to avoid fragmentation */
- if (file_is_cold(inode))
+ if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE))
return true;
return check_inplace_update_policy(inode, fio);
fio->old_blkaddr = ei.blk + page->index - ei.fofs;
if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
- DATA_GENERIC_ENHANCE))
+ DATA_GENERIC_ENHANCE)) {
+ f2fs_handle_error(fio->sbi,
+ ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
+ }
ipu_force = true;
fio->need_lock = LOCK_DONE;
!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
DATA_GENERIC_ENHANCE)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR);
goto out_writepage;
}
}
unlock_page(page);
if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
- !F2FS_I(inode)->cp_task && allow_balance)
+ !F2FS_I(inode)->wb_task && allow_balance)
f2fs_balance_fs(sbi, need_balance_fs);
if (unlikely(f2fs_cp_error(sbi))) {
struct writeback_control *wbc)
{
/* to avoid deadlock in path of data flush */
- if (F2FS_I(inode)->cp_task)
+ if (F2FS_I(inode)->wb_task)
return false;
if (!S_ISREG(inode->i_mode))
if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
DATA_GENERIC_ENHANCE_READ)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto fail;
}
err = f2fs_submit_page_read(inode, page, blkaddr, 0, true);
folio_mark_uptodate(folio);
BUG_ON(folio_test_swapcache(folio));
- if (!folio_test_dirty(folio)) {
- filemap_dirty_folio(mapping, folio);
+ if (filemap_dirty_folio(mapping, folio)) {
f2fs_update_dirty_folio(inode, folio);
return true;
}
if (ret < 0)
return ret;
+ stat_inc_swapfile_inode(inode);
set_inode_flag(inode, FI_PIN_FILE);
f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
return ret;
{
struct inode *inode = file_inode(file);
+ stat_dec_swapfile_inode(inode);
clear_inode_flag(inode, FI_PIN_FILE);
}
#else
si->ndirty_files = sbi->ndirty_inode[FILE_INODE];
si->nquota_files = sbi->nquota_files;
si->ndirty_all = sbi->ndirty_inode[DIRTY_META];
- si->aw_cnt = sbi->atomic_files;
+ si->aw_cnt = atomic_read(&sbi->atomic_files);
si->max_aw_cnt = atomic_read(&sbi->max_aw_cnt);
si->nr_dio_read = get_pages(sbi, F2FS_DIO_READ);
si->nr_dio_write = get_pages(sbi, F2FS_DIO_WRITE);
si->inline_inode = atomic_read(&sbi->inline_inode);
si->inline_dir = atomic_read(&sbi->inline_dir);
si->compr_inode = atomic_read(&sbi->compr_inode);
+ si->swapfile_inode = atomic_read(&sbi->swapfile_inode);
si->compr_blocks = atomic64_read(&sbi->compr_blocks);
si->append = sbi->im[APPEND_INO].ino_num;
si->update = sbi->im[UPDATE_INO].ino_num;
seq_printf(s, "\n=====[ partition info(%pg). #%d, %s, CP: %s]=====\n",
si->sbi->sb->s_bdev, i++,
- f2fs_readonly(si->sbi->sb) ? "RO": "RW",
+ f2fs_readonly(si->sbi->sb) ? "RO" : "RW",
is_set_ckpt_flags(si->sbi, CP_DISABLED_FLAG) ?
"Disabled" : (f2fs_cp_error(si->sbi) ? "Error" : "Good"));
if (si->sbi->s_flag) {
si->inline_dir);
seq_printf(s, " - Compressed Inode: %u, Blocks: %llu\n",
si->compr_inode, si->compr_blocks);
+ seq_printf(s, " - Swapfile Inode: %u\n",
+ si->swapfile_inode);
seq_printf(s, " - Orphan/Append/Update Inode: %u, %u, %u\n",
si->orphans, si->append, si->update);
seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n",
atomic_set(&sbi->inline_dir, 0);
atomic_set(&sbi->compr_inode, 0);
atomic64_set(&sbi->compr_blocks, 0);
+ atomic_set(&sbi->swapfile_inode, 0);
+ atomic_set(&sbi->atomic_files, 0);
atomic_set(&sbi->inplace_count, 0);
for (i = META_CP; i < META_MAX; i++)
atomic_set(&sbi->meta_count[i], 0);
__func__, le16_to_cpu(de->name_len));
set_sbi_flag(sbi, SBI_NEED_FSCK);
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_CORRUPTED_DIRENT);
goto out;
}
if (!et)
return;
- trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, len);
+ trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, len, 0);
write_lock(&et->lock);
org_end = dei.fofs + dei.len;
f2fs_bug_on(sbi, pos >= org_end);
- if (pos > dei.fofs && pos - dei.fofs >= F2FS_MIN_EXTENT_LEN) {
+ if (pos > dei.fofs && pos - dei.fofs >= F2FS_MIN_EXTENT_LEN) {
en->ei.len = pos - en->ei.fofs;
prev_en = en;
parts = 1;
struct rb_node **insert_p = NULL, *insert_parent = NULL;
bool leftmost = false;
- trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, llen);
+ trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, llen, c_len);
/* it is safe here to check FI_NO_EXTENT w/o et->lock in ro image */
if (is_inode_flag_set(inode, FI_NO_EXTENT))
if (!f2fs_may_extent_tree(inode))
return;
- set_inode_flag(inode, FI_NO_EXTENT);
-
write_lock(&et->lock);
+ set_inode_flag(inode, FI_NO_EXTENT);
__free_extent_tree(sbi, et);
if (et->largest.len) {
et->largest.len = 0;
* condition of read on truncated area
* by extent_cache
*/
+ DATA_GENERIC_ENHANCE_UPDATE, /*
+ * strong check on range and segment
+ * bitmap for update case
+ */
META_GENERIC,
};
ORPHAN_INO, /* for orphan ino list */
APPEND_INO, /* for append ino list */
UPDATE_INO, /* for update ino list */
- TRANS_DIR_INO, /* for trasactions dir ino list */
+ TRANS_DIR_INO, /* for transactions dir ino list */
FLUSH_INO, /* for multiple device flushing */
MAX_INO_ENTRY, /* max. list */
};
unsigned int clevel; /* maximum level of given file name */
struct task_struct *task; /* lookup and create consistency */
struct task_struct *cp_task; /* separate cp/wb IO stats*/
+ struct task_struct *wb_task; /* indicate inode is in context of writeback */
nid_t i_xattr_nid; /* node id that contains xattrs */
loff_t last_disk_size; /* lastly written file size */
spinlock_t i_size_lock; /* protect last_disk_size */
APP_BUFFERED_IO, /* app buffered write IOs */
APP_WRITE_IO, /* app write IOs */
APP_MAPPED_IO, /* app mapped IOs */
+ APP_BUFFERED_CDATA_IO, /* app buffered write IOs on compressed file */
+ APP_MAPPED_CDATA_IO, /* app mapped write IOs on compressed file */
FS_DATA_IO, /* data IOs from kworker/fsync/reclaimer */
+ FS_CDATA_IO, /* data IOs from kworker/fsync/reclaimer on compressed file */
FS_NODE_IO, /* node IOs from kworker/fsync/reclaimer */
FS_META_IO, /* meta IOs from kworker/reclaimer */
FS_GC_DATA_IO, /* data IOs from forground gc */
APP_BUFFERED_READ_IO, /* app buffered read IOs */
APP_READ_IO, /* app read IOs */
APP_MAPPED_READ_IO, /* app mapped read IOs */
+ APP_BUFFERED_CDATA_READ_IO, /* app buffered read IOs on compressed file */
+ APP_MAPPED_CDATA_READ_IO, /* app mapped read IOs on compressed file */
FS_DATA_READ_IO, /* data read IOs */
FS_GDATA_READ_IO, /* data read IOs from background gc */
FS_CDATA_READ_IO, /* compressed data read IOs */
DIR_INODE, /* for dirty dir inode */
FILE_INODE, /* for dirty regular/symlink inode */
DIRTY_META, /* for all dirtied inode metadata */
- ATOMIC_FILE, /* for all atomic files */
NR_INODE_TYPE,
};
unsigned int gc_mode; /* current GC state */
unsigned int next_victim_seg[2]; /* next segment in victim section */
spinlock_t gc_urgent_high_lock;
- bool gc_urgent_high_limited; /* indicates having limited trial count */
unsigned int gc_urgent_high_remaining; /* remaining trial count for GC_URGENT_HIGH */
/* for skip statistic */
- unsigned int atomic_files; /* # of opened atomic file */
unsigned long long skipped_gc_rwsem; /* FG_GC only */
/* threshold for gc trials on pinned files */
atomic_t inline_dir; /* # of inline_dentry inodes */
atomic_t compr_inode; /* # of compressed inodes */
atomic64_t compr_blocks; /* # of compressed blocks */
+ atomic_t swapfile_inode; /* # of swapfile inodes */
+ atomic_t atomic_files; /* # of opened atomic file */
atomic_t max_aw_cnt; /* max # of atomic writes */
unsigned int io_skip_bggc; /* skip background gc for in-flight IO */
unsigned int other_skip_bggc; /* skip background gc for other reasons */
struct workqueue_struct *post_read_wq; /* post read workqueue */
+ unsigned char errors[MAX_F2FS_ERRORS]; /* error flags */
+ spinlock_t error_lock; /* protect errors array */
+ bool error_dirty; /* errors of sb is dirty */
+
struct kmem_cache *inline_xattr_slab; /* inline xattr entry */
unsigned int inline_xattr_slab_size; /* default inline xattr slab size */
if (__cp_payload(sbi) > 0) {
if (flag == NAT_BITMAP)
- return &ckpt->sit_nat_version_bitmap;
+ return tmp_ptr;
else
return (unsigned char *)ckpt + F2FS_BLKSIZE;
} else {
int f2fs_quota_sync(struct super_block *sb, int type);
loff_t max_file_blocks(struct inode *inode);
void f2fs_quota_off_umount(struct super_block *sb);
+void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason);
+void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error);
int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover);
int f2fs_sync_fs(struct super_block *sb, int sync);
int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi);
/*
* checkpoint.c
*/
-void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io);
+void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
+ unsigned char reason);
+void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index);
struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index);
int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi);
void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio);
void f2fs_remove_dirty_inode(struct inode *inode);
-int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type);
+int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
+ bool from_cp);
void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt;
unsigned int cur_ckpt_time, peak_ckpt_time;
int inline_xattr, inline_inode, inline_dir, append, update, orphans;
- int compr_inode;
+ int compr_inode, swapfile_inode;
unsigned long long compr_blocks;
int aw_cnt, max_aw_cnt;
unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
(atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks))
#define stat_sub_compr_blocks(inode, blocks) \
(atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks))
+#define stat_inc_swapfile_inode(inode) \
+ (atomic_inc(&F2FS_I_SB(inode)->swapfile_inode))
+#define stat_dec_swapfile_inode(inode) \
+ (atomic_dec(&F2FS_I_SB(inode)->swapfile_inode))
+#define stat_inc_atomic_inode(inode) \
+ (atomic_inc(&F2FS_I_SB(inode)->atomic_files))
+#define stat_dec_atomic_inode(inode) \
+ (atomic_dec(&F2FS_I_SB(inode)->atomic_files))
#define stat_inc_meta_count(sbi, blkaddr) \
do { \
if (blkaddr < SIT_I(sbi)->sit_base_addr) \
(atomic_inc(&(sbi)->inplace_count))
#define stat_update_max_atomic_write(inode) \
do { \
- int cur = F2FS_I_SB(inode)->atomic_files; \
+ int cur = atomic_read(&F2FS_I_SB(inode)->atomic_files); \
int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt); \
if (cur > max) \
atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur); \
#define stat_dec_compr_inode(inode) do { } while (0)
#define stat_add_compr_blocks(inode, blocks) do { } while (0)
#define stat_sub_compr_blocks(inode, blocks) do { } while (0)
+#define stat_inc_swapfile_inode(inode) do { } while (0)
+#define stat_dec_swapfile_inode(inode) do { } while (0)
+#define stat_inc_atomic_inode(inode) do { } while (0)
+#define stat_dec_atomic_inode(inode) do { } while (0)
#define stat_update_max_atomic_write(inode) do { } while (0)
#define stat_inc_meta_count(sbi, blkaddr) do { } while (0)
#define stat_inc_seg_type(sbi, curseg) do { } while (0)
ret = filemap_fault(vmf);
if (!ret)
- f2fs_update_iostat(F2FS_I_SB(inode), APP_MAPPED_READ_IO,
- F2FS_BLKSIZE);
+ f2fs_update_iostat(F2FS_I_SB(inode), inode,
+ APP_MAPPED_READ_IO, F2FS_BLKSIZE);
trace_f2fs_filemap_fault(inode, vmf->pgoff, (unsigned long)ret);
if (!PageUptodate(page))
SetPageUptodate(page);
- f2fs_update_iostat(sbi, APP_MAPPED_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE);
f2fs_update_time(sbi, REQ_TIME);
trace_f2fs_vm_page_mkwrite(page, DATA);
/* disallow direct IO if any of devices has unaligned blksize */
if (f2fs_is_multi_device(sbi) && !sbi->aligned_blksize)
return true;
-
+ /*
+ * for blkzoned device, fallback direct IO to buffered IO, so
+ * all IOs can be serialized by log-structured write.
+ */
+ if (f2fs_sb_has_blkzoned(sbi) && (rw == WRITE))
+ return true;
if (f2fs_lfs_mode(sbi) && rw == WRITE && F2FS_IO_ALIGNED(sbi))
return true;
if (is_sbi_flag_set(sbi, SBI_CP_DISABLED))
inode->i_ctime = attr->ia_ctime;
if (ia_valid & ATTR_MODE) {
umode_t mode = attr->ia_mode;
- kgid_t kgid = i_gid_into_mnt(mnt_userns, inode);
+ vfsgid_t vfsgid = i_gid_into_vfsgid(mnt_userns, inode);
- if (!in_group_p(kgid) && !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID))
+ if (!vfsgid_in_group_p(vfsgid) &&
+ !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID))
mode &= ~S_ISGID;
set_acl_inode(inode, mode);
}
!f2fs_is_valid_blkaddr(sbi, *blkaddr,
DATA_GENERIC_ENHANCE)) {
f2fs_put_dnode(&dn);
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
}
if (!f2fs_is_valid_blkaddr(sbi, dn->data_blkaddr,
DATA_GENERIC_ENHANCE)) {
ret = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
break;
}
}
f2fs_i_size_write(fi->cow_inode, i_size_read(inode));
- spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
- sbi->atomic_files++;
- spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
+ stat_inc_atomic_inode(inode);
set_inode_flag(inode, FI_ATOMIC_FILE);
set_inode_flag(fi->cow_inode, FI_COW_FILE);
if (ret) {
if (ret == -EROFS) {
ret = 0;
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false,
+ STOP_CP_REASON_SHUTDOWN);
set_sbi_flag(sbi, SBI_IS_SHUTDOWN);
trace_f2fs_shutdown(sbi, in, ret);
}
ret = freeze_bdev(sb->s_bdev);
if (ret)
goto out;
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
set_sbi_flag(sbi, SBI_IS_SHUTDOWN);
thaw_bdev(sb->s_bdev);
break;
ret = f2fs_sync_fs(sb, 1);
if (ret)
goto out;
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
set_sbi_flag(sbi, SBI_IS_SHUTDOWN);
break;
case F2FS_GOING_DOWN_NOSYNC:
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
set_sbi_flag(sbi, SBI_IS_SHUTDOWN);
break;
case F2FS_GOING_DOWN_METAFLUSH:
f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_META_IO);
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
set_sbi_flag(sbi, SBI_IS_SHUTDOWN);
break;
case F2FS_GOING_DOWN_NEED_FSCK:
if (!__is_valid_data_blkaddr(blkaddr))
continue;
if (unlikely(!f2fs_is_valid_blkaddr(sbi, blkaddr,
- DATA_GENERIC_ENHANCE)))
+ DATA_GENERIC_ENHANCE))) {
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
+ }
}
while (count) {
if (!__is_valid_data_blkaddr(blkaddr))
continue;
if (unlikely(!f2fs_is_valid_blkaddr(sbi, blkaddr,
- DATA_GENERIC_ENHANCE)))
+ DATA_GENERIC_ENHANCE))) {
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
+ }
}
while (count) {
DATA_GENERIC_ENHANCE)) {
ret = -EFSCORRUPTED;
f2fs_put_dnode(&dn);
+ f2fs_handle_error(sbi,
+ ERROR_INVALID_BLKADDR);
goto out;
}
dec_page_count(sbi, F2FS_DIO_READ);
if (error)
return error;
- f2fs_update_iostat(sbi, APP_DIRECT_READ_IO, size);
+ f2fs_update_iostat(sbi, NULL, APP_DIRECT_READ_IO, size);
return 0;
}
} else {
ret = filemap_read(iocb, to, 0);
if (ret > 0)
- f2fs_update_iostat(F2FS_I_SB(inode), APP_BUFFERED_READ_IO, ret);
+ f2fs_update_iostat(F2FS_I_SB(inode), inode,
+ APP_BUFFERED_READ_IO, ret);
}
if (trace_f2fs_dataread_end_enabled())
trace_f2fs_dataread_end(inode, pos, ret);
if (ret > 0) {
iocb->ki_pos += ret;
- f2fs_update_iostat(F2FS_I_SB(inode), APP_BUFFERED_IO, ret);
+ f2fs_update_iostat(F2FS_I_SB(inode), inode,
+ APP_BUFFERED_IO, ret);
}
return ret;
}
dec_page_count(sbi, F2FS_DIO_WRITE);
if (error)
return error;
- f2fs_update_iostat(sbi, APP_DIRECT_IO, size);
+ f2fs_update_iostat(sbi, NULL, APP_DIRECT_IO, size);
return 0;
}
skip_write_trace:
/* Do the actual write. */
ret = dio ?
- f2fs_dio_write_iter(iocb, from, &may_need_sync):
+ f2fs_dio_write_iter(iocb, from, &may_need_sync) :
f2fs_buffered_write_iter(iocb, from);
if (trace_f2fs_datawrite_end_enabled())
if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false,
+ STOP_CP_REASON_FAULT_INJECT);
}
if (!sb_start_write_trylock(sbi->sb)) {
*/
if (sbi->gc_mode == GC_URGENT_HIGH) {
spin_lock(&sbi->gc_urgent_high_lock);
- if (sbi->gc_urgent_high_limited) {
- if (!sbi->gc_urgent_high_remaining) {
- sbi->gc_urgent_high_limited = false;
- spin_unlock(&sbi->gc_urgent_high_lock);
- sbi->gc_mode = GC_NORMAL;
- continue;
- }
+ if (sbi->gc_urgent_high_remaining) {
sbi->gc_urgent_high_remaining--;
+ if (!sbi->gc_urgent_high_remaining)
+ sbi->gc_mode = GC_NORMAL;
}
spin_unlock(&sbi->gc_urgent_high_lock);
}
/* let's select beginning hot/small space first in no_heap mode*/
if (f2fs_need_rand_seg(sbi))
- p->offset = prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec);
+ p->offset = prandom_u32_max(MAIN_SECS(sbi) * sbi->segs_per_sec);
else if (test_opt(sbi, NOHEAP) &&
(type == CURSEG_HOT_DATA || IS_NODESEG(type)))
p->offset = 0;
{
struct page *node_page;
nid_t nid;
- unsigned int ofs_in_node;
+ unsigned int ofs_in_node, max_addrs;
block_t source_blkaddr;
nid = le32_to_cpu(sum->nid);
return false;
}
+ max_addrs = IS_INODE(node_page) ? DEF_ADDRS_PER_INODE :
+ DEF_ADDRS_PER_BLOCK;
+ if (ofs_in_node >= max_addrs) {
+ f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%u, nid:%u, max:%u",
+ ofs_in_node, dni->ino, dni->nid, max_addrs);
+ return false;
+ }
+
*nofs = ofs_of_node(node_page);
source_blkaddr = data_blkaddr(NULL, node_page, ofs_in_node);
f2fs_put_page(node_page, 1);
if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
DATA_GENERIC_ENHANCE_READ))) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto put_page;
}
goto got_it;
if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
DATA_GENERIC_ENHANCE))) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto put_page;
}
got_it:
f2fs_put_page(fio.encrypted_page, 0);
f2fs_put_page(page, 1);
- f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
- f2fs_update_iostat(sbi, FS_GDATA_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE);
return 0;
put_encrypted_page:
goto up_out;
}
- f2fs_update_iostat(fio.sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
- f2fs_update_iostat(fio.sbi, FS_GDATA_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(fio.sbi, inode, FS_DATA_READ_IO,
+ F2FS_BLKSIZE);
+ f2fs_update_iostat(fio.sbi, NULL, FS_GDATA_READ_IO,
+ F2FS_BLKSIZE);
lock_page(mpage);
if (unlikely(mpage->mapping != META_MAPPING(fio.sbi) ||
goto put_page_out;
}
- f2fs_update_iostat(fio.sbi, FS_GC_DATA_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE);
f2fs_update_data_blkaddr(&dn, newaddr);
set_inode_flag(inode, FI_APPEND_WRITE);
f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT",
segno, type, GET_SUM_TYPE((&sum->footer)));
set_sbi_flag(sbi, SBI_NEED_FSCK);
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false,
+ STOP_CP_REASON_CORRUPTED_SUMMARY);
goto skip;
}
void f2fs_do_read_inline_data(struct page *page, struct page *ipage)
{
struct inode *inode = page->mapping->host;
- void *src_addr, *dst_addr;
if (PageUptodate(page))
return;
zero_user_segment(page, MAX_INLINE_DATA(inode), PAGE_SIZE);
/* Copy the whole inline data block */
- src_addr = inline_data_addr(inode, ipage);
- dst_addr = kmap_atomic(page);
- memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode));
- flush_dcache_page(page);
- kunmap_atomic(dst_addr);
+ memcpy_to_page(page, 0, inline_data_addr(inode, ipage),
+ MAX_INLINE_DATA(inode));
if (!PageUptodate(page))
SetPageUptodate(page);
}
set_sbi_flag(fio.sbi, SBI_NEED_FSCK);
f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.",
__func__, dn->inode->i_ino, dn->data_blkaddr);
+ f2fs_handle_error(fio.sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
}
int f2fs_write_inline_data(struct inode *inode, struct page *page)
{
- void *src_addr, *dst_addr;
struct dnode_of_data dn;
int err;
f2fs_bug_on(F2FS_I_SB(inode), page->index);
f2fs_wait_on_page_writeback(dn.inode_page, NODE, true, true);
- src_addr = kmap_atomic(page);
- dst_addr = inline_data_addr(inode, dn.inode_page);
- memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode));
- kunmap_atomic(src_addr);
+ memcpy_from_page(inline_data_addr(inode, dn.inode_page),
+ page, 0, MAX_INLINE_DATA(inode));
set_page_dirty(dn.inode_page);
f2fs_clear_page_cache_dirty_tag(page);
set_sbi_flag(F2FS_P_SB(page), SBI_NEED_FSCK);
f2fs_warn(F2FS_P_SB(page), "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.",
__func__, dir->i_ino, dn.data_blkaddr);
+ f2fs_handle_error(F2FS_P_SB(page), ERROR_INVALID_BLKADDR);
err = -EFSCORRUPTED;
goto out;
}
if (!__is_valid_data_blkaddr(addr))
return 1;
- if (!f2fs_is_valid_blkaddr(sbi, addr, DATA_GENERIC_ENHANCE))
+ if (!f2fs_is_valid_blkaddr(sbi, addr, DATA_GENERIC_ENHANCE)) {
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
return -EFSCORRUPTED;
+ }
return 0;
}
return true;
}
+static void init_idisk_time(struct inode *inode)
+{
+ struct f2fs_inode_info *fi = F2FS_I(inode);
+
+ fi->i_disk_time[0] = inode->i_atime;
+ fi->i_disk_time[1] = inode->i_ctime;
+ fi->i_disk_time[2] = inode->i_mtime;
+ fi->i_disk_time[3] = fi->i_crtime;
+}
+
static int do_read_inode(struct inode *inode)
{
struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
if (!sanity_check_inode(inode, node_page)) {
f2fs_put_page(node_page, 1);
+ f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
return -EFSCORRUPTED;
}
}
}
- fi->i_disk_time[0] = inode->i_atime;
- fi->i_disk_time[1] = inode->i_ctime;
- fi->i_disk_time[2] = inode->i_mtime;
- fi->i_disk_time[3] = fi->i_crtime;
+ init_idisk_time(inode);
f2fs_put_page(node_page, 1);
stat_inc_inline_xattr(inode);
return 0;
}
+static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino)
+{
+ return ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi) ||
+ ino == F2FS_COMPRESS_INO(sbi);
+}
+
struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
{
struct f2fs_sb_info *sbi = F2FS_SB(sb);
return ERR_PTR(-ENOMEM);
if (!(inode->i_state & I_NEW)) {
+ if (is_meta_ino(sbi, ino)) {
+ f2fs_err(sbi, "inaccessible inode: %lu, run fsck to repair", ino);
+ set_sbi_flag(sbi, SBI_NEED_FSCK);
+ ret = -EFSCORRUPTED;
+ trace_f2fs_iget_exit(inode, ret);
+ iput(inode);
+ f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
+ return ERR_PTR(ret);
+ }
+
trace_f2fs_iget(inode);
return inode;
}
- if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi))
- goto make_now;
-#ifdef CONFIG_F2FS_FS_COMPRESSION
- if (ino == F2FS_COMPRESS_INO(sbi))
+ if (is_meta_ino(sbi, ino))
goto make_now;
-#endif
ret = do_read_inode(inode);
if (ret)
if (inode->i_nlink == 0)
clear_page_private_inline(node_page);
- F2FS_I(inode)->i_disk_time[0] = inode->i_atime;
- F2FS_I(inode)->i_disk_time[1] = inode->i_ctime;
- F2FS_I(inode)->i_disk_time[2] = inode->i_mtime;
- F2FS_I(inode)->i_disk_time[3] = F2FS_I(inode)->i_crtime;
-
+ init_idisk_time(inode);
#ifdef CONFIG_F2FS_CHECK_FS
f2fs_inode_chksum_set(F2FS_I_SB(inode), node_page);
#endif
cond_resched();
goto retry;
} else if (err != -ENOENT) {
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false,
+ STOP_CP_REASON_UPDATE_INODE);
}
return;
}
/* print app write IOs */
seq_puts(seq, "[WRITE]\n");
- seq_printf(seq, "app buffered: %-16llu\n",
+ seq_printf(seq, "app buffered data: %-16llu\n",
sbi->rw_iostat[APP_BUFFERED_IO]);
- seq_printf(seq, "app direct: %-16llu\n",
+ seq_printf(seq, "app direct data: %-16llu\n",
sbi->rw_iostat[APP_DIRECT_IO]);
- seq_printf(seq, "app mapped: %-16llu\n",
+ seq_printf(seq, "app mapped data: %-16llu\n",
sbi->rw_iostat[APP_MAPPED_IO]);
+ seq_printf(seq, "app buffered cdata: %-16llu\n",
+ sbi->rw_iostat[APP_BUFFERED_CDATA_IO]);
+ seq_printf(seq, "app mapped cdata: %-16llu\n",
+ sbi->rw_iostat[APP_MAPPED_CDATA_IO]);
/* print fs write IOs */
- seq_printf(seq, "fs data: %-16llu\n",
+ seq_printf(seq, "fs data: %-16llu\n",
sbi->rw_iostat[FS_DATA_IO]);
- seq_printf(seq, "fs node: %-16llu\n",
+ seq_printf(seq, "fs cdata: %-16llu\n",
+ sbi->rw_iostat[FS_CDATA_IO]);
+ seq_printf(seq, "fs node: %-16llu\n",
sbi->rw_iostat[FS_NODE_IO]);
- seq_printf(seq, "fs meta: %-16llu\n",
+ seq_printf(seq, "fs meta: %-16llu\n",
sbi->rw_iostat[FS_META_IO]);
- seq_printf(seq, "fs gc data: %-16llu\n",
+ seq_printf(seq, "fs gc data: %-16llu\n",
sbi->rw_iostat[FS_GC_DATA_IO]);
- seq_printf(seq, "fs gc node: %-16llu\n",
+ seq_printf(seq, "fs gc node: %-16llu\n",
sbi->rw_iostat[FS_GC_NODE_IO]);
- seq_printf(seq, "fs cp data: %-16llu\n",
+ seq_printf(seq, "fs cp data: %-16llu\n",
sbi->rw_iostat[FS_CP_DATA_IO]);
- seq_printf(seq, "fs cp node: %-16llu\n",
+ seq_printf(seq, "fs cp node: %-16llu\n",
sbi->rw_iostat[FS_CP_NODE_IO]);
- seq_printf(seq, "fs cp meta: %-16llu\n",
+ seq_printf(seq, "fs cp meta: %-16llu\n",
sbi->rw_iostat[FS_CP_META_IO]);
/* print app read IOs */
seq_puts(seq, "[READ]\n");
- seq_printf(seq, "app buffered: %-16llu\n",
+ seq_printf(seq, "app buffered data: %-16llu\n",
sbi->rw_iostat[APP_BUFFERED_READ_IO]);
- seq_printf(seq, "app direct: %-16llu\n",
+ seq_printf(seq, "app direct data: %-16llu\n",
sbi->rw_iostat[APP_DIRECT_READ_IO]);
- seq_printf(seq, "app mapped: %-16llu\n",
+ seq_printf(seq, "app mapped data: %-16llu\n",
sbi->rw_iostat[APP_MAPPED_READ_IO]);
+ seq_printf(seq, "app buffered cdata: %-16llu\n",
+ sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO]);
+ seq_printf(seq, "app mapped cdata: %-16llu\n",
+ sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO]);
/* print fs read IOs */
- seq_printf(seq, "fs data: %-16llu\n",
+ seq_printf(seq, "fs data: %-16llu\n",
sbi->rw_iostat[FS_DATA_READ_IO]);
- seq_printf(seq, "fs gc data: %-16llu\n",
+ seq_printf(seq, "fs gc data: %-16llu\n",
sbi->rw_iostat[FS_GDATA_READ_IO]);
- seq_printf(seq, "fs compr_data: %-16llu\n",
+ seq_printf(seq, "fs cdata: %-16llu\n",
sbi->rw_iostat[FS_CDATA_READ_IO]);
- seq_printf(seq, "fs node: %-16llu\n",
+ seq_printf(seq, "fs node: %-16llu\n",
sbi->rw_iostat[FS_NODE_READ_IO]);
- seq_printf(seq, "fs meta: %-16llu\n",
+ seq_printf(seq, "fs meta: %-16llu\n",
sbi->rw_iostat[FS_META_READ_IO]);
/* print other IOs */
seq_puts(seq, "[OTHER]\n");
- seq_printf(seq, "fs discard: %-16llu\n",
+ seq_printf(seq, "fs discard: %-16llu\n",
sbi->rw_iostat[FS_DISCARD]);
return 0;
spin_unlock_irq(&sbi->iostat_lat_lock);
}
-void f2fs_update_iostat(struct f2fs_sb_info *sbi,
+void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,
enum iostat_type type, unsigned long long io_bytes)
{
unsigned long flags;
if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO)
sbi->rw_iostat[APP_READ_IO] += io_bytes;
+#ifdef CONFIG_F2FS_FS_COMPRESSION
+ if (inode && f2fs_compressed_file(inode)) {
+ if (type == APP_BUFFERED_IO)
+ sbi->rw_iostat[APP_BUFFERED_CDATA_IO] += io_bytes;
+
+ if (type == APP_BUFFERED_READ_IO)
+ sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO] += io_bytes;
+
+ if (type == APP_MAPPED_READ_IO)
+ sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO] += io_bytes;
+
+ if (type == APP_MAPPED_IO)
+ sbi->rw_iostat[APP_MAPPED_CDATA_IO] += io_bytes;
+
+ if (type == FS_DATA_READ_IO)
+ sbi->rw_iostat[FS_CDATA_READ_IO] += io_bytes;
+
+ if (type == FS_DATA_IO)
+ sbi->rw_iostat[FS_CDATA_IO] += io_bytes;
+ }
+#endif
+
spin_unlock_irqrestore(&sbi->iostat_lock, flags);
f2fs_record_iostat(sbi);
extern int __maybe_unused iostat_info_seq_show(struct seq_file *seq,
void *offset);
extern void f2fs_reset_iostat(struct f2fs_sb_info *sbi);
-extern void f2fs_update_iostat(struct f2fs_sb_info *sbi,
+extern void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,
enum iostat_type type, unsigned long long io_bytes);
struct bio_iostat_ctx {
extern int f2fs_init_iostat(struct f2fs_sb_info *sbi);
extern void f2fs_destroy_iostat(struct f2fs_sb_info *sbi);
#else
-static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi,
+static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,
enum iostat_type type, unsigned long long io_bytes) {}
static inline void iostat_update_and_unbind_ctx(struct bio *bio, int rw) {}
static inline void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
inode->i_blocks = 0;
inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode);
F2FS_I(inode)->i_crtime = inode->i_mtime;
- inode->i_generation = prandom_u32();
+ inode->i_generation = get_random_u32();
if (S_ISDIR(inode->i_mode))
F2FS_I(inode)->i_current_depth = 1;
}
static int __f2fs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode, bool is_whiteout,
+ struct file *file, umode_t mode, bool is_whiteout,
struct inode **new_inode)
{
struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
inode->i_state |= I_LINKABLE;
spin_unlock(&inode->i_lock);
} else {
- if (dentry)
- d_tmpfile(dentry, inode);
+ if (file)
+ d_tmpfile(file, inode);
else
f2fs_i_links_write(inode, false);
}
}
static int f2fs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
+ int err;
if (unlikely(f2fs_cp_error(sbi)))
return -EIO;
if (!f2fs_is_checkpoint_ready(sbi))
return -ENOSPC;
- return __f2fs_tmpfile(mnt_userns, dir, dentry, mode, false, NULL);
+ err = __f2fs_tmpfile(mnt_userns, dir, file, mode, false, NULL);
+
+ return finish_open_simple(file, err);
}
static int f2fs_create_whiteout(struct user_namespace *mnt_userns,
set_sbi_flag(sbi, SBI_NEED_FSCK);
f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.",
__func__, nid);
+ f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
return -EFSCORRUPTED;
}
return 0;
ne = nat_in_journal(journal, i);
node_info_from_raw_nat(ni, &ne);
}
- up_read(&curseg->journal_rwsem);
+ up_read(&curseg->journal_rwsem);
if (i >= 0) {
f2fs_up_read(&nm_i->nat_tree_lock);
goto cache;
if (unlikely(new_ni.blk_addr != NULL_ADDR)) {
err = -EFSCORRUPTED;
set_sbi_flag(sbi, SBI_NEED_FSCK);
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto fail;
}
#endif
err = f2fs_submit_page_bio(&fio);
if (!err)
- f2fs_update_iostat(sbi, FS_NODE_READ_IO, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE);
return err;
}
if (IS_INODE(&folio->page))
f2fs_inode_chksum_set(F2FS_M_SB(mapping), &folio->page);
#endif
- if (!folio_test_dirty(folio)) {
- filemap_dirty_folio(mapping, folio);
+ if (filemap_dirty_folio(mapping, folio)) {
inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES);
set_page_private_reference(&folio->page);
return true;
struct dnode_of_data tdn = *dn;
nid_t ino, nid;
struct inode *inode;
- unsigned int offset;
+ unsigned int offset, ofs_in_node, max_addrs;
block_t bidx;
int i;
got_it:
/* Use the locked dnode page and inode */
nid = le32_to_cpu(sum.nid);
+ ofs_in_node = le16_to_cpu(sum.ofs_in_node);
+
+ max_addrs = ADDRS_PER_PAGE(dn->node_page, dn->inode);
+ if (ofs_in_node >= max_addrs) {
+ f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%lu, nid:%u, max:%u",
+ ofs_in_node, dn->inode->i_ino, nid, max_addrs);
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUMMARY);
+ return -EFSCORRUPTED;
+ }
+
if (dn->inode->i_ino == nid) {
tdn.nid = nid;
if (!dn->inode_page_locked)
lock_page(dn->inode_page);
tdn.node_page = dn->inode_page;
- tdn.ofs_in_node = le16_to_cpu(sum.ofs_in_node);
+ tdn.ofs_in_node = ofs_in_node;
goto truncate_out;
} else if (dn->nid == nid) {
- tdn.ofs_in_node = le16_to_cpu(sum.ofs_in_node);
+ tdn.ofs_in_node = ofs_in_node;
goto truncate_out;
}
inode->i_ino, ofs_of_node(dn.node_page),
ofs_of_node(page));
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER);
goto err;
}
if (__is_valid_data_blkaddr(src) &&
!f2fs_is_valid_blkaddr(sbi, src, META_POR)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto err;
}
if (__is_valid_data_blkaddr(dest) &&
!f2fs_is_valid_blkaddr(sbi, dest, META_POR)) {
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
goto err;
}
goto err;
}
+ if (f2fs_is_valid_blkaddr(sbi, dest,
+ DATA_GENERIC_ENHANCE_UPDATE)) {
+ f2fs_err(sbi, "Inconsistent dest blkaddr:%u, ino:%lu, ofs:%u",
+ dest, inode->i_ino, dn.ofs_in_node);
+ err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi,
+ ERROR_INVALID_BLKADDR);
+ goto err;
+ }
+
/* write dummy data page */
f2fs_replace_block(sbi, &dn, src, dest,
ni.version, false, false);
void f2fs_abort_atomic_write(struct inode *inode, bool clean)
{
- struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
struct f2fs_inode_info *fi = F2FS_I(inode);
if (!f2fs_is_atomic_file(inode))
fi->cow_inode = NULL;
release_atomic_write_cnt(inode);
clear_inode_flag(inode, FI_ATOMIC_FILE);
-
- spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
- sbi->atomic_files--;
- spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
+ stat_dec_atomic_inode(inode);
}
static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
DATA_GENERIC_ENHANCE)) {
f2fs_put_dnode(&dn);
ret = -EFSCORRUPTED;
+ f2fs_handle_error(sbi,
+ ERROR_INVALID_BLKADDR);
goto out;
}
{
if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
}
/* balance_fs_bg is able to be pending */
mutex_lock(&sbi->flush_lock);
blk_start_plug(&plug);
- f2fs_sync_dirty_inodes(sbi, FILE_INODE);
+ f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
blk_finish_plug(&plug);
mutex_unlock(&sbi->flush_lock);
}
- f2fs_sync_fs(sbi->sb, true);
+ f2fs_sync_fs(sbi->sb, 1);
stat_inc_bg_cp_count(sbi->stat_info);
}
} while (ret && --count);
if (ret) {
- f2fs_stop_checkpoint(sbi, false);
+ f2fs_stop_checkpoint(sbi, false,
+ STOP_CP_REASON_FLUSH_FAIL);
break;
}
atomic_inc(&dcc->issued_discard);
- f2fs_update_iostat(sbi, FS_DISCARD, 1);
+ f2fs_update_iostat(sbi, NULL, FS_DISCARD, 1);
lstart += len;
start += len;
sanity_check_seg_type(sbi, seg_type);
if (f2fs_need_rand_seg(sbi))
- return prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec);
+ return prandom_u32_max(MAIN_SECS(sbi) * sbi->segs_per_sec);
/* if segs_per_sec is large than 1, we need to keep original policy. */
if (__is_large_section(sbi))
curseg->alloc_type = LFS;
if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
curseg->fragment_remained_chunk =
- prandom_u32() % sbi->max_fragment_chunk + 1;
+ prandom_u32_max(sbi->max_fragment_chunk) + 1;
}
static int __next_free_blkoff(struct f2fs_sb_info *sbi,
/* To allocate block chunks in different sizes, use random number */
if (--seg->fragment_remained_chunk <= 0) {
seg->fragment_remained_chunk =
- prandom_u32() % sbi->max_fragment_chunk + 1;
+ prandom_u32_max(sbi->max_fragment_chunk) + 1;
seg->next_blkoff +=
- prandom_u32() % sbi->max_fragment_hole + 1;
+ prandom_u32_max(sbi->max_fragment_hole) + 1;
}
}
}
f2fs_submit_page_write(&fio);
stat_inc_meta_count(sbi, page->index);
- f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
}
void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
set_summary(&sum, nid, 0, 0);
do_write_page(&sum, fio);
- f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
+ f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
}
void f2fs_outplace_write_data(struct dnode_of_data *dn,
do_write_page(&sum, fio);
f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
- f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
+ f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
}
int f2fs_inplace_write_data(struct f2fs_io_info *fio)
f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
__func__, segno);
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
goto drop_bio;
}
if (!err) {
f2fs_update_device_state(fio->sbi, fio->ino,
fio->new_blkaddr, 1);
- f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
+ f2fs_update_iostat(fio->sbi, fio->page->mapping->host,
+ fio->io_type, F2FS_BLKSIZE);
}
return err;
if (se->type >= NR_PERSISTENT_LOG) {
f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
se->type, start);
+ f2fs_handle_error(sbi,
+ ERROR_INCONSISTENT_SUM_TYPE);
return -EFSCORRUPTED;
}
f2fs_err(sbi, "Wrong journal entry on segno %u",
start);
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);
break;
}
f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
se->type, start);
err = -EFSCORRUPTED;
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
break;
}
if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
sit_valid_blocks[NODE], valid_node_count(sbi));
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);
return -EFSCORRUPTED;
}
f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
sit_valid_blocks[DATA], sit_valid_blocks[NODE],
valid_user_blocks(sbi));
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);
return -EFSCORRUPTED;
}
f2fs_err(sbi,
"Current segment has invalid alloc_type:%d",
curseg->alloc_type);
+ f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
return -EFSCORRUPTED;
}
"Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
i, curseg->segno, curseg->alloc_type,
curseg->next_blkoff, blkofs);
+ f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
return -EFSCORRUPTED;
}
}
f2fs_err(sbi, "Mismatch valid blocks %d vs. %d",
GET_SIT_VBLOCKS(raw_sit), valid_blocks);
set_sbi_flag(sbi, SBI_NEED_FSCK);
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_SIT);
return -EFSCORRUPTED;
}
f2fs_err(sbi, "Wrong valid blocks %d or segno %u",
GET_SIT_VBLOCKS(raw_sit), segno);
set_sbi_flag(sbi, SBI_NEED_FSCK);
+ f2fs_handle_error(sbi, ERROR_INCONSISTENT_SIT);
return -EFSCORRUPTED;
}
return 0;
static inline void limit_reserve_root(struct f2fs_sb_info *sbi)
{
- block_t limit = min((sbi->user_block_count << 1) / 1000,
+ block_t limit = min((sbi->user_block_count >> 3),
sbi->user_block_count - sbi->reserved_blocks);
- /* limit is 0.2% */
+ /* limit is 12.5% */
if (test_opt(sbi, RESERVE_ROOT) &&
F2FS_OPTION(sbi).root_reserved_blocks > limit) {
F2FS_OPTION(sbi).root_reserved_blocks = limit;
return -EINVAL;
}
+ if (test_opt(sbi, ATGC) && f2fs_lfs_mode(sbi)) {
+ f2fs_err(sbi, "LFS not compatible with ATGC");
+ return -EINVAL;
+ }
+
if (f2fs_sb_has_readonly(sbi) && !f2fs_readonly(sbi->sb)) {
f2fs_err(sbi, "Allow to mount readonly mode only");
return -EROFS;
if (is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY))
return -EINVAL;
- /* ensure no checkpoint required */
- if (!llist_empty(&F2FS_SB(sb)->cprc_info.issue_list))
- return -EINVAL;
+ /* Let's flush checkpoints and stop the thread. */
+ f2fs_flush_ckpt_thread(F2FS_SB(sb));
/* to avoid deadlock on f2fs_evict_inode->SB_FREEZE_FS */
set_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING);
f2fs_up_write(&sbi->gc_lock);
f2fs_sync_fs(sbi->sb, 1);
+
+ /* Let's ensure there's no pending checkpoint anymore */
+ f2fs_flush_ckpt_thread(sbi);
}
static int f2fs_remount(struct super_block *sb, int *flags, char *data)
f2fs_stop_ckpt_thread(sbi);
need_restart_ckpt = true;
} else {
+ /* Flush if the prevous checkpoint, if exists. */
+ f2fs_flush_ckpt_thread(sbi);
+
err = f2fs_start_ckpt_thread(sbi);
if (err) {
f2fs_err(sbi,
size_t toread;
loff_t i_size = i_size_read(inode);
struct page *page;
- char *kaddr;
if (off > i_size)
return 0;
return -EIO;
}
- kaddr = kmap_atomic(page);
- memcpy(data, kaddr + offset, tocopy);
- kunmap_atomic(kaddr);
+ memcpy_from_page(data, page, offset, tocopy);
f2fs_put_page(page, 1);
offset = 0;
size_t towrite = len;
struct page *page;
void *fsdata = NULL;
- char *kaddr;
int err = 0;
int tocopy;
break;
}
- kaddr = kmap_atomic(page);
- memcpy(kaddr + offset, data, tocopy);
- kunmap_atomic(kaddr);
- flush_dcache_page(page);
+ memcpy_to_page(page, offset, data, tocopy);
a_ops->write_end(NULL, mapping, off, tocopy, tocopy,
page, fsdata);
return err;
}
+void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason)
+{
+ struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
+ int err;
+
+ f2fs_down_write(&sbi->sb_lock);
+
+ if (raw_super->s_stop_reason[reason] < ((1 << BITS_PER_BYTE) - 1))
+ raw_super->s_stop_reason[reason]++;
+
+ err = f2fs_commit_super(sbi, false);
+ if (err)
+ f2fs_err(sbi, "f2fs_commit_super fails to record reason:%u err:%d",
+ reason, err);
+ f2fs_up_write(&sbi->sb_lock);
+}
+
+static void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag)
+{
+ spin_lock(&sbi->error_lock);
+ if (!test_bit(flag, (unsigned long *)sbi->errors)) {
+ set_bit(flag, (unsigned long *)sbi->errors);
+ sbi->error_dirty = true;
+ }
+ spin_unlock(&sbi->error_lock);
+}
+
+static bool f2fs_update_errors(struct f2fs_sb_info *sbi)
+{
+ bool need_update = false;
+
+ spin_lock(&sbi->error_lock);
+ if (sbi->error_dirty) {
+ memcpy(F2FS_RAW_SUPER(sbi)->s_errors, sbi->errors,
+ MAX_F2FS_ERRORS);
+ sbi->error_dirty = false;
+ need_update = true;
+ }
+ spin_unlock(&sbi->error_lock);
+
+ return need_update;
+}
+
+void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error)
+{
+ int err;
+
+ f2fs_save_errors(sbi, error);
+
+ f2fs_down_write(&sbi->sb_lock);
+
+ if (!f2fs_update_errors(sbi))
+ goto out_unlock;
+
+ err = f2fs_commit_super(sbi, false);
+ if (err)
+ f2fs_err(sbi, "f2fs_commit_super fails to record errors:%u, err:%d",
+ error, err);
+out_unlock:
+ f2fs_up_write(&sbi->sb_lock);
+}
+
static int f2fs_scan_devices(struct f2fs_sb_info *sbi)
{
struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
goto free_devices;
}
+ spin_lock_init(&sbi->error_lock);
+ memcpy(sbi->errors, raw_super->s_errors, MAX_F2FS_ERRORS);
+
sbi->total_valid_node_count =
le32_to_cpu(sbi->ckpt->valid_node_count);
percpu_counter_set(&sbi->total_valid_inode_count,
return sprintf(buf, "%lx\n", sbi->s_flag);
}
+static ssize_t cp_status_show(struct f2fs_attr *a,
+ struct f2fs_sb_info *sbi, char *buf)
+{
+ return sprintf(buf, "%x\n", le32_to_cpu(F2FS_CKPT(sbi)->ckpt_flags));
+}
+
static ssize_t pending_discard_show(struct f2fs_attr *a,
struct f2fs_sb_info *sbi, char *buf)
{
if (!strcmp(a->attr.name, "gc_urgent_high_remaining")) {
spin_lock(&sbi->gc_urgent_high_lock);
- sbi->gc_urgent_high_limited = t != 0;
sbi->gc_urgent_high_remaining = t;
spin_unlock(&sbi->gc_urgent_high_lock);
ATTRIBUTE_GROUPS(f2fs_feat);
F2FS_GENERAL_RO_ATTR(sb_status);
+F2FS_GENERAL_RO_ATTR(cp_status);
static struct attribute *f2fs_stat_attrs[] = {
ATTR_LIST(sb_status),
+ ATTR_LIST(cp_status),
NULL,
};
ATTRIBUTE_GROUPS(f2fs_stat);
size_t n = min_t(size_t, count,
PAGE_SIZE - offset_in_page(pos));
struct page *page;
- void *addr;
page = read_mapping_page(inode->i_mapping, pos >> PAGE_SHIFT,
NULL);
if (IS_ERR(page))
return PTR_ERR(page);
- addr = kmap_atomic(page);
- memcpy(buf, addr + offset_in_page(pos), n);
- kunmap_atomic(addr);
+ memcpy_from_page(buf, page, offset_in_page(pos), n);
put_page(page);
PAGE_SIZE - offset_in_page(pos));
struct page *page;
void *fsdata;
- void *addr;
int res;
res = aops->write_begin(NULL, mapping, pos, n, &page, &fsdata);
if (res)
return res;
- addr = kmap_atomic(page);
- memcpy(addr + offset_in_page(pos), buf, n);
- kunmap_atomic(addr);
+ memcpy_to_page(page, offset_in_page(pos), buf, n);
res = aops->write_end(NULL, mapping, pos, n, n, page, fsdata);
if (res < 0)
if (pos + size < pos || pos + size > inode->i_sb->s_maxbytes ||
pos < f2fs_verity_metadata_pos(inode) || size > INT_MAX) {
f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr");
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_CORRUPTED_VERITY_XATTR);
return -EFSCORRUPTED;
}
if (buf_size) {
pgoff_t index,
unsigned long num_ra_pages)
{
- DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
struct page *page;
index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT;
page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED);
if (!page || !PageUptodate(page)) {
+ DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index);
+
if (page)
put_page(page);
else if (num_ra_pages > 1)
inode->i_ino);
set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
err = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_CORRUPTED_XATTR);
goto out;
}
check:
inode->i_ino);
set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
error = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_CORRUPTED_XATTR);
goto cleanup;
}
inode->i_ino);
set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
error = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_CORRUPTED_XATTR);
goto exit;
}
inode->i_ino, ENTRY_SIZE(last));
set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
error = -EFSCORRUPTED;
+ f2fs_handle_error(F2FS_I_SB(inode),
+ ERROR_CORRUPTED_XATTR);
goto exit;
}
last = XATTR_NEXT_ENTRY(last);
inode->i_uid = sbi->options.fs_uid;
inode->i_gid = sbi->options.fs_gid;
inode_inc_iversion(inode);
- inode->i_generation = prandom_u32();
+ inode->i_generation = get_random_u32();
if ((de->attr & ATTR_DIR) && !IS_FREE(de->name)) {
inode->i_generation &= ~1;
*ret_fd = fd;
return file;
}
+EXPORT_SYMBOL(task_lookup_next_fd_rcu);
/*
* Lightweight file lookup - no refcnt increment if fd table isn't shared.
1 << PG_active |
1 << PG_workingset |
1 << PG_reclaim |
- 1 << PG_waiters))) {
+ 1 << PG_waiters |
+ LRU_GEN_MASK | LRU_REFS_MASK))) {
dump_page(page, "fuse: trying to steal weird page");
return 1;
}
*/
static int fuse_create_open(struct inode *dir, struct dentry *entry,
struct file *file, unsigned int flags,
- umode_t mode)
+ umode_t mode, u32 opcode)
{
int err;
struct inode *inode;
inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID;
}
- args.opcode = FUSE_CREATE;
+ args.opcode = opcode;
args.nodeid = get_node_id(dir);
args.in_numargs = 2;
args.in_args[0].size = sizeof(inarg);
if (fc->no_create)
goto mknod;
- err = fuse_create_open(dir, entry, file, flags, mode);
+ err = fuse_create_open(dir, entry, file, flags, mode, FUSE_CREATE);
if (err == -ENOSYS) {
fc->no_create = 1;
goto mknod;
return fuse_mknod(&init_user_ns, dir, entry, mode, 0);
}
+static int fuse_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
+ struct file *file, umode_t mode)
+{
+ struct fuse_conn *fc = get_fuse_conn(dir);
+ int err;
+
+ if (fc->no_tmpfile)
+ return -EOPNOTSUPP;
+
+ err = fuse_create_open(dir, file->f_path.dentry, file, file->f_flags, mode, FUSE_TMPFILE);
+ if (err == -ENOSYS) {
+ fc->no_tmpfile = 1;
+ err = -EOPNOTSUPP;
+ }
+ return err;
+}
+
static int fuse_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
struct dentry *entry, umode_t mode)
{
.setattr = fuse_setattr,
.create = fuse_create,
.atomic_open = fuse_atomic_open,
+ .tmpfile = fuse_tmpfile,
.mknod = fuse_mknod,
.permission = fuse_permission,
.getattr = fuse_getattr,
/* Does the filesystem support per inode DAX? */
unsigned int inode_dax:1;
+ /* Is tmpfile not implemented by fs? */
+ unsigned int no_tmpfile:1;
+
/** The number of requests waiting for completion */
atomic_t num_waiting;
return dlm_posix_lock(ls->ls_dlm, ip->i_no_addr, file, cmd, fl);
}
+static void __flock_holder_uninit(struct file *file, struct gfs2_holder *fl_gh)
+{
+ struct gfs2_glock *gl = fl_gh->gh_gl;
+
+ /*
+ * Make sure gfs2_glock_put() won't sleep under the file->f_lock
+ * spinlock.
+ */
+
+ gfs2_glock_hold(gl);
+ spin_lock(&file->f_lock);
+ gfs2_holder_uninit(fl_gh);
+ spin_unlock(&file->f_lock);
+ gfs2_glock_put(gl);
+}
+
static int do_flock(struct file *file, int cmd, struct file_lock *fl)
{
struct gfs2_file *fp = file->private_data;
int sleeptime;
state = (fl->fl_type == F_WRLCK) ? LM_ST_EXCLUSIVE : LM_ST_SHARED;
- flags = (IS_SETLKW(cmd) ? 0 : LM_FLAG_TRY_1CB) | GL_EXACT;
+ flags = GL_EXACT | GL_NOPID;
+ if (!IS_SETLKW(cmd))
+ flags |= LM_FLAG_TRY_1CB;
mutex_lock(&fp->f_fl_mutex);
&gfs2_flock_glops, CREATE, &gl);
if (error)
goto out;
+ spin_lock(&file->f_lock);
gfs2_holder_init(gl, state, flags, fl_gh);
+ spin_unlock(&file->f_lock);
gfs2_glock_put(gl);
}
for (sleeptime = 1; sleeptime <= 4; sleeptime <<= 1) {
error = gfs2_glock_nq(fl_gh);
if (error != GLR_TRYFAILED)
break;
- fl_gh->gh_flags = LM_FLAG_TRY | GL_EXACT;
+ fl_gh->gh_flags &= ~LM_FLAG_TRY_1CB;
+ fl_gh->gh_flags |= LM_FLAG_TRY;
msleep(sleeptime);
}
if (error) {
- gfs2_holder_uninit(fl_gh);
+ __flock_holder_uninit(file, fl_gh);
if (error == GLR_TRYFAILED)
error = -EAGAIN;
} else {
locks_lock_file_wait(file, fl);
if (gfs2_holder_initialized(fl_gh)) {
gfs2_glock_dq(fl_gh);
- gfs2_holder_uninit(fl_gh);
+ __flock_holder_uninit(file, fl_gh);
}
mutex_unlock(&fp->f_fl_mutex);
}
#include <linux/list_sort.h>
#include <linux/lockref.h>
#include <linux/rhashtable.h>
+#include <linux/pid_namespace.h>
+#include <linux/fdtable.h>
+#include <linux/file.h>
#include "gfs2.h"
#include "incore.h"
static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
static void __gfs2_glock_dq(struct gfs2_holder *gh);
+static void handle_callback(struct gfs2_glock *gl, unsigned int state,
+ unsigned long delay, bool remote);
static struct dentry *gfs2_root;
static struct workqueue_struct *glock_workqueue;
*
*/
-static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
+static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh,
+ unsigned int target)
__releases(&gl->gl_lockref.lock)
__acquires(&gl->gl_lockref.lock)
{
if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
gh && !(gh->gh_flags & LM_FLAG_NOEXP))
- return;
+ goto skip_inval;
+
lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
LM_FLAG_PRIORITY);
GLOCK_BUG_ON(gl, gl->gl_state == target);
(target != LM_ST_UNLOCKED ||
test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) {
if (!is_system_glock(gl)) {
+ handle_callback(gl, LM_ST_UNLOCKED, 0, false); /* sets demote */
+ /*
+ * Ordinarily, we would call dlm and its callback would call
+ * finish_xmote, which would call state_change() to the new state.
+ * Since we withdrew, we won't call dlm, so call state_change
+ * manually, but to the UNLOCKED state we desire.
+ */
+ state_change(gl, LM_ST_UNLOCKED);
+ /*
+ * We skip telling dlm to do the locking, so we won't get a
+ * reply that would otherwise clear GLF_LOCK. So we clear it here.
+ */
+ clear_bit(GLF_LOCK, &gl->gl_flags);
+ clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
goto out;
} else {
if (gfs2_queue_delete_work(gl, 5 * HZ))
return;
}
- goto out;
}
inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
GFS2_BLKST_UNLINKED);
- if (!IS_ERR_OR_NULL(inode)) {
+ if (IS_ERR(inode)) {
+ if (PTR_ERR(inode) == -EAGAIN &&
+ (gfs2_queue_delete_work(gl, 5 * HZ)))
+ return;
+ } else {
d_prune_aliases(inode);
iput(inode);
}
-out:
gfs2_glock_put(gl);
}
va_end(args);
}
+static inline bool pid_is_meaningful(const struct gfs2_holder *gh)
+{
+ if (!(gh->gh_flags & GL_NOPID))
+ return true;
+ if (gh->gh_state == LM_ST_UNLOCKED)
+ return true;
+ return false;
+}
+
/**
* add_to_queue - Add a holder to the wait queue (but look for recursion)
* @gh: the holder structure to add
}
list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
- if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
- (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK) &&
- !test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags)))
- goto trap_recursive;
+ if (likely(gh2->gh_owner_pid != gh->gh_owner_pid))
+ continue;
+ if (gh->gh_gl->gl_ops->go_type == LM_TYPE_FLOCK)
+ continue;
+ if (test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags))
+ continue;
+ if (!pid_is_meaningful(gh2))
+ continue;
+ goto trap_recursive;
+ }
+ list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
if (try_futile &&
!(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
fail:
dump_glock(NULL, gl, true);
}
+static void withdraw_dq(struct gfs2_glock *gl)
+{
+ spin_lock(&gl->gl_lockref.lock);
+ if (!__lockref_is_dead(&gl->gl_lockref) &&
+ glock_blocked_by_withdraw(gl))
+ do_error(gl, LM_OUT_ERROR); /* remove pending waiters */
+ spin_unlock(&gl->gl_lockref.lock);
+}
+
+void gfs2_gl_dq_holders(struct gfs2_sbd *sdp)
+{
+ glock_hash_walk(withdraw_dq, sdp);
+}
+
/**
* gfs2_gl_hash_clear - Empty out the glock hash table
* @sdp: the filesystem
static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
const char *fs_id_buf)
{
- struct task_struct *gh_owner = NULL;
+ const char *comm = "(none)";
+ pid_t owner_pid = 0;
char flags_buf[32];
rcu_read_lock();
- if (gh->gh_owner_pid)
+ if (pid_is_meaningful(gh)) {
+ struct task_struct *gh_owner;
+
+ comm = "(ended)";
+ owner_pid = pid_nr(gh->gh_owner_pid);
gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
+ if (gh_owner)
+ comm = gh_owner->comm;
+ }
gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
fs_id_buf, state2str(gh->gh_state),
hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
- gh->gh_error,
- gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
- gh_owner ? gh_owner->comm : "(ended)",
- (void *)gh->gh_ip);
+ gh->gh_error, (long)owner_pid, comm, (void *)gh->gh_ip);
rcu_read_unlock();
}
.release = gfs2_glocks_release,
};
+struct gfs2_glockfd_iter {
+ struct super_block *sb;
+ unsigned int tgid;
+ struct task_struct *task;
+ unsigned int fd;
+ struct file *file;
+};
+
+static struct task_struct *gfs2_glockfd_next_task(struct gfs2_glockfd_iter *i)
+{
+ struct pid_namespace *ns = task_active_pid_ns(current);
+ struct pid *pid;
+
+ if (i->task)
+ put_task_struct(i->task);
+
+ rcu_read_lock();
+retry:
+ i->task = NULL;
+ pid = find_ge_pid(i->tgid, ns);
+ if (pid) {
+ i->tgid = pid_nr_ns(pid, ns);
+ i->task = pid_task(pid, PIDTYPE_TGID);
+ if (!i->task) {
+ i->tgid++;
+ goto retry;
+ }
+ get_task_struct(i->task);
+ }
+ rcu_read_unlock();
+ return i->task;
+}
+
+static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i)
+{
+ if (i->file) {
+ fput(i->file);
+ i->file = NULL;
+ }
+
+ rcu_read_lock();
+ for(;; i->fd++) {
+ struct inode *inode;
+
+ i->file = task_lookup_next_fd_rcu(i->task, &i->fd);
+ if (!i->file) {
+ i->fd = 0;
+ break;
+ }
+ inode = file_inode(i->file);
+ if (inode->i_sb != i->sb)
+ continue;
+ if (get_file_rcu(i->file))
+ break;
+ }
+ rcu_read_unlock();
+ return i->file;
+}
+
+static void *gfs2_glockfd_seq_start(struct seq_file *seq, loff_t *pos)
+{
+ struct gfs2_glockfd_iter *i = seq->private;
+
+ if (*pos)
+ return NULL;
+ while (gfs2_glockfd_next_task(i)) {
+ if (gfs2_glockfd_next_file(i))
+ return i;
+ i->tgid++;
+ }
+ return NULL;
+}
+
+static void *gfs2_glockfd_seq_next(struct seq_file *seq, void *iter_ptr,
+ loff_t *pos)
+{
+ struct gfs2_glockfd_iter *i = seq->private;
+
+ (*pos)++;
+ i->fd++;
+ do {
+ if (gfs2_glockfd_next_file(i))
+ return i;
+ i->tgid++;
+ } while (gfs2_glockfd_next_task(i));
+ return NULL;
+}
+
+static void gfs2_glockfd_seq_stop(struct seq_file *seq, void *iter_ptr)
+{
+ struct gfs2_glockfd_iter *i = seq->private;
+
+ if (i->file)
+ fput(i->file);
+ if (i->task)
+ put_task_struct(i->task);
+}
+
+static void gfs2_glockfd_seq_show_flock(struct seq_file *seq,
+ struct gfs2_glockfd_iter *i)
+{
+ struct gfs2_file *fp = i->file->private_data;
+ struct gfs2_holder *fl_gh = &fp->f_fl_gh;
+ struct lm_lockname gl_name = { .ln_type = LM_TYPE_RESERVED };
+
+ if (!READ_ONCE(fl_gh->gh_gl))
+ return;
+
+ spin_lock(&i->file->f_lock);
+ if (gfs2_holder_initialized(fl_gh))
+ gl_name = fl_gh->gh_gl->gl_name;
+ spin_unlock(&i->file->f_lock);
+
+ if (gl_name.ln_type != LM_TYPE_RESERVED) {
+ seq_printf(seq, "%d %u %u/%llx\n",
+ i->tgid, i->fd, gl_name.ln_type,
+ (unsigned long long)gl_name.ln_number);
+ }
+}
+
+static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr)
+{
+ struct gfs2_glockfd_iter *i = seq->private;
+ struct inode *inode = file_inode(i->file);
+ struct gfs2_glock *gl;
+
+ inode_lock_shared(inode);
+ gl = GFS2_I(inode)->i_iopen_gh.gh_gl;
+ if (gl) {
+ seq_printf(seq, "%d %u %u/%llx\n",
+ i->tgid, i->fd, gl->gl_name.ln_type,
+ (unsigned long long)gl->gl_name.ln_number);
+ }
+ gfs2_glockfd_seq_show_flock(seq, i);
+ inode_unlock_shared(inode);
+ return 0;
+}
+
+static const struct seq_operations gfs2_glockfd_seq_ops = {
+ .start = gfs2_glockfd_seq_start,
+ .next = gfs2_glockfd_seq_next,
+ .stop = gfs2_glockfd_seq_stop,
+ .show = gfs2_glockfd_seq_show,
+};
+
+static int gfs2_glockfd_open(struct inode *inode, struct file *file)
+{
+ struct gfs2_glockfd_iter *i;
+ struct gfs2_sbd *sdp = inode->i_private;
+
+ i = __seq_open_private(file, &gfs2_glockfd_seq_ops,
+ sizeof(struct gfs2_glockfd_iter));
+ if (!i)
+ return -ENOMEM;
+ i->sb = sdp->sd_vfs;
+ return 0;
+}
+
+static const struct file_operations gfs2_glockfd_fops = {
+ .owner = THIS_MODULE,
+ .open = gfs2_glockfd_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release_private,
+};
+
DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
&gfs2_glocks_fops);
+ debugfs_create_file("glockfd", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
+ &gfs2_glockfd_fops);
+
debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
&gfs2_glstats_fops);
#define GL_ASYNC 0x0040
#define GL_EXACT 0x0080
#define GL_SKIP 0x0100
+#define GL_NOPID 0x0200
#define GL_NOCACHE 0x0400
/*
extern bool gfs2_delete_work_queued(const struct gfs2_glock *gl);
extern void gfs2_flush_delete_work(struct gfs2_sbd *sdp);
extern void gfs2_gl_hash_clear(struct gfs2_sbd *sdp);
+extern void gfs2_gl_dq_holders(struct gfs2_sbd *sdp);
extern void gfs2_glock_thaw(struct gfs2_sbd *sdp);
extern void gfs2_glock_add_to_lru(struct gfs2_glock *gl);
extern void gfs2_glock_free(struct gfs2_glock *gl);
if (inode->i_state & I_NEW) {
struct gfs2_sbd *sdp = GFS2_SB(inode);
struct gfs2_glock *io_gl;
+ int extra_flags = 0;
error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE,
&ip->i_gl);
if (unlikely(error))
goto fail;
- if (blktype != GFS2_BLKST_UNLINKED)
+ if (blktype == GFS2_BLKST_UNLINKED)
+ extra_flags |= LM_FLAG_TRY;
+ else
gfs2_cancel_delete_work(io_gl);
- error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT,
+ error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED,
+ GL_EXACT | GL_NOPID | extra_flags,
&ip->i_iopen_gh);
gfs2_glock_put(io_gl);
if (unlikely(error))
return inode;
fail:
+ if (error == GLR_TRYFAILED)
+ error = -EAGAIN;
if (gfs2_holder_initialized(&ip->i_iopen_gh))
gfs2_glock_dq_uninit(&ip->i_iopen_gh);
if (gfs2_holder_initialized(&i_gh))
error = insert_inode_locked4(inode, ip->i_no_addr, iget_test, &ip->i_no_addr);
BUG_ON(error);
- error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, &ip->i_iopen_gh);
+ error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT | GL_NOPID,
+ &ip->i_iopen_gh);
if (error)
goto fail_gunlock2;
if (error)
goto fail_shrinker;
- error = register_filesystem(&gfs2_fs_type);
- if (error)
- goto fail_fs1;
-
- error = register_filesystem(&gfs2meta_fs_type);
- if (error)
- goto fail_fs2;
-
error = -ENOMEM;
gfs_recovery_wq = alloc_workqueue("gfs_recovery",
WQ_MEM_RECLAIM | WQ_FREEZABLE, 0);
goto fail_mempool;
gfs2_register_debugfs();
+ error = register_filesystem(&gfs2_fs_type);
+ if (error)
+ goto fail_fs1;
+
+ error = register_filesystem(&gfs2meta_fs_type);
+ if (error)
+ goto fail_fs2;
+
pr_info("GFS2 installed\n");
return 0;
+fail_fs2:
+ unregister_filesystem(&gfs2_fs_type);
+fail_fs1:
+ mempool_destroy(gfs2_page_pool);
fail_mempool:
destroy_workqueue(gfs2_freeze_wq);
fail_wq3:
fail_wq2:
destroy_workqueue(gfs_recovery_wq);
fail_wq1:
- unregister_filesystem(&gfs2meta_fs_type);
-fail_fs2:
- unregister_filesystem(&gfs2_fs_type);
-fail_fs1:
unregister_shrinker(&gfs2_qd_shrinker);
fail_shrinker:
kmem_cache_destroy(gfs2_trans_cachep);
if (buffer_uptodate(first_bh))
goto out;
- if (!buffer_locked(first_bh))
- ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &first_bh);
+ bh_read_nowait(first_bh, REQ_META | REQ_PRIO);
dblock++;
extlen--;
while (extlen) {
bh = gfs2_getbuf(gl, dblock, CREATE);
- if (!buffer_uptodate(bh) && !buffer_locked(bh))
- ll_rw_block(REQ_OP_READ | REQ_RAHEAD | REQ_META |
- REQ_PRIO, 1, &bh);
+ bh_readahead(bh, REQ_RAHEAD | REQ_META | REQ_PRIO);
brelse(bh);
dblock++;
extlen--;
pr_warn("Invalid block size\n");
return -EINVAL;
}
-
+ if (sb->sb_bsize_shift != ffs(sb->sb_bsize) - 1) {
+ pr_warn("Invalid block size shift\n");
+ return -EINVAL;
+ }
return 0;
}
if (!table[0])
table = sdp->sd_vfs->s_id;
- strlcpy(sdp->sd_proto_name, proto, GFS2_FSNAME_LEN);
- strlcpy(sdp->sd_table_name, table, GFS2_FSNAME_LEN);
+ BUILD_BUG_ON(GFS2_LOCKNAME_LEN > GFS2_FSNAME_LEN);
+
+ strscpy(sdp->sd_proto_name, proto, GFS2_LOCKNAME_LEN);
+ strscpy(sdp->sd_table_name, table, GFS2_LOCKNAME_LEN);
table = sdp->sd_table_name;
while ((table = strchr(table, '/')))
error = gfs2_glock_nq_num(sdp,
GFS2_MOUNT_LOCK, &gfs2_nondisk_glops,
- LM_ST_EXCLUSIVE, LM_FLAG_NOEXP | GL_NOCACHE,
+ LM_ST_EXCLUSIVE,
+ LM_FLAG_NOEXP | GL_NOCACHE | GL_NOPID,
mount_gh);
if (error) {
fs_err(sdp, "can't acquire mount glock: %d\n", error);
error = gfs2_glock_nq_num(sdp,
GFS2_LIVE_LOCK, &gfs2_nondisk_glops,
LM_ST_SHARED,
- LM_FLAG_NOEXP | GL_EXACT,
+ LM_FLAG_NOEXP | GL_EXACT | GL_NOPID,
&sdp->sd_live_gh);
if (error) {
fs_err(sdp, "can't acquire live glock: %d\n", error);
iput(pn);
pn = NULL;
ip = GFS2_I(sdp->sd_sc_inode);
- error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0,
+ error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID,
&sdp->sd_sc_gh);
if (error) {
fs_err(sdp, "can't lock local \"sc\" file: %d\n", error);
error = gfs2_glock_nq_num(sdp, sdp->sd_lockstruct.ls_jid,
&gfs2_journal_glops,
LM_ST_EXCLUSIVE,
- LM_FLAG_NOEXP | GL_NOCACHE,
+ LM_FLAG_NOEXP | GL_NOCACHE | GL_NOPID,
&sdp->sd_journal_gh);
if (error) {
fs_err(sdp, "can't acquire journal glock: %d\n", error);
ip = GFS2_I(sdp->sd_jdesc->jd_inode);
sdp->sd_jinode_gl = ip->i_gl;
error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED,
- LM_FLAG_NOEXP | GL_EXACT | GL_NOCACHE,
+ LM_FLAG_NOEXP | GL_EXACT |
+ GL_NOCACHE | GL_NOPID,
&sdp->sd_jinode_gh);
if (error) {
fs_err(sdp, "can't acquire journal inode glock: %d\n",
pn = NULL;
ip = GFS2_I(sdp->sd_qc_inode);
- error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0,
+ error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID,
&sdp->sd_qc_gh);
if (error) {
fs_err(sdp, "can't lock local \"qc\" file: %d\n", error);
switch (o) {
case Opt_lockproto:
- strlcpy(args->ar_lockproto, param->string, GFS2_LOCKNAME_LEN);
+ strscpy(args->ar_lockproto, param->string, GFS2_LOCKNAME_LEN);
break;
case Opt_locktable:
- strlcpy(args->ar_locktable, param->string, GFS2_LOCKNAME_LEN);
+ strscpy(args->ar_locktable, param->string, GFS2_LOCKNAME_LEN);
break;
case Opt_hostdata:
- strlcpy(args->ar_hostdata, param->string, GFS2_LOCKNAME_LEN);
+ strscpy(args->ar_hostdata, param->string, GFS2_LOCKNAME_LEN);
break;
case Opt_spectator:
args->ar_spectator = 1;
}
if (PageUptodate(page))
set_buffer_uptodate(bh);
- if (!buffer_uptodate(bh)) {
- ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &bh);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh))
- goto unlock_out;
- }
+ if (bh_read(bh, REQ_META | REQ_PRIO) < 0)
+ goto unlock_out;
if (gfs2_is_jdata(ip))
gfs2_trans_add_data(ip->i_gl, bh);
else
}
error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_EXCLUSIVE,
- LM_FLAG_NOEXP, &sdp->sd_freeze_gh);
+ LM_FLAG_NOEXP | GL_NOPID,
+ &sdp->sd_freeze_gh);
if (error)
goto out;
}
if (!ret)
gfs2_make_fs_ro(sdp);
+ /*
+ * Dequeue any pending non-system glock holders that can no
+ * longer be granted because the file system is withdrawn.
+ */
+ gfs2_gl_dq_holders(sdp);
gfs2_freeze_unlock(&freeze_gh);
}
* exception code in glock_dq.
*/
iput(inode);
+ sdp->sd_jdesc->jd_inode = NULL;
/*
* Wait until the journal inode's glock is freed. This allows try locks
* on other nodes to be successful, otherwise we remain the owner of
*/
fs_warn(sdp, "Requesting recovery of jid %d.\n",
sdp->sd_lockstruct.ls_jid);
- gfs2_holder_reinit(LM_ST_EXCLUSIVE, LM_FLAG_TRY_1CB | LM_FLAG_NOEXP,
+ gfs2_holder_reinit(LM_ST_EXCLUSIVE,
+ LM_FLAG_TRY_1CB | LM_FLAG_NOEXP | GL_NOPID,
&sdp->sd_live_gh);
msleep(GL_GLOCK_MAX_HOLD);
/*
fs_warn(sdp, "Unable to recover our journal jid %d.\n",
sdp->sd_lockstruct.ls_jid);
gfs2_glock_dq_wait(&sdp->sd_live_gh);
- gfs2_holder_reinit(LM_ST_SHARED, LM_FLAG_NOEXP | GL_EXACT,
+ gfs2_holder_reinit(LM_ST_SHARED,
+ LM_FLAG_NOEXP | GL_EXACT | GL_NOPID,
&sdp->sd_live_gh);
gfs2_glock_nq(&sdp->sd_live_gh);
}
int pagenum;
int bytes_read;
int bytes_to_read;
- void *vaddr;
off += node->page_offset;
pagenum = off >> PAGE_SHIFT;
page = node->page[pagenum];
bytes_to_read = min_t(int, len - bytes_read, PAGE_SIZE - off);
- vaddr = kmap_atomic(page);
- memcpy(buf + bytes_read, vaddr + off, bytes_to_read);
- kunmap_atomic(vaddr);
+ memcpy_from_page(buf + bytes_read, page, off, bytes_to_read);
pagenum++;
off = 0; /* page offset only applies to the first page */
off += node->page_offset;
page = node->page[0];
- memcpy(kmap(page) + off, buf, len);
- kunmap(page);
+ memcpy_to_page(page, off, buf, len);
set_page_dirty(page);
}
off += node->page_offset;
page = node->page[0];
- memset(kmap(page) + off, 0, len);
- kunmap(page);
+ memzero_page(page, off, len);
set_page_dirty(page);
}
src_page = src_node->page[0];
dst_page = dst_node->page[0];
- memcpy(kmap(dst_page) + dst, kmap(src_page) + src, len);
- kunmap(src_page);
- kunmap(dst_page);
+ memcpy_page(dst_page, dst, src_page, src, len);
set_page_dirty(dst_page);
}
src += node->page_offset;
dst += node->page_offset;
page = node->page[0];
- ptr = kmap(page);
+ ptr = kmap_local_page(page);
memmove(ptr + dst, ptr + src, len);
- kunmap(page);
+ kunmap_local(ptr);
set_page_dirty(page);
}
if (!test_bit(HFS_BNODE_NEW, &node->flags))
return node;
- desc = (struct hfs_bnode_desc *)(kmap(node->page[0]) + node->page_offset);
+ desc = (struct hfs_bnode_desc *)(kmap_local_page(node->page[0]) +
+ node->page_offset);
node->prev = be32_to_cpu(desc->prev);
node->next = be32_to_cpu(desc->next);
node->num_recs = be16_to_cpu(desc->num_recs);
node->type = desc->type;
node->height = desc->height;
- kunmap(node->page[0]);
+ kunmap_local(desc);
switch (node->type) {
case HFS_NODE_HEADER:
}
pagep = node->page;
- memset(kmap(*pagep) + node->page_offset, 0,
- min((int)PAGE_SIZE, (int)tree->node_size));
+ memzero_page(*pagep, node->page_offset,
+ min((int)PAGE_SIZE, (int)tree->node_size));
set_page_dirty(*pagep);
- kunmap(*pagep);
for (i = 1; i < tree->pages_per_bnode; i++) {
- memset(kmap(*++pagep), 0, PAGE_SIZE);
+ memzero_page(*++pagep, 0, PAGE_SIZE);
set_page_dirty(*pagep);
- kunmap(*pagep);
}
clear_bit(HFS_BNODE_NEW, &node->flags);
wake_up(&node->lock_wq);
goto free_inode;
/* Load the header */
- head = (struct hfs_btree_header_rec *)(kmap(page) + sizeof(struct hfs_bnode_desc));
+ head = (struct hfs_btree_header_rec *)(kmap_local_page(page) +
+ sizeof(struct hfs_bnode_desc));
tree->root = be32_to_cpu(head->root);
tree->leaf_count = be32_to_cpu(head->leaf_count);
tree->leaf_head = be32_to_cpu(head->leaf_head);
tree->node_size_shift = ffs(size) - 1;
tree->pages_per_bnode = (tree->node_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
- kunmap(page);
+ kunmap_local(head);
put_page(page);
return tree;
fail_page:
+ kunmap_local(head);
put_page(page);
free_inode:
tree->inode->i_mapping->a_ops = &hfs_aops;
return;
/* Load the header */
page = node->page[0];
- head = (struct hfs_btree_header_rec *)(kmap(page) + sizeof(struct hfs_bnode_desc));
+ head = (struct hfs_btree_header_rec *)(kmap_local_page(page) +
+ sizeof(struct hfs_bnode_desc));
head->root = cpu_to_be32(tree->root);
head->leaf_count = cpu_to_be32(tree->leaf_count);
head->attributes = cpu_to_be32(tree->attributes);
head->depth = cpu_to_be16(tree->depth);
- kunmap(page);
+ kunmap_local(head);
set_page_dirty(page);
hfs_bnode_put(node);
}
off += node->page_offset;
pagep = node->page + (off >> PAGE_SHIFT);
- data = kmap(*pagep);
+ data = kmap_local_page(*pagep);
off &= ~PAGE_MASK;
idx = 0;
idx += i;
data[off] |= m;
set_page_dirty(*pagep);
- kunmap(*pagep);
+ kunmap_local(data);
tree->free_nodes--;
mark_inode_dirty(tree->inode);
hfs_bnode_put(node);
}
}
if (++off >= PAGE_SIZE) {
- kunmap(*pagep);
- data = kmap(*++pagep);
+ kunmap_local(data);
+ data = kmap_local_page(*++pagep);
off = 0;
}
idx += 8;
len--;
}
- kunmap(*pagep);
+ kunmap_local(data);
nidx = node->next;
if (!nidx) {
printk(KERN_DEBUG "create new bmap node...\n");
off = off16;
off += node->page_offset;
pagep = node->page + (off >> PAGE_SHIFT);
- data = kmap(*pagep);
+ data = kmap_local_page(*pagep);
off &= ~PAGE_MASK;
}
}
}
off += node->page_offset + nidx / 8;
page = node->page[off >> PAGE_SHIFT];
- data = kmap(page);
+ data = kmap_local_page(page);
off &= ~PAGE_MASK;
m = 1 << (~nidx & 7);
byte = data[off];
if (!(byte & m)) {
pr_crit("trying to free free bnode %u(%d)\n",
node->this, node->type);
- kunmap(page);
+ kunmap_local(data);
hfs_bnode_put(node);
return;
}
data[off] = byte & ~m;
set_page_dirty(page);
- kunmap(page);
+ kunmap_local(data);
hfs_bnode_put(node);
tree->free_nodes++;
mark_inode_dirty(tree->inode);
start = size;
goto out;
}
- pptr = kmap(page);
+ pptr = kmap_local_page(page);
curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32;
i = offset % 32;
offset &= ~(PAGE_CACHE_BITS - 1);
}
curr++;
}
- kunmap(page);
+ kunmap_local(pptr);
offset += PAGE_CACHE_BITS;
if (offset >= size)
break;
start = size;
goto out;
}
- curr = pptr = kmap(page);
+ curr = pptr = kmap_local_page(page);
if ((size ^ offset) / PAGE_CACHE_BITS)
end = pptr + PAGE_CACHE_BITS / 32;
else
len -= 32;
}
set_page_dirty(page);
- kunmap(page);
+ kunmap_local(pptr);
offset += PAGE_CACHE_BITS;
page = read_mapping_page(mapping, offset / PAGE_CACHE_BITS,
NULL);
start = size;
goto out;
}
- pptr = kmap(page);
+ pptr = kmap_local_page(page);
curr = pptr;
end = pptr + PAGE_CACHE_BITS / 32;
}
done:
*curr = cpu_to_be32(n);
set_page_dirty(page);
- kunmap(page);
+ kunmap_local(pptr);
*max = offset + (curr - pptr) * 32 + i - start;
sbi->free_blocks -= *max;
hfsplus_mark_mdb_dirty(sb);
page = read_mapping_page(mapping, pnr, NULL);
if (IS_ERR(page))
goto kaboom;
- pptr = kmap(page);
+ pptr = kmap_local_page(page);
curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32;
end = pptr + PAGE_CACHE_BITS / 32;
len = count;
if (!count)
break;
set_page_dirty(page);
- kunmap(page);
+ kunmap_local(pptr);
page = read_mapping_page(mapping, ++pnr, NULL);
if (IS_ERR(page))
goto kaboom;
- pptr = kmap(page);
+ pptr = kmap_local_page(page);
curr = pptr;
end = pptr + PAGE_CACHE_BITS / 32;
}
}
out:
set_page_dirty(page);
- kunmap(page);
+ kunmap_local(pptr);
sbi->free_blocks += len;
hfsplus_mark_mdb_dirty(sb);
mutex_unlock(&sbi->alloc_mutex);
off &= ~PAGE_MASK;
l = min_t(int, len, PAGE_SIZE - off);
- memcpy(buf, kmap(*pagep) + off, l);
- kunmap(*pagep);
+ memcpy_from_page(buf, *pagep, off, l);
while ((len -= l) != 0) {
buf += l;
l = min_t(int, len, PAGE_SIZE);
- memcpy(buf, kmap(*++pagep), l);
- kunmap(*pagep);
+ memcpy_from_page(buf, *++pagep, 0, l);
}
}
off &= ~PAGE_MASK;
l = min_t(int, len, PAGE_SIZE - off);
- memcpy(kmap(*pagep) + off, buf, l);
+ memcpy_to_page(*pagep, off, buf, l);
set_page_dirty(*pagep);
- kunmap(*pagep);
while ((len -= l) != 0) {
buf += l;
l = min_t(int, len, PAGE_SIZE);
- memcpy(kmap(*++pagep), buf, l);
+ memcpy_to_page(*++pagep, 0, buf, l);
set_page_dirty(*pagep);
- kunmap(*pagep);
}
}
off &= ~PAGE_MASK;
l = min_t(int, len, PAGE_SIZE - off);
- memset(kmap(*pagep) + off, 0, l);
+ memzero_page(*pagep, off, l);
set_page_dirty(*pagep);
- kunmap(*pagep);
while ((len -= l) != 0) {
l = min_t(int, len, PAGE_SIZE);
- memset(kmap(*++pagep), 0, l);
+ memzero_page(*++pagep, 0, l);
set_page_dirty(*pagep);
- kunmap(*pagep);
}
}
if (src == dst) {
l = min_t(int, len, PAGE_SIZE - src);
- memcpy(kmap(*dst_page) + src, kmap(*src_page) + src, l);
- kunmap(*src_page);
+ memcpy_page(*dst_page, src, *src_page, src, l);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
while ((len -= l) != 0) {
l = min_t(int, len, PAGE_SIZE);
- memcpy(kmap(*++dst_page), kmap(*++src_page), l);
- kunmap(*src_page);
+ memcpy_page(*++dst_page, 0, *++src_page, 0, l);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
}
} else {
void *src_ptr, *dst_ptr;
do {
- src_ptr = kmap(*src_page) + src;
- dst_ptr = kmap(*dst_page) + dst;
+ dst_ptr = kmap_local_page(*dst_page) + dst;
+ src_ptr = kmap_local_page(*src_page) + src;
if (PAGE_SIZE - src < PAGE_SIZE - dst) {
l = PAGE_SIZE - src;
src = 0;
}
l = min(len, l);
memcpy(dst_ptr, src_ptr, l);
- kunmap(*src_page);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
if (!dst)
dst_page++;
else
void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len)
{
struct page **src_page, **dst_page;
+ void *src_ptr, *dst_ptr;
int l;
hfs_dbg(BNODE_MOD, "movebytes: %u,%u,%u\n", dst, src, len);
if (src == dst) {
while (src < len) {
- memmove(kmap(*dst_page), kmap(*src_page), src);
- kunmap(*src_page);
+ dst_ptr = kmap_local_page(*dst_page);
+ src_ptr = kmap_local_page(*src_page);
+ memmove(dst_ptr, src_ptr, src);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
len -= src;
src = PAGE_SIZE;
src_page--;
dst_page--;
}
src -= len;
- memmove(kmap(*dst_page) + src,
- kmap(*src_page) + src, len);
- kunmap(*src_page);
+ dst_ptr = kmap_local_page(*dst_page);
+ src_ptr = kmap_local_page(*src_page);
+ memmove(dst_ptr + src, src_ptr + src, len);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
} else {
- void *src_ptr, *dst_ptr;
-
do {
- src_ptr = kmap(*src_page) + src;
- dst_ptr = kmap(*dst_page) + dst;
+ dst_ptr = kmap_local_page(*dst_page) + dst;
+ src_ptr = kmap_local_page(*src_page) + src;
if (src < dst) {
l = src;
src = PAGE_SIZE;
}
l = min(len, l);
memmove(dst_ptr - l, src_ptr - l, l);
- kunmap(*src_page);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
if (dst == PAGE_SIZE)
dst_page--;
else
if (src == dst) {
l = min_t(int, len, PAGE_SIZE - src);
- memmove(kmap(*dst_page) + src,
- kmap(*src_page) + src, l);
- kunmap(*src_page);
+
+ dst_ptr = kmap_local_page(*dst_page) + src;
+ src_ptr = kmap_local_page(*src_page) + src;
+ memmove(dst_ptr, src_ptr, l);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
while ((len -= l) != 0) {
l = min_t(int, len, PAGE_SIZE);
- memmove(kmap(*++dst_page),
- kmap(*++src_page), l);
- kunmap(*src_page);
+ dst_ptr = kmap_local_page(*++dst_page);
+ src_ptr = kmap_local_page(*++src_page);
+ memmove(dst_ptr, src_ptr, l);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
}
} else {
- void *src_ptr, *dst_ptr;
-
do {
- src_ptr = kmap(*src_page) + src;
- dst_ptr = kmap(*dst_page) + dst;
+ dst_ptr = kmap_local_page(*dst_page) + dst;
+ src_ptr = kmap_local_page(*src_page) + src;
if (PAGE_SIZE - src <
PAGE_SIZE - dst) {
l = PAGE_SIZE - src;
}
l = min(len, l);
memmove(dst_ptr, src_ptr, l);
- kunmap(*src_page);
+ kunmap_local(src_ptr);
set_page_dirty(*dst_page);
- kunmap(*dst_page);
+ kunmap_local(dst_ptr);
if (!dst)
dst_page++;
else
if (!test_bit(HFS_BNODE_NEW, &node->flags))
return node;
- desc = (struct hfs_bnode_desc *)(kmap(node->page[0]) +
- node->page_offset);
+ desc = (struct hfs_bnode_desc *)(kmap_local_page(node->page[0]) +
+ node->page_offset);
node->prev = be32_to_cpu(desc->prev);
node->next = be32_to_cpu(desc->next);
node->num_recs = be16_to_cpu(desc->num_recs);
node->type = desc->type;
node->height = desc->height;
- kunmap(node->page[0]);
+ kunmap_local(desc);
switch (node->type) {
case HFS_NODE_HEADER:
}
pagep = node->page;
- memset(kmap(*pagep) + node->page_offset, 0,
- min_t(int, PAGE_SIZE, tree->node_size));
+ memzero_page(*pagep, node->page_offset,
+ min_t(int, PAGE_SIZE, tree->node_size));
set_page_dirty(*pagep);
- kunmap(*pagep);
for (i = 1; i < tree->pages_per_bnode; i++) {
- memset(kmap(*++pagep), 0, PAGE_SIZE);
+ memzero_page(*++pagep, 0, PAGE_SIZE);
set_page_dirty(*pagep);
- kunmap(*pagep);
}
clear_bit(HFS_BNODE_NEW, &node->flags);
wake_up(&node->lock_wq);
goto free_inode;
/* Load the header */
- head = (struct hfs_btree_header_rec *)(kmap(page) +
+ head = (struct hfs_btree_header_rec *)(kmap_local_page(page) +
sizeof(struct hfs_bnode_desc));
tree->root = be32_to_cpu(head->root);
tree->leaf_count = be32_to_cpu(head->leaf_count);
(tree->node_size + PAGE_SIZE - 1) >>
PAGE_SHIFT;
- kunmap(page);
+ kunmap_local(head);
put_page(page);
return tree;
fail_page:
+ kunmap_local(head);
put_page(page);
free_inode:
tree->inode->i_mapping->a_ops = &hfsplus_aops;
return -EIO;
/* Load the header */
page = node->page[0];
- head = (struct hfs_btree_header_rec *)(kmap(page) +
+ head = (struct hfs_btree_header_rec *)(kmap_local_page(page) +
sizeof(struct hfs_bnode_desc));
head->root = cpu_to_be32(tree->root);
head->attributes = cpu_to_be32(tree->attributes);
head->depth = cpu_to_be16(tree->depth);
- kunmap(page);
+ kunmap_local(head);
set_page_dirty(page);
hfs_bnode_put(node);
return 0;
off += node->page_offset;
pagep = node->page + (off >> PAGE_SHIFT);
- data = kmap(*pagep);
+ data = kmap_local_page(*pagep);
off &= ~PAGE_MASK;
idx = 0;
idx += i;
data[off] |= m;
set_page_dirty(*pagep);
- kunmap(*pagep);
+ kunmap_local(data);
tree->free_nodes--;
mark_inode_dirty(tree->inode);
hfs_bnode_put(node);
}
}
if (++off >= PAGE_SIZE) {
- kunmap(*pagep);
- data = kmap(*++pagep);
+ kunmap_local(data);
+ data = kmap_local_page(*++pagep);
off = 0;
}
idx += 8;
len--;
}
- kunmap(*pagep);
+ kunmap_local(data);
nidx = node->next;
if (!nidx) {
hfs_dbg(BNODE_MOD, "create new bmap node\n");
off = off16;
off += node->page_offset;
pagep = node->page + (off >> PAGE_SHIFT);
- data = kmap(*pagep);
+ data = kmap_local_page(*pagep);
off &= ~PAGE_MASK;
}
}
}
off += node->page_offset + nidx / 8;
page = node->page[off >> PAGE_SHIFT];
- data = kmap(page);
+ data = kmap_local_page(page);
off &= ~PAGE_MASK;
m = 1 << (~nidx & 7);
byte = data[off];
pr_crit("trying to free free bnode "
"%u(%d)\n",
node->this, node->type);
- kunmap(page);
+ kunmap_local(data);
hfs_bnode_put(node);
return;
}
data[off] = byte & ~m;
set_page_dirty(page);
- kunmap(page);
+ kunmap_local(data);
hfs_bnode_put(node);
tree->free_nodes++;
mark_inode_dirty(tree->inode);
*/
BUG_ON(p + strlen(p) + 1 != name + PATH_MAX);
- strlcpy(name, root, PATH_MAX);
+ strscpy(name, root, PATH_MAX);
if (len > p - name) {
__putname(name);
return NULL;
return -EINVAL;
}
-static void remove_huge_page(struct page *page)
+static void hugetlb_delete_from_page_cache(struct page *page)
{
ClearPageDirty(page);
ClearPageUptodate(page);
delete_from_page_cache(page);
}
+/*
+ * Called with i_mmap_rwsem held for inode based vma maps. This makes
+ * sure vma (and vm_mm) will not go away. We also hold the hugetlb fault
+ * mutex for the page in the mapping. So, we can not race with page being
+ * faulted into the vma.
+ */
+static bool hugetlb_vma_maps_page(struct vm_area_struct *vma,
+ unsigned long addr, struct page *page)
+{
+ pte_t *ptep, pte;
+
+ ptep = huge_pte_offset(vma->vm_mm, addr,
+ huge_page_size(hstate_vma(vma)));
+
+ if (!ptep)
+ return false;
+
+ pte = huge_ptep_get(ptep);
+ if (huge_pte_none(pte) || !pte_present(pte))
+ return false;
+
+ if (pte_page(pte) == page)
+ return true;
+
+ return false;
+}
+
+/*
+ * Can vma_offset_start/vma_offset_end overflow on 32-bit arches?
+ * No, because the interval tree returns us only those vmas
+ * which overlap the truncated area starting at pgoff,
+ * and no vma on a 32-bit arch can span beyond the 4GB.
+ */
+static unsigned long vma_offset_start(struct vm_area_struct *vma, pgoff_t start)
+{
+ if (vma->vm_pgoff < start)
+ return (start - vma->vm_pgoff) << PAGE_SHIFT;
+ else
+ return 0;
+}
+
+static unsigned long vma_offset_end(struct vm_area_struct *vma, pgoff_t end)
+{
+ unsigned long t_end;
+
+ if (!end)
+ return vma->vm_end;
+
+ t_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) + vma->vm_start;
+ if (t_end > vma->vm_end)
+ t_end = vma->vm_end;
+ return t_end;
+}
+
+/*
+ * Called with hugetlb fault mutex held. Therefore, no more mappings to
+ * this folio can be created while executing the routine.
+ */
+static void hugetlb_unmap_file_folio(struct hstate *h,
+ struct address_space *mapping,
+ struct folio *folio, pgoff_t index)
+{
+ struct rb_root_cached *root = &mapping->i_mmap;
+ struct hugetlb_vma_lock *vma_lock;
+ struct page *page = &folio->page;
+ struct vm_area_struct *vma;
+ unsigned long v_start;
+ unsigned long v_end;
+ pgoff_t start, end;
+
+ start = index * pages_per_huge_page(h);
+ end = (index + 1) * pages_per_huge_page(h);
+
+ i_mmap_lock_write(mapping);
+retry:
+ vma_lock = NULL;
+ vma_interval_tree_foreach(vma, root, start, end - 1) {
+ v_start = vma_offset_start(vma, start);
+ v_end = vma_offset_end(vma, end);
+
+ if (!hugetlb_vma_maps_page(vma, vma->vm_start + v_start, page))
+ continue;
+
+ if (!hugetlb_vma_trylock_write(vma)) {
+ vma_lock = vma->vm_private_data;
+ /*
+ * If we can not get vma lock, we need to drop
+ * immap_sema and take locks in order. First,
+ * take a ref on the vma_lock structure so that
+ * we can be guaranteed it will not go away when
+ * dropping immap_sema.
+ */
+ kref_get(&vma_lock->refs);
+ break;
+ }
+
+ unmap_hugepage_range(vma, vma->vm_start + v_start, v_end,
+ NULL, ZAP_FLAG_DROP_MARKER);
+ hugetlb_vma_unlock_write(vma);
+ }
+
+ i_mmap_unlock_write(mapping);
+
+ if (vma_lock) {
+ /*
+ * Wait on vma_lock. We know it is still valid as we have
+ * a reference. We must 'open code' vma locking as we do
+ * not know if vma_lock is still attached to vma.
+ */
+ down_write(&vma_lock->rw_sema);
+ i_mmap_lock_write(mapping);
+
+ vma = vma_lock->vma;
+ if (!vma) {
+ /*
+ * If lock is no longer attached to vma, then just
+ * unlock, drop our reference and retry looking for
+ * other vmas.
+ */
+ up_write(&vma_lock->rw_sema);
+ kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
+ goto retry;
+ }
+
+ /*
+ * vma_lock is still attached to vma. Check to see if vma
+ * still maps page and if so, unmap.
+ */
+ v_start = vma_offset_start(vma, start);
+ v_end = vma_offset_end(vma, end);
+ if (hugetlb_vma_maps_page(vma, vma->vm_start + v_start, page))
+ unmap_hugepage_range(vma, vma->vm_start + v_start,
+ v_end, NULL,
+ ZAP_FLAG_DROP_MARKER);
+
+ kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
+ hugetlb_vma_unlock_write(vma);
+
+ goto retry;
+ }
+}
+
static void
hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
zap_flags_t zap_flags)
* an inclusive "last".
*/
vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
- unsigned long v_offset;
+ unsigned long v_start;
unsigned long v_end;
+ if (!hugetlb_vma_trylock_write(vma))
+ continue;
+
+ v_start = vma_offset_start(vma, start);
+ v_end = vma_offset_end(vma, end);
+
+ unmap_hugepage_range(vma, vma->vm_start + v_start, v_end,
+ NULL, zap_flags);
+
/*
- * Can the expression below overflow on 32-bit arches?
- * No, because the interval tree returns us only those vmas
- * which overlap the truncated area starting at pgoff,
- * and no vma on a 32-bit arch can span beyond the 4GB.
+ * Note that vma lock only exists for shared/non-private
+ * vmas. Therefore, lock is not held when calling
+ * unmap_hugepage_range for private vmas.
*/
- if (vma->vm_pgoff < start)
- v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
- else
- v_offset = 0;
-
- if (!end)
- v_end = vma->vm_end;
- else {
- v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT)
- + vma->vm_start;
- if (v_end > vma->vm_end)
- v_end = vma->vm_end;
- }
+ hugetlb_vma_unlock_write(vma);
+ }
+}
- unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
- NULL, zap_flags);
+/*
+ * Called with hugetlb fault mutex held.
+ * Returns true if page was actually removed, false otherwise.
+ */
+static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
+ struct address_space *mapping,
+ struct folio *folio, pgoff_t index,
+ bool truncate_op)
+{
+ bool ret = false;
+
+ /*
+ * If folio is mapped, it was faulted in after being
+ * unmapped in caller. Unmap (again) while holding
+ * the fault mutex. The mutex will prevent faults
+ * until we finish removing the folio.
+ */
+ if (unlikely(folio_mapped(folio)))
+ hugetlb_unmap_file_folio(h, mapping, folio, index);
+
+ folio_lock(folio);
+ /*
+ * We must remove the folio from page cache before removing
+ * the region/ reserve map (hugetlb_unreserve_pages). In
+ * rare out of memory conditions, removal of the region/reserve
+ * map could fail. Correspondingly, the subpool and global
+ * reserve usage count can need to be adjusted.
+ */
+ VM_BUG_ON(HPageRestoreReserve(&folio->page));
+ hugetlb_delete_from_page_cache(&folio->page);
+ ret = true;
+ if (!truncate_op) {
+ if (unlikely(hugetlb_unreserve_pages(inode, index,
+ index + 1, 1)))
+ hugetlb_fix_reserve_counts(inode);
}
+
+ folio_unlock(folio);
+ return ret;
}
/*
* truncation is indicated by end of range being LLONG_MAX
* In this case, we first scan the range and release found pages.
* After releasing pages, hugetlb_unreserve_pages cleans up region/reserve
- * maps and global counts. Page faults can not race with truncation
- * in this routine. hugetlb_no_page() holds i_mmap_rwsem and prevents
- * page faults in the truncated range by checking i_size. i_size is
- * modified while holding i_mmap_rwsem.
+ * maps and global counts. Page faults can race with truncation.
+ * During faults, hugetlb_no_page() checks i_size before page allocation,
+ * and again after obtaining page table lock. It will 'back out'
+ * allocations in the truncated range.
* hole punch is indicated if end is not LLONG_MAX
* In the hole punch case we scan the range and release found pages.
* Only when releasing a page is the associated region/reserve map
u32 hash = 0;
index = folio->index;
- if (!truncate_op) {
- /*
- * Only need to hold the fault mutex in the
- * hole punch case. This prevents races with
- * page faults. Races are not possible in the
- * case of truncation.
- */
- hash = hugetlb_fault_mutex_hash(mapping, index);
- mutex_lock(&hugetlb_fault_mutex_table[hash]);
- }
+ hash = hugetlb_fault_mutex_hash(mapping, index);
+ mutex_lock(&hugetlb_fault_mutex_table[hash]);
/*
- * If folio is mapped, it was faulted in after being
- * unmapped in caller. Unmap (again) now after taking
- * the fault mutex. The mutex will prevent faults
- * until we finish removing the folio.
- *
- * This race can only happen in the hole punch case.
- * Getting here in a truncate operation is a bug.
+ * Remove folio that was part of folio_batch.
*/
- if (unlikely(folio_mapped(folio))) {
- BUG_ON(truncate_op);
-
- mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_lock_write(mapping);
- mutex_lock(&hugetlb_fault_mutex_table[hash]);
- hugetlb_vmdelete_list(&mapping->i_mmap,
- index * pages_per_huge_page(h),
- (index + 1) * pages_per_huge_page(h),
- ZAP_FLAG_DROP_MARKER);
- i_mmap_unlock_write(mapping);
- }
-
- folio_lock(folio);
- /*
- * We must free the huge page and remove from page
- * cache (remove_huge_page) BEFORE removing the
- * region/reserve map (hugetlb_unreserve_pages). In
- * rare out of memory conditions, removal of the
- * region/reserve map could fail. Correspondingly,
- * the subpool and global reserve usage count can need
- * to be adjusted.
- */
- VM_BUG_ON(HPageRestoreReserve(&folio->page));
- remove_huge_page(&folio->page);
- freed++;
- if (!truncate_op) {
- if (unlikely(hugetlb_unreserve_pages(inode,
- index, index + 1, 1)))
- hugetlb_fix_reserve_counts(inode);
- }
-
- folio_unlock(folio);
- if (!truncate_op)
- mutex_unlock(&hugetlb_fault_mutex_table[hash]);
+ if (remove_inode_single_folio(h, inode, mapping, folio,
+ index, truncate_op))
+ freed++;
+
+ mutex_unlock(&hugetlb_fault_mutex_table[hash]);
}
folio_batch_release(&fbatch);
cond_resched();
BUG_ON(offset & ~huge_page_mask(h));
pgoff = offset >> PAGE_SHIFT;
- i_mmap_lock_write(mapping);
i_size_write(inode, offset);
+ i_mmap_lock_write(mapping);
if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
ZAP_FLAG_DROP_MARKER);
/* addr is the offset within the file (zero based) */
addr = index * hpage_size;
- /*
- * fault mutex taken here, protects against fault path
- * and hole punch. inode_lock previously taken protects
- * against truncation.
- */
+ /* mutex taken here, fault path and hole punch */
hash = hugetlb_fault_mutex_hash(mapping, index);
mutex_lock(&hugetlb_fault_mutex_table[hash]);
}
clear_huge_page(page, addr, pages_per_huge_page(h));
__SetPageUptodate(page);
- error = huge_add_to_page_cache(page, mapping, index);
+ error = hugetlb_add_to_page_cache(page, mapping, index);
if (unlikely(error)) {
restore_reserve_on_error(h, &pseudo_vma, addr, page);
put_page(page);
SetHPageMigratable(page);
/*
- * unlock_page because locked by huge_add_to_page_cache()
+ * unlock_page because locked by hugetlb_add_to_page_cache()
* put_page() due to reference from alloc_huge_page()
*/
unlock_page(page);
/*
* File creation. Allocate an inode, and we're done..
*/
-static int do_hugetlbfs_mknod(struct inode *dir,
- struct dentry *dentry,
- umode_t mode,
- dev_t dev,
- bool tmpfile)
+static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir,
+ struct dentry *dentry, umode_t mode, dev_t dev)
{
struct inode *inode;
- int error = -ENOSPC;
inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
- if (inode) {
- dir->i_ctime = dir->i_mtime = current_time(dir);
- if (tmpfile) {
- d_tmpfile(dentry, inode);
- } else {
- d_instantiate(dentry, inode);
- dget(dentry);/* Extra count - pin the dentry in core */
- }
- error = 0;
- }
- return error;
-}
-
-static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode, dev_t dev)
-{
- return do_hugetlbfs_mknod(dir, dentry, mode, dev, false);
+ if (!inode)
+ return -ENOSPC;
+ dir->i_ctime = dir->i_mtime = current_time(dir);
+ d_instantiate(dentry, inode);
+ dget(dentry);/* Extra count - pin the dentry in core */
+ return 0;
}
static int hugetlbfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
}
static int hugetlbfs_tmpfile(struct user_namespace *mnt_userns,
- struct inode *dir, struct dentry *dentry,
+ struct inode *dir, struct file *file,
umode_t mode)
{
- return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true);
+ struct inode *inode;
+
+ inode = hugetlbfs_get_inode(dir->i_sb, dir, mode | S_IFREG, 0);
+ if (!inode)
+ return -ENOSPC;
+ dir->i_ctime = dir->i_mtime = current_time(dir);
+ d_tmpfile(file, inode);
+ return finish_open_simple(file, 0);
}
static int hugetlbfs_symlink(struct user_namespace *mnt_userns,
struct inode *inode = mapping->host;
pgoff_t index = page->index;
- remove_huge_page(page);
+ hugetlb_delete_from_page_cache(page);
if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1)))
hugetlb_fix_reserve_counts(inode);
for ( i = 0 ; i < pcount ; i++ ) {
if (!pages[i])
continue;
- memset(page_address(pages[i]), 0, PAGE_SIZE);
- flush_dcache_page(pages[i]);
+ memzero_page(pages[i], 0, PAGE_SIZE);
SetPageUptodate(pages[i]);
}
return ((loff_t)pcount) << PAGE_SHIFT;
return 0;
}
haveblocks = isofs_get_blocks(inode, blocknum, bhs, needblocks);
- ll_rw_block(REQ_OP_READ, haveblocks, bhs);
+ bh_read_batch(haveblocks, bhs);
curbh = 0;
curpage = 0;
zerr != Z_STREAM_END) {
if (!stream.avail_out) {
if (pages[curpage]) {
- stream.next_out = page_address(pages[curpage])
+ stream.next_out = kmap_local_page(pages[curpage])
+ poffset;
stream.avail_out = PAGE_SIZE - poffset;
poffset = 0;
flush_dcache_page(pages[curpage]);
SetPageUptodate(pages[curpage]);
}
+ if (stream.next_out != (unsigned char *)zisofs_sink_page) {
+ kunmap_local(stream.next_out);
+ stream.next_out = NULL;
+ }
curpage++;
}
if (!stream.avail_in)
}
inflate_out:
zlib_inflateEnd(&stream);
+ if (stream.next_out && stream.next_out != (unsigned char *)zisofs_sink_page)
+ kunmap_local(stream.next_out);
z_eio:
mutex_unlock(&zisofs_zlib_lock);
}
if (poffset && *pages) {
- memset(page_address(*pages) + poffset, 0,
- PAGE_SIZE - poffset);
- flush_dcache_page(*pages);
+ memzero_page(*pages, poffset, PAGE_SIZE - poffset);
SetPageUptodate(*pages);
}
return 0;
for (i = 0; i < pcount; i++, index++) {
if (i != full_page)
pages[i] = grab_cache_page_nowait(mapping, index);
- if (pages[i]) {
+ if (pages[i])
ClearPageError(pages[i]);
- kmap(pages[i]);
- }
}
err = zisofs_fill_pages(inode, full_page, pcount, pages);
flush_dcache_page(pages[i]);
if (i == full_page && err)
SetPageError(pages[i]);
- kunmap(pages[i]);
unlock_page(pages[i]);
if (i != full_page)
put_page(pages[i]);
{
struct buffer_head *bh;
journal_superblock_t *sb;
- int err = -EIO;
+ int err;
bh = journal->j_sb_buffer;
J_ASSERT(bh != NULL);
- if (!buffer_uptodate(bh)) {
- ll_rw_block(REQ_OP_READ, 1, &bh);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh)) {
- printk(KERN_ERR
- "JBD2: IO error reading journal superblock\n");
- goto out;
- }
+ err = bh_read(bh, 0);
+ if (err < 0) {
+ printk(KERN_ERR
+ "JBD2: IO error reading journal superblock\n");
+ goto out;
}
if (buffer_verified(bh))
if (!buffer_uptodate(bh) && !buffer_locked(bh)) {
bufs[nbufs++] = bh;
if (nbufs == MAXBUF) {
- ll_rw_block(REQ_OP_READ, nbufs, bufs);
+ bh_readahead_batch(nbufs, bufs, 0);
journal_brelse_array(bufs, nbufs);
nbufs = 0;
}
}
if (nbufs)
- ll_rw_block(REQ_OP_READ, nbufs, bufs);
+ bh_readahead_batch(nbufs, bufs, 0);
err = 0;
failed:
return -ENOMEM;
if (!buffer_uptodate(bh)) {
- /* If this is a brand new buffer, start readahead.
- Otherwise, we assume we are already reading it. */
- if (!buffer_req(bh))
+ /*
+ * If this is a brand new buffer, start readahead.
+ * Otherwise, we assume we are already reading it.
+ */
+ bool need_readahead = !buffer_req(bh);
+
+ bh_read_nowait(bh, 0);
+ if (need_readahead)
do_readahead(journal, offset);
wait_on_buffer(bh);
}
mark_buffer_dirty(nbh);
BUFFER_TRACE(nbh, "marking uptodate");
++info->nr_replays;
- /* ll_rw_block(WRITE, 1, &nbh); */
unlock_buffer(nbh);
brelse(obh);
brelse(nbh);
#include <linux/mutex.h>
#include <linux/namei.h>
#include <linux/exportfs.h>
+#include <linux/iversion.h>
#include <linux/writeback.h>
#include <linux/buffer_head.h> /* sync_mapping_buffers */
#include <linux/fs_context.h>
#endif
}
EXPORT_SYMBOL(generic_set_encrypted_ci_d_ops);
+
+/**
+ * inode_maybe_inc_iversion - increments i_version
+ * @inode: inode with the i_version that should be updated
+ * @force: increment the counter even if it's not necessary?
+ *
+ * Every time the inode is modified, the i_version field must be seen to have
+ * changed by any observer.
+ *
+ * If "force" is set or the QUERIED flag is set, then ensure that we increment
+ * the value, and clear the queried flag.
+ *
+ * In the common case where neither is set, then we can return "false" without
+ * updating i_version.
+ *
+ * If this function returns false, and no other metadata has changed, then we
+ * can avoid logging the metadata.
+ */
+bool inode_maybe_inc_iversion(struct inode *inode, bool force)
+{
+ u64 cur, new;
+
+ /*
+ * The i_version field is not strictly ordered with any other inode
+ * information, but the legacy inode_inc_iversion code used a spinlock
+ * to serialize increments.
+ *
+ * Here, we add full memory barriers to ensure that any de-facto
+ * ordering with other info is preserved.
+ *
+ * This barrier pairs with the barrier in inode_query_iversion()
+ */
+ smp_mb();
+ cur = inode_peek_iversion_raw(inode);
+ do {
+ /* If flag is clear then we needn't do anything */
+ if (!force && !(cur & I_VERSION_QUERIED))
+ return false;
+
+ /* Since lowest bit is flag, add 2 to avoid it */
+ new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT;
+ } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new));
+ return true;
+}
+EXPORT_SYMBOL(inode_maybe_inc_iversion);
}
static int minix_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
int error;
struct inode *inode = minix_new_inode(dir, mode, &error);
if (inode) {
minix_set_inode(inode, 0);
mark_inode_dirty(inode);
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
}
- return error;
+ return finish_open_simple(file, error);
}
static int minix_create(struct user_namespace *mnt_userns, struct inode *dir,
* On non-idmapped mounts or if permission checking is to be performed on the
* raw inode simply passs init_user_ns.
*/
-struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns,
- struct dentry *dentry, umode_t mode, int open_flag)
+static int vfs_tmpfile(struct user_namespace *mnt_userns,
+ const struct path *parentpath,
+ struct file *file, umode_t mode)
{
- struct dentry *child = NULL;
- struct inode *dir = dentry->d_inode;
+ struct dentry *child;
+ struct inode *dir = d_inode(parentpath->dentry);
struct inode *inode;
int error;
/* we want directory to be writable */
error = inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC);
if (error)
- goto out_err;
- error = -EOPNOTSUPP;
+ return error;
if (!dir->i_op->tmpfile)
- goto out_err;
- error = -ENOMEM;
- child = d_alloc(dentry, &slash_name);
+ return -EOPNOTSUPP;
+ child = d_alloc(parentpath->dentry, &slash_name);
if (unlikely(!child))
- goto out_err;
+ return -ENOMEM;
+ file->f_path.mnt = parentpath->mnt;
+ file->f_path.dentry = child;
mode = vfs_prepare_mode(mnt_userns, dir, mode, mode, mode);
- error = dir->i_op->tmpfile(mnt_userns, dir, child, mode);
+ error = dir->i_op->tmpfile(mnt_userns, dir, file, mode);
+ dput(child);
if (error)
- goto out_err;
- error = -ENOENT;
- inode = child->d_inode;
- if (unlikely(!inode))
- goto out_err;
- if (!(open_flag & O_EXCL)) {
+ return error;
+ /* Don't check for other permissions, the inode was just created */
+ error = may_open(mnt_userns, &file->f_path, 0, file->f_flags);
+ if (error)
+ return error;
+ inode = file_inode(file);
+ if (!(file->f_flags & O_EXCL)) {
spin_lock(&inode->i_lock);
inode->i_state |= I_LINKABLE;
spin_unlock(&inode->i_lock);
}
ima_post_create_tmpfile(mnt_userns, inode);
- return child;
+ return 0;
+}
-out_err:
- dput(child);
- return ERR_PTR(error);
+/**
+ * vfs_tmpfile_open - open a tmpfile for kernel internal use
+ * @mnt_userns: user namespace of the mount the inode was found from
+ * @parentpath: path of the base directory
+ * @mode: mode of the new tmpfile
+ * @open_flag: flags
+ * @cred: credentials for open
+ *
+ * Create and open a temporary file. The file is not accounted in nr_files,
+ * hence this is only for kernel internal use, and must not be installed into
+ * file tables or such.
+ */
+struct file *vfs_tmpfile_open(struct user_namespace *mnt_userns,
+ const struct path *parentpath,
+ umode_t mode, int open_flag, const struct cred *cred)
+{
+ struct file *file;
+ int error;
+
+ file = alloc_empty_file_noaccount(open_flag, cred);
+ if (!IS_ERR(file)) {
+ error = vfs_tmpfile(mnt_userns, parentpath, file, mode);
+ if (error) {
+ fput(file);
+ file = ERR_PTR(error);
+ }
+ }
+ return file;
}
-EXPORT_SYMBOL(vfs_tmpfile);
+EXPORT_SYMBOL(vfs_tmpfile_open);
static int do_tmpfile(struct nameidata *nd, unsigned flags,
const struct open_flags *op,
struct file *file)
{
struct user_namespace *mnt_userns;
- struct dentry *child;
struct path path;
int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
+
if (unlikely(error))
return error;
error = mnt_want_write(path.mnt);
if (unlikely(error))
goto out;
mnt_userns = mnt_user_ns(path.mnt);
- child = vfs_tmpfile(mnt_userns, path.dentry, op->mode, op->open_flag);
- error = PTR_ERR(child);
- if (IS_ERR(child))
+ error = vfs_tmpfile(mnt_userns, &path, file, op->mode);
+ if (error)
goto out2;
- dput(path.dentry);
- path.dentry = child;
- audit_inode(nd->name, child, 0);
- /* Don't check for other permissions, the inode was just created */
- error = may_open(mnt_userns, &path, 0, op->open_flag);
- if (!error)
- error = vfs_open(&path, file);
+ audit_inode(nd->name, file->f_path.dentry, 0);
out2:
mnt_drop_write(path.mnt);
out:
const struct address_space_operations *aops = mapping->a_ops;
bool nofs = !mapping_gfp_constraint(mapping, __GFP_FS);
struct page *page;
- void *fsdata;
+ void *fsdata = NULL;
int err;
unsigned int flags;
goto out;
}
if (mntflags & NFS_MOUNT_WRITE_WAIT) {
- result = filemap_fdatawait_range(file->f_mapping,
- iocb->ki_pos - written,
- iocb->ki_pos - 1);
+ filemap_fdatawait_range(file->f_mapping,
+ iocb->ki_pos - written,
+ iocb->ki_pos - 1);
}
result = generic_write_sync(iocb, written);
if (result < 0)
#define FF_LAYOUT_POLL_RETRY_MAX (15*HZ)
#define FF_LAYOUTRETURN_MAXERR 20
+enum nfs4_ff_op_type {
+ NFS4_FF_OP_LAYOUTSTATS,
+ NFS4_FF_OP_LAYOUTRETURN,
+};
+
static unsigned short io_maxretrans;
static const struct pnfs_commit_ops ff_layout_commit_ops;
static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
struct nfs_pgio_header *hdr);
-static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
+static int
+ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
struct nfs42_layoutstat_devinfo *devinfo,
- int dev_limit);
+ int dev_limit, enum nfs4_ff_op_type type);
static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
const struct nfs42_layoutstat_devinfo *devinfo,
struct nfs4_ff_layout_mirror *mirror);
return -EIO;
}
+ if (!pnfs_is_valid_lseg(hdr->lseg)) {
+ rpc_exit(task, -EAGAIN);
+ return -EAGAIN;
+ }
+
ff_layout_read_record_layoutstats_start(task, hdr);
return 0;
}
return -EIO;
}
+ if (!pnfs_is_valid_lseg(hdr->lseg)) {
+ rpc_exit(task, -EAGAIN);
+ return -EAGAIN;
+ }
+
ff_layout_write_record_layoutstats_start(task, hdr);
return 0;
}
set_bit(NFS_LSEG_LAYOUTRETURN, &cdata->lseg->pls_flags);
}
-static void ff_layout_commit_prepare_common(struct rpc_task *task,
- struct nfs_commit_data *cdata)
+static int ff_layout_commit_prepare_common(struct rpc_task *task,
+ struct nfs_commit_data *cdata)
{
+ if (!pnfs_is_valid_lseg(cdata->lseg)) {
+ rpc_exit(task, -EAGAIN);
+ return -EAGAIN;
+ }
+
ff_layout_commit_record_layoutstats_start(task, cdata);
+ return 0;
}
static void ff_layout_commit_prepare_v3(struct rpc_task *task, void *data)
{
- ff_layout_commit_prepare_common(task, data);
+ if (ff_layout_commit_prepare_common(task, data))
+ return;
+
rpc_call_start(task);
}
ff_layout_initiate_commit);
}
+static bool ff_layout_match_rw(const struct rpc_task *task,
+ const struct nfs_pgio_header *hdr,
+ const struct pnfs_layout_segment *lseg)
+{
+ return hdr->lseg == lseg;
+}
+
+static bool ff_layout_match_commit(const struct rpc_task *task,
+ const struct nfs_commit_data *cdata,
+ const struct pnfs_layout_segment *lseg)
+{
+ return cdata->lseg == lseg;
+}
+
+static bool ff_layout_match_io(const struct rpc_task *task, const void *data)
+{
+ const struct rpc_call_ops *ops = task->tk_ops;
+
+ if (ops == &ff_layout_read_call_ops_v3 ||
+ ops == &ff_layout_read_call_ops_v4 ||
+ ops == &ff_layout_write_call_ops_v3 ||
+ ops == &ff_layout_write_call_ops_v4)
+ return ff_layout_match_rw(task, task->tk_calldata, data);
+ if (ops == &ff_layout_commit_call_ops_v3 ||
+ ops == &ff_layout_commit_call_ops_v4)
+ return ff_layout_match_commit(task, task->tk_calldata, data);
+ return false;
+}
+
+static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg)
+{
+ struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg);
+ struct nfs4_ff_layout_mirror *mirror;
+ struct nfs4_ff_layout_ds *mirror_ds;
+ struct nfs4_pnfs_ds *ds;
+ struct nfs_client *ds_clp;
+ struct rpc_clnt *clnt;
+ u32 idx;
+
+ for (idx = 0; idx < flseg->mirror_array_cnt; idx++) {
+ mirror = flseg->mirror_array[idx];
+ mirror_ds = mirror->mirror_ds;
+ if (!mirror_ds)
+ continue;
+ ds = mirror->mirror_ds->ds;
+ if (!ds)
+ continue;
+ ds_clp = ds->ds_clp;
+ if (!ds_clp)
+ continue;
+ clnt = ds_clp->cl_rpcclient;
+ if (!clnt)
+ continue;
+ if (!rpc_cancel_tasks(clnt, -EAGAIN, ff_layout_match_io, lseg))
+ continue;
+ rpc_clnt_disconnect(clnt);
+ }
+}
+
static struct pnfs_ds_commit_info *
ff_layout_get_ds_info(struct inode *inode)
{
FF_LAYOUTRETURN_MAXERR);
spin_lock(&args->inode->i_lock);
- ff_args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
- &ff_args->devinfo[0], ARRAY_SIZE(ff_args->devinfo));
+ ff_args->num_dev = ff_layout_mirror_prepare_stats(
+ &ff_layout->generic_hdr, &ff_args->devinfo[0],
+ ARRAY_SIZE(ff_args->devinfo), NFS4_FF_OP_LAYOUTRETURN);
spin_unlock(&args->inode->i_lock);
args->ld_private->ops = &layoutreturn_ops;
static int
ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
struct nfs42_layoutstat_devinfo *devinfo,
- int dev_limit)
+ int dev_limit, enum nfs4_ff_op_type type)
{
struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo);
struct nfs4_ff_layout_mirror *mirror;
break;
if (IS_ERR_OR_NULL(mirror->mirror_ds))
continue;
- if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags))
+ if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL,
+ &mirror->flags) &&
+ type != NFS4_FF_OP_LAYOUTRETURN)
continue;
/* mirror refcount put in cleanup_layoutstats */
if (!refcount_inc_not_zero(&mirror->ref))
spin_lock(&args->inode->i_lock);
ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout);
args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
- &args->devinfo[0], dev_count);
+ &args->devinfo[0],
+ dev_count,
+ NFS4_FF_OP_LAYOUTSTATS);
spin_unlock(&args->inode->i_lock);
if (!args->num_dev) {
kfree(args->devinfo);
.prepare_layoutreturn = ff_layout_prepare_layoutreturn,
.sync = pnfs_nfs_generic_sync,
.prepare_layoutstats = ff_layout_prepare_layoutstats,
+ .cancel_io = ff_layout_cancel_io,
};
static int __init nfs4flexfilelayout_init(void)
static int
nfs_find_actor(struct inode *inode, void *opaque)
{
- struct nfs_find_desc *desc = (struct nfs_find_desc *)opaque;
+ struct nfs_find_desc *desc = opaque;
struct nfs_fh *fh = desc->fh;
struct nfs_fattr *fattr = desc->fattr;
static int
nfs_init_locked(struct inode *inode, void *opaque)
{
- struct nfs_find_desc *desc = (struct nfs_find_desc *)opaque;
+ struct nfs_find_desc *desc = opaque;
struct nfs_fattr *fattr = desc->fattr;
set_nfs_fileid(inode, fattr->fileid);
static void init_once(void *foo)
{
- struct nfs_inode *nfsi = (struct nfs_inode *) foo;
+ struct nfs_inode *nfsi = foo;
inode_init_once(&nfsi->vfs_inode);
INIT_LIST_HEAD(&nfsi->open_files);
extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags);
extern bool nfs_check_cache_invalid(struct inode *, unsigned long);
extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode);
-extern int nfs_wait_atomic_killable(atomic_t *p, unsigned int mode);
/* super.c */
extern const struct super_operations nfs_sops;
const struct nfs_pgio_completion_ops *compl_ops);
extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio);
extern void nfs_commit_free(struct nfs_commit_data *p);
-extern void nfs_write_prepare(struct rpc_task *task, void *calldata);
extern void nfs_commit_prepare(struct rpc_task *task, void *calldata);
extern int nfs_initiate_commit(struct rpc_clnt *clnt,
struct nfs_commit_data *data,
ret = nfs4_call_sync(server->client, server, &msg, &args.seq_args,
&res.seq_res, 1);
+ trace_nfs4_removexattr(inode, name, ret);
if (!ret)
nfs4_update_changeattr(inode, &res.cinfo, timestamp, 0);
ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args,
&res.seq_res, 1);
+ trace_nfs4_setxattr(inode, name, ret);
for (; np > 0; np--)
put_page(pages[np - 1]);
ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args,
&res.seq_res, 0);
+ trace_nfs4_getxattr(inode, name, ret);
if (ret < 0)
return ret;
ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args,
&res.seq_res, 0);
+ trace_nfs4_listxattr(inode, ret);
if (ret >= 0) {
ret = res.copied;
static void nfs4_xattr_cache_init_once(void *p)
{
- struct nfs4_xattr_cache *cache = (struct nfs4_xattr_cache *)p;
+ struct nfs4_xattr_cache *cache = p;
spin_lock_init(&cache->listxattr_lock);
atomic_long_set(&cache->nent, 0);
*/
if (status == -ETOOSMALL)
status = -ERANGE;
+ /*
+ * Special case: for LISTXATTRS, NFS4ERR_NOXATTR
+ * should be translated to success with zero-length reply.
+ */
+ if (status == -ENODATA) {
+ res->eof = true;
+ status = 0;
+ }
goto out;
}
/* nfs4renewd.c */
extern void nfs4_schedule_state_renewal(struct nfs_client *);
-extern void nfs4_renewd_prepare_shutdown(struct nfs_server *);
extern void nfs4_kill_renewd(struct nfs_client *);
extern void nfs4_renew_state(struct work_struct *);
extern void nfs4_set_lease_period(struct nfs_client *clp, unsigned long lease);
goto error;
ip_addr = (const char *)buf;
}
- strlcpy(clp->cl_ipaddr, ip_addr, sizeof(clp->cl_ipaddr));
+ strscpy(clp->cl_ipaddr, ip_addr, sizeof(clp->cl_ipaddr));
err = nfs_idmap_new(clp);
if (err < 0) {
struct request_key_auth *rka = get_request_key_auth(authkey);
struct rpc_pipe_msg *msg;
struct idmap_msg *im;
- struct idmap *idmap = (struct idmap *)aux;
+ struct idmap *idmap = aux;
struct key *key = rka->target_key;
int ret = -ENOKEY;
struct nfs4_delegreturndata *d_data;
struct pnfs_layout_hdr *lo;
- d_data = (struct nfs4_delegreturndata *)data;
+ d_data = data;
if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task)) {
nfs4_sequence_done(task, &d_data->res.seq_res);
void nfs4_test_session_trunk(struct rpc_clnt *clnt, struct rpc_xprt *xprt,
void *data)
{
- struct nfs4_add_xprt_data *adata = (struct nfs4_add_xprt_data *)data;
+ struct nfs4_add_xprt_data *adata = data;
struct rpc_task *task;
int status;
sp = kzalloc(sizeof(*sp), gfp_flags);
if (!sp)
return NULL;
- sp->so_seqid.owner_id = ida_simple_get(&server->openowner_id, 0, 0,
- gfp_flags);
+ sp->so_seqid.owner_id = ida_alloc(&server->openowner_id, gfp_flags);
if (sp->so_seqid.owner_id < 0) {
kfree(sp);
return NULL;
{
nfs4_destroy_seqid_counter(&sp->so_seqid);
put_cred(sp->so_cred);
- ida_simple_remove(&sp->so_server->openowner_id, sp->so_seqid.owner_id);
+ ida_free(&sp->so_server->openowner_id, sp->so_seqid.owner_id);
kfree(sp);
}
refcount_set(&lsp->ls_count, 1);
lsp->ls_state = state;
lsp->ls_owner = fl_owner;
- lsp->ls_seqid.owner_id = ida_simple_get(&server->lockowner_id,
- 0, 0, GFP_KERNEL_ACCOUNT);
+ lsp->ls_seqid.owner_id = ida_alloc(&server->lockowner_id, GFP_KERNEL_ACCOUNT);
if (lsp->ls_seqid.owner_id < 0)
goto out_free;
INIT_LIST_HEAD(&lsp->ls_locks);
void nfs4_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp)
{
- ida_simple_remove(&server->lockowner_id, lsp->ls_seqid.owner_id);
+ ida_free(&server->lockowner_id, lsp->ls_seqid.owner_id);
nfs4_destroy_seqid_counter(&lsp->ls_seqid);
kfree(lsp);
}
)
);
+#ifdef CONFIG_NFS_V4_2
TRACE_DEFINE_ENUM(NFS4_CONTENT_DATA);
TRACE_DEFINE_ENUM(NFS4_CONTENT_HOLE);
{ NFS4_CONTENT_DATA, "DATA" }, \
{ NFS4_CONTENT_HOLE, "HOLE" })
-#ifdef CONFIG_NFS_V4_2
TRACE_EVENT(nfs4_llseek,
TP_PROTO(
const struct inode *inode,
__entry->stateid_seq, __entry->stateid_hash
)
);
+
+DECLARE_EVENT_CLASS(nfs4_xattr_event,
+ TP_PROTO(
+ const struct inode *inode,
+ const char *name,
+ int error
+ ),
+
+ TP_ARGS(inode, name, error),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, error)
+ __field(dev_t, dev)
+ __field(u32, fhandle)
+ __field(u64, fileid)
+ __string(name, name)
+ ),
+
+ TP_fast_assign(
+ __entry->error = error < 0 ? -error : 0;
+ __entry->dev = inode->i_sb->s_dev;
+ __entry->fileid = NFS_FILEID(inode);
+ __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
+ __assign_str(name, name);
+ ),
+
+ TP_printk(
+ "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x "
+ "name=%s",
+ -__entry->error, show_nfs4_status(__entry->error),
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ (unsigned long long)__entry->fileid,
+ __entry->fhandle, __get_str(name)
+ )
+);
+#define DEFINE_NFS4_XATTR_EVENT(name) \
+ DEFINE_EVENT(nfs4_xattr_event, name, \
+ TP_PROTO( \
+ const struct inode *inode, \
+ const char *name, \
+ int error \
+ ), \
+ TP_ARGS(inode, name, error))
+DEFINE_NFS4_XATTR_EVENT(nfs4_getxattr);
+DEFINE_NFS4_XATTR_EVENT(nfs4_setxattr);
+DEFINE_NFS4_XATTR_EVENT(nfs4_removexattr);
+
+DEFINE_NFS4_INODE_EVENT(nfs4_listxattr);
#endif /* CONFIG_NFS_V4_2 */
#endif /* CONFIG_NFS_V4_1 */
ROOT_DEV = Root_NFS;
if (line[0] == '/' || line[0] == ',' || (line[0] >= '0' && line[0] <= '9')) {
- strlcpy(nfs_root_parms, line, sizeof(nfs_root_parms));
+ strscpy(nfs_root_parms, line, sizeof(nfs_root_parms));
} else {
size_t n = strlen(line) + sizeof(NFS_ROOT) - 1;
if (n >= sizeof(nfs_root_parms))
u32 seq)
{
struct pnfs_layout_segment *lseg, *next;
+ struct nfs_server *server = NFS_SERVER(lo->plh_inode);
int remaining = 0;
dprintk("%s:Begin lo %p\n", __func__, lo);
"offset %llu length %llu\n", __func__,
lseg, lseg->pls_range.iomode, lseg->pls_seq,
lseg->pls_range.offset, lseg->pls_range.length);
- if (!mark_lseg_invalid(lseg, tmp_list))
- remaining++;
+ if (mark_lseg_invalid(lseg, tmp_list))
+ continue;
+ remaining++;
+ pnfs_lseg_cancel_io(server, lseg);
}
dprintk("%s:Return %i\n", __func__, remaining);
return remaining;
u32 seq)
{
struct pnfs_layout_segment *lseg, *next;
+ struct nfs_server *server = NFS_SERVER(lo->plh_inode);
int remaining = 0;
dprintk("%s:Begin lo %p\n", __func__, lo);
continue;
remaining++;
set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
+ pnfs_lseg_cancel_io(server, lseg);
}
if (remaining) {
void (*cleanup_layoutcommit) (struct nfs4_layoutcommit_data *data);
int (*prepare_layoutcommit) (struct nfs4_layoutcommit_args *args);
int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args);
+
+ void (*cancel_io)(struct pnfs_layout_segment *lseg);
};
struct pnfs_commit_ops {
req_offset(req), req_last);
}
+static inline void pnfs_lseg_cancel_io(struct nfs_server *server,
+ struct pnfs_layout_segment *lseg)
+{
+ if (server->pnfs_curr_ld->cancel_io)
+ server->pnfs_curr_ld->cancel_io(lseg);
+}
+
extern unsigned int layoutstats_timer;
#ifdef NFS_DEBUG
return NULL;
}
-/* pnfs_generic_search_commit_reqs - Search lists in @cinfo for the head reqest
+/* pnfs_generic_search_commit_reqs - Search lists in @cinfo for the head request
* for @page
* @cinfo - commit info for current inode
* @page - page to search for matching head request
*
- * Returns a the head request if one is found, otherwise returns NULL.
+ * Return: the head request if one is found, otherwise %NULL.
*/
struct nfs_page *
pnfs_generic_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page)
return false;
}
-/*
- * Return true if the file was unhashed.
- */
-static bool
+static void
nfsd_file_unhash_and_dispose(struct nfsd_file *nf, struct list_head *dispose)
{
trace_nfsd_file_unhash_and_dispose(nf);
- if (!nfsd_file_unhash(nf))
- return false;
- /* keep final reference for nfsd_file_lru_dispose */
- if (refcount_dec_not_one(&nf->nf_ref))
- return true;
-
- nfsd_file_lru_remove(nf);
- list_add(&nf->nf_lru, dispose);
- return true;
+ if (nfsd_file_unhash(nf)) {
+ /* caller must call nfsd_file_dispose_list() later */
+ nfsd_file_lru_remove(nf);
+ list_add(&nf->nf_lru, dispose);
+ }
}
static void
* @lock: LRU list lock (unused)
* @arg: dispose list
*
- * Note this can deadlock with nfsd_file_cache_purge.
- *
* Return values:
* %LRU_REMOVED: @item was removed from the LRU
* %LRU_ROTATE: @item is to be moved to the LRU tail
*
* Walk the LRU list and close any entries that have not been used since
* the last scan.
- *
- * Note this can deadlock with nfsd_file_cache_purge.
*/
static void
nfsd_file_delayed_close(struct work_struct *work)
goto out;
}
-/*
- * Note this can deadlock with nfsd_file_lru_cb.
- */
static void
__nfsd_file_cache_purge(struct net *net)
{
struct rhashtable_iter iter;
struct nfsd_file *nf;
LIST_HEAD(dispose);
- bool del;
rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter);
do {
while (!IS_ERR_OR_NULL(nf)) {
if (net && nf->nf_net != net)
continue;
- del = nfsd_file_unhash_and_dispose(nf, &dispose);
-
- /*
- * Deadlock detected! Something marked this entry as
- * unhased, but hasn't removed it from the hash list.
- */
- WARN_ON_ONCE(!del);
-
+ nfsd_file_unhash_and_dispose(nf, &dispose);
nf = rhashtable_walk_next(&iter);
}
.need = may_flags & NFSD_FILE_MAY_MASK,
.net = SVC_NET(rqstp),
};
- struct nfsd_file *nf, *new;
- bool retry = true;
+ bool open_retry = true;
+ struct nfsd_file *nf;
__be32 status;
+ int ret;
status = fh_verify(rqstp, fhp, S_IFREG,
may_flags|NFSD_MAY_OWNER_OVERRIDE);
key.cred = get_current_cred();
retry:
- /* Avoid allocation if the item is already in cache */
- nf = rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
- nfsd_file_rhash_params);
+ rcu_read_lock();
+ nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
+ nfsd_file_rhash_params);
if (nf)
nf = nfsd_file_get(nf);
+ rcu_read_unlock();
if (nf)
goto wait_for_construction;
- new = nfsd_file_alloc(&key, may_flags);
- if (!new) {
+ nf = nfsd_file_alloc(&key, may_flags);
+ if (!nf) {
status = nfserr_jukebox;
goto out_status;
}
- nf = rhashtable_lookup_get_insert_key(&nfsd_file_rhash_tbl,
- &key, &new->nf_rhash,
- nfsd_file_rhash_params);
- if (!nf) {
- nf = new;
- goto open_file;
- }
- if (IS_ERR(nf))
- goto insert_err;
- nf = nfsd_file_get(nf);
- if (nf == NULL) {
- nf = new;
+ ret = rhashtable_lookup_insert_key(&nfsd_file_rhash_tbl,
+ &key, &nf->nf_rhash,
+ nfsd_file_rhash_params);
+ if (likely(ret == 0))
goto open_file;
- }
- nfsd_file_slab_free(&new->nf_rcu);
+
+ nfsd_file_slab_free(&nf->nf_rcu);
+ if (ret == -EEXIST)
+ goto retry;
+ trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, ret);
+ status = nfserr_jukebox;
+ goto out_status;
wait_for_construction:
wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
/* Did construction of this file fail? */
if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf);
- if (!retry) {
+ if (!open_retry) {
status = nfserr_jukebox;
goto out;
}
- retry = false;
+ open_retry = false;
nfsd_file_put_noref(nf);
goto retry;
}
smp_mb__after_atomic();
wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
goto out;
-
-insert_err:
- nfsd_file_slab_free(&new->nf_rcu);
- trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, PTR_ERR(nf));
- nf = NULL;
- status = nfserr_jukebox;
- goto out_status;
}
/**
nn->nfsd4_grace = 90;
nn->somebody_reclaimed = false;
nn->track_reclaim_completes = false;
- nn->clverifier_counter = prandom_u32();
- nn->clientid_base = prandom_u32();
+ nn->clverifier_counter = get_random_u32();
+ nn->clientid_base = get_random_u32();
nn->clientid_counter = nn->clientid_base + 1;
nn->s2s_cp_cl_id = nn->clientid_counter++;
maxkey = nilfs_btree_node_get_key(node, nchildren - 1);
nextmaxkey = (nchildren > 1) ?
nilfs_btree_node_get_key(node, nchildren - 2) : 0;
- if (bh != NULL)
- brelse(bh);
+ brelse(bh);
return (maxkey == key) && (nextmaxkey < NILFS_BMAP_LARGE_LOW);
}
ptrs[i] = le64_to_cpu(dptrs[i]);
}
- if (bh != NULL)
- brelse(bh);
+ brelse(bh);
return nitems;
}
struct inode *inode;
struct nilfs_inode_info *ii;
struct nilfs_root *root;
+ struct buffer_head *bh;
int err = -ENOMEM;
ino_t ino;
ii->i_state = BIT(NILFS_I_NEW);
ii->i_root = root;
- err = nilfs_ifile_create_inode(root->ifile, &ino, &ii->i_bh);
+ err = nilfs_ifile_create_inode(root->ifile, &ino, &bh);
if (unlikely(err))
goto failed_ifile_create_inode;
/* reference count of i_bh inherits from nilfs_mdt_read_block() */
+ if (unlikely(ino < NILFS_USER_INO)) {
+ nilfs_warn(sb,
+ "inode bitmap is inconsistent for reserved inodes");
+ do {
+ brelse(bh);
+ err = nilfs_ifile_create_inode(root->ifile, &ino, &bh);
+ if (unlikely(err))
+ goto failed_ifile_create_inode;
+ } while (ino < NILFS_USER_INO);
+
+ nilfs_info(sb, "repaired inode bitmap for reserved inodes");
+ }
+ ii->i_bh = bh;
+
atomic64_inc(&root->inodes_count);
inode_init_owner(&init_user_ns, inode, dir, mode);
inode->i_ino = ino;
inode->i_atime.tv_nsec = le32_to_cpu(raw_inode->i_mtime_nsec);
inode->i_ctime.tv_nsec = le32_to_cpu(raw_inode->i_ctime_nsec);
inode->i_mtime.tv_nsec = le32_to_cpu(raw_inode->i_mtime_nsec);
+ if (nilfs_is_metadata_file_inode(inode) && !S_ISREG(inode->i_mode))
+ return -EIO; /* this inode is for metadata and corrupted */
if (inode->i_nlink == 0)
return -ESTALE; /* this inode is deleted */
sector_t start_blk,
sector_t *blkoff)
{
- unsigned int i;
+ unsigned int i, nr_folios;
pgoff_t index;
- unsigned int nblocks_in_page;
unsigned long length = 0;
- sector_t b;
- struct pagevec pvec;
- struct page *page;
+ struct folio_batch fbatch;
+ struct folio *folio;
if (inode->i_mapping->nrpages == 0)
return 0;
index = start_blk >> (PAGE_SHIFT - inode->i_blkbits);
- nblocks_in_page = 1U << (PAGE_SHIFT - inode->i_blkbits);
- pagevec_init(&pvec);
+ folio_batch_init(&fbatch);
repeat:
- pvec.nr = find_get_pages_contig(inode->i_mapping, index, PAGEVEC_SIZE,
- pvec.pages);
- if (pvec.nr == 0)
+ nr_folios = filemap_get_folios_contig(inode->i_mapping, &index, ULONG_MAX,
+ &fbatch);
+ if (nr_folios == 0)
return length;
- if (length > 0 && pvec.pages[0]->index > index)
- goto out;
-
- b = pvec.pages[0]->index << (PAGE_SHIFT - inode->i_blkbits);
i = 0;
do {
- page = pvec.pages[i];
+ folio = fbatch.folios[i];
- lock_page(page);
- if (page_has_buffers(page)) {
+ folio_lock(folio);
+ if (folio_buffers(folio)) {
struct buffer_head *bh, *head;
+ sector_t b;
- bh = head = page_buffers(page);
+ b = folio->index << (PAGE_SHIFT - inode->i_blkbits);
+ bh = head = folio_buffers(folio);
do {
if (b < start_blk)
continue;
} else {
if (length > 0)
goto out_locked;
-
- b += nblocks_in_page;
}
- unlock_page(page);
+ folio_unlock(folio);
- } while (++i < pagevec_count(&pvec));
+ } while (++i < nr_folios);
- index = page->index + 1;
- pagevec_release(&pvec);
+ folio_batch_release(&fbatch);
cond_resched();
goto repeat;
out_locked:
- unlock_page(page);
-out:
- pagevec_release(&pvec);
+ folio_unlock(folio);
+ folio_batch_release(&fbatch);
return length;
}
nilfs_mdt_mark_dirty(nilfs->ns_cpfile);
nilfs_cpfile_put_checkpoint(
nilfs->ns_cpfile, nilfs->ns_cno, bh_cp);
- } else
- WARN_ON(err == -EINVAL || err == -ENOENT);
-
+ } else if (err == -EINVAL || err == -ENOENT) {
+ nilfs_error(sci->sc_super,
+ "checkpoint creation failed due to metadata corruption.");
+ err = -EIO;
+ }
return err;
}
err = nilfs_cpfile_get_checkpoint(nilfs->ns_cpfile, nilfs->ns_cno, 0,
&raw_cp, &bh_cp);
if (unlikely(err)) {
- WARN_ON(err == -EINVAL || err == -ENOENT);
+ if (err == -EINVAL || err == -ENOENT) {
+ nilfs_error(sci->sc_super,
+ "checkpoint finalization failed due to metadata corruption.");
+ err = -EIO;
+ }
goto failed_ibh;
}
raw_cp->cp_snapshot_list.ssl_next = 0;
struct the_nilfs *nilfs = sb->s_fs_info;
struct nilfs_sc_info *sci = nilfs->ns_writer;
struct nilfs_transaction_info *ti;
- int err;
if (!sci)
return -EROFS;
/* A call inside transactions causes a deadlock. */
BUG_ON((ti = current->journal_info) && ti->ti_magic == NILFS_TI_MAGIC);
- err = nilfs_segctor_sync(sci);
- return err;
+ return nilfs_segctor_sync(sci);
}
/**
inode_attach_wb(nilfs->ns_bdev->bd_inode, NULL);
err = nilfs_segctor_start_thread(nilfs->ns_writer);
- if (err) {
- kfree(nilfs->ns_writer);
- nilfs->ns_writer = NULL;
- }
+ if (unlikely(err))
+ nilfs_detach_log_writer(sb);
+
return err;
}
for (;; a = (ATTR_RECORD*)((u8*)a + le32_to_cpu(a->length))) {
u8 *mrec_end = (u8 *)ctx->mrec +
le32_to_cpu(ctx->mrec->bytes_allocated);
- u8 *name_end = (u8 *)a + le16_to_cpu(a->name_offset) +
- a->name_length * sizeof(ntfschar);
- if ((u8*)a < (u8*)ctx->mrec || (u8*)a > mrec_end ||
- name_end > mrec_end)
+ u8 *name_end;
+
+ /* check whether ATTR_RECORD wrap */
+ if ((u8 *)a < (u8 *)ctx->mrec)
+ break;
+
+ /* check whether Attribute Record Header is within bounds */
+ if ((u8 *)a > mrec_end ||
+ (u8 *)a + sizeof(ATTR_RECORD) > mrec_end)
+ break;
+
+ /* check whether ATTR_RECORD's name is within bounds */
+ name_end = (u8 *)a + le16_to_cpu(a->name_offset) +
+ a->name_length * sizeof(ntfschar);
+ if (name_end > mrec_end)
break;
+
ctx->attr = a;
if (unlikely(le32_to_cpu(a->type) > le32_to_cpu(type) ||
a->type == AT_END))
return -ENOENT;
if (unlikely(!a->length))
break;
+
+ /* check whether ATTR_RECORD's length wrap */
+ if ((u8 *)a + le32_to_cpu(a->length) < (u8 *)a)
+ break;
+ /* check whether ATTR_RECORD's length is within bounds */
+ if ((u8 *)a + le32_to_cpu(a->length) > mrec_end)
+ break;
+
if (a->type != type)
continue;
/*
goto err_out;
}
+ /* Sanity check offset to the first attribute */
+ if (le16_to_cpu(m->attrs_offset) >= le32_to_cpu(m->bytes_allocated)) {
+ ntfs_error(sb, "Incorrect mft offset to the first attribute %u in superblock.",
+ le16_to_cpu(m->attrs_offset));
+ goto err_out;
+ }
+
/* Need this to sanity check attribute list references to $MFT. */
vi->i_generation = ni->seq_no = le16_to_cpu(m->sequence_number);
}
log_init_pg_hdr(log, page_size, page_size, 1, 1);
- log_create(log, l_size, 0, get_random_int(), false, false);
+ log_create(log, l_size, 0, get_random_u32(), false, false);
log->ra = ra;
/* Do some checks based on whether we have a valid log page. */
if (!rst_info.valid_page) {
- open_log_count = get_random_int();
+ open_log_count = get_random_u32();
goto init_log_instance;
}
open_log_count = le32_to_cpu(ra2->open_log_count);
memcpy(ra->clients, Add2Ptr(ra2, t16),
le16_to_cpu(ra2->ra_len) - t16);
- log->current_openlog_count = get_random_int();
+ log->current_openlog_count = get_random_u32();
ra->open_log_count = cpu_to_le32(log->current_openlog_count);
log->ra_size = offsetof(struct RESTART_AREA, clients) +
sizeof(struct CLIENT_REC);
bh->b_size = block_size;
off = vbo & (PAGE_SIZE - 1);
set_bh_page(bh, page, off);
- ll_rw_block(REQ_OP_READ, 1, &bh);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh)) {
- err = -EIO;
+ err = bh_read(bh, 0);
+ if (err < 0)
goto out;
- }
zero_user_segment(page, off + voff, off + block_size);
}
}
!buffer_new(bh) &&
ocfs2_should_read_blk(inode, page, block_start) &&
(block_start < from || block_end > to)) {
- ll_rw_block(REQ_OP_READ, 1, &bh);
+ bh_read_nowait(bh, 0);
*wait_bh++=bh;
}
* value -1 (0xFFFF) is OCFS2_INVALID_SLOT. This marks a slot empty.
*/
struct ocfs2_slot_map {
-/*00*/ __le16 sm_slots[0];
+/*00*/ DECLARE_FLEX_ARRAY(__le16, sm_slots);
/*
* Actual on-disk size is one block. OCFS2_MAX_SLOTS is 255,
* 255 * sizeof(__le16) == 512B, within the 512B block minimum blocksize.
* i_size.
*/
struct ocfs2_slot_map_extended {
-/*00*/ struct ocfs2_extended_slot se_slots[0];
+/*00*/ DECLARE_FLEX_ARRAY(struct ocfs2_extended_slot, se_slots);
/*
* Actual size is i_size of the slot_map system file. It should
* match s_max_slots * sizeof(struct ocfs2_extended_slot)
struct ocfs2_extent_list i_list;
struct ocfs2_truncate_log i_dealloc;
struct ocfs2_inline_data i_data;
- __u8 i_symlink[0];
+ DECLARE_FLEX_ARRAY(__u8, i_symlink);
} id2;
/* Actual on-disk size is one block */
};
/*30*/ struct ocfs2_block_check bg_check; /* Error checking */
__le64 bg_reserved2;
/*40*/ union {
- __u8 bg_bitmap[0];
+ DECLARE_FLEX_ARRAY(__u8, bg_bitmap);
struct {
/*
* Block groups may be discontiguous when
}
/*
- * Calculate out the start and number of virtual clusters we need to to CoW.
+ * Calculate out the start and number of virtual clusters we need to CoW.
*
* cpos is vitual start cluster position we want to do CoW in a
* file and write_len is the cluster length.
goto out;
}
- strlcpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1);
+ strscpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1);
new_conn->cc_namelen = grouplen;
if (cluster_name_len)
- strlcpy(new_conn->cc_cluster_name, cluster_name,
+ strscpy(new_conn->cc_cluster_name, cluster_name,
CLUSTER_NAME_MAX + 1);
new_conn->cc_cluster_name_len = cluster_name_len;
new_conn->cc_recovery_handler = recovery_handler;
u32 *cluster_start,
u32 *num_clusters);
/*
- * Use this variant of ocfs2_claim_clusters to specify a maxiumum
+ * Use this variant of ocfs2_claim_clusters to specify a maximum
* number of clusters smaller than the allocation reserved.
*/
int __ocfs2_claim_clusters(handle_t *handle,
if (!buffer_dirty(*bh))
clear_buffer_uptodate(*bh);
unlock_buffer(*bh);
- ll_rw_block(REQ_OP_READ, 1, bh);
- wait_on_buffer(*bh);
- if (!buffer_uptodate(*bh)) {
+ if (bh_read(*bh, 0) < 0) {
mlog_errno(-EIO);
brelse(*bh);
*bh = NULL;
goto out_journal;
}
- strlcpy(osb->vol_label, di->id2.i_super.s_label,
+ strscpy(osb->vol_label, di->id2.i_super.s_label,
OCFS2_MAX_VOL_LABEL_LEN);
osb->root_blkno = le64_to_cpu(di->id2.i_super.s_root_blkno);
osb->system_dir_blkno = le64_to_cpu(di->id2.i_super.s_system_dir_blkno);
const struct file_operations orangefs_dir_operations = {
.llseek = orangefs_dir_llseek,
.read = generic_read_dir,
- .iterate = orangefs_dir_iterate,
+ .iterate_shared = orangefs_dir_iterate,
.open = orangefs_dir_open,
.release = orangefs_dir_release
};
return ovl_real_fileattr_set(new, &newfa);
}
-static int ovl_copy_up_data(struct ovl_fs *ofs, const struct path *old,
- const struct path *new, loff_t len)
+static int ovl_copy_up_file(struct ovl_fs *ofs, struct dentry *dentry,
+ struct file *new_file, loff_t len)
{
+ struct path datapath;
struct file *old_file;
- struct file *new_file;
loff_t old_pos = 0;
loff_t new_pos = 0;
loff_t cloned;
bool skip_hole = false;
int error = 0;
- if (len == 0)
- return 0;
+ ovl_path_lowerdata(dentry, &datapath);
+ if (WARN_ON(datapath.dentry == NULL))
+ return -EIO;
- old_file = ovl_path_open(old, O_LARGEFILE | O_RDONLY);
+ old_file = ovl_path_open(&datapath, O_LARGEFILE | O_RDONLY);
if (IS_ERR(old_file))
return PTR_ERR(old_file);
- new_file = ovl_path_open(new, O_LARGEFILE | O_WRONLY);
- if (IS_ERR(new_file)) {
- error = PTR_ERR(new_file);
- goto out_fput;
- }
-
/* Try to use clone_file_range to clone up within the same fs */
cloned = do_clone_file_range(old_file, 0, new_file, 0, len, 0);
if (cloned == len)
- goto out;
+ goto out_fput;
/* Couldn't clone, so now we try to copy the data */
/* Check if lower fs supports seek operation */
len -= bytes;
}
-out:
if (!error && ovl_should_sync(ofs))
error = vfs_fsync(new_file, 0);
- fput(new_file);
out_fput:
fput(old_file);
return error;
return err;
}
-static int ovl_copy_up_inode(struct ovl_copy_up_ctx *c, struct dentry *temp)
+static int ovl_copy_up_data(struct ovl_copy_up_ctx *c, const struct path *temp)
{
struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
- struct inode *inode = d_inode(c->dentry);
- struct path upperpath, datapath;
+ struct file *new_file;
int err;
- ovl_path_upper(c->dentry, &upperpath);
- if (WARN_ON(upperpath.dentry != NULL))
- return -EIO;
+ if (!S_ISREG(c->stat.mode) || c->metacopy || !c->stat.size)
+ return 0;
- upperpath.dentry = temp;
+ new_file = ovl_path_open(temp, O_LARGEFILE | O_WRONLY);
+ if (IS_ERR(new_file))
+ return PTR_ERR(new_file);
- /*
- * Copy up data first and then xattrs. Writing data after
- * xattrs will remove security.capability xattr automatically.
- */
- if (S_ISREG(c->stat.mode) && !c->metacopy) {
- ovl_path_lowerdata(c->dentry, &datapath);
- err = ovl_copy_up_data(ofs, &datapath, &upperpath,
- c->stat.size);
- if (err)
- return err;
- }
+ err = ovl_copy_up_file(ofs, c->dentry, new_file, c->stat.size);
+ fput(new_file);
+
+ return err;
+}
+
+static int ovl_copy_up_metadata(struct ovl_copy_up_ctx *c, struct dentry *temp)
+{
+ struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
+ struct inode *inode = d_inode(c->dentry);
+ struct path upperpath = { .mnt = ovl_upper_mnt(ofs), .dentry = temp };
+ int err;
err = ovl_copy_xattr(c->dentry->d_sb, &c->lowerpath, temp);
if (err)
struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
struct inode *inode;
struct inode *udir = d_inode(c->destdir), *wdir = d_inode(c->workdir);
+ struct path path = { .mnt = ovl_upper_mnt(ofs) };
struct dentry *temp, *upper;
struct ovl_cu_creds cc;
int err;
if (IS_ERR(temp))
goto unlock;
- err = ovl_copy_up_inode(c, temp);
+ /*
+ * Copy up data first and then xattrs. Writing data after
+ * xattrs will remove security.capability xattr automatically.
+ */
+ path.dentry = temp;
+ err = ovl_copy_up_data(c, &path);
+ if (err)
+ goto cleanup;
+
+ err = ovl_copy_up_metadata(c, temp);
if (err)
goto cleanup;
struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
struct inode *udir = d_inode(c->destdir);
struct dentry *temp, *upper;
+ struct file *tmpfile;
struct ovl_cu_creds cc;
int err;
if (err)
return err;
- temp = ovl_do_tmpfile(ofs, c->workdir, c->stat.mode);
+ tmpfile = ovl_do_tmpfile(ofs, c->workdir, c->stat.mode);
ovl_revert_cu_creds(&cc);
- if (IS_ERR(temp))
- return PTR_ERR(temp);
+ if (IS_ERR(tmpfile))
+ return PTR_ERR(tmpfile);
- err = ovl_copy_up_inode(c, temp);
+ temp = tmpfile->f_path.dentry;
+ if (!c->metacopy && c->stat.size) {
+ err = ovl_copy_up_file(ofs, c->dentry, tmpfile, c->stat.size);
+ if (err)
+ return err;
+ }
+
+ err = ovl_copy_up_metadata(c, temp);
if (err)
- goto out_dput;
+ goto out_fput;
inode_lock_nested(udir, I_MUTEX_PARENT);
inode_unlock(udir);
if (err)
- goto out_dput;
+ goto out_fput;
if (!c->metacopy)
ovl_set_upperdata(d_inode(c->dentry));
- ovl_inode_update(d_inode(c->dentry), temp);
+ ovl_inode_update(d_inode(c->dentry), dget(temp));
- return 0;
-
-out_dput:
- dput(temp);
+out_fput:
+ fput(tmpfile);
return err;
}
static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c)
{
struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb);
- struct path upperpath, datapath;
+ struct path upperpath;
int err;
char *capability = NULL;
ssize_t cap_size;
if (WARN_ON(upperpath.dentry == NULL))
return -EIO;
- ovl_path_lowerdata(c->dentry, &datapath);
- if (WARN_ON(datapath.dentry == NULL))
- return -EIO;
-
if (c->stat.size) {
err = cap_size = ovl_getxattr_value(&upperpath, XATTR_NAME_CAPS,
&capability);
goto out;
}
- err = ovl_copy_up_data(ofs, &datapath, &upperpath, c->stat.size);
+ err = ovl_copy_up_data(c, &upperpath);
if (err)
goto out_free;
return err;
}
-static inline struct dentry *ovl_do_tmpfile(struct ovl_fs *ofs,
- struct dentry *dentry, umode_t mode)
+static inline struct file *ovl_do_tmpfile(struct ovl_fs *ofs,
+ struct dentry *dentry, umode_t mode)
{
- struct dentry *ret = vfs_tmpfile(ovl_upper_mnt_userns(ofs), dentry, mode, 0);
- int err = PTR_ERR_OR_ZERO(ret);
+ struct path path = { .mnt = ovl_upper_mnt(ofs), .dentry = dentry };
+ struct file *file = vfs_tmpfile_open(ovl_upper_mnt_userns(ofs), &path, mode,
+ O_LARGEFILE | O_WRONLY, current_cred());
+ int err = PTR_ERR_OR_ZERO(file);
pr_debug("tmpfile(%pd2, 0%o) = %i\n", dentry, mode, err);
- return ret;
+ return file;
}
static inline struct dentry *ovl_lookup_upper(struct ovl_fs *ofs,
#include <linux/seq_file.h>
#include <linux/posix_acl_xattr.h>
#include <linux/exportfs.h>
+#include <linux/file.h>
#include "overlayfs.h"
MODULE_AUTHOR("Miklos Szeredi <miklos@szeredi.hu>");
const struct path *workpath)
{
struct vfsmount *mnt = ovl_upper_mnt(ofs);
- struct dentry *temp, *workdir;
+ struct dentry *workdir;
+ struct file *tmpfile;
bool rename_whiteout;
bool d_type;
int fh_type;
pr_warn("upper fs needs to support d_type.\n");
/* Check if upper/work fs supports O_TMPFILE */
- temp = ovl_do_tmpfile(ofs, ofs->workdir, S_IFREG | 0);
- ofs->tmpfile = !IS_ERR(temp);
+ tmpfile = ovl_do_tmpfile(ofs, ofs->workdir, S_IFREG | 0);
+ ofs->tmpfile = !IS_ERR(tmpfile);
if (ofs->tmpfile)
- dput(temp);
+ fput(tmpfile);
else
pr_warn("upper fs does not support tmpfile.\n");
#include <linux/user_namespace.h>
#include <linux/namei.h>
#include <linux/mnt_idmapping.h>
+#include <linux/iversion.h>
static struct posix_acl **acl_by_type(struct inode *inode, int type)
{
}
inode->i_ctime = current_time(inode);
+ if (IS_I_VERSION(inode))
+ inode_inc_iversion(inode);
set_cached_acl(inode, type, acl);
return 0;
}
config PROC_CHILDREN
bool "Include /proc/<pid>/task/<tid>/children file"
+ depends on PROC_FS
default n
help
Provides a fast way to retrieve first level children pids of a task. See
GENRADIX(struct map_files_info) fa;
struct map_files_info *p;
int ret;
+ struct vma_iterator vmi;
genradix_init(&fa);
* routine might require mmap_lock taken in might_fault().
*/
- for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) {
+ pos = 2;
+ vma_iter_init(&vmi, mm, 0);
+ for_each_vma(vmi, vma) {
if (!vma->vm_file)
continue;
if (++pos <= ctx->pos)
return 0;
}
+static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns,
+ struct pid *pid, struct task_struct *task)
+{
+ struct mm_struct *mm;
+
+ mm = get_task_mm(task);
+ if (mm) {
+ seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items);
+ mmput(mm);
+ }
+
+ return 0;
+}
#endif /* CONFIG_KSM */
#ifdef CONFIG_STACKLEAK_METRICS
#endif
#ifdef CONFIG_KSM
ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages),
+ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat),
#endif
};
#endif
#ifdef CONFIG_KSM
ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages),
+ ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat),
#endif
};
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/blkdev.h>
+#include "internal.h"
static int devinfo_show(struct seq_file *f, void *v)
{
static int __init proc_devices_init(void)
{
- proc_create_seq("devices", 0, NULL, &devinfo_ops);
+ struct proc_dir_entry *pde;
+
+ pde = proc_create_seq("devices", 0, NULL, &devinfo_ops);
+ pde_make_permanent(pde);
return 0;
}
fs_initcall(proc_devices_init);
return pde->flags & PROC_ENTRY_PERMANENT;
}
+static inline void pde_make_permanent(struct proc_dir_entry *pde)
+{
+ pde->flags |= PROC_ENTRY_PERMANENT;
+}
+
extern struct kmem_cache *proc_dir_entry_cache;
void pde_free(struct proc_dir_entry *pde);
struct task_struct *task;
struct mm_struct *mm;
#ifdef CONFIG_MMU
- struct vm_area_struct *tail_vma;
+ struct vma_iterator iter;
#endif
#ifdef CONFIG_NUMA
struct mempolicy *task_mempolicy;
#include <linux/seq_file.h>
#include <linux/seqlock.h>
#include <linux/time.h>
+#include "internal.h"
static int loadavg_proc_show(struct seq_file *m, void *v)
{
static int __init proc_loadavg_init(void)
{
- proc_create_single("loadavg", 0, NULL, loadavg_proc_show);
+ struct proc_dir_entry *pde;
+
+ pde = proc_create_single("loadavg", 0, NULL, loadavg_proc_show);
+ pde_make_permanent(pde);
return 0;
}
fs_initcall(proc_loadavg_init);
static int __init proc_meminfo_init(void)
{
- proc_create_single("meminfo", 0, NULL, meminfo_proc_show);
+ struct proc_dir_entry *pde;
+
+ pde = proc_create_single("meminfo", 0, NULL, meminfo_proc_show);
+ pde_make_permanent(pde);
return 0;
}
fs_initcall(proc_meminfo_init);
}
static const struct proc_ops kpagecount_proc_ops = {
+ .proc_flags = PROC_ENTRY_PERMANENT,
.proc_lseek = mem_lseek,
.proc_read = kpagecount_read,
};
}
static const struct proc_ops kpageflags_proc_ops = {
+ .proc_flags = PROC_ENTRY_PERMANENT,
.proc_lseek = mem_lseek,
.proc_read = kpageflags_read,
};
}
static const struct proc_ops kpagecgroup_proc_ops = {
+ .proc_flags = PROC_ENTRY_PERMANENT,
.proc_lseek = mem_lseek,
.proc_read = kpagecgroup_read,
};
#include <linux/kernel_stat.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
+#include "internal.h"
/*
* /proc/softirqs ... display the number of softirqs
static int __init proc_softirqs_init(void)
{
- proc_create_single("softirqs", 0, NULL, show_softirqs);
+ struct proc_dir_entry *pde;
+
+ pde = proc_create_single("softirqs", 0, NULL, show_softirqs);
+ pde_make_permanent(pde);
return 0;
}
fs_initcall(proc_softirqs_init);
// SPDX-License-Identifier: GPL-2.0
#include <linux/pagewalk.h>
-#include <linux/vmacache.h>
#include <linux/mm_inline.h>
#include <linux/hugetlb.h>
#include <linux/huge_mm.h>
}
#endif
+static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv,
+ loff_t *ppos)
+{
+ struct vm_area_struct *vma = vma_next(&priv->iter);
+
+ if (vma) {
+ *ppos = vma->vm_start;
+ } else {
+ *ppos = -2UL;
+ vma = get_gate_vma(priv->mm);
+ }
+
+ return vma;
+}
+
static void *m_start(struct seq_file *m, loff_t *ppos)
{
struct proc_maps_private *priv = m->private;
unsigned long last_addr = *ppos;
struct mm_struct *mm;
- struct vm_area_struct *vma;
/* See m_next(). Zero at the start or after lseek. */
if (last_addr == -1UL)
return ERR_PTR(-EINTR);
}
+ vma_iter_init(&priv->iter, mm, last_addr);
hold_task_mempolicy(priv);
- priv->tail_vma = get_gate_vma(mm);
-
- vma = find_vma(mm, last_addr);
- if (vma)
- return vma;
+ if (last_addr == -2UL)
+ return get_gate_vma(mm);
- return priv->tail_vma;
+ return proc_get_vma(priv, ppos);
}
static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
{
- struct proc_maps_private *priv = m->private;
- struct vm_area_struct *next, *vma = v;
-
- if (vma == priv->tail_vma)
- next = NULL;
- else if (vma->vm_next)
- next = vma->vm_next;
- else
- next = priv->tail_vma;
-
- *ppos = next ? next->vm_start : -1UL;
-
- return next;
+ if (*ppos == -2UL) {
+ *ppos = -1UL;
+ return NULL;
+ }
+ return proc_get_vma(m->private, ppos);
}
static void m_stop(struct seq_file *m, void *v)
__show_smap(m, &mss, false);
seq_printf(m, "THPeligible: %d\n",
- hugepage_vma_check(vma, vma->vm_flags, true, false));
+ hugepage_vma_check(vma, vma->vm_flags, true, false, true));
if (arch_pkeys_enabled())
seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
{
struct proc_maps_private *priv = m->private;
struct mem_size_stats mss;
- struct mm_struct *mm;
+ struct mm_struct *mm = priv->mm;
struct vm_area_struct *vma;
- unsigned long last_vma_end = 0;
+ unsigned long vma_start = 0, last_vma_end = 0;
int ret = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
priv->task = get_proc_task(priv->inode);
if (!priv->task)
return -ESRCH;
- mm = priv->mm;
if (!mm || !mmget_not_zero(mm)) {
ret = -ESRCH;
goto out_put_task;
goto out_put_mm;
hold_task_mempolicy(priv);
+ vma = mas_find(&mas, 0);
+
+ if (unlikely(!vma))
+ goto empty_set;
- for (vma = priv->mm->mmap; vma;) {
+ vma_start = vma->vm_start;
+ do {
smap_gather_stats(vma, &mss, 0);
last_vma_end = vma->vm_end;
* access it for write request.
*/
if (mmap_lock_is_contended(mm)) {
+ mas_pause(&mas);
mmap_read_unlock(mm);
ret = mmap_read_lock_killable(mm);
if (ret) {
* contains last_vma_end.
* Iterate VMA' from last_vma_end.
*/
- vma = find_vma(mm, last_vma_end - 1);
+ vma = mas_find(&mas, ULONG_MAX);
/* Case 3 above */
if (!vma)
break;
smap_gather_stats(vma, &mss, last_vma_end);
}
/* Case 2 above */
- vma = vma->vm_next;
- }
+ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL);
- show_vma_header_prefix(m, priv->mm->mmap->vm_start,
- last_vma_end, 0, 0, 0, 0);
+empty_set:
+ show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
seq_pad(m, ' ');
seq_puts(m, "[rollup]\n");
return -ESRCH;
mm = get_task_mm(task);
if (mm) {
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
struct mmu_notifier_range range;
struct clear_refs_private cp = {
.type = type,
}
if (type == CLEAR_REFS_SOFT_DIRTY) {
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (!(vma->vm_flags & VM_SOFTDIRTY))
continue;
vma->vm_flags &= ~VM_SOFTDIRTY;
0, NULL, mm, 0, -1UL);
mmu_notifier_invalidate_range_start(&range);
}
- walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops,
- &cp);
+ walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp);
if (type == CLEAR_REFS_SOFT_DIRTY) {
mmu_notifier_invalidate_range_end(&range);
flush_tlb_mm(mm);
if (pte_swp_uffd_wp(pte))
flags |= PM_UFFD_WP;
entry = pte_to_swp_entry(pte);
- if (pm->show_pfn)
+ if (pm->show_pfn) {
+ pgoff_t offset;
+ /*
+ * For PFN swap offsets, keeping the offset field
+ * to be PFN only to be compatible with old smaps.
+ */
+ if (is_pfn_swap_entry(entry))
+ offset = swp_offset_pfn(entry);
+ else
+ offset = swp_offset(entry);
frame = swp_type(entry) |
- (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
+ (offset << MAX_SWAPFILES_SHIFT);
+ }
flags |= PM_SWAP;
migration = is_migration_entry(entry);
if (is_pfn_swap_entry(entry))
unsigned long offset;
if (pm->show_pfn) {
- offset = swp_offset(entry) +
+ if (is_pfn_swap_entry(entry))
+ offset = swp_offset_pfn(entry);
+ else
+ offset = swp_offset(entry);
+ offset = offset +
((addr & ~PMD_MASK) >> PAGE_SHIFT);
frame = swp_type(entry) |
(offset << MAX_SWAPFILES_SHIFT);
*/
void task_mem(struct seq_file *m, struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
struct vm_region *region;
- struct rb_node *p;
unsigned long bytes = 0, sbytes = 0, slack = 0, size;
-
- mmap_read_lock(mm);
- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
- vma = rb_entry(p, struct vm_area_struct, vm_rb);
+ mmap_read_lock(mm);
+ for_each_vma(vmi, vma) {
bytes += kobjsize(vma);
region = vma->vm_region;
unsigned long task_vsize(struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
- struct rb_node *p;
unsigned long vsize = 0;
mmap_read_lock(mm);
- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
- vma = rb_entry(p, struct vm_area_struct, vm_rb);
+ for_each_vma(vmi, vma)
vsize += vma->vm_end - vma->vm_start;
- }
mmap_read_unlock(mm);
return vsize;
}
unsigned long *shared, unsigned long *text,
unsigned long *data, unsigned long *resident)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
struct vm_region *region;
- struct rb_node *p;
unsigned long size = kobjsize(mm);
mmap_read_lock(mm);
- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
- vma = rb_entry(p, struct vm_area_struct, vm_rb);
+ for_each_vma(vmi, vma) {
size += kobjsize(vma);
region = vma->vm_region;
if (region) {
*/
static int show_map(struct seq_file *m, void *_p)
{
- struct rb_node *p = _p;
-
- return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb));
+ return nommu_vma_show(m, _p);
}
static void *m_start(struct seq_file *m, loff_t *pos)
{
struct proc_maps_private *priv = m->private;
struct mm_struct *mm;
- struct rb_node *p;
- loff_t n = *pos;
+ struct vm_area_struct *vma;
+ unsigned long addr = *pos;
+
+ /* See m_next(). Zero at the start or after lseek. */
+ if (addr == -1UL)
+ return NULL;
/* pin the task and mm whilst we play with them */
priv->task = get_proc_task(priv->inode);
return ERR_PTR(-EINTR);
}
- /* start from the Nth VMA */
- for (p = rb_first(&mm->mm_rb); p; p = rb_next(p))
- if (n-- == 0)
- return p;
+ /* start the next element from addr */
+ vma = find_vma(mm, addr);
+ if (vma)
+ return vma;
mmap_read_unlock(mm);
mmput(mm);
static void *m_next(struct seq_file *m, void *_p, loff_t *pos)
{
- struct rb_node *p = _p;
+ struct vm_area_struct *vma = _p;
- (*pos)++;
- return p ? rb_next(p) : NULL;
+ *pos = vma->vm_end;
+ return find_vma(vma->vm_mm, vma->vm_end);
}
static const struct seq_operations proc_pid_maps_ops = {
#include <linux/time.h>
#include <linux/time_namespace.h>
#include <linux/kernel_stat.h>
+#include "internal.h"
static int uptime_proc_show(struct seq_file *m, void *v)
{
static int __init proc_uptime_init(void)
{
- proc_create_single("uptime", 0, NULL, uptime_proc_show);
+ struct proc_dir_entry *pde;
+
+ pde = proc_create_single("uptime", 0, NULL, uptime_proc_show);
+ pde_make_permanent(pde);
return 0;
}
fs_initcall(proc_uptime_init);
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/utsname.h>
+#include "internal.h"
static int version_proc_show(struct seq_file *m, void *v)
{
static int __init proc_version_init(void)
{
- proc_create_single("version", 0, NULL, version_proc_show);
+ struct proc_dir_entry *pde;
+
+ pde = proc_create_single("version", 0, NULL, version_proc_show);
+ pde_make_permanent(pde);
return 0;
}
fs_initcall(proc_version_init);
out1:
iput(sbi->inodes);
out:
- if (bh1)
- brelse(bh1);
- if (bh2)
- brelse(bh2);
+ brelse(bh1);
+ brelse(bh2);
outnobh:
kfree(qs);
s->s_fs_info = NULL;
unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags)
{
- unsigned long maxpages, lpages, nr, loop, ret;
+ unsigned long maxpages, lpages, nr_folios, loop, ret, nr_pages, pfn;
struct inode *inode = file_inode(file);
- struct page **pages = NULL, **ptr, *page;
+ struct folio_batch fbatch;
loff_t isize;
/* the mapping mustn't extend beyond the EOF */
goto out;
/* gang-find the pages */
- pages = kcalloc(lpages, sizeof(struct page *), GFP_KERNEL);
- if (!pages)
- goto out_free;
-
- nr = find_get_pages_contig(inode->i_mapping, pgoff, lpages, pages);
- if (nr != lpages)
- goto out_free_pages; /* leave if some pages were missing */
+ folio_batch_init(&fbatch);
+ nr_pages = 0;
+repeat:
+ nr_folios = filemap_get_folios_contig(inode->i_mapping, &pgoff,
+ ULONG_MAX, &fbatch);
+ if (!nr_folios) {
+ ret = -ENOSYS;
+ return ret;
+ }
+ if (ret == -ENOSYS) {
+ ret = (unsigned long) folio_address(fbatch.folios[0]);
+ pfn = folio_pfn(fbatch.folios[0]);
+ }
/* check the pages for physical adjacency */
- ptr = pages;
- page = *ptr++;
- page++;
- for (loop = lpages; loop > 1; loop--)
- if (*ptr++ != page++)
- goto out_free_pages;
+ for (loop = 0; loop < nr_folios; loop++) {
+ if (pfn + nr_pages != folio_pfn(fbatch.folios[loop])) {
+ ret = -ENOSYS;
+ goto out_free; /* leave if not physical adjacent */
+ }
+ nr_pages += folio_nr_pages(fbatch.folios[loop]);
+ if (nr_pages >= lpages)
+ goto out_free; /* successfully found desired pages*/
+ }
+ if (nr_pages < lpages) {
+ folio_batch_release(&fbatch);
+ goto repeat; /* loop if pages are missing */
+ }
/* okay - all conditions fulfilled */
- ret = (unsigned long) page_address(pages[0]);
-out_free_pages:
- ptr = pages;
- for (loop = nr; loop > 0; loop--)
- put_page(*ptr++);
out_free:
- kfree(pages);
+ folio_batch_release(&fbatch);
out:
return ret;
}
}
static int ramfs_tmpfile(struct user_namespace *mnt_userns,
- struct inode *dir, struct dentry *dentry, umode_t mode)
+ struct inode *dir, struct file *file, umode_t mode)
{
struct inode *inode;
inode = ramfs_get_inode(dir->i_sb, dir, mode, 0);
if (!inode)
return -ENOSPC;
- d_tmpfile(dentry, inode);
- return 0;
+ d_tmpfile(file, inode);
+ return finish_open_simple(file, 0);
}
static const struct inode_operations ramfs_dir_inode_operations = {
*/
if (buffer_dirty(bh) && unlikely(bh->b_page->mapping == NULL)) {
spin_unlock(lock);
- ll_rw_block(REQ_OP_WRITE, 1, &bh);
+ write_dirty_buffer(bh, 0);
spin_lock(lock);
}
put_bh(bh);
if (tbh) {
if (buffer_dirty(tbh)) {
depth = reiserfs_write_unlock_nested(s);
- ll_rw_block(REQ_OP_WRITE, 1, &tbh);
+ write_dirty_buffer(tbh, 0);
reiserfs_write_lock_nested(s, depth);
}
put_bh(tbh) ;
}
}
/* read in the log blocks, memcpy to the corresponding real block */
- ll_rw_block(REQ_OP_READ, get_desc_trans_len(desc), log_blocks);
+ bh_read_batch(get_desc_trans_len(desc), log_blocks);
for (i = 0; i < get_desc_trans_len(desc); i++) {
wait_on_buffer(log_blocks[i]);
} else
bhlist[j++] = bh;
}
- ll_rw_block(REQ_OP_READ, j, bhlist);
+ bh = bhlist[0];
+ bh_read_nowait(bh, 0);
+ bh_readahead_batch(j - 1, &bhlist[1], 0);
for (i = 1; i < j; i++)
brelse(bhlist[i]);
- bh = bhlist[0];
wait_on_buffer(bh);
if (buffer_uptodate(bh))
return bh;
char *s;
/* Some block devices use /'s */
- strlcpy(b, sb->s_id, BDEVNAME_SIZE);
+ strscpy(b, sb->s_id, BDEVNAME_SIZE);
s = strchr(b, '/');
if (s)
*s = '!';
char *s;
/* Some block devices use /'s */
- strlcpy(b, sb->s_id, BDEVNAME_SIZE);
+ strscpy(b, sb->s_id, BDEVNAME_SIZE);
s = strchr(b, '/');
if (s)
*s = '!';
if (!buffer_uptodate(bh[j])) {
if (depth == -1)
depth = reiserfs_write_unlock_nested(s);
- ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, bh + j);
+ bh_readahead(bh[j], REQ_RAHEAD);
}
brelse(bh[j]);
}
if (!buffer_uptodate(bh) && depth == -1)
depth = reiserfs_write_unlock_nested(sb);
- ll_rw_block(REQ_OP_READ, 1, &bh);
+ bh_read_nowait(bh, 0);
wait_on_buffer(bh);
if (depth != -1)
/* after journal replay, reread all bitmap and super blocks */
static int reread_meta_blocks(struct super_block *s)
{
- ll_rw_block(REQ_OP_READ, 1, &SB_BUFFER_WITH_SB(s));
- wait_on_buffer(SB_BUFFER_WITH_SB(s));
- if (!buffer_uptodate(SB_BUFFER_WITH_SB(s))) {
+ if (bh_read(SB_BUFFER_WITH_SB(s), 0) < 0) {
reiserfs_warning(s, "reiserfs-2504", "error reading the super");
return 1;
}
#define SMB2_CREATE_REQUEST_LEASE "RqLs"
#define SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 "DH2Q"
#define SMB2_CREATE_DURABLE_HANDLE_RECONNECT_V2 "DH2C"
-#define SMB2_CREATE_TAG_POSIX "\x93\xAD\x25\x50\x9C\xB4\x11\xE7\xB4\x23\x83\xDE\x96\x8B\xCD\x7C"
+#define SMB2_CREATE_TAG_POSIX "\x93\xAD\x25\x50\x9C\xB4\x11\xE7\xB4\x23\x83\xDE\x96\x8B\xCD\x7C"
+#define SMB2_CREATE_APP_INSTANCE_ID "\x45\xBC\xA6\x6A\xEF\xA7\xF7\x4A\x90\x08\xFA\x46\x2E\x14\x4D\x74"
+#define SMB2_CREATE_APP_INSTANCE_VERSION "\xB9\x82\xD0\xB7\x3B\x56\x07\x4F\xA0\x7B\x52\x4A\x81\x16\xA0\x10"
+#define SVHDX_OPEN_DEVICE_CONTEXT "\x9C\xCB\xCF\x9E\x04\xC1\xE6\x43\x98\x0E\x15\x8D\xA1\xF6\xEC\x83"
+#define SMB2_CREATE_TAG_AAPL "AAPL"
/* Flag (SMB3 open response) values */
#define SMB2_CREATE_FLAG_REPARSEPOINT 0x01
return ubifs_check_dir_empty(inode) == 0;
}
+/**
+ * ubifs_encrypt - Encrypt data.
+ * @inode: inode which refers to the data node
+ * @dn: data node to encrypt
+ * @in_len: length of data to be compressed
+ * @out_len: allocated memory size for the data area of @dn
+ * @block: logical block number of the block
+ *
+ * This function encrypt a possibly-compressed data in the data node.
+ * The encrypted data length will store in @out_len.
+ */
int ubifs_encrypt(const struct inode *inode, struct ubifs_data_node *dn,
unsigned int in_len, unsigned int *out_len, int block)
{
static inline int chance(unsigned int n, unsigned int out_of)
{
- return !!((prandom_u32() % out_of) + 1 <= n);
+ return !!(prandom_u32_max(out_of) + 1 <= n);
}
if (chance(1, 2)) {
d->pc_delay = 1;
/* Fail within 1 minute */
- delay = prandom_u32() % 60000;
+ delay = prandom_u32_max(60000);
d->pc_timeout = jiffies;
d->pc_timeout += msecs_to_jiffies(delay);
ubifs_warn(c, "failing after %lums", delay);
} else {
d->pc_delay = 2;
- delay = prandom_u32() % 10000;
+ delay = prandom_u32_max(10000);
/* Fail within 10000 operations */
d->pc_cnt_max = delay;
ubifs_warn(c, "failing after %lu calls", delay);
unsigned int from, to, ffs = chance(1, 2);
unsigned char *p = (void *)buf;
- from = prandom_u32() % len;
+ from = prandom_u32_max(len);
/* Corruption span max to end of write unit */
to = min(len, ALIGN(from + 1, c->max_write_size));
if (ffs)
memset(p + from, 0xFF, to - from);
else
- prandom_bytes(p + from, to - from);
+ get_random_bytes(p + from, to - from);
return to;
}
* @c: UBIFS file-system description object
* @dir: parent directory inode
* @mode: inode mode flags
+ * @is_xattr: whether the inode is xattr inode
*
* This function finds an unused inode number, allocates new inode and
* initializes it. Returns new inode in case of success and an error code in
* case of failure.
*/
struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir,
- umode_t mode)
+ umode_t mode, bool is_xattr)
{
int err;
struct inode *inode;
current_time(inode);
inode->i_mapping->nrpages = 0;
- err = fscrypt_prepare_new_inode(dir, inode, &encrypted);
- if (err) {
- ubifs_err(c, "fscrypt_prepare_new_inode failed: %i", err);
- goto out_iput;
+ if (!is_xattr) {
+ err = fscrypt_prepare_new_inode(dir, inode, &encrypted);
+ if (err) {
+ ubifs_err(c, "fscrypt_prepare_new_inode failed: %i", err);
+ goto out_iput;
+ }
}
switch (mode & S_IFMT) {
sz_change = CALC_DENT_SIZE(fname_len(&nm));
- inode = ubifs_new_inode(c, dir, mode);
+ inode = ubifs_new_inode(c, dir, mode, false);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
goto out_fname;
if (err)
return ERR_PTR(err);
- inode = ubifs_new_inode(c, dir, mode);
+ inode = ubifs_new_inode(c, dir, mode, false);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
goto out_free;
}
static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
+ struct dentry *dentry = file->f_path.dentry;
struct inode *inode;
struct ubifs_info *c = dir->i_sb->s_fs_info;
struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1,
return err;
}
- inode = ubifs_new_inode(c, dir, mode);
+ inode = ubifs_new_inode(c, dir, mode, false);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
goto out_budg;
mutex_lock(&ui->ui_mutex);
insert_inode_hash(inode);
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
ubifs_assert(c, ui->dirty);
instantiated = 1;
ubifs_release_budget(c, &req);
- return 0;
+ return finish_open_simple(file, 0);
out_cancel:
unlock_2_inodes(dir, inode);
}
/**
- * check_dir_empty - check if a directory is empty or not.
+ * ubifs_check_dir_empty - check if a directory is empty or not.
* @dir: VFS inode object of the directory to check
*
* This function checks if directory @dir is empty. Returns zero if the
sz_change = CALC_DENT_SIZE(fname_len(&nm));
- inode = ubifs_new_inode(c, dir, S_IFDIR | mode);
+ inode = ubifs_new_inode(c, dir, S_IFDIR | mode, false);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
goto out_fname;
sz_change = CALC_DENT_SIZE(fname_len(&nm));
- inode = ubifs_new_inode(c, dir, mode);
+ inode = ubifs_new_inode(c, dir, mode, false);
if (IS_ERR(inode)) {
kfree(dev);
err = PTR_ERR(inode);
sz_change = CALC_DENT_SIZE(fname_len(&nm));
- inode = ubifs_new_inode(c, dir, S_IFLNK | S_IRWXUGO);
+ inode = ubifs_new_inode(c, dir, S_IFLNK | S_IRWXUGO, false);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
goto out_fname;
static void set_dent_cookie(struct ubifs_info *c, struct ubifs_dent_node *dent)
{
if (c->double_hash)
- dent->cookie = (__force __le32) prandom_u32();
+ dent->cookie = (__force __le32) get_random_u32();
else
dent->cookie = 0;
}
* @block: data block number
* @dn: data node to re-compress
* @new_len: new length
+ * @dn_size: size of the data node @dn in memory
*
* This function is used when an inode is truncated and the last data node of
* the inode has to be re-compressed/encrypted and re-written.
*/
static int truncate_data_node(const struct ubifs_info *c, const struct inode *inode,
unsigned int block, struct ubifs_data_node *dn,
- int *new_len)
+ int *new_len, int dn_size)
{
void *buf;
- int err, dlen, compr_type, out_len, old_dlen;
+ int err, dlen, compr_type, out_len, data_size;
out_len = le32_to_cpu(dn->size);
buf = kmalloc_array(out_len, WORST_COMPR_FACTOR, GFP_NOFS);
if (!buf)
return -ENOMEM;
- dlen = old_dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ;
+ dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ;
+ data_size = dn_size - UBIFS_DATA_NODE_SZ;
compr_type = le16_to_cpu(dn->compr_type);
if (IS_ENCRYPTED(inode)) {
}
if (IS_ENCRYPTED(inode)) {
- err = ubifs_encrypt(inode, dn, out_len, &old_dlen, block);
+ err = ubifs_encrypt(inode, dn, out_len, &data_size, block);
if (err)
goto out;
- out_len = old_dlen;
+ out_len = data_size;
} else {
dn->compr_size = 0;
}
struct ubifs_trun_node *trun;
struct ubifs_data_node *dn;
int err, dlen, len, lnum, offs, bit, sz, sync = IS_SYNC(inode);
+ int dn_size;
struct ubifs_inode *ui = ubifs_inode(inode);
ino_t inum = inode->i_ino;
unsigned int blk;
ubifs_assert(c, S_ISREG(inode->i_mode));
ubifs_assert(c, mutex_is_locked(&ui->ui_mutex));
- sz = UBIFS_TRUN_NODE_SZ + UBIFS_INO_NODE_SZ +
- UBIFS_MAX_DATA_NODE_SZ * WORST_COMPR_FACTOR;
+ dn_size = COMPRESSED_DATA_NODE_BUF_SZ;
- sz += ubifs_auth_node_sz(c);
+ if (IS_ENCRYPTED(inode))
+ dn_size += UBIFS_CIPHER_BLOCK_SIZE;
+
+ sz = UBIFS_TRUN_NODE_SZ + UBIFS_INO_NODE_SZ +
+ dn_size + ubifs_auth_node_sz(c);
ino = kmalloc(sz, GFP_NOFS);
if (!ino)
if (dn_len <= 0 || dn_len > UBIFS_BLOCK_SIZE) {
ubifs_err(c, "bad data node (block %u, inode %lu)",
blk, inode->i_ino);
- ubifs_dump_node(c, dn, sz - UBIFS_INO_NODE_SZ -
- UBIFS_TRUN_NODE_SZ);
+ ubifs_dump_node(c, dn, dn_size);
goto out_free;
}
if (dn_len <= dlen)
dlen = 0; /* Nothing to do */
else {
- err = truncate_data_node(c, inode, blk, dn, &dlen);
+ err = truncate_data_node(c, inode, blk, dn,
+ &dlen, dn_size);
if (err)
goto out_free;
}
if (!dbg_is_chk_gen(c))
return 0;
- if (prandom_u32() & 3)
+ if (prandom_u32_max(4))
return 0;
for (i = 0; i < c->lsave_cnt; i++)
c->lsave[i] = c->main_first;
list_for_each_entry(lprops, &c->empty_list, list)
- c->lsave[prandom_u32() % c->lsave_cnt] = lprops->lnum;
+ c->lsave[prandom_u32_max(c->lsave_cnt)] = lprops->lnum;
list_for_each_entry(lprops, &c->freeable_list, list)
- c->lsave[prandom_u32() % c->lsave_cnt] = lprops->lnum;
+ c->lsave[prandom_u32_max(c->lsave_cnt)] = lprops->lnum;
list_for_each_entry(lprops, &c->frdi_idx_list, list)
- c->lsave[prandom_u32() % c->lsave_cnt] = lprops->lnum;
+ c->lsave[prandom_u32_max(c->lsave_cnt)] = lprops->lnum;
heap = &c->lpt_heap[LPROPS_DIRTY_IDX - 1];
for (i = 0; i < heap->cnt; i++)
- c->lsave[prandom_u32() % c->lsave_cnt] = heap->arr[i]->lnum;
+ c->lsave[prandom_u32_max(c->lsave_cnt)] = heap->arr[i]->lnum;
heap = &c->lpt_heap[LPROPS_DIRTY - 1];
for (i = 0; i < heap->cnt; i++)
- c->lsave[prandom_u32() % c->lsave_cnt] = heap->arr[i]->lnum;
+ c->lsave[prandom_u32_max(c->lsave_cnt)] = heap->arr[i]->lnum;
heap = &c->lpt_heap[LPROPS_FREE - 1];
for (i = 0; i < heap->cnt; i++)
- c->lsave[prandom_u32() % c->lsave_cnt] = heap->arr[i]->lnum;
+ c->lsave[prandom_u32_max(c->lsave_cnt)] = heap->arr[i]->lnum;
return 1;
}
c->ilebs[c->ileb_cnt++] = lnum;
dbg_cmt("LEB %d", lnum);
}
- if (dbg_is_chk_index(c) && !(prandom_u32() & 7))
+ if (dbg_is_chk_index(c) && !prandom_u32_max(8))
return -ENOSPC;
return 0;
}
/* dir.c */
struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir,
- umode_t mode);
+ umode_t mode, bool is_xattr);
int ubifs_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat,
u32 request_mask, unsigned int flags);
int ubifs_check_dir_empty(struct inode *dir);
if (err)
return err;
- inode = ubifs_new_inode(c, host, S_IFREG | S_IRWXUGO);
+ inode = ubifs_new_inode(c, host, S_IFREG | S_IRWXUGO, true);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
goto out_budg;
brelse(tmp);
}
if (num) {
- ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha);
+ bh_readahead_batch(num, bha, REQ_RAHEAD);
for (i = 0; i < num; i++)
brelse(bha[i]);
}
brelse(tmp);
}
if (num) {
- ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha);
+ bh_readahead_batch(num, bha, REQ_RAHEAD);
for (i = 0; i < num; i++)
brelse(bha[i]);
}
if (!bh)
return NULL;
- if (buffer_uptodate(bh))
- return bh;
-
- ll_rw_block(REQ_OP_READ, 1, &bh);
-
- wait_on_buffer(bh);
- if (buffer_uptodate(bh))
+ if (bh_read(bh, 0) >= 0)
return bh;
brelse(bh);
}
static int udf_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
struct inode *inode = udf_new_inode(dir, mode);
inode->i_op = &udf_file_inode_operations;
inode->i_fop = &udf_file_operations;
mark_inode_dirty(inode);
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
unlock_new_inode(inode);
- return 0;
+ return finish_open_simple(file, 0);
}
static int udf_mknod(struct user_namespace *mnt_userns, struct inode *dir,
if (!buffer_mapped(bh))
map_bh(bh, inode->i_sb, oldb + pos);
- if (!buffer_uptodate(bh)) {
- ll_rw_block(REQ_OP_READ, 1, &bh);
- wait_on_buffer(bh);
- if (!buffer_uptodate(bh)) {
- ufs_error(inode->i_sb, __func__,
- "read of block failed\n");
- break;
- }
+ if (bh_read(bh, 0) < 0) {
+ ufs_error(inode->i_sb, __func__,
+ "read of block failed\n");
+ break;
}
UFSD(" change from %llu to %llu, pos %u\n",
#include <linux/security.h>
#include <linux/hugetlb.h>
#include <linux/swapops.h>
+#include <linux/miscdevice.h>
int sysctl_unprivileged_userfaultfd __read_mostly;
if (ctx->features & UFFD_FEATURE_SIGBUS)
goto out;
- if ((vmf->flags & FAULT_FLAG_USER) == 0 &&
- ctx->flags & UFFD_USER_MODE_ONLY) {
- printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd "
- "sysctl knob to 1 if kernel faults must be handled "
- "without obtaining CAP_SYS_PTRACE capability\n");
+ if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))
goto out;
- }
/*
* If it's already released don't get it. This avoids to loop
if (release_new_ctx) {
struct vm_area_struct *vma;
struct mm_struct *mm = release_new_ctx->mm;
+ VMA_ITERATOR(vmi, mm, 0);
/* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ for_each_vma(vmi, vma) {
if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vma->vm_flags &= ~__VM_UFFD_FLAGS;
}
+ }
mmap_write_unlock(mm);
userfaultfd_ctx_put(release_new_ctx);
return false;
}
-int userfaultfd_unmap_prep(struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- struct list_head *unmaps)
+int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
+ unsigned long end, struct list_head *unmaps)
{
- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
+ VMA_ITERATOR(vmi, mm, start);
+ struct vm_area_struct *vma;
+
+ for_each_vma_range(vmi, vma, end) {
struct userfaultfd_unmap_ctx *unmap_ctx;
struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
/* len == 0 means wake all */
struct userfaultfd_wake_range range = { .len = 0, };
unsigned long new_flags;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
WRITE_ONCE(ctx->released, true);
*/
mmap_write_lock(mm);
prev = NULL;
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
cond_resched();
BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
!!(vma->vm_flags & __VM_UFFD_FLAGS));
vma->vm_file, vma->vm_pgoff,
vma_policy(vma),
NULL_VM_UFFD_CTX, anon_vma_name(vma));
- if (prev)
+ if (prev) {
+ mas_pause(&mas);
vma = prev;
- else
+ } else {
prev = vma;
+ }
+
vma->vm_flags = new_flags;
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
}
bool found;
bool basic_ioctls;
unsigned long start, end, vma_end;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
user_uffdio_register = (struct uffdio_register __user *) arg;
goto out;
mmap_write_lock(mm);
- vma = find_vma_prev(mm, start, &prev);
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
if (!vma)
goto out_unlock;
*/
found = false;
basic_ioctls = false;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
}
BUG_ON(!found);
- if (vma->vm_start < start)
- prev = vma;
+ mas_set(&mas, start);
+ prev = mas_prev(&mas, 0);
+ if (prev != vma)
+ mas_next(&mas, ULONG_MAX);
ret = 0;
do {
((struct vm_userfaultfd_ctx){ ctx }),
anon_vma_name(vma));
if (prev) {
+ /* vma_merge() invalidated the mas */
+ mas_pause(&mas);
vma = prev;
goto next;
}
ret = split_vma(mm, vma, start, 1);
if (ret)
break;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
if (vma->vm_end > end) {
ret = split_vma(mm, vma, end, 0);
if (ret)
break;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
next:
/*
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
- } while (vma && vma->vm_start < end);
+ vma = mas_next(&mas, end - 1);
+ } while (vma);
out_unlock:
mmap_write_unlock(mm);
mmput(mm);
bool found;
unsigned long start, end, vma_end;
const void __user *buf = (void __user *)arg;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
ret = -EFAULT;
if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
goto out;
mmap_write_lock(mm);
- vma = find_vma_prev(mm, start, &prev);
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
if (!vma)
goto out_unlock;
*/
found = false;
ret = -EINVAL;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
}
BUG_ON(!found);
- if (vma->vm_start < start)
- prev = vma;
+ mas_set(&mas, start);
+ prev = mas_prev(&mas, 0);
+ if (prev != vma)
+ mas_next(&mas, ULONG_MAX);
ret = 0;
do {
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
- } while (vma && vma->vm_start < end);
+ vma = mas_next(&mas, end - 1);
+ } while (vma);
out_unlock:
mmap_write_unlock(mm);
mmput(mm);
seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
}
-SYSCALL_DEFINE1(userfaultfd, int, flags)
+static int new_userfaultfd(int flags)
{
struct userfaultfd_ctx *ctx;
int fd;
- if (!sysctl_unprivileged_userfaultfd &&
- (flags & UFFD_USER_MODE_ONLY) == 0 &&
- !capable(CAP_SYS_PTRACE)) {
- printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd "
- "sysctl knob to 1 if kernel faults must be handled "
- "without obtaining CAP_SYS_PTRACE capability\n");
- return -EPERM;
- }
-
BUG_ON(!current->mm);
/* Check the UFFD_* constants for consistency. */
return fd;
}
+static inline bool userfaultfd_syscall_allowed(int flags)
+{
+ /* Userspace-only page faults are always allowed */
+ if (flags & UFFD_USER_MODE_ONLY)
+ return true;
+
+ /*
+ * The user is requesting a userfaultfd which can handle kernel faults.
+ * Privileged users are always allowed to do this.
+ */
+ if (capable(CAP_SYS_PTRACE))
+ return true;
+
+ /* Otherwise, access to kernel fault handling is sysctl controlled. */
+ return sysctl_unprivileged_userfaultfd;
+}
+
+SYSCALL_DEFINE1(userfaultfd, int, flags)
+{
+ if (!userfaultfd_syscall_allowed(flags))
+ return -EPERM;
+
+ return new_userfaultfd(flags);
+}
+
+static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)
+{
+ if (cmd != USERFAULTFD_IOC_NEW)
+ return -EINVAL;
+
+ return new_userfaultfd(flags);
+}
+
+static const struct file_operations userfaultfd_dev_fops = {
+ .unlocked_ioctl = userfaultfd_dev_ioctl,
+ .compat_ioctl = userfaultfd_dev_ioctl,
+ .owner = THIS_MODULE,
+ .llseek = noop_llseek,
+};
+
+static struct miscdevice userfaultfd_misc = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "userfaultfd",
+ .fops = &userfaultfd_dev_fops
+};
+
static int __init userfaultfd_init(void)
{
+ int ret;
+
+ ret = misc_register(&userfaultfd_misc);
+ if (ret)
+ return ret;
+
userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
sizeof(struct userfaultfd_ctx),
0,
#ifdef DEBUG
/* Randomly don't execute the first algorithm. */
- if (prandom_u32() & 1)
+ if (prandom_u32_max(2))
return 0;
#endif
else
thispa = XFS_BMBT_PTR_ADDR(mp, block, j, dmxr);
if (*thispa == *pp) {
- xfs_warn(mp, "%s: thispa(%d) == pp(%d) %Ld",
+ xfs_warn(mp, "%s: thispa(%d) == pp(%d) %lld",
__func__, j, i,
(unsigned long long)be64_to_cpu(*thispa));
xfs_err(mp, "%s: ptrs are equal in node\n",
*/
mapp = kmem_alloc(sizeof(*mapp) * count, 0);
for (b = *bno, mapi = 0; b < *bno + count; ) {
- nmap = min(XFS_BMAP_MAX_NMAP, count);
c = (int)(*bno + count - b);
+ nmap = min(XFS_BMAP_MAX_NMAP, c);
error = xfs_bmapi_write(tp, dp, b, c,
xfs_bmapi_aflag(w)|XFS_BMAPI_METADATA,
args->total, &mapp[mapi], &nmap);
{
struct xfs_da_args *args;
int rval;
- int v; /* type-checking value */
+ bool v;
ASSERT(S_ISDIR(VFS_I(dp)->i_mode));
{
struct xfs_da_args *args;
int rval;
- int v; /* type-checking value */
+ bool v;
int lock_mode;
ASSERT(S_ISDIR(VFS_I(dp)->i_mode));
{
struct xfs_da_args *args;
int rval;
- int v; /* type-checking value */
+ bool v;
ASSERT(S_ISDIR(VFS_I(dp)->i_mode));
XFS_STATS_INC(dp->i_mount, xs_dir_remove);
{
struct xfs_da_args *args;
int rval;
- int v; /* type-checking value */
+ bool v;
ASSERT(S_ISDIR(VFS_I(dp)->i_mode));
int
xfs_dir2_isblock(
struct xfs_da_args *args,
- int *vp) /* out: 1 is block, 0 is not block */
+ bool *isblock)
{
- xfs_fileoff_t last; /* last file offset */
- int rval;
+ struct xfs_mount *mp = args->dp->i_mount;
+ xfs_fileoff_t eof;
+ int error;
- if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK)))
- return rval;
- rval = XFS_FSB_TO_B(args->dp->i_mount, last) == args->geo->blksize;
- if (XFS_IS_CORRUPT(args->dp->i_mount,
- rval != 0 &&
- args->dp->i_disk_size != args->geo->blksize))
+ error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK);
+ if (error)
+ return error;
+
+ *isblock = false;
+ if (XFS_FSB_TO_B(mp, eof) != args->geo->blksize)
+ return 0;
+
+ *isblock = true;
+ if (XFS_IS_CORRUPT(mp, args->dp->i_disk_size != args->geo->blksize))
return -EFSCORRUPTED;
- *vp = rval;
return 0;
}
int
xfs_dir2_isleaf(
struct xfs_da_args *args,
- int *vp) /* out: 1 is block, 0 is not block */
+ bool *isleaf)
{
- xfs_fileoff_t last; /* last file offset */
- int rval;
+ xfs_fileoff_t eof;
+ int error;
- if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK)))
- return rval;
- *vp = last == args->geo->leafblk + args->geo->fsbcount;
+ error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK);
+ if (error)
+ return error;
+
+ *isleaf = false;
+ if (eof != args->geo->leafblk + args->geo->fsbcount)
+ return 0;
+
+ *isleaf = true;
return 0;
}
/*
* Interface routines used by userspace utilities
*/
-extern int xfs_dir2_isblock(struct xfs_da_args *args, int *r);
-extern int xfs_dir2_isleaf(struct xfs_da_args *args, int *r);
+extern int xfs_dir2_isblock(struct xfs_da_args *args, bool *isblock);
+extern int xfs_dir2_isleaf(struct xfs_da_args *args, bool *isleaf);
extern int xfs_dir2_shrink_inode(struct xfs_da_args *args, xfs_dir2_db_t db,
struct xfs_buf *bp);
struct xfs_inode *dp = args->dp;
struct xfs_mount *mp = dp->i_mount;
int i; /* entry index */
- int error;
xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */
xfs_dir2_sf_hdr_t *sfp; /* shortform structure */
enum xfs_dacmp cmp; /* comparison result */
if (!ci_sfep)
return -ENOENT;
/* otherwise process the CI match as required by the caller */
- error = xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen);
- return error;
+ return xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen);
}
/*
/* randomly do sparse inode allocations */
if (xfs_has_sparseinodes(tp->t_mountp) &&
igeo->ialloc_min_blks < igeo->ialloc_blks)
- do_sparse = prandom_u32() & 1;
+ do_sparse = prandom_u32_max(2);
#endif
/*
* number from being easily guessable.
*/
error = xfs_ialloc_inode_init(args.mp, tp, NULL, newlen, pag->pag_agno,
- args.agbno, args.len, prandom_u32());
+ args.agbno, args.len, get_random_u32());
if (error)
return error;
*/
if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) {
xfs_warn(ip->i_mount,
- "corrupt inode %Lu (bad size %d for local fork, size = %zd).",
+ "corrupt inode %llu (bad size %d for local fork, size = %zd).",
(unsigned long long) ip->i_ino, size,
XFS_DFORK_SIZE(dip, ip->i_mount, whichfork));
xfs_inode_verifier_error(ip, -EFSCORRUPTED,
XFS_DFORK_SIZE(dip, mp, whichfork) ||
ifp->if_nextents > ip->i_nblocks) ||
level == 0 || level > XFS_BM_MAXLEVELS(mp, whichfork)) {
- xfs_warn(mp, "corrupt inode %Lu (btree).",
+ xfs_warn(mp, "corrupt inode %llu (btree).",
(unsigned long long) ip->i_ino);
xfs_inode_verifier_error(ip, -EFSCORRUPTED,
"xfs_iformat_btree", dfp, size,
xfs_dablk_t dabno;
xfs_dir2_db_t last_data_db = 0;
bool found;
- int is_block = 0;
+ bool is_block = false;
int error;
/* Ignore local format directories. */
*/
nv = xlog_kvmalloc(sizeof(struct xfs_attri_log_nameval) +
name_len + value_len);
- if (!nv)
- return nv;
nv->name.i_addr = nv + 1;
nv->name.i_len = name_len;
attr->xattri_nameval = xfs_attri_log_nameval_alloc(args->name,
args->namelen, args->value, args->valuelen);
}
- if (!attr->xattri_nameval)
- return ERR_PTR(-ENOMEM);
attrip = xfs_attri_init(mp, attr->xattri_nameval);
xfs_trans_add_item(tp, &attrip->attri_item);
nv = xfs_attri_log_nameval_alloc(attr_name,
attri_formatp->alfi_name_len, attr_value,
attri_formatp->alfi_value_len);
- if (!nv)
- return -ENOMEM;
attrip = xfs_attri_init(mp, nv);
error = xfs_attri_copy_format(&item->ri_buf[0], &attrip->attri_format);
{
struct xfs_da_args args = { NULL };
unsigned int lock_mode;
- int isblock;
+ bool isblock;
int error;
trace_xfs_readdir(dp);
ASSERT(error_tag < XFS_ERRTAG_MAX);
randfactor = mp->m_errortag[error_tag];
- if (!randfactor || prandom_u32() % randfactor)
+ if (!randfactor || prandom_u32_max(randfactor))
return false;
xfs_warn_ratelimited(mp,
*/
if (xfs_has_v3inodes(mp) &&
(flags & XFS_IGET_CREATE) && !xfs_has_ikeep(mp)) {
- VFS_I(ip)->i_generation = prandom_u32();
+ VFS_I(ip)->i_generation = get_random_u32();
} else {
struct xfs_buf *bp;
* ID or one of the supplementary group IDs, the S_ISGID bit is cleared
* (and only if the irix_sgid_inherit compatibility variable is set).
*/
- if (irix_sgid_inherit &&
- (inode->i_mode & S_ISGID) &&
- !in_group_p(i_gid_into_mnt(mnt_userns, inode)))
+ if (irix_sgid_inherit && (inode->i_mode & S_ISGID) &&
+ !vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode)))
inode->i_mode &= ~S_ISGID;
ip->i_disk_size = 0;
if (XFS_TEST_ERROR(dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC),
mp, XFS_ERRTAG_IFLUSH_1)) {
xfs_alert_tag(mp, XFS_PTAG_IFLUSH,
- "%s: Bad inode %Lu magic number 0x%x, ptr "PTR_FMT,
+ "%s: Bad inode %llu magic number 0x%x, ptr "PTR_FMT,
__func__, ip->i_ino, be16_to_cpu(dip->di_magic), dip);
goto flush_out;
}
ip->i_df.if_format != XFS_DINODE_FMT_BTREE,
mp, XFS_ERRTAG_IFLUSH_3)) {
xfs_alert_tag(mp, XFS_PTAG_IFLUSH,
- "%s: Bad regular inode %Lu, ptr "PTR_FMT,
+ "%s: Bad regular inode %llu, ptr "PTR_FMT,
__func__, ip->i_ino, ip);
goto flush_out;
}
ip->i_df.if_format != XFS_DINODE_FMT_LOCAL,
mp, XFS_ERRTAG_IFLUSH_4)) {
xfs_alert_tag(mp, XFS_PTAG_IFLUSH,
- "%s: Bad directory inode %Lu, ptr "PTR_FMT,
+ "%s: Bad directory inode %llu, ptr "PTR_FMT,
__func__, ip->i_ino, ip);
goto flush_out;
}
if (XFS_TEST_ERROR(ip->i_forkoff > mp->m_sb.sb_inodesize,
mp, XFS_ERRTAG_IFLUSH_6)) {
xfs_alert_tag(mp, XFS_PTAG_IFLUSH,
- "%s: bad inode %Lu, forkoff 0x%x, ptr "PTR_FMT,
+ "%s: bad inode %llu, forkoff 0x%x, ptr "PTR_FMT,
__func__, ip->i_ino, ip->i_forkoff, ip);
goto flush_out;
}
if (!bp || (ip->i_flags & XFS_ISTALE)) {
/*
- * Inode item/buffer is being being aborted due to cluster
+ * Inode item/buffer is being aborted due to cluster
* buffer deletion. Trigger a log force to have that operation
* completed and items removed from the AIL before the next push
* attempt.
*/
if (XFS_IS_CORRUPT(mp, !xfs_verify_magic16(bp, dip->di_magic))) {
xfs_alert(mp,
- "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %Ld",
+ "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %lld",
__func__, dip, bp, in_f->ilf_ino);
error = -EFSCORRUPTED;
goto out_release;
ldip = item->ri_buf[1].i_addr;
if (XFS_IS_CORRUPT(mp, ldip->di_magic != XFS_DINODE_MAGIC)) {
xfs_alert(mp,
- "%s: Bad inode log record, rec ptr "PTR_FMT", ino %Ld",
+ "%s: Bad inode log record, rec ptr "PTR_FMT", ino %lld",
__func__, item, in_f->ilf_ino);
error = -EFSCORRUPTED;
goto out_release;
struct dentry *dentry,
umode_t mode,
dev_t rdev,
- bool tmpfile) /* unnamed file */
+ struct file *tmpfile) /* unnamed file */
{
struct inode *inode;
struct xfs_inode *ip = NULL;
* d_tmpfile can immediately set it back to zero.
*/
set_nlink(inode, 1);
- d_tmpfile(dentry, inode);
+ d_tmpfile(tmpfile, inode);
} else
d_instantiate(dentry, inode);
umode_t mode,
dev_t rdev)
{
- return xfs_generic_create(mnt_userns, dir, dentry, mode, rdev, false);
+ return xfs_generic_create(mnt_userns, dir, dentry, mode, rdev, NULL);
}
STATIC int
umode_t mode,
bool flags)
{
- return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, false);
+ return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, NULL);
}
STATIC int
umode_t mode)
{
return xfs_generic_create(mnt_userns, dir, dentry, mode | S_IFDIR, 0,
- false);
+ NULL);
}
STATIC struct dentry *
struct inode *inode = d_inode(path->dentry);
struct xfs_inode *ip = XFS_I(inode);
struct xfs_mount *mp = ip->i_mount;
+ vfsuid_t vfsuid = i_uid_into_vfsuid(mnt_userns, inode);
+ vfsgid_t vfsgid = i_gid_into_vfsgid(mnt_userns, inode);
trace_xfs_getattr(ip);
stat->dev = inode->i_sb->s_dev;
stat->mode = inode->i_mode;
stat->nlink = inode->i_nlink;
- stat->uid = i_uid_into_mnt(mnt_userns, inode);
- stat->gid = i_gid_into_mnt(mnt_userns, inode);
+ stat->uid = vfsuid_into_kuid(vfsuid);
+ stat->gid = vfsgid_into_kgid(vfsgid);
stat->ino = ip->i_ino;
stat->atime = inode->i_atime;
stat->mtime = inode->i_mtime;
xfs_vn_tmpfile(
struct user_namespace *mnt_userns,
struct inode *dir,
- struct dentry *dentry,
+ struct file *file,
umode_t mode)
{
- return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, true);
+ int err = xfs_generic_create(mnt_userns, dir, file->f_path.dentry, mode, 0, file);
+
+ return finish_open_simple(file, err);
}
static const struct inode_operations xfs_inode_operations = {
extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size);
-extern void xfs_setattr_time(struct xfs_inode *ip, struct iattr *iattr);
int xfs_vn_setattr_size(struct user_namespace *mnt_userns,
struct dentry *dentry, struct iattr *vap);
struct xfs_bulkstat *buf = bc->buf;
xfs_extnum_t nextents;
int error = -EINVAL;
+ vfsuid_t vfsuid;
+ vfsgid_t vfsgid;
if (xfs_internal_inum(mp, ino))
goto out_advance;
ASSERT(ip != NULL);
ASSERT(ip->i_imap.im_blkno != 0);
inode = VFS_I(ip);
+ vfsuid = i_uid_into_vfsuid(mnt_userns, inode);
+ vfsgid = i_gid_into_vfsgid(mnt_userns, inode);
/* xfs_iget returns the following without needing
* further change.
*/
buf->bs_projectid = ip->i_projid;
buf->bs_ino = ino;
- buf->bs_uid = from_kuid(sb_userns, i_uid_into_mnt(mnt_userns, inode));
- buf->bs_gid = from_kgid(sb_userns, i_gid_into_mnt(mnt_userns, inode));
+ buf->bs_uid = from_kuid(sb_userns, vfsuid_into_kuid(vfsuid));
+ buf->bs_gid = from_kgid(sb_userns, vfsgid_into_kgid(vfsgid));
buf->bs_size = ip->i_disk_size;
buf->bs_nlink = inode->i_nlink;
if (head == &log->l_write_head) {
ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV);
return tic->t_unit_res;
- } else {
- if (tic->t_flags & XLOG_TIC_PERM_RESERV)
- return tic->t_unit_res * tic->t_cnt;
- else
- return tic->t_unit_res;
}
+
+ if (tic->t_flags & XLOG_TIC_PERM_RESERV)
+ return tic->t_unit_res * tic->t_cnt;
+
+ return tic->t_unit_res;
}
STATIC bool
tic->t_curr_res = unit_res;
tic->t_cnt = cnt;
tic->t_ocnt = cnt;
- tic->t_tid = prandom_u32();
+ tic->t_tid = get_random_u32();
if (permanent)
tic->t_flags |= XLOG_TIC_PERM_RESERV;
"alignment check failed: sunit/swidth vs. blocksize(%d)",
mp->m_sb.sb_blocksize);
return -EINVAL;
- } else {
- /*
- * Convert the stripe unit and width to FSBs.
- */
- mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign);
- if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) {
- xfs_warn(mp,
- "alignment check failed: sunit/swidth vs. agsize(%d)",
- mp->m_sb.sb_agblocks);
- return -EINVAL;
- } else if (mp->m_dalign) {
- mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth);
- } else {
- xfs_warn(mp,
- "alignment check failed: sunit(%d) less than bsize(%d)",
- mp->m_dalign, mp->m_sb.sb_blocksize);
- return -EINVAL;
- }
}
+ /*
+ * Convert the stripe unit and width to FSBs.
+ */
+ mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign);
+ if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) {
+ xfs_warn(mp,
+ "alignment check failed: sunit/swidth vs. agsize(%d)",
+ mp->m_sb.sb_agblocks);
+ return -EINVAL;
+ }
+
+ if (!mp->m_dalign) {
+ xfs_warn(mp,
+ "alignment check failed: sunit(%d) less than bsize(%d)",
+ mp->m_dalign, mp->m_sb.sb_blocksize);
+ return -EINVAL;
+ }
+
+ mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth);
+
if (!xfs_has_dalign(mp)) {
xfs_warn(mp,
"cannot change alignment: superblock does not support data alignment");
#include <linux/mm.h>
#include <linux/dax.h>
-struct failure_info {
+struct xfs_failure_info {
xfs_agblock_t startblock;
xfs_extlen_t blockcount;
int mf_flags;
+ bool want_shutdown;
};
static pgoff_t
xfs_failure_pgoff(
struct xfs_mount *mp,
const struct xfs_rmap_irec *rec,
- const struct failure_info *notify)
+ const struct xfs_failure_info *notify)
{
loff_t pos = XFS_FSB_TO_B(mp, rec->rm_offset);
xfs_failure_pgcnt(
struct xfs_mount *mp,
const struct xfs_rmap_irec *rec,
- const struct failure_info *notify)
+ const struct xfs_failure_info *notify)
{
xfs_agblock_t end_rec;
xfs_agblock_t end_notify;
{
struct xfs_mount *mp = cur->bc_mp;
struct xfs_inode *ip;
- struct failure_info *notify = data;
+ struct xfs_failure_info *notify = data;
int error = 0;
if (XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) ||
(rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))) {
- xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK);
- return -EFSCORRUPTED;
+ notify->want_shutdown = true;
+ return 0;
}
/* Get files that incore, filter out others that are not in use. */
/* Continue the rmap query if the inode isn't incore */
if (error == -ENODATA)
return 0;
- if (error)
- return error;
+ if (error) {
+ notify->want_shutdown = true;
+ return 0;
+ }
error = mf_dax_kill_procs(VFS_I(ip)->i_mapping,
xfs_failure_pgoff(mp, rec, notify),
xfs_daddr_t bblen,
int mf_flags)
{
+ struct xfs_failure_info notify = { .mf_flags = mf_flags };
struct xfs_trans *tp = NULL;
struct xfs_btree_cur *cur = NULL;
struct xfs_buf *agf_bp = NULL;
for (; agno <= end_agno; agno++) {
struct xfs_rmap_irec ri_low = { };
struct xfs_rmap_irec ri_high;
- struct failure_info notify;
struct xfs_agf *agf;
xfs_agblock_t agend;
struct xfs_perag *pag;
}
xfs_trans_cancel(tp);
+ if (error || notify.want_shutdown) {
+ xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK);
+ if (!error)
+ error = -EFSCORRUPTED;
+ }
return error;
}
if (fbno == NULLAGBLOCK) {
/* No shared blocks at all. */
return 0;
- } else if (fbno == agbno) {
+ }
+
+ if (fbno == agbno) {
/*
* The start of this extent is shared. Truncate the
* mapping at the end of the shared region so that a
irec->br_blockcount = flen;
*shared = true;
return 0;
- } else {
- /*
- * There's a shared extent midway through this extent.
- * Truncate the mapping at the start of the shared
- * extent so that a subsequent iteration starts at the
- * start of the shared region.
- */
- irec->br_blockcount = fbno - agbno;
- return 0;
}
+
+ /*
+ * There's a shared extent midway through this extent.
+ * Truncate the mapping at the start of the shared
+ * extent so that a subsequent iteration starts at the
+ * start of the shared region.
+ */
+ irec->br_blockcount = fbno - agbno;
+ return 0;
}
int
defer_relog += per_cpu_ptr(stats, i)->s.defer_relog;
}
- len += scnprintf(buf + len, PATH_MAX-len, "xpc %Lu %Lu %Lu\n",
+ len += scnprintf(buf + len, PATH_MAX-len, "xpc %llu %llu %llu\n",
xs_xstrat_bytes, xs_write_bytes, xs_read_bytes);
len += scnprintf(buf + len, PATH_MAX-len, "defer_relog %llu\n",
defer_relog);
{
int j;
- seq_printf(m, "qm");
+ seq_puts(m, "qm");
for (j = XFSSTAT_START_XQMSTAT; j < XFSSTAT_END_XQMSTAT; j++)
seq_printf(m, " %u", counter_val(xfsstats.xs_stats, j));
seq_putc(m, '\n');
__entry->ino_res_used = qtrx->qt_ino_res_used;
__entry->icount_delta = qtrx->qt_icount_delta;
),
- TP_printk("dev %d:%d dquot id 0x%x type %s flags %s"
+ TP_printk("dev %d:%d dquot id 0x%x type %s flags %s "
"blk_res %llu bcount_delta %lld delbcnt_delta %lld "
"rtblk_res %llu rtblk_res_used %llu rtbcount_delta %lld delrtb_delta %lld "
"ino_res %llu ino_res_used %llu icount_delta %lld",
__entry->caller_ip = caller_ip;
__entry->flags = flags;
),
- TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx"
+ TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx "
"flags %s caller %pS",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->ino,
#ifndef _ASM_GENERIC_CACHEFLUSH_H
#define _ASM_GENERIC_CACHEFLUSH_H
+#include <linux/instrumented.h>
+
struct mm_struct;
struct vm_area_struct;
struct page;
#ifndef copy_to_user_page
#define copy_to_user_page(vma, page, vaddr, dst, src, len) \
do { \
+ instrument_copy_to_user((void __user *)dst, src, len); \
memcpy(dst, src, len); \
flush_icache_user_page(vma, page, vaddr, len); \
} while (0)
#endif
+
#ifndef copy_from_user_page
-#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
- memcpy(dst, src, len)
+#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
+ do { \
+ instrument_copy_from_user_before(dst, (void __user *)src, \
+ len); \
+ memcpy(dst, src, len); \
+ instrument_copy_from_user_after(dst, (void __user *)src, len, \
+ 0); \
+ } while (0)
#endif
#endif /* _ASM_GENERIC_CACHEFLUSH_H */
}
/* Generate the guest OS identifier as described in the Hyper-V TLFS */
-static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version,
- __u64 d_info2)
+static inline u64 hv_generate_guest_id(u64 kernel_version)
{
- __u64 guest_id = 0;
+ u64 guest_id;
- guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48);
- guest_id |= (d_info1 << 48);
+ guest_id = (((u64)HV_LINUX_VENDOR_ID) << 48);
guest_id |= (kernel_version << 16);
- guest_id |= d_info2;
return guest_id;
}
__put_unaligned_le24(val, p);
}
-static inline void __put_unaligned_be48(const u64 val, __u8 *p)
+static inline void __put_unaligned_be48(const u64 val, u8 *p)
{
*p++ = val >> 40;
*p++ = val >> 32;
--- /dev/null
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2022 MediaTek Inc.
+ * Author: Hui Liu <hui.liu@mediatek.com>
+ */
+
+#ifndef __MEDIATEK_MT8188_PINFUNC_H
+#define __MEDIATEK_MT8188_PINFUNC_H
+
+#include "mt65xx.h"
+
+#define PINMUX_GPIO0__FUNC_B_GPIO0 (MTK_PIN_NO(0) | 0)
+#define PINMUX_GPIO0__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(0) | 1)
+#define PINMUX_GPIO0__FUNC_O_SPIM5_CSB (MTK_PIN_NO(0) | 2)
+#define PINMUX_GPIO0__FUNC_O_UTXD1 (MTK_PIN_NO(0) | 3)
+#define PINMUX_GPIO0__FUNC_O_DMIC3_CLK (MTK_PIN_NO(0) | 4)
+#define PINMUX_GPIO0__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(0) | 5)
+#define PINMUX_GPIO0__FUNC_O_I2SO2_MCK (MTK_PIN_NO(0) | 6)
+#define PINMUX_GPIO0__FUNC_B0_DBG_MON_A0 (MTK_PIN_NO(0) | 7)
+
+#define PINMUX_GPIO1__FUNC_B_GPIO1 (MTK_PIN_NO(1) | 0)
+#define PINMUX_GPIO1__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(1) | 1)
+#define PINMUX_GPIO1__FUNC_O_SPIM5_CLK (MTK_PIN_NO(1) | 2)
+#define PINMUX_GPIO1__FUNC_I1_URXD1 (MTK_PIN_NO(1) | 3)
+#define PINMUX_GPIO1__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(1) | 4)
+#define PINMUX_GPIO1__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(1) | 5)
+#define PINMUX_GPIO1__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(1) | 6)
+#define PINMUX_GPIO1__FUNC_B0_DBG_MON_A1 (MTK_PIN_NO(1) | 7)
+
+#define PINMUX_GPIO2__FUNC_B_GPIO2 (MTK_PIN_NO(2) | 0)
+#define PINMUX_GPIO2__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(2) | 1)
+#define PINMUX_GPIO2__FUNC_B0_SPIM5_MOSI (MTK_PIN_NO(2) | 2)
+#define PINMUX_GPIO2__FUNC_O_URTS1 (MTK_PIN_NO(2) | 3)
+#define PINMUX_GPIO2__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(2) | 4)
+#define PINMUX_GPIO2__FUNC_B0_I2SIN_WS (MTK_PIN_NO(2) | 5)
+#define PINMUX_GPIO2__FUNC_B0_I2SO2_WS (MTK_PIN_NO(2) | 6)
+#define PINMUX_GPIO2__FUNC_B0_DBG_MON_A2 (MTK_PIN_NO(2) | 7)
+
+#define PINMUX_GPIO3__FUNC_B_GPIO3 (MTK_PIN_NO(3) | 0)
+#define PINMUX_GPIO3__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(3) | 1)
+#define PINMUX_GPIO3__FUNC_B0_SPIM5_MISO (MTK_PIN_NO(3) | 2)
+#define PINMUX_GPIO3__FUNC_I1_UCTS1 (MTK_PIN_NO(3) | 3)
+#define PINMUX_GPIO3__FUNC_O_DMIC4_CLK (MTK_PIN_NO(3) | 4)
+#define PINMUX_GPIO3__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(3) | 5)
+#define PINMUX_GPIO3__FUNC_O_I2SO2_D0 (MTK_PIN_NO(3) | 6)
+#define PINMUX_GPIO3__FUNC_B0_DBG_MON_A3 (MTK_PIN_NO(3) | 7)
+
+#define PINMUX_GPIO4__FUNC_B_GPIO4 (MTK_PIN_NO(4) | 0)
+#define PINMUX_GPIO4__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(4) | 1)
+#define PINMUX_GPIO4__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(4) | 2)
+#define PINMUX_GPIO4__FUNC_O_I2SO1_MCK (MTK_PIN_NO(4) | 3)
+#define PINMUX_GPIO4__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(4) | 4)
+#define PINMUX_GPIO4__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(4) | 5)
+#define PINMUX_GPIO4__FUNC_O_I2SO2_D1 (MTK_PIN_NO(4) | 6)
+#define PINMUX_GPIO4__FUNC_B0_DBG_MON_A4 (MTK_PIN_NO(4) | 7)
+
+#define PINMUX_GPIO5__FUNC_B_GPIO5 (MTK_PIN_NO(5) | 0)
+#define PINMUX_GPIO5__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(5) | 1)
+#define PINMUX_GPIO5__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(5) | 2)
+#define PINMUX_GPIO5__FUNC_O_I2SO1_BCK (MTK_PIN_NO(5) | 3)
+#define PINMUX_GPIO5__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(5) | 4)
+#define PINMUX_GPIO5__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(5) | 5)
+#define PINMUX_GPIO5__FUNC_O_I2SO2_D2 (MTK_PIN_NO(5) | 6)
+#define PINMUX_GPIO5__FUNC_B0_DBG_MON_A5 (MTK_PIN_NO(5) | 7)
+
+#define PINMUX_GPIO6__FUNC_B_GPIO6 (MTK_PIN_NO(6) | 0)
+#define PINMUX_GPIO6__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(6) | 1)
+#define PINMUX_GPIO6__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(6) | 2)
+#define PINMUX_GPIO6__FUNC_O_I2SO1_WS (MTK_PIN_NO(6) | 3)
+#define PINMUX_GPIO6__FUNC_O_DMIC1_CLK (MTK_PIN_NO(6) | 4)
+#define PINMUX_GPIO6__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(6) | 5)
+#define PINMUX_GPIO6__FUNC_O_I2SO2_D3 (MTK_PIN_NO(6) | 6)
+#define PINMUX_GPIO6__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(6) | 7)
+
+#define PINMUX_GPIO7__FUNC_B_GPIO7 (MTK_PIN_NO(7) | 0)
+#define PINMUX_GPIO7__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(7) | 1)
+#define PINMUX_GPIO7__FUNC_O_SPIM3_CSB (MTK_PIN_NO(7) | 2)
+#define PINMUX_GPIO7__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(7) | 3)
+#define PINMUX_GPIO7__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(7) | 4)
+#define PINMUX_GPIO7__FUNC_O_CMVREF0 (MTK_PIN_NO(7) | 5)
+#define PINMUX_GPIO7__FUNC_O_CLKM0 (MTK_PIN_NO(7) | 6)
+#define PINMUX_GPIO7__FUNC_B0_DBG_MON_A6 (MTK_PIN_NO(7) | 7)
+
+#define PINMUX_GPIO8__FUNC_B_GPIO8 (MTK_PIN_NO(8) | 0)
+#define PINMUX_GPIO8__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(8) | 1)
+#define PINMUX_GPIO8__FUNC_O_SPIM3_CLK (MTK_PIN_NO(8) | 2)
+#define PINMUX_GPIO8__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(8) | 3)
+#define PINMUX_GPIO8__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(8) | 4)
+#define PINMUX_GPIO8__FUNC_O_CMVREF1 (MTK_PIN_NO(8) | 5)
+#define PINMUX_GPIO8__FUNC_O_CLKM1 (MTK_PIN_NO(8) | 6)
+#define PINMUX_GPIO8__FUNC_B0_DBG_MON_A7 (MTK_PIN_NO(8) | 7)
+
+#define PINMUX_GPIO9__FUNC_B_GPIO9 (MTK_PIN_NO(9) | 0)
+#define PINMUX_GPIO9__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(9) | 1)
+#define PINMUX_GPIO9__FUNC_B0_SPIM3_MOSI (MTK_PIN_NO(9) | 2)
+#define PINMUX_GPIO9__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(9) | 3)
+#define PINMUX_GPIO9__FUNC_O_DMIC2_CLK (MTK_PIN_NO(9) | 4)
+#define PINMUX_GPIO9__FUNC_O_CMFLASH0 (MTK_PIN_NO(9) | 5)
+#define PINMUX_GPIO9__FUNC_O_PWM_0 (MTK_PIN_NO(9) | 6)
+#define PINMUX_GPIO9__FUNC_B0_DBG_MON_A8 (MTK_PIN_NO(9) | 7)
+
+#define PINMUX_GPIO10__FUNC_B_GPIO10 (MTK_PIN_NO(10) | 0)
+#define PINMUX_GPIO10__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(10) | 1)
+#define PINMUX_GPIO10__FUNC_B0_SPIM3_MISO (MTK_PIN_NO(10) | 2)
+#define PINMUX_GPIO10__FUNC_I0_TDMIN_DI (MTK_PIN_NO(10) | 3)
+#define PINMUX_GPIO10__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(10) | 4)
+#define PINMUX_GPIO10__FUNC_O_CMFLASH1 (MTK_PIN_NO(10) | 5)
+#define PINMUX_GPIO10__FUNC_O_PWM_1 (MTK_PIN_NO(10) | 6)
+#define PINMUX_GPIO10__FUNC_B0_DBG_MON_A9 (MTK_PIN_NO(10) | 7)
+
+#define PINMUX_GPIO11__FUNC_B_GPIO11 (MTK_PIN_NO(11) | 0)
+#define PINMUX_GPIO11__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(11) | 1)
+#define PINMUX_GPIO11__FUNC_O_SPDIF_OUT (MTK_PIN_NO(11) | 2)
+#define PINMUX_GPIO11__FUNC_O_I2SO1_D0 (MTK_PIN_NO(11) | 3)
+#define PINMUX_GPIO11__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(11) | 4)
+#define PINMUX_GPIO11__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(11) | 5)
+#define PINMUX_GPIO11__FUNC_O_CMVREF6 (MTK_PIN_NO(11) | 6)
+#define PINMUX_GPIO11__FUNC_B0_DBG_MON_A10 (MTK_PIN_NO(11) | 7)
+
+#define PINMUX_GPIO12__FUNC_B_GPIO12 (MTK_PIN_NO(12) | 0)
+#define PINMUX_GPIO12__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(12) | 1)
+#define PINMUX_GPIO12__FUNC_O_SPIM4_CSB (MTK_PIN_NO(12) | 2)
+#define PINMUX_GPIO12__FUNC_B1_JTMS_SEL3 (MTK_PIN_NO(12) | 3)
+#define PINMUX_GPIO12__FUNC_B1_APU_JTAG_TMS (MTK_PIN_NO(12) | 4)
+#define PINMUX_GPIO12__FUNC_I0_VPU_UDI_TMS (MTK_PIN_NO(12) | 5)
+#define PINMUX_GPIO12__FUNC_I0_IPU_JTAG_TMS (MTK_PIN_NO(12) | 6)
+#define PINMUX_GPIO12__FUNC_I0_HDMITX20_HTPLG (MTK_PIN_NO(12) | 7)
+
+#define PINMUX_GPIO13__FUNC_B_GPIO13 (MTK_PIN_NO(13) | 0)
+#define PINMUX_GPIO13__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(13) | 1)
+#define PINMUX_GPIO13__FUNC_O_SPIM4_CLK (MTK_PIN_NO(13) | 2)
+#define PINMUX_GPIO13__FUNC_I0_JTCK_SEL3 (MTK_PIN_NO(13) | 3)
+#define PINMUX_GPIO13__FUNC_I0_APU_JTAG_TCK (MTK_PIN_NO(13) | 4)
+#define PINMUX_GPIO13__FUNC_I0_VPU_UDI_TCK (MTK_PIN_NO(13) | 5)
+#define PINMUX_GPIO13__FUNC_I0_IPU_JTAG_TCK (MTK_PIN_NO(13) | 6)
+#define PINMUX_GPIO13__FUNC_B1_HDMITX20_CEC (MTK_PIN_NO(13) | 7)
+
+#define PINMUX_GPIO14__FUNC_B_GPIO14 (MTK_PIN_NO(14) | 0)
+#define PINMUX_GPIO14__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(14) | 1)
+#define PINMUX_GPIO14__FUNC_B0_SPIM4_MOSI (MTK_PIN_NO(14) | 2)
+#define PINMUX_GPIO14__FUNC_I1_JTDI_SEL3 (MTK_PIN_NO(14) | 3)
+#define PINMUX_GPIO14__FUNC_I1_APU_JTAG_TDI (MTK_PIN_NO(14) | 4)
+#define PINMUX_GPIO14__FUNC_I0_VPU_UDI_TDI (MTK_PIN_NO(14) | 5)
+#define PINMUX_GPIO14__FUNC_I0_IPU_JTAG_TDI (MTK_PIN_NO(14) | 6)
+#define PINMUX_GPIO14__FUNC_B1_HDMITX20_SCL (MTK_PIN_NO(14) | 7)
+
+#define PINMUX_GPIO15__FUNC_B_GPIO15 (MTK_PIN_NO(15) | 0)
+#define PINMUX_GPIO15__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(15) | 1)
+#define PINMUX_GPIO15__FUNC_B0_SPIM4_MISO (MTK_PIN_NO(15) | 2)
+#define PINMUX_GPIO15__FUNC_O_JTDO_SEL3 (MTK_PIN_NO(15) | 3)
+#define PINMUX_GPIO15__FUNC_O_APU_JTAG_TDO (MTK_PIN_NO(15) | 4)
+#define PINMUX_GPIO15__FUNC_O_VPU_UDI_TDO (MTK_PIN_NO(15) | 5)
+#define PINMUX_GPIO15__FUNC_O_IPU_JTAG_TDO (MTK_PIN_NO(15) | 6)
+#define PINMUX_GPIO15__FUNC_B1_HDMITX20_SDA (MTK_PIN_NO(15) | 7)
+
+#define PINMUX_GPIO16__FUNC_B_GPIO16 (MTK_PIN_NO(16) | 0)
+#define PINMUX_GPIO16__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(16) | 1)
+#define PINMUX_GPIO16__FUNC_O_UTXD3 (MTK_PIN_NO(16) | 2)
+#define PINMUX_GPIO16__FUNC_I1_JTRSTn_SEL3 (MTK_PIN_NO(16) | 3)
+#define PINMUX_GPIO16__FUNC_I0_APU_JTAG_TRST (MTK_PIN_NO(16) | 4)
+#define PINMUX_GPIO16__FUNC_I0_VPU_UDI_NTRST (MTK_PIN_NO(16) | 5)
+#define PINMUX_GPIO16__FUNC_I0_IPU_JTAG_TRST (MTK_PIN_NO(16) | 6)
+#define PINMUX_GPIO16__FUNC_O_HDMITX20_PWR5V (MTK_PIN_NO(16) | 7)
+
+#define PINMUX_GPIO17__FUNC_B_GPIO17 (MTK_PIN_NO(17) | 0)
+#define PINMUX_GPIO17__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(17) | 1)
+#define PINMUX_GPIO17__FUNC_I1_URXD3 (MTK_PIN_NO(17) | 2)
+#define PINMUX_GPIO17__FUNC_O_CMFLASH2 (MTK_PIN_NO(17) | 3)
+#define PINMUX_GPIO17__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(17) | 4)
+#define PINMUX_GPIO17__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(17) | 5)
+#define PINMUX_GPIO17__FUNC_O_CMVREF7 (MTK_PIN_NO(17) | 6)
+#define PINMUX_GPIO17__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(17) | 7)
+
+#define PINMUX_GPIO18__FUNC_B_GPIO18 (MTK_PIN_NO(18) | 0)
+#define PINMUX_GPIO18__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(18) | 1)
+#define PINMUX_GPIO18__FUNC_O_CMFLASH0 (MTK_PIN_NO(18) | 2)
+#define PINMUX_GPIO18__FUNC_O_CMVREF4 (MTK_PIN_NO(18) | 3)
+#define PINMUX_GPIO18__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(18) | 4)
+#define PINMUX_GPIO18__FUNC_O_UTXD1 (MTK_PIN_NO(18) | 5)
+#define PINMUX_GPIO18__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(18) | 6)
+#define PINMUX_GPIO18__FUNC_B0_DBG_MON_A11 (MTK_PIN_NO(18) | 7)
+
+#define PINMUX_GPIO19__FUNC_B_GPIO19 (MTK_PIN_NO(19) | 0)
+#define PINMUX_GPIO19__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(19) | 1)
+#define PINMUX_GPIO19__FUNC_O_CMFLASH1 (MTK_PIN_NO(19) | 2)
+#define PINMUX_GPIO19__FUNC_O_CMVREF5 (MTK_PIN_NO(19) | 3)
+#define PINMUX_GPIO19__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(19) | 4)
+#define PINMUX_GPIO19__FUNC_I1_URXD1 (MTK_PIN_NO(19) | 5)
+#define PINMUX_GPIO19__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(19) | 6)
+#define PINMUX_GPIO19__FUNC_B0_DBG_MON_A12 (MTK_PIN_NO(19) | 7)
+
+#define PINMUX_GPIO20__FUNC_B_GPIO20 (MTK_PIN_NO(20) | 0)
+#define PINMUX_GPIO20__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(20) | 1)
+#define PINMUX_GPIO20__FUNC_O_CMFLASH2 (MTK_PIN_NO(20) | 2)
+#define PINMUX_GPIO20__FUNC_O_CLKM2 (MTK_PIN_NO(20) | 3)
+#define PINMUX_GPIO20__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(20) | 4)
+#define PINMUX_GPIO20__FUNC_O_URTS1 (MTK_PIN_NO(20) | 5)
+#define PINMUX_GPIO20__FUNC_O_TP_URTS1_AO (MTK_PIN_NO(20) | 6)
+#define PINMUX_GPIO20__FUNC_B0_DBG_MON_A13 (MTK_PIN_NO(20) | 7)
+
+#define PINMUX_GPIO21__FUNC_B_GPIO21 (MTK_PIN_NO(21) | 0)
+#define PINMUX_GPIO21__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(21) | 1)
+#define PINMUX_GPIO21__FUNC_O_CMFLASH3 (MTK_PIN_NO(21) | 2)
+#define PINMUX_GPIO21__FUNC_O_CLKM3 (MTK_PIN_NO(21) | 3)
+#define PINMUX_GPIO21__FUNC_I0_TDMIN_DI (MTK_PIN_NO(21) | 4)
+#define PINMUX_GPIO21__FUNC_I1_UCTS1 (MTK_PIN_NO(21) | 5)
+#define PINMUX_GPIO21__FUNC_I1_TP_UCTS1_AO (MTK_PIN_NO(21) | 6)
+#define PINMUX_GPIO21__FUNC_B0_DBG_MON_A14 (MTK_PIN_NO(21) | 7)
+
+#define PINMUX_GPIO22__FUNC_B_GPIO22 (MTK_PIN_NO(22) | 0)
+#define PINMUX_GPIO22__FUNC_O_CMMCLK0 (MTK_PIN_NO(22) | 1)
+#define PINMUX_GPIO22__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(22) | 5)
+#define PINMUX_GPIO22__FUNC_B0_DBG_MON_A15 (MTK_PIN_NO(22) | 7)
+
+#define PINMUX_GPIO23__FUNC_B_GPIO23 (MTK_PIN_NO(23) | 0)
+#define PINMUX_GPIO23__FUNC_O_CMMCLK1 (MTK_PIN_NO(23) | 1)
+#define PINMUX_GPIO23__FUNC_O_PWM_2 (MTK_PIN_NO(23) | 3)
+#define PINMUX_GPIO23__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(23) | 4)
+#define PINMUX_GPIO23__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(23) | 5)
+#define PINMUX_GPIO23__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(23) | 6)
+#define PINMUX_GPIO23__FUNC_B0_DBG_MON_A16 (MTK_PIN_NO(23) | 7)
+
+#define PINMUX_GPIO24__FUNC_B_GPIO24 (MTK_PIN_NO(24) | 0)
+#define PINMUX_GPIO24__FUNC_O_CMMCLK2 (MTK_PIN_NO(24) | 1)
+#define PINMUX_GPIO24__FUNC_O_PWM_3 (MTK_PIN_NO(24) | 3)
+#define PINMUX_GPIO24__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(24) | 4)
+#define PINMUX_GPIO24__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(24) | 5)
+#define PINMUX_GPIO24__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(24) | 6)
+#define PINMUX_GPIO24__FUNC_B0_MD32_0_GPIO2 (MTK_PIN_NO(24) | 7)
+
+#define PINMUX_GPIO25__FUNC_B_GPIO25 (MTK_PIN_NO(25) | 0)
+#define PINMUX_GPIO25__FUNC_O_LCM_RST (MTK_PIN_NO(25) | 1)
+#define PINMUX_GPIO25__FUNC_O_LCM1_RST (MTK_PIN_NO(25) | 2)
+#define PINMUX_GPIO25__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(25) | 3)
+
+#define PINMUX_GPIO26__FUNC_B_GPIO26 (MTK_PIN_NO(26) | 0)
+#define PINMUX_GPIO26__FUNC_I0_DSI_TE (MTK_PIN_NO(26) | 1)
+#define PINMUX_GPIO26__FUNC_I0_DSI1_TE (MTK_PIN_NO(26) | 2)
+#define PINMUX_GPIO26__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(26) | 3)
+
+#define PINMUX_GPIO27__FUNC_B_GPIO27 (MTK_PIN_NO(27) | 0)
+#define PINMUX_GPIO27__FUNC_O_LCM1_RST (MTK_PIN_NO(27) | 1)
+#define PINMUX_GPIO27__FUNC_O_LCM_RST (MTK_PIN_NO(27) | 2)
+#define PINMUX_GPIO27__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(27) | 3)
+#define PINMUX_GPIO27__FUNC_O_CMVREF2 (MTK_PIN_NO(27) | 4)
+#define PINMUX_GPIO27__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(27) | 5)
+#define PINMUX_GPIO27__FUNC_O_PWM_2 (MTK_PIN_NO(27) | 6)
+#define PINMUX_GPIO27__FUNC_B0_DBG_MON_A17 (MTK_PIN_NO(27) | 7)
+
+#define PINMUX_GPIO28__FUNC_B_GPIO28 (MTK_PIN_NO(28) | 0)
+#define PINMUX_GPIO28__FUNC_I0_DSI1_TE (MTK_PIN_NO(28) | 1)
+#define PINMUX_GPIO28__FUNC_I0_DSI_TE (MTK_PIN_NO(28) | 2)
+#define PINMUX_GPIO28__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(28) | 3)
+#define PINMUX_GPIO28__FUNC_O_CMVREF3 (MTK_PIN_NO(28) | 4)
+#define PINMUX_GPIO28__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(28) | 5)
+#define PINMUX_GPIO28__FUNC_O_PWM_3 (MTK_PIN_NO(28) | 6)
+#define PINMUX_GPIO28__FUNC_B0_DBG_MON_A18 (MTK_PIN_NO(28) | 7)
+
+#define PINMUX_GPIO29__FUNC_B_GPIO29 (MTK_PIN_NO(29) | 0)
+#define PINMUX_GPIO29__FUNC_O_DISP_PWM0 (MTK_PIN_NO(29) | 1)
+#define PINMUX_GPIO29__FUNC_O_DISP_PWM1 (MTK_PIN_NO(29) | 2)
+
+#define PINMUX_GPIO30__FUNC_B_GPIO30 (MTK_PIN_NO(30) | 0)
+#define PINMUX_GPIO30__FUNC_O_DISP_PWM1 (MTK_PIN_NO(30) | 1)
+#define PINMUX_GPIO30__FUNC_O_DISP_PWM0 (MTK_PIN_NO(30) | 2)
+#define PINMUX_GPIO30__FUNC_O_CMFLASH3 (MTK_PIN_NO(30) | 3)
+#define PINMUX_GPIO30__FUNC_O_PWM_1 (MTK_PIN_NO(30) | 4)
+#define PINMUX_GPIO30__FUNC_B0_DBG_MON_A19 (MTK_PIN_NO(30) | 7)
+
+#define PINMUX_GPIO31__FUNC_B_GPIO31 (MTK_PIN_NO(31) | 0)
+#define PINMUX_GPIO31__FUNC_O_UTXD0 (MTK_PIN_NO(31) | 1)
+#define PINMUX_GPIO31__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(31) | 2)
+#define PINMUX_GPIO31__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(31) | 3)
+#define PINMUX_GPIO31__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(31) | 4)
+#define PINMUX_GPIO31__FUNC_O_MD32_0_TXD (MTK_PIN_NO(31) | 5)
+#define PINMUX_GPIO31__FUNC_O_MD32_1_TXD (MTK_PIN_NO(31) | 6)
+#define PINMUX_GPIO31__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(31) | 7)
+
+#define PINMUX_GPIO32__FUNC_B_GPIO32 (MTK_PIN_NO(32) | 0)
+#define PINMUX_GPIO32__FUNC_I1_URXD0 (MTK_PIN_NO(32) | 1)
+#define PINMUX_GPIO32__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(32) | 2)
+#define PINMUX_GPIO32__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(32) | 3)
+#define PINMUX_GPIO32__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(32) | 4)
+#define PINMUX_GPIO32__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(32) | 5)
+#define PINMUX_GPIO32__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(32) | 6)
+#define PINMUX_GPIO32__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(32) | 7)
+
+#define PINMUX_GPIO33__FUNC_B_GPIO33 (MTK_PIN_NO(33) | 0)
+#define PINMUX_GPIO33__FUNC_O_UTXD1 (MTK_PIN_NO(33) | 1)
+#define PINMUX_GPIO33__FUNC_O_URTS2 (MTK_PIN_NO(33) | 2)
+#define PINMUX_GPIO33__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(33) | 3)
+#define PINMUX_GPIO33__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(33) | 4)
+#define PINMUX_GPIO33__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(33) | 5)
+#define PINMUX_GPIO33__FUNC_O_MD32_0_TXD (MTK_PIN_NO(33) | 6)
+#define PINMUX_GPIO33__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(33) | 7)
+
+#define PINMUX_GPIO34__FUNC_B_GPIO34 (MTK_PIN_NO(34) | 0)
+#define PINMUX_GPIO34__FUNC_I1_URXD1 (MTK_PIN_NO(34) | 1)
+#define PINMUX_GPIO34__FUNC_I1_UCTS2 (MTK_PIN_NO(34) | 2)
+#define PINMUX_GPIO34__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(34) | 3)
+#define PINMUX_GPIO34__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(34) | 4)
+#define PINMUX_GPIO34__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(34) | 5)
+#define PINMUX_GPIO34__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(34) | 6)
+#define PINMUX_GPIO34__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(34) | 7)
+
+#define PINMUX_GPIO35__FUNC_B_GPIO35 (MTK_PIN_NO(35) | 0)
+#define PINMUX_GPIO35__FUNC_O_UTXD2 (MTK_PIN_NO(35) | 1)
+#define PINMUX_GPIO35__FUNC_O_URTS1 (MTK_PIN_NO(35) | 2)
+#define PINMUX_GPIO35__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(35) | 3)
+#define PINMUX_GPIO35__FUNC_O_TP_URTS1_AO (MTK_PIN_NO(35) | 4)
+#define PINMUX_GPIO35__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(35) | 5)
+#define PINMUX_GPIO35__FUNC_O_MD32_1_TXD (MTK_PIN_NO(35) | 6)
+#define PINMUX_GPIO35__FUNC_B0_DBG_MON_A20 (MTK_PIN_NO(35) | 7)
+
+#define PINMUX_GPIO36__FUNC_B_GPIO36 (MTK_PIN_NO(36) | 0)
+#define PINMUX_GPIO36__FUNC_I1_URXD2 (MTK_PIN_NO(36) | 1)
+#define PINMUX_GPIO36__FUNC_I1_UCTS1 (MTK_PIN_NO(36) | 2)
+#define PINMUX_GPIO36__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(36) | 3)
+#define PINMUX_GPIO36__FUNC_I1_TP_UCTS1_AO (MTK_PIN_NO(36) | 4)
+#define PINMUX_GPIO36__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(36) | 5)
+#define PINMUX_GPIO36__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(36) | 6)
+#define PINMUX_GPIO36__FUNC_B0_DBG_MON_A21 (MTK_PIN_NO(36) | 7)
+
+#define PINMUX_GPIO37__FUNC_B_GPIO37 (MTK_PIN_NO(37) | 0)
+#define PINMUX_GPIO37__FUNC_B1_JTMS_SEL1 (MTK_PIN_NO(37) | 1)
+#define PINMUX_GPIO37__FUNC_I0_UDI_TMS (MTK_PIN_NO(37) | 2)
+#define PINMUX_GPIO37__FUNC_I1_SPM_JTAG_TMS (MTK_PIN_NO(37) | 3)
+#define PINMUX_GPIO37__FUNC_I1_ADSP_JTAG0_TMS (MTK_PIN_NO(37) | 4)
+#define PINMUX_GPIO37__FUNC_I1_SCP_JTAG0_TMS (MTK_PIN_NO(37) | 5)
+#define PINMUX_GPIO37__FUNC_I1_CCU0_JTAG_TMS (MTK_PIN_NO(37) | 6)
+#define PINMUX_GPIO37__FUNC_I1_MCUPM_JTAG_TMS (MTK_PIN_NO(37) | 7)
+
+#define PINMUX_GPIO38__FUNC_B_GPIO38 (MTK_PIN_NO(38) | 0)
+#define PINMUX_GPIO38__FUNC_I0_JTCK_SEL1 (MTK_PIN_NO(38) | 1)
+#define PINMUX_GPIO38__FUNC_I0_UDI_TCK (MTK_PIN_NO(38) | 2)
+#define PINMUX_GPIO38__FUNC_I1_SPM_JTAG_TCK (MTK_PIN_NO(38) | 3)
+#define PINMUX_GPIO38__FUNC_I0_ADSP_JTAG0_TCK (MTK_PIN_NO(38) | 4)
+#define PINMUX_GPIO38__FUNC_I1_SCP_JTAG0_TCK (MTK_PIN_NO(38) | 5)
+#define PINMUX_GPIO38__FUNC_I1_CCU0_JTAG_TCK (MTK_PIN_NO(38) | 6)
+#define PINMUX_GPIO38__FUNC_I1_MCUPM_JTAG_TCK (MTK_PIN_NO(38) | 7)
+
+#define PINMUX_GPIO39__FUNC_B_GPIO39 (MTK_PIN_NO(39) | 0)
+#define PINMUX_GPIO39__FUNC_I1_JTDI_SEL1 (MTK_PIN_NO(39) | 1)
+#define PINMUX_GPIO39__FUNC_I0_UDI_TDI (MTK_PIN_NO(39) | 2)
+#define PINMUX_GPIO39__FUNC_I1_SPM_JTAG_TDI (MTK_PIN_NO(39) | 3)
+#define PINMUX_GPIO39__FUNC_I1_ADSP_JTAG0_TDI (MTK_PIN_NO(39) | 4)
+#define PINMUX_GPIO39__FUNC_I1_SCP_JTAG0_TDI (MTK_PIN_NO(39) | 5)
+#define PINMUX_GPIO39__FUNC_I1_CCU0_JTAG_TDI (MTK_PIN_NO(39) | 6)
+#define PINMUX_GPIO39__FUNC_I1_MCUPM_JTAG_TDI (MTK_PIN_NO(39) | 7)
+
+#define PINMUX_GPIO40__FUNC_B_GPIO40 (MTK_PIN_NO(40) | 0)
+#define PINMUX_GPIO40__FUNC_O_JTDO_SEL1 (MTK_PIN_NO(40) | 1)
+#define PINMUX_GPIO40__FUNC_O_UDI_TDO (MTK_PIN_NO(40) | 2)
+#define PINMUX_GPIO40__FUNC_O_SPM_JTAG_TDO (MTK_PIN_NO(40) | 3)
+#define PINMUX_GPIO40__FUNC_O_ADSP_JTAG0_TDO (MTK_PIN_NO(40) | 4)
+#define PINMUX_GPIO40__FUNC_O_SCP_JTAG0_TDO (MTK_PIN_NO(40) | 5)
+#define PINMUX_GPIO40__FUNC_O_CCU0_JTAG_TDO (MTK_PIN_NO(40) | 6)
+#define PINMUX_GPIO40__FUNC_O_MCUPM_JTAG_TDO (MTK_PIN_NO(40) | 7)
+
+#define PINMUX_GPIO41__FUNC_B_GPIO41 (MTK_PIN_NO(41) | 0)
+#define PINMUX_GPIO41__FUNC_I1_JTRSTn_SEL1 (MTK_PIN_NO(41) | 1)
+#define PINMUX_GPIO41__FUNC_I0_UDI_NTRST (MTK_PIN_NO(41) | 2)
+#define PINMUX_GPIO41__FUNC_I0_SPM_JTAG_TRSTN (MTK_PIN_NO(41) | 3)
+#define PINMUX_GPIO41__FUNC_I1_ADSP_JTAG0_TRSTN (MTK_PIN_NO(41) | 4)
+#define PINMUX_GPIO41__FUNC_I0_SCP_JTAG0_TRSTN (MTK_PIN_NO(41) | 5)
+#define PINMUX_GPIO41__FUNC_I1_CCU0_JTAG_TRST (MTK_PIN_NO(41) | 6)
+#define PINMUX_GPIO41__FUNC_I0_MCUPM_JTAG_TRSTN (MTK_PIN_NO(41) | 7)
+
+#define PINMUX_GPIO42__FUNC_B_GPIO42 (MTK_PIN_NO(42) | 0)
+#define PINMUX_GPIO42__FUNC_B1_KPCOL0 (MTK_PIN_NO(42) | 1)
+
+#define PINMUX_GPIO43__FUNC_B_GPIO43 (MTK_PIN_NO(43) | 0)
+#define PINMUX_GPIO43__FUNC_B1_KPCOL1 (MTK_PIN_NO(43) | 1)
+#define PINMUX_GPIO43__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(43) | 2)
+#define PINMUX_GPIO43__FUNC_O_CMFLASH2 (MTK_PIN_NO(43) | 3)
+#define PINMUX_GPIO43__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(43) | 4)
+#define PINMUX_GPIO43__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(43) | 7)
+
+#define PINMUX_GPIO44__FUNC_B_GPIO44 (MTK_PIN_NO(44) | 0)
+#define PINMUX_GPIO44__FUNC_B1_KPROW0 (MTK_PIN_NO(44) | 1)
+
+#define PINMUX_GPIO45__FUNC_B_GPIO45 (MTK_PIN_NO(45) | 0)
+#define PINMUX_GPIO45__FUNC_B1_KPROW1 (MTK_PIN_NO(45) | 1)
+#define PINMUX_GPIO45__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(45) | 2)
+#define PINMUX_GPIO45__FUNC_O_CMFLASH3 (MTK_PIN_NO(45) | 3)
+#define PINMUX_GPIO45__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(45) | 4)
+#define PINMUX_GPIO45__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(45) | 7)
+
+#define PINMUX_GPIO46__FUNC_B_GPIO46 (MTK_PIN_NO(46) | 0)
+#define PINMUX_GPIO46__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(46) | 1)
+#define PINMUX_GPIO46__FUNC_O_PWM_0 (MTK_PIN_NO(46) | 2)
+#define PINMUX_GPIO46__FUNC_I0_VBUSVALID_2P (MTK_PIN_NO(46) | 3)
+#define PINMUX_GPIO46__FUNC_B0_DBG_MON_A22 (MTK_PIN_NO(46) | 7)
+
+#define PINMUX_GPIO47__FUNC_B_GPIO47 (MTK_PIN_NO(47) | 0)
+#define PINMUX_GPIO47__FUNC_I1_WAKEN (MTK_PIN_NO(47) | 1)
+#define PINMUX_GPIO47__FUNC_O_GDU_TROOPS_DET0 (MTK_PIN_NO(47) | 6)
+
+#define PINMUX_GPIO48__FUNC_B_GPIO48 (MTK_PIN_NO(48) | 0)
+#define PINMUX_GPIO48__FUNC_O_PERSTN (MTK_PIN_NO(48) | 1)
+#define PINMUX_GPIO48__FUNC_O_GDU_TROOPS_DET1 (MTK_PIN_NO(48) | 6)
+
+#define PINMUX_GPIO49__FUNC_B_GPIO49 (MTK_PIN_NO(49) | 0)
+#define PINMUX_GPIO49__FUNC_B1_CLKREQN (MTK_PIN_NO(49) | 1)
+#define PINMUX_GPIO49__FUNC_O_GDU_TROOPS_DET2 (MTK_PIN_NO(49) | 6)
+
+#define PINMUX_GPIO50__FUNC_B_GPIO50 (MTK_PIN_NO(50) | 0)
+#define PINMUX_GPIO50__FUNC_O_HDMITX20_PWR5V (MTK_PIN_NO(50) | 1)
+#define PINMUX_GPIO50__FUNC_I1_IDDIG_1P (MTK_PIN_NO(50) | 3)
+#define PINMUX_GPIO50__FUNC_I1_SCP_JTAG1_TMS (MTK_PIN_NO(50) | 4)
+#define PINMUX_GPIO50__FUNC_I1_SSPM_JTAG_TMS (MTK_PIN_NO(50) | 5)
+#define PINMUX_GPIO50__FUNC_I1_MD32_0_JTAG_TMS (MTK_PIN_NO(50) | 6)
+#define PINMUX_GPIO50__FUNC_I1_MD32_1_JTAG_TMS (MTK_PIN_NO(50) | 7)
+
+#define PINMUX_GPIO51__FUNC_B_GPIO51 (MTK_PIN_NO(51) | 0)
+#define PINMUX_GPIO51__FUNC_I0_HDMITX20_HTPLG (MTK_PIN_NO(51) | 1)
+#define PINMUX_GPIO51__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(51) | 2)
+#define PINMUX_GPIO51__FUNC_O_USB_DRVVBUS_1P (MTK_PIN_NO(51) | 3)
+#define PINMUX_GPIO51__FUNC_I1_SCP_JTAG1_TCK (MTK_PIN_NO(51) | 4)
+#define PINMUX_GPIO51__FUNC_I1_SSPM_JTAG_TCK (MTK_PIN_NO(51) | 5)
+#define PINMUX_GPIO51__FUNC_I1_MD32_0_JTAG_TCK (MTK_PIN_NO(51) | 6)
+#define PINMUX_GPIO51__FUNC_I1_MD32_1_JTAG_TCK (MTK_PIN_NO(51) | 7)
+
+#define PINMUX_GPIO52__FUNC_B_GPIO52 (MTK_PIN_NO(52) | 0)
+#define PINMUX_GPIO52__FUNC_B1_HDMITX20_CEC (MTK_PIN_NO(52) | 1)
+#define PINMUX_GPIO52__FUNC_I0_VBUSVALID_1P (MTK_PIN_NO(52) | 3)
+#define PINMUX_GPIO52__FUNC_I1_SCP_JTAG1_TDI (MTK_PIN_NO(52) | 4)
+#define PINMUX_GPIO52__FUNC_I1_SSPM_JTAG_TDI (MTK_PIN_NO(52) | 5)
+#define PINMUX_GPIO52__FUNC_I1_MD32_0_JTAG_TDI (MTK_PIN_NO(52) | 6)
+#define PINMUX_GPIO52__FUNC_I1_MD32_1_JTAG_TDI (MTK_PIN_NO(52) | 7)
+
+#define PINMUX_GPIO53__FUNC_B_GPIO53 (MTK_PIN_NO(53) | 0)
+#define PINMUX_GPIO53__FUNC_B1_HDMITX20_SCL (MTK_PIN_NO(53) | 1)
+#define PINMUX_GPIO53__FUNC_I1_IDDIG_2P (MTK_PIN_NO(53) | 3)
+#define PINMUX_GPIO53__FUNC_O_SCP_JTAG1_TDO (MTK_PIN_NO(53) | 4)
+#define PINMUX_GPIO53__FUNC_O_SSPM_JTAG_TDO (MTK_PIN_NO(53) | 5)
+#define PINMUX_GPIO53__FUNC_O_MD32_0_JTAG_TDO (MTK_PIN_NO(53) | 6)
+#define PINMUX_GPIO53__FUNC_O_MD32_1_JTAG_TDO (MTK_PIN_NO(53) | 7)
+
+#define PINMUX_GPIO54__FUNC_B_GPIO54 (MTK_PIN_NO(54) | 0)
+#define PINMUX_GPIO54__FUNC_B1_HDMITX20_SDA (MTK_PIN_NO(54) | 1)
+#define PINMUX_GPIO54__FUNC_O_USB_DRVVBUS_2P (MTK_PIN_NO(54) | 3)
+#define PINMUX_GPIO54__FUNC_I0_SCP_JTAG1_TRSTN (MTK_PIN_NO(54) | 4)
+#define PINMUX_GPIO54__FUNC_I0_SSPM_JTAG_TRSTN (MTK_PIN_NO(54) | 5)
+#define PINMUX_GPIO54__FUNC_I1_MD32_0_JTAG_TRST (MTK_PIN_NO(54) | 6)
+#define PINMUX_GPIO54__FUNC_I1_MD32_1_JTAG_TRST (MTK_PIN_NO(54) | 7)
+
+#define PINMUX_GPIO55__FUNC_B_GPIO55 (MTK_PIN_NO(55) | 0)
+#define PINMUX_GPIO55__FUNC_B1_SCL0 (MTK_PIN_NO(55) | 1)
+#define PINMUX_GPIO55__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(55) | 2)
+#define PINMUX_GPIO55__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(55) | 3)
+#define PINMUX_GPIO55__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(55) | 4)
+
+#define PINMUX_GPIO56__FUNC_B_GPIO56 (MTK_PIN_NO(56) | 0)
+#define PINMUX_GPIO56__FUNC_B1_SDA0 (MTK_PIN_NO(56) | 1)
+#define PINMUX_GPIO56__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(56) | 2)
+#define PINMUX_GPIO56__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(56) | 3)
+#define PINMUX_GPIO56__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(56) | 4)
+
+#define PINMUX_GPIO57__FUNC_B_GPIO57 (MTK_PIN_NO(57) | 0)
+#define PINMUX_GPIO57__FUNC_B1_SCL1 (MTK_PIN_NO(57) | 1)
+
+#define PINMUX_GPIO58__FUNC_B_GPIO58 (MTK_PIN_NO(58) | 0)
+#define PINMUX_GPIO58__FUNC_B1_SDA1 (MTK_PIN_NO(58) | 1)
+
+#define PINMUX_GPIO59__FUNC_B_GPIO59 (MTK_PIN_NO(59) | 0)
+#define PINMUX_GPIO59__FUNC_B1_SCL2 (MTK_PIN_NO(59) | 1)
+#define PINMUX_GPIO59__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(59) | 2)
+#define PINMUX_GPIO59__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(59) | 3)
+
+#define PINMUX_GPIO60__FUNC_B_GPIO60 (MTK_PIN_NO(60) | 0)
+#define PINMUX_GPIO60__FUNC_B1_SDA2 (MTK_PIN_NO(60) | 1)
+#define PINMUX_GPIO60__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(60) | 2)
+#define PINMUX_GPIO60__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(60) | 3)
+
+#define PINMUX_GPIO61__FUNC_B_GPIO61 (MTK_PIN_NO(61) | 0)
+#define PINMUX_GPIO61__FUNC_B1_SCL3 (MTK_PIN_NO(61) | 1)
+#define PINMUX_GPIO61__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(61) | 2)
+#define PINMUX_GPIO61__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(61) | 3)
+#define PINMUX_GPIO61__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(61) | 4)
+
+#define PINMUX_GPIO62__FUNC_B_GPIO62 (MTK_PIN_NO(62) | 0)
+#define PINMUX_GPIO62__FUNC_B1_SDA3 (MTK_PIN_NO(62) | 1)
+#define PINMUX_GPIO62__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(62) | 2)
+#define PINMUX_GPIO62__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(62) | 3)
+#define PINMUX_GPIO62__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(62) | 4)
+
+#define PINMUX_GPIO63__FUNC_B_GPIO63 (MTK_PIN_NO(63) | 0)
+#define PINMUX_GPIO63__FUNC_B1_SCL4 (MTK_PIN_NO(63) | 1)
+
+#define PINMUX_GPIO64__FUNC_B_GPIO64 (MTK_PIN_NO(64) | 0)
+#define PINMUX_GPIO64__FUNC_B1_SDA4 (MTK_PIN_NO(64) | 1)
+
+#define PINMUX_GPIO65__FUNC_B_GPIO65 (MTK_PIN_NO(65) | 0)
+#define PINMUX_GPIO65__FUNC_B1_SCL5 (MTK_PIN_NO(65) | 1)
+#define PINMUX_GPIO65__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(65) | 2)
+#define PINMUX_GPIO65__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(65) | 3)
+
+#define PINMUX_GPIO66__FUNC_B_GPIO66 (MTK_PIN_NO(66) | 0)
+#define PINMUX_GPIO66__FUNC_B1_SDA5 (MTK_PIN_NO(66) | 1)
+#define PINMUX_GPIO66__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(66) | 2)
+#define PINMUX_GPIO66__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(66) | 3)
+
+#define PINMUX_GPIO67__FUNC_B_GPIO67 (MTK_PIN_NO(67) | 0)
+#define PINMUX_GPIO67__FUNC_B1_SCL6 (MTK_PIN_NO(67) | 1)
+#define PINMUX_GPIO67__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(67) | 2)
+#define PINMUX_GPIO67__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(67) | 3)
+#define PINMUX_GPIO67__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(67) | 4)
+
+#define PINMUX_GPIO68__FUNC_B_GPIO68 (MTK_PIN_NO(68) | 0)
+#define PINMUX_GPIO68__FUNC_B1_SDA6 (MTK_PIN_NO(68) | 1)
+#define PINMUX_GPIO68__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(68) | 2)
+#define PINMUX_GPIO68__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(68) | 3)
+#define PINMUX_GPIO68__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(68) | 4)
+
+#define PINMUX_GPIO69__FUNC_B_GPIO69 (MTK_PIN_NO(69) | 0)
+#define PINMUX_GPIO69__FUNC_O_SPIM0_CSB (MTK_PIN_NO(69) | 1)
+#define PINMUX_GPIO69__FUNC_O_SCP_SPI0_CS (MTK_PIN_NO(69) | 2)
+#define PINMUX_GPIO69__FUNC_O_DMIC3_CLK (MTK_PIN_NO(69) | 3)
+#define PINMUX_GPIO69__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(69) | 4)
+#define PINMUX_GPIO69__FUNC_O_CMVREF0 (MTK_PIN_NO(69) | 5)
+#define PINMUX_GPIO69__FUNC_O_GDU_SUM_TROOP0_0 (MTK_PIN_NO(69) | 6)
+#define PINMUX_GPIO69__FUNC_B0_DBG_MON_A23 (MTK_PIN_NO(69) | 7)
+
+#define PINMUX_GPIO70__FUNC_B_GPIO70 (MTK_PIN_NO(70) | 0)
+#define PINMUX_GPIO70__FUNC_O_SPIM0_CLK (MTK_PIN_NO(70) | 1)
+#define PINMUX_GPIO70__FUNC_O_SCP_SPI0_CK (MTK_PIN_NO(70) | 2)
+#define PINMUX_GPIO70__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(70) | 3)
+#define PINMUX_GPIO70__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(70) | 4)
+#define PINMUX_GPIO70__FUNC_O_CMVREF1 (MTK_PIN_NO(70) | 5)
+#define PINMUX_GPIO70__FUNC_O_GDU_SUM_TROOP0_1 (MTK_PIN_NO(70) | 6)
+#define PINMUX_GPIO70__FUNC_B0_DBG_MON_A24 (MTK_PIN_NO(70) | 7)
+
+#define PINMUX_GPIO71__FUNC_B_GPIO71 (MTK_PIN_NO(71) | 0)
+#define PINMUX_GPIO71__FUNC_B0_SPIM0_MOSI (MTK_PIN_NO(71) | 1)
+#define PINMUX_GPIO71__FUNC_O_SCP_SPI0_MO (MTK_PIN_NO(71) | 2)
+#define PINMUX_GPIO71__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(71) | 3)
+#define PINMUX_GPIO71__FUNC_B0_MD32_1_GPIO2 (MTK_PIN_NO(71) | 4)
+#define PINMUX_GPIO71__FUNC_O_CMVREF2 (MTK_PIN_NO(71) | 5)
+#define PINMUX_GPIO71__FUNC_O_GDU_SUM_TROOP0_2 (MTK_PIN_NO(71) | 6)
+#define PINMUX_GPIO71__FUNC_B0_DBG_MON_A25 (MTK_PIN_NO(71) | 7)
+
+#define PINMUX_GPIO72__FUNC_B_GPIO72 (MTK_PIN_NO(72) | 0)
+#define PINMUX_GPIO72__FUNC_B0_SPIM0_MISO (MTK_PIN_NO(72) | 1)
+#define PINMUX_GPIO72__FUNC_I0_SCP_SPI0_MI (MTK_PIN_NO(72) | 2)
+#define PINMUX_GPIO72__FUNC_O_DMIC4_CLK (MTK_PIN_NO(72) | 3)
+#define PINMUX_GPIO72__FUNC_O_CMVREF3 (MTK_PIN_NO(72) | 5)
+#define PINMUX_GPIO72__FUNC_O_GDU_SUM_TROOP1_0 (MTK_PIN_NO(72) | 6)
+#define PINMUX_GPIO72__FUNC_B0_DBG_MON_A26 (MTK_PIN_NO(72) | 7)
+
+#define PINMUX_GPIO73__FUNC_B_GPIO73 (MTK_PIN_NO(73) | 0)
+#define PINMUX_GPIO73__FUNC_B0_SPIM0_MIO2 (MTK_PIN_NO(73) | 1)
+#define PINMUX_GPIO73__FUNC_O_UTXD3 (MTK_PIN_NO(73) | 2)
+#define PINMUX_GPIO73__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(73) | 3)
+#define PINMUX_GPIO73__FUNC_O_CLKM0 (MTK_PIN_NO(73) | 4)
+#define PINMUX_GPIO73__FUNC_O_CMVREF4 (MTK_PIN_NO(73) | 5)
+#define PINMUX_GPIO73__FUNC_O_GDU_SUM_TROOP1_1 (MTK_PIN_NO(73) | 6)
+#define PINMUX_GPIO73__FUNC_B0_DBG_MON_A27 (MTK_PIN_NO(73) | 7)
+
+#define PINMUX_GPIO74__FUNC_B_GPIO74 (MTK_PIN_NO(74) | 0)
+#define PINMUX_GPIO74__FUNC_B0_SPIM0_MIO3 (MTK_PIN_NO(74) | 1)
+#define PINMUX_GPIO74__FUNC_I1_URXD3 (MTK_PIN_NO(74) | 2)
+#define PINMUX_GPIO74__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(74) | 3)
+#define PINMUX_GPIO74__FUNC_O_CLKM1 (MTK_PIN_NO(74) | 4)
+#define PINMUX_GPIO74__FUNC_O_CMVREF5 (MTK_PIN_NO(74) | 5)
+#define PINMUX_GPIO74__FUNC_O_GDU_SUM_TROOP1_2 (MTK_PIN_NO(74) | 6)
+#define PINMUX_GPIO74__FUNC_B0_DBG_MON_A28 (MTK_PIN_NO(74) | 7)
+
+#define PINMUX_GPIO75__FUNC_B_GPIO75 (MTK_PIN_NO(75) | 0)
+#define PINMUX_GPIO75__FUNC_O_SPIM1_CSB (MTK_PIN_NO(75) | 1)
+#define PINMUX_GPIO75__FUNC_O_SCP_SPI1_A_CS (MTK_PIN_NO(75) | 2)
+#define PINMUX_GPIO75__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(75) | 3)
+#define PINMUX_GPIO75__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(75) | 4)
+#define PINMUX_GPIO75__FUNC_O_CMVREF6 (MTK_PIN_NO(75) | 5)
+#define PINMUX_GPIO75__FUNC_O_GDU_SUM_TROOP2_0 (MTK_PIN_NO(75) | 6)
+#define PINMUX_GPIO75__FUNC_B0_DBG_MON_A29 (MTK_PIN_NO(75) | 7)
+
+#define PINMUX_GPIO76__FUNC_B_GPIO76 (MTK_PIN_NO(76) | 0)
+#define PINMUX_GPIO76__FUNC_O_SPIM1_CLK (MTK_PIN_NO(76) | 1)
+#define PINMUX_GPIO76__FUNC_O_SCP_SPI1_A_CK (MTK_PIN_NO(76) | 2)
+#define PINMUX_GPIO76__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(76) | 3)
+#define PINMUX_GPIO76__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(76) | 4)
+#define PINMUX_GPIO76__FUNC_O_CMVREF7 (MTK_PIN_NO(76) | 5)
+#define PINMUX_GPIO76__FUNC_O_GDU_SUM_TROOP2_1 (MTK_PIN_NO(76) | 6)
+#define PINMUX_GPIO76__FUNC_B0_DBG_MON_A30 (MTK_PIN_NO(76) | 7)
+
+#define PINMUX_GPIO77__FUNC_B_GPIO77 (MTK_PIN_NO(77) | 0)
+#define PINMUX_GPIO77__FUNC_B0_SPIM1_MOSI (MTK_PIN_NO(77) | 1)
+#define PINMUX_GPIO77__FUNC_O_SCP_SPI1_A_MO (MTK_PIN_NO(77) | 2)
+#define PINMUX_GPIO77__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(77) | 3)
+#define PINMUX_GPIO77__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(77) | 4)
+#define PINMUX_GPIO77__FUNC_O_GDU_SUM_TROOP2_2 (MTK_PIN_NO(77) | 6)
+#define PINMUX_GPIO77__FUNC_B0_DBG_MON_A31 (MTK_PIN_NO(77) | 7)
+
+#define PINMUX_GPIO78__FUNC_B_GPIO78 (MTK_PIN_NO(78) | 0)
+#define PINMUX_GPIO78__FUNC_B0_SPIM1_MISO (MTK_PIN_NO(78) | 1)
+#define PINMUX_GPIO78__FUNC_I0_SCP_SPI1_A_MI (MTK_PIN_NO(78) | 2)
+#define PINMUX_GPIO78__FUNC_I0_TDMIN_DI (MTK_PIN_NO(78) | 3)
+#define PINMUX_GPIO78__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(78) | 4)
+#define PINMUX_GPIO78__FUNC_B0_DBG_MON_A32 (MTK_PIN_NO(78) | 7)
+
+#define PINMUX_GPIO79__FUNC_B_GPIO79 (MTK_PIN_NO(79) | 0)
+#define PINMUX_GPIO79__FUNC_O_SPIM2_CSB (MTK_PIN_NO(79) | 1)
+#define PINMUX_GPIO79__FUNC_O_SCP_SPI2_CS (MTK_PIN_NO(79) | 2)
+#define PINMUX_GPIO79__FUNC_O_I2SO1_MCK (MTK_PIN_NO(79) | 3)
+#define PINMUX_GPIO79__FUNC_O_UTXD2 (MTK_PIN_NO(79) | 4)
+#define PINMUX_GPIO79__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(79) | 5)
+#define PINMUX_GPIO79__FUNC_B0_PCM_SYNC (MTK_PIN_NO(79) | 6)
+#define PINMUX_GPIO79__FUNC_B0_DBG_MON_B0 (MTK_PIN_NO(79) | 7)
+
+#define PINMUX_GPIO80__FUNC_B_GPIO80 (MTK_PIN_NO(80) | 0)
+#define PINMUX_GPIO80__FUNC_O_SPIM2_CLK (MTK_PIN_NO(80) | 1)
+#define PINMUX_GPIO80__FUNC_O_SCP_SPI2_CK (MTK_PIN_NO(80) | 2)
+#define PINMUX_GPIO80__FUNC_O_I2SO1_BCK (MTK_PIN_NO(80) | 3)
+#define PINMUX_GPIO80__FUNC_I1_URXD2 (MTK_PIN_NO(80) | 4)
+#define PINMUX_GPIO80__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(80) | 5)
+#define PINMUX_GPIO80__FUNC_B0_PCM_CLK (MTK_PIN_NO(80) | 6)
+#define PINMUX_GPIO80__FUNC_B0_DBG_MON_B1 (MTK_PIN_NO(80) | 7)
+
+#define PINMUX_GPIO81__FUNC_B_GPIO81 (MTK_PIN_NO(81) | 0)
+#define PINMUX_GPIO81__FUNC_B0_SPIM2_MOSI (MTK_PIN_NO(81) | 1)
+#define PINMUX_GPIO81__FUNC_O_SCP_SPI2_MO (MTK_PIN_NO(81) | 2)
+#define PINMUX_GPIO81__FUNC_O_I2SO1_WS (MTK_PIN_NO(81) | 3)
+#define PINMUX_GPIO81__FUNC_O_URTS2 (MTK_PIN_NO(81) | 4)
+#define PINMUX_GPIO81__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(81) | 5)
+#define PINMUX_GPIO81__FUNC_O_PCM_DO (MTK_PIN_NO(81) | 6)
+#define PINMUX_GPIO81__FUNC_B0_DBG_MON_B2 (MTK_PIN_NO(81) | 7)
+
+#define PINMUX_GPIO82__FUNC_B_GPIO82 (MTK_PIN_NO(82) | 0)
+#define PINMUX_GPIO82__FUNC_B0_SPIM2_MISO (MTK_PIN_NO(82) | 1)
+#define PINMUX_GPIO82__FUNC_I0_SCP_SPI2_MI (MTK_PIN_NO(82) | 2)
+#define PINMUX_GPIO82__FUNC_O_I2SO1_D0 (MTK_PIN_NO(82) | 3)
+#define PINMUX_GPIO82__FUNC_I1_UCTS2 (MTK_PIN_NO(82) | 4)
+#define PINMUX_GPIO82__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(82) | 5)
+#define PINMUX_GPIO82__FUNC_I0_PCM_DI (MTK_PIN_NO(82) | 6)
+#define PINMUX_GPIO82__FUNC_B0_DBG_MON_B3 (MTK_PIN_NO(82) | 7)
+
+#define PINMUX_GPIO83__FUNC_B_GPIO83 (MTK_PIN_NO(83) | 0)
+#define PINMUX_GPIO83__FUNC_I1_IDDIG (MTK_PIN_NO(83) | 1)
+
+#define PINMUX_GPIO84__FUNC_B_GPIO84 (MTK_PIN_NO(84) | 0)
+#define PINMUX_GPIO84__FUNC_O_USB_DRVVBUS (MTK_PIN_NO(84) | 1)
+
+#define PINMUX_GPIO85__FUNC_B_GPIO85 (MTK_PIN_NO(85) | 0)
+#define PINMUX_GPIO85__FUNC_I0_VBUSVALID (MTK_PIN_NO(85) | 1)
+
+#define PINMUX_GPIO86__FUNC_B_GPIO86 (MTK_PIN_NO(86) | 0)
+#define PINMUX_GPIO86__FUNC_I1_IDDIG_1P (MTK_PIN_NO(86) | 1)
+#define PINMUX_GPIO86__FUNC_O_UTXD1 (MTK_PIN_NO(86) | 2)
+#define PINMUX_GPIO86__FUNC_O_URTS2 (MTK_PIN_NO(86) | 3)
+#define PINMUX_GPIO86__FUNC_O_PWM_2 (MTK_PIN_NO(86) | 4)
+#define PINMUX_GPIO86__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(86) | 5)
+#define PINMUX_GPIO86__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(86) | 6)
+#define PINMUX_GPIO86__FUNC_B0_DBG_MON_B4 (MTK_PIN_NO(86) | 7)
+
+#define PINMUX_GPIO87__FUNC_B_GPIO87 (MTK_PIN_NO(87) | 0)
+#define PINMUX_GPIO87__FUNC_O_USB_DRVVBUS_1P (MTK_PIN_NO(87) | 1)
+#define PINMUX_GPIO87__FUNC_I1_URXD1 (MTK_PIN_NO(87) | 2)
+#define PINMUX_GPIO87__FUNC_I1_UCTS2 (MTK_PIN_NO(87) | 3)
+#define PINMUX_GPIO87__FUNC_O_PWM_3 (MTK_PIN_NO(87) | 4)
+#define PINMUX_GPIO87__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(87) | 5)
+#define PINMUX_GPIO87__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(87) | 6)
+#define PINMUX_GPIO87__FUNC_B0_DBG_MON_B5 (MTK_PIN_NO(87) | 7)
+
+#define PINMUX_GPIO88__FUNC_B_GPIO88 (MTK_PIN_NO(88) | 0)
+#define PINMUX_GPIO88__FUNC_I0_VBUSVALID_1P (MTK_PIN_NO(88) | 1)
+#define PINMUX_GPIO88__FUNC_O_UTXD2 (MTK_PIN_NO(88) | 2)
+#define PINMUX_GPIO88__FUNC_O_URTS1 (MTK_PIN_NO(88) | 3)
+#define PINMUX_GPIO88__FUNC_O_CLKM2 (MTK_PIN_NO(88) | 4)
+#define PINMUX_GPIO88__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(88) | 5)
+#define PINMUX_GPIO88__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(88) | 6)
+#define PINMUX_GPIO88__FUNC_B0_DBG_MON_B6 (MTK_PIN_NO(88) | 7)
+
+#define PINMUX_GPIO89__FUNC_B_GPIO89 (MTK_PIN_NO(89) | 0)
+#define PINMUX_GPIO89__FUNC_I1_IDDIG_2P (MTK_PIN_NO(89) | 1)
+#define PINMUX_GPIO89__FUNC_I1_URXD2 (MTK_PIN_NO(89) | 2)
+#define PINMUX_GPIO89__FUNC_I1_UCTS1 (MTK_PIN_NO(89) | 3)
+#define PINMUX_GPIO89__FUNC_O_CLKM3 (MTK_PIN_NO(89) | 4)
+#define PINMUX_GPIO89__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(89) | 5)
+#define PINMUX_GPIO89__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(89) | 6)
+#define PINMUX_GPIO89__FUNC_B0_DBG_MON_B7 (MTK_PIN_NO(89) | 7)
+
+#define PINMUX_GPIO90__FUNC_B_GPIO90 (MTK_PIN_NO(90) | 0)
+#define PINMUX_GPIO90__FUNC_O_USB_DRVVBUS_2P (MTK_PIN_NO(90) | 1)
+#define PINMUX_GPIO90__FUNC_O_UTXD3 (MTK_PIN_NO(90) | 2)
+#define PINMUX_GPIO90__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(90) | 3)
+#define PINMUX_GPIO90__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(90) | 4)
+#define PINMUX_GPIO90__FUNC_O_MD32_0_TXD (MTK_PIN_NO(90) | 5)
+#define PINMUX_GPIO90__FUNC_O_MD32_1_TXD (MTK_PIN_NO(90) | 6)
+#define PINMUX_GPIO90__FUNC_B0_DBG_MON_B8 (MTK_PIN_NO(90) | 7)
+
+#define PINMUX_GPIO91__FUNC_B_GPIO91 (MTK_PIN_NO(91) | 0)
+#define PINMUX_GPIO91__FUNC_I0_VBUSVALID_2P (MTK_PIN_NO(91) | 1)
+#define PINMUX_GPIO91__FUNC_I1_URXD3 (MTK_PIN_NO(91) | 2)
+#define PINMUX_GPIO91__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(91) | 3)
+#define PINMUX_GPIO91__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(91) | 4)
+#define PINMUX_GPIO91__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(91) | 5)
+#define PINMUX_GPIO91__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(91) | 6)
+#define PINMUX_GPIO91__FUNC_B0_DBG_MON_B9 (MTK_PIN_NO(91) | 7)
+
+#define PINMUX_GPIO92__FUNC_B_GPIO92 (MTK_PIN_NO(92) | 0)
+#define PINMUX_GPIO92__FUNC_O_PWRAP_SPI0_CSN (MTK_PIN_NO(92) | 1)
+
+#define PINMUX_GPIO93__FUNC_B_GPIO93 (MTK_PIN_NO(93) | 0)
+#define PINMUX_GPIO93__FUNC_O_PWRAP_SPI0_CK (MTK_PIN_NO(93) | 1)
+
+#define PINMUX_GPIO94__FUNC_B_GPIO94 (MTK_PIN_NO(94) | 0)
+#define PINMUX_GPIO94__FUNC_B0_PWRAP_SPI0_MO (MTK_PIN_NO(94) | 1)
+#define PINMUX_GPIO94__FUNC_B0_PWRAP_SPI0_MI (MTK_PIN_NO(94) | 2)
+
+#define PINMUX_GPIO95__FUNC_B_GPIO95 (MTK_PIN_NO(95) | 0)
+#define PINMUX_GPIO95__FUNC_B0_PWRAP_SPI0_MI (MTK_PIN_NO(95) | 1)
+#define PINMUX_GPIO95__FUNC_B0_PWRAP_SPI0_MO (MTK_PIN_NO(95) | 2)
+
+#define PINMUX_GPIO96__FUNC_B_GPIO96 (MTK_PIN_NO(96) | 0)
+#define PINMUX_GPIO96__FUNC_O_SRCLKENA0 (MTK_PIN_NO(96) | 1)
+
+#define PINMUX_GPIO97__FUNC_B_GPIO97 (MTK_PIN_NO(97) | 0)
+#define PINMUX_GPIO97__FUNC_O_SRCLKENA1 (MTK_PIN_NO(97) | 1)
+
+#define PINMUX_GPIO98__FUNC_B_GPIO98 (MTK_PIN_NO(98) | 0)
+#define PINMUX_GPIO98__FUNC_O_SCP_VREQ_VAO (MTK_PIN_NO(98) | 1)
+#define PINMUX_GPIO98__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(98) | 2)
+
+#define PINMUX_GPIO99__FUNC_B_GPIO99 (MTK_PIN_NO(99) | 0)
+#define PINMUX_GPIO99__FUNC_I0_RTC32K_CK (MTK_PIN_NO(99) | 1)
+
+#define PINMUX_GPIO100__FUNC_B_GPIO100 (MTK_PIN_NO(100) | 0)
+#define PINMUX_GPIO100__FUNC_O_WATCHDOG (MTK_PIN_NO(100) | 1)
+
+#define PINMUX_GPIO101__FUNC_B_GPIO101 (MTK_PIN_NO(101) | 0)
+#define PINMUX_GPIO101__FUNC_O_AUD_CLK_MOSI (MTK_PIN_NO(101) | 1)
+#define PINMUX_GPIO101__FUNC_O_I2SO1_MCK (MTK_PIN_NO(101) | 2)
+#define PINMUX_GPIO101__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(101) | 3)
+
+#define PINMUX_GPIO102__FUNC_B_GPIO102 (MTK_PIN_NO(102) | 0)
+#define PINMUX_GPIO102__FUNC_O_AUD_SYNC_MOSI (MTK_PIN_NO(102) | 1)
+#define PINMUX_GPIO102__FUNC_O_I2SO1_BCK (MTK_PIN_NO(102) | 2)
+#define PINMUX_GPIO102__FUNC_B0_I2SIN_WS (MTK_PIN_NO(102) | 3)
+
+#define PINMUX_GPIO103__FUNC_B_GPIO103 (MTK_PIN_NO(103) | 0)
+#define PINMUX_GPIO103__FUNC_O_AUD_DAT_MOSI0 (MTK_PIN_NO(103) | 1)
+#define PINMUX_GPIO103__FUNC_O_I2SO1_WS (MTK_PIN_NO(103) | 2)
+#define PINMUX_GPIO103__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(103) | 3)
+
+#define PINMUX_GPIO104__FUNC_B_GPIO104 (MTK_PIN_NO(104) | 0)
+#define PINMUX_GPIO104__FUNC_O_AUD_DAT_MOSI1 (MTK_PIN_NO(104) | 1)
+#define PINMUX_GPIO104__FUNC_O_I2SO1_D0 (MTK_PIN_NO(104) | 2)
+#define PINMUX_GPIO104__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(104) | 3)
+
+#define PINMUX_GPIO105__FUNC_B_GPIO105 (MTK_PIN_NO(105) | 0)
+#define PINMUX_GPIO105__FUNC_I0_AUD_DAT_MISO0 (MTK_PIN_NO(105) | 1)
+#define PINMUX_GPIO105__FUNC_I0_VOW_DAT_MISO (MTK_PIN_NO(105) | 2)
+#define PINMUX_GPIO105__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(105) | 3)
+
+#define PINMUX_GPIO106__FUNC_B_GPIO106 (MTK_PIN_NO(106) | 0)
+#define PINMUX_GPIO106__FUNC_I0_AUD_DAT_MISO1 (MTK_PIN_NO(106) | 1)
+#define PINMUX_GPIO106__FUNC_I0_VOW_CLK_MISO (MTK_PIN_NO(106) | 2)
+#define PINMUX_GPIO106__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(106) | 3)
+
+#define PINMUX_GPIO107__FUNC_B_GPIO107 (MTK_PIN_NO(107) | 0)
+#define PINMUX_GPIO107__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(107) | 1)
+#define PINMUX_GPIO107__FUNC_I0_SPLIN_MCK (MTK_PIN_NO(107) | 2)
+#define PINMUX_GPIO107__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(107) | 3)
+#define PINMUX_GPIO107__FUNC_O_CMVREF4 (MTK_PIN_NO(107) | 4)
+#define PINMUX_GPIO107__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(107) | 5)
+#define PINMUX_GPIO107__FUNC_O_PGD_LV_LSC_PWR0 (MTK_PIN_NO(107) | 6)
+
+#define PINMUX_GPIO108__FUNC_B_GPIO108 (MTK_PIN_NO(108) | 0)
+#define PINMUX_GPIO108__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(108) | 1)
+#define PINMUX_GPIO108__FUNC_I0_SPLIN_LRCK (MTK_PIN_NO(108) | 2)
+#define PINMUX_GPIO108__FUNC_O_DMIC4_CLK (MTK_PIN_NO(108) | 3)
+#define PINMUX_GPIO108__FUNC_O_CMVREF5 (MTK_PIN_NO(108) | 4)
+#define PINMUX_GPIO108__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(108) | 5)
+#define PINMUX_GPIO108__FUNC_O_PGD_LV_LSC_PWR1 (MTK_PIN_NO(108) | 6)
+#define PINMUX_GPIO108__FUNC_B0_DBG_MON_B10 (MTK_PIN_NO(108) | 7)
+
+#define PINMUX_GPIO109__FUNC_B_GPIO109 (MTK_PIN_NO(109) | 0)
+#define PINMUX_GPIO109__FUNC_B0_I2SIN_WS (MTK_PIN_NO(109) | 1)
+#define PINMUX_GPIO109__FUNC_I0_SPLIN_BCK (MTK_PIN_NO(109) | 2)
+#define PINMUX_GPIO109__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(109) | 3)
+#define PINMUX_GPIO109__FUNC_O_CMVREF6 (MTK_PIN_NO(109) | 4)
+#define PINMUX_GPIO109__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(109) | 5)
+#define PINMUX_GPIO109__FUNC_O_PGD_LV_LSC_PWR2 (MTK_PIN_NO(109) | 6)
+#define PINMUX_GPIO109__FUNC_B0_DBG_MON_B11 (MTK_PIN_NO(109) | 7)
+
+#define PINMUX_GPIO110__FUNC_B_GPIO110 (MTK_PIN_NO(110) | 0)
+#define PINMUX_GPIO110__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(110) | 1)
+#define PINMUX_GPIO110__FUNC_I0_SPLIN_D0 (MTK_PIN_NO(110) | 2)
+#define PINMUX_GPIO110__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(110) | 3)
+#define PINMUX_GPIO110__FUNC_O_CMVREF7 (MTK_PIN_NO(110) | 4)
+#define PINMUX_GPIO110__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(110) | 5)
+#define PINMUX_GPIO110__FUNC_O_PGD_LV_LSC_PWR3 (MTK_PIN_NO(110) | 6)
+#define PINMUX_GPIO110__FUNC_B0_DBG_MON_B12 (MTK_PIN_NO(110) | 7)
+
+#define PINMUX_GPIO111__FUNC_B_GPIO111 (MTK_PIN_NO(111) | 0)
+#define PINMUX_GPIO111__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(111) | 1)
+#define PINMUX_GPIO111__FUNC_I0_SPLIN_D1 (MTK_PIN_NO(111) | 2)
+#define PINMUX_GPIO111__FUNC_O_DMIC3_CLK (MTK_PIN_NO(111) | 3)
+#define PINMUX_GPIO111__FUNC_O_SPDIF_OUT (MTK_PIN_NO(111) | 4)
+#define PINMUX_GPIO111__FUNC_O_PGD_LV_LSC_PWR4 (MTK_PIN_NO(111) | 6)
+#define PINMUX_GPIO111__FUNC_B0_DBG_MON_B13 (MTK_PIN_NO(111) | 7)
+
+#define PINMUX_GPIO112__FUNC_B_GPIO112 (MTK_PIN_NO(112) | 0)
+#define PINMUX_GPIO112__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(112) | 1)
+#define PINMUX_GPIO112__FUNC_I0_SPLIN_D2 (MTK_PIN_NO(112) | 2)
+#define PINMUX_GPIO112__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(112) | 3)
+#define PINMUX_GPIO112__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(112) | 4)
+#define PINMUX_GPIO112__FUNC_O_I2SO1_WS (MTK_PIN_NO(112) | 5)
+#define PINMUX_GPIO112__FUNC_O_PGD_LV_LSC_PWR5 (MTK_PIN_NO(112) | 6)
+#define PINMUX_GPIO112__FUNC_B0_DBG_MON_B14 (MTK_PIN_NO(112) | 7)
+
+#define PINMUX_GPIO113__FUNC_B_GPIO113 (MTK_PIN_NO(113) | 0)
+#define PINMUX_GPIO113__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(113) | 1)
+#define PINMUX_GPIO113__FUNC_I0_SPLIN_D3 (MTK_PIN_NO(113) | 2)
+#define PINMUX_GPIO113__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(113) | 3)
+#define PINMUX_GPIO113__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(113) | 4)
+#define PINMUX_GPIO113__FUNC_O_I2SO1_D0 (MTK_PIN_NO(113) | 5)
+#define PINMUX_GPIO113__FUNC_B0_DBG_MON_B15 (MTK_PIN_NO(113) | 7)
+
+#define PINMUX_GPIO114__FUNC_B_GPIO114 (MTK_PIN_NO(114) | 0)
+#define PINMUX_GPIO114__FUNC_O_I2SO2_MCK (MTK_PIN_NO(114) | 1)
+#define PINMUX_GPIO114__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(114) | 2)
+#define PINMUX_GPIO114__FUNC_I1_MCUPM_JTAG_TMS (MTK_PIN_NO(114) | 3)
+#define PINMUX_GPIO114__FUNC_B1_APU_JTAG_TMS (MTK_PIN_NO(114) | 4)
+#define PINMUX_GPIO114__FUNC_I1_SCP_JTAG1_TMS (MTK_PIN_NO(114) | 5)
+#define PINMUX_GPIO114__FUNC_I1_SPM_JTAG_TMS (MTK_PIN_NO(114) | 6)
+#define PINMUX_GPIO114__FUNC_B0_DBG_MON_B16 (MTK_PIN_NO(114) | 7)
+
+#define PINMUX_GPIO115__FUNC_B_GPIO115 (MTK_PIN_NO(115) | 0)
+#define PINMUX_GPIO115__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(115) | 1)
+#define PINMUX_GPIO115__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(115) | 2)
+#define PINMUX_GPIO115__FUNC_I1_MCUPM_JTAG_TCK (MTK_PIN_NO(115) | 3)
+#define PINMUX_GPIO115__FUNC_I0_APU_JTAG_TCK (MTK_PIN_NO(115) | 4)
+#define PINMUX_GPIO115__FUNC_I1_SCP_JTAG1_TCK (MTK_PIN_NO(115) | 5)
+#define PINMUX_GPIO115__FUNC_I1_SPM_JTAG_TCK (MTK_PIN_NO(115) | 6)
+#define PINMUX_GPIO115__FUNC_B0_DBG_MON_B17 (MTK_PIN_NO(115) | 7)
+
+#define PINMUX_GPIO116__FUNC_B_GPIO116 (MTK_PIN_NO(116) | 0)
+#define PINMUX_GPIO116__FUNC_B0_I2SO2_WS (MTK_PIN_NO(116) | 1)
+#define PINMUX_GPIO116__FUNC_B0_I2SIN_WS (MTK_PIN_NO(116) | 2)
+#define PINMUX_GPIO116__FUNC_I1_MCUPM_JTAG_TDI (MTK_PIN_NO(116) | 3)
+#define PINMUX_GPIO116__FUNC_I1_APU_JTAG_TDI (MTK_PIN_NO(116) | 4)
+#define PINMUX_GPIO116__FUNC_I1_SCP_JTAG1_TDI (MTK_PIN_NO(116) | 5)
+#define PINMUX_GPIO116__FUNC_I1_SPM_JTAG_TDI (MTK_PIN_NO(116) | 6)
+#define PINMUX_GPIO116__FUNC_B0_DBG_MON_B18 (MTK_PIN_NO(116) | 7)
+
+#define PINMUX_GPIO117__FUNC_B_GPIO117 (MTK_PIN_NO(117) | 0)
+#define PINMUX_GPIO117__FUNC_O_I2SO2_D0 (MTK_PIN_NO(117) | 1)
+#define PINMUX_GPIO117__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(117) | 2)
+#define PINMUX_GPIO117__FUNC_O_MCUPM_JTAG_TDO (MTK_PIN_NO(117) | 3)
+#define PINMUX_GPIO117__FUNC_O_APU_JTAG_TDO (MTK_PIN_NO(117) | 4)
+#define PINMUX_GPIO117__FUNC_O_SCP_JTAG1_TDO (MTK_PIN_NO(117) | 5)
+#define PINMUX_GPIO117__FUNC_O_SPM_JTAG_TDO (MTK_PIN_NO(117) | 6)
+#define PINMUX_GPIO117__FUNC_B0_DBG_MON_B19 (MTK_PIN_NO(117) | 7)
+
+#define PINMUX_GPIO118__FUNC_B_GPIO118 (MTK_PIN_NO(118) | 0)
+#define PINMUX_GPIO118__FUNC_O_I2SO2_D1 (MTK_PIN_NO(118) | 1)
+#define PINMUX_GPIO118__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(118) | 2)
+#define PINMUX_GPIO118__FUNC_I0_MCUPM_JTAG_TRSTN (MTK_PIN_NO(118) | 3)
+#define PINMUX_GPIO118__FUNC_I0_APU_JTAG_TRST (MTK_PIN_NO(118) | 4)
+#define PINMUX_GPIO118__FUNC_I0_SCP_JTAG1_TRSTN (MTK_PIN_NO(118) | 5)
+#define PINMUX_GPIO118__FUNC_I0_SPM_JTAG_TRSTN (MTK_PIN_NO(118) | 6)
+#define PINMUX_GPIO118__FUNC_B0_DBG_MON_B20 (MTK_PIN_NO(118) | 7)
+
+#define PINMUX_GPIO119__FUNC_B_GPIO119 (MTK_PIN_NO(119) | 0)
+#define PINMUX_GPIO119__FUNC_O_I2SO2_D2 (MTK_PIN_NO(119) | 1)
+#define PINMUX_GPIO119__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(119) | 2)
+#define PINMUX_GPIO119__FUNC_O_UTXD3 (MTK_PIN_NO(119) | 3)
+#define PINMUX_GPIO119__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(119) | 4)
+#define PINMUX_GPIO119__FUNC_O_I2SO1_MCK (MTK_PIN_NO(119) | 5)
+#define PINMUX_GPIO119__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(119) | 6)
+#define PINMUX_GPIO119__FUNC_B0_DBG_MON_B21 (MTK_PIN_NO(119) | 7)
+
+#define PINMUX_GPIO120__FUNC_B_GPIO120 (MTK_PIN_NO(120) | 0)
+#define PINMUX_GPIO120__FUNC_O_I2SO2_D3 (MTK_PIN_NO(120) | 1)
+#define PINMUX_GPIO120__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(120) | 2)
+#define PINMUX_GPIO120__FUNC_I1_URXD3 (MTK_PIN_NO(120) | 3)
+#define PINMUX_GPIO120__FUNC_I0_TDMIN_DI (MTK_PIN_NO(120) | 4)
+#define PINMUX_GPIO120__FUNC_O_I2SO1_BCK (MTK_PIN_NO(120) | 5)
+#define PINMUX_GPIO120__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(120) | 6)
+#define PINMUX_GPIO120__FUNC_B0_DBG_MON_B22 (MTK_PIN_NO(120) | 7)
+
+#define PINMUX_GPIO121__FUNC_B_GPIO121 (MTK_PIN_NO(121) | 0)
+#define PINMUX_GPIO121__FUNC_B0_PCM_CLK (MTK_PIN_NO(121) | 1)
+#define PINMUX_GPIO121__FUNC_O_SPIM4_CSB (MTK_PIN_NO(121) | 2)
+#define PINMUX_GPIO121__FUNC_O_SCP_SPI1_B_CS (MTK_PIN_NO(121) | 3)
+#define PINMUX_GPIO121__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(121) | 4)
+#define PINMUX_GPIO121__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(121) | 5)
+#define PINMUX_GPIO121__FUNC_O_PGD_DA_EFUSE_RDY (MTK_PIN_NO(121) | 6)
+#define PINMUX_GPIO121__FUNC_B0_DBG_MON_B23 (MTK_PIN_NO(121) | 7)
+
+#define PINMUX_GPIO122__FUNC_B_GPIO122 (MTK_PIN_NO(122) | 0)
+#define PINMUX_GPIO122__FUNC_B0_PCM_SYNC (MTK_PIN_NO(122) | 1)
+#define PINMUX_GPIO122__FUNC_O_SPIM4_CLK (MTK_PIN_NO(122) | 2)
+#define PINMUX_GPIO122__FUNC_O_SCP_SPI1_B_CK (MTK_PIN_NO(122) | 3)
+#define PINMUX_GPIO122__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(122) | 4)
+#define PINMUX_GPIO122__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(122) | 5)
+#define PINMUX_GPIO122__FUNC_O_PGD_DA_EFUSE_RDY_PRE (MTK_PIN_NO(122) | 6)
+#define PINMUX_GPIO122__FUNC_B0_DBG_MON_B24 (MTK_PIN_NO(122) | 7)
+
+#define PINMUX_GPIO123__FUNC_B_GPIO123 (MTK_PIN_NO(123) | 0)
+#define PINMUX_GPIO123__FUNC_O_PCM_DO (MTK_PIN_NO(123) | 1)
+#define PINMUX_GPIO123__FUNC_B0_SPIM4_MOSI (MTK_PIN_NO(123) | 2)
+#define PINMUX_GPIO123__FUNC_O_SCP_SPI1_B_MO (MTK_PIN_NO(123) | 3)
+#define PINMUX_GPIO123__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(123) | 4)
+#define PINMUX_GPIO123__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(123) | 5)
+#define PINMUX_GPIO123__FUNC_O_PGD_DA_PWRGD_RESET (MTK_PIN_NO(123) | 6)
+#define PINMUX_GPIO123__FUNC_B0_DBG_MON_B25 (MTK_PIN_NO(123) | 7)
+
+#define PINMUX_GPIO124__FUNC_B_GPIO124 (MTK_PIN_NO(124) | 0)
+#define PINMUX_GPIO124__FUNC_I0_PCM_DI (MTK_PIN_NO(124) | 1)
+#define PINMUX_GPIO124__FUNC_B0_SPIM4_MISO (MTK_PIN_NO(124) | 2)
+#define PINMUX_GPIO124__FUNC_I0_SCP_SPI1_B_MI (MTK_PIN_NO(124) | 3)
+#define PINMUX_GPIO124__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(124) | 4)
+#define PINMUX_GPIO124__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(124) | 5)
+#define PINMUX_GPIO124__FUNC_O_PGD_DA_PWRGD_ENB (MTK_PIN_NO(124) | 6)
+#define PINMUX_GPIO124__FUNC_B0_DBG_MON_B26 (MTK_PIN_NO(124) | 7)
+
+#define PINMUX_GPIO125__FUNC_B_GPIO125 (MTK_PIN_NO(125) | 0)
+#define PINMUX_GPIO125__FUNC_O_DMIC1_CLK (MTK_PIN_NO(125) | 1)
+#define PINMUX_GPIO125__FUNC_O_SPINOR_CK (MTK_PIN_NO(125) | 2)
+#define PINMUX_GPIO125__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(125) | 3)
+#define PINMUX_GPIO125__FUNC_O_LVTS_FOUT (MTK_PIN_NO(125) | 6)
+#define PINMUX_GPIO125__FUNC_B0_DBG_MON_B27 (MTK_PIN_NO(125) | 7)
+
+#define PINMUX_GPIO126__FUNC_B_GPIO126 (MTK_PIN_NO(126) | 0)
+#define PINMUX_GPIO126__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(126) | 1)
+#define PINMUX_GPIO126__FUNC_O_SPINOR_CS (MTK_PIN_NO(126) | 2)
+#define PINMUX_GPIO126__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(126) | 3)
+#define PINMUX_GPIO126__FUNC_O_LVTS_SDO (MTK_PIN_NO(126) | 6)
+#define PINMUX_GPIO126__FUNC_B0_DBG_MON_B28 (MTK_PIN_NO(126) | 7)
+
+#define PINMUX_GPIO127__FUNC_B_GPIO127 (MTK_PIN_NO(127) | 0)
+#define PINMUX_GPIO127__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(127) | 1)
+#define PINMUX_GPIO127__FUNC_B0_SPINOR_IO0 (MTK_PIN_NO(127) | 2)
+#define PINMUX_GPIO127__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(127) | 3)
+#define PINMUX_GPIO127__FUNC_I0_LVTS_26M (MTK_PIN_NO(127) | 6)
+#define PINMUX_GPIO127__FUNC_B0_DBG_MON_B29 (MTK_PIN_NO(127) | 7)
+
+#define PINMUX_GPIO128__FUNC_B_GPIO128 (MTK_PIN_NO(128) | 0)
+#define PINMUX_GPIO128__FUNC_O_DMIC2_CLK (MTK_PIN_NO(128) | 1)
+#define PINMUX_GPIO128__FUNC_B0_SPINOR_IO1 (MTK_PIN_NO(128) | 2)
+#define PINMUX_GPIO128__FUNC_I0_TDMIN_DI (MTK_PIN_NO(128) | 3)
+#define PINMUX_GPIO128__FUNC_I0_LVTS_SCF (MTK_PIN_NO(128) | 6)
+#define PINMUX_GPIO128__FUNC_B0_DBG_MON_B30 (MTK_PIN_NO(128) | 7)
+
+#define PINMUX_GPIO129__FUNC_B_GPIO129 (MTK_PIN_NO(129) | 0)
+#define PINMUX_GPIO129__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(129) | 1)
+#define PINMUX_GPIO129__FUNC_B0_SPINOR_IO2 (MTK_PIN_NO(129) | 2)
+#define PINMUX_GPIO129__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(129) | 3)
+#define PINMUX_GPIO129__FUNC_I0_LVTS_SCK (MTK_PIN_NO(129) | 6)
+#define PINMUX_GPIO129__FUNC_B0_DBG_MON_B31 (MTK_PIN_NO(129) | 7)
+
+#define PINMUX_GPIO130__FUNC_B_GPIO130 (MTK_PIN_NO(130) | 0)
+#define PINMUX_GPIO130__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(130) | 1)
+#define PINMUX_GPIO130__FUNC_B0_SPINOR_IO3 (MTK_PIN_NO(130) | 2)
+#define PINMUX_GPIO130__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(130) | 3)
+#define PINMUX_GPIO130__FUNC_I0_LVTS_SDI (MTK_PIN_NO(130) | 6)
+#define PINMUX_GPIO130__FUNC_B0_DBG_MON_B32 (MTK_PIN_NO(130) | 7)
+
+#define PINMUX_GPIO131__FUNC_B_GPIO131 (MTK_PIN_NO(131) | 0)
+#define PINMUX_GPIO131__FUNC_O_DPI_D0 (MTK_PIN_NO(131) | 1)
+#define PINMUX_GPIO131__FUNC_O_GBE_TXD3 (MTK_PIN_NO(131) | 2)
+#define PINMUX_GPIO131__FUNC_O_DMIC1_CLK (MTK_PIN_NO(131) | 3)
+#define PINMUX_GPIO131__FUNC_O_I2SO2_MCK (MTK_PIN_NO(131) | 4)
+#define PINMUX_GPIO131__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(131) | 5)
+#define PINMUX_GPIO131__FUNC_O_SPIM5_CSB (MTK_PIN_NO(131) | 6)
+#define PINMUX_GPIO131__FUNC_O_PGD_LV_HSC_PWR0 (MTK_PIN_NO(131) | 7)
+
+#define PINMUX_GPIO132__FUNC_B_GPIO132 (MTK_PIN_NO(132) | 0)
+#define PINMUX_GPIO132__FUNC_O_DPI_D1 (MTK_PIN_NO(132) | 1)
+#define PINMUX_GPIO132__FUNC_O_GBE_TXD2 (MTK_PIN_NO(132) | 2)
+#define PINMUX_GPIO132__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(132) | 3)
+#define PINMUX_GPIO132__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(132) | 4)
+#define PINMUX_GPIO132__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(132) | 5)
+#define PINMUX_GPIO132__FUNC_O_SPIM5_CLK (MTK_PIN_NO(132) | 6)
+#define PINMUX_GPIO132__FUNC_O_PGD_LV_HSC_PWR1 (MTK_PIN_NO(132) | 7)
+
+#define PINMUX_GPIO133__FUNC_B_GPIO133 (MTK_PIN_NO(133) | 0)
+#define PINMUX_GPIO133__FUNC_O_DPI_D2 (MTK_PIN_NO(133) | 1)
+#define PINMUX_GPIO133__FUNC_O_GBE_TXD1 (MTK_PIN_NO(133) | 2)
+#define PINMUX_GPIO133__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(133) | 3)
+#define PINMUX_GPIO133__FUNC_B0_I2SO2_WS (MTK_PIN_NO(133) | 4)
+#define PINMUX_GPIO133__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(133) | 5)
+#define PINMUX_GPIO133__FUNC_B0_SPIM5_MOSI (MTK_PIN_NO(133) | 6)
+#define PINMUX_GPIO133__FUNC_O_PGD_LV_HSC_PWR2 (MTK_PIN_NO(133) | 7)
+
+#define PINMUX_GPIO134__FUNC_B_GPIO134 (MTK_PIN_NO(134) | 0)
+#define PINMUX_GPIO134__FUNC_O_DPI_D3 (MTK_PIN_NO(134) | 1)
+#define PINMUX_GPIO134__FUNC_O_GBE_TXD0 (MTK_PIN_NO(134) | 2)
+#define PINMUX_GPIO134__FUNC_O_DMIC2_CLK (MTK_PIN_NO(134) | 3)
+#define PINMUX_GPIO134__FUNC_O_I2SO2_D0 (MTK_PIN_NO(134) | 4)
+#define PINMUX_GPIO134__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(134) | 5)
+#define PINMUX_GPIO134__FUNC_B0_SPIM5_MISO (MTK_PIN_NO(134) | 6)
+#define PINMUX_GPIO134__FUNC_O_PGD_LV_HSC_PWR3 (MTK_PIN_NO(134) | 7)
+
+#define PINMUX_GPIO135__FUNC_B_GPIO135 (MTK_PIN_NO(135) | 0)
+#define PINMUX_GPIO135__FUNC_O_DPI_D4 (MTK_PIN_NO(135) | 1)
+#define PINMUX_GPIO135__FUNC_I0_GBE_RXD3 (MTK_PIN_NO(135) | 2)
+#define PINMUX_GPIO135__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(135) | 3)
+#define PINMUX_GPIO135__FUNC_O_I2SO2_D1 (MTK_PIN_NO(135) | 4)
+#define PINMUX_GPIO135__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(135) | 5)
+#define PINMUX_GPIO135__FUNC_I1_WAKEN (MTK_PIN_NO(135) | 6)
+#define PINMUX_GPIO135__FUNC_O_PGD_LV_HSC_PWR4 (MTK_PIN_NO(135) | 7)
+
+#define PINMUX_GPIO136__FUNC_B_GPIO136 (MTK_PIN_NO(136) | 0)
+#define PINMUX_GPIO136__FUNC_O_DPI_D5 (MTK_PIN_NO(136) | 1)
+#define PINMUX_GPIO136__FUNC_I0_GBE_RXD2 (MTK_PIN_NO(136) | 2)
+#define PINMUX_GPIO136__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(136) | 3)
+#define PINMUX_GPIO136__FUNC_O_I2SO2_D2 (MTK_PIN_NO(136) | 4)
+#define PINMUX_GPIO136__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(136) | 5)
+#define PINMUX_GPIO136__FUNC_O_PERSTN (MTK_PIN_NO(136) | 6)
+#define PINMUX_GPIO136__FUNC_O_PGD_LV_HSC_PWR5 (MTK_PIN_NO(136) | 7)
+
+#define PINMUX_GPIO137__FUNC_B_GPIO137 (MTK_PIN_NO(137) | 0)
+#define PINMUX_GPIO137__FUNC_O_DPI_D6 (MTK_PIN_NO(137) | 1)
+#define PINMUX_GPIO137__FUNC_I0_GBE_RXD1 (MTK_PIN_NO(137) | 2)
+#define PINMUX_GPIO137__FUNC_O_DMIC3_CLK (MTK_PIN_NO(137) | 3)
+#define PINMUX_GPIO137__FUNC_O_I2SO2_D3 (MTK_PIN_NO(137) | 4)
+#define PINMUX_GPIO137__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(137) | 5)
+#define PINMUX_GPIO137__FUNC_B1_CLKREQN (MTK_PIN_NO(137) | 6)
+#define PINMUX_GPIO137__FUNC_O_PWM_0 (MTK_PIN_NO(137) | 7)
+
+#define PINMUX_GPIO138__FUNC_B_GPIO138 (MTK_PIN_NO(138) | 0)
+#define PINMUX_GPIO138__FUNC_O_DPI_D7 (MTK_PIN_NO(138) | 1)
+#define PINMUX_GPIO138__FUNC_I0_GBE_RXD0 (MTK_PIN_NO(138) | 2)
+#define PINMUX_GPIO138__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(138) | 3)
+#define PINMUX_GPIO138__FUNC_O_CLKM2 (MTK_PIN_NO(138) | 4)
+#define PINMUX_GPIO138__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(138) | 5)
+#define PINMUX_GPIO138__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(138) | 7)
+
+#define PINMUX_GPIO139__FUNC_B_GPIO139 (MTK_PIN_NO(139) | 0)
+#define PINMUX_GPIO139__FUNC_O_DPI_D8 (MTK_PIN_NO(139) | 1)
+#define PINMUX_GPIO139__FUNC_B0_GBE_TXC (MTK_PIN_NO(139) | 2)
+#define PINMUX_GPIO139__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(139) | 3)
+#define PINMUX_GPIO139__FUNC_O_CLKM3 (MTK_PIN_NO(139) | 4)
+#define PINMUX_GPIO139__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(139) | 5)
+#define PINMUX_GPIO139__FUNC_O_UTXD2 (MTK_PIN_NO(139) | 6)
+#define PINMUX_GPIO139__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(139) | 7)
+
+#define PINMUX_GPIO140__FUNC_B_GPIO140 (MTK_PIN_NO(140) | 0)
+#define PINMUX_GPIO140__FUNC_O_DPI_D9 (MTK_PIN_NO(140) | 1)
+#define PINMUX_GPIO140__FUNC_I0_GBE_RXC (MTK_PIN_NO(140) | 2)
+#define PINMUX_GPIO140__FUNC_O_DMIC4_CLK (MTK_PIN_NO(140) | 3)
+#define PINMUX_GPIO140__FUNC_O_PWM_2 (MTK_PIN_NO(140) | 4)
+#define PINMUX_GPIO140__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(140) | 5)
+#define PINMUX_GPIO140__FUNC_I1_URXD2 (MTK_PIN_NO(140) | 6)
+#define PINMUX_GPIO140__FUNC_B0_MD32_0_GPIO2 (MTK_PIN_NO(140) | 7)
+
+#define PINMUX_GPIO141__FUNC_B_GPIO141 (MTK_PIN_NO(141) | 0)
+#define PINMUX_GPIO141__FUNC_O_DPI_D10 (MTK_PIN_NO(141) | 1)
+#define PINMUX_GPIO141__FUNC_I0_GBE_RXDV (MTK_PIN_NO(141) | 2)
+#define PINMUX_GPIO141__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(141) | 3)
+#define PINMUX_GPIO141__FUNC_O_PWM_3 (MTK_PIN_NO(141) | 4)
+#define PINMUX_GPIO141__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(141) | 5)
+#define PINMUX_GPIO141__FUNC_O_URTS2 (MTK_PIN_NO(141) | 6)
+#define PINMUX_GPIO141__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(141) | 7)
+
+#define PINMUX_GPIO142__FUNC_B_GPIO142 (MTK_PIN_NO(142) | 0)
+#define PINMUX_GPIO142__FUNC_O_DPI_D11 (MTK_PIN_NO(142) | 1)
+#define PINMUX_GPIO142__FUNC_O_GBE_TXEN (MTK_PIN_NO(142) | 2)
+#define PINMUX_GPIO142__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(142) | 3)
+#define PINMUX_GPIO142__FUNC_O_PWM_1 (MTK_PIN_NO(142) | 4)
+#define PINMUX_GPIO142__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(142) | 5)
+#define PINMUX_GPIO142__FUNC_I1_UCTS2 (MTK_PIN_NO(142) | 6)
+#define PINMUX_GPIO142__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(142) | 7)
+
+#define PINMUX_GPIO143__FUNC_B_GPIO143 (MTK_PIN_NO(143) | 0)
+#define PINMUX_GPIO143__FUNC_O_DPI_D12 (MTK_PIN_NO(143) | 1)
+#define PINMUX_GPIO143__FUNC_O_GBE_MDC (MTK_PIN_NO(143) | 2)
+#define PINMUX_GPIO143__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(143) | 3)
+#define PINMUX_GPIO143__FUNC_O_CLKM0 (MTK_PIN_NO(143) | 4)
+#define PINMUX_GPIO143__FUNC_O_SPIM3_CSB (MTK_PIN_NO(143) | 5)
+#define PINMUX_GPIO143__FUNC_O_UTXD1 (MTK_PIN_NO(143) | 6)
+#define PINMUX_GPIO143__FUNC_B0_MD32_1_GPIO2 (MTK_PIN_NO(143) | 7)
+
+#define PINMUX_GPIO144__FUNC_B_GPIO144 (MTK_PIN_NO(144) | 0)
+#define PINMUX_GPIO144__FUNC_O_DPI_D13 (MTK_PIN_NO(144) | 1)
+#define PINMUX_GPIO144__FUNC_B1_GBE_MDIO (MTK_PIN_NO(144) | 2)
+#define PINMUX_GPIO144__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(144) | 3)
+#define PINMUX_GPIO144__FUNC_O_CLKM1 (MTK_PIN_NO(144) | 4)
+#define PINMUX_GPIO144__FUNC_O_SPIM3_CLK (MTK_PIN_NO(144) | 5)
+#define PINMUX_GPIO144__FUNC_I1_URXD1 (MTK_PIN_NO(144) | 6)
+#define PINMUX_GPIO144__FUNC_O_PGD_HV_HSC_PWR0 (MTK_PIN_NO(144) | 7)
+
+#define PINMUX_GPIO145__FUNC_B_GPIO145 (MTK_PIN_NO(145) | 0)
+#define PINMUX_GPIO145__FUNC_O_DPI_D14 (MTK_PIN_NO(145) | 1)
+#define PINMUX_GPIO145__FUNC_O_GBE_TXER (MTK_PIN_NO(145) | 2)
+#define PINMUX_GPIO145__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(145) | 3)
+#define PINMUX_GPIO145__FUNC_O_CMFLASH0 (MTK_PIN_NO(145) | 4)
+#define PINMUX_GPIO145__FUNC_B0_SPIM3_MOSI (MTK_PIN_NO(145) | 5)
+#define PINMUX_GPIO145__FUNC_B0_GBE_AUX_PPS2 (MTK_PIN_NO(145) | 6)
+#define PINMUX_GPIO145__FUNC_O_PGD_HV_HSC_PWR1 (MTK_PIN_NO(145) | 7)
+
+#define PINMUX_GPIO146__FUNC_B_GPIO146 (MTK_PIN_NO(146) | 0)
+#define PINMUX_GPIO146__FUNC_O_DPI_D15 (MTK_PIN_NO(146) | 1)
+#define PINMUX_GPIO146__FUNC_I0_GBE_RXER (MTK_PIN_NO(146) | 2)
+#define PINMUX_GPIO146__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(146) | 3)
+#define PINMUX_GPIO146__FUNC_O_CMFLASH1 (MTK_PIN_NO(146) | 4)
+#define PINMUX_GPIO146__FUNC_B0_SPIM3_MISO (MTK_PIN_NO(146) | 5)
+#define PINMUX_GPIO146__FUNC_B0_GBE_AUX_PPS3 (MTK_PIN_NO(146) | 6)
+#define PINMUX_GPIO146__FUNC_O_PGD_HV_HSC_PWR2 (MTK_PIN_NO(146) | 7)
+
+#define PINMUX_GPIO147__FUNC_B_GPIO147 (MTK_PIN_NO(147) | 0)
+#define PINMUX_GPIO147__FUNC_O_DPI_HSYNC (MTK_PIN_NO(147) | 1)
+#define PINMUX_GPIO147__FUNC_I0_GBE_COL (MTK_PIN_NO(147) | 2)
+#define PINMUX_GPIO147__FUNC_O_I2SO1_MCK (MTK_PIN_NO(147) | 3)
+#define PINMUX_GPIO147__FUNC_O_CMVREF0 (MTK_PIN_NO(147) | 4)
+#define PINMUX_GPIO147__FUNC_O_SPDIF_OUT (MTK_PIN_NO(147) | 5)
+#define PINMUX_GPIO147__FUNC_O_URTS1 (MTK_PIN_NO(147) | 6)
+#define PINMUX_GPIO147__FUNC_O_PGD_HV_HSC_PWR3 (MTK_PIN_NO(147) | 7)
+
+#define PINMUX_GPIO148__FUNC_B_GPIO148 (MTK_PIN_NO(148) | 0)
+#define PINMUX_GPIO148__FUNC_O_DPI_VSYNC (MTK_PIN_NO(148) | 1)
+#define PINMUX_GPIO148__FUNC_I0_GBE_INTR (MTK_PIN_NO(148) | 2)
+#define PINMUX_GPIO148__FUNC_O_I2SO1_BCK (MTK_PIN_NO(148) | 3)
+#define PINMUX_GPIO148__FUNC_O_CMVREF1 (MTK_PIN_NO(148) | 4)
+#define PINMUX_GPIO148__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(148) | 5)
+#define PINMUX_GPIO148__FUNC_I1_UCTS1 (MTK_PIN_NO(148) | 6)
+#define PINMUX_GPIO148__FUNC_O_PGD_HV_HSC_PWR4 (MTK_PIN_NO(148) | 7)
+
+#define PINMUX_GPIO149__FUNC_B_GPIO149 (MTK_PIN_NO(149) | 0)
+#define PINMUX_GPIO149__FUNC_O_DPI_DE (MTK_PIN_NO(149) | 1)
+#define PINMUX_GPIO149__FUNC_B0_GBE_AUX_PPS0 (MTK_PIN_NO(149) | 2)
+#define PINMUX_GPIO149__FUNC_O_I2SO1_WS (MTK_PIN_NO(149) | 3)
+#define PINMUX_GPIO149__FUNC_O_CMVREF2 (MTK_PIN_NO(149) | 4)
+#define PINMUX_GPIO149__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(149) | 5)
+#define PINMUX_GPIO149__FUNC_O_UTXD3 (MTK_PIN_NO(149) | 6)
+#define PINMUX_GPIO149__FUNC_O_PGD_HV_HSC_PWR5 (MTK_PIN_NO(149) | 7)
+
+#define PINMUX_GPIO150__FUNC_B_GPIO150 (MTK_PIN_NO(150) | 0)
+#define PINMUX_GPIO150__FUNC_O_DPI_CK (MTK_PIN_NO(150) | 1)
+#define PINMUX_GPIO150__FUNC_B0_GBE_AUX_PPS1 (MTK_PIN_NO(150) | 2)
+#define PINMUX_GPIO150__FUNC_O_I2SO1_D0 (MTK_PIN_NO(150) | 3)
+#define PINMUX_GPIO150__FUNC_O_CMVREF3 (MTK_PIN_NO(150) | 4)
+#define PINMUX_GPIO150__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(150) | 5)
+#define PINMUX_GPIO150__FUNC_I1_URXD3 (MTK_PIN_NO(150) | 6)
+
+#define PINMUX_GPIO151__FUNC_B_GPIO151 (MTK_PIN_NO(151) | 0)
+#define PINMUX_GPIO151__FUNC_B1_MSDC0_DAT7 (MTK_PIN_NO(151) | 1)
+
+#define PINMUX_GPIO152__FUNC_B_GPIO152 (MTK_PIN_NO(152) | 0)
+#define PINMUX_GPIO152__FUNC_B1_MSDC0_DAT6 (MTK_PIN_NO(152) | 1)
+
+#define PINMUX_GPIO153__FUNC_B_GPIO153 (MTK_PIN_NO(153) | 0)
+#define PINMUX_GPIO153__FUNC_B1_MSDC0_DAT5 (MTK_PIN_NO(153) | 1)
+
+#define PINMUX_GPIO154__FUNC_B_GPIO154 (MTK_PIN_NO(154) | 0)
+#define PINMUX_GPIO154__FUNC_B1_MSDC0_DAT4 (MTK_PIN_NO(154) | 1)
+
+#define PINMUX_GPIO155__FUNC_B_GPIO155 (MTK_PIN_NO(155) | 0)
+#define PINMUX_GPIO155__FUNC_O_MSDC0_RSTB (MTK_PIN_NO(155) | 1)
+
+#define PINMUX_GPIO156__FUNC_B_GPIO156 (MTK_PIN_NO(156) | 0)
+#define PINMUX_GPIO156__FUNC_B1_MSDC0_CMD (MTK_PIN_NO(156) | 1)
+
+#define PINMUX_GPIO157__FUNC_B_GPIO157 (MTK_PIN_NO(157) | 0)
+#define PINMUX_GPIO157__FUNC_B1_MSDC0_CLK (MTK_PIN_NO(157) | 1)
+
+#define PINMUX_GPIO158__FUNC_B_GPIO158 (MTK_PIN_NO(158) | 0)
+#define PINMUX_GPIO158__FUNC_B1_MSDC0_DAT3 (MTK_PIN_NO(158) | 1)
+
+#define PINMUX_GPIO159__FUNC_B_GPIO159 (MTK_PIN_NO(159) | 0)
+#define PINMUX_GPIO159__FUNC_B1_MSDC0_DAT2 (MTK_PIN_NO(159) | 1)
+
+#define PINMUX_GPIO160__FUNC_B_GPIO160 (MTK_PIN_NO(160) | 0)
+#define PINMUX_GPIO160__FUNC_B1_MSDC0_DAT1 (MTK_PIN_NO(160) | 1)
+
+#define PINMUX_GPIO161__FUNC_B_GPIO161 (MTK_PIN_NO(161) | 0)
+#define PINMUX_GPIO161__FUNC_B1_MSDC0_DAT0 (MTK_PIN_NO(161) | 1)
+
+#define PINMUX_GPIO162__FUNC_B_GPIO162 (MTK_PIN_NO(162) | 0)
+#define PINMUX_GPIO162__FUNC_B0_MSDC0_DSL (MTK_PIN_NO(162) | 1)
+
+#define PINMUX_GPIO163__FUNC_B_GPIO163 (MTK_PIN_NO(163) | 0)
+#define PINMUX_GPIO163__FUNC_B1_MSDC1_CMD (MTK_PIN_NO(163) | 1)
+#define PINMUX_GPIO163__FUNC_O_SPDIF_OUT (MTK_PIN_NO(163) | 2)
+#define PINMUX_GPIO163__FUNC_I1_MD32_0_JTAG_TMS (MTK_PIN_NO(163) | 3)
+#define PINMUX_GPIO163__FUNC_I1_ADSP_JTAG0_TMS (MTK_PIN_NO(163) | 4)
+#define PINMUX_GPIO163__FUNC_I1_SCP_JTAG0_TMS (MTK_PIN_NO(163) | 5)
+#define PINMUX_GPIO163__FUNC_I1_CCU0_JTAG_TMS (MTK_PIN_NO(163) | 6)
+#define PINMUX_GPIO163__FUNC_I0_IPU_JTAG_TMS (MTK_PIN_NO(163) | 7)
+
+#define PINMUX_GPIO164__FUNC_B_GPIO164 (MTK_PIN_NO(164) | 0)
+#define PINMUX_GPIO164__FUNC_B1_MSDC1_CLK (MTK_PIN_NO(164) | 1)
+#define PINMUX_GPIO164__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(164) | 2)
+#define PINMUX_GPIO164__FUNC_I1_MD32_0_JTAG_TCK (MTK_PIN_NO(164) | 3)
+#define PINMUX_GPIO164__FUNC_I0_ADSP_JTAG0_TCK (MTK_PIN_NO(164) | 4)
+#define PINMUX_GPIO164__FUNC_I1_SCP_JTAG0_TCK (MTK_PIN_NO(164) | 5)
+#define PINMUX_GPIO164__FUNC_I1_CCU0_JTAG_TCK (MTK_PIN_NO(164) | 6)
+#define PINMUX_GPIO164__FUNC_I0_IPU_JTAG_TCK (MTK_PIN_NO(164) | 7)
+
+#define PINMUX_GPIO165__FUNC_B_GPIO165 (MTK_PIN_NO(165) | 0)
+#define PINMUX_GPIO165__FUNC_B1_MSDC1_DAT0 (MTK_PIN_NO(165) | 1)
+#define PINMUX_GPIO165__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(165) | 2)
+#define PINMUX_GPIO165__FUNC_I1_MD32_0_JTAG_TDI (MTK_PIN_NO(165) | 3)
+#define PINMUX_GPIO165__FUNC_I1_ADSP_JTAG0_TDI (MTK_PIN_NO(165) | 4)
+#define PINMUX_GPIO165__FUNC_I1_SCP_JTAG0_TDI (MTK_PIN_NO(165) | 5)
+#define PINMUX_GPIO165__FUNC_I1_CCU0_JTAG_TDI (MTK_PIN_NO(165) | 6)
+#define PINMUX_GPIO165__FUNC_I0_IPU_JTAG_TDI (MTK_PIN_NO(165) | 7)
+
+#define PINMUX_GPIO166__FUNC_B_GPIO166 (MTK_PIN_NO(166) | 0)
+#define PINMUX_GPIO166__FUNC_B1_MSDC1_DAT1 (MTK_PIN_NO(166) | 1)
+#define PINMUX_GPIO166__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(166) | 2)
+#define PINMUX_GPIO166__FUNC_O_MD32_0_JTAG_TDO (MTK_PIN_NO(166) | 3)
+#define PINMUX_GPIO166__FUNC_O_ADSP_JTAG0_TDO (MTK_PIN_NO(166) | 4)
+#define PINMUX_GPIO166__FUNC_O_SCP_JTAG0_TDO (MTK_PIN_NO(166) | 5)
+#define PINMUX_GPIO166__FUNC_O_CCU0_JTAG_TDO (MTK_PIN_NO(166) | 6)
+#define PINMUX_GPIO166__FUNC_O_IPU_JTAG_TDO (MTK_PIN_NO(166) | 7)
+
+#define PINMUX_GPIO167__FUNC_B_GPIO167 (MTK_PIN_NO(167) | 0)
+#define PINMUX_GPIO167__FUNC_B1_MSDC1_DAT2 (MTK_PIN_NO(167) | 1)
+#define PINMUX_GPIO167__FUNC_O_PWM_0 (MTK_PIN_NO(167) | 2)
+#define PINMUX_GPIO167__FUNC_I1_MD32_0_JTAG_TRST (MTK_PIN_NO(167) | 3)
+#define PINMUX_GPIO167__FUNC_I1_ADSP_JTAG0_TRSTN (MTK_PIN_NO(167) | 4)
+#define PINMUX_GPIO167__FUNC_I0_SCP_JTAG0_TRSTN (MTK_PIN_NO(167) | 5)
+#define PINMUX_GPIO167__FUNC_I1_CCU0_JTAG_TRST (MTK_PIN_NO(167) | 6)
+#define PINMUX_GPIO167__FUNC_I0_IPU_JTAG_TRST (MTK_PIN_NO(167) | 7)
+
+#define PINMUX_GPIO168__FUNC_B_GPIO168 (MTK_PIN_NO(168) | 0)
+#define PINMUX_GPIO168__FUNC_B1_MSDC1_DAT3 (MTK_PIN_NO(168) | 1)
+#define PINMUX_GPIO168__FUNC_O_PWM_1 (MTK_PIN_NO(168) | 2)
+#define PINMUX_GPIO168__FUNC_O_CLKM0 (MTK_PIN_NO(168) | 3)
+
+#define PINMUX_GPIO169__FUNC_B_GPIO169 (MTK_PIN_NO(169) | 0)
+#define PINMUX_GPIO169__FUNC_B1_MSDC2_CMD (MTK_PIN_NO(169) | 1)
+#define PINMUX_GPIO169__FUNC_O_LVTS_FOUT (MTK_PIN_NO(169) | 2)
+#define PINMUX_GPIO169__FUNC_I1_MD32_1_JTAG_TMS (MTK_PIN_NO(169) | 3)
+#define PINMUX_GPIO169__FUNC_I0_UDI_TMS (MTK_PIN_NO(169) | 4)
+#define PINMUX_GPIO169__FUNC_I0_VPU_UDI_TMS (MTK_PIN_NO(169) | 5)
+#define PINMUX_GPIO169__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(169) | 6)
+#define PINMUX_GPIO169__FUNC_I1_SSPM_JTAG_TMS (MTK_PIN_NO(169) | 7)
+
+#define PINMUX_GPIO170__FUNC_B_GPIO170 (MTK_PIN_NO(170) | 0)
+#define PINMUX_GPIO170__FUNC_B1_MSDC2_CLK (MTK_PIN_NO(170) | 1)
+#define PINMUX_GPIO170__FUNC_O_LVTS_SDO (MTK_PIN_NO(170) | 2)
+#define PINMUX_GPIO170__FUNC_I1_MD32_1_JTAG_TCK (MTK_PIN_NO(170) | 3)
+#define PINMUX_GPIO170__FUNC_I0_UDI_TCK (MTK_PIN_NO(170) | 4)
+#define PINMUX_GPIO170__FUNC_I0_VPU_UDI_TCK (MTK_PIN_NO(170) | 5)
+#define PINMUX_GPIO170__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(170) | 6)
+#define PINMUX_GPIO170__FUNC_I1_SSPM_JTAG_TCK (MTK_PIN_NO(170) | 7)
+
+#define PINMUX_GPIO171__FUNC_B_GPIO171 (MTK_PIN_NO(171) | 0)
+#define PINMUX_GPIO171__FUNC_B1_MSDC2_DAT0 (MTK_PIN_NO(171) | 1)
+#define PINMUX_GPIO171__FUNC_I0_LVTS_26M (MTK_PIN_NO(171) | 2)
+#define PINMUX_GPIO171__FUNC_I1_MD32_1_JTAG_TDI (MTK_PIN_NO(171) | 3)
+#define PINMUX_GPIO171__FUNC_I0_UDI_TDI (MTK_PIN_NO(171) | 4)
+#define PINMUX_GPIO171__FUNC_I0_VPU_UDI_TDI (MTK_PIN_NO(171) | 5)
+#define PINMUX_GPIO171__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(171) | 6)
+#define PINMUX_GPIO171__FUNC_I1_SSPM_JTAG_TDI (MTK_PIN_NO(171) | 7)
+
+#define PINMUX_GPIO172__FUNC_B_GPIO172 (MTK_PIN_NO(172) | 0)
+#define PINMUX_GPIO172__FUNC_B1_MSDC2_DAT1 (MTK_PIN_NO(172) | 1)
+#define PINMUX_GPIO172__FUNC_I0_LVTS_SCF (MTK_PIN_NO(172) | 2)
+#define PINMUX_GPIO172__FUNC_O_MD32_1_JTAG_TDO (MTK_PIN_NO(172) | 3)
+#define PINMUX_GPIO172__FUNC_O_UDI_TDO (MTK_PIN_NO(172) | 4)
+#define PINMUX_GPIO172__FUNC_O_VPU_UDI_TDO (MTK_PIN_NO(172) | 5)
+#define PINMUX_GPIO172__FUNC_I0_TDMIN_DI (MTK_PIN_NO(172) | 6)
+#define PINMUX_GPIO172__FUNC_O_SSPM_JTAG_TDO (MTK_PIN_NO(172) | 7)
+
+#define PINMUX_GPIO173__FUNC_B_GPIO173 (MTK_PIN_NO(173) | 0)
+#define PINMUX_GPIO173__FUNC_B1_MSDC2_DAT2 (MTK_PIN_NO(173) | 1)
+#define PINMUX_GPIO173__FUNC_I0_LVTS_SCK (MTK_PIN_NO(173) | 2)
+#define PINMUX_GPIO173__FUNC_I1_MD32_1_JTAG_TRST (MTK_PIN_NO(173) | 3)
+#define PINMUX_GPIO173__FUNC_I0_UDI_NTRST (MTK_PIN_NO(173) | 4)
+#define PINMUX_GPIO173__FUNC_I0_VPU_UDI_NTRST (MTK_PIN_NO(173) | 5)
+#define PINMUX_GPIO173__FUNC_I0_SSPM_JTAG_TRSTN (MTK_PIN_NO(173) | 7)
+
+#define PINMUX_GPIO174__FUNC_B_GPIO174 (MTK_PIN_NO(174) | 0)
+#define PINMUX_GPIO174__FUNC_B1_MSDC2_DAT3 (MTK_PIN_NO(174) | 1)
+#define PINMUX_GPIO174__FUNC_I0_LVTS_SDI (MTK_PIN_NO(174) | 2)
+
+#define PINMUX_GPIO175__FUNC_B_GPIO175 (MTK_PIN_NO(175) | 0)
+#define PINMUX_GPIO175__FUNC_B0_SPMI_M_SCL (MTK_PIN_NO(175) | 1)
+
+#define PINMUX_GPIO176__FUNC_B_GPIO176 (MTK_PIN_NO(176) | 0)
+#define PINMUX_GPIO176__FUNC_B0_SPMI_M_SDA (MTK_PIN_NO(176) | 1)
+
+#endif /* __MEDIATEK_MT8188-PINFUNC_H */
* Copyright (C) 2021 Emil Renner Berthing <kernel@esmil.dk>
*/
-#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_H__
-#define __DT_BINDINGS_PINCTRL_STARFIVE_H__
+#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__
+#define __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__
#define PAD_GPIO_OFFSET 0
#define PAD_FUNC_SHARE_OFFSET 64
#define GPI_NONE 0xff
-#endif /* __DT_BINDINGS_PINCTRL_STARFIVE_H__ */
+#endif /* __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__ */
#ifndef __DT_BINDINGS_PINCTRL_SAMSUNG_H__
#define __DT_BINDINGS_PINCTRL_SAMSUNG_H__
+/*
+ * These bindings are deprecated, because they do not match the actual
+ * concept of bindings but rather contain pure register values.
+ * Instead include the header in the DTS source directory.
+ */
+#warning "These bindings are deprecated. Instead use the header in the DTS source directory."
+
#define EXYNOS_PIN_PULL_NONE 0
#define EXYNOS_PIN_PULL_DOWN 1
#define EXYNOS_PIN_PULL_UP 3
/**
* struct kunit_assert - Data for printing a failed assertion or expectation.
- * @format: a function which formats the data in this kunit_assert to a string.
*
* Represents a failed expectation/assertion. Contains all the data necessary to
* format a string to a user reporting the failure.
*/
-struct kunit_assert {
- void (*format)(const struct kunit_assert *assert,
- const struct va_format *message,
- struct string_stream *stream);
-};
+struct kunit_assert {};
+
+typedef void (*assert_format_t)(const struct kunit_assert *assert,
+ const struct va_format *message,
+ struct string_stream *stream);
void kunit_assert_prologue(const struct kunit_loc *loc,
enum kunit_assert_type type,
const struct va_format *message,
struct string_stream *stream);
-/**
- * KUNIT_INIT_FAIL_ASSERT_STRUCT - Initializer for &struct kunit_fail_assert.
- *
- * Initializes a &struct kunit_fail_assert. Intended to be used in
- * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros.
- */
-#define KUNIT_INIT_FAIL_ASSERT_STRUCT { \
- .assert = { .format = kunit_fail_assert_format }, \
-}
-
/**
* struct kunit_unary_assert - Represents a KUNIT_{EXPECT|ASSERT}_{TRUE|FALSE}
* @assert: The parent of this type.
* KUNIT_EXPECT_* and KUNIT_ASSERT_* macros.
*/
#define KUNIT_INIT_UNARY_ASSERT_STRUCT(cond, expect_true) { \
- .assert = { .format = kunit_unary_assert_format }, \
.condition = cond, \
.expected_true = expect_true \
}
* KUNIT_EXPECT_* and KUNIT_ASSERT_* macros.
*/
#define KUNIT_INIT_PTR_NOT_ERR_STRUCT(txt, val) { \
- .assert = { .format = kunit_ptr_not_err_assert_format }, \
.text = txt, \
.value = val \
}
* KUNIT_INIT_BINARY_ASSERT_STRUCT() - Initializes a binary assert like
* kunit_binary_assert, kunit_binary_ptr_assert, etc.
*
- * @format_func: a function which formats the assert to a string.
* @text_: Pointer to a kunit_binary_assert_text.
* @left_val: The actual evaluated value of the expression in the left slot.
* @right_val: The actual evaluated value of the expression in the right slot.
* fields but with different types for left_val/right_val.
* This is ultimately used by binary assertion macros like KUNIT_EXPECT_EQ, etc.
*/
-#define KUNIT_INIT_BINARY_ASSERT_STRUCT(format_func, \
- text_, \
+#define KUNIT_INIT_BINARY_ASSERT_STRUCT(text_, \
left_val, \
right_val) { \
- .assert = { .format = format_func }, \
.text = text_, \
.left_value = left_val, \
.right_value = right_val \
struct kunit_resource *res,
void *match_data);
-/**
- * kunit_resource_instance_match() - Match a resource with the same instance.
- * @test: Test case to which the resource belongs.
- * @res: The resource.
- * @match_data: The resource pointer to match against.
- *
- * An instance of kunit_resource_match_t that matches a resource whose
- * allocation matches @match_data.
- */
-static inline bool kunit_resource_instance_match(struct kunit *test,
- struct kunit_resource *res,
- void *match_data)
-{
- return res->data == match_data;
-}
-
/**
* kunit_resource_name_match() - Match a resource with the same name.
* @test: Test case to which the resource belongs.
const struct kunit_loc *loc,
enum kunit_assert_type type,
const struct kunit_assert *assert,
+ assert_format_t assert_format,
const char *fmt, ...);
-#define KUNIT_ASSERTION(test, assert_type, pass, assert_class, INITIALIZER, fmt, ...) do { \
- if (unlikely(!(pass))) { \
- static const struct kunit_loc __loc = KUNIT_CURRENT_LOC; \
- struct assert_class __assertion = INITIALIZER; \
- kunit_do_failed_assertion(test, \
- &__loc, \
- assert_type, \
- &__assertion.assert, \
- fmt, \
- ##__VA_ARGS__); \
- } \
+#define _KUNIT_FAILED(test, assert_type, assert_class, assert_format, INITIALIZER, fmt, ...) do { \
+ static const struct kunit_loc __loc = KUNIT_CURRENT_LOC; \
+ const struct assert_class __assertion = INITIALIZER; \
+ kunit_do_failed_assertion(test, \
+ &__loc, \
+ assert_type, \
+ &__assertion.assert, \
+ assert_format, \
+ fmt, \
+ ##__VA_ARGS__); \
} while (0)
#define KUNIT_FAIL_ASSERTION(test, assert_type, fmt, ...) \
- KUNIT_ASSERTION(test, \
- assert_type, \
- false, \
- kunit_fail_assert, \
- KUNIT_INIT_FAIL_ASSERT_STRUCT, \
- fmt, \
- ##__VA_ARGS__)
+ _KUNIT_FAILED(test, \
+ assert_type, \
+ kunit_fail_assert, \
+ kunit_fail_assert_format, \
+ {}, \
+ fmt, \
+ ##__VA_ARGS__)
/**
* KUNIT_FAIL() - Always causes a test to fail when evaluated.
expected_true, \
fmt, \
...) \
- KUNIT_ASSERTION(test, \
- assert_type, \
- !!(condition) == !!expected_true, \
- kunit_unary_assert, \
- KUNIT_INIT_UNARY_ASSERT_STRUCT(#condition, \
- expected_true), \
- fmt, \
- ##__VA_ARGS__)
+do { \
+ if (likely(!!(condition) == !!expected_true)) \
+ break; \
+ \
+ _KUNIT_FAILED(test, \
+ assert_type, \
+ kunit_unary_assert, \
+ kunit_unary_assert_format, \
+ KUNIT_INIT_UNARY_ASSERT_STRUCT(#condition, \
+ expected_true), \
+ fmt, \
+ ##__VA_ARGS__); \
+} while (0)
#define KUNIT_TRUE_MSG_ASSERTION(test, assert_type, condition, fmt, ...) \
KUNIT_UNARY_ASSERTION(test, \
.right_text = #right, \
}; \
\
- KUNIT_ASSERTION(test, \
- assert_type, \
- __left op __right, \
- assert_class, \
- KUNIT_INIT_BINARY_ASSERT_STRUCT(format_func, \
- &__text, \
- __left, \
- __right), \
- fmt, \
- ##__VA_ARGS__); \
+ if (likely(__left op __right)) \
+ break; \
+ \
+ _KUNIT_FAILED(test, \
+ assert_type, \
+ assert_class, \
+ format_func, \
+ KUNIT_INIT_BINARY_ASSERT_STRUCT(&__text, \
+ __left, \
+ __right), \
+ fmt, \
+ ##__VA_ARGS__); \
} while (0)
#define KUNIT_BINARY_INT_ASSERTION(test, \
.right_text = #right, \
}; \
\
- KUNIT_ASSERTION(test, \
- assert_type, \
- strcmp(__left, __right) op 0, \
- kunit_binary_str_assert, \
- KUNIT_INIT_BINARY_ASSERT_STRUCT(kunit_binary_str_assert_format,\
- &__text, \
- __left, \
- __right), \
- fmt, \
- ##__VA_ARGS__); \
+ if (likely(strcmp(__left, __right) op 0)) \
+ break; \
+ \
+ \
+ _KUNIT_FAILED(test, \
+ assert_type, \
+ kunit_binary_str_assert, \
+ kunit_binary_str_assert_format, \
+ KUNIT_INIT_BINARY_ASSERT_STRUCT(&__text, \
+ __left, \
+ __right), \
+ fmt, \
+ ##__VA_ARGS__); \
} while (0)
#define KUNIT_PTR_NOT_ERR_OR_NULL_MSG_ASSERTION(test, \
do { \
const typeof(ptr) __ptr = (ptr); \
\
- KUNIT_ASSERTION(test, \
- assert_type, \
- !IS_ERR_OR_NULL(__ptr), \
- kunit_ptr_not_err_assert, \
- KUNIT_INIT_PTR_NOT_ERR_STRUCT(#ptr, \
- __ptr), \
- fmt, \
- ##__VA_ARGS__); \
+ if (!IS_ERR_OR_NULL(__ptr)) \
+ break; \
+ \
+ _KUNIT_FAILED(test, \
+ assert_type, \
+ kunit_ptr_not_err_assert, \
+ kunit_ptr_not_err_assert_format, \
+ KUNIT_INIT_PTR_NOT_ERR_STRUCT(#ptr, __ptr), \
+ fmt, \
+ ##__VA_ARGS__); \
} while (0)
/**
const typeof(*(ptr)) mask__ = (mask), bits__ = (bits); \
typeof(*(ptr)) old__, new__; \
\
+ old__ = READ_ONCE(*(ptr)); \
do { \
- old__ = READ_ONCE(*(ptr)); \
new__ = (old__ & ~mask__) | bits__; \
- } while (cmpxchg(ptr, old__, new__) != old__); \
+ } while (!try_cmpxchg(ptr, &old__, new__)); \
\
old__; \
})
const typeof(*(ptr)) clear__ = (clear), test__ = (test);\
typeof(*(ptr)) old__, new__; \
\
+ old__ = READ_ONCE(*(ptr)); \
do { \
- old__ = READ_ONCE(*(ptr)); \
+ if (old__ & test__) \
+ break; \
new__ = old__ & ~clear__; \
- } while (!(old__ & test__) && \
- cmpxchg(ptr, old__, new__) != old__); \
+ } while (!try_cmpxchg(ptr, &old__, new__)); \
\
!(old__ & test__); \
})
#define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */
#define QUEUE_FLAG_SQ_SCHED 30 /* single queue style io dispatch */
-#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
- (1 << QUEUE_FLAG_SAME_COMP) | \
- (1 << QUEUE_FLAG_NOWAIT))
+#define QUEUE_FLAG_MQ_DEFAULT ((1UL << QUEUE_FLAG_IO_STAT) | \
+ (1UL << QUEUE_FLAG_SAME_COMP) | \
+ (1UL << QUEUE_FLAG_NOWAIT))
void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
unsigned char lg_hyst; /* Low-G hysterisis */
unsigned char lg_dur; /* Low-G duration */
unsigned char lg_thres; /* Low-G threshold */
- unsigned char range; /* one of BMA0150_RANGE_xxx */
- unsigned char bandwidth; /* one of BMA0150_BW_xxx */
+ unsigned char range; /* one of BMA150_RANGE_xxx */
+ unsigned char bandwidth; /* one of BMA150_BW_xxx */
};
struct bma150_platform_data {
void __brelse(struct buffer_head *);
void __bforget(struct buffer_head *);
void __breadahead(struct block_device *, sector_t block, unsigned int size);
-void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
- gfp_t gfp);
struct buffer_head *__bread_gfp(struct block_device *,
sector_t block, unsigned size, gfp_t gfp);
void invalidate_bh_lrus(void);
void free_buffer_head(struct buffer_head * bh);
void unlock_buffer(struct buffer_head *bh);
void __lock_buffer(struct buffer_head *bh);
-void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]);
int sync_dirty_buffer(struct buffer_head *bh);
int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
void write_boundary_block(struct block_device *bdev,
sector_t bblock, unsigned blocksize);
int bh_uptodate_or_lock(struct buffer_head *bh);
-int bh_submit_read(struct buffer_head *bh);
+int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait);
+void __bh_read_batch(int nr, struct buffer_head *bhs[],
+ blk_opf_t op_flags, bool force_lock);
extern int buffer_heads_over_limit;
__breadahead(sb->s_bdev, block, sb->s_blocksize);
}
-static inline void
-sb_breadahead_unmovable(struct super_block *sb, sector_t block)
-{
- __breadahead_gfp(sb->s_bdev, block, sb->s_blocksize, 0);
-}
-
static inline struct buffer_head *
sb_getblk(struct super_block *sb, sector_t block)
{
return __getblk_gfp(bdev, block, size, __GFP_MOVABLE);
}
+static inline void bh_readahead(struct buffer_head *bh, blk_opf_t op_flags)
+{
+ if (!buffer_uptodate(bh) && trylock_buffer(bh)) {
+ if (!buffer_uptodate(bh))
+ __bh_read(bh, op_flags, false);
+ else
+ unlock_buffer(bh);
+ }
+}
+
+static inline void bh_read_nowait(struct buffer_head *bh, blk_opf_t op_flags)
+{
+ if (!bh_uptodate_or_lock(bh))
+ __bh_read(bh, op_flags, false);
+}
+
+/* Returns 1 if buffer uptodated, 0 on success, and -EIO on error. */
+static inline int bh_read(struct buffer_head *bh, blk_opf_t op_flags)
+{
+ if (bh_uptodate_or_lock(bh))
+ return 1;
+ return __bh_read(bh, op_flags, true);
+}
+
+static inline void bh_read_batch(int nr, struct buffer_head *bhs[])
+{
+ __bh_read_batch(nr, bhs, 0, true);
+}
+
+static inline void bh_readahead_batch(int nr, struct buffer_head *bhs[],
+ blk_opf_t op_flags)
+{
+ __bh_read_batch(nr, bhs, op_flags, false);
+}
+
/**
* __bread() - reads a specified block and returns the bh
* @bdev: the block_device to read from
#define cache_line_size() L1_CACHE_BYTES
#endif
+/*
+ * Helper to add padding within a struct to ensure data fall into separate
+ * cachelines.
+ */
+#if defined(CONFIG_SMP)
+struct cacheline_padding {
+ char x[0];
+} ____cacheline_internodealigned_in_smp;
+#define CACHELINE_PADDING(name) struct cacheline_padding name
+#else
+#define CACHELINE_PADDING(name)
+#endif
+
#endif /* __LINUX_CACHE_H */
struct ceph_msg_data *data; /* current data item */
size_t resid; /* bytes not yet consumed */
- bool last_piece; /* current is last piece */
bool need_crc; /* crc update needed */
union {
#ifdef CONFIG_BLOCK
void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor,
struct ceph_msg *msg, size_t length);
struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,
- size_t *page_offset, size_t *length,
- bool *last_piece);
+ size_t *page_offset, size_t *length);
void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor, size_t bytes);
u32 ceph_crc32c_page(u32 crc, struct page *page, unsigned int page_offset,
struct cgroup_file procs_file; /* handle for "cgroup.procs" */
struct cgroup_file events_file; /* handle for "cgroup.events" */
+ /* handles for "{cpu,memory,io,irq}.pressure" */
+ struct cgroup_file psi_files[NR_PSI_RESOURCES];
+
/*
* The bitmask of subsystems enabled on the child cgroups.
* ->subtree_control is the one configured through
css_put(&cgrp->self);
}
+extern struct mutex cgroup_mutex;
+
+static inline void cgroup_lock(void)
+{
+ mutex_lock(&cgroup_mutex);
+}
+
+static inline void cgroup_unlock(void)
+{
+ mutex_unlock(&cgroup_mutex);
+}
+
/**
* task_css_set_check - obtain a task's css_set with extra access conditions
* @task: the task to obtain css_set for
* as locks used during the cgroup_subsys::attach() methods.
*/
#ifdef CONFIG_PROVE_RCU
-extern struct mutex cgroup_mutex;
extern spinlock_t css_set_lock;
#define task_css_set_check(task, __c) \
rcu_dereference_check((task)->cgroups, \
pr_cont_kernfs_path(cgrp->kn);
}
-static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
-{
- return cgrp->psi;
-}
-
bool cgroup_psi_enabled(void);
static inline void cgroup_init_kthreadd(void)
static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; }
static inline void css_get(struct cgroup_subsys_state *css) {}
static inline void css_put(struct cgroup_subsys_state *css) {}
+static inline void cgroup_lock(void) {}
+static inline void cgroup_unlock(void) {}
static inline int cgroup_attach_task_all(struct task_struct *from,
struct task_struct *t) { return 0; }
static inline int cgroupstats_build(struct cgroupstats *stats,
* struct clk_rate_request - Structure encoding the clk constraints that
* a clock user might require.
*
+ * Should be initialized by calling clk_hw_init_rate_request().
+ *
* @rate: Requested clock rate. This field will be adjusted by
* clock drivers according to hardware capabilities.
* @min_rate: Minimum rate imposed by clk users.
struct clk_hw *best_parent_hw;
};
+void clk_hw_init_rate_request(const struct clk_hw *hw,
+ struct clk_rate_request *req,
+ unsigned long rate);
+void clk_hw_forward_rate_request(const struct clk_hw *core,
+ const struct clk_rate_request *old_req,
+ const struct clk_hw *parent,
+ struct clk_rate_request *req,
+ unsigned long parent_rate);
+
/**
* struct clk_duty - Struture encoding the duty cycle ratio of a clock
*
*
* @recalc_rate Recalculate the rate of this clock, by querying hardware. The
* parent rate is an input parameter. It is up to the caller to
- * ensure that the prepare_mutex is held across this call.
- * Returns the calculated rate. Optional, but recommended - if
+ * ensure that the prepare_mutex is held across this call. If the
+ * driver cannot figure out a rate for this clock, it must return
+ * 0. Returns the calculated rate. Optional, but recommended - if
* this op is not set then clock rate will be initialized to 0.
*
* @round_rate: Given a target rate as input, returns the closest rate actually
struct clk_rate_request *req,
unsigned long flags);
void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent);
+void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate,
+ unsigned long *max_rate);
void clk_hw_set_rate_range(struct clk_hw *hw, unsigned long min_rate,
unsigned long max_rate);
*
* Returns true if @parent is a possible parent for @clk, false otherwise.
*/
-bool clk_has_parent(struct clk *clk, struct clk *parent);
+bool clk_has_parent(const struct clk *clk, const struct clk *parent);
/**
* clk_set_rate_range - set a rate range for a clock source
#ifndef AT91_PMC_H
#define AT91_PMC_H
+#include <linux/bits.h>
+
#define AT91_PMC_V1 (1) /* PMC version 1 */
#define AT91_PMC_V2 (2) /* PMC version 2 [SAM9X60] */
#define AT91_PMC_PCSR 0x18 /* Peripheral Clock Status Register */
#define AT91_PMC_PLL_ACR 0x18 /* PLL Analog Control Register [for SAM9X60] */
-#define AT91_PMC_PLL_ACR_DEFAULT_UPLL 0x12020010UL /* Default PLL ACR value for UPLL */
-#define AT91_PMC_PLL_ACR_DEFAULT_PLLA 0x00020010UL /* Default PLL ACR value for PLLA */
+#define AT91_PMC_PLL_ACR_DEFAULT_UPLL UL(0x12020010) /* Default PLL ACR value for UPLL */
+#define AT91_PMC_PLL_ACR_DEFAULT_PLLA UL(0x00020010) /* Default PLL ACR value for PLLA */
#define AT91_PMC_PLL_ACR_UTMIVR (1 << 12) /* UPLL Voltage regulator Control */
#define AT91_PMC_PLL_ACR_UTMIBG (1 << 13) /* UPLL Bandgap Control */
#ifndef __LINUX_CLK_SPEAR_H
#define __LINUX_CLK_SPEAR_H
+#ifdef CONFIG_ARCH_SPEAR3XX
+void __init spear3xx_clk_init(void __iomem *misc_base,
+ void __iomem *soc_config_base);
+#else
+static inline void __init spear3xx_clk_init(void __iomem *misc_base,
+ void __iomem *soc_config_base) {}
+#endif
+
+#ifdef CONFIG_ARCH_SPEAR6XX
+void __init spear6xx_clk_init(void __iomem *misc_base);
+#else
+static inline void __init spear6xx_clk_init(void __iomem *misc_base) {}
+#endif
+
#ifdef CONFIG_MACH_SPEAR1310
void __init spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base);
#else
#define __no_sanitize_undefined
#endif
+#if __has_feature(memory_sanitizer)
+#define __SANITIZE_MEMORY__
+/*
+ * Unlike other sanitizers, KMSAN still inserts code into functions marked with
+ * no_sanitize("kernel-memory"). Using disable_sanitizer_instrumentation
+ * provides the behavior consistent with other __no_sanitize_ attributes,
+ * guaranteeing that __no_sanitize_memory functions remain uninstrumented.
+ */
+#define __no_sanitize_memory __disable_sanitizer_instrumentation
+
+/*
+ * The __no_kmsan_checks attribute ensures that a function does not produce
+ * false positive reports by:
+ * - initializing all local variables and memory stores in this function;
+ * - skipping all shadow checks;
+ * - passing initialized arguments to this function's callees.
+ */
+#define __no_kmsan_checks __attribute__((no_sanitize("kernel-memory")))
+#else
+#define __no_sanitize_memory
+#define __no_kmsan_checks
+#endif
+
/*
* Support for __has_feature(coverage_sanitizer) was added in Clang 13 together
* with no_sanitize("coverage"). Prior versions of Clang support coverage
#define __SANITIZE_ADDRESS__
#endif
+/*
+ * GCC does not support KMSAN.
+ */
+#define __no_sanitize_memory
+#define __no_kmsan_checks
+
/*
* Turn individual warnings and errors on and off locally, depending
* on version.
/* Section for code which can't be instrumented at all */
#define noinstr \
noinline notrace __attribute((__section__(".noinstr.text"))) \
- __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage
+ __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage \
+ __no_sanitize_memory
#endif /* __KERNEL__ */
static inline
unsigned int cpumask_next(int n, const struct cpumask *srcp)
{
- /* n is a prior cpu */
- cpumask_check(n + 1);
+ /* -1 is a legal arg here. */
+ if (n != -1)
+ cpumask_check(n);
return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
}
*/
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
{
- /* n is a prior cpu */
- cpumask_check(n + 1);
+ /* -1 is a legal arg here. */
+ if (n != -1)
+ cpumask_check(n);
return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
}
unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
const struct cpumask *src2p)
{
- /* n is a prior cpu */
- cpumask_check(n + 1);
+ /* -1 is a legal arg here. */
+ if (n != -1)
+ cpumask_check(n);
return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
nr_cpumask_bits, n + 1);
}
unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
{
cpumask_check(start);
- /* n is a prior cpu */
- cpumask_check(n + 1);
+ if (n != -1)
+ cpumask_check(n);
/*
* Return the first available CPU when wrapping, or when starting before cpu0,
};
/**
- * struct damos - Represents a Data Access Monitoring-based Operation Scheme.
+ * struct damos_access_pattern - Target access pattern of the given scheme.
* @min_sz_region: Minimum size of target regions.
* @max_sz_region: Maximum size of target regions.
* @min_nr_accesses: Minimum ``->nr_accesses`` of target regions.
* @max_nr_accesses: Maximum ``->nr_accesses`` of target regions.
* @min_age_region: Minimum age of target regions.
* @max_age_region: Maximum age of target regions.
+ */
+struct damos_access_pattern {
+ unsigned long min_sz_region;
+ unsigned long max_sz_region;
+ unsigned int min_nr_accesses;
+ unsigned int max_nr_accesses;
+ unsigned int min_age_region;
+ unsigned int max_age_region;
+};
+
+/**
+ * struct damos - Represents a Data Access Monitoring-based Operation Scheme.
+ * @pattern: Access pattern of target regions.
* @action: &damo_action to be applied to the target regions.
* @quota: Control the aggressiveness of this scheme.
* @wmarks: Watermarks for automated (in)activation of this scheme.
* @list: List head for siblings.
*
* For each aggregation interval, DAMON finds regions which fit in the
- * condition (&min_sz_region, &max_sz_region, &min_nr_accesses,
- * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to
- * those. To avoid consuming too much CPU time or IO resources for the
- * &action, "a is used.
+ * &pattern and applies &action to those. To avoid consuming too much
+ * CPU time or IO resources for the &action, "a is used.
*
* To do the work only when needed, schemes can be activated for specific
* system situations using &wmarks. If all schemes that registered to the
* &action is applied.
*/
struct damos {
- unsigned long min_sz_region;
- unsigned long max_sz_region;
- unsigned int min_nr_accesses;
- unsigned int max_nr_accesses;
- unsigned int min_age_region;
- unsigned int max_age_region;
+ struct damos_access_pattern pattern;
enum damos_action action;
struct damos_quota quota;
struct damos_watermarks wmarks;
unsigned long (*apply_scheme)(struct damon_ctx *context,
struct damon_target *t, struct damon_region *r,
struct damos *scheme);
- bool (*target_valid)(void *target);
+ bool (*target_valid)(struct damon_target *t);
void (*cleanup)(struct damon_ctx *context);
};
};
/**
- * struct damon_ctx - Represents a context for each monitoring. This is the
- * main interface that allows users to set the attributes and get the results
- * of the monitoring.
+ * struct damon_attrs - Monitoring attributes for accuracy/overhead control.
*
* @sample_interval: The time between access samplings.
* @aggr_interval: The time between monitor results aggregations.
* @ops_update_interval: The time between monitoring operations updates.
+ * @min_nr_regions: The minimum number of adaptive monitoring
+ * regions.
+ * @max_nr_regions: The maximum number of adaptive monitoring
+ * regions.
*
* For each @sample_interval, DAMON checks whether each region is accessed or
* not. It aggregates and keeps the access information (number of accesses to
* @ops_update_interval. All time intervals are in micro-seconds.
* Please refer to &struct damon_operations and &struct damon_callback for more
* detail.
+ */
+struct damon_attrs {
+ unsigned long sample_interval;
+ unsigned long aggr_interval;
+ unsigned long ops_update_interval;
+ unsigned long min_nr_regions;
+ unsigned long max_nr_regions;
+};
+
+/**
+ * struct damon_ctx - Represents a context for each monitoring. This is the
+ * main interface that allows users to set the attributes and get the results
+ * of the monitoring.
*
+ * @attrs: Monitoring attributes for accuracy/overhead control.
* @kdamond: Kernel thread who does the monitoring.
* @kdamond_lock: Mutex for the synchronizations with @kdamond.
*
* @ops: Set of monitoring operations for given use cases.
* @callback: Set of callbacks for monitoring events notifications.
*
- * @min_nr_regions: The minimum number of adaptive monitoring regions.
- * @max_nr_regions: The maximum number of adaptive monitoring regions.
* @adaptive_targets: Head of monitoring targets (&damon_target) list.
* @schemes: Head of schemes (&damos) list.
*/
struct damon_ctx {
- unsigned long sample_interval;
- unsigned long aggr_interval;
- unsigned long ops_update_interval;
+ struct damon_attrs attrs;
/* private: internal use only */
struct timespec64 last_aggregation;
struct damon_operations ops;
struct damon_callback callback;
- unsigned long min_nr_regions;
- unsigned long max_nr_regions;
struct list_head adaptive_targets;
struct list_head schemes;
};
return list_last_entry(&t->regions_list, struct damon_region, list);
}
+static inline struct damon_region *damon_first_region(struct damon_target *t)
+{
+ return list_first_entry(&t->regions_list, struct damon_region, list);
+}
+
+static inline unsigned long damon_sz_region(struct damon_region *r)
+{
+ return r->ar.end - r->ar.start;
+}
+
+
#define damon_for_each_region(r, t) \
list_for_each_entry(r, &t->regions_list, list)
+#define damon_for_each_region_from(r, t) \
+ list_for_each_entry_from(r, &t->regions_list, list)
+
#define damon_for_each_region_safe(r, next, t) \
list_for_each_entry_safe(r, next, &t->regions_list, list)
int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
unsigned int nr_ranges);
-struct damos *damon_new_scheme(
- unsigned long min_sz_region, unsigned long max_sz_region,
- unsigned int min_nr_accesses, unsigned int max_nr_accesses,
- unsigned int min_age_region, unsigned int max_age_region,
- enum damos_action action, struct damos_quota *quota,
- struct damos_watermarks *wmarks);
+struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
+ enum damos_action action, struct damos_quota *quota,
+ struct damos_watermarks *wmarks);
void damon_add_scheme(struct damon_ctx *ctx, struct damos *s);
void damon_destroy_scheme(struct damos *s);
struct damon_ctx *damon_new_ctx(void);
void damon_destroy_ctx(struct damon_ctx *ctx);
-int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int,
- unsigned long aggr_int, unsigned long ops_upd_int,
- unsigned long min_nr_reg, unsigned long max_nr_reg);
-int damon_set_schemes(struct damon_ctx *ctx,
+int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs);
+void damon_set_schemes(struct damon_ctx *ctx,
struct damos **schemes, ssize_t nr_schemes);
int damon_nr_running_ctxs(void);
bool damon_is_registered_ops(enum damon_ops_id id);
int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive);
int damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
+int damon_set_region_biggest_system_ram_default(struct damon_target *t,
+ unsigned long *start, unsigned long *end);
+
#endif /* CONFIG_DAMON */
#endif /* _DAMON_H */
#include <linux/wait.h>
struct path;
+struct file;
struct vfsmount;
/*
/* <clickety>-<click> the ramfs-type tree */
extern void d_genocide(struct dentry *);
-extern void d_tmpfile(struct dentry *, struct inode *);
+extern void d_tmpfile(struct file *, struct inode *);
extern struct dentry *d_find_alias(struct inode *);
extern void d_prune_aliases(struct inode *);
extern __u64 __delayacct_blkio_ticks(struct task_struct *);
extern void __delayacct_freepages_start(void);
extern void __delayacct_freepages_end(void);
-extern void __delayacct_thrashing_start(void);
-extern void __delayacct_thrashing_end(void);
+extern void __delayacct_thrashing_start(bool *in_thrashing);
+extern void __delayacct_thrashing_end(bool *in_thrashing);
extern void __delayacct_swapin_start(void);
extern void __delayacct_swapin_end(void);
extern void __delayacct_compact_start(void);
__delayacct_freepages_end();
}
-static inline void delayacct_thrashing_start(void)
+static inline void delayacct_thrashing_start(bool *in_thrashing)
{
if (!static_branch_unlikely(&delayacct_key))
return;
if (current->delays)
- __delayacct_thrashing_start();
+ __delayacct_thrashing_start(in_thrashing);
}
-static inline void delayacct_thrashing_end(void)
+static inline void delayacct_thrashing_end(bool *in_thrashing)
{
if (!static_branch_unlikely(&delayacct_key))
return;
if (current->delays)
- __delayacct_thrashing_end();
+ __delayacct_thrashing_end(in_thrashing);
}
static inline void delayacct_swapin_start(void)
{}
static inline void delayacct_freepages_end(void)
{}
-static inline void delayacct_thrashing_start(void)
+static inline void delayacct_thrashing_start(bool *in_thrashing)
{}
-static inline void delayacct_thrashing_end(void)
+static inline void delayacct_thrashing_end(bool *in_thrashing)
{}
static inline void delayacct_swapin_start(void)
{}
/**
* arch_do_signal_or_restart - Architecture specific signal delivery function
* @regs: Pointer to currents pt_regs
- * @has_signal: actual signal to handle
*
* Invoked from exit_to_user_mode_loop().
*/
__le32 total_segments;
} __packed;
+/* reason of stop_checkpoint */
+enum stop_cp_reason {
+ STOP_CP_REASON_SHUTDOWN,
+ STOP_CP_REASON_FAULT_INJECT,
+ STOP_CP_REASON_META_PAGE,
+ STOP_CP_REASON_WRITE_FAIL,
+ STOP_CP_REASON_CORRUPTED_SUMMARY,
+ STOP_CP_REASON_UPDATE_INODE,
+ STOP_CP_REASON_FLUSH_FAIL,
+ STOP_CP_REASON_MAX,
+};
+
+#define MAX_STOP_REASON 32
+
+/* detail reason for EFSCORRUPTED */
+enum f2fs_error {
+ ERROR_CORRUPTED_CLUSTER,
+ ERROR_FAIL_DECOMPRESSION,
+ ERROR_INVALID_BLKADDR,
+ ERROR_CORRUPTED_DIRENT,
+ ERROR_CORRUPTED_INODE,
+ ERROR_INCONSISTENT_SUMMARY,
+ ERROR_INCONSISTENT_FOOTER,
+ ERROR_INCONSISTENT_SUM_TYPE,
+ ERROR_CORRUPTED_JOURNAL,
+ ERROR_INCONSISTENT_NODE_COUNT,
+ ERROR_INCONSISTENT_BLOCK_COUNT,
+ ERROR_INVALID_CURSEG,
+ ERROR_INCONSISTENT_SIT,
+ ERROR_CORRUPTED_VERITY_XATTR,
+ ERROR_CORRUPTED_XATTR,
+ ERROR_MAX,
+};
+
+#define MAX_F2FS_ERRORS 16
+
struct f2fs_super_block {
__le32 magic; /* Magic Number */
__le16 major_ver; /* Major Version */
__u8 hot_ext_count; /* # of hot file extension */
__le16 s_encoding; /* Filename charset encoding */
__le16 s_encoding_flags; /* Filename charset encoding flags */
- __u8 reserved[306]; /* valid reserved region */
+ __u8 s_stop_reason[MAX_STOP_REASON]; /* stop checkpoint reason */
+ __u8 s_errors[MAX_F2FS_ERRORS]; /* reason of image corrupts */
+ __u8 reserved[258]; /* valid reserved region */
__le32 crc; /* checksum of superblock */
} __packed;
* __struct_size() vs __member_size() must be captured here to avoid
* evaluating argument side-effects further into the macro layers.
*/
+#ifndef CONFIG_KMSAN
#define memset(p, c, s) __fortify_memset_chk(p, c, s, \
__struct_size(p), __member_size(p))
+#endif
/*
* To make sure the compiler can enforce protection against buffer overflows,
WHITEOUT_DEV);
}
-struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns,
- struct dentry *dentry, umode_t mode, int open_flag);
+struct file *vfs_tmpfile_open(struct user_namespace *mnt_userns,
+ const struct path *parentpath,
+ umode_t mode, int open_flag, const struct cred *cred);
int vfs_mkobj(struct dentry *, umode_t,
int (*f)(struct dentry *, umode_t, void *),
struct file *, unsigned open_flag,
umode_t create_mode);
int (*tmpfile) (struct user_namespace *, struct inode *,
- struct dentry *, umode_t);
+ struct file *, umode_t);
int (*set_acl)(struct user_namespace *, struct inode *,
struct posix_acl *, int);
int (*fileattr_set)(struct user_namespace *mnt_userns,
int (*open)(struct inode *, struct file *));
extern int finish_no_open(struct file *file, struct dentry *dentry);
+/* Helper for the simple case when original dentry is used */
+static inline int finish_open_simple(struct file *file, int error)
+{
+ if (error)
+ return error;
+
+ return finish_open(file, file->f_path.dentry, NULL);
+}
+
/* fs/dcache.c */
extern void __init vfs_caches_init_early(void);
extern void __init vfs_caches_init(void);
static inline void gameport_set_name(struct gameport *gameport, const char *name)
{
- strlcpy(gameport->name, name, sizeof(gameport->name));
+ strscpy(gameport->name, name, sizeof(gameport->name));
}
/*
VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
+ BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE);
+ BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >>
+ GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC);
if (unlikely(page_group_by_mobility_disabled))
return MIGRATE_UNMOVABLE;
return !!(gfp_flags & __GFP_DIRECT_RECLAIM);
}
-/**
- * gfpflags_normal_context - is gfp_flags a normal sleepable context?
- * @gfp_flags: gfp_flags to test
- *
- * Test whether @gfp_flags indicates that the allocation is from the
- * %current context and allowed to sleep.
- *
- * An allocation being allowed to block doesn't mean it owns the %current
- * context. When direct reclaim path tries to allocate memory, the
- * allocation context is nested inside whatever %current was doing at the
- * time of the original allocation. The nested allocation may be allowed
- * to block but modifying anything %current owns can corrupt the outer
- * context's expectations.
- *
- * %true result from this function indicates that the allocation context
- * can sleep and use anything that's associated with %current.
- */
-static inline bool gfpflags_normal_context(const gfp_t gfp_flags)
-{
- return (gfp_flags & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC)) ==
- __GFP_DIRECT_RECLAIM;
-}
-
#ifdef CONFIG_HIGHMEM
#define OPT_ZONE_HIGHMEM ZONE_HIGHMEM
#else
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/cacheflush.h>
+#include <linux/kmsan.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
copy_user_page(vto, vfrom, vaddr, to);
+ kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
kunmap_local(vto);
kunmap_local(vfrom);
}
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
copy_page(vto, vfrom);
+ kmsan_copy_page_meta(to, from);
kunmap_local(vto);
kunmap_local(vfrom);
}
!inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
}
-bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags,
- bool smaps, bool in_pf);
+bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags,
+ bool smaps, bool in_pf, bool enforce_sysfs);
#define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \
int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags,
int advice);
+int madvise_collapse(struct vm_area_struct *vma,
+ struct vm_area_struct **prev,
+ unsigned long start, unsigned long end);
void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start,
unsigned long end, long adjust_next);
spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma);
}
static inline bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags,
- bool smaps, bool in_pf)
+ unsigned long vm_flags, bool smaps,
+ bool in_pf, bool enforce_sysfs)
{
return false;
}
static inline int hugepage_madvise(struct vm_area_struct *vma,
unsigned long *vm_flags, int advice)
{
- BUG();
- return 0;
+ return -EINVAL;
+}
+
+static inline int madvise_collapse(struct vm_area_struct *vma,
+ struct vm_area_struct **prev,
+ unsigned long start, unsigned long end)
+{
+ return -EINVAL;
}
+
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
unsigned long start,
unsigned long end,
return split_huge_page_to_list(&folio->page, list);
}
+static inline int split_folio(struct folio *folio)
+{
+ return split_folio_to_list(folio, NULL);
+}
+
/*
* archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to
* limitations in the implementation like arm64 MTE can override this to
struct ctl_table;
struct user_struct;
struct mmu_gather;
+struct node;
#ifndef CONFIG_ARCH_HAS_HUGEPD
typedef struct { unsigned long pd; } hugepd_t;
#endif
};
+struct hugetlb_vma_lock {
+ struct kref refs;
+ struct rw_semaphore rw_sema;
+ struct vm_area_struct *vma;
+};
+
extern struct resv_map *resv_map_alloc(void);
void resv_map_release(struct kref *ref);
long min_hpages);
void hugepage_put_subpool(struct hugepage_subpool *spool);
-void reset_vma_resv_huge_pages(struct vm_area_struct *vma);
+void hugetlb_dup_vma_private(struct vm_area_struct *vma);
void clear_vma_resv_huge_pages(struct vm_area_struct *vma);
int hugetlb_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *);
int hugetlb_overcommit_handler(struct ctl_table *, int, void *, size_t *,
struct page *follow_huge_pd(struct vm_area_struct *vma,
unsigned long address, hugepd_t hpd,
int flags, int pdshift);
-struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
- pmd_t *pmd, int flags);
+struct page *follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address,
+ int flags);
struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
pud_t *pud, int flags);
struct page *follow_huge_pgd(struct mm_struct *mm, unsigned long address,
pgd_t *pgd, int flags);
+void hugetlb_vma_lock_read(struct vm_area_struct *vma);
+void hugetlb_vma_unlock_read(struct vm_area_struct *vma);
+void hugetlb_vma_lock_write(struct vm_area_struct *vma);
+void hugetlb_vma_unlock_write(struct vm_area_struct *vma);
+int hugetlb_vma_trylock_write(struct vm_area_struct *vma);
+void hugetlb_vma_assert_locked(struct vm_area_struct *vma);
+void hugetlb_vma_lock_release(struct kref *kref);
+
int pmd_huge(pmd_t pmd);
int pud_huge(pud_t pud);
unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
#else /* !CONFIG_HUGETLB_PAGE */
-static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
+static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma)
{
}
return NULL;
}
-static inline struct page *follow_huge_pmd(struct mm_struct *mm,
- unsigned long address, pmd_t *pmd, int flags)
+static inline struct page *follow_huge_pmd_pte(struct vm_area_struct *vma,
+ unsigned long address, int flags)
{
return NULL;
}
return -EINVAL;
}
+static inline void hugetlb_vma_lock_read(struct vm_area_struct *vma)
+{
+}
+
+static inline void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
+{
+}
+
+static inline void hugetlb_vma_lock_write(struct vm_area_struct *vma)
+{
+}
+
+static inline void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
+{
+}
+
+static inline int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
+{
+ return 1;
+}
+
+static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
+{
+}
+
static inline int pmd_huge(pmd_t pmd)
{
return 0;
nodemask_t *nmask, gfp_t gfp_mask);
struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
unsigned long address);
-int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
+int hugetlb_add_to_page_cache(struct page *page, struct address_space *mapping,
pgoff_t idx);
void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
unsigned long address, struct page *page);
}
#endif
+#ifdef CONFIG_NUMA
+void hugetlb_register_node(struct node *node);
+void hugetlb_unregister_node(struct node *node);
+#endif
+
#else /* CONFIG_HUGETLB_PAGE */
struct hstate {};
pte_t *ptep, pte_t pte)
{
}
+
+static inline void hugetlb_register_node(struct node *node)
+{
+}
+
+static inline void hugetlb_unregister_node(struct node *node)
+{
+}
#endif /* CONFIG_HUGETLB_PAGE */
static inline spinlock_t *huge_pte_lock(struct hstate *h,
#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
extern void __init hugetlb_cma_reserve(int order);
-extern void __init hugetlb_cma_check(void);
#else
static inline __init void hugetlb_cma_reserve(int order)
{
}
-static inline __init void hugetlb_cma_check(void)
-{
-}
#endif
bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr);
return __hugetlb_cgroup_from_page(page, true);
}
-static inline int __set_hugetlb_cgroup(struct page *page,
+static inline void __set_hugetlb_cgroup(struct page *page,
struct hugetlb_cgroup *h_cg, bool rsvd)
{
VM_BUG_ON_PAGE(!PageHuge(page), page);
if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER)
- return -1;
+ return;
if (rsvd)
set_page_private(page + SUBPAGE_INDEX_CGROUP_RSVD,
(unsigned long)h_cg);
else
set_page_private(page + SUBPAGE_INDEX_CGROUP,
(unsigned long)h_cg);
- return 0;
}
-static inline int set_hugetlb_cgroup(struct page *page,
+static inline void set_hugetlb_cgroup(struct page *page,
struct hugetlb_cgroup *h_cg)
{
- return __set_hugetlb_cgroup(page, h_cg, false);
+ __set_hugetlb_cgroup(page, h_cg, false);
}
-static inline int set_hugetlb_cgroup_rsvd(struct page *page,
+static inline void set_hugetlb_cgroup_rsvd(struct page *page,
struct hugetlb_cgroup *h_cg)
{
- return __set_hugetlb_cgroup(page, h_cg, true);
+ __set_hugetlb_cgroup(page, h_cg, true);
}
static inline bool hugetlb_cgroup_disabled(void)
return NULL;
}
-static inline int set_hugetlb_cgroup(struct page *page,
+static inline void set_hugetlb_cgroup(struct page *page,
struct hugetlb_cgroup *h_cg)
{
- return 0;
}
-static inline int set_hugetlb_cgroup_rsvd(struct page *page,
+static inline void set_hugetlb_cgroup_rsvd(struct page *page,
struct hugetlb_cgroup *h_cg)
{
- return 0;
}
static inline bool hugetlb_cgroup_disabled(void)
extern initcall_entry_t __con_initcall_start[], __con_initcall_end[];
-/* Used for contructor calls. */
+/* Used for constructor calls. */
typedef void (*ctor_fn_t)(void);
struct file_system_type;
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Driver for AUO in-cell touchscreens
- *
- * Copyright (c) 2011 Heiko Stuebner <heiko@sntech.de>
- *
- * based on auo_touch.h from Dell Streak kernel
- *
- * Copyright (c) 2008 QUALCOMM Incorporated.
- * Copyright (c) 2008 QUALCOMM USA, INC.
- */
-
-#ifndef __AUO_PIXCIR_TS_H__
-#define __AUO_PIXCIR_TS_H__
-
-/*
- * Interrupt modes:
- * periodical: interrupt is asserted periodicaly
- * compare coordinates: interrupt is asserted when coordinates change
- * indicate touch: interrupt is asserted during touch
- */
-#define AUO_PIXCIR_INT_PERIODICAL 0x00
-#define AUO_PIXCIR_INT_COMP_COORD 0x01
-#define AUO_PIXCIR_INT_TOUCH_IND 0x02
-
-/*
- * @gpio_int interrupt gpio
- * @int_setting one of AUO_PIXCIR_INT_*
- * @init_hw hardwarespecific init
- * @exit_hw hardwarespecific shutdown
- * @x_max x-resolution
- * @y_max y-resolution
- */
-struct auo_pixcir_ts_platdata {
- int gpio_int;
- int gpio_rst;
-
- int int_setting;
-
- unsigned int x_max;
- unsigned int y_max;
-};
-
-#endif
/*
* This header provides generic wrappers for memory access instrumentation that
- * the compiler cannot emit for: KASAN, KCSAN.
+ * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
*/
#ifndef _LINUX_INSTRUMENTED_H
#define _LINUX_INSTRUMENTED_H
#include <linux/compiler.h>
#include <linux/kasan-checks.h>
#include <linux/kcsan-checks.h>
+#include <linux/kmsan-checks.h>
#include <linux/types.h>
/**
{
kasan_check_read(from, n);
kcsan_check_read(from, n);
+ kmsan_copy_to_user(to, from, n, 0);
}
/**
- * instrument_copy_from_user - instrument writes of copy_from_user
+ * instrument_copy_from_user_before - add instrumentation before copy_from_user
*
* Instrument writes to kernel memory, that are due to copy_from_user (and
* variants). The instrumentation should be inserted before the accesses.
* @n number of bytes to copy
*/
static __always_inline void
-instrument_copy_from_user(const void *to, const void __user *from, unsigned long n)
+instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
{
kasan_check_write(to, n);
kcsan_check_write(to, n);
}
+/**
+ * instrument_copy_from_user_after - add instrumentation after copy_from_user
+ *
+ * Instrument writes to kernel memory, that are due to copy_from_user (and
+ * variants). The instrumentation should be inserted after the accesses.
+ *
+ * @to destination address
+ * @from source address
+ * @n number of bytes to copy
+ * @left number of bytes not copied (as returned by copy_from_user)
+ */
+static __always_inline void
+instrument_copy_from_user_after(const void *to, const void __user *from,
+ unsigned long n, unsigned long left)
+{
+ kmsan_unpoison_memory(to, n - left);
+}
+
+/**
+ * instrument_get_user() - add instrumentation to get_user()-like macros
+ *
+ * get_user() and friends are fragile, so it may depend on the implementation
+ * whether the instrumentation happens before or after the data is copied from
+ * the userspace.
+ *
+ * @to destination variable, may not be address-taken
+ */
+#define instrument_get_user(to) \
+({ \
+ u64 __tmp = (u64)(to); \
+ kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \
+ to = __tmp; \
+})
+
+
+/**
+ * instrument_put_user() - add instrumentation to put_user()-like macros
+ *
+ * put_user() and friends are fragile, so it may depend on the implementation
+ * whether the instrumentation happens before or after the data is copied from
+ * the userspace.
+ *
+ * @from source address
+ * @ptr userspace pointer to copy to
+ * @size number of bytes to copy
+ */
+#define instrument_put_user(from, ptr, size) \
+({ \
+ kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \
+})
+
#endif /* _LINUX_INSTRUMENTED_H */
unsigned int alloc_hint;
};
-struct io_notif;
-struct io_notif_slot;
-
struct io_hash_bucket {
spinlock_t lock;
struct hlist_head list;
unsigned nr_user_files;
unsigned nr_user_bufs;
struct io_mapped_ubuf **user_bufs;
- struct io_notif_slot *notif_slots;
- unsigned nr_notif_slots;
struct io_submit_state submit_state;
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates.
+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+#ifndef _IOVA_BITMAP_H_
+#define _IOVA_BITMAP_H_
+
+#include <linux/types.h>
+
+struct iova_bitmap;
+
+typedef int (*iova_bitmap_fn_t)(struct iova_bitmap *bitmap,
+ unsigned long iova, size_t length,
+ void *opaque);
+
+struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length,
+ unsigned long page_size,
+ u64 __user *data);
+void iova_bitmap_free(struct iova_bitmap *bitmap);
+int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque,
+ iova_bitmap_fn_t fn);
+void iova_bitmap_set(struct iova_bitmap *bitmap,
+ unsigned long iova, size_t length);
+
+#endif
#include <linux/refcount.h>
#include <linux/rhashtable-types.h>
#include <linux/sysctl.h>
+#include <linux/percpu_counter.h>
struct user_namespace;
unsigned int msg_ctlmax;
unsigned int msg_ctlmnb;
unsigned int msg_ctlmni;
- atomic_t msg_bytes;
- atomic_t msg_hdrs;
+ struct percpu_counter percpu_msg_bytes;
+ struct percpu_counter percpu_msg_hdrs;
size_t shm_ctlmax;
size_t shm_ctlall;
#define IRQCHIP_MATCH(compat, fn) { .compatible = compat, \
.data = typecheck_irq_init_cb(fn), },
-#define IRQCHIP_PLATFORM_DRIVER_END(drv_name) \
+
+#define IRQCHIP_PLATFORM_DRIVER_END(drv_name, ...) \
{}, \
}; \
MODULE_DEVICE_TABLE(of, drv_name##_irqchip_match_table); \
.owner = THIS_MODULE, \
.of_match_table = drv_name##_irqchip_match_table, \
.suppress_bind_attrs = true, \
+ __VA_ARGS__ \
}, \
}; \
builtin_platform_driver(drv_name##_driver)
* conversion failed.
*/
int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq);
+int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq);
int generic_handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq);
#endif
static inline void
inode_set_max_iversion_raw(struct inode *inode, u64 val)
{
- u64 cur, old;
+ u64 cur = inode_peek_iversion_raw(inode);
- cur = inode_peek_iversion_raw(inode);
- for (;;) {
+ do {
if (cur > val)
break;
- old = atomic64_cmpxchg(&inode->i_version, cur, val);
- if (likely(old == cur))
- break;
- cur = old;
- }
+ } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, val));
}
/**
I_VERSION_QUERIED);
}
-/**
- * inode_maybe_inc_iversion - increments i_version
- * @inode: inode with the i_version that should be updated
- * @force: increment the counter even if it's not necessary?
- *
- * Every time the inode is modified, the i_version field must be seen to have
- * changed by any observer.
- *
- * If "force" is set or the QUERIED flag is set, then ensure that we increment
- * the value, and clear the queried flag.
- *
- * In the common case where neither is set, then we can return "false" without
- * updating i_version.
- *
- * If this function returns false, and no other metadata has changed, then we
- * can avoid logging the metadata.
- */
-static inline bool
-inode_maybe_inc_iversion(struct inode *inode, bool force)
-{
- u64 cur, old, new;
-
- /*
- * The i_version field is not strictly ordered with any other inode
- * information, but the legacy inode_inc_iversion code used a spinlock
- * to serialize increments.
- *
- * Here, we add full memory barriers to ensure that any de-facto
- * ordering with other info is preserved.
- *
- * This barrier pairs with the barrier in inode_query_iversion()
- */
- smp_mb();
- cur = inode_peek_iversion_raw(inode);
- for (;;) {
- /* If flag is clear then we needn't do anything */
- if (!force && !(cur & I_VERSION_QUERIED))
- return false;
-
- /* Since lowest bit is flag, add 2 to avoid it */
- new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT;
-
- old = atomic64_cmpxchg(&inode->i_version, cur, new);
- if (likely(old == cur))
- break;
- cur = old;
- }
- return true;
-}
-
+bool inode_maybe_inc_iversion(struct inode *inode, bool force);
/**
* inode_inc_iversion - forcibly increment i_version
static inline u64
inode_query_iversion(struct inode *inode)
{
- u64 cur, old, new;
+ u64 cur, new;
cur = inode_peek_iversion_raw(inode);
- for (;;) {
+ do {
/* If flag is already set, then no need to swap */
if (cur & I_VERSION_QUERIED) {
/*
}
new = cur | I_VERSION_QUERIED;
- old = atomic64_cmpxchg(&inode->i_version, cur, new);
- if (likely(old == cur))
- break;
- cur = old;
- }
+ } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new));
return cur >> I_VERSION_QUERIED_SHIFT;
}
#ifdef CONFIG_KASAN
struct kasan_cache {
+#ifdef CONFIG_KASAN_GENERIC
int alloc_meta_offset;
int free_meta_offset;
+#endif
bool is_kmalloc;
};
-slab_flags_t __kasan_never_merge(void);
-static __always_inline slab_flags_t kasan_never_merge(void)
-{
- if (kasan_enabled())
- return __kasan_never_merge();
- return 0;
-}
-
void __kasan_unpoison_range(const void *addr, size_t size);
static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
{
__kasan_unpoison_pages(page, order, init);
}
-void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
- slab_flags_t *flags);
-static __always_inline void kasan_cache_create(struct kmem_cache *cache,
- unsigned int *size, slab_flags_t *flags)
-{
- if (kasan_enabled())
- __kasan_cache_create(cache, size, flags);
-}
-
void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
{
__kasan_cache_create_kmalloc(cache);
}
-size_t __kasan_metadata_size(struct kmem_cache *cache);
-static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
-{
- if (kasan_enabled())
- return __kasan_metadata_size(cache);
- return 0;
-}
-
void __kasan_poison_slab(struct slab *slab);
static __always_inline void kasan_poison_slab(struct slab *slab)
{
#else /* CONFIG_KASAN */
-static inline slab_flags_t kasan_never_merge(void)
-{
- return 0;
-}
static inline void kasan_unpoison_range(const void *address, size_t size) {}
static inline void kasan_poison_pages(struct page *page, unsigned int order,
bool init) {}
static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
bool init) {}
-static inline void kasan_cache_create(struct kmem_cache *cache,
- unsigned int *size,
- slab_flags_t *flags) {}
static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
-static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
static inline void kasan_poison_slab(struct slab *slab) {}
static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
void *object) {}
#ifdef CONFIG_KASAN_GENERIC
+size_t kasan_metadata_size(struct kmem_cache *cache);
+slab_flags_t kasan_never_merge(void);
+void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
+ slab_flags_t *flags);
+
void kasan_cache_shrink(struct kmem_cache *cache);
void kasan_cache_shutdown(struct kmem_cache *cache);
void kasan_record_aux_stack(void *ptr);
#else /* CONFIG_KASAN_GENERIC */
+/* Tag-based KASAN modes do not use per-object metadata. */
+static inline size_t kasan_metadata_size(struct kmem_cache *cache)
+{
+ return 0;
+}
+/* And thus nothing prevents cache merging. */
+static inline slab_flags_t kasan_never_merge(void)
+{
+ return 0;
+}
+/* And no cache-related metadata initialization is required. */
+static inline void kasan_cache_create(struct kmem_cache *cache,
+ unsigned int *size,
+ slab_flags_t *flags) {}
+
static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
static inline void kasan_record_aux_stack(void *ptr) {}
extern bool kexec_in_progress;
int crash_shrink_memory(unsigned long new_size);
-size_t crash_get_memory_size(void);
+ssize_t crash_get_memory_size(void);
#ifndef arch_kexec_protect_crashkres
/*
unsigned long vm_flags);
extern void khugepaged_min_free_kbytes_update(void);
#ifdef CONFIG_SHMEM
-extern void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr);
+extern int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr,
+ bool install_pmd);
#else
-static inline void collapse_pte_mapped_thp(struct mm_struct *mm,
- unsigned long addr)
+static inline int collapse_pte_mapped_thp(struct mm_struct *mm,
+ unsigned long addr, bool install_pmd)
{
+ return 0;
}
#endif
unsigned long vm_flags)
{
}
-static inline void collapse_pte_mapped_thp(struct mm_struct *mm,
- unsigned long addr)
+static inline int collapse_pte_mapped_thp(struct mm_struct *mm,
+ unsigned long addr, bool install_pmd)
{
+ return 0;
}
static inline void khugepaged_min_free_kbytes_update(void)
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN checks to be used for one-off annotations in subsystems.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#ifndef _LINUX_KMSAN_CHECKS_H
+#define _LINUX_KMSAN_CHECKS_H
+
+#include <linux/types.h>
+
+#ifdef CONFIG_KMSAN
+
+/**
+ * kmsan_poison_memory() - Mark the memory range as uninitialized.
+ * @address: address to start with.
+ * @size: size of buffer to poison.
+ * @flags: GFP flags for allocations done by this function.
+ *
+ * Until other data is written to this range, KMSAN will treat it as
+ * uninitialized. Error reports for this memory will reference the call site of
+ * kmsan_poison_memory() as origin.
+ */
+void kmsan_poison_memory(const void *address, size_t size, gfp_t flags);
+
+/**
+ * kmsan_unpoison_memory() - Mark the memory range as initialized.
+ * @address: address to start with.
+ * @size: size of buffer to unpoison.
+ *
+ * Until other data is written to this range, KMSAN will treat it as
+ * initialized.
+ */
+void kmsan_unpoison_memory(const void *address, size_t size);
+
+/**
+ * kmsan_check_memory() - Check the memory range for being initialized.
+ * @address: address to start with.
+ * @size: size of buffer to check.
+ *
+ * If any piece of the given range is marked as uninitialized, KMSAN will report
+ * an error.
+ */
+void kmsan_check_memory(const void *address, size_t size);
+
+/**
+ * kmsan_copy_to_user() - Notify KMSAN about a data transfer to userspace.
+ * @to: destination address in the userspace.
+ * @from: source address in the kernel.
+ * @to_copy: number of bytes to copy.
+ * @left: number of bytes not copied.
+ *
+ * If this is a real userspace data transfer, KMSAN checks the bytes that were
+ * actually copied to ensure there was no information leak. If @to belongs to
+ * the kernel space (which is possible for compat syscalls), KMSAN just copies
+ * the metadata.
+ */
+void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
+ size_t left);
+
+#else
+
+static inline void kmsan_poison_memory(const void *address, size_t size,
+ gfp_t flags)
+{
+}
+static inline void kmsan_unpoison_memory(const void *address, size_t size)
+{
+}
+static inline void kmsan_check_memory(const void *address, size_t size)
+{
+}
+static inline void kmsan_copy_to_user(void __user *to, const void *from,
+ size_t to_copy, size_t left)
+{
+}
+
+#endif
+
+#endif /* _LINUX_KMSAN_CHECKS_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN API for subsystems.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+#ifndef _LINUX_KMSAN_H
+#define _LINUX_KMSAN_H
+
+#include <linux/dma-direction.h>
+#include <linux/gfp.h>
+#include <linux/kmsan-checks.h>
+#include <linux/types.h>
+
+struct page;
+struct kmem_cache;
+struct task_struct;
+struct scatterlist;
+struct urb;
+
+#ifdef CONFIG_KMSAN
+
+/**
+ * kmsan_task_create() - Initialize KMSAN state for the task.
+ * @task: task to initialize.
+ */
+void kmsan_task_create(struct task_struct *task);
+
+/**
+ * kmsan_task_exit() - Notify KMSAN that a task has exited.
+ * @task: task about to finish.
+ */
+void kmsan_task_exit(struct task_struct *task);
+
+/**
+ * kmsan_init_shadow() - Initialize KMSAN shadow at boot time.
+ *
+ * Allocate and initialize KMSAN metadata for early allocations.
+ */
+void __init kmsan_init_shadow(void);
+
+/**
+ * kmsan_init_runtime() - Initialize KMSAN state and enable KMSAN.
+ */
+void __init kmsan_init_runtime(void);
+
+/**
+ * kmsan_memblock_free_pages() - handle freeing of memblock pages.
+ * @page: struct page to free.
+ * @order: order of @page.
+ *
+ * Freed pages are either returned to buddy allocator or held back to be used
+ * as metadata pages.
+ */
+bool __init kmsan_memblock_free_pages(struct page *page, unsigned int order);
+
+/**
+ * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call.
+ * @page: struct page pointer returned by alloc_pages().
+ * @order: order of allocated struct page.
+ * @flags: GFP flags used by alloc_pages()
+ *
+ * KMSAN marks 1<<@order pages starting at @page as uninitialized, unless
+ * @flags contain __GFP_ZERO.
+ */
+void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags);
+
+/**
+ * kmsan_free_page() - Notify KMSAN about a free_pages() call.
+ * @page: struct page pointer passed to free_pages().
+ * @order: order of deallocated struct page.
+ *
+ * KMSAN marks freed memory as uninitialized.
+ */
+void kmsan_free_page(struct page *page, unsigned int order);
+
+/**
+ * kmsan_copy_page_meta() - Copy KMSAN metadata between two pages.
+ * @dst: destination page.
+ * @src: source page.
+ *
+ * KMSAN copies the contents of metadata pages for @src into the metadata pages
+ * for @dst. If @dst has no associated metadata pages, nothing happens.
+ * If @src has no associated metadata pages, @dst metadata pages are unpoisoned.
+ */
+void kmsan_copy_page_meta(struct page *dst, struct page *src);
+
+/**
+ * kmsan_slab_alloc() - Notify KMSAN about a slab allocation.
+ * @s: slab cache the object belongs to.
+ * @object: object pointer.
+ * @flags: GFP flags passed to the allocator.
+ *
+ * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the
+ * newly created object, marking it as initialized or uninitialized.
+ */
+void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
+
+/**
+ * kmsan_slab_free() - Notify KMSAN about a slab deallocation.
+ * @s: slab cache the object belongs to.
+ * @object: object pointer.
+ *
+ * KMSAN marks the freed object as uninitialized.
+ */
+void kmsan_slab_free(struct kmem_cache *s, void *object);
+
+/**
+ * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation.
+ * @ptr: object pointer.
+ * @size: object size.
+ * @flags: GFP flags passed to the allocator.
+ *
+ * Similar to kmsan_slab_alloc(), but for large allocations.
+ */
+void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
+
+/**
+ * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation.
+ * @ptr: object pointer.
+ *
+ * Similar to kmsan_slab_free(), but for large allocations.
+ */
+void kmsan_kfree_large(const void *ptr);
+
+/**
+ * kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap.
+ * @start: start of vmapped range.
+ * @end: end of vmapped range.
+ * @prot: page protection flags used for vmap.
+ * @pages: array of pages.
+ * @page_shift: page_shift passed to vmap_range_noflush().
+ *
+ * KMSAN maps shadow and origin pages of @pages into contiguous ranges in
+ * vmalloc metadata address range.
+ */
+void kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
+ pgprot_t prot, struct page **pages,
+ unsigned int page_shift);
+
+/**
+ * kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap.
+ * @start: start of vunmapped range.
+ * @end: end of vunmapped range.
+ *
+ * KMSAN unmaps the contiguous metadata ranges created by
+ * kmsan_map_kernel_range_noflush().
+ */
+void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end);
+
+/**
+ * kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call.
+ * @addr: range start.
+ * @end: range end.
+ * @phys_addr: physical range start.
+ * @prot: page protection flags used for ioremap_page_range().
+ * @page_shift: page_shift argument passed to vmap_range_noflush().
+ *
+ * KMSAN creates new metadata pages for the physical pages mapped into the
+ * virtual memory.
+ */
+void kmsan_ioremap_page_range(unsigned long addr, unsigned long end,
+ phys_addr_t phys_addr, pgprot_t prot,
+ unsigned int page_shift);
+
+/**
+ * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call.
+ * @start: range start.
+ * @end: range end.
+ *
+ * KMSAN unmaps the metadata pages for the given range and, unlike for
+ * vunmap_page_range(), also deallocates them.
+ */
+void kmsan_iounmap_page_range(unsigned long start, unsigned long end);
+
+/**
+ * kmsan_handle_dma() - Handle a DMA data transfer.
+ * @page: first page of the buffer.
+ * @offset: offset of the buffer within the first page.
+ * @size: buffer size.
+ * @dir: one of possible dma_data_direction values.
+ *
+ * Depending on @direction, KMSAN:
+ * * checks the buffer, if it is copied to device;
+ * * initializes the buffer, if it is copied from device;
+ * * does both, if this is a DMA_BIDIRECTIONAL transfer.
+ */
+void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
+ enum dma_data_direction dir);
+
+/**
+ * kmsan_handle_dma_sg() - Handle a DMA transfer using scatterlist.
+ * @sg: scatterlist holding DMA buffers.
+ * @nents: number of scatterlist entries.
+ * @dir: one of possible dma_data_direction values.
+ *
+ * Depending on @direction, KMSAN:
+ * * checks the buffers in the scatterlist, if they are copied to device;
+ * * initializes the buffers, if they are copied from device;
+ * * does both, if this is a DMA_BIDIRECTIONAL transfer.
+ */
+void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
+ enum dma_data_direction dir);
+
+/**
+ * kmsan_handle_urb() - Handle a USB data transfer.
+ * @urb: struct urb pointer.
+ * @is_out: data transfer direction (true means output to hardware).
+ *
+ * If @is_out is true, KMSAN checks the transfer buffer of @urb. Otherwise,
+ * KMSAN initializes the transfer buffer.
+ */
+void kmsan_handle_urb(const struct urb *urb, bool is_out);
+
+/**
+ * kmsan_unpoison_entry_regs() - Handle pt_regs in low-level entry code.
+ * @regs: struct pt_regs pointer received from assembly code.
+ *
+ * KMSAN unpoisons the contents of the passed pt_regs, preventing potential
+ * false positive reports. Unlike kmsan_unpoison_memory(),
+ * kmsan_unpoison_entry_regs() can be called from the regions where
+ * kmsan_in_runtime() returns true, which is the case in early entry code.
+ */
+void kmsan_unpoison_entry_regs(const struct pt_regs *regs);
+
+#else
+
+static inline void kmsan_init_shadow(void)
+{
+}
+
+static inline void kmsan_init_runtime(void)
+{
+}
+
+static inline bool kmsan_memblock_free_pages(struct page *page,
+ unsigned int order)
+{
+ return true;
+}
+
+static inline void kmsan_task_create(struct task_struct *task)
+{
+}
+
+static inline void kmsan_task_exit(struct task_struct *task)
+{
+}
+
+static inline int kmsan_alloc_page(struct page *page, unsigned int order,
+ gfp_t flags)
+{
+ return 0;
+}
+
+static inline void kmsan_free_page(struct page *page, unsigned int order)
+{
+}
+
+static inline void kmsan_copy_page_meta(struct page *dst, struct page *src)
+{
+}
+
+static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object,
+ gfp_t flags)
+{
+}
+
+static inline void kmsan_slab_free(struct kmem_cache *s, void *object)
+{
+}
+
+static inline void kmsan_kmalloc_large(const void *ptr, size_t size,
+ gfp_t flags)
+{
+}
+
+static inline void kmsan_kfree_large(const void *ptr)
+{
+}
+
+static inline void kmsan_vmap_pages_range_noflush(unsigned long start,
+ unsigned long end,
+ pgprot_t prot,
+ struct page **pages,
+ unsigned int page_shift)
+{
+}
+
+static inline void kmsan_vunmap_range_noflush(unsigned long start,
+ unsigned long end)
+{
+}
+
+static inline void kmsan_ioremap_page_range(unsigned long start,
+ unsigned long end,
+ phys_addr_t phys_addr,
+ pgprot_t prot,
+ unsigned int page_shift)
+{
+}
+
+static inline void kmsan_iounmap_page_range(unsigned long start,
+ unsigned long end)
+{
+}
+
+static inline void kmsan_handle_dma(struct page *page, size_t offset,
+ size_t size, enum dma_data_direction dir)
+{
+}
+
+static inline void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
+ enum dma_data_direction dir)
+{
+}
+
+static inline void kmsan_handle_urb(const struct urb *urb, bool is_out)
+{
+}
+
+static inline void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
+{
+}
+
+#endif
+
+#endif /* _LINUX_KMSAN_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * A minimal header declaring types added by KMSAN to existing kernel structs.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+#ifndef _LINUX_KMSAN_TYPES_H
+#define _LINUX_KMSAN_TYPES_H
+
+/* These constants are defined in the MSan LLVM instrumentation pass. */
+#define KMSAN_RETVAL_SIZE 800
+#define KMSAN_PARAM_SIZE 800
+
+struct kmsan_context_state {
+ char param_tls[KMSAN_PARAM_SIZE];
+ char retval_tls[KMSAN_RETVAL_SIZE];
+ char va_arg_tls[KMSAN_PARAM_SIZE];
+ char va_arg_origin_tls[KMSAN_PARAM_SIZE];
+ u64 va_arg_overflow_size_tls;
+ char param_origin_tls[KMSAN_PARAM_SIZE];
+ u32 retval_origin_tls;
+};
+
+#undef KMSAN_PARAM_SIZE
+#undef KMSAN_RETVAL_SIZE
+
+struct kmsan_ctx {
+ struct kmsan_context_state cstate;
+ int kmsan_in_runtime;
+ bool allow_reporting;
+};
+
+#endif /* _LINUX_KMSAN_TYPES_H */
#include <linux/sched.h>
#include <linux/sched/coredump.h>
-struct stable_node;
-struct mem_cgroup;
-
#ifdef CONFIG_KSM
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
unsigned long end, int advice, unsigned long *vm_flags);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef _LINUX_MAPLE_TREE_H
+#define _LINUX_MAPLE_TREE_H
+/*
+ * Maple Tree - An RCU-safe adaptive tree for storing ranges
+ * Copyright (c) 2018-2022 Oracle
+ * Authors: Liam R. Howlett <Liam.Howlett@Oracle.com>
+ * Matthew Wilcox <willy@infradead.org>
+ */
+
+#include <linux/kernel.h>
+#include <linux/rcupdate.h>
+#include <linux/spinlock.h>
+/* #define CONFIG_MAPLE_RCU_DISABLED */
+/* #define CONFIG_DEBUG_MAPLE_TREE_VERBOSE */
+
+/*
+ * Allocated nodes are mutable until they have been inserted into the tree,
+ * at which time they cannot change their type until they have been removed
+ * from the tree and an RCU grace period has passed.
+ *
+ * Removed nodes have their ->parent set to point to themselves. RCU readers
+ * check ->parent before relying on the value that they loaded from the
+ * slots array. This lets us reuse the slots array for the RCU head.
+ *
+ * Nodes in the tree point to their parent unless bit 0 is set.
+ */
+#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64)
+/* 64bit sizes */
+#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */
+#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */
+#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */
+#define MAPLE_ARANGE64_META_MAX 15 /* Out of range for metadata */
+#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1)
+#else
+/* 32bit sizes */
+#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */
+#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */
+#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */
+#define MAPLE_ARANGE64_META_MAX 31 /* Out of range for metadata */
+#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2)
+#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */
+
+#define MAPLE_NODE_MASK 255UL
+
+/*
+ * The node->parent of the root node has bit 0 set and the rest of the pointer
+ * is a pointer to the tree itself. No more bits are available in this pointer
+ * (on m68k, the data structure may only be 2-byte aligned).
+ *
+ * Internal non-root nodes can only have maple_range_* nodes as parents. The
+ * parent pointer is 256B aligned like all other tree nodes. When storing a 32
+ * or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an
+ * extra bit to store the offset. This extra bit comes from a reuse of the last
+ * bit in the node type. This is possible by using bit 1 to indicate if bit 2
+ * is part of the type or the slot.
+ *
+ * Once the type is decided, the decision of an allocation range type or a range
+ * type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE
+ * flag.
+ *
+ * Node types:
+ * 0x??1 = Root
+ * 0x?00 = 16 bit nodes
+ * 0x010 = 32 bit nodes
+ * 0x110 = 64 bit nodes
+ *
+ * Slot size and location in the parent pointer:
+ * type : slot location
+ * 0x??1 : Root
+ * 0x?00 : 16 bit values, type in 0-1, slot in 2-6
+ * 0x010 : 32 bit values, type in 0-2, slot in 3-6
+ * 0x110 : 64 bit values, type in 0-2, slot in 3-6
+ */
+
+/*
+ * This metadata is used to optimize the gap updating code and in reverse
+ * searching for gaps or any other code that needs to find the end of the data.
+ */
+struct maple_metadata {
+ unsigned char end;
+ unsigned char gap;
+};
+
+/*
+ * Leaf nodes do not store pointers to nodes, they store user data. Users may
+ * store almost any bit pattern. As noted above, the optimisation of storing an
+ * entry at 0 in the root pointer cannot be done for data which have the bottom
+ * two bits set to '10'. We also reserve values with the bottom two bits set to
+ * '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs
+ * return errnos as a negative errno shifted right by two bits and the bottom
+ * two bits set to '10', and while choosing to store these values in the array
+ * is not an error, it may lead to confusion if you're testing for an error with
+ * mas_is_err().
+ *
+ * Non-leaf nodes store the type of the node pointed to (enum maple_type in bits
+ * 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now.
+ *
+ * In regular B-Tree terms, pivots are called keys. The term pivot is used to
+ * indicate that the tree is specifying ranges, Pivots may appear in the
+ * subtree with an entry attached to the value whereas keys are unique to a
+ * specific position of a B-tree. Pivot values are inclusive of the slot with
+ * the same index.
+ */
+
+struct maple_range_64 {
+ struct maple_pnode *parent;
+ unsigned long pivot[MAPLE_RANGE64_SLOTS - 1];
+ union {
+ void __rcu *slot[MAPLE_RANGE64_SLOTS];
+ struct {
+ void __rcu *pad[MAPLE_RANGE64_SLOTS - 1];
+ struct maple_metadata meta;
+ };
+ };
+};
+
+/*
+ * At tree creation time, the user can specify that they're willing to trade off
+ * storing fewer entries in a tree in return for storing more information in
+ * each node.
+ *
+ * The maple tree supports recording the largest range of NULL entries available
+ * in this node, also called gaps. This optimises the tree for allocating a
+ * range.
+ */
+struct maple_arange_64 {
+ struct maple_pnode *parent;
+ unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1];
+ void __rcu *slot[MAPLE_ARANGE64_SLOTS];
+ unsigned long gap[MAPLE_ARANGE64_SLOTS];
+ struct maple_metadata meta;
+};
+
+struct maple_alloc {
+ unsigned long total;
+ unsigned char node_count;
+ unsigned int request_count;
+ struct maple_alloc *slot[MAPLE_ALLOC_SLOTS];
+};
+
+struct maple_topiary {
+ struct maple_pnode *parent;
+ struct maple_enode *next; /* Overlaps the pivot */
+};
+
+enum maple_type {
+ maple_dense,
+ maple_leaf_64,
+ maple_range_64,
+ maple_arange_64,
+};
+
+
+/**
+ * DOC: Maple tree flags
+ *
+ * * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree
+ * * MT_FLAGS_USE_RCU - Operate in RCU mode
+ * * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags
+ * * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value
+ * * MT_FLAGS_LOCK_MASK - How the mt_lock is used
+ * * MT_FLAGS_LOCK_IRQ - Acquired irq-safe
+ * * MT_FLAGS_LOCK_BH - Acquired bh-safe
+ * * MT_FLAGS_LOCK_EXTERN - mt_lock is not used
+ *
+ * MAPLE_HEIGHT_MAX The largest height that can be stored
+ */
+#define MT_FLAGS_ALLOC_RANGE 0x01
+#define MT_FLAGS_USE_RCU 0x02
+#define MT_FLAGS_HEIGHT_OFFSET 0x02
+#define MT_FLAGS_HEIGHT_MASK 0x7C
+#define MT_FLAGS_LOCK_MASK 0x300
+#define MT_FLAGS_LOCK_IRQ 0x100
+#define MT_FLAGS_LOCK_BH 0x200
+#define MT_FLAGS_LOCK_EXTERN 0x300
+
+#define MAPLE_HEIGHT_MAX 31
+
+
+#define MAPLE_NODE_TYPE_MASK 0x0F
+#define MAPLE_NODE_TYPE_SHIFT 0x03
+
+#define MAPLE_RESERVED_RANGE 4096
+
+#ifdef CONFIG_LOCKDEP
+typedef struct lockdep_map *lockdep_map_p;
+#define mt_lock_is_held(mt) lock_is_held(mt->ma_external_lock)
+#define mt_set_external_lock(mt, lock) \
+ (mt)->ma_external_lock = &(lock)->dep_map
+#else
+typedef struct { /* nothing */ } lockdep_map_p;
+#define mt_lock_is_held(mt) 1
+#define mt_set_external_lock(mt, lock) do { } while (0)
+#endif
+
+/*
+ * If the tree contains a single entry at index 0, it is usually stored in
+ * tree->ma_root. To optimise for the page cache, an entry which ends in '00',
+ * '01' or '11' is stored in the root, but an entry which ends in '10' will be
+ * stored in a node. Bits 3-6 are used to store enum maple_type.
+ *
+ * The flags are used both to store some immutable information about this tree
+ * (set at tree creation time) and dynamic information set under the spinlock.
+ *
+ * Another use of flags are to indicate global states of the tree. This is the
+ * case with the MAPLE_USE_RCU flag, which indicates the tree is currently in
+ * RCU mode. This mode was added to allow the tree to reuse nodes instead of
+ * re-allocating and RCU freeing nodes when there is a single user.
+ */
+struct maple_tree {
+ union {
+ spinlock_t ma_lock;
+ lockdep_map_p ma_external_lock;
+ };
+ void __rcu *ma_root;
+ unsigned int ma_flags;
+};
+
+/**
+ * MTREE_INIT() - Initialize a maple tree
+ * @name: The maple tree name
+ * @__flags: The maple tree flags
+ *
+ */
+#define MTREE_INIT(name, __flags) { \
+ .ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), \
+ .ma_flags = __flags, \
+ .ma_root = NULL, \
+}
+
+/**
+ * MTREE_INIT_EXT() - Initialize a maple tree with an external lock.
+ * @name: The tree name
+ * @__flags: The maple tree flags
+ * @__lock: The external lock
+ */
+#ifdef CONFIG_LOCKDEP
+#define MTREE_INIT_EXT(name, __flags, __lock) { \
+ .ma_external_lock = &(__lock).dep_map, \
+ .ma_flags = (__flags), \
+ .ma_root = NULL, \
+}
+#else
+#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags)
+#endif
+
+#define DEFINE_MTREE(name) \
+ struct maple_tree name = MTREE_INIT(name, 0)
+
+#define mtree_lock(mt) spin_lock((&(mt)->ma_lock))
+#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock))
+
+/*
+ * The Maple Tree squeezes various bits in at various points which aren't
+ * necessarily obvious. Usually, this is done by observing that pointers are
+ * N-byte aligned and thus the bottom log_2(N) bits are available for use. We
+ * don't use the high bits of pointers to store additional information because
+ * we don't know what bits are unused on any given architecture.
+ *
+ * Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8
+ * low bits for our own purposes. Nodes are currently of 4 types:
+ * 1. Single pointer (Range is 0-0)
+ * 2. Non-leaf Allocation Range nodes
+ * 3. Non-leaf Range nodes
+ * 4. Leaf Range nodes All nodes consist of a number of node slots,
+ * pivots, and a parent pointer.
+ */
+
+struct maple_node {
+ union {
+ struct {
+ struct maple_pnode *parent;
+ void __rcu *slot[MAPLE_NODE_SLOTS];
+ };
+ struct {
+ void *pad;
+ struct rcu_head rcu;
+ struct maple_enode *piv_parent;
+ unsigned char parent_slot;
+ enum maple_type type;
+ unsigned char slot_len;
+ unsigned int ma_flags;
+ };
+ struct maple_range_64 mr64;
+ struct maple_arange_64 ma64;
+ struct maple_alloc alloc;
+ };
+};
+
+/*
+ * More complicated stores can cause two nodes to become one or three and
+ * potentially alter the height of the tree. Either half of the tree may need
+ * to be rebalanced against the other. The ma_topiary struct is used to track
+ * which nodes have been 'cut' from the tree so that the change can be done
+ * safely at a later date. This is done to support RCU.
+ */
+struct ma_topiary {
+ struct maple_enode *head;
+ struct maple_enode *tail;
+ struct maple_tree *mtree;
+};
+
+void *mtree_load(struct maple_tree *mt, unsigned long index);
+
+int mtree_insert(struct maple_tree *mt, unsigned long index,
+ void *entry, gfp_t gfp);
+int mtree_insert_range(struct maple_tree *mt, unsigned long first,
+ unsigned long last, void *entry, gfp_t gfp);
+int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp,
+ void *entry, unsigned long size, unsigned long min,
+ unsigned long max, gfp_t gfp);
+int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp,
+ void *entry, unsigned long size, unsigned long min,
+ unsigned long max, gfp_t gfp);
+
+int mtree_store_range(struct maple_tree *mt, unsigned long first,
+ unsigned long last, void *entry, gfp_t gfp);
+int mtree_store(struct maple_tree *mt, unsigned long index,
+ void *entry, gfp_t gfp);
+void *mtree_erase(struct maple_tree *mt, unsigned long index);
+
+void mtree_destroy(struct maple_tree *mt);
+void __mt_destroy(struct maple_tree *mt);
+
+/**
+ * mtree_empty() - Determine if a tree has any present entries.
+ * @mt: Maple Tree.
+ *
+ * Context: Any context.
+ * Return: %true if the tree contains only NULL pointers.
+ */
+static inline bool mtree_empty(const struct maple_tree *mt)
+{
+ return mt->ma_root == NULL;
+}
+
+/* Advanced API */
+
+/*
+ * The maple state is defined in the struct ma_state and is used to keep track
+ * of information during operations, and even between operations when using the
+ * advanced API.
+ *
+ * If state->node has bit 0 set then it references a tree location which is not
+ * a node (eg the root). If bit 1 is set, the rest of the bits are a negative
+ * errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the
+ * node type.
+ *
+ * state->alloc either has a request number of nodes or an allocated node. If
+ * stat->alloc has a requested number of nodes, the first bit will be set (0x1)
+ * and the remaining bits are the value. If state->alloc is a node, then the
+ * node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for
+ * storing more allocated nodes, a total number of nodes allocated, and the
+ * node_count in this node. node_count is the number of allocated nodes in this
+ * node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further
+ * nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc
+ * by removing a node from the state->alloc node until state->alloc->node_count
+ * is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted
+ * to state->alloc. Nodes are pushed onto state->alloc by putting the current
+ * state->alloc into the pushed node's slot[0].
+ *
+ * The state also contains the implied min/max of the state->node, the depth of
+ * this search, and the offset. The implied min/max are either from the parent
+ * node or are 0-oo for the root node. The depth is incremented or decremented
+ * every time a node is walked down or up. The offset is the slot/pivot of
+ * interest in the node - either for reading or writing.
+ *
+ * When returning a value the maple state index and last respectively contain
+ * the start and end of the range for the entry. Ranges are inclusive in the
+ * Maple Tree.
+ */
+struct ma_state {
+ struct maple_tree *tree; /* The tree we're operating in */
+ unsigned long index; /* The index we're operating on - range start */
+ unsigned long last; /* The last index we're operating on - range end */
+ struct maple_enode *node; /* The node containing this entry */
+ unsigned long min; /* The minimum index of this node - implied pivot min */
+ unsigned long max; /* The maximum index of this node - implied pivot max */
+ struct maple_alloc *alloc; /* Allocated nodes for this operation */
+ unsigned char depth; /* depth of tree descent during write */
+ unsigned char offset;
+ unsigned char mas_flags;
+};
+
+struct ma_wr_state {
+ struct ma_state *mas;
+ struct maple_node *node; /* Decoded mas->node */
+ unsigned long r_min; /* range min */
+ unsigned long r_max; /* range max */
+ enum maple_type type; /* mas->node type */
+ unsigned char offset_end; /* The offset where the write ends */
+ unsigned char node_end; /* mas->node end */
+ unsigned long *pivots; /* mas->node->pivots pointer */
+ unsigned long end_piv; /* The pivot at the offset end */
+ void __rcu **slots; /* mas->node->slots pointer */
+ void *entry; /* The entry to write */
+ void *content; /* The existing entry that is being overwritten */
+};
+
+#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock))
+#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock))
+
+
+/*
+ * Special values for ma_state.node.
+ * MAS_START means we have not searched the tree.
+ * MAS_ROOT means we have searched the tree and the entry we found lives in
+ * the root of the tree (ie it has index 0, length 1 and is the only entry in
+ * the tree).
+ * MAS_NONE means we have searched the tree and there is no node in the
+ * tree for this entry. For example, we searched for index 1 in an empty
+ * tree. Or we have a tree which points to a full leaf node and we
+ * searched for an entry which is larger than can be contained in that
+ * leaf node.
+ * MA_ERROR represents an errno. After dropping the lock and attempting
+ * to resolve the error, the walk would have to be restarted from the
+ * top of the tree as the tree may have been modified.
+ */
+#define MAS_START ((struct maple_enode *)1UL)
+#define MAS_ROOT ((struct maple_enode *)5UL)
+#define MAS_NONE ((struct maple_enode *)9UL)
+#define MAS_PAUSE ((struct maple_enode *)17UL)
+#define MA_ERROR(err) \
+ ((struct maple_enode *)(((unsigned long)err << 2) | 2UL))
+
+#define MA_STATE(name, mt, first, end) \
+ struct ma_state name = { \
+ .tree = mt, \
+ .index = first, \
+ .last = end, \
+ .node = MAS_START, \
+ .min = 0, \
+ .max = ULONG_MAX, \
+ .alloc = NULL, \
+ }
+
+#define MA_WR_STATE(name, ma_state, wr_entry) \
+ struct ma_wr_state name = { \
+ .mas = ma_state, \
+ .content = NULL, \
+ .entry = wr_entry, \
+ }
+
+#define MA_TOPIARY(name, tree) \
+ struct ma_topiary name = { \
+ .head = NULL, \
+ .tail = NULL, \
+ .mtree = tree, \
+ }
+
+void *mas_walk(struct ma_state *mas);
+void *mas_store(struct ma_state *mas, void *entry);
+void *mas_erase(struct ma_state *mas);
+int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp);
+void mas_store_prealloc(struct ma_state *mas, void *entry);
+void *mas_find(struct ma_state *mas, unsigned long max);
+void *mas_find_rev(struct ma_state *mas, unsigned long min);
+int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp);
+bool mas_is_err(struct ma_state *mas);
+
+bool mas_nomem(struct ma_state *mas, gfp_t gfp);
+void mas_pause(struct ma_state *mas);
+void maple_tree_init(void);
+void mas_destroy(struct ma_state *mas);
+int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries);
+
+void *mas_prev(struct ma_state *mas, unsigned long min);
+void *mas_next(struct ma_state *mas, unsigned long max);
+
+int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max,
+ unsigned long size);
+
+/* Checks if a mas has not found anything */
+static inline bool mas_is_none(struct ma_state *mas)
+{
+ return mas->node == MAS_NONE;
+}
+
+/* Checks if a mas has been paused */
+static inline bool mas_is_paused(struct ma_state *mas)
+{
+ return mas->node == MAS_PAUSE;
+}
+
+void mas_dup_tree(struct ma_state *oldmas, struct ma_state *mas);
+void mas_dup_store(struct ma_state *mas, void *entry);
+
+/*
+ * This finds an empty area from the highest address to the lowest.
+ * AKA "Topdown" version,
+ */
+int mas_empty_area_rev(struct ma_state *mas, unsigned long min,
+ unsigned long max, unsigned long size);
+/**
+ * mas_reset() - Reset a Maple Tree operation state.
+ * @mas: Maple Tree operation state.
+ *
+ * Resets the error or walk state of the @mas so future walks of the
+ * array will start from the root. Use this if you have dropped the
+ * lock and want to reuse the ma_state.
+ *
+ * Context: Any context.
+ */
+static inline void mas_reset(struct ma_state *mas)
+{
+ mas->node = MAS_START;
+}
+
+/**
+ * mas_for_each() - Iterate over a range of the maple tree.
+ * @__mas: Maple Tree operation state (maple_state)
+ * @__entry: Entry retrieved from the tree
+ * @__max: maximum index to retrieve from the tree
+ *
+ * When returned, mas->index and mas->last will hold the entire range for the
+ * entry.
+ *
+ * Note: may return the zero entry.
+ *
+ */
+#define mas_for_each(__mas, __entry, __max) \
+ while (((__entry) = mas_find((__mas), (__max))) != NULL)
+
+
+/**
+ * mas_set_range() - Set up Maple Tree operation state for a different index.
+ * @mas: Maple Tree operation state.
+ * @start: New start of range in the Maple Tree.
+ * @last: New end of range in the Maple Tree.
+ *
+ * Move the operation state to refer to a different range. This will
+ * have the effect of starting a walk from the top; see mas_next()
+ * to move to an adjacent index.
+ */
+static inline
+void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last)
+{
+ mas->index = start;
+ mas->last = last;
+ mas->node = MAS_START;
+}
+
+/**
+ * mas_set() - Set up Maple Tree operation state for a different index.
+ * @mas: Maple Tree operation state.
+ * @index: New index into the Maple Tree.
+ *
+ * Move the operation state to refer to a different index. This will
+ * have the effect of starting a walk from the top; see mas_next()
+ * to move to an adjacent index.
+ */
+static inline void mas_set(struct ma_state *mas, unsigned long index)
+{
+
+ mas_set_range(mas, index, index);
+}
+
+static inline bool mt_external_lock(const struct maple_tree *mt)
+{
+ return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN;
+}
+
+/**
+ * mt_init_flags() - Initialise an empty maple tree with flags.
+ * @mt: Maple Tree
+ * @flags: maple tree flags.
+ *
+ * If you need to initialise a Maple Tree with special flags (eg, an
+ * allocation tree), use this function.
+ *
+ * Context: Any context.
+ */
+static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags)
+{
+ mt->ma_flags = flags;
+ if (!mt_external_lock(mt))
+ spin_lock_init(&mt->ma_lock);
+ rcu_assign_pointer(mt->ma_root, NULL);
+}
+
+/**
+ * mt_init() - Initialise an empty maple tree.
+ * @mt: Maple Tree
+ *
+ * An empty Maple Tree.
+ *
+ * Context: Any context.
+ */
+static inline void mt_init(struct maple_tree *mt)
+{
+ mt_init_flags(mt, 0);
+}
+
+static inline bool mt_in_rcu(struct maple_tree *mt)
+{
+#ifdef CONFIG_MAPLE_RCU_DISABLED
+ return false;
+#endif
+ return mt->ma_flags & MT_FLAGS_USE_RCU;
+}
+
+/**
+ * mt_clear_in_rcu() - Switch the tree to non-RCU mode.
+ * @mt: The Maple Tree
+ */
+static inline void mt_clear_in_rcu(struct maple_tree *mt)
+{
+ if (!mt_in_rcu(mt))
+ return;
+
+ if (mt_external_lock(mt)) {
+ BUG_ON(!mt_lock_is_held(mt));
+ mt->ma_flags &= ~MT_FLAGS_USE_RCU;
+ } else {
+ mtree_lock(mt);
+ mt->ma_flags &= ~MT_FLAGS_USE_RCU;
+ mtree_unlock(mt);
+ }
+}
+
+/**
+ * mt_set_in_rcu() - Switch the tree to RCU safe mode.
+ * @mt: The Maple Tree
+ */
+static inline void mt_set_in_rcu(struct maple_tree *mt)
+{
+ if (mt_in_rcu(mt))
+ return;
+
+ if (mt_external_lock(mt)) {
+ BUG_ON(!mt_lock_is_held(mt));
+ mt->ma_flags |= MT_FLAGS_USE_RCU;
+ } else {
+ mtree_lock(mt);
+ mt->ma_flags |= MT_FLAGS_USE_RCU;
+ mtree_unlock(mt);
+ }
+}
+
+void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max);
+void *mt_find_after(struct maple_tree *mt, unsigned long *index,
+ unsigned long max);
+void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min);
+void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max);
+
+/**
+ * mt_for_each - Iterate over each entry starting at index until max.
+ * @__tree: The Maple Tree
+ * @__entry: The current entry
+ * @__index: The index to update to track the location in the tree
+ * @__max: The maximum limit for @index
+ *
+ * Note: Will not return the zero entry.
+ */
+#define mt_for_each(__tree, __entry, __index, __max) \
+ for (__entry = mt_find(__tree, &(__index), __max); \
+ __entry; __entry = mt_find_after(__tree, &(__index), __max))
+
+
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+extern atomic_t maple_tree_tests_run;
+extern atomic_t maple_tree_tests_passed;
+
+void mt_dump(const struct maple_tree *mt);
+void mt_validate(struct maple_tree *mt);
+#define MT_BUG_ON(__tree, __x) do { \
+ atomic_inc(&maple_tree_tests_run); \
+ if (__x) { \
+ pr_info("BUG at %s:%d (%u)\n", \
+ __func__, __LINE__, __x); \
+ mt_dump(__tree); \
+ pr_info("Pass: %u Run:%u\n", \
+ atomic_read(&maple_tree_tests_passed), \
+ atomic_read(&maple_tree_tests_run)); \
+ dump_stack(); \
+ } else { \
+ atomic_inc(&maple_tree_tests_passed); \
+ } \
+} while (0)
+#else
+#define MT_BUG_ON(__tree, __x) BUG_ON(__x)
+#endif /* CONFIG_DEBUG_MAPLE_TREE */
+
+#endif /*_LINUX_MAPLE_TREE_H */
#ifndef MDEV_H
#define MDEV_H
+#include <linux/device.h>
+#include <linux/uuid.h>
+
struct mdev_type;
struct mdev_device {
bool active;
};
-static inline struct mdev_device *to_mdev_device(struct device *dev)
-{
- return container_of(dev, struct mdev_device, dev);
-}
+struct mdev_type {
+ /* set by the driver before calling mdev_register parent: */
+ const char *sysfs_name;
+ const char *pretty_name;
-unsigned int mdev_get_type_group_id(struct mdev_device *mdev);
-unsigned int mtype_get_type_group_id(struct mdev_type *mtype);
-struct device *mtype_get_parent_dev(struct mdev_type *mtype);
+ /* set by the core, can be used drivers */
+ struct mdev_parent *parent;
-/* interface for exporting mdev supported type attributes */
-struct mdev_type_attribute {
- struct attribute attr;
- ssize_t (*show)(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf);
- ssize_t (*store)(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, const char *buf,
- size_t count);
+ /* internal only */
+ struct kobject kobj;
+ struct kobject *devices_kobj;
};
-#define MDEV_TYPE_ATTR(_name, _mode, _show, _store) \
-struct mdev_type_attribute mdev_type_attr_##_name = \
- __ATTR(_name, _mode, _show, _store)
-#define MDEV_TYPE_ATTR_RW(_name) \
- struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RW(_name)
-#define MDEV_TYPE_ATTR_RO(_name) \
- struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RO(_name)
-#define MDEV_TYPE_ATTR_WO(_name) \
- struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_WO(_name)
+/* embedded into the struct device that the mdev devices hang off */
+struct mdev_parent {
+ struct device *dev;
+ struct mdev_driver *mdev_driver;
+ struct kset *mdev_types_kset;
+ /* Synchronize device creation/removal with parent unregistration */
+ struct rw_semaphore unreg_sem;
+ struct mdev_type **types;
+ unsigned int nr_types;
+ atomic_t available_instances;
+};
+
+static inline struct mdev_device *to_mdev_device(struct device *dev)
+{
+ return container_of(dev, struct mdev_device, dev);
+}
/**
* struct mdev_driver - Mediated device driver
+ * @device_api: string to return for the device_api sysfs
+ * @max_instances: maximum number of instances supported (optional)
* @probe: called when new device created
* @remove: called when device removed
- * @supported_type_groups: Attributes to define supported types. It is mandatory
- * to provide supported types.
+ * @get_available: Return the max number of instances that can be created
+ * @show_description: Print a description of the mtype
* @driver: device driver structure
- *
**/
struct mdev_driver {
+ const char *device_api;
+ unsigned int max_instances;
int (*probe)(struct mdev_device *dev);
void (*remove)(struct mdev_device *dev);
- struct attribute_group **supported_type_groups;
+ unsigned int (*get_available)(struct mdev_type *mtype);
+ ssize_t (*show_description)(struct mdev_type *mtype, char *buf);
struct device_driver driver;
};
-extern struct bus_type mdev_bus_type;
-
-int mdev_register_device(struct device *dev, struct mdev_driver *mdev_driver);
-void mdev_unregister_device(struct device *dev);
+int mdev_register_parent(struct mdev_parent *parent, struct device *dev,
+ struct mdev_driver *mdev_driver, struct mdev_type **types,
+ unsigned int nr_types);
+void mdev_unregister_parent(struct mdev_parent *parent);
int mdev_register_driver(struct mdev_driver *drv);
void mdev_unregister_driver(struct mdev_driver *drv);
-struct device *mdev_parent_dev(struct mdev_device *mdev);
static inline struct device *mdev_dev(struct mdev_device *mdev)
{
return &mdev->dev;
}
-static inline struct mdev_device *mdev_from_dev(struct device *dev)
-{
- return dev->bus == &mdev_bus_type ? to_mdev_device(dev) : NULL;
-}
#endif /* MDEV_H */
MEM_CGROUP_NTARGETS,
};
-struct memcg_vmstats_percpu {
- /* Local (CPU and cgroup) page state & events */
- long state[MEMCG_NR_STAT];
- unsigned long events[NR_VM_EVENT_ITEMS];
-
- /* Delta calculation for lockless upward propagation */
- long state_prev[MEMCG_NR_STAT];
- unsigned long events_prev[NR_VM_EVENT_ITEMS];
-
- /* Cgroup1: threshold notifications & softlimit tree updates */
- unsigned long nr_page_events;
- unsigned long targets[MEM_CGROUP_NTARGETS];
-};
-
-struct memcg_vmstats {
- /* Aggregated (CPU and subtree) page state & events */
- long state[MEMCG_NR_STAT];
- unsigned long events[NR_VM_EVENT_ITEMS];
-
- /* Pending child counts during tree propagation */
- long state_pending[MEMCG_NR_STAT];
- unsigned long events_pending[NR_VM_EVENT_ITEMS];
-};
+struct memcg_vmstats_percpu;
+struct memcg_vmstats;
struct mem_cgroup_reclaim_iter {
struct mem_cgroup *position;
struct mem_cgroup_threshold_ary *spare;
};
-#if defined(CONFIG_SMP)
-struct memcg_padding {
- char x[0];
-} ____cacheline_internodealigned_in_smp;
-#define MEMCG_PADDING(name) struct memcg_padding name
-#else
-#define MEMCG_PADDING(name)
-#endif
-
/*
* Remember four most recent foreign writebacks with dirty pages in this
* cgroup. Inode sharing is expected to be uncommon and, even if we miss
spinlock_t move_lock;
unsigned long move_lock_flags;
- MEMCG_PADDING(_pad1_);
+ CACHELINE_PADDING(_pad1_);
/* memory.stat */
- struct memcg_vmstats vmstats;
+ struct memcg_vmstats *vmstats;
/* memory.events */
atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS];
struct list_head objcg_list;
#endif
- MEMCG_PADDING(_pad2_);
+ CACHELINE_PADDING(_pad2_);
/*
* set > 0 if pages under this cgroup are moving to other cgroup.
struct deferred_split deferred_split_queue;
#endif
+#ifdef CONFIG_LRU_GEN
+ /* per-memcg mm_struct list */
+ struct lru_gen_mm_list mm_list;
+#endif
+
struct mem_cgroup_per_node *nodeinfo[];
};
/*
- * size of first charge trial. "32" comes from vmscan.c's magic value.
- * TODO: maybe necessary to use big numbers in big irons.
+ * size of first charge trial.
+ * TODO: maybe necessary to use big numbers in big irons or dynamic based of the
+ * workload.
*/
-#define MEMCG_CHARGE_BATCH 32U
+#define MEMCG_CHARGE_BATCH 64U
extern struct mem_cgroup *root_mem_cgroup;
* - LRU isolation
* - lock_page_memcg()
* - exclusive reference
+ * - mem_cgroup_trylock_pages()
*
* For a kmem folio a caller should hold an rcu read lock to protect memcg
* associated with a kmem folio from being released.
* - LRU isolation
* - lock_page_memcg()
* - exclusive reference
+ * - mem_cgroup_trylock_pages()
*
* For a kmem page a caller should hold an rcu read lock to protect memcg
* associated with a kmem page from being released.
return __mem_cgroup_charge(folio, mm, gfp);
}
-int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
+int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
gfp_t gfp, swp_entry_t entry);
void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
+/* try to stablize folio_memcg() for all the pages in a memcg */
+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
+{
+ rcu_read_lock();
+
+ if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account))
+ return true;
+
+ rcu_read_unlock();
+ return false;
+}
+
+static inline void mem_cgroup_unlock_pages(void)
+{
+ rcu_read_unlock();
+}
+
/* idx can be of type enum memcg_stat_item or node_stat_item */
static inline void mod_memcg_state(struct mem_cgroup *memcg,
int idx, int val)
rcu_read_unlock();
}
-static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
-{
- long x = READ_ONCE(memcg->vmstats.state[idx]);
-#ifdef CONFIG_SMP
- if (x < 0)
- x = 0;
-#endif
- return x;
-}
+unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx);
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
enum node_stat_item idx)
return 0;
}
-static inline int mem_cgroup_swapin_charge_page(struct page *page,
+static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,
struct mm_struct *mm, gfp_t gfp, swp_entry_t entry)
{
return 0;
{
}
+static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg)
+{
+ /* to match folio_memcg_rcu() */
+ rcu_read_lock();
+ return true;
+}
+
+static inline void mem_cgroup_unlock_pages(void)
+{
+ rcu_read_unlock();
+}
+
static inline void mem_cgroup_handle_over_high(void)
{
}
{
struct mem_cgroup *memcg;
- if (mem_cgroup_kmem_disabled())
+ if (!memcg_kmem_enabled())
return;
rcu_read_lock();
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_MEMORY_TIERS_H
+#define _LINUX_MEMORY_TIERS_H
+
+#include <linux/types.h>
+#include <linux/nodemask.h>
+#include <linux/kref.h>
+#include <linux/mmzone.h>
+/*
+ * Each tier cover a abstrace distance chunk size of 128
+ */
+#define MEMTIER_CHUNK_BITS 7
+#define MEMTIER_CHUNK_SIZE (1 << MEMTIER_CHUNK_BITS)
+/*
+ * Smaller abstract distance values imply faster (higher) memory tiers. Offset
+ * the DRAM adistance so that we can accommodate devices with a slightly lower
+ * adistance value (slightly faster) than default DRAM adistance to be part of
+ * the same memory tier.
+ */
+#define MEMTIER_ADISTANCE_DRAM ((4 * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1))
+#define MEMTIER_HOTPLUG_PRIO 100
+
+struct memory_tier;
+struct memory_dev_type {
+ /* list of memory types that are part of same tier as this type */
+ struct list_head tier_sibiling;
+ /* abstract distance for this specific memory type */
+ int adistance;
+ /* Nodes of same abstract distance */
+ nodemask_t nodes;
+ struct kref kref;
+};
+
+#ifdef CONFIG_NUMA
+extern bool numa_demotion_enabled;
+struct memory_dev_type *alloc_memory_type(int adistance);
+void destroy_memory_type(struct memory_dev_type *memtype);
+void init_node_memory_type(int node, struct memory_dev_type *default_type);
+void clear_node_memory_type(int node, struct memory_dev_type *memtype);
+#ifdef CONFIG_MIGRATION
+int next_demotion_node(int node);
+void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets);
+bool node_is_toptier(int node);
+#else
+static inline int next_demotion_node(int node)
+{
+ return NUMA_NO_NODE;
+}
+
+static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets)
+{
+ *targets = NODE_MASK_NONE;
+}
+
+static inline bool node_is_toptier(int node)
+{
+ return true;
+}
+#endif
+
+#else
+
+#define numa_demotion_enabled false
+/*
+ * CONFIG_NUMA implementation returns non NULL error.
+ */
+static inline struct memory_dev_type *alloc_memory_type(int adistance)
+{
+ return NULL;
+}
+
+static inline void destroy_memory_type(struct memory_dev_type *memtype)
+{
+
+}
+
+static inline void init_node_memory_type(int node, struct memory_dev_type *default_type)
+{
+
+}
+
+static inline void clear_node_memory_type(int node, struct memory_dev_type *memtype)
+{
+
+}
+
+static inline int next_demotion_node(int node)
+{
+ return NUMA_NO_NODE;
+}
+
+static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets)
+{
+ *targets = NODE_MASK_NONE;
+}
+
+static inline bool node_is_toptier(int node)
+{
+ return true;
+}
+#endif /* CONFIG_NUMA */
+#endif /* _LINUX_MEMORY_TIERS_H */
struct zone;
struct pglist_data;
struct mem_section;
-struct memory_block;
struct memory_group;
struct resource;
struct vmem_altmap;
({ \
memblock_alloc(sizeof(*pgdat), SMP_CACHE_BYTES); \
})
-/*
- * This definition is just for error path in node hotadd.
- * For node hotremove, we have to replace this.
- */
-#define generic_free_nodedata(pgdat) kfree(pgdat)
extern pg_data_t *node_data[];
static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat)
BUG();
return NULL;
}
-static inline void generic_free_nodedata(pg_data_t *pgdat)
-{
-}
static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat)
{
}
void mem_hotplug_begin(void);
void mem_hotplug_done(void);
+/* See kswapd_is_running() */
+static inline void pgdat_kswapd_lock(pg_data_t *pgdat)
+{
+ mutex_lock(&pgdat->kswapd_lock);
+}
+
+static inline void pgdat_kswapd_unlock(pg_data_t *pgdat)
+{
+ mutex_unlock(&pgdat->kswapd_lock);
+}
+
+static inline void pgdat_kswapd_lock_init(pg_data_t *pgdat)
+{
+ mutex_init(&pgdat->kswapd_lock);
+}
+
#else /* ! CONFIG_MEMORY_HOTPLUG */
#define pfn_to_online_page(pfn) \
({ \
{
return false;
}
+
+static inline void pgdat_kswapd_lock(pg_data_t *pgdat) {}
+static inline void pgdat_kswapd_unlock(pg_data_t *pgdat) {}
+static inline void pgdat_kswapd_lock_init(pg_data_t *pgdat) {}
#endif /* ! CONFIG_MEMORY_HOTPLUG */
/*
extern void remove_pfn_range_from_zone(struct zone *zone,
unsigned long start_pfn,
unsigned long nr_pages);
-extern bool is_memblock_offlined(struct memory_block *mem);
extern int sparse_add_section(int nid, unsigned long pfn,
unsigned long nr_pages, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap);
const nodemask_t *mask);
extern nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy);
-static inline nodemask_t *policy_nodemask_current(gfp_t gfp)
-{
- struct mempolicy *mpol = get_task_policy(current);
-
- return policy_nodemask(gfp, mpol);
-}
-
extern unsigned int mempolicy_slab_node(void);
extern enum zone_type policy_zone;
return (pol->mode == MPOL_PREFERRED_MANY);
}
+extern bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone);
#else
{
}
-static inline nodemask_t *policy_nodemask_current(gfp_t gfp)
-{
- return NULL;
-}
-
static inline bool mpol_is_preferred_many(struct mempolicy *pol)
{
return false;
}
#ifdef CONFIG_ZONE_DEVICE
+void zone_device_page_init(struct page *page);
void *memremap_pages(struct dev_pagemap *pgmap, int nid);
void memunmap_pages(struct dev_pagemap *pgmap);
void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);
#ifdef CONFIG_MIGRATION
extern void putback_movable_pages(struct list_head *l);
+int migrate_folio_extra(struct address_space *mapping, struct folio *dst,
+ struct folio *src, enum migrate_mode mode, int extra_count);
int migrate_folio(struct address_space *mapping, struct folio *dst,
struct folio *src, enum migrate_mode mode);
extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
#endif /* CONFIG_MIGRATION */
-#if defined(CONFIG_MIGRATION) && defined(CONFIG_NUMA)
-extern void set_migration_target_nodes(void);
-extern void migrate_on_reclaim_init(void);
-extern bool numa_demotion_enabled;
-extern int next_demotion_node(int node);
-#else
-static inline void set_migration_target_nodes(void) {}
-static inline void migrate_on_reclaim_init(void) {}
-static inline int next_demotion_node(int node)
-{
- return NUMA_NO_NODE;
-}
-#define numa_demotion_enabled false
-#endif
-
#ifdef CONFIG_COMPACTION
bool PageMovable(struct page *page);
void __SetPageMovable(struct page *page, const struct movable_operations *ops);
*/
void *pgmap_owner;
unsigned long flags;
+
+ /*
+ * Set to vmf->page if this is being called to migrate a page as part of
+ * a migrate_to_ram() callback.
+ */
+ struct page *fault_page;
};
int migrate_vma_setup(struct migrate_vma *args);
void migrate_vma_pages(struct migrate_vma *migrate);
void migrate_vma_finalize(struct migrate_vma *migrate);
+int migrate_device_range(unsigned long *src_pfns, unsigned long start,
+ unsigned long npages);
+void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns,
+ unsigned long npages);
+void migrate_device_finalize(unsigned long *src_pfns,
+ unsigned long *dst_pfns, unsigned long npages);
+
#endif /* CONFIG_MIGRATION */
#endif /* _LINUX_MIGRATE_H */
return vma->vm_flags & VM_ACCESS_FLAGS;
}
+static inline
+struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max)
+{
+ return mas_find(&vmi->mas, max);
+}
+
+static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi)
+{
+ /*
+ * Uses vma_find() to get the first VMA when the iterator starts.
+ * Calling mas_next() could skip the first entry.
+ */
+ return vma_find(vmi, ULONG_MAX);
+}
+
+static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi)
+{
+ return mas_prev(&vmi->mas, 0);
+}
+
+static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
+{
+ return vmi->mas.index;
+}
+
+#define for_each_vma(__vmi, __vma) \
+ while (((__vma) = vma_next(&(__vmi))) != NULL)
+
+/* The MM code likes to work with exclusive end addresses */
+#define for_each_vma_range(__vmi, __vma, __end) \
+ while (((__vma) = vma_find(&(__vmi), (__end) - 1)) != NULL)
+
#ifdef CONFIG_SHMEM
/*
* The vma_is_shmem is not inline because it is used only by slow
*/
static inline unsigned int folio_order(struct folio *folio)
{
- return compound_order(&folio->page);
+ if (!folio_test_large(folio))
+ return 0;
+ return folio->_folio_order;
}
#include <linux/huge_mm.h>
}
#ifdef CONFIG_NUMA_BALANCING
+/* page access time bits needs to hold at least 4 seconds */
+#define PAGE_ACCESS_TIME_MIN_BITS 12
+#if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS
+#define PAGE_ACCESS_TIME_BUCKETS \
+ (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT)
+#else
+#define PAGE_ACCESS_TIME_BUCKETS 0
+#endif
+
+#define PAGE_ACCESS_TIME_MASK \
+ (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS)
+
static inline int cpu_pid_to_cpupid(int cpu, int pid)
{
return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
}
#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
+
+static inline int xchg_page_access_time(struct page *page, int time)
+{
+ int last_time;
+
+ last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS);
+ return last_time << PAGE_ACCESS_TIME_BUCKETS;
+}
#else /* !CONFIG_NUMA_BALANCING */
static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
{
return page_to_nid(page); /* XXX */
}
+static inline int xchg_page_access_time(struct page *page, int time)
+{
+ return 0;
+}
+
static inline int page_cpupid_last(struct page *page)
{
return page_to_nid(page); /* XXX */
return page_to_pfn(&folio->page);
}
+static inline struct folio *pfn_folio(unsigned long pfn)
+{
+ return page_folio(pfn_to_page(pfn));
+}
+
static inline atomic_t *folio_pincount_ptr(struct folio *folio)
{
return &folio_page(folio, 1)->compound_pincount;
*/
static inline long folio_nr_pages(struct folio *folio)
{
- return compound_nr(&folio->page);
+ if (!folio_test_large(folio))
+ return 1;
+#ifdef CONFIG_64BIT
+ return folio->_folio_nr_pages;
+#else
+ return 1L << folio->_folio_order;
+#endif
}
/**
*/
#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
-extern void show_free_areas(unsigned int flags, nodemask_t *nodemask);
+extern void __show_free_areas(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);
+static void __maybe_unused show_free_areas(unsigned int flags, nodemask_t *nodemask)
+{
+ __show_free_areas(flags, nodemask, MAX_NR_ZONES - 1);
+}
#ifdef CONFIG_MMU
extern bool can_do_mlock(void);
unsigned long size);
void zap_page_range(struct vm_area_struct *vma, unsigned long address,
unsigned long size);
-void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
- unsigned long start, unsigned long end);
+void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt,
+ struct vm_area_struct *start_vma, unsigned long start,
+ unsigned long end);
struct mmu_notifier_range;
unsigned long end_pfn);
extern void get_pfn_range_for_nid(unsigned int nid,
unsigned long *start_pfn, unsigned long *end_pfn);
-extern unsigned long find_min_pfn_with_active_regions(void);
#ifndef CONFIG_NUMA
static inline int early_pfn_to_nid(unsigned long pfn)
extern int __meminit init_per_zone_wmark_min(void);
extern void mem_init(void);
extern void __init mmap_init(void);
-extern void show_mem(unsigned int flags, nodemask_t *nodemask);
+
+extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);
+static inline void show_mem(unsigned int flags, nodemask_t *nodemask)
+{
+ __show_mem(flags, nodemask, MAX_NR_ZONES - 1);
+}
extern long si_mem_available(void);
extern void si_meminfo(struct sysinfo * val);
extern void si_meminfo_node(struct sysinfo *val, int nid);
extern int split_vma(struct mm_struct *, struct vm_area_struct *,
unsigned long addr, int new_below);
extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
-extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
- struct rb_node **, struct rb_node *);
extern void unlink_file_vma(struct vm_area_struct *);
extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
unsigned long addr, unsigned long len, pgoff_t pgoff,
bool *need_rmap_locks);
extern void exit_mmap(struct mm_struct *);
+void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas);
+void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas);
+
static inline int check_data_rlimit(unsigned long rlim,
unsigned long new,
unsigned long start,
extern unsigned long do_mmap(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot, unsigned long flags,
unsigned long pgoff, unsigned long *populate, struct list_head *uf);
-extern int __do_munmap(struct mm_struct *, unsigned long, size_t,
- struct list_head *uf, bool downgrade);
+extern int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm,
+ unsigned long start, size_t len, struct list_head *uf,
+ bool downgrade);
extern int do_munmap(struct mm_struct *, unsigned long, size_t,
struct list_head *uf);
extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);
extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
struct vm_area_struct **pprev);
-/**
- * find_vma_intersection() - Look up the first VMA which intersects the interval
- * @mm: The process address space.
- * @start_addr: The inclusive start user address.
- * @end_addr: The exclusive end user address.
- *
- * Returns: The first VMA within the provided range, %NULL otherwise. Assumes
- * start_addr < end_addr.
+/*
+ * Look up the first VMA which intersects the interval [start_addr, end_addr)
+ * NULL if none. Assume start_addr < end_addr.
*/
-static inline
struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
- unsigned long start_addr,
- unsigned long end_addr)
-{
- struct vm_area_struct *vma = find_vma(mm, start_addr);
-
- if (vma && end_addr <= vma->vm_start)
- vma = NULL;
- return vma;
-}
+ unsigned long start_addr, unsigned long end_addr);
/**
* vma_lookup() - Find a VMA at a specific address
static inline
struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr)
{
- struct vm_area_struct *vma = find_vma(mm, addr);
-
- if (vma && addr < vma->vm_start)
- vma = NULL;
-
- return vma;
+ return mtree_load(&mm->mm_mt, addr);
}
static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
unsigned long vm_start, unsigned long vm_end)
{
- struct vm_area_struct *vma = find_vma(mm, vm_start);
+ struct vm_area_struct *vma = vma_lookup(mm, vm_start);
if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
vma = NULL;
* and return without waiting upon it */
#define FOLL_NOFAULT 0x80 /* do not fault in pages */
#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
-#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
* PageAnonExclusive() has to protect against concurrent GUP:
* * Ordinary GUP: Using the PT lock
* * GUP-fast and fork(): mm->write_protect_seq
- * * GUP-fast and KSM or temporary unmapping (swap, migration):
- * clear/invalidate+flush of the page table entry
+ * * GUP-fast and KSM or temporary unmapping (swap, migration): see
+ * page_try_share_anon_rmap()
*
* Must be called with the (sub)page that's actually referenced via the
* page table entry, which might not necessarily be the head page for a
*/
if (!PageAnon(page))
return false;
+
+ /* Paired with a memory barrier in page_try_share_anon_rmap(). */
+ if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
+ smp_rmb();
+
/*
* Note that PageKsm() pages cannot be exclusive, and consequently,
* cannot get pinned.
return !PageAnonExclusive(page);
}
+/*
+ * Indicates whether GUP can follow a PROT_NONE mapped page, or whether
+ * a (NUMA hinting) fault is required.
+ */
+static inline bool gup_can_follow_protnone(unsigned int flags)
+{
+ /*
+ * FOLL_FORCE has to be able to make progress even if the VMA is
+ * inaccessible. Further, FOLL_FORCE access usually does not represent
+ * application behaviour and we should avoid triggering NUMA hinting
+ * faults.
+ */
+ return flags & FOLL_FORCE;
+}
+
typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);
extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
unsigned long size, pte_fn_t fn, void *data);
unsigned long address, unsigned long size,
pte_fn_t fn, void *data);
-extern void init_mem_debugging_and_hardening(void);
+extern void __init init_mem_debugging_and_hardening(void);
#ifdef CONFIG_PAGE_POISONING
extern void __kernel_poison_pages(struct page *page, int numpages);
extern void __kernel_unpoison_pages(struct page *page, int numpages);
return folio_is_file_lru(page_folio(page));
}
-static __always_inline void update_lru_size(struct lruvec *lruvec,
+static __always_inline void __update_lru_size(struct lruvec *lruvec,
enum lru_list lru, enum zone_type zid,
long nr_pages)
{
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+ lockdep_assert_held(&lruvec->lru_lock);
+ WARN_ON_ONCE(nr_pages != (int)nr_pages);
+
__mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
__mod_zone_page_state(&pgdat->node_zones[zid],
NR_ZONE_LRU_BASE + lru, nr_pages);
+}
+
+static __always_inline void update_lru_size(struct lruvec *lruvec,
+ enum lru_list lru, enum zone_type zid,
+ long nr_pages)
+{
+ __update_lru_size(lruvec, lru, zid, nr_pages);
#ifdef CONFIG_MEMCG
mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
#endif
__folio_clear_unevictable(folio);
}
-static __always_inline void __clear_page_lru_flags(struct page *page)
-{
- __folio_clear_lru_flags(page_folio(page));
-}
-
/**
* folio_lru_list - Which LRU list should a folio be on?
* @folio: The folio to test.
return lru;
}
+#ifdef CONFIG_LRU_GEN
+
+#ifdef CONFIG_LRU_GEN_ENABLED
+static inline bool lru_gen_enabled(void)
+{
+ DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]);
+
+ return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]);
+}
+#else
+static inline bool lru_gen_enabled(void)
+{
+ DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]);
+
+ return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]);
+}
+#endif
+
+static inline bool lru_gen_in_fault(void)
+{
+ return current->in_lru_fault;
+}
+
+static inline int lru_gen_from_seq(unsigned long seq)
+{
+ return seq % MAX_NR_GENS;
+}
+
+static inline int lru_hist_from_seq(unsigned long seq)
+{
+ return seq % NR_HIST_GENS;
+}
+
+static inline int lru_tier_from_refs(int refs)
+{
+ VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));
+
+ /* see the comment in folio_lru_refs() */
+ return order_base_2(refs + 1);
+}
+
+static inline int folio_lru_refs(struct folio *folio)
+{
+ unsigned long flags = READ_ONCE(folio->flags);
+ bool workingset = flags & BIT(PG_workingset);
+
+ /*
+ * Return the number of accesses beyond PG_referenced, i.e., N-1 if the
+ * total number of accesses is N>1, since N=0,1 both map to the first
+ * tier. lru_tier_from_refs() will account for this off-by-one. Also see
+ * the comment on MAX_NR_TIERS.
+ */
+ return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset;
+}
+
+static inline int folio_lru_gen(struct folio *folio)
+{
+ unsigned long flags = READ_ONCE(folio->flags);
+
+ return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+}
+
+static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
+{
+ unsigned long max_seq = lruvec->lrugen.max_seq;
+
+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
+
+ /* see the comment on MIN_NR_GENS */
+ return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
+}
+
+static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
+ int old_gen, int new_gen)
+{
+ int type = folio_is_file_lru(folio);
+ int zone = folio_zonenum(folio);
+ int delta = folio_nr_pages(folio);
+ enum lru_list lru = type * LRU_INACTIVE_FILE;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
+ VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
+ VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);
+
+ if (old_gen >= 0)
+ WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
+ lrugen->nr_pages[old_gen][type][zone] - delta);
+ if (new_gen >= 0)
+ WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
+ lrugen->nr_pages[new_gen][type][zone] + delta);
+
+ /* addition */
+ if (old_gen < 0) {
+ if (lru_gen_is_active(lruvec, new_gen))
+ lru += LRU_ACTIVE;
+ __update_lru_size(lruvec, lru, zone, delta);
+ return;
+ }
+
+ /* deletion */
+ if (new_gen < 0) {
+ if (lru_gen_is_active(lruvec, old_gen))
+ lru += LRU_ACTIVE;
+ __update_lru_size(lruvec, lru, zone, -delta);
+ return;
+ }
+
+ /* promotion */
+ if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
+ __update_lru_size(lruvec, lru, zone, -delta);
+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
+ }
+
+ /* demotion requires isolation, e.g., lru_deactivate_fn() */
+ VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
+}
+
+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+ unsigned long seq;
+ unsigned long flags;
+ int gen = folio_lru_gen(folio);
+ int type = folio_is_file_lru(folio);
+ int zone = folio_zonenum(folio);
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);
+
+ if (folio_test_unevictable(folio) || !lrugen->enabled)
+ return false;
+ /*
+ * There are three common cases for this page:
+ * 1. If it's hot, e.g., freshly faulted in or previously hot and
+ * migrated, add it to the youngest generation.
+ * 2. If it's cold but can't be evicted immediately, i.e., an anon page
+ * not in swapcache or a dirty page pending writeback, add it to the
+ * second oldest generation.
+ * 3. Everything else (clean, cold) is added to the oldest generation.
+ */
+ if (folio_test_active(folio))
+ seq = lrugen->max_seq;
+ else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) ||
+ (folio_test_reclaim(folio) &&
+ (folio_test_dirty(folio) || folio_test_writeback(folio))))
+ seq = lrugen->min_seq[type] + 1;
+ else
+ seq = lrugen->min_seq[type];
+
+ gen = lru_gen_from_seq(seq);
+ flags = (gen + 1UL) << LRU_GEN_PGOFF;
+ /* see the comment on MIN_NR_GENS about PG_active */
+ set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);
+
+ lru_gen_update_size(lruvec, folio, -1, gen);
+ /* for folio_rotate_reclaimable() */
+ if (reclaiming)
+ list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
+ else
+ list_add(&folio->lru, &lrugen->lists[gen][type][zone]);
+
+ return true;
+}
+
+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+ unsigned long flags;
+ int gen = folio_lru_gen(folio);
+
+ if (gen < 0)
+ return false;
+
+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
+
+ /* for folio_migrate_flags() */
+ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
+ flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);
+ gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+
+ lru_gen_update_size(lruvec, folio, gen, -1);
+ list_del(&folio->lru);
+
+ return true;
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static inline bool lru_gen_enabled(void)
+{
+ return false;
+}
+
+static inline bool lru_gen_in_fault(void)
+{
+ return false;
+}
+
+static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+ return false;
+}
+
+static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+ return false;
+}
+
+#endif /* CONFIG_LRU_GEN */
+
static __always_inline
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
{
enum lru_list lru = folio_lru_list(folio);
+ if (lru_gen_add_folio(lruvec, folio, false))
+ return;
+
update_lru_size(lruvec, lru, folio_zonenum(folio),
folio_nr_pages(folio));
if (lru != LRU_UNEVICTABLE)
{
enum lru_list lru = folio_lru_list(folio);
+ if (lru_gen_add_folio(lruvec, folio, true))
+ return;
+
update_lru_size(lruvec, lru, folio_zonenum(folio),
folio_nr_pages(folio));
/* This is not expected to be used on LRU_UNEVICTABLE */
list_add_tail(&folio->lru, &lruvec->lists[lru]);
}
-static __always_inline void add_page_to_lru_list_tail(struct page *page,
- struct lruvec *lruvec)
-{
- lruvec_add_folio_tail(lruvec, page_folio(page));
-}
-
static __always_inline
void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
{
enum lru_list lru = folio_lru_list(folio);
+ if (lru_gen_del_folio(lruvec, folio, false))
+ return;
+
if (lru != LRU_UNEVICTABLE)
list_del(&folio->lru);
update_lru_size(lruvec, lru, folio_zonenum(folio),
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/rbtree.h>
+#include <linux/maple_tree.h>
#include <linux/rwsem.h>
#include <linux/completion.h>
#include <linux/cpumask.h>
not kmapped, ie. highmem) */
#endif /* WANT_PAGE_VIRTUAL */
+#ifdef CONFIG_KMSAN
+ /*
+ * KMSAN metadata for this page:
+ * - shadow page: every bit indicates whether the corresponding
+ * bit of the original page is initialized (0) or not (1);
+ * - origin page: every 4 bytes contain an id of the stack trace
+ * where the uninitialized value was created.
+ */
+ struct page *kmsan_shadow;
+ struct page *kmsan_origin;
+#endif
+
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
int _last_cpupid;
#endif
* @_refcount: Do not access this member directly. Use folio_ref_count()
* to find how many references there are to this folio.
* @memcg_data: Memory Control Group data.
+ * @_flags_1: For large folios, additional page flags.
+ * @__head: Points to the folio. Do not use.
+ * @_folio_dtor: Which destructor to use for this folio.
+ * @_folio_order: Do not use directly, call folio_order().
+ * @_total_mapcount: Do not use directly, call folio_entire_mapcount().
+ * @_pincount: Do not use directly, call folio_maybe_dma_pinned().
+ * @_folio_nr_pages: Do not use directly, call folio_nr_pages().
*
* A folio is a physically, virtually and logically contiguous set
* of bytes. It is a power-of-two in size, and it is aligned to that
};
struct page page;
};
+ unsigned long _flags_1;
+ unsigned long __head;
+ unsigned char _folio_dtor;
+ unsigned char _folio_order;
+ atomic_t _total_mapcount;
+ atomic_t _pincount;
+#ifdef CONFIG_64BIT
+ unsigned int _folio_nr_pages;
+#endif
};
-static_assert(sizeof(struct page) == sizeof(struct folio));
#define FOLIO_MATCH(pg, fl) \
static_assert(offsetof(struct page, pg) == offsetof(struct folio, fl))
FOLIO_MATCH(flags, flags);
FOLIO_MATCH(memcg_data, memcg_data);
#endif
#undef FOLIO_MATCH
+#define FOLIO_MATCH(pg, fl) \
+ static_assert(offsetof(struct folio, fl) == \
+ offsetof(struct page, pg) + sizeof(struct page))
+FOLIO_MATCH(flags, _flags_1);
+FOLIO_MATCH(compound_head, __head);
+FOLIO_MATCH(compound_dtor, _folio_dtor);
+FOLIO_MATCH(compound_order, _folio_order);
+FOLIO_MATCH(compound_mapcount, _total_mapcount);
+FOLIO_MATCH(compound_pincount, _pincount);
+#ifdef CONFIG_64BIT
+FOLIO_MATCH(compound_nr, _folio_nr_pages);
+#endif
+#undef FOLIO_MATCH
static inline atomic_t *folio_mapcount_ptr(struct folio *folio)
{
unsigned long vm_end; /* The first byte after our end address
within vm_mm. */
- /* linked list of VM areas per task, sorted by address */
- struct vm_area_struct *vm_next, *vm_prev;
-
- struct rb_node vm_rb;
-
- /*
- * Largest free memory gap in bytes to the left of this VMA.
- * Either between this VMA and vma->vm_prev, or between one of the
- * VMAs below us in the VMA rbtree and its ->vm_prev. This helps
- * get_unmapped_area find a free area of the right size.
- */
- unsigned long rb_subtree_gap;
-
- /* Second cache line starts here. */
-
struct mm_struct *vm_mm; /* The address space we belong to. */
/*
struct kioctx_table;
struct mm_struct {
struct {
- struct vm_area_struct *mmap; /* list of VMAs */
- struct rb_root mm_rb;
- u64 vmacache_seqnum; /* per-thread vmacache */
+ struct maple_tree mm_mt;
#ifdef CONFIG_MMU
unsigned long (*get_unmapped_area) (struct file *filp,
unsigned long addr, unsigned long len,
unsigned long mmap_compat_legacy_base;
#endif
unsigned long task_size; /* size of task vm space */
- unsigned long highest_vm_end; /* highest vma end address */
pgd_t * pgd;
#ifdef CONFIG_MEMBARRIER
#endif
#ifdef CONFIG_NUMA_BALANCING
/*
- * numa_next_scan is the next time that the PTEs will be marked
- * pte_numa. NUMA hinting faults will gather statistics and
- * migrate pages to new nodes if necessary.
+ * numa_next_scan is the next time that PTEs will be remapped
+ * PROT_NONE to trigger NUMA hinting faults; such faults gather
+ * statistics and migrate pages to new nodes if necessary.
*/
unsigned long numa_next_scan;
- /* Restart point for scanning and setting pte_numa */
+ /* Restart point for scanning and remapping PTEs. */
unsigned long numa_scan_offset;
- /* numa_scan_seq prevents two threads setting pte_numa */
+ /* numa_scan_seq prevents two threads remapping PTEs. */
int numa_scan_seq;
#endif
/*
* An operation with batched TLB flushing is going on. Anything
* that can move process memory needs to flush the TLB when
- * moving a PROT_NONE or PROT_NUMA mapped page.
+ * moving a PROT_NONE mapped page.
*/
atomic_t tlb_flush_pending;
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
* merging.
*/
unsigned long ksm_merging_pages;
+ /*
+ * Represent how many pages are checked for ksm merging
+ * including merged and not merged.
+ */
+ unsigned long ksm_rmap_items;
+#endif
+#ifdef CONFIG_LRU_GEN
+ struct {
+ /* this mm_struct is on lru_gen_mm_list */
+ struct list_head list;
+ /*
+ * Set when switching to this mm_struct, as a hint of
+ * whether it has been used since the last time per-node
+ * page table walkers cleared the corresponding bits.
+ */
+ unsigned long bitmap;
+#ifdef CONFIG_MEMCG
+ /* points to the memcg of "owner" above */
+ struct mem_cgroup *memcg;
#endif
+ } lru_gen;
+#endif /* CONFIG_LRU_GEN */
} __randomize_layout;
/*
unsigned long cpu_bitmap[];
};
+#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN)
extern struct mm_struct init_mm;
/* Pointer magic because the dynamic array size confuses some compilers. */
return (struct cpumask *)&mm->cpu_bitmap;
}
+#ifdef CONFIG_LRU_GEN
+
+struct lru_gen_mm_list {
+ /* mm_struct list for page table walkers */
+ struct list_head fifo;
+ /* protects the list above */
+ spinlock_t lock;
+};
+
+void lru_gen_add_mm(struct mm_struct *mm);
+void lru_gen_del_mm(struct mm_struct *mm);
+#ifdef CONFIG_MEMCG
+void lru_gen_migrate_mm(struct mm_struct *mm);
+#endif
+
+static inline void lru_gen_init_mm(struct mm_struct *mm)
+{
+ INIT_LIST_HEAD(&mm->lru_gen.list);
+ mm->lru_gen.bitmap = 0;
+#ifdef CONFIG_MEMCG
+ mm->lru_gen.memcg = NULL;
+#endif
+}
+
+static inline void lru_gen_use_mm(struct mm_struct *mm)
+{
+ /*
+ * When the bitmap is set, page reclaim knows this mm_struct has been
+ * used since the last time it cleared the bitmap. So it might be worth
+ * walking the page tables of this mm_struct to clear the accessed bit.
+ */
+ WRITE_ONCE(mm->lru_gen.bitmap, -1);
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static inline void lru_gen_add_mm(struct mm_struct *mm)
+{
+}
+
+static inline void lru_gen_del_mm(struct mm_struct *mm)
+{
+}
+
+#ifdef CONFIG_MEMCG
+static inline void lru_gen_migrate_mm(struct mm_struct *mm)
+{
+}
+#endif
+
+static inline void lru_gen_init_mm(struct mm_struct *mm)
+{
+}
+
+static inline void lru_gen_use_mm(struct mm_struct *mm)
+{
+}
+
+#endif /* CONFIG_LRU_GEN */
+
+struct vma_iterator {
+ struct ma_state mas;
+};
+
+#define VMA_ITERATOR(name, __mm, __addr) \
+ struct vma_iterator name = { \
+ .mas = { \
+ .tree = &(__mm)->mm_mt, \
+ .index = __addr, \
+ .node = MAS_START, \
+ }, \
+ }
+
+static inline void vma_iter_init(struct vma_iterator *vmi,
+ struct mm_struct *mm, unsigned long addr)
+{
+ vmi->mas.tree = &mm->mm_mt;
+ vmi->mas.index = addr;
+ vmi->mas.node = MAS_START;
+}
+
struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
-/*
- * The per task VMA cache array:
- */
-#define VMACACHE_BITS 2
-#define VMACACHE_SIZE (1U << VMACACHE_BITS)
-#define VMACACHE_MASK (VMACACHE_SIZE - 1)
-
-struct vmacache {
- u64 seqnum;
- struct vm_area_struct *vmas[VMACACHE_SIZE];
-};
-
/*
* When updating this, please also update struct resident_page_types[] in
* kernel/fork.c
#define MMC_QUIRK_BROKEN_IRQ_POLLING (1<<11) /* Polling SDIO_CCCR_INTx could create a fake interrupt */
#define MMC_QUIRK_TRIM_BROKEN (1<<12) /* Skip trim */
#define MMC_QUIRK_BROKEN_HPI (1<<13) /* Disable broken HPI support */
+#define MMC_QUIRK_BROKEN_SD_DISCARD (1<<14) /* Disable broken SD discard support */
bool reenable_cmdq; /* Re-enable Command Queue */
#include <asm/page.h>
/* Free memory management - zoned buddy allocator. */
-#ifndef CONFIG_FORCE_MAX_ZONEORDER
+#ifndef CONFIG_ARCH_FORCE_MAX_ORDER
#define MAX_ORDER 11
#else
-#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
+#define MAX_ORDER CONFIG_ARCH_FORCE_MAX_ORDER
#endif
#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
struct pglist_data;
-/*
- * Add a wild amount of padding here to ensure data fall into separate
- * cachelines. There are very few zone structures in the machine, so space
- * consumption is not a concern here.
- */
-#if defined(CONFIG_SMP)
-struct zone_padding {
- char x[0];
-} ____cacheline_internodealigned_in_smp;
-#define ZONE_PADDING(name) struct zone_padding name;
-#else
-#define ZONE_PADDING(name)
-#endif
-
#ifdef CONFIG_NUMA
enum numa_stat_item {
NUMA_HIT, /* allocated in intended node */
#endif
#ifdef CONFIG_NUMA_BALANCING
PGPROMOTE_SUCCESS, /* promote successfully */
+ PGPROMOTE_CANDIDATE, /* candidate pages to promote */
#endif
NR_VM_NODE_STAT_ITEMS
};
return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
}
+#define WORKINGSET_ANON 0
+#define WORKINGSET_FILE 1
#define ANON_AND_FILE 2
enum lruvec_flags {
*/
};
+#endif /* !__GENERATING_BOUNDS_H */
+
+/*
+ * Evictable pages are divided into multiple generations. The youngest and the
+ * oldest generation numbers, max_seq and min_seq, are monotonically increasing.
+ * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
+ * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the
+ * corresponding generation. The gen counter in folio->flags stores gen+1 while
+ * a page is on one of lrugen->lists[]. Otherwise it stores 0.
+ *
+ * A page is added to the youngest generation on faulting. The aging needs to
+ * check the accessed bit at least twice before handing this page over to the
+ * eviction. The first check takes care of the accessed bit set on the initial
+ * fault; the second check makes sure this page hasn't been used since then.
+ * This process, AKA second chance, requires a minimum of two generations,
+ * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive
+ * LRU, e.g., /proc/vmstat, these two generations are considered active; the
+ * rest of generations, if they exist, are considered inactive. See
+ * lru_gen_is_active().
+ *
+ * PG_active is always cleared while a page is on one of lrugen->lists[] so that
+ * the aging needs not to worry about it. And it's set again when a page
+ * considered active is isolated for non-reclaiming purposes, e.g., migration.
+ * See lru_gen_add_folio() and lru_gen_del_folio().
+ *
+ * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the
+ * number of categories of the active/inactive LRU when keeping track of
+ * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits
+ * in folio->flags.
+ */
+#define MIN_NR_GENS 2U
+#define MAX_NR_GENS 4U
+
+/*
+ * Each generation is divided into multiple tiers. A page accessed N times
+ * through file descriptors is in tier order_base_2(N). A page in the first tier
+ * (N=0,1) is marked by PG_referenced unless it was faulted in through page
+ * tables or read ahead. A page in any other tier (N>1) is marked by
+ * PG_referenced and PG_workingset. This implies a minimum of two tiers is
+ * supported without using additional bits in folio->flags.
+ *
+ * In contrast to moving across generations which requires the LRU lock, moving
+ * across tiers only involves atomic operations on folio->flags and therefore
+ * has a negligible cost in the buffered access path. In the eviction path,
+ * comparisons of refaulted/(evicted+protected) from the first tier and the
+ * rest infer whether pages accessed multiple times through file descriptors
+ * are statistically hot and thus worth protecting.
+ *
+ * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
+ * number of categories of the active/inactive LRU when keeping track of
+ * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
+ * folio->flags.
+ */
+#define MAX_NR_TIERS 4U
+
+#ifndef __GENERATING_BOUNDS_H
+
+struct lruvec;
+struct page_vma_mapped_walk;
+
+#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
+#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
+
+#ifdef CONFIG_LRU_GEN
+
+enum {
+ LRU_GEN_ANON,
+ LRU_GEN_FILE,
+};
+
+enum {
+ LRU_GEN_CORE,
+ LRU_GEN_MM_WALK,
+ LRU_GEN_NONLEAF_YOUNG,
+ NR_LRU_GEN_CAPS
+};
+
+#define MIN_LRU_BATCH BITS_PER_LONG
+#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64)
+
+/* whether to keep historical stats from evicted generations */
+#ifdef CONFIG_LRU_GEN_STATS
+#define NR_HIST_GENS MAX_NR_GENS
+#else
+#define NR_HIST_GENS 1U
+#endif
+
+/*
+ * The youngest generation number is stored in max_seq for both anon and file
+ * types as they are aged on an equal footing. The oldest generation numbers are
+ * stored in min_seq[] separately for anon and file types as clean file pages
+ * can be evicted regardless of swap constraints.
+ *
+ * Normally anon and file min_seq are in sync. But if swapping is constrained,
+ * e.g., out of swap space, file min_seq is allowed to advance and leave anon
+ * min_seq behind.
+ *
+ * The number of pages in each generation is eventually consistent and therefore
+ * can be transiently negative when reset_batch_size() is pending.
+ */
+struct lru_gen_struct {
+ /* the aging increments the youngest generation number */
+ unsigned long max_seq;
+ /* the eviction increments the oldest generation numbers */
+ unsigned long min_seq[ANON_AND_FILE];
+ /* the birth time of each generation in jiffies */
+ unsigned long timestamps[MAX_NR_GENS];
+ /* the multi-gen LRU lists, lazily sorted on eviction */
+ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+ /* the multi-gen LRU sizes, eventually consistent */
+ long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+ /* the exponential moving average of refaulted */
+ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
+ /* the exponential moving average of evicted+protected */
+ unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS];
+ /* the first tier doesn't need protection, hence the minus one */
+ unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1];
+ /* can be modified without holding the LRU lock */
+ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
+ atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
+ /* whether the multi-gen LRU is enabled */
+ bool enabled;
+};
+
+enum {
+ MM_LEAF_TOTAL, /* total leaf entries */
+ MM_LEAF_OLD, /* old leaf entries */
+ MM_LEAF_YOUNG, /* young leaf entries */
+ MM_NONLEAF_TOTAL, /* total non-leaf entries */
+ MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */
+ MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */
+ NR_MM_STATS
+};
+
+/* double-buffering Bloom filters */
+#define NR_BLOOM_FILTERS 2
+
+struct lru_gen_mm_state {
+ /* set to max_seq after each iteration */
+ unsigned long seq;
+ /* where the current iteration continues (inclusive) */
+ struct list_head *head;
+ /* where the last iteration ended (exclusive) */
+ struct list_head *tail;
+ /* to wait for the last page table walker to finish */
+ struct wait_queue_head wait;
+ /* Bloom filters flip after each iteration */
+ unsigned long *filters[NR_BLOOM_FILTERS];
+ /* the mm stats for debugging */
+ unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
+ /* the number of concurrent page table walkers */
+ int nr_walkers;
+};
+
+struct lru_gen_mm_walk {
+ /* the lruvec under reclaim */
+ struct lruvec *lruvec;
+ /* unstable max_seq from lru_gen_struct */
+ unsigned long max_seq;
+ /* the next address within an mm to scan */
+ unsigned long next_addr;
+ /* to batch promoted pages */
+ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
+ /* to batch the mm stats */
+ int mm_stats[NR_MM_STATS];
+ /* total batched items */
+ int batched;
+ bool can_swap;
+ bool force_scan;
+};
+
+void lru_gen_init_lruvec(struct lruvec *lruvec);
+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw);
+
+#ifdef CONFIG_MEMCG
+void lru_gen_init_memcg(struct mem_cgroup *memcg);
+void lru_gen_exit_memcg(struct mem_cgroup *memcg);
+#endif
+
+#else /* !CONFIG_LRU_GEN */
+
+static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
+{
+}
+
+static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+{
+}
+
+#ifdef CONFIG_MEMCG
+static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
+{
+}
+
+static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+{
+}
+#endif
+
+#endif /* CONFIG_LRU_GEN */
+
struct lruvec {
struct list_head lists[NR_LRU_LISTS];
/* per lruvec lru_lock for memcg */
unsigned long refaults[ANON_AND_FILE];
/* Various lruvec state flags (enum lruvec_flags) */
unsigned long flags;
+#ifdef CONFIG_LRU_GEN
+ /* evictable pages divided into generations */
+ struct lru_gen_struct lrugen;
+ /* to concurrently iterate lru_gen_mm_list */
+ struct lru_gen_mm_state mm_state;
+#endif
#ifdef CONFIG_MEMCG
struct pglist_data *pgdat;
#endif
#define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1))
#define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP)
-/*
- * Shift to encode migratetype and order in the same integer, with order
- * in the least significant bits.
- */
-#define NR_PCP_ORDER_WIDTH 8
-#define NR_PCP_ORDER_MASK ((1<<NR_PCP_ORDER_WIDTH) - 1)
-
#define min_wmark_pages(z) (z->_watermark[WMARK_MIN] + z->watermark_boost)
#define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost)
#define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost)
int initialized;
/* Write-intensive fields used from the page allocator */
- ZONE_PADDING(_pad1_)
+ CACHELINE_PADDING(_pad1_);
/* free areas of different sizes */
struct free_area free_area[MAX_ORDER];
spinlock_t lock;
/* Write-intensive fields used by compaction and vmstats. */
- ZONE_PADDING(_pad2_)
+ CACHELINE_PADDING(_pad2_);
/*
* When free pages are below this point, additional steps are taken
bool contiguous;
- ZONE_PADDING(_pad3_)
+ CACHELINE_PADDING(_pad3_);
/* Zone statistics */
atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
+#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH)
+#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH)
/*
* Define the bit shifts to access each section. For non-existent
atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */
unsigned long nr_reclaim_start; /* nr pages written while throttled
* when throttling started. */
- struct task_struct *kswapd; /* Protected by
- mem_hotplug_begin/done() */
+#ifdef CONFIG_MEMORY_HOTPLUG
+ struct mutex kswapd_lock;
+#endif
+ struct task_struct *kswapd; /* Protected by kswapd_lock */
int kswapd_order;
enum zone_type kswapd_highest_zoneidx;
#endif /* CONFIG_NUMA */
/* Write-intensive fields used by page reclaim */
- ZONE_PADDING(_pad1_)
+ CACHELINE_PADDING(_pad1_);
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
/*
struct deferred_split deferred_split_queue;
#endif
+#ifdef CONFIG_NUMA_BALANCING
+ /* start time in ms of current promote rate limit period */
+ unsigned int nbp_rl_start;
+ /* number of promote candidate pages at start time of current rate limit period */
+ unsigned long nbp_rl_nr_cand;
+ /* promote threshold in ms */
+ unsigned int nbp_threshold;
+ /* start time in ms of current promote threshold adjustment period */
+ unsigned int nbp_th_start;
+ /*
+ * number of promote candidate pages at stat time of current promote
+ * threshold adjustment period
+ */
+ unsigned long nbp_th_nr_cand;
+#endif
/* Fields commonly accessed by the page reclaim scanner */
/*
unsigned long flags;
- ZONE_PADDING(_pad2_)
+#ifdef CONFIG_LRU_GEN
+ /* kswap mm walk data */
+ struct lru_gen_mm_walk mm_walk;
+#endif
+
+ CACHELINE_PADDING(_pad2_);
/* Per-node vmstats */
struct per_cpu_nodestat __percpu *per_cpu_nodestats;
atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
+#ifdef CONFIG_NUMA
+ struct memory_tier __rcu *memtier;
+#endif
} pg_data_t;
#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
return pgdat->node_start_pfn + pgdat->node_spanned_pages;
}
-static inline bool pgdat_is_empty(pg_data_t *pgdat)
-{
- return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
-}
-
#include <linux/memory_hotplug.h>
void build_all_zonelists(pg_data_t *pgdat);
/*
* include/linux/node.h - generic node definition
*
- * This is mainly for topological representation. We define the
- * basic 'struct node' here, which can be embedded in per-arch
+ * This is mainly for topological representation. We define the
+ * basic 'struct node' here, which can be embedded in per-arch
* definitions of processors.
*
* Basic handling of the devices is done in drivers/base/node.c
- * and system devices are handled in drivers/base/sys.c.
+ * and system devices are handled in drivers/base/sys.c.
*
* Nodes are exported via driverfs in the class/node/devices/
- * directory.
+ * directory.
*/
#ifndef _LINUX_NODE_H_
#define _LINUX_NODE_H_
#include <linux/device.h>
#include <linux/cpumask.h>
#include <linux/list.h>
-#include <linux/workqueue.h>
/**
* struct node_hmem_attrs - heterogeneous memory performance attributes
struct node {
struct device dev;
struct list_head access_list;
-
-#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_HUGETLBFS)
- struct work_struct node_work;
-#endif
#ifdef CONFIG_HMEM_REPORTING
struct list_head cache_attrs;
struct device *cache_dev;
struct memory_block;
extern struct node *node_devices[];
-typedef void (*node_registration_func_t)(struct node *);
#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_NUMA)
void register_memory_blocks_under_node(int nid, unsigned long start_pfn,
extern int register_memory_node_under_compute_node(unsigned int mem_nid,
unsigned int cpu_nid,
unsigned access);
-
-#ifdef CONFIG_HUGETLBFS
-extern void register_hugetlbfs_with_node(node_registration_func_t doregister,
- node_registration_func_t unregister);
-#endif
#else
static inline void node_dev_init(void)
{
static inline void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
{
}
-
-static inline void register_hugetlbfs_with_node(node_registration_func_t reg,
- node_registration_func_t unreg)
-{
-}
#endif
#define to_node(device) container_of(device, struct node, dev)
-static inline bool node_is_toptier(int node)
-{
- return node_state(node, N_CPU);
-}
-
#endif /* _LINUX_NODE_H_ */
#define first_online_node 0
#define first_memory_node 0
#define next_online_node(nid) (MAX_NUMNODES)
+#define next_memory_node(nid) (MAX_NUMNODES)
#define nr_node_ids 1U
#define nr_online_nodes 1U
static inline int node_random(const nodemask_t *maskp)
{
#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
- int w, bit = NUMA_NO_NODE;
+ int w, bit;
w = nodes_weight(*maskp);
- if (w)
- bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_int() % w);
+ switch (w) {
+ case 0:
+ bit = NUMA_NO_NODE;
+ break;
+ case 1:
+ bit = first_node(*maskp);
+ break;
+ default:
+ bit = find_nth_bit(maskp->bits, MAX_NUMNODES, prandom_u32_max(w));
+ break;
+ }
return bit;
#else
return 0;
extern int of_irq_to_resource(struct device_node *dev, int index,
struct resource *r);
-extern void of_irq_init(const struct of_device_id *matches);
-
#ifdef CONFIG_OF_IRQ
+extern void of_irq_init(const struct of_device_id *matches);
extern int of_irq_parse_one(struct device_node *device, int index,
struct of_phandle_args *out_irq);
extern int of_irq_count(struct device_node *dev);
extern void of_msi_configure(struct device *dev, struct device_node *np);
u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in);
#else
+static inline void of_irq_init(const struct of_device_id *matches)
+{
+}
static inline int of_irq_parse_one(struct device_node *device, int index,
struct of_phandle_args *out_irq)
{
return tsk->signal->oom_mm;
}
-/*
- * Use this helper if tsk->mm != mm and the victim mm needs a special
- * handling. This is guaranteed to stay true after once set.
- */
-static inline bool mm_is_oom_victim(struct mm_struct *mm)
-{
- return test_bit(MMF_OOM_VICTIM, &mm->flags);
-}
-
/*
* Checks whether a page fault on the given mm is still reliable.
* This is no longer true if the oom reaper started to reap the
return 0;
}
-bool __oom_reap_task_mm(struct mm_struct *mm);
-
long oom_badness(struct task_struct *p,
unsigned long totalpages);
#define SECTIONS_WIDTH 0
#endif
-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \
+ <= BITS_PER_LONG - NR_PAGEFLAGS
#define NODES_WIDTH NODES_SHIFT
#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
#error "Vmemmap: No space for nodes field in page flags"
#define LAST_CPUPID_SHIFT 0
#endif
-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \
- <= BITS_PER_LONG - NR_PAGEFLAGS
+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
+ KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
#else
#define LAST_CPUPID_WIDTH 0
#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
#endif
-#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \
- > BITS_PER_LONG - NR_PAGEFLAGS
+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
+ KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
#error "Not enough bits in page flags"
#endif
+/* see the comment on MAX_NR_TIERS */
+#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \
+ ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
+ NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)
+
#endif
#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
1UL << PG_private | 1UL << PG_private_2 | \
1UL << PG_writeback | 1UL << PG_reserved | \
1UL << PG_slab | 1UL << PG_active | \
- 1UL << PG_unevictable | __PG_MLOCKED)
+ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK)
/*
* Flags checked when a page is prepped for return by the page allocator.
* alloc-free cycle to prevent from reusing the page.
*/
#define PAGE_FLAGS_CHECK_AT_PREP \
- (PAGEFLAGS_MASK & ~__PG_HWPOISON)
+ ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK)
#define PAGE_FLAGS_PRIVATE \
(1UL << PG_private | 1UL << PG_private_2)
#define _LINUX_PAGE_COUNTER_H
#include <linux/atomic.h>
+#include <linux/cache.h>
#include <linux/kernel.h>
#include <asm/page.h>
struct page_counter {
+ /*
+ * Make sure 'usage' does not share cacheline with any other field. The
+ * memcg->memory.usage is a hot member of struct mem_cgroup.
+ */
atomic_long_t usage;
- unsigned long min;
- unsigned long low;
- unsigned long high;
- unsigned long max;
+ CACHELINE_PADDING(_pad1_);
/* effective memory.min and memory.min usage tracking */
unsigned long emin;
atomic_long_t low_usage;
atomic_long_t children_low_usage;
- /* legacy */
unsigned long watermark;
unsigned long failcnt;
- /*
- * 'parent' is placed here to be far from 'usage' to reduce
- * cache false sharing, as 'usage' is written mostly while
- * parent is frequently read for cgroup's hierarchical
- * counting nature.
- */
+ /* Keep all the read most fields in a separete cacheline. */
+ CACHELINE_PADDING(_pad2_);
+
+ unsigned long min;
+ unsigned long low;
+ unsigned long high;
+ unsigned long max;
struct page_counter *parent;
-};
+} ____cacheline_internodealigned_in_smp;
#if BITS_PER_LONG == 32
#define PAGE_COUNTER_MAX LONG_MAX
unsigned long flags;
};
+extern bool early_page_ext;
extern unsigned long page_ext_size;
extern void pgdat_page_ext_init(struct pglist_data *pgdat);
+static inline bool early_page_ext_enabled(void)
+{
+ return early_page_ext;
+}
+
#ifdef CONFIG_SPARSEMEM
static inline void page_ext_init_flatmem(void)
{
}
#endif
-struct page_ext *lookup_page_ext(const struct page *page);
+extern struct page_ext *page_ext_get(struct page *page);
+extern void page_ext_put(struct page_ext *page_ext);
static inline struct page_ext *page_ext_next(struct page_ext *curr)
{
#else /* !CONFIG_PAGE_EXTENSION */
struct page_ext;
-static inline void pgdat_page_ext_init(struct pglist_data *pgdat)
+static inline bool early_page_ext_enabled(void)
{
+ return false;
}
-static inline struct page_ext *lookup_page_ext(const struct page *page)
+static inline void pgdat_page_ext_init(struct pglist_data *pgdat)
{
- return NULL;
}
static inline void page_ext_init(void)
static inline void page_ext_init_flatmem(void)
{
}
+
+static inline struct page_ext *page_ext_get(struct page *page)
+{
+ return NULL;
+}
+
+static inline void page_ext_put(struct page_ext *page_ext)
+{
+}
#endif /* CONFIG_PAGE_EXTENSION */
#endif /* __LINUX_PAGE_EXT_H */
* If there is not enough space to store Idle and Young bits in page flags, use
* page ext flags instead.
*/
-
static inline bool folio_test_young(struct folio *folio)
{
- struct page_ext *page_ext = lookup_page_ext(&folio->page);
+ struct page_ext *page_ext = page_ext_get(&folio->page);
+ bool page_young;
if (unlikely(!page_ext))
return false;
- return test_bit(PAGE_EXT_YOUNG, &page_ext->flags);
+ page_young = test_bit(PAGE_EXT_YOUNG, &page_ext->flags);
+ page_ext_put(page_ext);
+
+ return page_young;
}
static inline void folio_set_young(struct folio *folio)
{
- struct page_ext *page_ext = lookup_page_ext(&folio->page);
+ struct page_ext *page_ext = page_ext_get(&folio->page);
if (unlikely(!page_ext))
return;
set_bit(PAGE_EXT_YOUNG, &page_ext->flags);
+ page_ext_put(page_ext);
}
static inline bool folio_test_clear_young(struct folio *folio)
{
- struct page_ext *page_ext = lookup_page_ext(&folio->page);
+ struct page_ext *page_ext = page_ext_get(&folio->page);
+ bool page_young;
if (unlikely(!page_ext))
return false;
- return test_and_clear_bit(PAGE_EXT_YOUNG, &page_ext->flags);
+ page_young = test_and_clear_bit(PAGE_EXT_YOUNG, &page_ext->flags);
+ page_ext_put(page_ext);
+
+ return page_young;
}
static inline bool folio_test_idle(struct folio *folio)
{
- struct page_ext *page_ext = lookup_page_ext(&folio->page);
+ struct page_ext *page_ext = page_ext_get(&folio->page);
+ bool page_idle;
if (unlikely(!page_ext))
return false;
- return test_bit(PAGE_EXT_IDLE, &page_ext->flags);
+ page_idle = test_bit(PAGE_EXT_IDLE, &page_ext->flags);
+ page_ext_put(page_ext);
+
+ return page_idle;
}
static inline void folio_set_idle(struct folio *folio)
{
- struct page_ext *page_ext = lookup_page_ext(&folio->page);
+ struct page_ext *page_ext = page_ext_get(&folio->page);
if (unlikely(!page_ext))
return;
set_bit(PAGE_EXT_IDLE, &page_ext->flags);
+ page_ext_put(page_ext);
}
static inline void folio_clear_idle(struct folio *folio)
{
- struct page_ext *page_ext = lookup_page_ext(&folio->page);
+ struct page_ext *page_ext = page_ext_get(&folio->page);
if (unlikely(!page_ext))
return;
clear_bit(PAGE_EXT_IDLE, &page_ext->flags);
+ page_ext_put(page_ext);
}
#endif /* !CONFIG_64BIT */
#endif /* CONFIG_HUGETLB_PAGE */
#define pageblock_nr_pages (1UL << pageblock_order)
+#define pageblock_align(pfn) ALIGN((pfn), pageblock_nr_pages)
+#define pageblock_aligned(pfn) IS_ALIGNED((pfn), pageblock_nr_pages)
+#define pageblock_start_pfn(pfn) ALIGN_DOWN((pfn), pageblock_nr_pages)
+#define pageblock_end_pfn(pfn) ALIGN((pfn) + 1, pageblock_nr_pages)
/* Forward declaration */
struct page;
unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
pgoff_t end, struct folio_batch *fbatch);
-unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start,
- unsigned int nr_pages, struct page **pages);
+unsigned filemap_get_folios_contig(struct address_space *mapping,
+ pgoff_t *start, pgoff_t end, struct folio_batch *fbatch);
unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
struct page **pages);
}
/*
- * lock_page_or_retry - Lock the page, unless this would block and the
+ * folio_lock_or_retry - Lock the folio, unless this would block and the
* caller indicated that it can handle a retry.
*
* Return value and mmap_lock implications depend on flags; see
* __folio_lock_or_retry().
*/
-static inline bool lock_page_or_retry(struct page *page, struct mm_struct *mm,
- unsigned int flags)
+static inline bool folio_lock_or_retry(struct folio *folio,
+ struct mm_struct *mm, unsigned int flags)
{
- struct folio *folio;
might_sleep();
-
- folio = page_folio(page);
return folio_trylock(folio) || __folio_lock_or_retry(folio, mm, flags);
}
return folio_wait_locked_killable(page_folio(page));
}
-int folio_put_wait_locked(struct folio *folio, int state);
void wait_on_page_writeback(struct page *page);
void folio_wait_writeback(struct folio *folio);
int folio_wait_writeback_killable(struct folio *folio);
* this handler is required to be able to handle
* pmd_trans_huge() pmds. They may simply choose to
* split_huge_page() instead of handling it explicitly.
- * @pte_entry: if set, called for each non-empty PTE (lowest-level)
- * entry
+ * @pte_entry: if set, called for each PTE (lowest-level) entry,
+ * including empty ones
* @pte_hole: if set, called for each hole at all levels,
- * depth is -1 if not known, 0:PGD, 1:P4D, 2:PUD, 3:PMD
- * 4:PTE. Any folded depths (where PTRS_PER_P?D is equal
- * to 1) are skipped.
+ * depth is -1 if not known, 0:PGD, 1:P4D, 2:PUD, 3:PMD.
+ * Any folded depths (where PTRS_PER_P?D is equal to 1)
+ * are skipped.
* @hugetlb_entry: if set, called for each hugetlb entry
* @test_walk: caller specific callback function to determine whether
* we walk over the current vma or not. Returning 0 means
unsigned int broken_cmd_compl:1; /* No compl for some cmds */
#endif
#ifdef CONFIG_PCIE_PTM
+ u16 ptm_cap; /* PTM Capability */
unsigned int ptm_root:1;
unsigned int ptm_enabled:1;
u8 ptm_granularity;
#ifdef CONFIG_PCIE_PTM
int pci_enable_ptm(struct pci_dev *dev, u8 *granularity);
+void pci_disable_ptm(struct pci_dev *dev);
bool pcie_ptm_enabled(struct pci_dev *dev);
#else
static inline int pci_enable_ptm(struct pci_dev *dev, u8 *granularity)
{ return -EINVAL; }
+static inline void pci_disable_ptm(struct pci_dev *dev) { }
static inline bool pcie_ptm_enabled(struct pci_dev *dev)
{ return false; }
#endif
#include <linux/types.h>
#include <linux/gfp.h>
+/* percpu_counter batch for local add or sub */
+#define PERCPU_COUNTER_LOCAL_BATCH INT_MAX
+
#ifdef CONFIG_SMP
struct percpu_counter {
percpu_counter_add_batch(fbc, amount, percpu_counter_batch);
}
+/*
+ * With percpu_counter_add_local() and percpu_counter_sub_local(), counts
+ * are accumulated in local per cpu counter and not in fbc->count until
+ * local count overflows PERCPU_COUNTER_LOCAL_BATCH. This makes counter
+ * write efficient.
+ * But percpu_counter_sum(), instead of percpu_counter_read(), needs to be
+ * used to add up the counts from each CPU to account for all the local
+ * counts. So percpu_counter_add_local() and percpu_counter_sub_local()
+ * should be used when a counter is updated frequently and read rarely.
+ */
+static inline void
+percpu_counter_add_local(struct percpu_counter *fbc, s64 amount)
+{
+ percpu_counter_add_batch(fbc, amount, PERCPU_COUNTER_LOCAL_BATCH);
+}
+
static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
{
s64 ret = __percpu_counter_sum(fbc);
preempt_enable();
}
+/* non-SMP percpu_counter_add_local is the same with percpu_counter_add */
+static inline void
+percpu_counter_add_local(struct percpu_counter *fbc, s64 amount)
+{
+ percpu_counter_add(fbc, amount);
+}
+
static inline void
percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
{
percpu_counter_add(fbc, -amount);
}
+static inline void
+percpu_counter_sub_local(struct percpu_counter *fbc, s64 amount)
+{
+ percpu_counter_add_local(fbc, -amount);
+}
+
#endif /* _LINUX_PERCPU_COUNTER_H */
#endif
#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
unsigned long address,
pmd_t *pmdp)
BUILD_BUG();
return 0;
}
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
#endif
#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
+#ifndef arch_has_hw_pte_young
+/*
+ * Return whether the accessed bit is supported on the local CPU.
+ *
+ * This stub assumes accessing through an old PTE triggers a page fault.
+ * Architectures that automatically set the access bit should overwrite it.
+ */
+static inline bool arch_has_hw_pte_young(void)
+{
+ return false;
+}
+#endif
+
#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
unsigned long address,
#endif
#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
- (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
- !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))
+ !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
static inline int pud_trans_huge(pud_t pud)
{
return 0;
#endif
#ifndef has_transparent_hugepage
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-#define has_transparent_hugepage() 1
-#else
-#define has_transparent_hugepage() 0
-#endif
+#define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE)
#endif
/*
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Analog Devices ADP5588 I/O Expander and QWERTY Keypad Controller
- *
- * Copyright 2009-2010 Analog Devices Inc.
- */
-
-#ifndef _ADP5588_H
-#define _ADP5588_H
-
-#define DEV_ID 0x00 /* Device ID */
-#define CFG 0x01 /* Configuration Register1 */
-#define INT_STAT 0x02 /* Interrupt Status Register */
-#define KEY_LCK_EC_STAT 0x03 /* Key Lock and Event Counter Register */
-#define Key_EVENTA 0x04 /* Key Event Register A */
-#define Key_EVENTB 0x05 /* Key Event Register B */
-#define Key_EVENTC 0x06 /* Key Event Register C */
-#define Key_EVENTD 0x07 /* Key Event Register D */
-#define Key_EVENTE 0x08 /* Key Event Register E */
-#define Key_EVENTF 0x09 /* Key Event Register F */
-#define Key_EVENTG 0x0A /* Key Event Register G */
-#define Key_EVENTH 0x0B /* Key Event Register H */
-#define Key_EVENTI 0x0C /* Key Event Register I */
-#define Key_EVENTJ 0x0D /* Key Event Register J */
-#define KP_LCK_TMR 0x0E /* Keypad Lock1 to Lock2 Timer */
-#define UNLOCK1 0x0F /* Unlock Key1 */
-#define UNLOCK2 0x10 /* Unlock Key2 */
-#define GPIO_INT_STAT1 0x11 /* GPIO Interrupt Status */
-#define GPIO_INT_STAT2 0x12 /* GPIO Interrupt Status */
-#define GPIO_INT_STAT3 0x13 /* GPIO Interrupt Status */
-#define GPIO_DAT_STAT1 0x14 /* GPIO Data Status, Read twice to clear */
-#define GPIO_DAT_STAT2 0x15 /* GPIO Data Status, Read twice to clear */
-#define GPIO_DAT_STAT3 0x16 /* GPIO Data Status, Read twice to clear */
-#define GPIO_DAT_OUT1 0x17 /* GPIO DATA OUT */
-#define GPIO_DAT_OUT2 0x18 /* GPIO DATA OUT */
-#define GPIO_DAT_OUT3 0x19 /* GPIO DATA OUT */
-#define GPIO_INT_EN1 0x1A /* GPIO Interrupt Enable */
-#define GPIO_INT_EN2 0x1B /* GPIO Interrupt Enable */
-#define GPIO_INT_EN3 0x1C /* GPIO Interrupt Enable */
-#define KP_GPIO1 0x1D /* Keypad or GPIO Selection */
-#define KP_GPIO2 0x1E /* Keypad or GPIO Selection */
-#define KP_GPIO3 0x1F /* Keypad or GPIO Selection */
-#define GPI_EM1 0x20 /* GPI Event Mode 1 */
-#define GPI_EM2 0x21 /* GPI Event Mode 2 */
-#define GPI_EM3 0x22 /* GPI Event Mode 3 */
-#define GPIO_DIR1 0x23 /* GPIO Data Direction */
-#define GPIO_DIR2 0x24 /* GPIO Data Direction */
-#define GPIO_DIR3 0x25 /* GPIO Data Direction */
-#define GPIO_INT_LVL1 0x26 /* GPIO Edge/Level Detect */
-#define GPIO_INT_LVL2 0x27 /* GPIO Edge/Level Detect */
-#define GPIO_INT_LVL3 0x28 /* GPIO Edge/Level Detect */
-#define Debounce_DIS1 0x29 /* Debounce Disable */
-#define Debounce_DIS2 0x2A /* Debounce Disable */
-#define Debounce_DIS3 0x2B /* Debounce Disable */
-#define GPIO_PULL1 0x2C /* GPIO Pull Disable */
-#define GPIO_PULL2 0x2D /* GPIO Pull Disable */
-#define GPIO_PULL3 0x2E /* GPIO Pull Disable */
-#define CMP_CFG_STAT 0x30 /* Comparator Configuration and Status Register */
-#define CMP_CONFG_SENS1 0x31 /* Sensor1 Comparator Configuration Register */
-#define CMP_CONFG_SENS2 0x32 /* L2 Light Sensor Reference Level, Output Falling for Sensor 1 */
-#define CMP1_LVL2_TRIP 0x33 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 1 */
-#define CMP1_LVL2_HYS 0x34 /* L3 Light Sensor Reference Level, Output Falling For Sensor 1 */
-#define CMP1_LVL3_TRIP 0x35 /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 1 */
-#define CMP1_LVL3_HYS 0x36 /* Sensor 2 Comparator Configuration Register */
-#define CMP2_LVL2_TRIP 0x37 /* L2 Light Sensor Reference Level, Output Falling for Sensor 2 */
-#define CMP2_LVL2_HYS 0x38 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 2 */
-#define CMP2_LVL3_TRIP 0x39 /* L3 Light Sensor Reference Level, Output Falling For Sensor 2 */
-#define CMP2_LVL3_HYS 0x3A /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 2 */
-#define CMP1_ADC_DAT_R1 0x3B /* Comparator 1 ADC data Register1 */
-#define CMP1_ADC_DAT_R2 0x3C /* Comparator 1 ADC data Register2 */
-#define CMP2_ADC_DAT_R1 0x3D /* Comparator 2 ADC data Register1 */
-#define CMP2_ADC_DAT_R2 0x3E /* Comparator 2 ADC data Register2 */
-
-#define ADP5588_DEVICE_ID_MASK 0xF
-
- /* Configuration Register1 */
-#define ADP5588_AUTO_INC (1 << 7)
-#define ADP5588_GPIEM_CFG (1 << 6)
-#define ADP5588_OVR_FLOW_M (1 << 5)
-#define ADP5588_INT_CFG (1 << 4)
-#define ADP5588_OVR_FLOW_IEN (1 << 3)
-#define ADP5588_K_LCK_IM (1 << 2)
-#define ADP5588_GPI_IEN (1 << 1)
-#define ADP5588_KE_IEN (1 << 0)
-
-/* Interrupt Status Register */
-#define ADP5588_CMP2_INT (1 << 5)
-#define ADP5588_CMP1_INT (1 << 4)
-#define ADP5588_OVR_FLOW_INT (1 << 3)
-#define ADP5588_K_LCK_INT (1 << 2)
-#define ADP5588_GPI_INT (1 << 1)
-#define ADP5588_KE_INT (1 << 0)
-
-/* Key Lock and Event Counter Register */
-#define ADP5588_K_LCK_EN (1 << 6)
-#define ADP5588_LCK21 0x30
-#define ADP5588_KEC 0xF
-
-#define ADP5588_MAXGPIO 18
-#define ADP5588_BANK(offs) ((offs) >> 3)
-#define ADP5588_BIT(offs) (1u << ((offs) & 0x7))
-
-/* Put one of these structures in i2c_board_info platform_data */
-
-#define ADP5588_KEYMAPSIZE 80
-
-#define GPI_PIN_ROW0 97
-#define GPI_PIN_ROW1 98
-#define GPI_PIN_ROW2 99
-#define GPI_PIN_ROW3 100
-#define GPI_PIN_ROW4 101
-#define GPI_PIN_ROW5 102
-#define GPI_PIN_ROW6 103
-#define GPI_PIN_ROW7 104
-#define GPI_PIN_COL0 105
-#define GPI_PIN_COL1 106
-#define GPI_PIN_COL2 107
-#define GPI_PIN_COL3 108
-#define GPI_PIN_COL4 109
-#define GPI_PIN_COL5 110
-#define GPI_PIN_COL6 111
-#define GPI_PIN_COL7 112
-#define GPI_PIN_COL8 113
-#define GPI_PIN_COL9 114
-
-#define GPI_PIN_ROW_BASE GPI_PIN_ROW0
-#define GPI_PIN_ROW_END GPI_PIN_ROW7
-#define GPI_PIN_COL_BASE GPI_PIN_COL0
-#define GPI_PIN_COL_END GPI_PIN_COL9
-
-#define GPI_PIN_BASE GPI_PIN_ROW_BASE
-#define GPI_PIN_END GPI_PIN_COL_END
-
-#define ADP5588_GPIMAPSIZE_MAX (GPI_PIN_END - GPI_PIN_BASE + 1)
-
-struct adp5588_gpi_map {
- unsigned short pin;
- unsigned short sw_evt;
-};
-
-struct adp5588_kpad_platform_data {
- int rows; /* Number of rows */
- int cols; /* Number of columns */
- const unsigned short *keymap; /* Pointer to keymap */
- unsigned short keymapsize; /* Keymap size */
- unsigned repeat:1; /* Enable key repeat */
- unsigned en_keylock:1; /* Enable Key Lock feature */
- unsigned short unlock_key1; /* Unlock Key 1 */
- unsigned short unlock_key2; /* Unlock Key 2 */
- const struct adp5588_gpi_map *gpimap;
- unsigned short gpimapsize;
- const struct adp5588_gpio_platform_data *gpio_data;
-};
-
-struct i2c_client; /* forward declaration */
-
-struct adp5588_gpio_platform_data {
- int gpio_start; /* GPIO Chip base # */
- const char *const *names;
- unsigned irq_base; /* interrupt base # */
- unsigned pullup_dis_mask; /* Pull-Up Disable Mask */
- int (*setup)(struct i2c_client *client,
- unsigned gpio, unsigned ngpio,
- void *context);
- int (*teardown)(struct i2c_client *client,
- unsigned gpio, unsigned ngpio,
- void *context);
- void *context;
-};
-
-#endif
}
#ifdef CONFIG_PM
-#define _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \
- runtime_resume_fn, idle_fn, sec, ns) \
- _DEFINE_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \
- runtime_resume_fn, idle_fn); \
- __EXPORT_SYMBOL(name, sec, ns)
+#define _EXPORT_DEV_PM_OPS(name, sec, ns) \
+ const struct dev_pm_ops name; \
+ __EXPORT_SYMBOL(name, sec, ns); \
+ const struct dev_pm_ops name
#else
-#define _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \
- runtime_resume_fn, idle_fn, sec, ns) \
-static __maybe_unused _DEFINE_DEV_PM_OPS(__static_##name, suspend_fn, \
- resume_fn, runtime_suspend_fn, \
- runtime_resume_fn, idle_fn)
+#define _EXPORT_DEV_PM_OPS(name, sec, ns) \
+ static __maybe_unused const struct dev_pm_ops __static_##name
#endif
+#define EXPORT_DEV_PM_OPS(name) _EXPORT_DEV_PM_OPS(name, "", "")
+#define EXPORT_GPL_DEV_PM_OPS(name) _EXPORT_DEV_PM_OPS(name, "_gpl", "")
+#define EXPORT_NS_DEV_PM_OPS(name, ns) _EXPORT_DEV_PM_OPS(name, "", #ns)
+#define EXPORT_NS_GPL_DEV_PM_OPS(name, ns) _EXPORT_DEV_PM_OPS(name, "_gpl", #ns)
+
/*
* Use this if you want to use the same suspend and resume callbacks for suspend
* to RAM and hibernation.
_DEFINE_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL)
#define EXPORT_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \
- _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "", "")
+ EXPORT_DEV_PM_OPS(name) = { \
+ SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \
+ }
#define EXPORT_GPL_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \
- _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "_gpl", "")
+ EXPORT_GPL_DEV_PM_OPS(name) = { \
+ SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \
+ }
#define EXPORT_NS_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn, ns) \
- _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "", #ns)
+ EXPORT_NS_DEV_PM_OPS(name, ns) = { \
+ SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \
+ }
#define EXPORT_NS_GPL_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn, ns) \
- _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "_gpl", #ns)
+ EXPORT_NS_GPL_DEV_PM_OPS(name, ns) = { \
+ SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \
+ }
/* Deprecated. Use DEFINE_SIMPLE_DEV_PM_OPS() instead. */
#define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \
resume_fn, idle_fn)
#define EXPORT_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
- _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \
- suspend_fn, resume_fn, idle_fn, "", "")
+ EXPORT_DEV_PM_OPS(name) = { \
+ RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
+ }
#define EXPORT_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \
- _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \
- suspend_fn, resume_fn, idle_fn, "_gpl", "")
+ EXPORT_GPL_DEV_PM_OPS(name) = { \
+ RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
+ }
#define EXPORT_NS_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \
- _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \
- suspend_fn, resume_fn, idle_fn, "", #ns)
+ EXPORT_NS_DEV_PM_OPS(name, ns) = { \
+ RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
+ }
#define EXPORT_NS_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \
- _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \
- suspend_fn, resume_fn, idle_fn, "_gpl", #ns)
+ EXPORT_NS_GPL_DEV_PM_OPS(name, ns) = { \
+ RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \
+ }
#ifdef CONFIG_PM
extern struct workqueue_struct *pm_wq;
#include <linux/percpu.h>
#include <linux/random.h>
-/* Deprecated: use get_random_u32 instead. */
-static inline u32 prandom_u32(void)
-{
- return get_random_u32();
-}
-
-/* Deprecated: use get_random_bytes instead. */
-static inline void prandom_bytes(void *buf, size_t nbytes)
-{
- return get_random_bytes(buf, nbytes);
-}
-
struct rnd_state {
__u32 s1, s2, s3, s4;
};
#include <linux/sched.h>
#include <linux/poll.h>
#include <linux/cgroup-defs.h>
+#include <linux/cgroup.h>
struct seq_file;
struct css_set;
void psi_init(void);
-void psi_task_change(struct task_struct *task, int clear, int set);
-void psi_task_switch(struct task_struct *prev, struct task_struct *next,
- bool sleep);
-
void psi_memstall_enter(unsigned long *flags);
void psi_memstall_leave(unsigned long *flags);
poll_table *wait);
#ifdef CONFIG_CGROUPS
+static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
+{
+ return cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
+}
+
int psi_cgroup_alloc(struct cgroup *cgrp);
void psi_cgroup_free(struct cgroup *cgrp);
void cgroup_move_task(struct task_struct *p, struct css_set *to);
+void psi_cgroup_restart(struct psi_group *group);
#endif
#else /* CONFIG_PSI */
{
rcu_assign_pointer(p->cgroups, to);
}
+static inline void psi_cgroup_restart(struct psi_group *group) {}
#endif
#endif /* CONFIG_PSI */
NR_IOWAIT,
NR_MEMSTALL,
NR_RUNNING,
- /*
- * This can't have values other than 0 or 1 and could be
- * implemented as a bit flag. But for now we still have room
- * in the first cacheline of psi_group_cpu, and this way we
- * don't have to special case any state tracking for it.
- */
- NR_ONCPU,
/*
* For IO and CPU stalls the presence of running/oncpu tasks
* in the domain means a partial rather than a full stall.
* threads and memstall ones.
*/
NR_MEMSTALL_RUNNING,
- NR_PSI_TASK_COUNTS = 5,
+ NR_PSI_TASK_COUNTS = 4,
};
/* Task state bitmasks */
#define TSK_IOWAIT (1 << NR_IOWAIT)
#define TSK_MEMSTALL (1 << NR_MEMSTALL)
#define TSK_RUNNING (1 << NR_RUNNING)
-#define TSK_ONCPU (1 << NR_ONCPU)
#define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING)
+/* Only one task can be scheduled, no corresponding task count */
+#define TSK_ONCPU (1 << NR_PSI_TASK_COUNTS)
+
/* Resources that workloads could be stalled on */
enum psi_res {
PSI_IO,
PSI_MEM,
PSI_CPU,
- NR_PSI_RESOURCES = 3,
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+ PSI_IRQ,
+#endif
+ NR_PSI_RESOURCES,
};
/*
PSI_MEM_FULL,
PSI_CPU_SOME,
PSI_CPU_FULL,
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+ PSI_IRQ_FULL,
+#endif
/* Only per-CPU, to weigh the CPU in the global average: */
PSI_NONIDLE,
- NR_PSI_STATES = 7,
+ NR_PSI_STATES,
};
+/* Use one bit in the state mask to track TSK_ONCPU */
+#define PSI_ONCPU (1 << NR_PSI_STATES)
+
enum psi_aggregators {
PSI_AVGS = 0,
PSI_POLL,
};
struct psi_group {
+ struct psi_group *parent;
+ bool enabled;
+
/* Protects data used by the aggregator */
struct mutex avgs_lock;
#else /* CONFIG_PSI */
+#define NR_PSI_RESOURCES 0
+
struct psi_group { };
#endif /* CONFIG_PSI */
u16 get_random_u16(void);
u32 get_random_u32(void);
u64 get_random_u64(void);
-static inline unsigned int get_random_int(void)
-{
- return get_random_u32();
-}
static inline unsigned long get_random_long(void)
{
#if BITS_PER_LONG == 64
declare_get_random_var_wait(u16, u16)
declare_get_random_var_wait(u32, u32)
declare_get_random_var_wait(u64, u32)
-declare_get_random_var_wait(int, unsigned int)
declare_get_random_var_wait(long, unsigned long)
#undef declare_get_random_var
unlink_anon_vmas(next);
}
-struct anon_vma *page_get_anon_vma(struct page *page);
+struct anon_vma *folio_get_anon_vma(struct folio *folio);
/* RMAP flags, currently only relevant for some anon rmap operations. */
typedef int __bitwise rmap_t;
* @page: the exclusive anonymous page to try marking possibly shared
*
* The caller needs to hold the PT lock and has to have the page table entry
- * cleared/invalidated+flushed, to properly sync against GUP-fast.
+ * cleared/invalidated.
*
* This is similar to page_try_dup_anon_rmap(), however, not used during fork()
* to duplicate a mapping, but instead to prepare for KSM or temporarily
{
VM_BUG_ON_PAGE(!PageAnon(page) || !PageAnonExclusive(page), page);
- /* See page_try_dup_anon_rmap(). */
- if (likely(!is_device_private_page(page) &&
- unlikely(page_maybe_dma_pinned(page))))
- return -EBUSY;
+ /* device private pages cannot get pinned via GUP. */
+ if (unlikely(is_device_private_page(page))) {
+ ClearPageAnonExclusive(page);
+ return 0;
+ }
+
+ /*
+ * We have to make sure that when we clear PageAnonExclusive, that
+ * the page is not pinned and that concurrent GUP-fast won't succeed in
+ * concurrently pinning the page.
+ *
+ * Conceptually, PageAnonExclusive clearing consists of:
+ * (A1) Clear PTE
+ * (A2) Check if the page is pinned; back off if so.
+ * (A3) Clear PageAnonExclusive
+ * (A4) Restore PTE (optional, but certainly not writable)
+ *
+ * When clearing PageAnonExclusive, we cannot possibly map the page
+ * writable again, because anon pages that may be shared must never
+ * be writable. So in any case, if the PTE was writable it cannot
+ * be writable anymore afterwards and there would be a PTE change. Only
+ * if the PTE wasn't writable, there might not be a PTE change.
+ *
+ * Conceptually, GUP-fast pinning of an anon page consists of:
+ * (B1) Read the PTE
+ * (B2) FOLL_WRITE: check if the PTE is not writable; back off if so.
+ * (B3) Pin the mapped page
+ * (B4) Check if the PTE changed by re-reading it; back off if so.
+ * (B5) If the original PTE is not writable, check if
+ * PageAnonExclusive is not set; back off if so.
+ *
+ * If the PTE was writable, we only have to make sure that GUP-fast
+ * observes a PTE change and properly backs off.
+ *
+ * If the PTE was not writable, we have to make sure that GUP-fast either
+ * detects a (temporary) PTE change or that PageAnonExclusive is cleared
+ * and properly backs off.
+ *
+ * Consequently, when clearing PageAnonExclusive(), we have to make
+ * sure that (A1), (A2)/(A3) and (A4) happen in the right memory
+ * order. In GUP-fast pinning code, we have to make sure that (B3),(B4)
+ * and (B5) happen in the right memory order.
+ *
+ * We assume that there might not be a memory barrier after
+ * clearing/invalidating the PTE (A1) and before restoring the PTE (A4),
+ * so we use explicit ones here.
+ */
+ /* Paired with the memory barrier in try_grab_folio(). */
+ if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
+ smp_mb();
+
+ if (unlikely(page_maybe_dma_pinned(page)))
+ return -EBUSY;
ClearPageAnonExclusive(page);
+
+ /*
+ * This is conceptually a smp_wmb() paired with the smp_rmb() in
+ * gup_must_unshare().
+ */
+ if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
+ smp_mb__after_atomic();
return 0;
}
void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc);
void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc);
-
-/*
- * Called by memory-failure.c to kill processes.
- */
struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,
struct rmap_walk_control *rwc);
-void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
#else /* !CONFIG_MMU */
#include <linux/pid.h>
#include <linux/sem.h>
#include <linux/shm.h>
+#include <linux/kmsan_types.h>
#include <linux/mutex.h>
#include <linux/plist.h>
#include <linux/hrtimer.h>
struct mm_struct *mm;
struct mm_struct *active_mm;
- /* Per-thread vma caching: */
- struct vmacache vmacache;
-
#ifdef SPLIT_RSS_COUNTING
struct task_rss_stat rss_stat;
#endif
#ifdef CONFIG_MEMCG
unsigned in_user_fault:1;
#endif
+#ifdef CONFIG_LRU_GEN
+ /* whether the LRU algorithm may apply to this access */
+ unsigned in_lru_fault:1;
+#endif
#ifdef CONFIG_COMPAT_BRK
unsigned brk_randomized:1;
#endif
#ifdef CONFIG_CPU_SUP_INTEL
unsigned reported_split_lock:1;
#endif
+#ifdef CONFIG_TASK_DELAY_ACCT
+ /* delay due to memory thrashing */
+ unsigned in_thrashing:1;
+#endif
unsigned long atomic_flags; /* Flags requiring atomic access. */
#endif
#endif
+#ifdef CONFIG_KMSAN
+ struct kmsan_ctx kmsan_ctx;
+#endif
+
#if IS_ENABLED(CONFIG_KUNIT)
struct kunit *kunit_test;
#endif
#define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */
#define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */
#define MMF_DISABLE_THP 24 /* disable THP for all VMAs */
-#define MMF_OOM_VICTIM 25 /* mm is the oom victim */
-#define MMF_OOM_REAP_QUEUED 26 /* mm was queued for oom_reaper */
-#define MMF_MULTIPROCESS 27 /* mm is shared between processes */
+#define MMF_OOM_REAP_QUEUED 25 /* mm was queued for oom_reaper */
+#define MMF_MULTIPROCESS 26 /* mm is shared between processes */
/*
* MMF_HAS_PINNED: Whether this mm has pinned any pages. This can be either
* replaced in the future by mm.pinned_vm when it becomes stable, or grow into
* pinned pages were unpinned later on, we'll still keep this bit set for the
* lifecycle of this mm, just for simplicity.
*/
-#define MMF_HAS_PINNED 28 /* FOLL_PIN has run, never cleared */
+#define MMF_HAS_PINNED 27 /* FOLL_PIN has run, never cleared */
#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP)
#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
#ifdef CONFIG_NUMA_BALANCING
extern int sysctl_numa_balancing_mode;
+extern unsigned int sysctl_numa_balancing_promote_rate_limit;
#else
#define sysctl_numa_balancing_mode 0
#endif
void put_task_struct_rcu_user(struct task_struct *task);
+/* Free all architecture-specific resources held by a thread. */
+void release_thread(struct task_struct *dead_task);
+
#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
extern int arch_task_struct_size __read_mostly;
#else
extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end);
int shmem_unuse(unsigned int type);
-extern bool shmem_is_huge(struct vm_area_struct *vma,
- struct inode *inode, pgoff_t index);
-static inline bool shmem_huge_enabled(struct vm_area_struct *vma)
+extern bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
+ pgoff_t index, bool shmem_huge_force);
+static inline bool shmem_huge_enabled(struct vm_area_struct *vma,
+ bool shmem_huge_force)
{
- return shmem_is_huge(vma, file_inode(vma->vm_file), vma->vm_pgoff);
+ return shmem_is_huge(vma, file_inode(vma->vm_file), vma->vm_pgoff,
+ shmem_huge_force);
}
extern unsigned long shmem_swap_usage(struct vm_area_struct *vma);
extern unsigned long shmem_partial_swap_usage(struct address_space *mapping,
pgoff_t start, pgoff_t end);
-/* Flag allocation requirements to shmem_getpage */
+/* Flag allocation requirements to shmem_get_folio */
enum sgp_type {
SGP_READ, /* don't exceed i_size, don't allocate page */
SGP_NOALLOC, /* similar, but fail on hole or use fallocated page */
SGP_FALLOC, /* like SGP_WRITE, but make existing page Uptodate */
};
-extern int shmem_getpage(struct inode *inode, pgoff_t index,
- struct page **pagep, enum sgp_type sgp);
+int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
+ enum sgp_type sgp);
static inline struct page *shmem_read_mapping_page(
struct address_space *mapping, pgoff_t index)
* @csum_level: indicates the number of consecutive checksums found in
* the packet minus one that have been verified as
* CHECKSUM_UNNECESSARY (max 3)
+ * @scm_io_uring: SKB holds io_uring registered files
* @dst_pending_confirm: need to confirm neighbour
* @decrypted: Decrypted SKB
* @slow_gro: state present at GRO time, slower prepare step required
#endif
__u8 slow_gro:1;
__u8 csum_not_inet:1;
+ __u8 scm_io_uring:1;
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
# define SLAB_ACCOUNT 0
#endif
-#ifdef CONFIG_KASAN
+#ifdef CONFIG_KASAN_GENERIC
#define SLAB_KASAN ((slab_flags_t __force)0x08000000U)
#else
#define SLAB_KASAN 0
*/
#define SLAB_NO_USER_FLAGS ((slab_flags_t __force)0x10000000U)
+#ifdef CONFIG_KFENCE
+#define SLAB_SKIP_KFENCE ((slab_flags_t __force)0x20000000U)
+#else
+#define SLAB_SKIP_KFENCE 0
+#endif
+
/* The following flags affect the page allocator grouping pages by mobility */
/* Objects are reclaimable */
#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U)
size_t colour; /* cache colouring range */
unsigned int colour_off; /* colour offset */
- struct kmem_cache *freelist_cache;
unsigned int freelist_size;
/* constructor func */
#include <linux/gfp.h>
typedef u32 depot_stack_handle_t;
+/*
+ * Number of bits in the handle that stack depot doesn't use. Users may store
+ * information in them.
+ */
+#define STACK_DEPOT_EXTRA_BITS 5
depot_stack_handle_t __stack_depot_save(unsigned long *entries,
unsigned int nr_entries,
+ unsigned int extra_bits,
gfp_t gfp_flags, bool can_alloc);
/*
unsigned int stack_depot_fetch(depot_stack_handle_t handle,
unsigned long **entries);
+unsigned int stack_depot_get_extra_bits(depot_stack_handle_t handle);
+
int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size,
int spaces);
bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt,
const struct sockaddr *sap);
void rpc_clnt_xprt_set_online(struct rpc_clnt *clnt, struct rpc_xprt *xprt);
+void rpc_clnt_disconnect(struct rpc_clnt *clnt);
void rpc_cleanup_clids(void);
static inline int rpc_reply_expected(struct rpc_task *task)
struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req);
void rpc_put_task(struct rpc_task *);
void rpc_put_task_async(struct rpc_task *);
+bool rpc_task_set_rpc_status(struct rpc_task *task, int rpc_status);
+void rpc_task_try_cancel(struct rpc_task *task, int error);
void rpc_signal_task(struct rpc_task *);
void rpc_exit_task(struct rpc_task *);
void rpc_exit(struct rpc_task *, int);
void rpc_release_calldata(const struct rpc_call_ops *, void *);
void rpc_killall_tasks(struct rpc_clnt *);
+unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
+ bool (*fnmatch)(const struct rpc_task *,
+ const void *),
+ const void *data);
void rpc_execute(struct rpc_task *);
void rpc_init_priority_wait_queue(struct rpc_wait_queue *, const char *);
void rpc_init_wait_queue(struct rpc_wait_queue *, const char *);
*/
struct reclaim_state {
unsigned long reclaimed_slab;
+#ifdef CONFIG_LRU_GEN
+ /* per-thread mm walk data */
+ struct lru_gen_mm_walk *mm_walk;
+#endif
};
#ifdef __KERNEL__
return entry;
}
+static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
+{
+ folio->private = (void *)entry.val;
+}
+
/* linux/mm/workingset.c */
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
/* linux/mm/swap.c */
-extern void lru_note_cost(struct lruvec *lruvec, bool file,
- unsigned int nr_pages);
-extern void lru_note_cost_folio(struct folio *);
-extern void folio_add_lru(struct folio *);
-extern void lru_cache_add(struct page *);
+void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages);
+void lru_note_cost_folio(struct folio *);
+void folio_add_lru(struct folio *);
+void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
+void lru_cache_add(struct page *);
void mark_page_accessed(struct page *);
void folio_mark_accessed(struct folio *);
extern void si_swapinfo(struct sysinfo *);
swp_entry_t folio_alloc_swap(struct folio *folio);
-extern void put_swap_page(struct page *page, swp_entry_t entry);
+bool folio_free_swap(struct folio *folio);
+void put_swap_folio(struct folio *folio, swp_entry_t entry);
extern swp_entry_t get_swap_page_of_type(int);
extern int get_swap_pages(int n, swp_entry_t swp_entries[], int entry_size);
extern int add_swap_count_continuation(swp_entry_t, gfp_t);
extern int swp_swapcount(swp_entry_t entry);
extern struct swap_info_struct *page_swap_info(struct page *);
extern struct swap_info_struct *swp_swap_info(swp_entry_t entry);
-extern int try_to_free_swap(struct page *);
struct backing_dev_info;
extern int init_swap_address_space(unsigned int type, unsigned long nr_pages);
extern void exit_swap_address_space(unsigned int type);
{
}
-static inline void put_swap_page(struct page *page, swp_entry_t swp)
+static inline void put_swap_folio(struct folio *folio, swp_entry_t swp)
{
}
return 0;
}
-static inline int try_to_free_swap(struct page *page)
-{
- return 0;
-}
-
static inline swp_entry_t folio_alloc_swap(struct folio *folio)
{
swp_entry_t entry;
return entry;
}
+static inline bool folio_free_swap(struct folio *folio)
+{
+ return false;
+}
+
static inline int add_swap_extent(struct swap_info_struct *sis,
unsigned long start_page,
unsigned long nr_pages, sector_t start_block)
cgroup_throttle_swaprate(&folio->page, gfp);
}
-#ifdef CONFIG_MEMCG_SWAP
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry);
int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry);
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
}
extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
-extern bool mem_cgroup_swap_full(struct page *page);
+extern bool mem_cgroup_swap_full(struct folio *folio);
#else
static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
{
return get_nr_swap_pages();
}
-static inline bool mem_cgroup_swap_full(struct page *page)
+static inline bool mem_cgroup_swap_full(struct folio *folio)
{
return vm_swap_full();
}
#include <linux/swap.h>
-#ifdef CONFIG_MEMCG_SWAP
+#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
extern unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
unsigned short old, unsigned short new);
return;
}
-#endif /* CONFIG_MEMCG_SWAP */
+#endif
#endif /* __LINUX_SWAP_CGROUP_H */
*/
extern struct swap_info_struct *swap_info[];
extern unsigned long generic_max_swapfile_size(void);
-extern unsigned long max_swapfile_size(void);
+unsigned long arch_max_swapfile_size(void);
+
+/* Maximum swapfile size supported for the arch (not inclusive). */
+extern unsigned long swapfile_maximum_size;
+/* Whether swap migration entry supports storing A/D bits for the arch */
+extern bool swap_migration_ad_supported;
#endif /* _LINUX_SWAPFILE_H */
#ifdef CONFIG_MMU
+#ifdef CONFIG_SWAP
+#include <linux/swapfile.h>
+#endif /* CONFIG_SWAP */
+
/*
* swapcache pages are stored in the swapper_space radix tree. We want to
* get good packing density in that tree, so the index should be dense in
#define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
#define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1)
+/*
+ * Definitions only for PFN swap entries (see is_pfn_swap_entry()). To
+ * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries
+ * can use the extra bits to store other information besides PFN.
+ */
+#ifdef MAX_PHYSMEM_BITS
+#define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
+#else /* MAX_PHYSMEM_BITS */
+#define SWP_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
+#endif /* MAX_PHYSMEM_BITS */
+#define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1)
+
+/**
+ * Migration swap entry specific bitfield definitions. Layout:
+ *
+ * |----------+--------------------|
+ * | swp_type | swp_offset |
+ * |----------+--------+-+-+-------|
+ * | | resv |D|A| PFN |
+ * |----------+--------+-+-+-------|
+ *
+ * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A)
+ * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D)
+ *
+ * Note: A/D bits will be stored in migration entries iff there're enough
+ * free bits in arch specific swp offset. By default we'll ignore A/D bits
+ * when migrating a page. Please refer to migration_entry_supports_ad()
+ * for more information. If there're more bits besides PFN and A/D bits,
+ * they should be reserved and always be zeros.
+ */
+#define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS)
+#define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1)
+#define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2)
+
+#define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT)
+#define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT)
+
+static inline bool is_pfn_swap_entry(swp_entry_t entry);
+
/* Clear all flags but only keep swp_entry_t related information */
static inline pte_t pte_swp_clear_flags(pte_t pte)
{
return entry.val & SWP_OFFSET_MASK;
}
+/*
+ * This should only be called upon a pfn swap entry to get the PFN stored
+ * in the swap entry. Please refers to is_pfn_swap_entry() for definition
+ * of pfn swap entry.
+ */
+static inline unsigned long swp_offset_pfn(swp_entry_t entry)
+{
+ VM_BUG_ON(!is_pfn_swap_entry(entry));
+ return swp_offset(entry) & SWP_PFN_MASK;
+}
+
/* check whether a pte points to a swap entry */
static inline int is_swap_pte(pte_t pte)
{
return swp_entry(SWP_MIGRATION_WRITE, offset);
}
+/*
+ * Returns whether the host has large enough swap offset field to support
+ * carrying over pgtable A/D bits for page migrations. The result is
+ * pretty much arch specific.
+ */
+static inline bool migration_entry_supports_ad(void)
+{
+#ifdef CONFIG_SWAP
+ return swap_migration_ad_supported;
+#else /* CONFIG_SWAP */
+ return false;
+#endif /* CONFIG_SWAP */
+}
+
+static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)
+{
+ if (migration_entry_supports_ad())
+ return swp_entry(swp_type(entry),
+ swp_offset(entry) | SWP_MIG_YOUNG);
+ return entry;
+}
+
+static inline bool is_migration_entry_young(swp_entry_t entry)
+{
+ if (migration_entry_supports_ad())
+ return swp_offset(entry) & SWP_MIG_YOUNG;
+ /* Keep the old behavior of aging page after migration */
+ return false;
+}
+
+static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
+{
+ if (migration_entry_supports_ad())
+ return swp_entry(swp_type(entry),
+ swp_offset(entry) | SWP_MIG_DIRTY);
+ return entry;
+}
+
+static inline bool is_migration_entry_dirty(swp_entry_t entry)
+{
+ if (migration_entry_supports_ad())
+ return swp_offset(entry) & SWP_MIG_DIRTY;
+ /* Keep the old behavior of clean page after migration */
+ return false;
+}
+
extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
spinlock_t *ptl);
extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
#ifdef CONFIG_HUGETLB_PAGE
extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl);
extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte);
-#endif
-#else
+#endif /* CONFIG_HUGETLB_PAGE */
+#else /* CONFIG_MIGRATION */
static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
{
return swp_entry(0, 0);
#ifdef CONFIG_HUGETLB_PAGE
static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { }
static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { }
-#endif
+#endif /* CONFIG_HUGETLB_PAGE */
static inline int is_writable_migration_entry(swp_entry_t entry)
{
return 0;
return 0;
}
-#endif
+static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)
+{
+ return entry;
+}
+
+static inline bool is_migration_entry_young(swp_entry_t entry)
+{
+ return false;
+}
+
+static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
+{
+ return entry;
+}
+
+static inline bool is_migration_entry_dirty(swp_entry_t entry)
+{
+ return false;
+}
+#endif /* CONFIG_MIGRATION */
typedef unsigned long pte_marker;
static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
{
- struct page *p = pfn_to_page(swp_offset(entry));
+ struct page *p = pfn_to_page(swp_offset_pfn(entry));
/*
* Any use of migration entries may only occur while the
*/
static inline bool is_pfn_swap_entry(swp_entry_t entry)
{
+ /* Make sure the swp offset can always store the needed fields */
+ BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);
+
return is_migration_entry(entry) || is_device_private_entry(entry) ||
is_device_exclusive_entry(entry);
}
{
return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd));
}
-#else
+#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
struct page *page)
{
{
return 0;
}
-#endif
+#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
#ifdef CONFIG_MEMORY_FAILURE
return swp_type(entry) == SWP_HWPOISON;
}
-static inline unsigned long hwpoison_entry_to_pfn(swp_entry_t entry)
-{
- return swp_offset(entry);
-}
-
static inline void num_poisoned_pages_inc(void)
{
atomic_long_inc(&num_poisoned_pages);
}
-static inline void num_poisoned_pages_dec(void)
-{
- atomic_long_dec(&num_poisoned_pages);
-}
-
static inline void num_poisoned_pages_sub(long i)
{
atomic_long_sub(i, &num_poisoned_pages);
}
-#else
+#else /* CONFIG_MEMORY_FAILURE */
static inline swp_entry_t make_hwpoison_entry(struct page *page)
{
static inline void num_poisoned_pages_sub(long i)
{
}
-#endif
+#endif /* CONFIG_MEMORY_FAILURE */
static inline int non_swap_entry(swp_entry_t entry)
{
void thermal_of_zone_unregister(struct thermal_zone_device *tz);
-int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
- struct device_node *sensor_np,
- u32 *id);
#else
static inline
struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data,
struct thermal_zone_device *tz)
{
}
-
-static inline int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
- struct device_node *sensor_np,
- u32 *id)
-{
- return -ENOENT;
-}
#endif
#ifdef CONFIG_THERMAL
static __always_inline __must_check unsigned long
__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
{
- instrument_copy_from_user(to, from, n);
+ unsigned long res;
+
+ instrument_copy_from_user_before(to, from, n);
check_object_size(to, n, false);
- return raw_copy_from_user(to, from, n);
+ res = raw_copy_from_user(to, from, n);
+ instrument_copy_from_user_after(to, from, n, res);
+ return res;
}
static __always_inline __must_check unsigned long
__copy_from_user(void *to, const void __user *from, unsigned long n)
{
+ unsigned long res;
+
might_fault();
+ instrument_copy_from_user_before(to, from, n);
if (should_fail_usercopy())
return n;
- instrument_copy_from_user(to, from, n);
check_object_size(to, n, false);
- return raw_copy_from_user(to, from, n);
+ res = raw_copy_from_user(to, from, n);
+ instrument_copy_from_user_after(to, from, n, res);
+ return res;
}
/**
unsigned long res = n;
might_fault();
if (!should_fail_usercopy() && likely(access_ok(from, n))) {
- instrument_copy_from_user(to, from, n);
+ instrument_copy_from_user_before(to, from, n);
res = raw_copy_from_user(to, from, n);
+ instrument_copy_from_user_after(to, from, n, res);
}
if (unlikely(res))
memset(to + (n - res), 0, res);
unsigned long start,
unsigned long end);
-extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- struct list_head *uf);
+extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
+ unsigned long end, struct list_head *uf);
extern void userfaultfd_unmap_complete(struct mm_struct *mm,
struct list_head *uf);
return true;
}
-static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
+static inline int userfaultfd_unmap_prep(struct mm_struct *mm,
unsigned long start, unsigned long end,
struct list_head *uf)
{
};
struct vdpa_dev_set_config {
+ u64 device_features;
struct {
u8 mac[ETH_ALEN];
u16 mtu;
#include <linux/workqueue.h>
#include <linux/poll.h>
#include <uapi/linux/vfio.h>
+#include <linux/iova_bitmap.h>
struct kvm;
struct device *dev;
const struct vfio_device_ops *ops;
/*
- * mig_ops is a static property of the vfio_device which must be set
- * prior to registering the vfio_device.
+ * mig_ops/log_ops is a static property of the vfio_device which must
+ * be set prior to registering the vfio_device.
*/
const struct vfio_migration_ops *mig_ops;
+ const struct vfio_log_ops *log_ops;
struct vfio_group *group;
struct vfio_device_set *dev_set;
struct list_head dev_set_list;
struct kvm *kvm;
/* Members below here are private, not for driver use */
- refcount_t refcount;
+ unsigned int index;
+ struct device device; /* device.kref covers object life circle */
+ refcount_t refcount; /* user count on registered device*/
unsigned int open_count;
struct completion comp;
struct list_head group_next;
/**
* struct vfio_device_ops - VFIO bus driver device callbacks
*
+ * @init: initialize private fields in device structure
+ * @release: Reclaim private fields in device structure
* @open_device: Called when the first file descriptor is opened for this device
* @close_device: Opposite of open_device
* @read: Perform read(2) on device file descriptor
*/
struct vfio_device_ops {
char *name;
+ int (*init)(struct vfio_device *vdev);
+ void (*release)(struct vfio_device *vdev);
int (*open_device)(struct vfio_device *vdev);
void (*close_device)(struct vfio_device *vdev);
ssize_t (*read)(struct vfio_device *vdev, char __user *buf,
enum vfio_device_mig_state *curr_state);
};
+/**
+ * @log_start: Optional callback to ask the device start DMA logging.
+ * @log_stop: Optional callback to ask the device stop DMA logging.
+ * @log_read_and_clear: Optional callback to ask the device read
+ * and clear the dirty DMAs in some given range.
+ *
+ * The vfio core implementation of the DEVICE_FEATURE_DMA_LOGGING_ set
+ * of features does not track logging state relative to the device,
+ * therefore the device implementation of vfio_log_ops must handle
+ * arbitrary user requests. This includes rejecting subsequent calls
+ * to log_start without an intervening log_stop, as well as graceful
+ * handling of log_stop and log_read_and_clear from invalid states.
+ */
+struct vfio_log_ops {
+ int (*log_start)(struct vfio_device *device,
+ struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
+ int (*log_stop)(struct vfio_device *device);
+ int (*log_read_and_clear)(struct vfio_device *device,
+ unsigned long iova, unsigned long length,
+ struct iova_bitmap *dirty);
+};
+
/**
* vfio_check_feature - Validate user input for the VFIO_DEVICE_FEATURE ioctl
* @flags: Arg from the device_feature op
return 1;
}
-void vfio_init_group_dev(struct vfio_device *device, struct device *dev,
- const struct vfio_device_ops *ops);
-void vfio_uninit_group_dev(struct vfio_device *device);
+struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
+ const struct vfio_device_ops *ops);
+#define vfio_alloc_device(dev_struct, member, dev, ops) \
+ container_of(_vfio_alloc_device(sizeof(struct dev_struct) + \
+ BUILD_BUG_ON_ZERO(offsetof( \
+ struct dev_struct, member)), \
+ dev, ops), \
+ struct dev_struct, member)
+
+int vfio_init_device(struct vfio_device *device, struct device *dev,
+ const struct vfio_device_ops *ops);
+void vfio_free_device(struct vfio_device *device);
+static inline void vfio_put_device(struct vfio_device *device)
+{
+ put_device(&device->device);
+}
+
int vfio_register_group_dev(struct vfio_device *device);
int vfio_register_emulated_iommu_dev(struct vfio_device *device);
void vfio_unregister_group_dev(struct vfio_device *device);
* External user API
*/
struct iommu_group *vfio_file_iommu_group(struct file *file);
+bool vfio_file_is_group(struct file *file);
bool vfio_file_enforced_coherent(struct file *file);
void vfio_file_set_kvm(struct file *file, struct kvm *kvm);
bool vfio_file_has_dev(struct file *file, struct vfio_device *device);
#define VFIO_PCI_CORE_H
#define VFIO_PCI_OFFSET_SHIFT 40
-
#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
-/* Special capability IDs predefined access */
-#define PCI_CAP_ID_INVALID 0xFF /* default raw access */
-#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */
-
-/* Cap maximum number of ioeventfds per device (arbitrary) */
-#define VFIO_PCI_IOEVENTFD_MAX 1000
-
-struct vfio_pci_ioeventfd {
- struct list_head next;
- struct vfio_pci_core_device *vdev;
- struct virqfd *virqfd;
- void __iomem *addr;
- uint64_t data;
- loff_t pos;
- int bar;
- int count;
- bool test_mem;
-};
-
-struct vfio_pci_irq_ctx {
- struct eventfd_ctx *trigger;
- struct virqfd *unmask;
- struct virqfd *mask;
- char *name;
- bool masked;
- struct irq_bypass_producer producer;
-};
-
struct vfio_pci_core_device;
struct vfio_pci_region;
u32 flags;
};
-struct vfio_pci_dummy_resource {
- struct resource resource;
- int index;
- struct list_head res_next;
-};
-
-struct vfio_pci_vf_token {
- struct mutex lock;
- uuid_t uuid;
- int users;
-};
-
-struct vfio_pci_mmap_vma {
- struct vm_area_struct *vma;
- struct list_head vma_next;
-};
-
struct vfio_pci_core_device {
struct vfio_device vdev;
struct pci_dev *pdev;
bool needs_reset;
bool nointx;
bool needs_pm_restore;
+ bool pm_intx_masked;
+ bool pm_runtime_engaged;
struct pci_saved_state *pci_saved_state;
struct pci_saved_state *pm_save;
int ioeventfds_nr;
struct eventfd_ctx *err_trigger;
struct eventfd_ctx *req_trigger;
+ struct eventfd_ctx *pm_wake_eventfd_ctx;
struct list_head dummy_resources_list;
struct mutex ioeventfds_lock;
struct list_head ioeventfds_list;
struct rw_semaphore memory_lock;
};
-#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
-#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX)
-#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX)
-#define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev)))
-#define irq_is(vdev, type) (vdev->irq_type == type)
-
-void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);
-void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);
-
-int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev,
- uint32_t flags, unsigned index,
- unsigned start, unsigned count, void *data);
-
-ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
- char __user *buf, size_t count,
- loff_t *ppos, bool iswrite);
-
-ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
- size_t count, loff_t *ppos, bool iswrite);
-
-#ifdef CONFIG_VFIO_PCI_VGA
-ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
- size_t count, loff_t *ppos, bool iswrite);
-#else
-static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev,
- char __user *buf, size_t count,
- loff_t *ppos, bool iswrite)
-{
- return -EINVAL;
-}
-#endif
-
-long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
- uint64_t data, int count, int fd);
-
-int vfio_pci_init_perm_bits(void);
-void vfio_pci_uninit_perm_bits(void);
-
-int vfio_config_init(struct vfio_pci_core_device *vdev);
-void vfio_config_free(struct vfio_pci_core_device *vdev);
-
-int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev,
- unsigned int type, unsigned int subtype,
- const struct vfio_pci_regops *ops,
- size_t size, u32 flags, void *data);
-
-int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,
- pci_power_t state);
-
-bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev);
-void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev);
-u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);
-void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,
- u16 cmd);
-
-#ifdef CONFIG_VFIO_PCI_IGD
-int vfio_pci_igd_init(struct vfio_pci_core_device *vdev);
-#else
-static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
-{
- return -ENODEV;
-}
-#endif
-
-#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
-int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
- struct vfio_info_cap *caps);
-int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev);
-void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev);
-#else
-static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
- struct vfio_info_cap *caps)
-{
- return -ENODEV;
-}
-
-static inline int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev)
-{
- return 0;
-}
-
-static inline void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev)
-{}
-#endif
-
/* Will be exported for vfio pci drivers usage */
+int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
+ unsigned int type, unsigned int subtype,
+ const struct vfio_pci_regops *ops,
+ size_t size, u32 flags, void *data);
void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga,
bool is_disable_idle_d3);
void vfio_pci_core_close_device(struct vfio_device *core_vdev);
-void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
- struct pci_dev *pdev,
- const struct vfio_device_ops *vfio_pci_ops);
+int vfio_pci_core_init_dev(struct vfio_device *core_vdev);
+void vfio_pci_core_release_dev(struct vfio_device *core_vdev);
int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev);
-void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev);
void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev);
extern const struct pci_error_handlers vfio_pci_core_err_handlers;
int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
pci_channel_state_t state);
-static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
-{
- return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
-}
-
#endif /* VFIO_PCI_CORE_H */
u16 index, u32 queue_pfn);
bool vp_legacy_get_queue_enable(struct virtio_pci_legacy_device *ldev,
u16 idx);
-void vp_legacy_set_queue_size(struct virtio_pci_legacy_device *ldev,
- u16 idx, u16 size);
u16 vp_legacy_get_queue_size(struct virtio_pci_legacy_device *ldev,
u16 idx);
int vp_legacy_probe(struct virtio_pci_legacy_device *ldev);
NR_TLB_LOCAL_FLUSH_ALL,
NR_TLB_LOCAL_FLUSH_ONE,
#endif /* CONFIG_DEBUG_TLBFLUSH */
-#ifdef CONFIG_DEBUG_VM_VMACACHE
- VMACACHE_FIND_CALLS,
- VMACACHE_FIND_HITS,
-#endif
#ifdef CONFIG_SWAP
SWAP_RA,
SWAP_RA_HIT,
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_VMACACHE_H
-#define __LINUX_VMACACHE_H
-
-#include <linux/sched.h>
-#include <linux/mm.h>
-
-static inline void vmacache_flush(struct task_struct *tsk)
-{
- memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas));
-}
-
-extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma);
-extern struct vm_area_struct *vmacache_find(struct mm_struct *mm,
- unsigned long addr);
-
-#ifndef CONFIG_MMU
-extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
- unsigned long start,
- unsigned long end);
-#endif
-
-static inline void vmacache_invalidate(struct mm_struct *mm)
-{
- mm->vmacache_seqnum++;
-}
-
-#endif /* __LINUX_VMACACHE_H */
#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
#endif
-#ifdef CONFIG_DEBUG_VM_VMACACHE
-#define count_vm_vmacache_event(x) count_vm_event(x)
-#else
-#define count_vm_vmacache_event(x) do {} while (0)
-#endif
-
#define __count_zid_vm_events(item, zid, delta) \
__count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
struct __compat_iw_event {
__u16 len; /* Real length of this stuff */
__u16 cmd; /* Wireless IOCTL */
- compat_caddr_t pointer;
+
+ union {
+ compat_caddr_t pointer;
+
+ /* we need ptr_bytes to make memcpy() run-time destination
+ * buffer bounds checking happy, nothing special
+ */
+ DECLARE_FLEX_ARRAY(__u8, ptr_bytes);
+ };
};
#define IW_EV_COMPAT_LCP_LEN offsetof(struct __compat_iw_event, pointer)
#define IW_EV_COMPAT_POINT_OFF offsetof(struct compat_iw_point, length)
DECLARE_PER_CPU(int, dirty_throttle_leaks);
/*
- * The 1/4 region under the global dirty thresh is for smooth dirty throttling:
- *
- * (thresh - thresh/DIRTY_FULL_SCOPE, thresh)
- *
- * Further beyond, all dirtier tasks will enter a loop waiting (possibly long
- * time) for the dirty pages to drop, unless written enough pages.
- *
* The global dirty threshold is normally equal to the global dirty limit,
* except when the system suddenly allocates a lot of anonymous memory and
* knocks down the global dirty threshold quickly, in which case the global
* dirty limit will follow down slowly to prevent livelocking all dirtier tasks.
*/
#define DIRTY_SCOPE 8
-#define DIRTY_FULL_SCOPE (DIRTY_SCOPE / 2)
struct backing_dev_info;
/* size of header for zero copy read/write */
#define P9_ZC_HDR_SZ 4096
+/* maximum length of an error string */
+#define P9_ERRMAX 128
+
/**
* struct p9_qid - file system entity information
* @type: 8-bit type &p9_qid_t
* @list: used to maintain a list of currently available transports
* @name: the human-readable name of the transport
* @maxsize: transport provided maximum packet size
+ * @pooled_rbuffers: currently only set for RDMA transport which pulls the
+ * response buffers from a shared pool, and accordingly
+ * we're less flexible when choosing the response message
+ * size in this case
* @def: set if this transport should be considered the default
* @create: member function to create a new connection on this transport
* @close: member function to discard a connection on this transport
struct list_head list;
char *name; /* name of transport */
int maxsize; /* max message size of transport */
+ bool pooled_rbuffers;
int def; /* this transport should be default */
struct module *owner;
int (*create)(struct p9_client *client,
ieee802154_sockaddr_check_size(struct sockaddr_ieee802154 *daddr, int len)
{
struct ieee802154_addr_sa *sa;
+ int ret = 0;
sa = &daddr->addr;
if (len < IEEE802154_MIN_NAMELEN)
return -EINVAL;
switch (sa->addr_type) {
+ case IEEE802154_ADDR_NONE:
+ break;
case IEEE802154_ADDR_SHORT:
if (len < IEEE802154_NAMELEN_SHORT)
- return -EINVAL;
+ ret = -EINVAL;
break;
case IEEE802154_ADDR_LONG:
if (len < IEEE802154_NAMELEN_LONG)
- return -EINVAL;
+ ret = -EINVAL;
+ break;
+ default:
+ ret = -EINVAL;
break;
}
- return 0;
+ return ret;
}
static inline void ieee802154_addr_from_sa(struct ieee802154_addr *a,
void ipv6_local_error(struct sock *sk, int err, struct flowi6 *fl6, u32 info);
void ipv6_local_rxpmtu(struct sock *sk, struct flowi6 *fl6, u32 mtu);
+void inet6_cleanup_sock(struct sock *sk);
+void inet6_sock_destruct(struct sock *sk);
int inet6_release(struct socket *sock);
int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
int inet6_getname(struct socket *sock, struct sockaddr *uaddr,
static inline void init_hashrandom(u32 *jhash_initval)
{
while (*jhash_initval == 0)
- *jhash_initval = prandom_u32();
+ *jhash_initval = get_random_u32();
}
static inline u32 hash_v4(const struct iphdr *iph, u32 initval)
static inline u32 red_random(const struct red_parms *p)
{
- return reciprocal_divide(prandom_u32(), p->max_P_reciprocal);
+ return reciprocal_divide(get_random_u32(), p->max_P_reciprocal);
}
static inline int red_mark_probability(const struct red_parms *p,
static inline u32 net_tx_rndhash(void)
{
- u32 v = prandom_u32();
+ u32 v = get_random_u32();
return v ?: 1;
}
}
/* net/ipv4/udp.c */
-void udp_destruct_sock(struct sock *sk);
+void udp_destruct_common(struct sock *sk);
void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len);
int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb);
void udp_skb_destructor(struct sock *sk, struct sk_buff *skb);
return copy_from_iter_full(to, len, &msg->msg_iter) ? 0 : -EFAULT;
}
-/* Designate sk as UDP-Lite socket */
-static inline int udplite_sk_init(struct sock *sk)
-{
- udp_init_sock(sk);
- udp_sk(sk)->pcflag = UDPLITE_BIT;
- return 0;
-}
-
/*
* Checksumming routines
*/
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Atmel Power Management
- *
- * Copyright (C) 2020 Atmel
- *
- * Author: Lee Jones <lee.jones@linaro.org>
- */
-
-#ifndef __SOC_ATMEL_PM_H
-#define __SOC_ATMEL_PM_H
-
-void at91_pinctrl_gpio_suspend(void);
-void at91_pinctrl_gpio_resume(void);
-
-#endif /* __SOC_ATMEL_PM_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * SiFive Composable Cache Controller header file
+ *
+ */
+
+#ifndef __SOC_SIFIVE_CCACHE_H
+#define __SOC_SIFIVE_CCACHE_H
+
+extern int register_sifive_ccache_error_notifier(struct notifier_block *nb);
+extern int unregister_sifive_ccache_error_notifier(struct notifier_block *nb);
+
+#define SIFIVE_CCACHE_ERR_TYPE_CE 0
+#define SIFIVE_CCACHE_ERR_TYPE_UE 1
+
+#endif /* __SOC_SIFIVE_CCACHE_H */
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * SiFive L2 Cache Controller header file
- *
- */
-
-#ifndef __SOC_SIFIVE_L2_CACHE_H
-#define __SOC_SIFIVE_L2_CACHE_H
-
-extern int register_sifive_l2_error_notifier(struct notifier_block *nb);
-extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb);
-
-#define SIFIVE_L2_ERR_TYPE_CE 0
-#define SIFIVE_L2_ERR_TYPE_UE 1
-
-#endif /* __SOC_SIFIVE_L2_CACHE_H */
#define snd_hdac_stream_readb(dev, reg) \
snd_hdac_reg_readb((dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg)
#define snd_hdac_stream_readb_poll(dev, reg, val, cond, delay_us, timeout_us) \
- readb_poll_timeout((dev)->sd_addr + AZX_REG_ ## reg, val, cond, \
- delay_us, timeout_us)
+ read_poll_timeout_atomic(snd_hdac_reg_readb, val, cond, delay_us, timeout_us, \
+ false, (dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg)
#define snd_hdac_stream_readl_poll(dev, reg, val, cond, delay_us, timeout_us) \
- readl_poll_timeout((dev)->sd_addr + AZX_REG_ ## reg, val, cond, \
- delay_us, timeout_us)
+ read_poll_timeout_atomic(snd_hdac_reg_readl, val, cond, delay_us, timeout_us, \
+ false, (dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg)
/* update a register, pass without AZX_REG_ prefix */
#define snd_hdac_stream_updatel(dev, reg, mask, val) \
TRACE_EVENT(f2fs_update_extent_tree_range,
TP_PROTO(struct inode *inode, unsigned int pgofs, block_t blkaddr,
- unsigned int len),
+ unsigned int len,
+ unsigned int c_len),
- TP_ARGS(inode, pgofs, blkaddr, len),
+ TP_ARGS(inode, pgofs, blkaddr, len, c_len),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(unsigned int, pgofs)
__field(u32, blk)
__field(unsigned int, len)
+ __field(unsigned int, c_len)
),
TP_fast_assign(
__entry->pgofs = pgofs;
__entry->blk = blkaddr;
__entry->len = len;
+ __entry->c_len = c_len;
),
TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, "
- "blkaddr = %u, len = %u",
+ "blkaddr = %u, len = %u, "
+ "c_len = %u",
show_dev_ino(__entry),
__entry->pgofs,
__entry->blk,
- __entry->len)
+ __entry->len,
+ __entry->c_len)
);
TRACE_EVENT(f2fs_shrink_extent_tree,
__field(unsigned long long, app_bio)
__field(unsigned long long, app_wio)
__field(unsigned long long, app_mio)
+ __field(unsigned long long, app_bcdio)
+ __field(unsigned long long, app_mcdio)
__field(unsigned long long, fs_dio)
+ __field(unsigned long long, fs_cdio)
__field(unsigned long long, fs_nio)
__field(unsigned long long, fs_mio)
__field(unsigned long long, fs_gc_dio)
__field(unsigned long long, app_brio)
__field(unsigned long long, app_rio)
__field(unsigned long long, app_mrio)
+ __field(unsigned long long, app_bcrio)
+ __field(unsigned long long, app_mcrio)
__field(unsigned long long, fs_drio)
__field(unsigned long long, fs_gdrio)
__field(unsigned long long, fs_cdrio)
__entry->app_bio = iostat[APP_BUFFERED_IO];
__entry->app_wio = iostat[APP_WRITE_IO];
__entry->app_mio = iostat[APP_MAPPED_IO];
+ __entry->app_bcdio = iostat[APP_BUFFERED_CDATA_IO];
+ __entry->app_mcdio = iostat[APP_MAPPED_CDATA_IO];
__entry->fs_dio = iostat[FS_DATA_IO];
+ __entry->fs_cdio = iostat[FS_CDATA_IO];
__entry->fs_nio = iostat[FS_NODE_IO];
__entry->fs_mio = iostat[FS_META_IO];
__entry->fs_gc_dio = iostat[FS_GC_DATA_IO];
__entry->app_brio = iostat[APP_BUFFERED_READ_IO];
__entry->app_rio = iostat[APP_READ_IO];
__entry->app_mrio = iostat[APP_MAPPED_READ_IO];
+ __entry->app_bcrio = iostat[APP_BUFFERED_CDATA_READ_IO];
+ __entry->app_mcrio = iostat[APP_MAPPED_CDATA_READ_IO];
__entry->fs_drio = iostat[FS_DATA_READ_IO];
__entry->fs_gdrio = iostat[FS_GDATA_READ_IO];
__entry->fs_cdrio = iostat[FS_CDATA_READ_IO];
),
TP_printk("dev = (%d,%d), "
- "app [write=%llu (direct=%llu, buffered=%llu), mapped=%llu], "
- "fs [data=%llu, node=%llu, meta=%llu, discard=%llu], "
+ "app [write=%llu (direct=%llu, buffered=%llu), mapped=%llu, "
+ "compr(buffered=%llu, mapped=%llu)], "
+ "fs [data=%llu, cdata=%llu, node=%llu, meta=%llu, discard=%llu], "
"gc [data=%llu, node=%llu], "
"cp [data=%llu, node=%llu, meta=%llu], "
"app [read=%llu (direct=%llu, buffered=%llu), mapped=%llu], "
- "fs [data=%llu, (gc_data=%llu, compr_data=%llu), "
+ "compr(buffered=%llu, mapped=%llu)], "
+ "fs [data=%llu, (gc_data=%llu, cdata=%llu), "
"node=%llu, meta=%llu]",
show_dev(__entry->dev), __entry->app_wio, __entry->app_dio,
- __entry->app_bio, __entry->app_mio, __entry->fs_dio,
+ __entry->app_bio, __entry->app_mio, __entry->app_bcdio,
+ __entry->app_mcdio, __entry->fs_dio, __entry->fs_cdio,
__entry->fs_nio, __entry->fs_mio, __entry->fs_discard,
__entry->fs_gc_dio, __entry->fs_gc_nio, __entry->fs_cp_dio,
__entry->fs_cp_nio, __entry->fs_cp_mio,
__entry->app_rio, __entry->app_drio, __entry->app_brio,
- __entry->app_mrio, __entry->fs_drio, __entry->fs_gdrio,
+ __entry->app_mrio, __entry->app_bcrio, __entry->app_mcrio,
+ __entry->fs_drio, __entry->fs_gdrio,
__entry->fs_cdrio, __entry->fs_nrio, __entry->fs_mrio)
);
EM( SCAN_FAIL, "failed") \
EM( SCAN_SUCCEED, "succeeded") \
EM( SCAN_PMD_NULL, "pmd_null") \
+ EM( SCAN_PMD_NONE, "pmd_none") \
+ EM( SCAN_PMD_MAPPED, "page_pmd_mapped") \
EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \
EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \
EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \
EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \
EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \
+ EM( SCAN_PTE_MAPPED_HUGEPAGE, "pte_mapped_hugepage") \
EM( SCAN_PAGE_RO, "no_writable_page") \
EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \
EM( SCAN_PAGE_NULL, "page_null") \
__entry->ret)
);
+TRACE_EVENT(mm_khugepaged_scan_file,
+
+ TP_PROTO(struct mm_struct *mm, struct page *page, const char *filename,
+ int present, int swap, int result),
+
+ TP_ARGS(mm, page, filename, present, swap, result),
+
+ TP_STRUCT__entry(
+ __field(struct mm_struct *, mm)
+ __field(unsigned long, pfn)
+ __string(filename, filename)
+ __field(int, present)
+ __field(int, swap)
+ __field(int, result)
+ ),
+
+ TP_fast_assign(
+ __entry->mm = mm;
+ __entry->pfn = page ? page_to_pfn(page) : -1;
+ __assign_str(filename, filename);
+ __entry->present = present;
+ __entry->swap = swap;
+ __entry->result = result;
+ ),
+
+ TP_printk("mm=%p, scan_pfn=0x%lx, filename=%s, present=%d, swap=%d, result=%s",
+ __entry->mm,
+ __entry->pfn,
+ __get_str(filename),
+ __entry->present,
+ __entry->swap,
+ __print_symbolic(__entry->result, SCAN_STATUS))
+);
+
#endif /* __HUGE_MEMORY_H */
#include <trace/define_trace.h>
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM maple_tree
+
+#if !defined(_TRACE_MM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_MM_H
+
+
+#include <linux/tracepoint.h>
+
+struct ma_state;
+
+TRACE_EVENT(ma_op,
+
+ TP_PROTO(const char *fn, struct ma_state *mas),
+
+ TP_ARGS(fn, mas),
+
+ TP_STRUCT__entry(
+ __field(const char *, fn)
+ __field(unsigned long, min)
+ __field(unsigned long, max)
+ __field(unsigned long, index)
+ __field(unsigned long, last)
+ __field(void *, node)
+ ),
+
+ TP_fast_assign(
+ __entry->fn = fn;
+ __entry->min = mas->min;
+ __entry->max = mas->max;
+ __entry->index = mas->index;
+ __entry->last = mas->last;
+ __entry->node = mas->node;
+ ),
+
+ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu",
+ __entry->fn,
+ (void *) __entry->node,
+ (unsigned long) __entry->min,
+ (unsigned long) __entry->max,
+ (unsigned long) __entry->index,
+ (unsigned long) __entry->last
+ )
+)
+TRACE_EVENT(ma_read,
+
+ TP_PROTO(const char *fn, struct ma_state *mas),
+
+ TP_ARGS(fn, mas),
+
+ TP_STRUCT__entry(
+ __field(const char *, fn)
+ __field(unsigned long, min)
+ __field(unsigned long, max)
+ __field(unsigned long, index)
+ __field(unsigned long, last)
+ __field(void *, node)
+ ),
+
+ TP_fast_assign(
+ __entry->fn = fn;
+ __entry->min = mas->min;
+ __entry->max = mas->max;
+ __entry->index = mas->index;
+ __entry->last = mas->last;
+ __entry->node = mas->node;
+ ),
+
+ TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu",
+ __entry->fn,
+ (void *) __entry->node,
+ (unsigned long) __entry->min,
+ (unsigned long) __entry->max,
+ (unsigned long) __entry->index,
+ (unsigned long) __entry->last
+ )
+)
+
+TRACE_EVENT(ma_write,
+
+ TP_PROTO(const char *fn, struct ma_state *mas, unsigned long piv,
+ void *val),
+
+ TP_ARGS(fn, mas, piv, val),
+
+ TP_STRUCT__entry(
+ __field(const char *, fn)
+ __field(unsigned long, min)
+ __field(unsigned long, max)
+ __field(unsigned long, index)
+ __field(unsigned long, last)
+ __field(unsigned long, piv)
+ __field(void *, val)
+ __field(void *, node)
+ ),
+
+ TP_fast_assign(
+ __entry->fn = fn;
+ __entry->min = mas->min;
+ __entry->max = mas->max;
+ __entry->index = mas->index;
+ __entry->last = mas->last;
+ __entry->piv = piv;
+ __entry->val = val;
+ __entry->node = mas->node;
+ ),
+
+ TP_printk("%s\tNode %p (%lu %lu) range:%lu-%lu piv (%lu) val %p",
+ __entry->fn,
+ (void *) __entry->node,
+ (unsigned long) __entry->min,
+ (unsigned long) __entry->max,
+ (unsigned long) __entry->index,
+ (unsigned long) __entry->last,
+ (unsigned long) __entry->piv,
+ (void *) __entry->val
+ )
+)
+#endif /* _TRACE_MM_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
__entry->low_limit, __entry->high_limit, __entry->align_mask,
__entry->align_offset)
);
+
+TRACE_EVENT(vma_mas_szero,
+ TP_PROTO(struct maple_tree *mt, unsigned long start,
+ unsigned long end),
+
+ TP_ARGS(mt, start, end),
+
+ TP_STRUCT__entry(
+ __field(struct maple_tree *, mt)
+ __field(unsigned long, start)
+ __field(unsigned long, end)
+ ),
+
+ TP_fast_assign(
+ __entry->mt = mt;
+ __entry->start = start;
+ __entry->end = end;
+ ),
+
+ TP_printk("mt_mod %p, (NULL), SNULL, %lu, %lu,",
+ __entry->mt,
+ (unsigned long) __entry->start,
+ (unsigned long) __entry->end
+ )
+);
+
+TRACE_EVENT(vma_store,
+ TP_PROTO(struct maple_tree *mt, struct vm_area_struct *vma),
+
+ TP_ARGS(mt, vma),
+
+ TP_STRUCT__entry(
+ __field(struct maple_tree *, mt)
+ __field(struct vm_area_struct *, vma)
+ __field(unsigned long, vm_start)
+ __field(unsigned long, vm_end)
+ ),
+
+ TP_fast_assign(
+ __entry->mt = mt;
+ __entry->vma = vma;
+ __entry->vm_start = vma->vm_start;
+ __entry->vm_end = vma->vm_end - 1;
+ ),
+
+ TP_printk("mt_mod %p, (%p), STORE, %lu, %lu,",
+ __entry->mt, __entry->vma,
+ (unsigned long) __entry->vm_start,
+ (unsigned long) __entry->vm_end
+ )
+);
+
+
+TRACE_EVENT(exit_mmap,
+ TP_PROTO(struct mm_struct *mm),
+
+ TP_ARGS(mm),
+
+ TP_STRUCT__entry(
+ __field(struct mm_struct *, mm)
+ __field(struct maple_tree *, mt)
+ ),
+
+ TP_fast_assign(
+ __entry->mm = mm;
+ __entry->mt = &mm->mm_mt;
+ ),
+
+ TP_printk("mt_mod %p, DESTROY\n",
+ __entry->mt
+ )
+);
+
#endif
/* This part must be outside protection */
#define HUGETLB_FLAG_ENCODE_SHIFT 26
#define HUGETLB_FLAG_ENCODE_MASK 0x3f
-#define HUGETLB_FLAG_ENCODE_16KB (14 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_64KB (16 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_512KB (19 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_1MB (20 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_2MB (21 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_8MB (23 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_16MB (24 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_32MB (25 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_256MB (28 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_512MB (29 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_1GB (30 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_2GB (31 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_16GB (34 << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_16KB (14U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_64KB (16U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_512KB (19U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_1MB (20U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_2MB (21U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_8MB (23U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_16MB (24U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_32MB (25U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_256MB (28U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_512MB (29U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_1GB (30U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_2GB (31U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_16GB (34U << HUGETLB_FLAG_ENCODE_SHIFT)
#endif /* _ASM_GENERIC_HUGETLB_ENCODE_H_ */
#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */
+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */
+
/* compatibility flags */
#define MAP_FILE 0
* - add FUSE_SECURITY_CTX init flag
* - add security context to create, mkdir, symlink, and mknod requests
* - add FUSE_HAS_INODE_DAX, FUSE_ATTR_DAX
+ *
+ * 7.37
+ * - add FUSE_TMPFILE
*/
#ifndef _LINUX_FUSE_H
#define FUSE_KERNEL_VERSION 7
/** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 36
+#define FUSE_KERNEL_MINOR_VERSION 37
/** The node ID of the root inode */
#define FUSE_ROOT_ID 1
FUSE_SETUPMAPPING = 48,
FUSE_REMOVEMAPPING = 49,
FUSE_SYNCFS = 50,
+ FUSE_TMPFILE = 51,
/* CUSE specific operations */
CUSE_INIT = 4096,
#define ABS_TOOL_WIDTH 0x1c
#define ABS_VOLUME 0x20
+#define ABS_PROFILE 0x21
#define ABS_MISC 0x28
#define KVM_CAP_VM_DISABLE_NX_HUGE_PAGES 220
#define KVM_CAP_S390_ZPCI_OP 221
#define KVM_CAP_S390_CPU_TOPOLOGY 222
+#define KVM_CAP_DIRTY_LOG_RING_ACQ_REL 223
#ifdef KVM_CAP_IRQ_ROUTING
#include <linux/types.h>
+/* ioctls for /dev/userfaultfd */
+#define USERFAULTFD_IOC 0xAA
+#define USERFAULTFD_IOC_NEW _IO(USERFAULTFD_IOC, 0x00)
+
/*
* If the UFFDIO_API is upgraded someday, the UFFDIO_UNREGISTER and
* UFFDIO_WAKE ioctls should be defined as _IOW and not as _IOR. In
VDPA_ATTR_DEV_NEGOTIATED_FEATURES, /* u64 */
VDPA_ATTR_DEV_MGMTDEV_MAX_VQS, /* u32 */
+ /* virtio features that are supported by the vDPA management device */
VDPA_ATTR_DEV_SUPPORTED_FEATURES, /* u64 */
VDPA_ATTR_DEV_QUEUE_INDEX, /* u32 */
VDPA_ATTR_DEV_VENDOR_ATTR_NAME, /* string */
VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, /* u64 */
+ VDPA_ATTR_DEV_FEATURES, /* u64 */
+
+ /* virtio features that are supported by the vDPA device */
+ VDPA_ATTR_VDPA_DEV_SUPPORTED_FEATURES, /* u64 */
+
/* new attributes must be added above here */
VDPA_ATTR_MAX,
};
VFIO_DEVICE_STATE_RUNNING_P2P = 5,
};
+/*
+ * Upon VFIO_DEVICE_FEATURE_SET, allow the device to be moved into a low power
+ * state with the platform-based power management. Device use of lower power
+ * states depends on factors managed by the runtime power management core,
+ * including system level support and coordinating support among dependent
+ * devices. Enabling device low power entry does not guarantee lower power
+ * usage by the device, nor is a mechanism provided through this feature to
+ * know the current power state of the device. If any device access happens
+ * (either from the host or through the vfio uAPI) when the device is in the
+ * low power state, then the host will move the device out of the low power
+ * state as necessary prior to the access. Once the access is completed, the
+ * device may re-enter the low power state. For single shot low power support
+ * with wake-up notification, see
+ * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP below. Access to mmap'd
+ * device regions is disabled on LOW_POWER_ENTRY and may only be resumed after
+ * calling LOW_POWER_EXIT.
+ */
+#define VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY 3
+
+/*
+ * This device feature has the same behavior as
+ * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY with the exception that the user
+ * provides an eventfd for wake-up notification. When the device moves out of
+ * the low power state for the wake-up, the host will not allow the device to
+ * re-enter a low power state without a subsequent user call to one of the low
+ * power entry device feature IOCTLs. Access to mmap'd device regions is
+ * disabled on LOW_POWER_ENTRY_WITH_WAKEUP and may only be resumed after the
+ * low power exit. The low power exit can happen either through LOW_POWER_EXIT
+ * or through any other access (where the wake-up notification has been
+ * generated). The access to mmap'd device regions will not trigger low power
+ * exit.
+ *
+ * The notification through the provided eventfd will be generated only when
+ * the device has entered and is resumed from a low power state after
+ * calling this device feature IOCTL. A device that has not entered low power
+ * state, as managed through the runtime power management core, will not
+ * generate a notification through the provided eventfd on access. Calling the
+ * LOW_POWER_EXIT feature is optional in the case where notification has been
+ * signaled on the provided eventfd that a resume from low power has occurred.
+ */
+struct vfio_device_low_power_entry_with_wakeup {
+ __s32 wakeup_eventfd;
+ __u32 reserved;
+};
+
+#define VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP 4
+
+/*
+ * Upon VFIO_DEVICE_FEATURE_SET, disallow use of device low power states as
+ * previously enabled via VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY or
+ * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP device features.
+ * This device feature IOCTL may itself generate a wakeup eventfd notification
+ * in the latter case if the device had previously entered a low power state.
+ */
+#define VFIO_DEVICE_FEATURE_LOW_POWER_EXIT 5
+
+/*
+ * Upon VFIO_DEVICE_FEATURE_SET start/stop device DMA logging.
+ * VFIO_DEVICE_FEATURE_PROBE can be used to detect if the device supports
+ * DMA logging.
+ *
+ * DMA logging allows a device to internally record what DMAs the device is
+ * initiating and report them back to userspace. It is part of the VFIO
+ * migration infrastructure that allows implementing dirty page tracking
+ * during the pre copy phase of live migration. Only DMA WRITEs are logged,
+ * and this API is not connected to VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE.
+ *
+ * When DMA logging is started a range of IOVAs to monitor is provided and the
+ * device can optimize its logging to cover only the IOVA range given. Each
+ * DMA that the device initiates inside the range will be logged by the device
+ * for later retrieval.
+ *
+ * page_size is an input that hints what tracking granularity the device
+ * should try to achieve. If the device cannot do the hinted page size then
+ * it's the driver choice which page size to pick based on its support.
+ * On output the device will return the page size it selected.
+ *
+ * ranges is a pointer to an array of
+ * struct vfio_device_feature_dma_logging_range.
+ *
+ * The core kernel code guarantees to support by minimum num_ranges that fit
+ * into a single kernel page. User space can try higher values but should give
+ * up if the above can't be achieved as of some driver limitations.
+ *
+ * A single call to start device DMA logging can be issued and a matching stop
+ * should follow at the end. Another start is not allowed in the meantime.
+ */
+struct vfio_device_feature_dma_logging_control {
+ __aligned_u64 page_size;
+ __u32 num_ranges;
+ __u32 __reserved;
+ __aligned_u64 ranges;
+};
+
+struct vfio_device_feature_dma_logging_range {
+ __aligned_u64 iova;
+ __aligned_u64 length;
+};
+
+#define VFIO_DEVICE_FEATURE_DMA_LOGGING_START 6
+
+/*
+ * Upon VFIO_DEVICE_FEATURE_SET stop device DMA logging that was started
+ * by VFIO_DEVICE_FEATURE_DMA_LOGGING_START
+ */
+#define VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP 7
+
+/*
+ * Upon VFIO_DEVICE_FEATURE_GET read back and clear the device DMA log
+ *
+ * Query the device's DMA log for written pages within the given IOVA range.
+ * During querying the log is cleared for the IOVA range.
+ *
+ * bitmap is a pointer to an array of u64s that will hold the output bitmap
+ * with 1 bit reporting a page_size unit of IOVA. The mapping of IOVA to bits
+ * is given by:
+ * bitmap[(addr - iova)/page_size] & (1ULL << (addr % 64))
+ *
+ * The input page_size can be any power of two value and does not have to
+ * match the value given to VFIO_DEVICE_FEATURE_DMA_LOGGING_START. The driver
+ * will format its internal logging to match the reporting page size, possibly
+ * by replicating bits if the internal page size is lower than requested.
+ *
+ * The LOGGING_REPORT will only set bits in the bitmap and never clear or
+ * perform any initialization of the user provided bitmap.
+ *
+ * If any error is returned userspace should assume that the dirty log is
+ * corrupted. Error recovery is to consider all memory dirty and try to
+ * restart the dirty tracking, or to abort/restart the whole migration.
+ *
+ * If DMA logging is not enabled, an error will be returned.
+ *
+ */
+struct vfio_device_feature_dma_logging_report {
+ __aligned_u64 iova;
+ __aligned_u64 length;
+ __aligned_u64 page_size;
+ __aligned_u64 bitmap;
+};
+
+#define VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT 8
+
/* -------- API for Type1 VFIO IOMMU -------- */
/**
#define VIRTIO_BLK_F_MQ 12 /* support more than one vq */
#define VIRTIO_BLK_F_DISCARD 13 /* DISCARD is supported */
#define VIRTIO_BLK_F_WRITE_ZEROES 14 /* WRITE ZEROES is supported */
+#define VIRTIO_BLK_F_SECURE_ERASE 16 /* Secure Erase is supported */
/* Legacy feature bits */
#ifndef VIRTIO_BLK_NO_LEGACY
__u8 write_zeroes_may_unmap;
__u8 unused1[3];
+
+ /* the next 3 entries are guarded by VIRTIO_BLK_F_SECURE_ERASE */
+ /*
+ * The maximum secure erase sectors (in 512-byte sectors) for
+ * one segment.
+ */
+ __virtio32 max_secure_erase_sectors;
+ /*
+ * The maximum number of secure erase segments in a
+ * secure erase command.
+ */
+ __virtio32 max_secure_erase_seg;
+ /* Secure erase commands must be aligned to this number of sectors. */
+ __virtio32 secure_erase_sector_alignment;
+
} __attribute__((packed));
/*
/* Write zeroes command */
#define VIRTIO_BLK_T_WRITE_ZEROES 13
+/* Secure erase command */
+#define VIRTIO_BLK_T_SECURE_ERASE 14
+
#ifndef VIRTIO_BLK_NO_LEGACY
/* Barrier before this op. */
#define VIRTIO_BLK_T_BARRIER 0x80000000
* @vid_hdr_offset: VID header offset (use defaults if %0)
* @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs
* @padding: reserved for future, not used, has to be zeroed
+ * @disable_fm: whether disable fastmap
*
* This data structure is used to specify MTD device UBI has to attach and the
* parameters it has to use. The number which should be assigned to the new UBI
* eraseblocks for new bad eraseblocks, but attempts to use available
* eraseblocks (if any). The accepted range is 0-768. If 0 is given, the
* default kernel value of %CONFIG_MTD_UBI_BEB_LIMIT will be used.
+ *
+ * If @disable_fm is not zero, ubi doesn't create new fastmap even the module
+ * param 'fm_autoconvert' is set, and existed old fastmap will be destroyed
+ * after doing full scanning.
*/
struct ubi_attach_req {
__s32 ubi_num;
__s32 mtd_num;
__s32 vid_hdr_offset;
__s16 max_beb_per1024;
- __s8 padding[10];
+ __s8 disable_fm;
+ __s8 padding[9];
};
/*
void xen_grant_setup_dma_ops(struct device *dev);
bool xen_is_grant_dma_device(struct device *dev);
bool xen_virtio_mem_acc(struct virtio_device *dev);
+bool xen_virtio_restricted_mem_acc(struct virtio_device *dev);
#else
static inline void xen_grant_setup_dma_ops(struct device *dev)
{
{
return false;
}
+
+static inline bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
+{
+ return false;
+}
#endif /* CONFIG_XEN_GRANT_DMA_OPS */
#endif /* INCLUDE_XEN_OPS_H */
help
Provides control over the memory footprint of tasks in a cgroup.
-config MEMCG_SWAP
- bool
- depends on MEMCG && SWAP
- default y
-
config MEMCG_KMEM
bool
depends on MEMCG && !SLOB
config CHECKPOINT_RESTORE
bool "Checkpoint/restore support"
+ depends on PROC_FS
select PROC_CHILDREN
select KCMP
default n
static int __init root_dev_setup(char *line)
{
- strlcpy(saved_root_name, line, sizeof(saved_root_name));
+ strscpy(saved_root_name, line, sizeof(saved_root_name));
return 1;
}
int count = 1;
char *p = page;
- strlcpy(p, root_fs_names, size);
+ strscpy(p, root_fs_names, size);
while (*p++) {
if (p[-1] == ',') {
p[-1] = '\0';
return origLen;
}
-static unsigned long my_inptr; /* index of next byte to be processed in inbuf */
+static unsigned long my_inptr __initdata; /* index of next byte to be processed in inbuf */
#include <linux/decompress/generic.h>
#include <linux/percpu.h>
#include <linux/kmod.h>
#include <linux/kprobes.h>
+#include <linux/kmsan.h>
#include <linux/vmalloc.h>
#include <linux/kernel_stat.h>
#include <linux/start_kernel.h>
extern void init_IRQ(void);
extern void radix_tree_init(void);
+extern void maple_tree_init(void);
/*
* Debug helper: via this flag we know that we are in 'early bootup code'
if (!data)
data = xbc_get_embedded_bootconfig(&size);
- strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
+ strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
bootconfig_params);
return;
/* All fall through to do_early_param. */
- strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
+ strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
parse_early_options(tmp_cmdline);
done = 1;
}
init_mem_debugging_and_hardening();
kfence_alloc_pool();
report_meminit();
+ kmsan_init_shadow();
stack_depot_early_init();
mem_init();
mem_init_print_info();
pgtable_init();
debug_objects_mem_init();
vmalloc_init();
+ /* Should be run after vmap initialization */
+ if (early_page_ext_enabled())
+ page_ext_init();
/* Should be run before the first non-init thread is created */
init_espfix_bsp();
/* Should be run after espfix64 is set up. */
pti_init();
+ kmsan_init_runtime();
}
#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
"Interrupts were enabled *very* early, fixing it\n"))
local_irq_disable();
radix_tree_init();
+ maple_tree_init();
/*
* Set up housekeeping before setting up workqueues to allow the unbound
static __init_or_module void
trace_initcall_start_cb(void *data, initcall_t fn)
{
- ktime_t *calltime = (ktime_t *)data;
+ ktime_t *calltime = data;
printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current));
*calltime = ktime_get();
static __init_or_module void
trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
{
- ktime_t rettime, *calltime = (ktime_t *)data;
+ ktime_t rettime, *calltime = data;
rettime = ktime_get();
printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
padata_init();
page_alloc_init_late();
/* Initialize page ext after all struct pages are initialized. */
- page_ext_init();
+ if (!early_page_ext_enabled())
+ page_ext_init();
do_basic_setup();
sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
if (sq_idx > sq_mask)
continue;
- sqe = &ctx->sq_sqes[sq_idx << 1];
+ sqe = &ctx->sq_sqes[sq_idx << sq_shift];
seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, "
"addr:0x%llx, rw_flags:0x%x, buf_index:%d "
"user_data:%llu",
if (!llist_add(&req->io_task_work.node, &ctx->work_llist))
return;
+ /* need it for the following io_cqring_wake() */
+ smp_mb__after_atomic();
if (unlikely(atomic_read(&req->task->io_uring->in_idle))) {
io_move_task_work_from_local(ctx);
if (ctx->has_evfd)
io_eventfd_signal(ctx);
- io_cqring_wake(ctx);
-
+ __io_cqring_wake(ctx);
}
static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local)
static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
io_sq_thread_finish(ctx);
-
- if (ctx->mm_account) {
- mmdrop(ctx->mm_account);
- ctx->mm_account = NULL;
- }
-
io_rsrc_refs_drop(ctx);
/* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
io_wait_rsrc_data(ctx->buf_data);
}
#endif
WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
- WARN_ON_ONCE(ctx->notif_slots || ctx->nr_notif_slots);
+ if (ctx->mm_account) {
+ mmdrop(ctx->mm_account);
+ ctx->mm_account = NULL;
+ }
io_mem_free(ctx->rings);
io_mem_free(ctx->sq_sqes);
mutex_unlock(&ctx->uring_lock);
goto out;
}
- if ((flags & IORING_ENTER_GETEVENTS) && ctx->syscall_iopoll)
- goto iopoll_locked;
+ if (flags & IORING_ENTER_GETEVENTS) {
+ if (ctx->syscall_iopoll)
+ goto iopoll_locked;
+ /*
+ * Ignore errors, we'll soon call io_cqring_wait() and
+ * it should handle ownership problems if any.
+ */
+ if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+ (void)io_run_local_work_locked(ctx);
+ }
mutex_unlock(&ctx->uring_lock);
}
if (fd < 0)
return fd;
- ret = __io_uring_add_tctx_node(ctx, false);
+ ret = __io_uring_add_tctx_node(ctx);
if (ret) {
put_unused_fd(fd);
return ret;
if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))
return -ENXIO;
+ if (ctx->submitter_task && ctx->submitter_task != current)
+ return -EEXIST;
+
if (ctx->restricted) {
if (opcode >= IORING_REGISTER_LAST)
return -EINVAL;
smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
}
-static inline void io_cqring_wake(struct io_ring_ctx *ctx)
+/* requires smb_mb() prior, see wq_has_sleeper() */
+static inline void __io_cqring_wake(struct io_ring_ctx *ctx)
{
/*
* wake_up_all() may seem excessive, but io_wake_function() and
* io_should_wake() handle the termination of the loop and only
* wake as many waiters as we need to.
*/
- if (wq_has_sleeper(&ctx->cq_wait))
+ if (waitqueue_active(&ctx->cq_wait))
wake_up_all(&ctx->cq_wait);
}
+static inline void io_cqring_wake(struct io_ring_ctx *ctx)
+{
+ smp_mb();
+ __io_cqring_wake(ctx);
+}
+
static inline bool io_sqring_full(struct io_ring_ctx *ctx)
{
struct io_rings *r = ctx->rings;
return ret;
}
+static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
+{
+ if (llist_empty(&ctx->work_llist))
+ return 0;
+ return __io_run_local_work(ctx, true);
+}
+
static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
{
if (!*locked) {
struct file *file;
struct sockaddr __user *addr;
int addr_len;
+ bool in_progress;
};
struct io_sr_msg {
conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
conn->addr_len = READ_ONCE(sqe->addr2);
+ conn->in_progress = false;
return 0;
}
int ret;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
+ if (connect->in_progress) {
+ struct socket *socket;
+
+ ret = -ENOTSOCK;
+ socket = sock_from_file(req->file);
+ if (socket)
+ ret = sock_error(socket->sk);
+ goto out;
+ }
+
if (req_has_async_data(req)) {
io = req->async_data;
} else {
ret = __sys_connect_file(req->file, &io->address,
connect->addr_len, file_flags);
if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
- if (req_has_async_data(req))
- return -EAGAIN;
- if (io_alloc_async_data(req)) {
- ret = -ENOMEM;
- goto out;
+ if (ret == -EINPROGRESS) {
+ connect->in_progress = true;
+ } else {
+ if (req_has_async_data(req))
+ return -EAGAIN;
+ if (io_alloc_async_data(req)) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memcpy(req->async_data, &__io, sizeof(__io));
}
- memcpy(req->async_data, &__io, sizeof(__io));
return -EAGAIN;
}
if (ret == -ERESTARTSYS)
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
- .audit_skip = 1,
.ioprio = 1,
.manual_alloc = 1,
#if defined(CONFIG_NET)
UNIXCB(skb).fp = fpl;
skb->sk = sk;
+ skb->scm_io_uring = 1;
skb->destructor = unix_destruct_scm;
refcount_add(skb->truesize, &sk->sk_wmem_alloc);
}
}
}
+/*
+ * Trigger the notifications after having done some IO, and finish the write
+ * accounting, if any.
+ */
+static void io_req_io_end(struct io_kiocb *req)
+{
+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+
+ WARN_ON(!in_task());
+
+ if (rw->kiocb.ki_flags & IOCB_WRITE) {
+ kiocb_end_write(req);
+ fsnotify_modify(req->file);
+ } else {
+ fsnotify_access(req->file);
+ }
+}
+
static bool __io_complete_rw_common(struct io_kiocb *req, long res)
{
if (unlikely(res != req->cqe.res)) {
if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
io_rw_should_reissue(req)) {
+ /*
+ * Reissue will start accounting again, finish the
+ * current cycle.
+ */
+ io_req_io_end(req);
req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO;
return true;
}
static void io_req_rw_complete(struct io_kiocb *req, bool *locked)
{
- struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
-
- if (rw->kiocb.ki_flags & IOCB_WRITE) {
- kiocb_end_write(req);
- fsnotify_modify(req->file);
- } else {
- fsnotify_access(req->file);
- }
-
+ io_req_io_end(req);
io_req_task_complete(req, locked);
}
req->file->f_pos = rw->kiocb.ki_pos;
if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {
if (!__io_complete_rw_common(req, ret)) {
+ /*
+ * Safe to call io_end from here as we're inline
+ * from the submission path.
+ */
+ io_req_io_end(req);
io_req_set_res(req, final_ret,
io_put_kbuf(req, issue_flags));
return IOU_OK;
goto copy_iov;
if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) {
- struct io_async_rw *rw;
+ struct io_async_rw *io;
trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,
req->cqe.res, ret2);
iov_iter_save_state(&s->iter, &s->iter_state);
ret = io_setup_async_rw(req, iovec, s, true);
- rw = req->async_data;
- if (rw)
- rw->bytes_done += ret2;
+ io = req->async_data;
+ if (io)
+ io->bytes_done += ret2;
if (kiocb->ki_flags & IOCB_WRITE)
kiocb_end_write(req);
return 0;
}
-static int io_register_submitter(struct io_ring_ctx *ctx)
-{
- int ret = 0;
-
- mutex_lock(&ctx->uring_lock);
- if (!ctx->submitter_task)
- ctx->submitter_task = get_task_struct(current);
- else if (ctx->submitter_task != current)
- ret = -EEXIST;
- mutex_unlock(&ctx->uring_lock);
-
- return ret;
-}
-
-int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter)
+int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
{
struct io_uring_task *tctx = current->io_uring;
struct io_tctx_node *node;
int ret;
- if ((ctx->flags & IORING_SETUP_SINGLE_ISSUER) && submitter) {
- ret = io_register_submitter(ctx);
- if (ret)
- return ret;
- }
-
if (unlikely(!tctx)) {
ret = io_uring_alloc_task_context(current, ctx);
if (unlikely(ret))
list_add(&node->ctx_node, &ctx->tctx_list);
mutex_unlock(&ctx->uring_lock);
}
- if (submitter)
- tctx->last = ctx;
+ return 0;
+}
+
+int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx)
+{
+ int ret;
+
+ if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
+ && ctx->submitter_task != current)
+ return -EEXIST;
+
+ ret = __io_uring_add_tctx_node(ctx);
+ if (ret)
+ return ret;
+
+ current->io_uring->last = ctx;
return 0;
}
return -EINVAL;
mutex_unlock(&ctx->uring_lock);
- ret = __io_uring_add_tctx_node(ctx, false);
+ ret = __io_uring_add_tctx_node(ctx);
mutex_lock(&ctx->uring_lock);
if (ret)
return ret;
int io_uring_alloc_task_context(struct task_struct *task,
struct io_ring_ctx *ctx);
void io_uring_del_tctx_node(unsigned long index);
-int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter);
+int __io_uring_add_tctx_node(struct io_ring_ctx *ctx);
+int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx);
void io_uring_clean_tctx(struct io_uring_task *tctx);
void io_uring_unreg_ringfd(void);
if (likely(tctx && tctx->last == ctx))
return 0;
- return __io_uring_add_tctx_node(ctx, true);
+
+ return __io_uring_add_tctx_node_from_submit(ctx);
}
out_unlock:
inode_unlock(d_inode(mnt->mnt_root));
- if (inode)
- iput(inode);
+ iput(inode);
mnt_drop_write(mnt);
out_name:
putname(name);
#include <linux/nsproxy.h>
#include <linux/ipc_namespace.h>
#include <linux/rhashtable.h>
+#include <linux/percpu_counter.h>
#include <asm/current.h>
#include <linux/uaccess.h>
rcu_read_unlock();
list_for_each_entry_safe(msg, t, &msq->q_messages, m_list) {
- atomic_dec(&ns->msg_hdrs);
+ percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1);
free_msg(msg);
}
- atomic_sub(msq->q_cbytes, &ns->msg_bytes);
+ percpu_counter_sub_local(&ns->percpu_msg_bytes, msq->q_cbytes);
ipc_update_pid(&msq->q_lspid, NULL);
ipc_update_pid(&msq->q_lrpid, NULL);
ipc_rcu_putref(&msq->q_perm, msg_rcu_free);
msginfo->msgssz = MSGSSZ;
msginfo->msgseg = MSGSEG;
down_read(&msg_ids(ns).rwsem);
- if (cmd == MSG_INFO) {
+ if (cmd == MSG_INFO)
msginfo->msgpool = msg_ids(ns).in_use;
- msginfo->msgmap = atomic_read(&ns->msg_hdrs);
- msginfo->msgtql = atomic_read(&ns->msg_bytes);
+ max_idx = ipc_get_maxidx(&msg_ids(ns));
+ up_read(&msg_ids(ns).rwsem);
+ if (cmd == MSG_INFO) {
+ msginfo->msgmap = min_t(int,
+ percpu_counter_sum(&ns->percpu_msg_hdrs),
+ INT_MAX);
+ msginfo->msgtql = min_t(int,
+ percpu_counter_sum(&ns->percpu_msg_bytes),
+ INT_MAX);
} else {
msginfo->msgmap = MSGMAP;
msginfo->msgpool = MSGPOOL;
msginfo->msgtql = MSGTQL;
}
- max_idx = ipc_get_maxidx(&msg_ids(ns));
- up_read(&msg_ids(ns).rwsem);
return (max_idx < 0) ? 0 : max_idx;
}
list_add_tail(&msg->m_list, &msq->q_messages);
msq->q_cbytes += msgsz;
msq->q_qnum++;
- atomic_add(msgsz, &ns->msg_bytes);
- atomic_inc(&ns->msg_hdrs);
+ percpu_counter_add_local(&ns->percpu_msg_bytes, msgsz);
+ percpu_counter_add_local(&ns->percpu_msg_hdrs, 1);
}
err = 0;
msq->q_rtime = ktime_get_real_seconds();
ipc_update_pid(&msq->q_lrpid, task_tgid(current));
msq->q_cbytes -= msg->m_ts;
- atomic_sub(msg->m_ts, &ns->msg_bytes);
- atomic_dec(&ns->msg_hdrs);
+ percpu_counter_sub_local(&ns->percpu_msg_bytes, msg->m_ts);
+ percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1);
ss_wakeup(msq, &wake_q, false);
goto out_unlock0;
}
#endif
-void msg_init_ns(struct ipc_namespace *ns)
+int msg_init_ns(struct ipc_namespace *ns)
{
+ int ret;
+
ns->msg_ctlmax = MSGMAX;
ns->msg_ctlmnb = MSGMNB;
ns->msg_ctlmni = MSGMNI;
- atomic_set(&ns->msg_bytes, 0);
- atomic_set(&ns->msg_hdrs, 0);
+ ret = percpu_counter_init(&ns->percpu_msg_bytes, 0, GFP_KERNEL);
+ if (ret)
+ goto fail_msg_bytes;
+ ret = percpu_counter_init(&ns->percpu_msg_hdrs, 0, GFP_KERNEL);
+ if (ret)
+ goto fail_msg_hdrs;
ipc_init_ids(&ns->ids[IPC_MSG_IDS]);
+ return 0;
+
+fail_msg_hdrs:
+ percpu_counter_destroy(&ns->percpu_msg_bytes);
+fail_msg_bytes:
+ return ret;
}
#ifdef CONFIG_IPC_NS
void msg_exit_ns(struct ipc_namespace *ns)
{
+ percpu_counter_destroy(&ns->percpu_msg_bytes);
+ percpu_counter_destroy(&ns->percpu_msg_hdrs);
free_ipcs(ns, &msg_ids(ns), freeque);
idr_destroy(&ns->ids[IPC_MSG_IDS].ipcs_idr);
rhashtable_destroy(&ns->ids[IPC_MSG_IDS].key_ht);
if (!setup_ipc_sysctls(ns))
goto fail_mq;
+ err = msg_init_ns(ns);
+ if (err)
+ goto fail_put;
+
sem_init_ns(ns);
- msg_init_ns(ns);
shm_init_ns(ns);
return ns;
#ifdef CONFIG_MMU
loff_t size = 0;
struct file *file;
- struct vm_area_struct *next;
+ VMA_ITERATOR(vmi, mm, addr);
#endif
if (addr & ~PAGE_MASK)
* match the usual checks anyway. So assume all vma's are
* above the starting address given.
*/
- vma = find_vma(mm, addr);
#ifdef CONFIG_MMU
- while (vma) {
- next = vma->vm_next;
-
+ for_each_vma(vmi, vma) {
/*
* Check if the starting address would match, i.e. it's
* a fragment created by mprotect() and/or munmap(), or it
file = vma->vm_file;
size = i_size_read(file_inode(vma->vm_file));
do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
+ mas_pause(&vmi.mas);
/*
* We discovered the size of the shm segment, so
* break out of here and fall through to the next
* searching for matching vma's.
*/
retval = 0;
- vma = next;
+ vma = vma_next(&vmi);
break;
}
- vma = next;
}
/*
*/
size = PAGE_ALIGN(size);
while (vma && (loff_t)(vma->vm_end - addr) <= size) {
- next = vma->vm_next;
-
/* finding a matching vma now does not alter retval */
if ((vma->vm_ops == &shm_vm_ops) &&
((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
- (vma->vm_file == file))
+ (vma->vm_file == file)) {
do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
- vma = next;
+ mas_pause(&vmi.mas);
+ }
+
+ vma = vma_next(&vmi);
}
#else /* CONFIG_MMU */
+ vma = vma_lookup(mm, addr);
/* under NOMMU conditions, the exact address to be destroyed must be
* given
*/
return iter->pid_ns;
}
-/*
- * This routine locks the ipc structure found at least at position pos.
+/**
+ * sysvipc_find_ipc - Find and lock the ipc structure based on seq pos
+ * @ids: ipc identifier set
+ * @pos: expected position
+ *
+ * The function finds an ipc structure, based on the sequence file
+ * position @pos. If there is no ipc structure at position @pos, then
+ * the successor is selected.
+ * If a structure is found, then it is locked (both rcu_read_lock() and
+ * ipc_lock_object()) and @pos is set to the position needed to locate
+ * the found ipc structure.
+ * If nothing is found (i.e. EOF), @pos is not modified.
+ *
+ * The function returns the found ipc structure, or NULL at EOF.
*/
-static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t pos,
- loff_t *new_pos)
+static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t *pos)
{
- struct kern_ipc_perm *ipc = NULL;
- int max_idx = ipc_get_maxidx(ids);
+ int tmpidx;
+ struct kern_ipc_perm *ipc;
- if (max_idx == -1 || pos > max_idx)
- goto out;
+ /* convert from position to idr index -> "-1" */
+ tmpidx = *pos - 1;
- for (; pos <= max_idx; pos++) {
- ipc = idr_find(&ids->ipcs_idr, pos);
- if (ipc != NULL) {
- rcu_read_lock();
- ipc_lock_object(ipc);
- break;
- }
+ ipc = idr_get_next(&ids->ipcs_idr, &tmpidx);
+ if (ipc != NULL) {
+ rcu_read_lock();
+ ipc_lock_object(ipc);
+
+ /* convert from idr index to position -> "+1" */
+ *pos = tmpidx + 1;
}
-out:
- *new_pos = pos + 1;
return ipc;
}
if (ipc && ipc != SEQ_START_TOKEN)
ipc_unlock(ipc);
- return sysvipc_find_ipc(&iter->ns->ids[iface->ids], *pos, pos);
+ /* Next -> search for *pos+1 */
+ (*pos)++;
+ return sysvipc_find_ipc(&iter->ns->ids[iface->ids], pos);
}
/*
- * File positions: pos 0 -> header, pos n -> ipc id = n - 1.
+ * File positions: pos 0 -> header, pos n -> ipc idx = n - 1.
* SeqFile iterator: iterator value locked ipc pointer or SEQ_TOKEN_START.
*/
static void *sysvipc_proc_start(struct seq_file *s, loff_t *pos)
if (*pos == 0)
return SEQ_START_TOKEN;
- /* Find the (pos-1)th ipc */
- return sysvipc_find_ipc(ids, *pos - 1, pos);
+ /* Otherwise return the correct ipc structure */
+ return sysvipc_find_ipc(ids, pos);
}
static void sysvipc_proc_stop(struct seq_file *s, void *it)
#ifdef CONFIG_SYSVIPC
void sem_init_ns(struct ipc_namespace *ns);
-void msg_init_ns(struct ipc_namespace *ns);
+int msg_init_ns(struct ipc_namespace *ns);
void shm_init_ns(struct ipc_namespace *ns);
void sem_exit_ns(struct ipc_namespace *ns);
void shm_exit_ns(struct ipc_namespace *ns);
#else
static inline void sem_init_ns(struct ipc_namespace *ns) { }
-static inline void msg_init_ns(struct ipc_namespace *ns) { }
+static inline int msg_init_ns(struct ipc_namespace *ns) { return 0; }
static inline void shm_init_ns(struct ipc_namespace *ns) { }
static inline void sem_exit_ns(struct ipc_namespace *ns) { }
KASAN_SANITIZE_kcov.o := n
KCSAN_SANITIZE_kcov.o := n
UBSAN_SANITIZE_kcov.o := n
+KMSAN_SANITIZE_kcov.o := n
CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector
# Don't instrument error handlers
unsigned long vsize = 0;
if (group_dead && current->mm) {
+ struct mm_struct *mm = current->mm;
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
- mmap_read_lock(current->mm);
- vma = current->mm->mmap;
- while (vma) {
+ mmap_read_lock(mm);
+ for_each_vma(vmi, vma)
vsize += vma->vm_end - vma->vm_start;
- vma = vma->vm_next;
- }
- mmap_read_unlock(current->mm);
+ mmap_read_unlock(mm);
}
spin_lock_irq(¤t->sighand->siglock);
DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
#endif
DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
+#ifdef CONFIG_LRU_GEN
+ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
+ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2);
+#else
+ DEFINE(LRU_GEN_WIDTH, 0);
+ DEFINE(__LRU_REFS_WIDTH, 0);
+#endif
/* End of constants */
return 0;
attr->value_size / sizeof(u32);
if (!(attr->map_flags & BPF_F_ZERO_SEED))
- bloom->hash_seed = get_random_int();
+ bloom->hash_seed = get_random_u32();
return &bloom->map;
}
hdr->size = size;
hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
PAGE_SIZE - sizeof(*hdr));
- start = (get_random_int() % hole) & ~(alignment - 1);
+ start = prandom_u32_max(hole) & ~(alignment - 1);
/* Leave a random number of instructions before BPF code. */
*image_ptr = &hdr->image[start];
hole = min_t(unsigned int, size - (proglen + sizeof(*ro_header)),
BPF_PROG_CHUNK_SIZE - sizeof(*ro_header));
- start = (get_random_int() % hole) & ~(alignment - 1);
+ start = prandom_u32_max(hole) & ~(alignment - 1);
*image_ptr = &ro_header->image[start];
*rw_image = &(*rw_header)->image[start];
bool emit_zext)
{
struct bpf_insn *to = to_buff;
- u32 imm_rnd = get_random_int();
+ u32 imm_rnd = get_random_u32();
s16 off;
BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG);
static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \
{ \
u64 stack[stack_size / sizeof(u64)]; \
- u64 regs[MAX_BPF_EXT_REG]; \
+ u64 regs[MAX_BPF_EXT_REG] = {}; \
\
FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
ARG1 = (u64) (unsigned long) ctx; \
if (htab->map.map_flags & BPF_F_ZERO_SEED)
htab->hashrnd = 0;
else
- htab->hashrnd = get_random_int();
+ htab->hashrnd = get_random_u32();
htab_init_buckets(htab);
};
enum bpf_task_vma_iter_find_op {
- task_vma_iter_first_vma, /* use mm->mmap */
- task_vma_iter_next_vma, /* use curr_vma->vm_next */
+ task_vma_iter_first_vma, /* use find_vma() with addr 0 */
+ task_vma_iter_next_vma, /* use vma_next() with curr_vma */
task_vma_iter_find_vma, /* use find_vma() to find next vma */
};
switch (op) {
case task_vma_iter_first_vma:
- curr_vma = curr_task->mm->mmap;
+ curr_vma = find_vma(curr_task->mm, 0);
break;
case task_vma_iter_next_vma:
- curr_vma = curr_vma->vm_next;
+ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end);
break;
case task_vma_iter_find_vma:
/* We dropped mmap_lock so it is necessary to use find_vma
if (curr_vma &&
curr_vma->vm_start == info->prev_vm_start &&
curr_vma->vm_end == info->prev_vm_end)
- curr_vma = curr_vma->vm_next;
+ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end);
break;
}
if (!curr_vma) {
aux[adj_idx].ptr_type == PTR_TO_CTX)
continue;
- imm_rnd = get_random_int();
+ imm_rnd = get_random_u32();
rnd_hi32_patch[0] = insn;
rnd_hi32_patch[1].imm = imm_rnd;
rnd_hi32_patch[3].dst_reg = load_reg;
#define DEFINE_CGROUP_MGCTX(name) \
struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name)
-extern struct mutex cgroup_mutex;
extern spinlock_t css_set_lock;
extern struct cgroup_subsys *cgroup_subsys[];
extern struct list_head cgroup_roots;
static int cgroup_io_pressure_show(struct seq_file *seq, void *v)
{
struct cgroup *cgrp = seq_css(seq)->cgroup;
- struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
+ struct psi_group *psi = cgroup_psi(cgrp);
return psi_show(seq, psi, PSI_IO);
}
static int cgroup_memory_pressure_show(struct seq_file *seq, void *v)
{
struct cgroup *cgrp = seq_css(seq)->cgroup;
- struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
+ struct psi_group *psi = cgroup_psi(cgrp);
return psi_show(seq, psi, PSI_MEM);
}
static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
{
struct cgroup *cgrp = seq_css(seq)->cgroup;
- struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
+ struct psi_group *psi = cgroup_psi(cgrp);
return psi_show(seq, psi, PSI_CPU);
}
-static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,
- size_t nbytes, enum psi_res res)
+static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
+ size_t nbytes, enum psi_res res)
{
struct cgroup_file_ctx *ctx = of->priv;
struct psi_trigger *new;
return -EBUSY;
}
- psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
+ psi = cgroup_psi(cgrp);
new = psi_trigger_create(psi, buf, res);
if (IS_ERR(new)) {
cgroup_put(cgrp);
char *buf, size_t nbytes,
loff_t off)
{
- return cgroup_pressure_write(of, buf, nbytes, PSI_IO);
+ return pressure_write(of, buf, nbytes, PSI_IO);
}
static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,
char *buf, size_t nbytes,
loff_t off)
{
- return cgroup_pressure_write(of, buf, nbytes, PSI_MEM);
+ return pressure_write(of, buf, nbytes, PSI_MEM);
}
static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,
char *buf, size_t nbytes,
loff_t off)
{
- return cgroup_pressure_write(of, buf, nbytes, PSI_CPU);
+ return pressure_write(of, buf, nbytes, PSI_CPU);
+}
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+static int cgroup_irq_pressure_show(struct seq_file *seq, void *v)
+{
+ struct cgroup *cgrp = seq_css(seq)->cgroup;
+ struct psi_group *psi = cgroup_psi(cgrp);
+
+ return psi_show(seq, psi, PSI_IRQ);
+}
+
+static ssize_t cgroup_irq_pressure_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes,
+ loff_t off)
+{
+ return pressure_write(of, buf, nbytes, PSI_IRQ);
+}
+#endif
+
+static int cgroup_pressure_show(struct seq_file *seq, void *v)
+{
+ struct cgroup *cgrp = seq_css(seq)->cgroup;
+ struct psi_group *psi = cgroup_psi(cgrp);
+
+ seq_printf(seq, "%d\n", psi->enabled);
+
+ return 0;
+}
+
+static ssize_t cgroup_pressure_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes,
+ loff_t off)
+{
+ ssize_t ret;
+ int enable;
+ struct cgroup *cgrp;
+ struct psi_group *psi;
+
+ ret = kstrtoint(strstrip(buf), 0, &enable);
+ if (ret)
+ return ret;
+
+ if (enable < 0 || enable > 1)
+ return -ERANGE;
+
+ cgrp = cgroup_kn_lock_live(of->kn, false);
+ if (!cgrp)
+ return -ENOENT;
+
+ psi = cgroup_psi(cgrp);
+ if (psi->enabled != enable) {
+ int i;
+
+ /* show or hide {cpu,memory,io,irq}.pressure files */
+ for (i = 0; i < NR_PSI_RESOURCES; i++)
+ cgroup_file_show(&cgrp->psi_files[i], enable);
+
+ psi->enabled = enable;
+ if (enable)
+ psi_cgroup_restart(psi);
+ }
+
+ cgroup_kn_unlock(of->kn);
+
+ return nbytes;
}
static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,
bool cgroup_psi_enabled(void)
{
+ if (static_branch_likely(&psi_disabled))
+ return false;
+
return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0;
}
#ifdef CONFIG_PSI
{
.name = "io.pressure",
+ .file_offset = offsetof(struct cgroup, psi_files[PSI_IO]),
.seq_show = cgroup_io_pressure_show,
.write = cgroup_io_pressure_write,
.poll = cgroup_pressure_poll,
},
{
.name = "memory.pressure",
+ .file_offset = offsetof(struct cgroup, psi_files[PSI_MEM]),
.seq_show = cgroup_memory_pressure_show,
.write = cgroup_memory_pressure_write,
.poll = cgroup_pressure_poll,
},
{
.name = "cpu.pressure",
+ .file_offset = offsetof(struct cgroup, psi_files[PSI_CPU]),
.seq_show = cgroup_cpu_pressure_show,
.write = cgroup_cpu_pressure_write,
.poll = cgroup_pressure_poll,
.release = cgroup_pressure_release,
},
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+ {
+ .name = "irq.pressure",
+ .file_offset = offsetof(struct cgroup, psi_files[PSI_IRQ]),
+ .seq_show = cgroup_irq_pressure_show,
+ .write = cgroup_irq_pressure_write,
+ .poll = cgroup_pressure_poll,
+ .release = cgroup_pressure_release,
+ },
+#endif
+ {
+ .name = "cgroup.pressure",
+ .seq_show = cgroup_pressure_show,
+ .write = cgroup_pressure_write,
+ },
#endif /* CONFIG_PSI */
{ } /* terminate */
};
#include <linux/pid.h>
#include <linux/smp.h>
#include <linux/mm.h>
-#include <linux/vmacache.h>
#include <linux/rcupdate.h>
#include <linux/irq.h>
#include <linux/security.h>
if (!CACHE_FLUSH_IS_SAFE)
return;
- if (current->mm) {
- int i;
-
- for (i = 0; i < VMACACHE_SIZE; i++) {
- if (!current->vmacache.vmas[i])
- continue;
- flush_cache_range(current->vmacache.vmas[i],
- addr, addr + BREAK_INSTR_SIZE);
- }
- }
-
/* Force flush instruction cache if it was outside the mm */
flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
}
¤t->delays->freepages_count);
}
-void __delayacct_thrashing_start(void)
+void __delayacct_thrashing_start(bool *in_thrashing)
{
+ *in_thrashing = !!current->in_thrashing;
+ if (*in_thrashing)
+ return;
+
+ current->in_thrashing = 1;
current->delays->thrashing_start = local_clock();
}
-void __delayacct_thrashing_end(void)
+void __delayacct_thrashing_end(bool *in_thrashing)
{
+ if (*in_thrashing)
+ return;
+
+ current->in_thrashing = 0;
delayacct_end(¤t->delays->lock,
¤t->delays->thrashing_start,
¤t->delays->thrashing_delay,
#include <linux/dma-map-ops.h>
#include <linux/export.h>
#include <linux/gfp.h>
+#include <linux/kmsan.h>
#include <linux/of_device.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
else
addr = ops->map_page(dev, page, offset, size, dir, attrs);
+ kmsan_handle_dma(page, offset, size, dir);
debug_dma_map_page(dev, page, offset, size, dir, addr, attrs);
return addr;
else
ents = ops->map_sg(dev, sg, nents, dir, attrs);
- if (ents > 0)
+ if (ents > 0) {
+ kmsan_handle_dma_sg(sg, nents, dir);
debug_dma_map_sg(dev, sg, nents, ents, dir, attrs);
- else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
- ents != -EIO && ents != -EREMOTEIO))
+ } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
+ ents != -EIO && ents != -EREMOTEIO)) {
return -EIO;
+ }
return ents;
}
#include <linux/resume_user_mode.h>
#include <linux/highmem.h>
#include <linux/jump_label.h>
+#include <linux/kmsan.h>
#include <linux/livepatch.h>
#include <linux/audit.h>
#include <linux/tick.h>
user_exit_irqoff();
instrumentation_begin();
+ kmsan_unpoison_entry_regs(regs);
trace_hardirqs_off_finish();
instrumentation_end();
}
lockdep_hardirqs_off(CALLER_ADDR0);
ct_irq_enter();
instrumentation_begin();
+ kmsan_unpoison_entry_regs(regs);
trace_hardirqs_off_finish();
instrumentation_end();
*/
lockdep_hardirqs_off(CALLER_ADDR0);
instrumentation_begin();
+ kmsan_unpoison_entry_regs(regs);
rcu_irq_enter_check_tick();
trace_hardirqs_off_finish();
instrumentation_end();
ct_nmi_enter();
instrumentation_begin();
+ kmsan_unpoison_entry_regs(regs);
trace_hardirqs_off_finish();
ftrace_nmi_enter();
instrumentation_end();
struct perf_addr_filter_range *fr)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (!vma->vm_file)
continue;
#include <linux/export.h>
#include <linux/rmap.h> /* anon_vma_prepare */
#include <linux/mmu_notifier.h> /* set_pte_at_notify */
-#include <linux/swap.h> /* try_to_free_swap */
+#include <linux/swap.h> /* folio_free_swap */
#include <linux/ptrace.h> /* user_enable_single_step */
#include <linux/kdebug.h> /* notifier mechanism */
#include "../../mm/internal.h" /* munlock_vma_page */
static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
struct page *old_page, struct page *new_page)
{
+ struct folio *old_folio = page_folio(old_page);
+ struct folio *new_folio;
struct mm_struct *mm = vma->vm_mm;
- DEFINE_FOLIO_VMA_WALK(pvmw, page_folio(old_page), vma, addr, 0);
+ DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0);
int err;
struct mmu_notifier_range range;
addr + PAGE_SIZE);
if (new_page) {
- err = mem_cgroup_charge(page_folio(new_page), vma->vm_mm,
- GFP_KERNEL);
+ new_folio = page_folio(new_page);
+ err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL);
if (err)
return err;
}
- /* For try_to_free_swap() below */
- lock_page(old_page);
+ /* For folio_free_swap() below */
+ folio_lock(old_folio);
mmu_notifier_invalidate_range_start(&range);
err = -EAGAIN;
VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
if (new_page) {
- get_page(new_page);
+ folio_get(new_folio);
page_add_new_anon_rmap(new_page, vma, addr);
- lru_cache_add_inactive_or_unevictable(new_page, vma);
+ folio_add_lru_vma(new_folio, vma);
} else
/* no new page, just dec_mm_counter for old_page */
dec_mm_counter(mm, MM_ANONPAGES);
- if (!PageAnon(old_page)) {
+ if (!folio_test_anon(old_folio)) {
dec_mm_counter(mm, mm_counter_file(old_page));
inc_mm_counter(mm, MM_ANONPAGES);
}
mk_pte(new_page, vma->vm_page_prot));
page_remove_rmap(old_page, vma, false);
- if (!page_mapped(old_page))
- try_to_free_swap(old_page);
+ if (!folio_mapped(old_folio))
+ folio_free_swap(old_folio);
page_vma_mapped_walk_done(&pvmw);
- put_page(old_page);
+ folio_put(old_folio);
err = 0;
unlock:
mmu_notifier_invalidate_range_end(&range);
- unlock_page(old_page);
+ folio_unlock(old_folio);
return err;
}
static struct vm_area_struct *
find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *tmp;
- for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
+ for_each_vma(vmi, tmp)
if (valid_ref_ctr_vma(uprobe, tmp))
return tmp;
/* try collapse pmd for compound page */
if (!ret && orig_page_huge)
- collapse_pte_mapped_thp(mm, vaddr);
+ collapse_pte_mapped_thp(mm, vaddr, false);
return ret;
}
static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
int err = 0;
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
unsigned long vaddr;
loff_t offset;
static void mmf_recalc_uprobes(struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (!valid_vma(vma, false))
continue;
/*
#include <linux/writeback.h>
#include <linux/shm.h>
#include <linux/kcov.h>
+#include <linux/kmsan.h>
#include <linux/random.h>
#include <linux/rcuwait.h>
#include <linux/compat.h>
call_rcu(&task->rcu, delayed_put_task_struct);
}
+void __weak release_thread(struct task_struct *dead_task)
+{
+}
+
void release_task(struct task_struct *p)
{
struct task_struct *leader;
goto retry;
}
WRITE_ONCE(mm->owner, c);
+ lru_gen_migrate_mm(mm);
task_unlock(c);
put_task_struct(c);
}
WARN_ON(tsk->plug);
kcov_task_exit(tsk);
+ kmsan_task_exit(tsk);
coredump_task_exit(tsk);
ptrace_event(PTRACE_EVENT_EXIT, code);
/* cut off if it is too long */
if (count > KSYM_NAME_LEN)
count = KSYM_NAME_LEN;
- buf = kmalloc(count + 1, GFP_KERNEL);
- if (!buf)
- return -ENOMEM;
- if (copy_from_user(buf, buffer, count)) {
- ret = -EFAULT;
- goto out_free;
- }
- buf[count] = '\0';
+ buf = memdup_user_nul(buffer, count);
+ if (IS_ERR(buf))
+ return PTR_ERR(buf);
+
sym = strstrip(buf);
mutex_lock(&fei_lock);
}
ret = register_kprobe(&attr->kp);
- if (!ret)
- fei_debugfs_add_attr(attr);
- if (ret < 0)
- fei_attr_remove(attr);
- else {
- list_add_tail(&attr->list, &fei_attr_list);
- ret = count;
+ if (ret) {
+ fei_attr_free(attr);
+ goto out;
}
+ fei_debugfs_add_attr(attr);
+ list_add_tail(&attr->list, &fei_attr_list);
+ ret = count;
out:
mutex_unlock(&fei_lock);
-out_free:
kfree(buf);
return ret;
}
#include <linux/fdtable.h>
#include <linux/iocontext.h>
#include <linux/key.h>
+#include <linux/kmsan.h>
#include <linux/binfmts.h>
#include <linux/mman.h>
#include <linux/mmu_notifier.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
-#include <linux/vmacache.h>
#include <linux/nsproxy.h>
#include <linux/capability.h>
#include <linux/cpu.h>
#include <linux/scs.h>
#include <linux/io_uring.h>
#include <linux/bpf.h>
-#include <linux/sched/mm.h>
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
*/
*new = data_race(*orig);
INIT_LIST_HEAD(&new->anon_vma_chain);
- new->vm_next = new->vm_prev = NULL;
dup_anon_vma_name(orig, new);
}
return new;
static __latent_entropy int dup_mmap(struct mm_struct *mm,
struct mm_struct *oldmm)
{
- struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
- struct rb_node **rb_link, *rb_parent;
+ struct vm_area_struct *mpnt, *tmp;
int retval;
- unsigned long charge;
+ unsigned long charge = 0;
LIST_HEAD(uf);
+ MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
uprobe_start_dup_mmap();
if (mmap_write_lock_killable(oldmm)) {
mm->exec_vm = oldmm->exec_vm;
mm->stack_vm = oldmm->stack_vm;
- rb_link = &mm->mm_rb.rb_node;
- rb_parent = NULL;
- pprev = &mm->mmap;
retval = ksm_fork(mm, oldmm);
if (retval)
goto out;
khugepaged_fork(mm, oldmm);
- prev = NULL;
- for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
+ retval = mas_expected_entries(&mas, oldmm->map_count);
+ if (retval)
+ goto out;
+
+ mas_for_each(&old_mas, mpnt, ULONG_MAX) {
struct file *file;
if (mpnt->vm_flags & VM_DONTCOPY) {
*/
if (fatal_signal_pending(current)) {
retval = -EINTR;
- goto out;
+ goto loop_out;
}
if (mpnt->vm_flags & VM_ACCOUNT) {
unsigned long len = vma_pages(mpnt);
}
/*
- * Clear hugetlb-related page reserves for children. This only
- * affects MAP_PRIVATE mappings. Faults generated by the child
- * are not guaranteed to succeed, even if read-only
+ * Copy/update hugetlb private vma information.
*/
if (is_vm_hugetlb_page(tmp))
- reset_vma_resv_huge_pages(tmp);
-
- /*
- * Link in the new vma and copy the page table entries.
- */
- *pprev = tmp;
- pprev = &tmp->vm_next;
- tmp->vm_prev = prev;
- prev = tmp;
+ hugetlb_dup_vma_private(tmp);
- __vma_link_rb(mm, tmp, rb_link, rb_parent);
- rb_link = &tmp->vm_rb.rb_right;
- rb_parent = &tmp->vm_rb;
+ /* Link the vma into the MT */
+ mas.index = tmp->vm_start;
+ mas.last = tmp->vm_end - 1;
+ mas_store(&mas, tmp);
+ if (mas_is_err(&mas))
+ goto fail_nomem_mas_store;
mm->map_count++;
if (!(tmp->vm_flags & VM_WIPEONFORK))
tmp->vm_ops->open(tmp);
if (retval)
- goto out;
+ goto loop_out;
}
/* a new mm has just been created */
retval = arch_dup_mmap(oldmm, mm);
+loop_out:
+ mas_destroy(&mas);
out:
mmap_write_unlock(mm);
flush_tlb_mm(oldmm);
fail_uprobe_end:
uprobe_end_dup_mmap();
return retval;
+
+fail_nomem_mas_store:
+ unlink_anon_vmas(tmp);
fail_nomem_anon_vma_fork:
mpol_put(vma_policy(tmp));
fail_nomem_policy:
fail_nomem:
retval = -ENOMEM;
vm_unacct_memory(charge);
- goto out;
+ goto loop_out;
}
static inline int mm_alloc_pgd(struct mm_struct *mm)
tsk->worker_private = NULL;
kcov_task_init(tsk);
+ kmsan_task_create(tsk);
kmap_local_fork(tsk);
#ifdef CONFIG_FAULT_INJECTION
static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
struct user_namespace *user_ns)
{
- mm->mmap = NULL;
- mm->mm_rb = RB_ROOT;
- mm->vmacache_seqnum = 0;
+ mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
+ mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
atomic_set(&mm->mm_users, 1);
atomic_set(&mm->mm_count, 1);
seqcount_init(&mm->write_protect_seq);
goto fail_nocontext;
mm->user_ns = get_user_ns(user_ns);
+ lru_gen_init_mm(mm);
return mm;
fail_nocontext:
}
if (mm->binfmt)
module_put(mm->binfmt->module);
+ lru_gen_del_mm(mm);
mmdrop(mm);
}
/* Forbid mm->exe_file change if old file still mapped. */
old_exe_file = get_mm_exe_file(mm);
if (old_exe_file) {
+ VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (!vma->vm_file)
continue;
if (path_equal(&vma->vm_file->f_path,
- &old_exe_file->f_path))
+ &old_exe_file->f_path)) {
ret = -EBUSY;
+ break;
+ }
}
mmap_read_unlock(mm);
fput(old_exe_file);
if (!oldmm)
return 0;
- /* initialize the new vmacache entries */
- vmacache_flush(tsk);
-
if (clone_flags & CLONE_VM) {
mmget(oldmm);
mm = oldmm;
get_task_struct(p);
}
+ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) {
+ /* lock the task to synchronize with memcg migration */
+ task_lock(p);
+ lru_gen_add_mm(p->mm);
+ task_unlock(p);
+ }
+
wake_up_new_task(p);
/* forking complete and child started to run, tell ptracer */
}
EXPORT_SYMBOL_GPL(generic_handle_domain_irq);
+ /**
+ * generic_handle_irq_safe - Invoke the handler for a HW irq belonging
+ * to a domain from any context.
+ * @domain: The domain where to perform the lookup
+ * @hwirq: The HW irq number to convert to a logical one
+ *
+ * Returns: 0 on success, a negative value on error.
+ *
+ * This function can be called from any context (IRQ or process
+ * context). If the interrupt is marked as 'enforce IRQ-context only' then
+ * the function must be invoked from hard interrupt context.
+ */
+int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq)
+{
+ unsigned long flags;
+ int ret;
+
+ local_irq_save(flags);
+ ret = handle_irq_desc(irq_resolve_mapping(domain, hwirq));
+ local_irq_restore(flags);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(generic_handle_domain_irq_safe);
+
/**
* generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging
* to a domain.
#include <linux/fs.h>
#include <linux/hashtable.h>
#include <linux/init.h>
+#include <linux/kmsan-checks.h>
#include <linux/mm.h>
#include <linux/preempt.h>
#include <linux/printk.h>
INIT_LIST_HEAD(&area->list);
area->size = size;
list_add(&area->list, &kcov_remote_areas);
+ /*
+ * KMSAN doesn't instrument this file, so it may not know area->list
+ * is initialized. Unpoison it explicitly to avoid reports in
+ * kcov_remote_area_get().
+ */
+ kmsan_unpoison_memory(&area->list, sizeof(area->list));
}
static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
static bool __init test_requires(void)
{
/* random should be initialized for the below tests */
- return prandom_u32() + prandom_u32() != 0;
+ return get_random_u32() + get_random_u32() != 0;
}
/*
unsigned long addr;
size_t verif_size;
- prandom_bytes(&addr, sizeof(addr));
+ get_random_bytes(&addr, sizeof(addr));
if (addr < PAGE_SIZE)
addr = PAGE_SIZE;
/*
* Because we write directly to the reserved memory region when loading
- * crash kernels we need a mutex here to prevent multiple crash kernels
- * from attempting to load simultaneously, and to prevent a crash kernel
- * from loading over the top of a in use crash kernel.
- *
- * KISS: always take the mutex.
+ * crash kernels we need a serialization here to prevent multiple crash
+ * kernels from attempting to load simultaneously.
*/
- if (!mutex_trylock(&kexec_mutex))
+ if (!kexec_trylock())
return -EBUSY;
if (flags & KEXEC_ON_CRASH) {
kimage_free(image);
out_unlock:
- mutex_unlock(&kexec_mutex);
+ kexec_unlock();
return ret;
}
#include <crypto/hash.h>
#include "kexec_internal.h"
-DEFINE_MUTEX(kexec_mutex);
+atomic_t __kexec_lock = ATOMIC_INIT(0);
/* Per cpu memory for storing cpu states in case of system crash. */
note_buf_t __percpu *crash_notes;
if (result < 0)
goto out;
- ptr = kmap(page);
+ ptr = kmap_local_page(page);
/* Start with a clear page */
clear_page(ptr);
ptr += maddr & ~PAGE_MASK;
memcpy(ptr, kbuf, uchunk);
else
result = copy_from_user(ptr, buf, uchunk);
- kunmap(page);
+ kunmap_local(ptr);
if (result) {
result = -EFAULT;
goto out;
goto out;
}
arch_kexec_post_alloc_pages(page_address(page), 1, 0);
- ptr = kmap(page);
+ ptr = kmap_local_page(page);
ptr += maddr & ~PAGE_MASK;
mchunk = min_t(size_t, mbytes,
PAGE_SIZE - (maddr & ~PAGE_MASK));
else
result = copy_from_user(ptr, buf, uchunk);
kexec_flush_icache_page(page);
- kunmap(page);
+ kunmap_local(ptr);
arch_kexec_pre_free_pages(page_address(page), 1);
if (result) {
result = -EFAULT;
*/
void __noclone __crash_kexec(struct pt_regs *regs)
{
- /* Take the kexec_mutex here to prevent sys_kexec_load
+ /* Take the kexec_lock here to prevent sys_kexec_load
* running on one cpu from replacing the crash kernel
* we are using after a panic on a different cpu.
*
* of memory the xchg(&kexec_crash_image) would be
* sufficient. But since I reuse the memory...
*/
- if (mutex_trylock(&kexec_mutex)) {
+ if (kexec_trylock()) {
if (kexec_crash_image) {
struct pt_regs fixed_regs;
machine_crash_shutdown(&fixed_regs);
machine_kexec(kexec_crash_image);
}
- mutex_unlock(&kexec_mutex);
+ kexec_unlock();
}
}
STACK_FRAME_NON_STANDARD(__crash_kexec);
}
}
-size_t crash_get_memory_size(void)
+ssize_t crash_get_memory_size(void)
{
- size_t size = 0;
+ ssize_t size = 0;
+
+ if (!kexec_trylock())
+ return -EBUSY;
- mutex_lock(&kexec_mutex);
if (crashk_res.end != crashk_res.start)
size = resource_size(&crashk_res);
- mutex_unlock(&kexec_mutex);
+
+ kexec_unlock();
return size;
}
unsigned long old_size;
struct resource *ram_res;
- mutex_lock(&kexec_mutex);
+ if (!kexec_trylock())
+ return -EBUSY;
if (kexec_crash_image) {
ret = -ENOENT;
insert_resource(&iomem_resource, ram_res);
unlock:
- mutex_unlock(&kexec_mutex);
+ kexec_unlock();
return ret;
}
{
int error = 0;
- if (!mutex_trylock(&kexec_mutex))
+ if (!kexec_trylock())
return -EBUSY;
if (!kexec_image) {
error = -EINVAL;
#endif
Unlock:
- mutex_unlock(&kexec_mutex);
+ kexec_unlock();
return error;
}
image = NULL;
- if (!mutex_trylock(&kexec_mutex))
+ if (!kexec_trylock())
return -EBUSY;
dest_image = &kexec_image;
if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image)
arch_kexec_protect_crashkres();
- mutex_unlock(&kexec_mutex);
+ kexec_unlock();
kimage_free(image);
return ret;
}
int kimage_is_destination_range(struct kimage *image,
unsigned long start, unsigned long end);
-extern struct mutex kexec_mutex;
+/*
+ * Whatever is used to serialize accesses to the kexec_crash_image needs to be
+ * NMI safe, as __crash_kexec() can happen during nmi_panic(), so here we use a
+ * "simple" atomic variable that is acquired with a cmpxchg().
+ */
+extern atomic_t __kexec_lock;
+static inline bool kexec_trylock(void)
+{
+ return atomic_cmpxchg_acquire(&__kexec_lock, 0, 1) == 0;
+}
+static inline void kexec_unlock(void)
+{
+ atomic_set_release(&__kexec_lock, 0);
+}
#ifdef CONFIG_KEXEC_FILE
#include <linux/purgatory.h>
static ssize_t kexec_crash_size_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
- return sprintf(buf, "%zu\n", crash_get_memory_size());
+ ssize_t size = crash_get_memory_size();
+
+ if (size < 0)
+ return size;
+
+ return sprintf(buf, "%zd\n", size);
}
static ssize_t kexec_crash_size_store(struct kobject *kobj,
struct kobj_attribute *attr,
account_global_scheduler_latency(struct task_struct *tsk,
struct latency_record *lat)
{
- int firstnonnull = MAXLR + 1;
+ int firstnonnull = MAXLR;
int i;
/* skip kernel threads for now */
}
i = firstnonnull;
- if (i >= MAXLR - 1)
+ if (i >= MAXLR)
return;
/* Allocted a new one: */
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
-# Avoid recursion lockdep -> KCSAN -> ... -> lockdep.
+# Avoid recursion lockdep -> sanitizer -> ... -> lockdep.
KCSAN_SANITIZE_lockdep.o := n
+KMSAN_SANITIZE_lockdep.o := n
ifdef CONFIG_FUNCTION_TRACER
CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE)
order[n] = n;
for (n = count - 1; n > 1; n--) {
- r = get_random_int() % (n + 1);
+ r = prandom_u32_max(n + 1);
if (r != n) {
tmp = order[n];
order[n] = order[r];
{
struct stress *stress = container_of(work, typeof(*stress), work);
const int nlocks = stress->nlocks;
- struct ww_mutex *lock = stress->locks + (get_random_int() % nlocks);
+ struct ww_mutex *lock = stress->locks + prandom_u32_max(nlocks);
int err;
do {
{
return idr_get_next(&ns->idr, &nr);
}
+EXPORT_SYMBOL_GPL(find_ge_pid);
struct pid *pidfd_get_pid(unsigned int fd, unsigned int *flags)
{
static const char schedstr[] = "schedule";
static const char sleepstr[] = "sleep";
static const char kvmstr[] = "kvm";
+ const char *select = NULL;
int par;
if (!strncmp(str, sleepstr, strlen(sleepstr))) {
#ifdef CONFIG_SCHEDSTATS
force_schedstat_enabled();
prof_on = SLEEP_PROFILING;
- if (str[strlen(sleepstr)] == ',')
- str += strlen(sleepstr) + 1;
- if (get_option(&str, &par))
- prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
- pr_info("kernel sleep profiling enabled (shift: %u)\n",
- prof_shift);
+ select = sleepstr;
#else
pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
#endif /* CONFIG_SCHEDSTATS */
} else if (!strncmp(str, schedstr, strlen(schedstr))) {
prof_on = SCHED_PROFILING;
- if (str[strlen(schedstr)] == ',')
- str += strlen(schedstr) + 1;
- if (get_option(&str, &par))
- prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
- pr_info("kernel schedule profiling enabled (shift: %u)\n",
- prof_shift);
+ select = schedstr;
} else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
prof_on = KVM_PROFILING;
- if (str[strlen(kvmstr)] == ',')
- str += strlen(kvmstr) + 1;
- if (get_option(&str, &par))
- prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
- pr_info("kernel KVM profiling enabled (shift: %u)\n",
- prof_shift);
+ select = kvmstr;
} else if (get_option(&str, &par)) {
prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
prof_on = CPU_PROFILING;
pr_info("kernel profiling enabled (shift: %u)\n",
prof_shift);
}
+
+ if (select) {
+ if (str[strlen(select)] == ',')
+ str += strlen(select) + 1;
+ if (get_option(&str, &par))
+ prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
+ pr_info("kernel %s profiling enabled (shift: %u)\n",
+ select, prof_shift);
+ }
+
return 1;
}
__setup("profile=", profile_setup);
*/
static struct page **relay_alloc_page_array(unsigned int n_pages)
{
- const size_t pa_size = n_pages * sizeof(struct page *);
- if (pa_size > PAGE_SIZE)
- return vzalloc(pa_size);
- return kzalloc(pa_size, GFP_KERNEL);
+ return kvcalloc(n_pages, sizeof(struct page *), GFP_KERNEL);
}
/*
rq->prev_irq_time += irq_delta;
delta -= irq_delta;
+ psi_account_irqtime(rq->curr, irq_delta);
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
if (static_key_false((¶virt_steal_rq_enabled))) {
}
#ifdef CONFIG_PROC_SYSCTL
+static void reset_memory_tiering(void)
+{
+ struct pglist_data *pgdat;
+
+ for_each_online_pgdat(pgdat) {
+ pgdat->nbp_threshold = 0;
+ pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+ pgdat->nbp_th_start = jiffies_to_msecs(jiffies);
+ }
+}
+
int sysctl_numa_balancing(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos)
{
if (err < 0)
return err;
if (write) {
+ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+ (state & NUMA_BALANCING_MEMORY_TIERING))
+ reset_memory_tiering();
sysctl_numa_balancing_mode = state;
__set_numabalancing_state(state);
}
* finish_task_switch()'s mmdrop().
*/
switch_mm_irqs_off(prev->active_mm, next->mm, next);
+ lru_gen_use_mm(next->mm);
if (!prev->mm) { // from kernel
/* will mmdrop() in finish_task_switch(). */
debugfs_create_u32("scan_period_min_ms", 0644, numa, &sysctl_numa_balancing_scan_period_min);
debugfs_create_u32("scan_period_max_ms", 0644, numa, &sysctl_numa_balancing_scan_period_max);
debugfs_create_u32("scan_size_mb", 0644, numa, &sysctl_numa_balancing_scan_size);
+ debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
#endif
debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
#include <linux/cpuidle.h>
#include <linux/interrupt.h>
+#include <linux/memory-tiers.h>
#include <linux/mempolicy.h>
#include <linux/mutex_api.h>
#include <linux/profile.h>
/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
unsigned int sysctl_numa_balancing_scan_delay = 1000;
+/* The page with hint page fault latency < threshold in ms is considered hot */
+unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC;
+
+/* Restrict the NUMA promotion throughput (MB/s) for each target node. */
+unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
+
struct numa_group {
refcount_t refcount;
return 1000 * faults / total_faults;
}
+/*
+ * If memory tiering mode is enabled, cpupid of slow memory page is
+ * used to record scan time instead of CPU and PID. When tiering mode
+ * is disabled at run time, the scan time (in cpupid) will be
+ * interpreted as CPU and PID. So CPU needs to be checked to avoid to
+ * access out of array bound.
+ */
+static inline bool cpupid_valid(int cpupid)
+{
+ return cpupid_to_cpu(cpupid) < nr_cpu_ids;
+}
+
+/*
+ * For memory tiering mode, if there are enough free pages (more than
+ * enough watermark defined here) in fast memory node, to take full
+ * advantage of fast memory capacity, all recently accessed slow
+ * memory pages will be migrated to fast memory node without
+ * considering hot threshold.
+ */
+static bool pgdat_free_space_enough(struct pglist_data *pgdat)
+{
+ int z;
+ unsigned long enough_wmark;
+
+ enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT,
+ pgdat->node_present_pages >> 4);
+ for (z = pgdat->nr_zones - 1; z >= 0; z--) {
+ struct zone *zone = pgdat->node_zones + z;
+
+ if (!populated_zone(zone))
+ continue;
+
+ if (zone_watermark_ok(zone, 0,
+ wmark_pages(zone, WMARK_PROMO) + enough_wmark,
+ ZONE_MOVABLE, 0))
+ return true;
+ }
+ return false;
+}
+
+/*
+ * For memory tiering mode, when page tables are scanned, the scan
+ * time will be recorded in struct page in addition to make page
+ * PROT_NONE for slow memory page. So when the page is accessed, in
+ * hint page fault handler, the hint page fault latency is calculated
+ * via,
+ *
+ * hint page fault latency = hint page fault time - scan time
+ *
+ * The smaller the hint page fault latency, the higher the possibility
+ * for the page to be hot.
+ */
+static int numa_hint_fault_latency(struct page *page)
+{
+ int last_time, time;
+
+ time = jiffies_to_msecs(jiffies);
+ last_time = xchg_page_access_time(page, time);
+
+ return (time - last_time) & PAGE_ACCESS_TIME_MASK;
+}
+
+/*
+ * For memory tiering mode, too high promotion/demotion throughput may
+ * hurt application latency. So we provide a mechanism to rate limit
+ * the number of pages that are tried to be promoted.
+ */
+static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
+ unsigned long rate_limit, int nr)
+{
+ unsigned long nr_cand;
+ unsigned int now, start;
+
+ now = jiffies_to_msecs(jiffies);
+ mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
+ nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+ start = pgdat->nbp_rl_start;
+ if (now - start > MSEC_PER_SEC &&
+ cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
+ pgdat->nbp_rl_nr_cand = nr_cand;
+ if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit)
+ return true;
+ return false;
+}
+
+#define NUMA_MIGRATION_ADJUST_STEPS 16
+
+static void numa_promotion_adjust_threshold(struct pglist_data *pgdat,
+ unsigned long rate_limit,
+ unsigned int ref_th)
+{
+ unsigned int now, start, th_period, unit_th, th;
+ unsigned long nr_cand, ref_cand, diff_cand;
+
+ now = jiffies_to_msecs(jiffies);
+ th_period = sysctl_numa_balancing_scan_period_max;
+ start = pgdat->nbp_th_start;
+ if (now - start > th_period &&
+ cmpxchg(&pgdat->nbp_th_start, start, now) == start) {
+ ref_cand = rate_limit *
+ sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC;
+ nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+ diff_cand = nr_cand - pgdat->nbp_th_nr_cand;
+ unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS;
+ th = pgdat->nbp_threshold ? : ref_th;
+ if (diff_cand > ref_cand * 11 / 10)
+ th = max(th - unit_th, unit_th);
+ else if (diff_cand < ref_cand * 9 / 10)
+ th = min(th + unit_th, ref_th * 2);
+ pgdat->nbp_th_nr_cand = nr_cand;
+ pgdat->nbp_threshold = th;
+ }
+}
+
bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
int src_nid, int dst_cpu)
{
int dst_nid = cpu_to_node(dst_cpu);
int last_cpupid, this_cpupid;
+ /*
+ * The pages in slow memory node should be migrated according
+ * to hot/cold instead of private/shared.
+ */
+ if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
+ !node_is_toptier(src_nid)) {
+ struct pglist_data *pgdat;
+ unsigned long rate_limit;
+ unsigned int latency, th, def_th;
+
+ pgdat = NODE_DATA(dst_nid);
+ if (pgdat_free_space_enough(pgdat)) {
+ /* workload changed, reset hot threshold */
+ pgdat->nbp_threshold = 0;
+ return true;
+ }
+
+ def_th = sysctl_numa_balancing_hot_threshold;
+ rate_limit = sysctl_numa_balancing_promote_rate_limit << \
+ (20 - PAGE_SHIFT);
+ numa_promotion_adjust_threshold(pgdat, rate_limit, def_th);
+
+ th = pgdat->nbp_threshold ? : def_th;
+ latency = numa_hint_fault_latency(page);
+ if (latency >= th)
+ return false;
+
+ return !numa_promotion_rate_limit(pgdat, rate_limit,
+ thp_nr_pages(page));
+ }
+
this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
+ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+ !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid))
+ return false;
+
/*
* Allow first faults or private faults to migrate immediately early in
* the lifetime of a task. The magic number 4 is based on waiting for
if (!p->mm)
return;
+ /*
+ * NUMA faults statistics are unnecessary for the slow memory
+ * node for memory tiering mode.
+ */
+ if (!node_is_toptier(mem_node) &&
+ (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ||
+ !cpupid_valid(last_cpupid)))
+ return;
+
/* Allocate buffer to track faults on a per-node basis */
if (unlikely(!p->numa_faults)) {
int size = sizeof(*p->numa_faults) *
struct task_struct *p = current;
struct mm_struct *mm = p->mm;
u64 runtime = p->se.sum_exec_runtime;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
struct vm_area_struct *vma;
unsigned long start, end;
unsigned long nr_pte_updates = 0;
if (!mmap_read_trylock(mm))
return;
- vma = find_vma(mm, start);
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
if (!vma) {
reset_ptenuma_scan(p);
start = 0;
- vma = mm->mmap;
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
}
- for (; vma; vma = vma->vm_next) {
+
+ for (; vma; vma = mas_find(&mas, ULONG_MAX)) {
if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
continue;
{
int cpu;
+ group->enabled = true;
for_each_possible_cpu(cpu)
seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq);
group->avg_last_update = sched_clock();
{
if (!psi_enable) {
static_branch_enable(&psi_disabled);
+ static_branch_disable(&psi_cgroups_enabled);
return;
}
group_init(&psi_system);
}
-static bool test_state(unsigned int *tasks, enum psi_states state)
+static bool test_state(unsigned int *tasks, enum psi_states state, bool oncpu)
{
switch (state) {
case PSI_IO_SOME:
return unlikely(tasks[NR_MEMSTALL] &&
tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);
case PSI_CPU_SOME:
- return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]);
+ return unlikely(tasks[NR_RUNNING] > oncpu);
case PSI_CPU_FULL:
- return unlikely(tasks[NR_RUNNING] && !tasks[NR_ONCPU]);
+ return unlikely(tasks[NR_RUNNING] && !oncpu);
case PSI_NONIDLE:
return tasks[NR_IOWAIT] || tasks[NR_MEMSTALL] ||
tasks[NR_RUNNING];
bool wake_clock)
{
struct psi_group_cpu *groupc;
- u32 state_mask = 0;
unsigned int t, m;
enum psi_states s;
+ u32 state_mask;
groupc = per_cpu_ptr(group->pcpu, cpu);
/*
- * First we assess the aggregate resource states this CPU's
- * tasks have been in since the last change, and account any
- * SOME and FULL time these may have resulted in.
- *
- * Then we update the task counts according to the state
+ * First we update the task counts according to the state
* change requested through the @clear and @set bits.
+ *
+ * Then if the cgroup PSI stats accounting enabled, we
+ * assess the aggregate resource states this CPU's tasks
+ * have been in since the last change, and account any
+ * SOME and FULL time these may have resulted in.
*/
write_seqcount_begin(&groupc->seq);
- record_times(groupc, now);
+ /*
+ * Start with TSK_ONCPU, which doesn't have a corresponding
+ * task count - it's just a boolean flag directly encoded in
+ * the state mask. Clear, set, or carry the current state if
+ * no changes are requested.
+ */
+ if (unlikely(clear & TSK_ONCPU)) {
+ state_mask = 0;
+ clear &= ~TSK_ONCPU;
+ } else if (unlikely(set & TSK_ONCPU)) {
+ state_mask = PSI_ONCPU;
+ set &= ~TSK_ONCPU;
+ } else {
+ state_mask = groupc->state_mask & PSI_ONCPU;
+ }
+ /*
+ * The rest of the state mask is calculated based on the task
+ * counts. Update those first, then construct the mask.
+ */
for (t = 0, m = clear; m; m &= ~(1 << t), t++) {
if (!(m & (1 << t)))
continue;
if (groupc->tasks[t]) {
groupc->tasks[t]--;
} else if (!psi_bug) {
- printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n",
+ printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n",
cpu, t, groupc->tasks[0],
groupc->tasks[1], groupc->tasks[2],
- groupc->tasks[3], groupc->tasks[4],
- clear, set);
+ groupc->tasks[3], clear, set);
psi_bug = 1;
}
}
if (set & (1 << t))
groupc->tasks[t]++;
- /* Calculate state mask representing active states */
+ if (!group->enabled) {
+ /*
+ * On the first group change after disabling PSI, conclude
+ * the current state and flush its time. This is unlikely
+ * to matter to the user, but aggregation (get_recent_times)
+ * may have already incorporated the live state into times_prev;
+ * avoid a delta sample underflow when PSI is later re-enabled.
+ */
+ if (unlikely(groupc->state_mask & (1 << PSI_NONIDLE)))
+ record_times(groupc, now);
+
+ groupc->state_mask = state_mask;
+
+ write_seqcount_end(&groupc->seq);
+ return;
+ }
+
for (s = 0; s < NR_PSI_STATES; s++) {
- if (test_state(groupc->tasks, s))
+ if (test_state(groupc->tasks, s, state_mask & PSI_ONCPU))
state_mask |= (1 << s);
}
* task in a cgroup is in_memstall, the corresponding groupc
* on that cpu is in PSI_MEM_FULL state.
*/
- if (unlikely(groupc->tasks[NR_ONCPU] && cpu_curr(cpu)->in_memstall))
+ if (unlikely((state_mask & PSI_ONCPU) && cpu_curr(cpu)->in_memstall))
state_mask |= (1 << PSI_MEM_FULL);
+ record_times(groupc, now);
+
groupc->state_mask = state_mask;
write_seqcount_end(&groupc->seq);
schedule_delayed_work(&group->avgs_work, PSI_FREQ);
}
-static struct psi_group *iterate_groups(struct task_struct *task, void **iter)
+static inline struct psi_group *task_psi_group(struct task_struct *task)
{
- if (*iter == &psi_system)
- return NULL;
-
#ifdef CONFIG_CGROUPS
- if (static_branch_likely(&psi_cgroups_enabled)) {
- struct cgroup *cgroup = NULL;
-
- if (!*iter)
- cgroup = task->cgroups->dfl_cgrp;
- else
- cgroup = cgroup_parent(*iter);
-
- if (cgroup && cgroup_parent(cgroup)) {
- *iter = cgroup;
- return cgroup_psi(cgroup);
- }
- }
+ if (static_branch_likely(&psi_cgroups_enabled))
+ return cgroup_psi(task_dfl_cgroup(task));
#endif
- *iter = &psi_system;
return &psi_system;
}
{
int cpu = task_cpu(task);
struct psi_group *group;
- bool wake_clock = true;
- void *iter = NULL;
u64 now;
if (!task->pid)
psi_flags_change(task, clear, set);
now = cpu_clock(cpu);
- /*
- * Periodic aggregation shuts off if there is a period of no
- * task changes, so we wake it back up if necessary. However,
- * don't do this if the task change is the aggregation worker
- * itself going to sleep, or we'll ping-pong forever.
- */
- if (unlikely((clear & TSK_RUNNING) &&
- (task->flags & PF_WQ_WORKER) &&
- wq_worker_last_func(task) == psi_avgs_work))
- wake_clock = false;
- while ((group = iterate_groups(task, &iter)))
- psi_group_change(group, cpu, clear, set, now, wake_clock);
+ group = task_psi_group(task);
+ do {
+ psi_group_change(group, cpu, clear, set, now, true);
+ } while ((group = group->parent));
}
void psi_task_switch(struct task_struct *prev, struct task_struct *next,
{
struct psi_group *group, *common = NULL;
int cpu = task_cpu(prev);
- void *iter;
u64 now = cpu_clock(cpu);
if (next->pid) {
- bool identical_state;
-
psi_flags_change(next, 0, TSK_ONCPU);
/*
- * When switching between tasks that have an identical
- * runtime state, the cgroup that contains both tasks
- * we reach the first common ancestor. Iterate @next's
- * ancestors only until we encounter @prev's ONCPU.
+ * Set TSK_ONCPU on @next's cgroups. If @next shares any
+ * ancestors with @prev, those will already have @prev's
+ * TSK_ONCPU bit set, and we can stop the iteration there.
*/
- identical_state = prev->psi_flags == next->psi_flags;
- iter = NULL;
- while ((group = iterate_groups(next, &iter))) {
- if (identical_state &&
- per_cpu_ptr(group->pcpu, cpu)->tasks[NR_ONCPU]) {
+ group = task_psi_group(next);
+ do {
+ if (per_cpu_ptr(group->pcpu, cpu)->state_mask &
+ PSI_ONCPU) {
common = group;
break;
}
psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);
- }
+ } while ((group = group->parent));
}
if (prev->pid) {
int clear = TSK_ONCPU, set = 0;
+ bool wake_clock = true;
/*
* When we're going to sleep, psi_dequeue() lets us
clear |= TSK_MEMSTALL_RUNNING;
if (prev->in_iowait)
set |= TSK_IOWAIT;
+
+ /*
+ * Periodic aggregation shuts off if there is a period of no
+ * task changes, so we wake it back up if necessary. However,
+ * don't do this if the task change is the aggregation worker
+ * itself going to sleep, or we'll ping-pong forever.
+ */
+ if (unlikely((prev->flags & PF_WQ_WORKER) &&
+ wq_worker_last_func(prev) == psi_avgs_work))
+ wake_clock = false;
}
psi_flags_change(prev, clear, set);
- iter = NULL;
- while ((group = iterate_groups(prev, &iter)) && group != common)
- psi_group_change(group, cpu, clear, set, now, true);
+ group = task_psi_group(prev);
+ do {
+ if (group == common)
+ break;
+ psi_group_change(group, cpu, clear, set, now, wake_clock);
+ } while ((group = group->parent));
/*
- * TSK_ONCPU is handled up to the common ancestor. If we're tasked
- * with dequeuing too, finish that for the rest of the hierarchy.
+ * TSK_ONCPU is handled up to the common ancestor. If there are
+ * any other differences between the two tasks (e.g. prev goes
+ * to sleep, or only one task is memstall), finish propagating
+ * those differences all the way up to the root.
*/
- if (sleep) {
+ if ((prev->psi_flags ^ next->psi_flags) & ~TSK_ONCPU) {
clear &= ~TSK_ONCPU;
- for (; group; group = iterate_groups(prev, &iter))
- psi_group_change(group, cpu, clear, set, now, true);
+ for (; group; group = group->parent)
+ psi_group_change(group, cpu, clear, set, now, wake_clock);
}
}
}
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+void psi_account_irqtime(struct task_struct *task, u32 delta)
+{
+ int cpu = task_cpu(task);
+ struct psi_group *group;
+ struct psi_group_cpu *groupc;
+ u64 now;
+
+ if (!task->pid)
+ return;
+
+ now = cpu_clock(cpu);
+
+ group = task_psi_group(task);
+ do {
+ if (!group->enabled)
+ continue;
+
+ groupc = per_cpu_ptr(group->pcpu, cpu);
+
+ write_seqcount_begin(&groupc->seq);
+
+ record_times(groupc, now);
+ groupc->times[PSI_IRQ_FULL] += delta;
+
+ write_seqcount_end(&groupc->seq);
+
+ if (group->poll_states & (1 << PSI_IRQ_FULL))
+ psi_schedule_poll_work(group, 1);
+ } while ((group = group->parent));
+}
+#endif
+
/**
* psi_memstall_enter - mark the beginning of a memory stall section
* @flags: flags to handle nested sections
#ifdef CONFIG_CGROUPS
int psi_cgroup_alloc(struct cgroup *cgroup)
{
- if (static_branch_likely(&psi_disabled))
+ if (!static_branch_likely(&psi_cgroups_enabled))
return 0;
cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL);
return -ENOMEM;
}
group_init(cgroup->psi);
+ cgroup->psi->parent = cgroup_psi(cgroup_parent(cgroup));
return 0;
}
void psi_cgroup_free(struct cgroup *cgroup)
{
- if (static_branch_likely(&psi_disabled))
+ if (!static_branch_likely(&psi_cgroups_enabled))
return;
cancel_delayed_work_sync(&cgroup->psi->avgs_work);
struct rq_flags rf;
struct rq *rq;
- if (static_branch_likely(&psi_disabled)) {
+ if (!static_branch_likely(&psi_cgroups_enabled)) {
/*
* Lame to do this here, but the scheduler cannot be locked
* from the outside, so we move cgroups from inside sched/.
task_rq_unlock(rq, task, &rf);
}
+
+void psi_cgroup_restart(struct psi_group *group)
+{
+ int cpu;
+
+ /*
+ * After we disable psi_group->enabled, we don't actually
+ * stop percpu tasks accounting in each psi_group_cpu,
+ * instead only stop test_state() loop, record_times()
+ * and averaging worker, see psi_group_change() for details.
+ *
+ * When disable cgroup PSI, this function has nothing to sync
+ * since cgroup pressure files are hidden and percpu psi_group_cpu
+ * would see !psi_group->enabled and only do task accounting.
+ *
+ * When re-enable cgroup PSI, this function use psi_group_change()
+ * to get correct state mask from test_state() loop on tasks[],
+ * and restart groupc->state_start from now, use .clear = .set = 0
+ * here since no task status really changed.
+ */
+ if (!group->enabled)
+ return;
+
+ for_each_possible_cpu(cpu) {
+ struct rq *rq = cpu_rq(cpu);
+ struct rq_flags rf;
+ u64 now;
+
+ rq_lock_irq(rq, &rf);
+ now = cpu_clock(cpu);
+ psi_group_change(group, cpu, 0, 0, now, true);
+ rq_unlock_irq(rq, &rf);
+ }
+}
#endif /* CONFIG_CGROUPS */
int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
{
+ bool only_full = false;
int full;
u64 now;
group->avg_next_update = update_averages(group, now);
mutex_unlock(&group->avgs_lock);
- for (full = 0; full < 2; full++) {
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+ only_full = res == PSI_IRQ;
+#endif
+
+ for (full = 0; full < 2 - only_full; full++) {
unsigned long avg[3] = { 0, };
u64 total = 0;
int w;
}
seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
- full ? "full" : "some",
+ full || only_full ? "full" : "some",
LOAD_INT(avg[0]), LOAD_FRAC(avg[0]),
LOAD_INT(avg[1]), LOAD_FRAC(avg[1]),
LOAD_INT(avg[2]), LOAD_FRAC(avg[2]),
else
return ERR_PTR(-EINVAL);
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+ if (res == PSI_IRQ && --state != PSI_IRQ_FULL)
+ return ERR_PTR(-EINVAL);
+#endif
+
if (state >= PSI_NONIDLE)
return ERR_PTR(-EINVAL);
.proc_release = psi_fop_release,
};
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+static int psi_irq_show(struct seq_file *m, void *v)
+{
+ return psi_show(m, &psi_system, PSI_IRQ);
+}
+
+static int psi_irq_open(struct inode *inode, struct file *file)
+{
+ return psi_open(file, psi_irq_show);
+}
+
+static ssize_t psi_irq_write(struct file *file, const char __user *user_buf,
+ size_t nbytes, loff_t *ppos)
+{
+ return psi_write(file, user_buf, nbytes, PSI_IRQ);
+}
+
+static const struct proc_ops psi_irq_proc_ops = {
+ .proc_open = psi_irq_open,
+ .proc_read = seq_read,
+ .proc_lseek = seq_lseek,
+ .proc_write = psi_irq_write,
+ .proc_poll = psi_fop_poll,
+ .proc_release = psi_fop_release,
+};
+#endif
+
static int __init psi_proc_init(void)
{
if (psi_enable) {
proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops);
proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops);
proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops);
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+ proc_create("pressure/irq", 0666, NULL, &psi_irq_proc_ops);
+#endif
}
return 0;
}
extern unsigned int sysctl_numa_balancing_scan_period_min;
extern unsigned int sysctl_numa_balancing_scan_period_max;
extern unsigned int sysctl_numa_balancing_scan_size;
+extern unsigned int sysctl_numa_balancing_hot_threshold;
#endif
#ifdef CONFIG_SCHED_HRTICK
}
#ifdef CONFIG_PSI
+void psi_task_change(struct task_struct *task, int clear, int set);
+void psi_task_switch(struct task_struct *prev, struct task_struct *next,
+ bool sleep);
+void psi_account_irqtime(struct task_struct *task, u32 delta);
+
/*
* PSI tracks state that persists across sleeps, such as iowaits and
* memory stalls. As a result, it has to distinguish between sleeps,
static inline void psi_sched_switch(struct task_struct *prev,
struct task_struct *next,
bool sleep) {}
+static inline void psi_account_irqtime(struct task_struct *task, u32 delta) {}
#endif /* CONFIG_PSI */
#ifdef CONFIG_SCHED_INFO
/* The outgoing CPU will normally get done quite quickly. */
if (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) == CPU_DEAD)
- goto update_state;
+ goto update_state_early;
udelay(5);
/* But if the outgoing CPU dawdles, wait increasingly long times. */
break;
sleep_jf = DIV_ROUND_UP(sleep_jf * 11, 10);
}
-update_state:
+update_state_early:
oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
+update_state:
if (oldstate == CPU_DEAD) {
/* Outgoing CPU died normally, update state. */
smp_mb(); /* atomic_read() before update. */
atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_POST_DEAD);
} else {
/* Outgoing CPU still hasn't died, set state accordingly. */
- if (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
- oldstate, CPU_BROKEN) != oldstate)
+ if (!atomic_try_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
+ &oldstate, CPU_BROKEN))
goto update_state;
ret = false;
}
int newstate;
int cpu = smp_processor_id();
+ oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
do {
- oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu));
if (oldstate != CPU_BROKEN)
newstate = CPU_DEAD;
else
newstate = CPU_DEAD_FROZEN;
- } while (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
- oldstate, newstate) != oldstate);
+ } while (!atomic_try_cmpxchg(&per_cpu(cpu_hotplug_state, cpu),
+ &oldstate, newstate));
return newstate == CPU_DEAD;
}
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_FOUR,
},
+ {
+ .procname = "numa_balancing_promote_rate_limit_MBps",
+ .data = &sysctl_numa_balancing_promote_rate_limit,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = SYSCTL_ZERO,
+ },
#endif /* CONFIG_NUMA_BALANCING */
{
.procname = "panic",
/* record the work call stack in order to print it in KASAN reports */
kasan_record_aux_stack(work);
+ head = READ_ONCE(task->task_works);
do {
- head = READ_ONCE(task->task_works);
if (unlikely(head == &work_exited))
return -ESRCH;
work->next = head;
- } while (cmpxchg(&task->task_works, head, work) != head);
+ } while (!try_cmpxchg(&task->task_works, &head, work));
switch (notify) {
case TWA_NONE:
* we raced with task_work_run(), *pprev == NULL/exited.
*/
raw_spin_lock_irqsave(&task->pi_lock, flags);
- while ((work = READ_ONCE(*pprev))) {
- if (!match(work, data))
+ work = READ_ONCE(*pprev);
+ while (work) {
+ if (!match(work, data)) {
pprev = &work->next;
- else if (cmpxchg(pprev, work, work->next) == work)
+ work = READ_ONCE(*pprev);
+ } else if (try_cmpxchg(pprev, &work, work->next))
break;
}
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
* work->func() can do task_work_add(), do not set
* work_exited unless the list is empty.
*/
+ work = READ_ONCE(task->task_works);
do {
head = NULL;
- work = READ_ONCE(task->task_works);
if (!work) {
if (task->flags & PF_EXITING)
head = &work_exited;
else
break;
}
- } while (cmpxchg(&task->task_works, work, head) != work);
+ } while (!try_cmpxchg(&task->task_works, &work, head));
if (!work)
break;
* CPUs that are currently online.
*/
for (i = 1; i < n; i++) {
- cpu = prandom_u32() % nr_cpu_ids;
+ cpu = prandom_u32_max(nr_cpu_ids);
cpu = cpumask_next(cpu - 1, cpu_online_mask);
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(cpu_online_mask);
static void print_ip_ins(const char *fmt, const unsigned char *p)
{
char ins[MCOUNT_INSN_SIZE];
- int i;
if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) {
printk(KERN_CONT "%s[FAULT] %px\n", fmt, p);
}
printk(KERN_CONT "%s", fmt);
-
- for (i = 0; i < MCOUNT_INSN_SIZE; i++)
- printk(KERN_CONT "%s%02x", i ? ":" : "", ins[i]);
+ pr_cont("%*phC", MCOUNT_INSN_SIZE, ins);
}
enum ftrace_bug_type ftrace_bug_type;
}
/**
- * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
+ * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
* @buffer: The ring_buffer to get the number of pages from
* @cpu: The cpu of the ring_buffer to get the number of pages from
*
EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
/**
- * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
+ * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
* @buffer: The ring buffer to reset a per cpu buffer of
* @cpu: The CPU buffer to be reset
*/
EXPORT_SYMBOL_GPL(ring_buffer_reset);
/**
- * rind_buffer_empty - is the ring buffer empty?
+ * ring_buffer_empty - is the ring buffer empty?
* @buffer: The ring buffer to test
*/
bool ring_buffer_empty(struct trace_buffer *buffer)
#include "trace_dynevent.h"
#include "trace_probe.h"
#include "trace_probe_tmpl.h"
+#include "trace_probe_kernel.h"
#define EPROBE_EVENT_SYSTEM "eprobes"
static nokprobe_inline int
fetch_store_strlen_user(unsigned long addr)
{
- const void __user *uaddr = (__force const void __user *)addr;
-
- return strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
+ return kern_fetch_store_strlen_user(addr);
}
/* Return the length of string -- including null terminal byte */
static nokprobe_inline int
fetch_store_strlen(unsigned long addr)
{
- int ret, len = 0;
- u8 c;
-
-#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
- if (addr < TASK_SIZE)
- return fetch_store_strlen_user(addr);
-#endif
-
- do {
- ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1);
- len++;
- } while (c && ret == 0 && len < MAX_STRING_SIZE);
-
- return (ret < 0) ? ret : len;
+ return kern_fetch_store_strlen(addr);
}
/*
static nokprobe_inline int
fetch_store_string_user(unsigned long addr, void *dest, void *base)
{
- const void __user *uaddr = (__force const void __user *)addr;
- int maxlen = get_loc_len(*(u32 *)dest);
- void *__dest;
- long ret;
-
- if (unlikely(!maxlen))
- return -ENOMEM;
-
- __dest = get_loc_data(dest, base);
-
- ret = strncpy_from_user_nofault(__dest, uaddr, maxlen);
- if (ret >= 0)
- *(u32 *)dest = make_data_loc(ret, __dest - base);
-
- return ret;
+ return kern_fetch_store_string_user(addr, dest, base);
}
/*
static nokprobe_inline int
fetch_store_string(unsigned long addr, void *dest, void *base)
{
- int maxlen = get_loc_len(*(u32 *)dest);
- void *__dest;
- long ret;
-
-#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
- if ((unsigned long)addr < TASK_SIZE)
- return fetch_store_string_user(addr, dest, base);
-#endif
-
- if (unlikely(!maxlen))
- return -ENOMEM;
-
- __dest = get_loc_data(dest, base);
-
- /*
- * Try to get string again, since the string can be changed while
- * probing.
- */
- ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen);
- if (ret >= 0)
- *(u32 *)dest = make_data_loc(ret, __dest - base);
-
- return ret;
+ return kern_fetch_store_string(addr, dest, base);
}
static nokprobe_inline int
/* for gfp flag names */
#include <linux/trace_events.h>
#include <trace/events/mmflags.h>
+#include "trace_probe.h"
+#include "trace_probe_kernel.h"
#include "trace_synth.h"
{
unsigned int len = 0;
char *str_field;
+ int ret;
if (is_dynamic) {
u32 data_offset;
data_offset += event->n_u64 * sizeof(u64);
data_offset += data_size;
- str_field = (char *)entry + data_offset;
-
- len = strlen(str_val) + 1;
- strscpy(str_field, str_val, len);
+ len = kern_fetch_store_strlen((unsigned long)str_val);
data_offset |= len << 16;
*(u32 *)&entry->fields[*n_u64] = data_offset;
+ ret = kern_fetch_store_string((unsigned long)str_val, &entry->fields[*n_u64], entry);
+
(*n_u64)++;
} else {
str_field = (char *)&entry->fields[*n_u64];
- strscpy(str_field, str_val, STR_VAR_LEN_MAX);
+#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
+ if ((unsigned long)str_val < TASK_SIZE)
+ ret = strncpy_from_user_nofault(str_field, str_val, STR_VAR_LEN_MAX);
+ else
+#endif
+ ret = strncpy_from_kernel_nofault(str_field, str_val, STR_VAR_LEN_MAX);
+
+ if (ret < 0)
+ strcpy(str_field, FAULT_STRING);
+
(*n_u64) += STR_VAR_LEN_MAX / sizeof(u64);
}
val_idx = var_ref_idx[field_pos];
str_val = (char *)(long)var_ref_vals[val_idx];
- len = strlen(str_val) + 1;
+ len = kern_fetch_store_strlen((unsigned long)str_val);
fields_size += len;
}
#include "trace_kprobe_selftest.h"
#include "trace_probe.h"
#include "trace_probe_tmpl.h"
+#include "trace_probe_kernel.h"
#define KPROBE_EVENT_SYSTEM "kprobes"
#define KRETPROBE_MAXACTIVE_MAX 4096
static nokprobe_inline int
fetch_store_strlen_user(unsigned long addr)
{
- const void __user *uaddr = (__force const void __user *)addr;
-
- return strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
+ return kern_fetch_store_strlen_user(addr);
}
/* Return the length of string -- including null terminal byte */
static nokprobe_inline int
fetch_store_strlen(unsigned long addr)
{
- int ret, len = 0;
- u8 c;
-
-#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
- if (addr < TASK_SIZE)
- return fetch_store_strlen_user(addr);
-#endif
-
- do {
- ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1);
- len++;
- } while (c && ret == 0 && len < MAX_STRING_SIZE);
-
- return (ret < 0) ? ret : len;
+ return kern_fetch_store_strlen(addr);
}
/*
static nokprobe_inline int
fetch_store_string_user(unsigned long addr, void *dest, void *base)
{
- const void __user *uaddr = (__force const void __user *)addr;
- int maxlen = get_loc_len(*(u32 *)dest);
- void *__dest;
- long ret;
-
- if (unlikely(!maxlen))
- return -ENOMEM;
-
- __dest = get_loc_data(dest, base);
-
- ret = strncpy_from_user_nofault(__dest, uaddr, maxlen);
- if (ret >= 0)
- *(u32 *)dest = make_data_loc(ret, __dest - base);
-
- return ret;
+ return kern_fetch_store_string_user(addr, dest, base);
}
/*
static nokprobe_inline int
fetch_store_string(unsigned long addr, void *dest, void *base)
{
- int maxlen = get_loc_len(*(u32 *)dest);
- void *__dest;
- long ret;
-
-#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
- if ((unsigned long)addr < TASK_SIZE)
- return fetch_store_string_user(addr, dest, base);
-#endif
-
- if (unlikely(!maxlen))
- return -ENOMEM;
-
- __dest = get_loc_data(dest, base);
-
- /*
- * Try to get string again, since the string can be changed while
- * probing.
- */
- ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen);
- if (ret >= 0)
- *(u32 *)dest = make_data_loc(ret, __dest - base);
-
- return ret;
+ return kern_fetch_store_string(addr, dest, base);
}
static nokprobe_inline int
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __TRACE_PROBE_KERNEL_H_
+#define __TRACE_PROBE_KERNEL_H_
+
+#define FAULT_STRING "(fault)"
+
+/*
+ * This depends on trace_probe.h, but can not include it due to
+ * the way trace_probe_tmpl.h is used by trace_kprobe.c and trace_eprobe.c.
+ * Which means that any other user must include trace_probe.h before including
+ * this file.
+ */
+/* Return the length of string -- including null terminal byte */
+static nokprobe_inline int
+kern_fetch_store_strlen_user(unsigned long addr)
+{
+ const void __user *uaddr = (__force const void __user *)addr;
+ int ret;
+
+ ret = strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
+ /*
+ * strnlen_user_nofault returns zero on fault, insert the
+ * FAULT_STRING when that occurs.
+ */
+ if (ret <= 0)
+ return strlen(FAULT_STRING) + 1;
+ return ret;
+}
+
+/* Return the length of string -- including null terminal byte */
+static nokprobe_inline int
+kern_fetch_store_strlen(unsigned long addr)
+{
+ int ret, len = 0;
+ u8 c;
+
+#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
+ if (addr < TASK_SIZE)
+ return kern_fetch_store_strlen_user(addr);
+#endif
+
+ do {
+ ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1);
+ len++;
+ } while (c && ret == 0 && len < MAX_STRING_SIZE);
+
+ /* For faults, return enough to hold the FAULT_STRING */
+ return (ret < 0) ? strlen(FAULT_STRING) + 1 : len;
+}
+
+static nokprobe_inline void set_data_loc(int ret, void *dest, void *__dest, void *base, int len)
+{
+ if (ret >= 0) {
+ *(u32 *)dest = make_data_loc(ret, __dest - base);
+ } else {
+ strscpy(__dest, FAULT_STRING, len);
+ ret = strlen(__dest) + 1;
+ }
+}
+
+/*
+ * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf
+ * with max length and relative data location.
+ */
+static nokprobe_inline int
+kern_fetch_store_string_user(unsigned long addr, void *dest, void *base)
+{
+ const void __user *uaddr = (__force const void __user *)addr;
+ int maxlen = get_loc_len(*(u32 *)dest);
+ void *__dest;
+ long ret;
+
+ if (unlikely(!maxlen))
+ return -ENOMEM;
+
+ __dest = get_loc_data(dest, base);
+
+ ret = strncpy_from_user_nofault(__dest, uaddr, maxlen);
+ set_data_loc(ret, dest, __dest, base, maxlen);
+
+ return ret;
+}
+
+/*
+ * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
+ * length and relative data location.
+ */
+static nokprobe_inline int
+kern_fetch_store_string(unsigned long addr, void *dest, void *base)
+{
+ int maxlen = get_loc_len(*(u32 *)dest);
+ void *__dest;
+ long ret;
+
+#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
+ if ((unsigned long)addr < TASK_SIZE)
+ return kern_fetch_store_string_user(addr, dest, base);
+#endif
+
+ if (unlikely(!maxlen))
+ return -ENOMEM;
+
+ __dest = get_loc_data(dest, base);
+
+ /*
+ * Try to get string again, since the string can be changed while
+ * probing.
+ */
+ ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen);
+ set_data_loc(ret, dest, __dest, base, maxlen);
+
+ return ret;
+}
+
+#endif /* __TRACE_PROBE_KERNEL_H_ */
static DEFINE_CTL_TABLE_POLL(domainname_poll);
static struct ctl_table uts_kern_table[] = {
+ {
+ .procname = "arch",
+ .data = init_uts_ns.name.machine,
+ .maxlen = sizeof(init_uts_ns.name.machine),
+ .mode = 0444,
+ .proc_handler = proc_do_uts_string,
+ },
{
.procname = "ostype",
.data = init_uts_ns.name.sysname,
in the "Debug information" choice below, indicating that debug
information will be generated for build targets.
+# Clang is known to generate .{s,u}leb128 with symbol deltas with DWARF5, which
+# some targets may not support: https://sourceware.org/bugzilla/show_bug.cgi?id=27215
+config AS_HAS_NON_CONST_LEB128
+ def_bool $(as-instr,.uleb128 .Lexpr_end4 - .Lexpr_start3\n.Lexpr_start3:\n.Lexpr_end4:)
+
choice
prompt "Debug information"
depends on DEBUG_KERNEL
config DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT
bool "Rely on the toolchain's implicit default DWARF version"
select DEBUG_INFO
+ depends on !CC_IS_CLANG || AS_IS_LLVM || CLANG_VERSION < 140000 || (AS_IS_GNU && AS_VERSION >= 23502 && AS_HAS_NON_CONST_LEB128)
help
The implicit default version of DWARF debug info produced by a
toolchain changes over time.
config DEBUG_INFO_DWARF4
bool "Generate DWARF Version 4 debuginfo"
select DEBUG_INFO
- depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502)))
+ depends on !CC_IS_CLANG || AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502)
help
Generate DWARF v4 debug info. This requires gcc 4.5+, binutils 2.35.2
if using clang without clang's integrated assembler, and gdb 7.0+.
config DEBUG_INFO_DWARF5
bool "Generate DWARF Version 5 debuginfo"
select DEBUG_INFO
- depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502)))
+ depends on !CC_IS_CLANG || AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502 && AS_HAS_NON_CONST_LEB128)
help
Generate DWARF v5 debug info. Requires binutils 2.35.2, gcc 5.0+ (gcc
5.0+ accepts the -gdwarf-5 flag but only had partial support for some
If unsure, say N.
-config DEBUG_VM_VMACACHE
- bool "Debug VMA caching"
+config DEBUG_VM_MAPLE_TREE
+ bool "Debug VM maple trees"
depends on DEBUG_VM
+ select DEBUG_MAPLE_TREE
help
- Enable this to turn on VMA caching debug information. Doing so
- can cause significant overhead, so only enable it in non-production
- environments.
+ Enable VM maple tree debugging information and extra validations.
If unsure, say N.
source "lib/Kconfig.kasan"
source "lib/Kconfig.kfence"
+source "lib/Kconfig.kmsan"
endmenu # "Memory Debugging"
If unsure, say N.
+config DEBUG_MAPLE_TREE
+ bool "Debug maple trees"
+ depends on DEBUG_KERNEL
+ help
+ Enable maple tree debugging information and extra validations.
+
+ If unsure, say N.
+
endmenu
config DEBUG_CREDENTIALS
as well, as it adds inline-style instrumentation that is run
unconditionally.
-config KASAN_TAGS_IDENTIFY
- bool "Memory corruption type identification"
- depends on KASAN_SW_TAGS || KASAN_HW_TAGS
- help
- Enables best-effort identification of the bug types (use-after-free
- or out-of-bounds) at the cost of increased memory consumption.
- Only applicable for the tag-based KASAN modes.
-
config KASAN_VMALLOC
bool "Check accesses to vmalloc allocations"
depends on HAVE_ARCH_KASAN_VMALLOC
config KDB_KEYBOARD
bool "KGDB_KDB: keyboard as input device"
- depends on VT && KGDB_KDB
+ depends on VT && KGDB_KDB && !PARISC
default n
help
KDB can use a PS/2 type keyboard for an input device
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+config HAVE_ARCH_KMSAN
+ bool
+
+config HAVE_KMSAN_COMPILER
+ # Clang versions <14.0.0 also support -fsanitize=kernel-memory, but not
+ # all the features necessary to build the kernel with KMSAN.
+ depends on CC_IS_CLANG && CLANG_VERSION >= 140000
+ def_bool $(cc-option,-fsanitize=kernel-memory -mllvm -msan-disable-checks=1)
+
+config KMSAN
+ bool "KMSAN: detector of uninitialized values use"
+ depends on HAVE_ARCH_KMSAN && HAVE_KMSAN_COMPILER
+ depends on SLUB && DEBUG_KERNEL && !KASAN && !KCSAN
+ select STACKDEPOT
+ select STACKDEPOT_ALWAYS_INIT
+ help
+ KernelMemorySanitizer (KMSAN) is a dynamic detector of uses of
+ uninitialized values in the kernel. It is based on compiler
+ instrumentation provided by Clang and thus requires Clang to build.
+
+ An important note is that KMSAN is not intended for production use,
+ because it drastically increases kernel memory footprint and slows
+ the whole system down.
+
+ See <file:Documentation/dev-tools/kmsan.rst> for more details.
+
+if KMSAN
+
+config HAVE_KMSAN_PARAM_RETVAL
+ # -fsanitize-memory-param-retval is supported only by Clang >= 14.
+ depends on HAVE_KMSAN_COMPILER
+ def_bool $(cc-option,-fsanitize=kernel-memory -fsanitize-memory-param-retval)
+
+config KMSAN_CHECK_PARAM_RETVAL
+ bool "Check for uninitialized values passed to and returned from functions"
+ default y
+ depends on HAVE_KMSAN_PARAM_RETVAL
+ help
+ If the compiler supports -fsanitize-memory-param-retval, KMSAN will
+ eagerly check every function parameter passed by value and every
+ function return value.
+
+ Disabling KMSAN_CHECK_PARAM_RETVAL will result in tracking shadow for
+ function parameters and return values across function borders. This
+ is a more relaxed mode, but it generates more instrumentation code and
+ may potentially report errors in corner cases when non-instrumented
+ functions call instrumented ones.
+
+config KMSAN_KUNIT_TEST
+ tristate "KMSAN integration test suite" if !KUNIT_ALL_TESTS
+ default KUNIT_ALL_TESTS
+ depends on TRACEPOINTS && KUNIT
+ help
+ Test suite for KMSAN, testing various error detection scenarios,
+ and checking that reports are correctly output to console.
+
+ Say Y here if you want the test to be built into the kernel and run
+ during boot; say M if you want the test to build as a module; say N
+ if you are unsure.
+
+endif
lib-y := ctype.o string.o vsprintf.o cmdline.o \
rbtree.o radix-tree.o timerqueue.o xarray.o \
- idr.o extable.o irq_regs.o argv_split.o \
+ maple_tree.o idr.o extable.o irq_regs.o argv_split.o \
flex_proportions.o ratelimit.o show_mem.o \
is_single_threaded.o plist.o decompress.o kobject_uevent.o \
earlycpio.o seq_buf.o siphash.o dec_and_lock.o \
obj-$(CONFIG_TEST_SIPHASH) += test_siphash.o
obj-$(CONFIG_HASH_KUNIT_TEST) += test_hash.o
obj-$(CONFIG_TEST_IDA) += test_ida.o
-obj-$(CONFIG_KASAN_KUNIT_TEST) += test_kasan.o
-CFLAGS_test_kasan.o += -fno-builtin
-CFLAGS_test_kasan.o += $(call cc-disable-warning, vla)
-obj-$(CONFIG_KASAN_MODULE_TEST) += test_kasan_module.o
-CFLAGS_test_kasan_module.o += -fno-builtin
obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o
CFLAGS_test_ubsan.o += $(call cc-disable-warning, vla)
UBSAN_SANITIZE_test_ubsan.o := y
CFLAGS_stackdepot.o += -fno-builtin
obj-$(CONFIG_STACKDEPOT) += stackdepot.o
KASAN_SANITIZE_stackdepot.o := n
+# In particular, instrumenting stackdepot.c with KMSAN will result in infinite
+# recursion.
+KMSAN_SANITIZE_stackdepot.o := n
KCOV_INSTRUMENT_stackdepot.o := n
obj-$(CONFIG_REF_TRACKER) += ref_tracker.o
args[i-1] = '\0';
}
}
- if (quoted && args[i-1] == '"')
+ if (quoted && i > 0 && args[i-1] == '"')
args[i-1] = '\0';
if (args[i]) {
int rc = cmdline_test_values[i];
int offset;
- sprintf(in, "%u%s", get_random_int() % 256, str);
+ sprintf(in, "%u%s", get_random_u8(), str);
/* Only first '-' after the number will advance the pointer */
offset = strlen(in) - strlen(str) + !!(rc == 2);
cmdline_do_one_test(test, in, rc, offset);
int rc = strcmp(str, "") ? (strcmp(str, "-") ? 0 : 1) : 1;
int offset;
- sprintf(in, "%s%u", str, get_random_int() % 256);
+ sprintf(in, "%s%u", str, get_random_u8());
/*
* Only first and leading '-' not followed by integer
* will advance the pointer.
"File %s exceeding MAX_CPIO_FILE_NAME [%d]\n",
p, MAX_CPIO_FILE_NAME);
}
- strlcpy(cd.name, p + mypathsize, MAX_CPIO_FILE_NAME);
+ strscpy(cd.name, p + mypathsize, MAX_CPIO_FILE_NAME);
cd.data = (void *)dptr;
cd.size = ch[C_FILESIZE];
return false;
}
- if (attr->probability <= prandom_u32() % 100)
+ if (attr->probability <= prandom_u32_max(100))
return false;
if (!fail_stacktrace(attr))
bitmap_zero(bitmap2, BITMAP_LEN);
while (nbits--) {
- __set_bit(prandom_u32() % BITMAP_LEN, bitmap);
- __set_bit(prandom_u32() % BITMAP_LEN, bitmap2);
+ __set_bit(prandom_u32_max(BITMAP_LEN), bitmap);
+ __set_bit(prandom_u32_max(BITMAP_LEN), bitmap2);
}
test_find_next_bit(bitmap, BITMAP_LEN);
*/
#include <linux/pci.h>
#include <linux/io.h>
+#include <linux/kmsan-checks.h>
#include <linux/export.h>
#define mmio_read64be(addr) swab64(readq(addr))
#endif
+/*
+ * Here and below, we apply __no_kmsan_checks to functions reading data from
+ * hardware, to ensure that KMSAN marks their return values as initialized.
+ */
+__no_kmsan_checks
unsigned int ioread8(const void __iomem *addr)
{
IO_COND(addr, return inb(port), return readb(addr));
return 0xff;
}
+__no_kmsan_checks
unsigned int ioread16(const void __iomem *addr)
{
IO_COND(addr, return inw(port), return readw(addr));
return 0xffff;
}
+__no_kmsan_checks
unsigned int ioread16be(const void __iomem *addr)
{
IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr));
return 0xffff;
}
+__no_kmsan_checks
unsigned int ioread32(const void __iomem *addr)
{
IO_COND(addr, return inl(port), return readl(addr));
return 0xffffffff;
}
+__no_kmsan_checks
unsigned int ioread32be(const void __iomem *addr)
{
IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr));
return lo | (hi << 32);
}
+__no_kmsan_checks
u64 ioread64_lo_hi(const void __iomem *addr)
{
IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr));
return 0xffffffffffffffffULL;
}
+__no_kmsan_checks
u64 ioread64_hi_lo(const void __iomem *addr)
{
IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr));
return 0xffffffffffffffffULL;
}
+__no_kmsan_checks
u64 ioread64be_lo_hi(const void __iomem *addr)
{
IO_COND(addr, return pio_read64be_lo_hi(port),
return 0xffffffffffffffffULL;
}
+__no_kmsan_checks
u64 ioread64be_hi_lo(const void __iomem *addr)
{
IO_COND(addr, return pio_read64be_hi_lo(port),
void iowrite8(u8 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, outb(val,port), writeb(val, addr));
}
void iowrite16(u16 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, outw(val,port), writew(val, addr));
}
void iowrite16be(u16 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr));
}
void iowrite32(u32 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, outl(val,port), writel(val, addr));
}
void iowrite32be(u32 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr));
}
EXPORT_SYMBOL(iowrite8);
void iowrite64_lo_hi(u64 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, pio_write64_lo_hi(val, port),
writeq(val, addr));
}
void iowrite64_hi_lo(u64 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, pio_write64_hi_lo(val, port),
writeq(val, addr));
}
void iowrite64be_lo_hi(u64 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, pio_write64be_lo_hi(val, port),
mmio_write64be(val, addr));
}
void iowrite64be_hi_lo(u64 val, void __iomem *addr)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(&val, sizeof(val));
IO_COND(addr, pio_write64be_hi_lo(val, port),
mmio_write64be(val, addr));
}
void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count)
{
IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count));
+ /* KMSAN must treat values read from devices as initialized. */
+ kmsan_unpoison_memory(dst, count);
}
void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count)
{
IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count));
+ /* KMSAN must treat values read from devices as initialized. */
+ kmsan_unpoison_memory(dst, count * 2);
}
void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count)
{
IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count));
+ /* KMSAN must treat values read from devices as initialized. */
+ kmsan_unpoison_memory(dst, count * 4);
}
EXPORT_SYMBOL(ioread8_rep);
EXPORT_SYMBOL(ioread16_rep);
void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(src, count);
IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count));
}
void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(src, count * 2);
IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count));
}
void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count)
{
+ /* Make sure uninitialized memory isn't copied to devices. */
+ kmsan_check_memory(src, count * 4);
IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count));
}
EXPORT_SYMBOL(iowrite8_rep);
static int copyin(void *to, const void __user *from, size_t n)
{
+ size_t res = n;
+
if (should_fail_usercopy())
return n;
if (access_ok(from, n)) {
- instrument_copy_from_user(to, from, n);
- n = raw_copy_from_user(to, from, n);
+ instrument_copy_from_user_before(to, from, n);
+ res = raw_copy_from_user(to, from, n);
+ instrument_copy_from_user_after(to, from, n, res);
}
- return n;
+ return res;
}
static inline struct pipe_buffer *pipe_buf(const struct pipe_inode_info *pipe,
{
struct kobject *kobj = container_of(kref, struct kobject, kref);
#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
- unsigned long delay = HZ + HZ * (get_random_int() & 0x3);
+ unsigned long delay = HZ + HZ * prandom_u32_max(4);
pr_info("kobject: '%s' (%p): %s, parent %p (delayed %ld)\n",
kobject_name(kobj), kobj, __func__, kobj->parent, delay);
INIT_DELAYED_WORK(&kobj->release, kobject_delayed_cleanup);
kunit_put_resource(res);
}
+static inline bool kunit_resource_instance_match(struct kunit *test,
+ struct kunit_resource *res,
+ void *match_data)
+{
+ return res->data == match_data;
+}
+
/*
* Note: tests below use kunit_alloc_and_get_resource(), so as a consequence
* they have a reference to the associated resource that they must release
#include "string-stream.h"
-struct string_stream_fragment_alloc_context {
- struct kunit *test;
- int len;
- gfp_t gfp;
-};
-static int string_stream_fragment_init(struct kunit_resource *res,
- void *context)
+static struct string_stream_fragment *alloc_string_stream_fragment(
+ struct kunit *test, int len, gfp_t gfp)
{
- struct string_stream_fragment_alloc_context *ctx = context;
struct string_stream_fragment *frag;
- frag = kunit_kzalloc(ctx->test, sizeof(*frag), ctx->gfp);
+ frag = kunit_kzalloc(test, sizeof(*frag), gfp);
if (!frag)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
- frag->test = ctx->test;
- frag->fragment = kunit_kmalloc(ctx->test, ctx->len, ctx->gfp);
+ frag->fragment = kunit_kmalloc(test, len, gfp);
if (!frag->fragment)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
- res->data = frag;
-
- return 0;
+ return frag;
}
-static void string_stream_fragment_free(struct kunit_resource *res)
+static void string_stream_fragment_destroy(struct kunit *test,
+ struct string_stream_fragment *frag)
{
- struct string_stream_fragment *frag = res->data;
-
list_del(&frag->node);
- kunit_kfree(frag->test, frag->fragment);
- kunit_kfree(frag->test, frag);
-}
-
-static struct string_stream_fragment *alloc_string_stream_fragment(
- struct kunit *test, int len, gfp_t gfp)
-{
- struct string_stream_fragment_alloc_context context = {
- .test = test,
- .len = len,
- .gfp = gfp
- };
-
- return kunit_alloc_resource(test,
- string_stream_fragment_init,
- string_stream_fragment_free,
- gfp,
- &context);
-}
-
-static int string_stream_fragment_destroy(struct string_stream_fragment *frag)
-{
- return kunit_destroy_resource(frag->test,
- kunit_resource_instance_match,
- frag);
+ kunit_kfree(test, frag->fragment);
+ kunit_kfree(test, frag);
}
int string_stream_vadd(struct string_stream *stream,
frag_container_safe,
&stream->fragments,
node) {
- string_stream_fragment_destroy(frag_container);
+ string_stream_fragment_destroy(stream->test, frag_container);
}
stream->length = 0;
spin_unlock(&stream->lock);
gfp_t gfp;
};
-static int string_stream_init(struct kunit_resource *res, void *context)
+struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
{
struct string_stream *stream;
- struct string_stream_alloc_context *ctx = context;
- stream = kunit_kzalloc(ctx->test, sizeof(*stream), ctx->gfp);
+ stream = kunit_kzalloc(test, sizeof(*stream), gfp);
if (!stream)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
- res->data = stream;
- stream->gfp = ctx->gfp;
- stream->test = ctx->test;
+ stream->gfp = gfp;
+ stream->test = test;
INIT_LIST_HEAD(&stream->fragments);
spin_lock_init(&stream->lock);
- return 0;
+ return stream;
}
-static void string_stream_free(struct kunit_resource *res)
+void string_stream_destroy(struct string_stream *stream)
{
- struct string_stream *stream = res->data;
-
string_stream_clear(stream);
}
-
-struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp)
-{
- struct string_stream_alloc_context context = {
- .test = test,
- .gfp = gfp
- };
-
- return kunit_alloc_resource(test,
- string_stream_init,
- string_stream_free,
- gfp,
- &context);
-}
-
-int string_stream_destroy(struct string_stream *stream)
-{
- return kunit_destroy_resource(stream->test,
- kunit_resource_instance_match,
- stream);
-}
#include <linux/stdarg.h>
struct string_stream_fragment {
- struct kunit *test;
struct list_head node;
char *fragment;
};
bool string_stream_is_empty(struct string_stream *stream);
-int string_stream_destroy(struct string_stream *stream);
+void string_stream_destroy(struct string_stream *stream);
#endif /* _KUNIT_STRING_STREAM_H */
static void kunit_fail(struct kunit *test, const struct kunit_loc *loc,
enum kunit_assert_type type, const struct kunit_assert *assert,
- const struct va_format *message)
+ assert_format_t assert_format, const struct va_format *message)
{
struct string_stream *stream;
}
kunit_assert_prologue(loc, type, stream);
- assert->format(assert, message, stream);
+ assert_format(assert, message, stream);
kunit_print_string_stream(test, stream);
- WARN_ON(string_stream_destroy(stream));
+ string_stream_destroy(stream);
}
static void __noreturn kunit_abort(struct kunit *test)
const struct kunit_loc *loc,
enum kunit_assert_type type,
const struct kunit_assert *assert,
+ assert_format_t assert_format,
const char *fmt, ...)
{
va_list args;
message.fmt = fmt;
message.va = &args;
- kunit_fail(test, loc, type, assert, &message);
+ kunit_fail(test, loc, type, assert, assert_format, &message);
va_end(args);
}
EXPORT_SYMBOL_GPL(kunit_kmalloc_array);
-void kunit_kfree(struct kunit *test, const void *ptr)
+static inline bool kunit_kfree_match(struct kunit *test,
+ struct kunit_resource *res, void *match_data)
{
- struct kunit_resource *res;
-
- res = kunit_find_resource(test, kunit_resource_instance_match,
- (void *)ptr);
-
- /*
- * Removing the resource from the list of resources drops the
- * reference count to 1; the final put will trigger the free.
- */
- kunit_remove_resource(test, res);
+ /* Only match resources allocated with kunit_kmalloc() and friends. */
+ return res->free == kunit_kmalloc_array_free && res->data == match_data;
+}
- kunit_put_resource(res);
+void kunit_kfree(struct kunit *test, const void *ptr)
+{
+ if (!ptr)
+ return;
+ if (kunit_destroy_resource(test, kunit_kfree_match, (void *)ptr))
+ KUNIT_FAIL(test, "kunit_kfree: %px already freed or not allocated by kunit", ptr);
}
EXPORT_SYMBOL_GPL(kunit_kfree);
do {
new_last->next = first = READ_ONCE(head->first);
- } while (cmpxchg(&head->first, first, new_first) != first);
+ } while (!try_cmpxchg(&head->first, &first, new_first));
return !first;
}
*/
struct llist_node *llist_del_first(struct llist_head *head)
{
- struct llist_node *entry, *old_entry, *next;
+ struct llist_node *entry, *next;
entry = smp_load_acquire(&head->first);
- for (;;) {
+ do {
if (entry == NULL)
return NULL;
- old_entry = entry;
next = READ_ONCE(entry->next);
- entry = cmpxchg(&head->first, old_entry, next);
- if (entry == old_entry)
- break;
- }
+ } while (!try_cmpxchg(&head->first, &entry, next));
return entry;
}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Maple Tree implementation
+ * Copyright (c) 2018-2022 Oracle Corporation
+ * Authors: Liam R. Howlett <Liam.Howlett@oracle.com>
+ * Matthew Wilcox <willy@infradead.org>
+ */
+
+/*
+ * DOC: Interesting implementation details of the Maple Tree
+ *
+ * Each node type has a number of slots for entries and a number of slots for
+ * pivots. In the case of dense nodes, the pivots are implied by the position
+ * and are simply the slot index + the minimum of the node.
+ *
+ * In regular B-Tree terms, pivots are called keys. The term pivot is used to
+ * indicate that the tree is specifying ranges, Pivots may appear in the
+ * subtree with an entry attached to the value where as keys are unique to a
+ * specific position of a B-tree. Pivot values are inclusive of the slot with
+ * the same index.
+ *
+ *
+ * The following illustrates the layout of a range64 nodes slots and pivots.
+ *
+ *
+ * Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 |
+ * ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬
+ * │ │ │ │ │ │ │ │ └─ Implied maximum
+ * │ │ │ │ │ │ │ └─ Pivot 14
+ * │ │ │ │ │ │ └─ Pivot 13
+ * │ │ │ │ │ └─ Pivot 12
+ * │ │ │ │ └─ Pivot 11
+ * │ │ │ └─ Pivot 2
+ * │ │ └─ Pivot 1
+ * │ └─ Pivot 0
+ * └─ Implied minimum
+ *
+ * Slot contents:
+ * Internal (non-leaf) nodes contain pointers to other nodes.
+ * Leaf nodes contain entries.
+ *
+ * The location of interest is often referred to as an offset. All offsets have
+ * a slot, but the last offset has an implied pivot from the node above (or
+ * UINT_MAX for the root node.
+ *
+ * Ranges complicate certain write activities. When modifying any of
+ * the B-tree variants, it is known that one entry will either be added or
+ * deleted. When modifying the Maple Tree, one store operation may overwrite
+ * the entire data set, or one half of the tree, or the middle half of the tree.
+ *
+ */
+
+
+#include <linux/maple_tree.h>
+#include <linux/xarray.h>
+#include <linux/types.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/limits.h>
+#include <asm/barrier.h>
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/maple_tree.h>
+
+#define MA_ROOT_PARENT 1
+
+/*
+ * Maple state flags
+ * * MA_STATE_BULK - Bulk insert mode
+ * * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert
+ * * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation
+ */
+#define MA_STATE_BULK 1
+#define MA_STATE_REBALANCE 2
+#define MA_STATE_PREALLOC 4
+
+#define ma_parent_ptr(x) ((struct maple_pnode *)(x))
+#define ma_mnode_ptr(x) ((struct maple_node *)(x))
+#define ma_enode_ptr(x) ((struct maple_enode *)(x))
+static struct kmem_cache *maple_node_cache;
+
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+static const unsigned long mt_max[] = {
+ [maple_dense] = MAPLE_NODE_SLOTS,
+ [maple_leaf_64] = ULONG_MAX,
+ [maple_range_64] = ULONG_MAX,
+ [maple_arange_64] = ULONG_MAX,
+};
+#define mt_node_max(x) mt_max[mte_node_type(x)]
+#endif
+
+static const unsigned char mt_slots[] = {
+ [maple_dense] = MAPLE_NODE_SLOTS,
+ [maple_leaf_64] = MAPLE_RANGE64_SLOTS,
+ [maple_range_64] = MAPLE_RANGE64_SLOTS,
+ [maple_arange_64] = MAPLE_ARANGE64_SLOTS,
+};
+#define mt_slot_count(x) mt_slots[mte_node_type(x)]
+
+static const unsigned char mt_pivots[] = {
+ [maple_dense] = 0,
+ [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1,
+ [maple_range_64] = MAPLE_RANGE64_SLOTS - 1,
+ [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1,
+};
+#define mt_pivot_count(x) mt_pivots[mte_node_type(x)]
+
+static const unsigned char mt_min_slots[] = {
+ [maple_dense] = MAPLE_NODE_SLOTS / 2,
+ [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2,
+ [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2,
+ [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1,
+};
+#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)]
+
+#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2)
+#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1)
+
+struct maple_big_node {
+ struct maple_pnode *parent;
+ unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1];
+ union {
+ struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS];
+ struct {
+ unsigned long padding[MAPLE_BIG_NODE_GAPS];
+ unsigned long gap[MAPLE_BIG_NODE_GAPS];
+ };
+ };
+ unsigned char b_end;
+ enum maple_type type;
+};
+
+/*
+ * The maple_subtree_state is used to build a tree to replace a segment of an
+ * existing tree in a more atomic way. Any walkers of the older tree will hit a
+ * dead node and restart on updates.
+ */
+struct maple_subtree_state {
+ struct ma_state *orig_l; /* Original left side of subtree */
+ struct ma_state *orig_r; /* Original right side of subtree */
+ struct ma_state *l; /* New left side of subtree */
+ struct ma_state *m; /* New middle of subtree (rare) */
+ struct ma_state *r; /* New right side of subtree */
+ struct ma_topiary *free; /* nodes to be freed */
+ struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */
+ struct maple_big_node *bn;
+};
+
+/* Functions */
+static inline struct maple_node *mt_alloc_one(gfp_t gfp)
+{
+ return kmem_cache_alloc(maple_node_cache, gfp | __GFP_ZERO);
+}
+
+static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes)
+{
+ return kmem_cache_alloc_bulk(maple_node_cache, gfp | __GFP_ZERO, size,
+ nodes);
+}
+
+static inline void mt_free_bulk(size_t size, void __rcu **nodes)
+{
+ kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes);
+}
+
+static void mt_free_rcu(struct rcu_head *head)
+{
+ struct maple_node *node = container_of(head, struct maple_node, rcu);
+
+ kmem_cache_free(maple_node_cache, node);
+}
+
+/*
+ * ma_free_rcu() - Use rcu callback to free a maple node
+ * @node: The node to free
+ *
+ * The maple tree uses the parent pointer to indicate this node is no longer in
+ * use and will be freed.
+ */
+static void ma_free_rcu(struct maple_node *node)
+{
+ node->parent = ma_parent_ptr(node);
+ call_rcu(&node->rcu, mt_free_rcu);
+}
+
+static unsigned int mt_height(const struct maple_tree *mt)
+{
+ return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET;
+}
+
+static void mas_set_height(struct ma_state *mas)
+{
+ unsigned int new_flags = mas->tree->ma_flags;
+
+ new_flags &= ~MT_FLAGS_HEIGHT_MASK;
+ BUG_ON(mas->depth > MAPLE_HEIGHT_MAX);
+ new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET;
+ mas->tree->ma_flags = new_flags;
+}
+
+static unsigned int mas_mt_height(struct ma_state *mas)
+{
+ return mt_height(mas->tree);
+}
+
+static inline enum maple_type mte_node_type(const struct maple_enode *entry)
+{
+ return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) &
+ MAPLE_NODE_TYPE_MASK;
+}
+
+static inline bool ma_is_dense(const enum maple_type type)
+{
+ return type < maple_leaf_64;
+}
+
+static inline bool ma_is_leaf(const enum maple_type type)
+{
+ return type < maple_range_64;
+}
+
+static inline bool mte_is_leaf(const struct maple_enode *entry)
+{
+ return ma_is_leaf(mte_node_type(entry));
+}
+
+/*
+ * We also reserve values with the bottom two bits set to '10' which are
+ * below 4096
+ */
+static inline bool mt_is_reserved(const void *entry)
+{
+ return ((unsigned long)entry < MAPLE_RESERVED_RANGE) &&
+ xa_is_internal(entry);
+}
+
+static inline void mas_set_err(struct ma_state *mas, long err)
+{
+ mas->node = MA_ERROR(err);
+}
+
+static inline bool mas_is_ptr(struct ma_state *mas)
+{
+ return mas->node == MAS_ROOT;
+}
+
+static inline bool mas_is_start(struct ma_state *mas)
+{
+ return mas->node == MAS_START;
+}
+
+bool mas_is_err(struct ma_state *mas)
+{
+ return xa_is_err(mas->node);
+}
+
+static inline bool mas_searchable(struct ma_state *mas)
+{
+ if (mas_is_none(mas))
+ return false;
+
+ if (mas_is_ptr(mas))
+ return false;
+
+ return true;
+}
+
+static inline struct maple_node *mte_to_node(const struct maple_enode *entry)
+{
+ return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK);
+}
+
+/*
+ * mte_to_mat() - Convert a maple encoded node to a maple topiary node.
+ * @entry: The maple encoded node
+ *
+ * Return: a maple topiary pointer
+ */
+static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry)
+{
+ return (struct maple_topiary *)
+ ((unsigned long)entry & ~MAPLE_NODE_MASK);
+}
+
+/*
+ * mas_mn() - Get the maple state node.
+ * @mas: The maple state
+ *
+ * Return: the maple node (not encoded - bare pointer).
+ */
+static inline struct maple_node *mas_mn(const struct ma_state *mas)
+{
+ return mte_to_node(mas->node);
+}
+
+/*
+ * mte_set_node_dead() - Set a maple encoded node as dead.
+ * @mn: The maple encoded node.
+ */
+static inline void mte_set_node_dead(struct maple_enode *mn)
+{
+ mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn));
+ smp_wmb(); /* Needed for RCU */
+}
+
+/* Bit 1 indicates the root is a node */
+#define MAPLE_ROOT_NODE 0x02
+/* maple_type stored bit 3-6 */
+#define MAPLE_ENODE_TYPE_SHIFT 0x03
+/* Bit 2 means a NULL somewhere below */
+#define MAPLE_ENODE_NULL 0x04
+
+static inline struct maple_enode *mt_mk_node(const struct maple_node *node,
+ enum maple_type type)
+{
+ return (void *)((unsigned long)node |
+ (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL);
+}
+
+static inline void *mte_mk_root(const struct maple_enode *node)
+{
+ return (void *)((unsigned long)node | MAPLE_ROOT_NODE);
+}
+
+static inline void *mte_safe_root(const struct maple_enode *node)
+{
+ return (void *)((unsigned long)node & ~MAPLE_ROOT_NODE);
+}
+
+static inline void mte_set_full(const struct maple_enode *node)
+{
+ node = (void *)((unsigned long)node & ~MAPLE_ENODE_NULL);
+}
+
+static inline void mte_clear_full(const struct maple_enode *node)
+{
+ node = (void *)((unsigned long)node | MAPLE_ENODE_NULL);
+}
+
+static inline bool ma_is_root(struct maple_node *node)
+{
+ return ((unsigned long)node->parent & MA_ROOT_PARENT);
+}
+
+static inline bool mte_is_root(const struct maple_enode *node)
+{
+ return ma_is_root(mte_to_node(node));
+}
+
+static inline bool mas_is_root_limits(const struct ma_state *mas)
+{
+ return !mas->min && mas->max == ULONG_MAX;
+}
+
+static inline bool mt_is_alloc(struct maple_tree *mt)
+{
+ return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE);
+}
+
+/*
+ * The Parent Pointer
+ * Excluding root, the parent pointer is 256B aligned like all other tree nodes.
+ * When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16
+ * bit values need an extra bit to store the offset. This extra bit comes from
+ * a reuse of the last bit in the node type. This is possible by using bit 1 to
+ * indicate if bit 2 is part of the type or the slot.
+ *
+ * Note types:
+ * 0x??1 = Root
+ * 0x?00 = 16 bit nodes
+ * 0x010 = 32 bit nodes
+ * 0x110 = 64 bit nodes
+ *
+ * Slot size and alignment
+ * 0b??1 : Root
+ * 0b?00 : 16 bit values, type in 0-1, slot in 2-7
+ * 0b010 : 32 bit values, type in 0-2, slot in 3-7
+ * 0b110 : 64 bit values, type in 0-2, slot in 3-7
+ */
+
+#define MAPLE_PARENT_ROOT 0x01
+
+#define MAPLE_PARENT_SLOT_SHIFT 0x03
+#define MAPLE_PARENT_SLOT_MASK 0xF8
+
+#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02
+#define MAPLE_PARENT_16B_SLOT_MASK 0xFC
+
+#define MAPLE_PARENT_RANGE64 0x06
+#define MAPLE_PARENT_RANGE32 0x04
+#define MAPLE_PARENT_NOT_RANGE16 0x02
+
+/*
+ * mte_parent_shift() - Get the parent shift for the slot storage.
+ * @parent: The parent pointer cast as an unsigned long
+ * Return: The shift into that pointer to the star to of the slot
+ */
+static inline unsigned long mte_parent_shift(unsigned long parent)
+{
+ /* Note bit 1 == 0 means 16B */
+ if (likely(parent & MAPLE_PARENT_NOT_RANGE16))
+ return MAPLE_PARENT_SLOT_SHIFT;
+
+ return MAPLE_PARENT_16B_SLOT_SHIFT;
+}
+
+/*
+ * mte_parent_slot_mask() - Get the slot mask for the parent.
+ * @parent: The parent pointer cast as an unsigned long.
+ * Return: The slot mask for that parent.
+ */
+static inline unsigned long mte_parent_slot_mask(unsigned long parent)
+{
+ /* Note bit 1 == 0 means 16B */
+ if (likely(parent & MAPLE_PARENT_NOT_RANGE16))
+ return MAPLE_PARENT_SLOT_MASK;
+
+ return MAPLE_PARENT_16B_SLOT_MASK;
+}
+
+/*
+ * mas_parent_enum() - Return the maple_type of the parent from the stored
+ * parent type.
+ * @mas: The maple state
+ * @node: The maple_enode to extract the parent's enum
+ * Return: The node->parent maple_type
+ */
+static inline
+enum maple_type mte_parent_enum(struct maple_enode *p_enode,
+ struct maple_tree *mt)
+{
+ unsigned long p_type;
+
+ p_type = (unsigned long)p_enode;
+ if (p_type & MAPLE_PARENT_ROOT)
+ return 0; /* Validated in the caller. */
+
+ p_type &= MAPLE_NODE_MASK;
+ p_type = p_type & ~(MAPLE_PARENT_ROOT | mte_parent_slot_mask(p_type));
+
+ switch (p_type) {
+ case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */
+ if (mt_is_alloc(mt))
+ return maple_arange_64;
+ return maple_range_64;
+ }
+
+ return 0;
+}
+
+static inline
+enum maple_type mas_parent_enum(struct ma_state *mas, struct maple_enode *enode)
+{
+ return mte_parent_enum(ma_enode_ptr(mte_to_node(enode)->parent), mas->tree);
+}
+
+/*
+ * mte_set_parent() - Set the parent node and encode the slot
+ * @enode: The encoded maple node.
+ * @parent: The encoded maple node that is the parent of @enode.
+ * @slot: The slot that @enode resides in @parent.
+ *
+ * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the
+ * parent type.
+ */
+static inline
+void mte_set_parent(struct maple_enode *enode, const struct maple_enode *parent,
+ unsigned char slot)
+{
+ unsigned long val = (unsigned long) parent;
+ unsigned long shift;
+ unsigned long type;
+ enum maple_type p_type = mte_node_type(parent);
+
+ BUG_ON(p_type == maple_dense);
+ BUG_ON(p_type == maple_leaf_64);
+
+ switch (p_type) {
+ case maple_range_64:
+ case maple_arange_64:
+ shift = MAPLE_PARENT_SLOT_SHIFT;
+ type = MAPLE_PARENT_RANGE64;
+ break;
+ default:
+ case maple_dense:
+ case maple_leaf_64:
+ shift = type = 0;
+ break;
+ }
+
+ val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */
+ val |= (slot << shift) | type;
+ mte_to_node(enode)->parent = ma_parent_ptr(val);
+}
+
+/*
+ * mte_parent_slot() - get the parent slot of @enode.
+ * @enode: The encoded maple node.
+ *
+ * Return: The slot in the parent node where @enode resides.
+ */
+static inline unsigned int mte_parent_slot(const struct maple_enode *enode)
+{
+ unsigned long val = (unsigned long) mte_to_node(enode)->parent;
+
+ /* Root. */
+ if (val & 1)
+ return 0;
+
+ /*
+ * Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost
+ * by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT
+ */
+ return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val);
+}
+
+/*
+ * mte_parent() - Get the parent of @node.
+ * @node: The encoded maple node.
+ *
+ * Return: The parent maple node.
+ */
+static inline struct maple_node *mte_parent(const struct maple_enode *enode)
+{
+ return (void *)((unsigned long)
+ (mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK);
+}
+
+/*
+ * ma_dead_node() - check if the @enode is dead.
+ * @enode: The encoded maple node
+ *
+ * Return: true if dead, false otherwise.
+ */
+static inline bool ma_dead_node(const struct maple_node *node)
+{
+ struct maple_node *parent = (void *)((unsigned long)
+ node->parent & ~MAPLE_NODE_MASK);
+
+ return (parent == node);
+}
+/*
+ * mte_dead_node() - check if the @enode is dead.
+ * @enode: The encoded maple node
+ *
+ * Return: true if dead, false otherwise.
+ */
+static inline bool mte_dead_node(const struct maple_enode *enode)
+{
+ struct maple_node *parent, *node;
+
+ node = mte_to_node(enode);
+ parent = mte_parent(enode);
+ return (parent == node);
+}
+
+/*
+ * mas_allocated() - Get the number of nodes allocated in a maple state.
+ * @mas: The maple state
+ *
+ * The ma_state alloc member is overloaded to hold a pointer to the first
+ * allocated node or to the number of requested nodes to allocate. If bit 0 is
+ * set, then the alloc contains the number of requested nodes. If there is an
+ * allocated node, then the total allocated nodes is in that node.
+ *
+ * Return: The total number of nodes allocated
+ */
+static inline unsigned long mas_allocated(const struct ma_state *mas)
+{
+ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1))
+ return 0;
+
+ return mas->alloc->total;
+}
+
+/*
+ * mas_set_alloc_req() - Set the requested number of allocations.
+ * @mas: the maple state
+ * @count: the number of allocations.
+ *
+ * The requested number of allocations is either in the first allocated node,
+ * located in @mas->alloc->request_count, or directly in @mas->alloc if there is
+ * no allocated node. Set the request either in the node or do the necessary
+ * encoding to store in @mas->alloc directly.
+ */
+static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count)
+{
+ if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) {
+ if (!count)
+ mas->alloc = NULL;
+ else
+ mas->alloc = (struct maple_alloc *)(((count) << 1U) | 1U);
+ return;
+ }
+
+ mas->alloc->request_count = count;
+}
+
+/*
+ * mas_alloc_req() - get the requested number of allocations.
+ * @mas: The maple state
+ *
+ * The alloc count is either stored directly in @mas, or in
+ * @mas->alloc->request_count if there is at least one node allocated. Decode
+ * the request count if it's stored directly in @mas->alloc.
+ *
+ * Return: The allocation request count.
+ */
+static inline unsigned int mas_alloc_req(const struct ma_state *mas)
+{
+ if ((unsigned long)mas->alloc & 0x1)
+ return (unsigned long)(mas->alloc) >> 1;
+ else if (mas->alloc)
+ return mas->alloc->request_count;
+ return 0;
+}
+
+/*
+ * ma_pivots() - Get a pointer to the maple node pivots.
+ * @node - the maple node
+ * @type - the node type
+ *
+ * Return: A pointer to the maple node pivots
+ */
+static inline unsigned long *ma_pivots(struct maple_node *node,
+ enum maple_type type)
+{
+ switch (type) {
+ case maple_arange_64:
+ return node->ma64.pivot;
+ case maple_range_64:
+ case maple_leaf_64:
+ return node->mr64.pivot;
+ case maple_dense:
+ return NULL;
+ }
+ return NULL;
+}
+
+/*
+ * ma_gaps() - Get a pointer to the maple node gaps.
+ * @node - the maple node
+ * @type - the node type
+ *
+ * Return: A pointer to the maple node gaps
+ */
+static inline unsigned long *ma_gaps(struct maple_node *node,
+ enum maple_type type)
+{
+ switch (type) {
+ case maple_arange_64:
+ return node->ma64.gap;
+ case maple_range_64:
+ case maple_leaf_64:
+ case maple_dense:
+ return NULL;
+ }
+ return NULL;
+}
+
+/*
+ * mte_pivot() - Get the pivot at @piv of the maple encoded node.
+ * @mn: The maple encoded node.
+ * @piv: The pivot.
+ *
+ * Return: the pivot at @piv of @mn.
+ */
+static inline unsigned long mte_pivot(const struct maple_enode *mn,
+ unsigned char piv)
+{
+ struct maple_node *node = mte_to_node(mn);
+
+ if (piv >= mt_pivots[piv]) {
+ WARN_ON(1);
+ return 0;
+ }
+ switch (mte_node_type(mn)) {
+ case maple_arange_64:
+ return node->ma64.pivot[piv];
+ case maple_range_64:
+ case maple_leaf_64:
+ return node->mr64.pivot[piv];
+ case maple_dense:
+ return 0;
+ }
+ return 0;
+}
+
+/*
+ * mas_safe_pivot() - get the pivot at @piv or mas->max.
+ * @mas: The maple state
+ * @pivots: The pointer to the maple node pivots
+ * @piv: The pivot to fetch
+ * @type: The maple node type
+ *
+ * Return: The pivot at @piv within the limit of the @pivots array, @mas->max
+ * otherwise.
+ */
+static inline unsigned long
+mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots,
+ unsigned char piv, enum maple_type type)
+{
+ if (piv >= mt_pivots[type])
+ return mas->max;
+
+ return pivots[piv];
+}
+
+/*
+ * mas_safe_min() - Return the minimum for a given offset.
+ * @mas: The maple state
+ * @pivots: The pointer to the maple node pivots
+ * @offset: The offset into the pivot array
+ *
+ * Return: The minimum range value that is contained in @offset.
+ */
+static inline unsigned long
+mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset)
+{
+ if (likely(offset))
+ return pivots[offset - 1] + 1;
+
+ return mas->min;
+}
+
+/*
+ * mas_logical_pivot() - Get the logical pivot of a given offset.
+ * @mas: The maple state
+ * @pivots: The pointer to the maple node pivots
+ * @offset: The offset into the pivot array
+ * @type: The maple node type
+ *
+ * When there is no value at a pivot (beyond the end of the data), then the
+ * pivot is actually @mas->max.
+ *
+ * Return: the logical pivot of a given @offset.
+ */
+static inline unsigned long
+mas_logical_pivot(struct ma_state *mas, unsigned long *pivots,
+ unsigned char offset, enum maple_type type)
+{
+ unsigned long lpiv = mas_safe_pivot(mas, pivots, offset, type);
+
+ if (likely(lpiv))
+ return lpiv;
+
+ if (likely(offset))
+ return mas->max;
+
+ return lpiv;
+}
+
+/*
+ * mte_set_pivot() - Set a pivot to a value in an encoded maple node.
+ * @mn: The encoded maple node
+ * @piv: The pivot offset
+ * @val: The value of the pivot
+ */
+static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv,
+ unsigned long val)
+{
+ struct maple_node *node = mte_to_node(mn);
+ enum maple_type type = mte_node_type(mn);
+
+ BUG_ON(piv >= mt_pivots[type]);
+ switch (type) {
+ default:
+ case maple_range_64:
+ case maple_leaf_64:
+ node->mr64.pivot[piv] = val;
+ break;
+ case maple_arange_64:
+ node->ma64.pivot[piv] = val;
+ break;
+ case maple_dense:
+ break;
+ }
+
+}
+
+/*
+ * ma_slots() - Get a pointer to the maple node slots.
+ * @mn: The maple node
+ * @mt: The maple node type
+ *
+ * Return: A pointer to the maple node slots
+ */
+static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt)
+{
+ switch (mt) {
+ default:
+ case maple_arange_64:
+ return mn->ma64.slot;
+ case maple_range_64:
+ case maple_leaf_64:
+ return mn->mr64.slot;
+ case maple_dense:
+ return mn->slot;
+ }
+}
+
+static inline bool mt_locked(const struct maple_tree *mt)
+{
+ return mt_external_lock(mt) ? mt_lock_is_held(mt) :
+ lockdep_is_held(&mt->ma_lock);
+}
+
+static inline void *mt_slot(const struct maple_tree *mt,
+ void __rcu **slots, unsigned char offset)
+{
+ return rcu_dereference_check(slots[offset], mt_locked(mt));
+}
+
+/*
+ * mas_slot_locked() - Get the slot value when holding the maple tree lock.
+ * @mas: The maple state
+ * @slots: The pointer to the slots
+ * @offset: The offset into the slots array to fetch
+ *
+ * Return: The entry stored in @slots at the @offset.
+ */
+static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots,
+ unsigned char offset)
+{
+ return rcu_dereference_protected(slots[offset], mt_locked(mas->tree));
+}
+
+/*
+ * mas_slot() - Get the slot value when not holding the maple tree lock.
+ * @mas: The maple state
+ * @slots: The pointer to the slots
+ * @offset: The offset into the slots array to fetch
+ *
+ * Return: The entry stored in @slots at the @offset
+ */
+static inline void *mas_slot(struct ma_state *mas, void __rcu **slots,
+ unsigned char offset)
+{
+ return mt_slot(mas->tree, slots, offset);
+}
+
+/*
+ * mas_root() - Get the maple tree root.
+ * @mas: The maple state.
+ *
+ * Return: The pointer to the root of the tree
+ */
+static inline void *mas_root(struct ma_state *mas)
+{
+ return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree));
+}
+
+static inline void *mt_root_locked(struct maple_tree *mt)
+{
+ return rcu_dereference_protected(mt->ma_root, mt_locked(mt));
+}
+
+/*
+ * mas_root_locked() - Get the maple tree root when holding the maple tree lock.
+ * @mas: The maple state.
+ *
+ * Return: The pointer to the root of the tree
+ */
+static inline void *mas_root_locked(struct ma_state *mas)
+{
+ return mt_root_locked(mas->tree);
+}
+
+static inline struct maple_metadata *ma_meta(struct maple_node *mn,
+ enum maple_type mt)
+{
+ switch (mt) {
+ case maple_arange_64:
+ return &mn->ma64.meta;
+ default:
+ return &mn->mr64.meta;
+ }
+}
+
+/*
+ * ma_set_meta() - Set the metadata information of a node.
+ * @mn: The maple node
+ * @mt: The maple node type
+ * @offset: The offset of the highest sub-gap in this node.
+ * @end: The end of the data in this node.
+ */
+static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt,
+ unsigned char offset, unsigned char end)
+{
+ struct maple_metadata *meta = ma_meta(mn, mt);
+
+ meta->gap = offset;
+ meta->end = end;
+}
+
+/*
+ * ma_meta_end() - Get the data end of a node from the metadata
+ * @mn: The maple node
+ * @mt: The maple node type
+ */
+static inline unsigned char ma_meta_end(struct maple_node *mn,
+ enum maple_type mt)
+{
+ struct maple_metadata *meta = ma_meta(mn, mt);
+
+ return meta->end;
+}
+
+/*
+ * ma_meta_gap() - Get the largest gap location of a node from the metadata
+ * @mn: The maple node
+ * @mt: The maple node type
+ */
+static inline unsigned char ma_meta_gap(struct maple_node *mn,
+ enum maple_type mt)
+{
+ BUG_ON(mt != maple_arange_64);
+
+ return mn->ma64.meta.gap;
+}
+
+/*
+ * ma_set_meta_gap() - Set the largest gap location in a nodes metadata
+ * @mn: The maple node
+ * @mn: The maple node type
+ * @offset: The location of the largest gap.
+ */
+static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt,
+ unsigned char offset)
+{
+
+ struct maple_metadata *meta = ma_meta(mn, mt);
+
+ meta->gap = offset;
+}
+
+/*
+ * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.
+ * @mat - the ma_topiary, a linked list of dead nodes.
+ * @dead_enode - the node to be marked as dead and added to the tail of the list
+ *
+ * Add the @dead_enode to the linked list in @mat.
+ */
+static inline void mat_add(struct ma_topiary *mat,
+ struct maple_enode *dead_enode)
+{
+ mte_set_node_dead(dead_enode);
+ mte_to_mat(dead_enode)->next = NULL;
+ if (!mat->tail) {
+ mat->tail = mat->head = dead_enode;
+ return;
+ }
+
+ mte_to_mat(mat->tail)->next = dead_enode;
+ mat->tail = dead_enode;
+}
+
+static void mte_destroy_walk(struct maple_enode *, struct maple_tree *);
+static inline void mas_free(struct ma_state *mas, struct maple_enode *used);
+
+/*
+ * mas_mat_free() - Free all nodes in a dead list.
+ * @mas - the maple state
+ * @mat - the ma_topiary linked list of dead nodes to free.
+ *
+ * Free walk a dead list.
+ */
+static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat)
+{
+ struct maple_enode *next;
+
+ while (mat->head) {
+ next = mte_to_mat(mat->head)->next;
+ mas_free(mas, mat->head);
+ mat->head = next;
+ }
+}
+
+/*
+ * mas_mat_destroy() - Free all nodes and subtrees in a dead list.
+ * @mas - the maple state
+ * @mat - the ma_topiary linked list of dead nodes to free.
+ *
+ * Destroy walk a dead list.
+ */
+static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat)
+{
+ struct maple_enode *next;
+
+ while (mat->head) {
+ next = mte_to_mat(mat->head)->next;
+ mte_destroy_walk(mat->head, mat->mtree);
+ mat->head = next;
+ }
+}
+/*
+ * mas_descend() - Descend into the slot stored in the ma_state.
+ * @mas - the maple state.
+ *
+ * Note: Not RCU safe, only use in write side or debug code.
+ */
+static inline void mas_descend(struct ma_state *mas)
+{
+ enum maple_type type;
+ unsigned long *pivots;
+ struct maple_node *node;
+ void __rcu **slots;
+
+ node = mas_mn(mas);
+ type = mte_node_type(mas->node);
+ pivots = ma_pivots(node, type);
+ slots = ma_slots(node, type);
+
+ if (mas->offset)
+ mas->min = pivots[mas->offset - 1] + 1;
+ mas->max = mas_safe_pivot(mas, pivots, mas->offset, type);
+ mas->node = mas_slot(mas, slots, mas->offset);
+}
+
+/*
+ * mte_set_gap() - Set a maple node gap.
+ * @mn: The encoded maple node
+ * @gap: The offset of the gap to set
+ * @val: The gap value
+ */
+static inline void mte_set_gap(const struct maple_enode *mn,
+ unsigned char gap, unsigned long val)
+{
+ switch (mte_node_type(mn)) {
+ default:
+ break;
+ case maple_arange_64:
+ mte_to_node(mn)->ma64.gap[gap] = val;
+ break;
+ }
+}
+
+/*
+ * mas_ascend() - Walk up a level of the tree.
+ * @mas: The maple state
+ *
+ * Sets the @mas->max and @mas->min to the correct values when walking up. This
+ * may cause several levels of walking up to find the correct min and max.
+ * May find a dead node which will cause a premature return.
+ * Return: 1 on dead node, 0 otherwise
+ */
+static int mas_ascend(struct ma_state *mas)
+{
+ struct maple_enode *p_enode; /* parent enode. */
+ struct maple_enode *a_enode; /* ancestor enode. */
+ struct maple_node *a_node; /* ancestor node. */
+ struct maple_node *p_node; /* parent node. */
+ unsigned char a_slot;
+ enum maple_type a_type;
+ unsigned long min, max;
+ unsigned long *pivots;
+ unsigned char offset;
+ bool set_max = false, set_min = false;
+
+ a_node = mas_mn(mas);
+ if (ma_is_root(a_node)) {
+ mas->offset = 0;
+ return 0;
+ }
+
+ p_node = mte_parent(mas->node);
+ if (unlikely(a_node == p_node))
+ return 1;
+ a_type = mas_parent_enum(mas, mas->node);
+ offset = mte_parent_slot(mas->node);
+ a_enode = mt_mk_node(p_node, a_type);
+
+ /* Check to make sure all parent information is still accurate */
+ if (p_node != mte_parent(mas->node))
+ return 1;
+
+ mas->node = a_enode;
+ mas->offset = offset;
+
+ if (mte_is_root(a_enode)) {
+ mas->max = ULONG_MAX;
+ mas->min = 0;
+ return 0;
+ }
+
+ min = 0;
+ max = ULONG_MAX;
+ do {
+ p_enode = a_enode;
+ a_type = mas_parent_enum(mas, p_enode);
+ a_node = mte_parent(p_enode);
+ a_slot = mte_parent_slot(p_enode);
+ pivots = ma_pivots(a_node, a_type);
+ a_enode = mt_mk_node(a_node, a_type);
+
+ if (!set_min && a_slot) {
+ set_min = true;
+ min = pivots[a_slot - 1] + 1;
+ }
+
+ if (!set_max && a_slot < mt_pivots[a_type]) {
+ set_max = true;
+ max = pivots[a_slot];
+ }
+
+ if (unlikely(ma_dead_node(a_node)))
+ return 1;
+
+ if (unlikely(ma_is_root(a_node)))
+ break;
+
+ } while (!set_min || !set_max);
+
+ mas->max = max;
+ mas->min = min;
+ return 0;
+}
+
+/*
+ * mas_pop_node() - Get a previously allocated maple node from the maple state.
+ * @mas: The maple state
+ *
+ * Return: A pointer to a maple node.
+ */
+static inline struct maple_node *mas_pop_node(struct ma_state *mas)
+{
+ struct maple_alloc *ret, *node = mas->alloc;
+ unsigned long total = mas_allocated(mas);
+
+ /* nothing or a request pending. */
+ if (unlikely(!total))
+ return NULL;
+
+ if (total == 1) {
+ /* single allocation in this ma_state */
+ mas->alloc = NULL;
+ ret = node;
+ goto single_node;
+ }
+
+ if (!node->node_count) {
+ /* Single allocation in this node. */
+ mas->alloc = node->slot[0];
+ node->slot[0] = NULL;
+ mas->alloc->total = node->total - 1;
+ ret = node;
+ goto new_head;
+ }
+
+ node->total--;
+ ret = node->slot[node->node_count];
+ node->slot[node->node_count--] = NULL;
+
+single_node:
+new_head:
+ ret->total = 0;
+ ret->node_count = 0;
+ if (ret->request_count) {
+ mas_set_alloc_req(mas, ret->request_count + 1);
+ ret->request_count = 0;
+ }
+ return (struct maple_node *)ret;
+}
+
+/*
+ * mas_push_node() - Push a node back on the maple state allocation.
+ * @mas: The maple state
+ * @used: The used maple node
+ *
+ * Stores the maple node back into @mas->alloc for reuse. Updates allocated and
+ * requested node count as necessary.
+ */
+static inline void mas_push_node(struct ma_state *mas, struct maple_node *used)
+{
+ struct maple_alloc *reuse = (struct maple_alloc *)used;
+ struct maple_alloc *head = mas->alloc;
+ unsigned long count;
+ unsigned int requested = mas_alloc_req(mas);
+
+ memset(reuse, 0, sizeof(*reuse));
+ count = mas_allocated(mas);
+
+ if (count && (head->node_count < MAPLE_ALLOC_SLOTS - 1)) {
+ if (head->slot[0])
+ head->node_count++;
+ head->slot[head->node_count] = reuse;
+ head->total++;
+ goto done;
+ }
+
+ reuse->total = 1;
+ if ((head) && !((unsigned long)head & 0x1)) {
+ head->request_count = 0;
+ reuse->slot[0] = head;
+ reuse->total += head->total;
+ }
+
+ mas->alloc = reuse;
+done:
+ if (requested > 1)
+ mas_set_alloc_req(mas, requested - 1);
+}
+
+/*
+ * mas_alloc_nodes() - Allocate nodes into a maple state
+ * @mas: The maple state
+ * @gfp: The GFP Flags
+ */
+static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp)
+{
+ struct maple_alloc *node;
+ struct maple_alloc **nodep = &mas->alloc;
+ unsigned long allocated = mas_allocated(mas);
+ unsigned long success = allocated;
+ unsigned int requested = mas_alloc_req(mas);
+ unsigned int count;
+ void **slots = NULL;
+ unsigned int max_req = 0;
+
+ if (!requested)
+ return;
+
+ mas_set_alloc_req(mas, 0);
+ if (mas->mas_flags & MA_STATE_PREALLOC) {
+ if (allocated)
+ return;
+ WARN_ON(!allocated);
+ }
+
+ if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS - 1) {
+ node = (struct maple_alloc *)mt_alloc_one(gfp);
+ if (!node)
+ goto nomem_one;
+
+ if (allocated)
+ node->slot[0] = mas->alloc;
+
+ success++;
+ mas->alloc = node;
+ requested--;
+ }
+
+ node = mas->alloc;
+ while (requested) {
+ max_req = MAPLE_ALLOC_SLOTS;
+ if (node->slot[0]) {
+ unsigned int offset = node->node_count + 1;
+
+ slots = (void **)&node->slot[offset];
+ max_req -= offset;
+ } else {
+ slots = (void **)&node->slot;
+ }
+
+ max_req = min(requested, max_req);
+ count = mt_alloc_bulk(gfp, max_req, slots);
+ if (!count)
+ goto nomem_bulk;
+
+ node->node_count += count;
+ /* zero indexed. */
+ if (slots == (void **)&node->slot)
+ node->node_count--;
+
+ success += count;
+ nodep = &node->slot[0];
+ node = *nodep;
+ requested -= count;
+ }
+ mas->alloc->total = success;
+ return;
+
+nomem_bulk:
+ /* Clean up potential freed allocations on bulk failure */
+ memset(slots, 0, max_req * sizeof(unsigned long));
+nomem_one:
+ mas_set_alloc_req(mas, requested);
+ if (mas->alloc && !(((unsigned long)mas->alloc & 0x1)))
+ mas->alloc->total = success;
+ mas_set_err(mas, -ENOMEM);
+ return;
+
+}
+
+/*
+ * mas_free() - Free an encoded maple node
+ * @mas: The maple state
+ * @used: The encoded maple node to free.
+ *
+ * Uses rcu free if necessary, pushes @used back on the maple state allocations
+ * otherwise.
+ */
+static inline void mas_free(struct ma_state *mas, struct maple_enode *used)
+{
+ struct maple_node *tmp = mte_to_node(used);
+
+ if (mt_in_rcu(mas->tree))
+ ma_free_rcu(tmp);
+ else
+ mas_push_node(mas, tmp);
+}
+
+/*
+ * mas_node_count() - Check if enough nodes are allocated and request more if
+ * there is not enough nodes.
+ * @mas: The maple state
+ * @count: The number of nodes needed
+ * @gfp: the gfp flags
+ */
+static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp)
+{
+ unsigned long allocated = mas_allocated(mas);
+
+ if (allocated < count) {
+ mas_set_alloc_req(mas, count - allocated);
+ mas_alloc_nodes(mas, gfp);
+ }
+}
+
+/*
+ * mas_node_count() - Check if enough nodes are allocated and request more if
+ * there is not enough nodes.
+ * @mas: The maple state
+ * @count: The number of nodes needed
+ *
+ * Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags.
+ */
+static void mas_node_count(struct ma_state *mas, int count)
+{
+ return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN);
+}
+
+/*
+ * mas_start() - Sets up maple state for operations.
+ * @mas: The maple state.
+ *
+ * If mas->node == MAS_START, then set the min, max, depth, and offset to
+ * defaults.
+ *
+ * Return:
+ * - If mas->node is an error or not MAS_START, return NULL.
+ * - If it's an empty tree: NULL & mas->node == MAS_NONE
+ * - If it's a single entry: The entry & mas->node == MAS_ROOT
+ * - If it's a tree: NULL & mas->node == safe root node.
+ */
+static inline struct maple_enode *mas_start(struct ma_state *mas)
+{
+ if (likely(mas_is_start(mas))) {
+ struct maple_enode *root;
+
+ mas->node = MAS_NONE;
+ mas->min = 0;
+ mas->max = ULONG_MAX;
+ mas->depth = 0;
+ mas->offset = 0;
+
+ root = mas_root(mas);
+ /* Tree with nodes */
+ if (likely(xa_is_node(root))) {
+ mas->node = mte_safe_root(root);
+ return NULL;
+ }
+
+ /* empty tree */
+ if (unlikely(!root)) {
+ mas->offset = MAPLE_NODE_SLOTS;
+ return NULL;
+ }
+
+ /* Single entry tree */
+ mas->node = MAS_ROOT;
+ mas->offset = MAPLE_NODE_SLOTS;
+
+ /* Single entry tree. */
+ if (mas->index > 0)
+ return NULL;
+
+ return root;
+ }
+
+ return NULL;
+}
+
+/*
+ * ma_data_end() - Find the end of the data in a node.
+ * @node: The maple node
+ * @type: The maple node type
+ * @pivots: The array of pivots in the node
+ * @max: The maximum value in the node
+ *
+ * Uses metadata to find the end of the data when possible.
+ * Return: The zero indexed last slot with data (may be null).
+ */
+static inline unsigned char ma_data_end(struct maple_node *node,
+ enum maple_type type,
+ unsigned long *pivots,
+ unsigned long max)
+{
+ unsigned char offset;
+
+ if (type == maple_arange_64)
+ return ma_meta_end(node, type);
+
+ offset = mt_pivots[type] - 1;
+ if (likely(!pivots[offset]))
+ return ma_meta_end(node, type);
+
+ if (likely(pivots[offset] == max))
+ return offset;
+
+ return mt_pivots[type];
+}
+
+/*
+ * mas_data_end() - Find the end of the data (slot).
+ * @mas: the maple state
+ *
+ * This method is optimized to check the metadata of a node if the node type
+ * supports data end metadata.
+ *
+ * Return: The zero indexed last slot with data (may be null).
+ */
+static inline unsigned char mas_data_end(struct ma_state *mas)
+{
+ enum maple_type type;
+ struct maple_node *node;
+ unsigned char offset;
+ unsigned long *pivots;
+
+ type = mte_node_type(mas->node);
+ node = mas_mn(mas);
+ if (type == maple_arange_64)
+ return ma_meta_end(node, type);
+
+ pivots = ma_pivots(node, type);
+ offset = mt_pivots[type] - 1;
+ if (likely(!pivots[offset]))
+ return ma_meta_end(node, type);
+
+ if (likely(pivots[offset] == mas->max))
+ return offset;
+
+ return mt_pivots[type];
+}
+
+/*
+ * mas_leaf_max_gap() - Returns the largest gap in a leaf node
+ * @mas - the maple state
+ *
+ * Return: The maximum gap in the leaf.
+ */
+static unsigned long mas_leaf_max_gap(struct ma_state *mas)
+{
+ enum maple_type mt;
+ unsigned long pstart, gap, max_gap;
+ struct maple_node *mn;
+ unsigned long *pivots;
+ void __rcu **slots;
+ unsigned char i;
+ unsigned char max_piv;
+
+ mt = mte_node_type(mas->node);
+ mn = mas_mn(mas);
+ slots = ma_slots(mn, mt);
+ max_gap = 0;
+ if (unlikely(ma_is_dense(mt))) {
+ gap = 0;
+ for (i = 0; i < mt_slots[mt]; i++) {
+ if (slots[i]) {
+ if (gap > max_gap)
+ max_gap = gap;
+ gap = 0;
+ } else {
+ gap++;
+ }
+ }
+ if (gap > max_gap)
+ max_gap = gap;
+ return max_gap;
+ }
+
+ /*
+ * Check the first implied pivot optimizes the loop below and slot 1 may
+ * be skipped if there is a gap in slot 0.
+ */
+ pivots = ma_pivots(mn, mt);
+ if (likely(!slots[0])) {
+ max_gap = pivots[0] - mas->min + 1;
+ i = 2;
+ } else {
+ i = 1;
+ }
+
+ /* reduce max_piv as the special case is checked before the loop */
+ max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1;
+ /*
+ * Check end implied pivot which can only be a gap on the right most
+ * node.
+ */
+ if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) {
+ gap = ULONG_MAX - pivots[max_piv];
+ if (gap > max_gap)
+ max_gap = gap;
+ }
+
+ for (; i <= max_piv; i++) {
+ /* data == no gap. */
+ if (likely(slots[i]))
+ continue;
+
+ pstart = pivots[i - 1];
+ gap = pivots[i] - pstart;
+ if (gap > max_gap)
+ max_gap = gap;
+
+ /* There cannot be two gaps in a row. */
+ i++;
+ }
+ return max_gap;
+}
+
+/*
+ * ma_max_gap() - Get the maximum gap in a maple node (non-leaf)
+ * @node: The maple node
+ * @gaps: The pointer to the gaps
+ * @mt: The maple node type
+ * @*off: Pointer to store the offset location of the gap.
+ *
+ * Uses the metadata data end to scan backwards across set gaps.
+ *
+ * Return: The maximum gap value
+ */
+static inline unsigned long
+ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt,
+ unsigned char *off)
+{
+ unsigned char offset, i;
+ unsigned long max_gap = 0;
+
+ i = offset = ma_meta_end(node, mt);
+ do {
+ if (gaps[i] > max_gap) {
+ max_gap = gaps[i];
+ offset = i;
+ }
+ } while (i--);
+
+ *off = offset;
+ return max_gap;
+}
+
+/*
+ * mas_max_gap() - find the largest gap in a non-leaf node and set the slot.
+ * @mas: The maple state.
+ *
+ * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap.
+ *
+ * Return: The gap value.
+ */
+static inline unsigned long mas_max_gap(struct ma_state *mas)
+{
+ unsigned long *gaps;
+ unsigned char offset;
+ enum maple_type mt;
+ struct maple_node *node;
+
+ mt = mte_node_type(mas->node);
+ if (ma_is_leaf(mt))
+ return mas_leaf_max_gap(mas);
+
+ node = mas_mn(mas);
+ offset = ma_meta_gap(node, mt);
+ if (offset == MAPLE_ARANGE64_META_MAX)
+ return 0;
+
+ gaps = ma_gaps(node, mt);
+ return gaps[offset];
+}
+
+/*
+ * mas_parent_gap() - Set the parent gap and any gaps above, as needed
+ * @mas: The maple state
+ * @offset: The gap offset in the parent to set
+ * @new: The new gap value.
+ *
+ * Set the parent gap then continue to set the gap upwards, using the metadata
+ * of the parent to see if it is necessary to check the node above.
+ */
+static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset,
+ unsigned long new)
+{
+ unsigned long meta_gap = 0;
+ struct maple_node *pnode;
+ struct maple_enode *penode;
+ unsigned long *pgaps;
+ unsigned char meta_offset;
+ enum maple_type pmt;
+
+ pnode = mte_parent(mas->node);
+ pmt = mas_parent_enum(mas, mas->node);
+ penode = mt_mk_node(pnode, pmt);
+ pgaps = ma_gaps(pnode, pmt);
+
+ascend:
+ meta_offset = ma_meta_gap(pnode, pmt);
+ if (meta_offset == MAPLE_ARANGE64_META_MAX)
+ meta_gap = 0;
+ else
+ meta_gap = pgaps[meta_offset];
+
+ pgaps[offset] = new;
+
+ if (meta_gap == new)
+ return;
+
+ if (offset != meta_offset) {
+ if (meta_gap > new)
+ return;
+
+ ma_set_meta_gap(pnode, pmt, offset);
+ } else if (new < meta_gap) {
+ meta_offset = 15;
+ new = ma_max_gap(pnode, pgaps, pmt, &meta_offset);
+ ma_set_meta_gap(pnode, pmt, meta_offset);
+ }
+
+ if (ma_is_root(pnode))
+ return;
+
+ /* Go to the parent node. */
+ pnode = mte_parent(penode);
+ pmt = mas_parent_enum(mas, penode);
+ pgaps = ma_gaps(pnode, pmt);
+ offset = mte_parent_slot(penode);
+ penode = mt_mk_node(pnode, pmt);
+ goto ascend;
+}
+
+/*
+ * mas_update_gap() - Update a nodes gaps and propagate up if necessary.
+ * @mas - the maple state.
+ */
+static inline void mas_update_gap(struct ma_state *mas)
+{
+ unsigned char pslot;
+ unsigned long p_gap;
+ unsigned long max_gap;
+
+ if (!mt_is_alloc(mas->tree))
+ return;
+
+ if (mte_is_root(mas->node))
+ return;
+
+ max_gap = mas_max_gap(mas);
+
+ pslot = mte_parent_slot(mas->node);
+ p_gap = ma_gaps(mte_parent(mas->node),
+ mas_parent_enum(mas, mas->node))[pslot];
+
+ if (p_gap != max_gap)
+ mas_parent_gap(mas, pslot, max_gap);
+}
+
+/*
+ * mas_adopt_children() - Set the parent pointer of all nodes in @parent to
+ * @parent with the slot encoded.
+ * @mas - the maple state (for the tree)
+ * @parent - the maple encoded node containing the children.
+ */
+static inline void mas_adopt_children(struct ma_state *mas,
+ struct maple_enode *parent)
+{
+ enum maple_type type = mte_node_type(parent);
+ struct maple_node *node = mas_mn(mas);
+ void __rcu **slots = ma_slots(node, type);
+ unsigned long *pivots = ma_pivots(node, type);
+ struct maple_enode *child;
+ unsigned char offset;
+
+ offset = ma_data_end(node, type, pivots, mas->max);
+ do {
+ child = mas_slot_locked(mas, slots, offset);
+ mte_set_parent(child, parent, offset);
+ } while (offset--);
+}
+
+/*
+ * mas_replace() - Replace a maple node in the tree with mas->node. Uses the
+ * parent encoding to locate the maple node in the tree.
+ * @mas - the ma_state to use for operations.
+ * @advanced - boolean to adopt the child nodes and free the old node (false) or
+ * leave the node (true) and handle the adoption and free elsewhere.
+ */
+static inline void mas_replace(struct ma_state *mas, bool advanced)
+ __must_hold(mas->tree->lock)
+{
+ struct maple_node *mn = mas_mn(mas);
+ struct maple_enode *old_enode;
+ unsigned char offset = 0;
+ void __rcu **slots = NULL;
+
+ if (ma_is_root(mn)) {
+ old_enode = mas_root_locked(mas);
+ } else {
+ offset = mte_parent_slot(mas->node);
+ slots = ma_slots(mte_parent(mas->node),
+ mas_parent_enum(mas, mas->node));
+ old_enode = mas_slot_locked(mas, slots, offset);
+ }
+
+ if (!advanced && !mte_is_leaf(mas->node))
+ mas_adopt_children(mas, mas->node);
+
+ if (mte_is_root(mas->node)) {
+ mn->parent = ma_parent_ptr(
+ ((unsigned long)mas->tree | MA_ROOT_PARENT));
+ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
+ mas_set_height(mas);
+ } else {
+ rcu_assign_pointer(slots[offset], mas->node);
+ }
+
+ if (!advanced)
+ mas_free(mas, old_enode);
+}
+
+/*
+ * mas_new_child() - Find the new child of a node.
+ * @mas: the maple state
+ * @child: the maple state to store the child.
+ */
+static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child)
+ __must_hold(mas->tree->lock)
+{
+ enum maple_type mt;
+ unsigned char offset;
+ unsigned char end;
+ unsigned long *pivots;
+ struct maple_enode *entry;
+ struct maple_node *node;
+ void __rcu **slots;
+
+ mt = mte_node_type(mas->node);
+ node = mas_mn(mas);
+ slots = ma_slots(node, mt);
+ pivots = ma_pivots(node, mt);
+ end = ma_data_end(node, mt, pivots, mas->max);
+ for (offset = mas->offset; offset <= end; offset++) {
+ entry = mas_slot_locked(mas, slots, offset);
+ if (mte_parent(entry) == node) {
+ *child = *mas;
+ mas->offset = offset + 1;
+ child->offset = offset;
+ mas_descend(child);
+ child->offset = 0;
+ return true;
+ }
+ }
+ return false;
+}
+
+/*
+ * mab_shift_right() - Shift the data in mab right. Note, does not clean out the
+ * old data or set b_node->b_end.
+ * @b_node: the maple_big_node
+ * @shift: the shift count
+ */
+static inline void mab_shift_right(struct maple_big_node *b_node,
+ unsigned char shift)
+{
+ unsigned long size = b_node->b_end * sizeof(unsigned long);
+
+ memmove(b_node->pivot + shift, b_node->pivot, size);
+ memmove(b_node->slot + shift, b_node->slot, size);
+ if (b_node->type == maple_arange_64)
+ memmove(b_node->gap + shift, b_node->gap, size);
+}
+
+/*
+ * mab_middle_node() - Check if a middle node is needed (unlikely)
+ * @b_node: the maple_big_node that contains the data.
+ * @size: the amount of data in the b_node
+ * @split: the potential split location
+ * @slot_count: the size that can be stored in a single node being considered.
+ *
+ * Return: true if a middle node is required.
+ */
+static inline bool mab_middle_node(struct maple_big_node *b_node, int split,
+ unsigned char slot_count)
+{
+ unsigned char size = b_node->b_end;
+
+ if (size >= 2 * slot_count)
+ return true;
+
+ if (!b_node->slot[split] && (size >= 2 * slot_count - 1))
+ return true;
+
+ return false;
+}
+
+/*
+ * mab_no_null_split() - ensure the split doesn't fall on a NULL
+ * @b_node: the maple_big_node with the data
+ * @split: the suggested split location
+ * @slot_count: the number of slots in the node being considered.
+ *
+ * Return: the split location.
+ */
+static inline int mab_no_null_split(struct maple_big_node *b_node,
+ unsigned char split, unsigned char slot_count)
+{
+ if (!b_node->slot[split]) {
+ /*
+ * If the split is less than the max slot && the right side will
+ * still be sufficient, then increment the split on NULL.
+ */
+ if ((split < slot_count - 1) &&
+ (b_node->b_end - split) > (mt_min_slots[b_node->type]))
+ split++;
+ else
+ split--;
+ }
+ return split;
+}
+
+/*
+ * mab_calc_split() - Calculate the split location and if there needs to be two
+ * splits.
+ * @bn: The maple_big_node with the data
+ * @mid_split: The second split, if required. 0 otherwise.
+ *
+ * Return: The first split location. The middle split is set in @mid_split.
+ */
+static inline int mab_calc_split(struct ma_state *mas,
+ struct maple_big_node *bn, unsigned char *mid_split, unsigned long min)
+{
+ unsigned char b_end = bn->b_end;
+ int split = b_end / 2; /* Assume equal split. */
+ unsigned char slot_min, slot_count = mt_slots[bn->type];
+
+ /*
+ * To support gap tracking, all NULL entries are kept together and a node cannot
+ * end on a NULL entry, with the exception of the left-most leaf. The
+ * limitation means that the split of a node must be checked for this condition
+ * and be able to put more data in one direction or the other.
+ */
+ if (unlikely((mas->mas_flags & MA_STATE_BULK))) {
+ *mid_split = 0;
+ split = b_end - mt_min_slots[bn->type];
+
+ if (!ma_is_leaf(bn->type))
+ return split;
+
+ mas->mas_flags |= MA_STATE_REBALANCE;
+ if (!bn->slot[split])
+ split--;
+ return split;
+ }
+
+ /*
+ * Although extremely rare, it is possible to enter what is known as the 3-way
+ * split scenario. The 3-way split comes about by means of a store of a range
+ * that overwrites the end and beginning of two full nodes. The result is a set
+ * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can
+ * also be located in different parent nodes which are also full. This can
+ * carry upwards all the way to the root in the worst case.
+ */
+ if (unlikely(mab_middle_node(bn, split, slot_count))) {
+ split = b_end / 3;
+ *mid_split = split * 2;
+ } else {
+ slot_min = mt_min_slots[bn->type];
+
+ *mid_split = 0;
+ /*
+ * Avoid having a range less than the slot count unless it
+ * causes one node to be deficient.
+ * NOTE: mt_min_slots is 1 based, b_end and split are zero.
+ */
+ while (((bn->pivot[split] - min) < slot_count - 1) &&
+ (split < slot_count - 1) && (b_end - split > slot_min))
+ split++;
+ }
+
+ /* Avoid ending a node on a NULL entry */
+ split = mab_no_null_split(bn, split, slot_count);
+ if (!(*mid_split))
+ return split;
+
+ *mid_split = mab_no_null_split(bn, *mid_split, slot_count);
+
+ return split;
+}
+
+/*
+ * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node
+ * and set @b_node->b_end to the next free slot.
+ * @mas: The maple state
+ * @mas_start: The starting slot to copy
+ * @mas_end: The end slot to copy (inclusively)
+ * @b_node: The maple_big_node to place the data
+ * @mab_start: The starting location in maple_big_node to store the data.
+ */
+static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start,
+ unsigned char mas_end, struct maple_big_node *b_node,
+ unsigned char mab_start)
+{
+ enum maple_type mt;
+ struct maple_node *node;
+ void __rcu **slots;
+ unsigned long *pivots, *gaps;
+ int i = mas_start, j = mab_start;
+ unsigned char piv_end;
+
+ node = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ pivots = ma_pivots(node, mt);
+ if (!i) {
+ b_node->pivot[j] = pivots[i++];
+ if (unlikely(i > mas_end))
+ goto complete;
+ j++;
+ }
+
+ piv_end = min(mas_end, mt_pivots[mt]);
+ for (; i < piv_end; i++, j++) {
+ b_node->pivot[j] = pivots[i];
+ if (unlikely(!b_node->pivot[j]))
+ break;
+
+ if (unlikely(mas->max == b_node->pivot[j]))
+ goto complete;
+ }
+
+ if (likely(i <= mas_end))
+ b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt);
+
+complete:
+ b_node->b_end = ++j;
+ j -= mab_start;
+ slots = ma_slots(node, mt);
+ memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j);
+ if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) {
+ gaps = ma_gaps(node, mt);
+ memcpy(b_node->gap + mab_start, gaps + mas_start,
+ sizeof(unsigned long) * j);
+ }
+}
+
+/*
+ * mas_leaf_set_meta() - Set the metadata of a leaf if possible.
+ * @mas: The maple state
+ * @node: The maple node
+ * @pivots: pointer to the maple node pivots
+ * @mt: The maple type
+ * @end: The assumed end
+ *
+ * Note, end may be incremented within this function but not modified at the
+ * source. This is fine since the metadata is the last thing to be stored in a
+ * node during a write.
+ */
+static inline void mas_leaf_set_meta(struct ma_state *mas,
+ struct maple_node *node, unsigned long *pivots,
+ enum maple_type mt, unsigned char end)
+{
+ /* There is no room for metadata already */
+ if (mt_pivots[mt] <= end)
+ return;
+
+ if (pivots[end] && pivots[end] < mas->max)
+ end++;
+
+ if (end < mt_slots[mt] - 1)
+ ma_set_meta(node, mt, 0, end);
+}
+
+/*
+ * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node.
+ * @b_node: the maple_big_node that has the data
+ * @mab_start: the start location in @b_node.
+ * @mab_end: The end location in @b_node (inclusively)
+ * @mas: The maple state with the maple encoded node.
+ */
+static inline void mab_mas_cp(struct maple_big_node *b_node,
+ unsigned char mab_start, unsigned char mab_end,
+ struct ma_state *mas, bool new_max)
+{
+ int i, j = 0;
+ enum maple_type mt = mte_node_type(mas->node);
+ struct maple_node *node = mte_to_node(mas->node);
+ void __rcu **slots = ma_slots(node, mt);
+ unsigned long *pivots = ma_pivots(node, mt);
+ unsigned long *gaps = NULL;
+ unsigned char end;
+
+ if (mab_end - mab_start > mt_pivots[mt])
+ mab_end--;
+
+ if (!pivots[mt_pivots[mt] - 1])
+ slots[mt_pivots[mt]] = NULL;
+
+ i = mab_start;
+ do {
+ pivots[j++] = b_node->pivot[i++];
+ } while (i <= mab_end && likely(b_node->pivot[i]));
+
+ memcpy(slots, b_node->slot + mab_start,
+ sizeof(void *) * (i - mab_start));
+
+ if (new_max)
+ mas->max = b_node->pivot[i - 1];
+
+ end = j - 1;
+ if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) {
+ unsigned long max_gap = 0;
+ unsigned char offset = 15;
+
+ gaps = ma_gaps(node, mt);
+ do {
+ gaps[--j] = b_node->gap[--i];
+ if (gaps[j] > max_gap) {
+ offset = j;
+ max_gap = gaps[j];
+ }
+ } while (j);
+
+ ma_set_meta(node, mt, offset, end);
+ } else {
+ mas_leaf_set_meta(mas, node, pivots, mt, end);
+ }
+}
+
+/*
+ * mas_descend_adopt() - Descend through a sub-tree and adopt children.
+ * @mas: the maple state with the maple encoded node of the sub-tree.
+ *
+ * Descend through a sub-tree and adopt children who do not have the correct
+ * parents set. Follow the parents which have the correct parents as they are
+ * the new entries which need to be followed to find other incorrectly set
+ * parents.
+ */
+static inline void mas_descend_adopt(struct ma_state *mas)
+{
+ struct ma_state list[3], next[3];
+ int i, n;
+
+ /*
+ * At each level there may be up to 3 correct parent pointers which indicates
+ * the new nodes which need to be walked to find any new nodes at a lower level.
+ */
+
+ for (i = 0; i < 3; i++) {
+ list[i] = *mas;
+ list[i].offset = 0;
+ next[i].offset = 0;
+ }
+ next[0] = *mas;
+
+ while (!mte_is_leaf(list[0].node)) {
+ n = 0;
+ for (i = 0; i < 3; i++) {
+ if (mas_is_none(&list[i]))
+ continue;
+
+ if (i && list[i-1].node == list[i].node)
+ continue;
+
+ while ((n < 3) && (mas_new_child(&list[i], &next[n])))
+ n++;
+
+ mas_adopt_children(&list[i], list[i].node);
+ }
+
+ while (n < 3)
+ next[n++].node = MAS_NONE;
+
+ /* descend by setting the list to the children */
+ for (i = 0; i < 3; i++)
+ list[i] = next[i];
+ }
+}
+
+/*
+ * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert.
+ * @mas: The maple state
+ * @end: The maple node end
+ * @mt: The maple node type
+ */
+static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end,
+ enum maple_type mt)
+{
+ if (!(mas->mas_flags & MA_STATE_BULK))
+ return;
+
+ if (mte_is_root(mas->node))
+ return;
+
+ if (end > mt_min_slots[mt]) {
+ mas->mas_flags &= ~MA_STATE_REBALANCE;
+ return;
+ }
+}
+
+/*
+ * mas_store_b_node() - Store an @entry into the b_node while also copying the
+ * data from a maple encoded node.
+ * @wr_mas: the maple write state
+ * @b_node: the maple_big_node to fill with data
+ * @offset_end: the offset to end copying
+ *
+ * Return: The actual end of the data stored in @b_node
+ */
+static inline void mas_store_b_node(struct ma_wr_state *wr_mas,
+ struct maple_big_node *b_node, unsigned char offset_end)
+{
+ unsigned char slot;
+ unsigned char b_end;
+ /* Possible underflow of piv will wrap back to 0 before use. */
+ unsigned long piv;
+ struct ma_state *mas = wr_mas->mas;
+
+ b_node->type = wr_mas->type;
+ b_end = 0;
+ slot = mas->offset;
+ if (slot) {
+ /* Copy start data up to insert. */
+ mas_mab_cp(mas, 0, slot - 1, b_node, 0);
+ b_end = b_node->b_end;
+ piv = b_node->pivot[b_end - 1];
+ } else
+ piv = mas->min - 1;
+
+ if (piv + 1 < mas->index) {
+ /* Handle range starting after old range */
+ b_node->slot[b_end] = wr_mas->content;
+ if (!wr_mas->content)
+ b_node->gap[b_end] = mas->index - 1 - piv;
+ b_node->pivot[b_end++] = mas->index - 1;
+ }
+
+ /* Store the new entry. */
+ mas->offset = b_end;
+ b_node->slot[b_end] = wr_mas->entry;
+ b_node->pivot[b_end] = mas->last;
+
+ /* Appended. */
+ if (mas->last >= mas->max)
+ goto b_end;
+
+ /* Handle new range ending before old range ends */
+ piv = mas_logical_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type);
+ if (piv > mas->last) {
+ if (piv == ULONG_MAX)
+ mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type);
+
+ if (offset_end != slot)
+ wr_mas->content = mas_slot_locked(mas, wr_mas->slots,
+ offset_end);
+
+ b_node->slot[++b_end] = wr_mas->content;
+ if (!wr_mas->content)
+ b_node->gap[b_end] = piv - mas->last + 1;
+ b_node->pivot[b_end] = piv;
+ }
+
+ slot = offset_end + 1;
+ if (slot > wr_mas->node_end)
+ goto b_end;
+
+ /* Copy end data to the end of the node. */
+ mas_mab_cp(mas, slot, wr_mas->node_end + 1, b_node, ++b_end);
+ b_node->b_end--;
+ return;
+
+b_end:
+ b_node->b_end = b_end;
+}
+
+/*
+ * mas_prev_sibling() - Find the previous node with the same parent.
+ * @mas: the maple state
+ *
+ * Return: True if there is a previous sibling, false otherwise.
+ */
+static inline bool mas_prev_sibling(struct ma_state *mas)
+{
+ unsigned int p_slot = mte_parent_slot(mas->node);
+
+ if (mte_is_root(mas->node))
+ return false;
+
+ if (!p_slot)
+ return false;
+
+ mas_ascend(mas);
+ mas->offset = p_slot - 1;
+ mas_descend(mas);
+ return true;
+}
+
+/*
+ * mas_next_sibling() - Find the next node with the same parent.
+ * @mas: the maple state
+ *
+ * Return: true if there is a next sibling, false otherwise.
+ */
+static inline bool mas_next_sibling(struct ma_state *mas)
+{
+ MA_STATE(parent, mas->tree, mas->index, mas->last);
+
+ if (mte_is_root(mas->node))
+ return false;
+
+ parent = *mas;
+ mas_ascend(&parent);
+ parent.offset = mte_parent_slot(mas->node) + 1;
+ if (parent.offset > mas_data_end(&parent))
+ return false;
+
+ *mas = parent;
+ mas_descend(mas);
+ return true;
+}
+
+/*
+ * mte_node_or_node() - Return the encoded node or MAS_NONE.
+ * @enode: The encoded maple node.
+ *
+ * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state.
+ *
+ * Return: @enode or MAS_NONE
+ */
+static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode)
+{
+ if (enode)
+ return enode;
+
+ return ma_enode_ptr(MAS_NONE);
+}
+
+/*
+ * mas_wr_node_walk() - Find the correct offset for the index in the @mas.
+ * @wr_mas: The maple write state
+ *
+ * Uses mas_slot_locked() and does not need to worry about dead nodes.
+ */
+static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+ unsigned char count;
+ unsigned char offset;
+ unsigned long index, min, max;
+
+ if (unlikely(ma_is_dense(wr_mas->type))) {
+ wr_mas->r_max = wr_mas->r_min = mas->index;
+ mas->offset = mas->index = mas->min;
+ return;
+ }
+
+ wr_mas->node = mas_mn(wr_mas->mas);
+ wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type);
+ count = wr_mas->node_end = ma_data_end(wr_mas->node, wr_mas->type,
+ wr_mas->pivots, mas->max);
+ offset = mas->offset;
+ min = mas_safe_min(mas, wr_mas->pivots, offset);
+ if (unlikely(offset == count))
+ goto max;
+
+ max = wr_mas->pivots[offset];
+ index = mas->index;
+ if (unlikely(index <= max))
+ goto done;
+
+ if (unlikely(!max && offset))
+ goto max;
+
+ min = max + 1;
+ while (++offset < count) {
+ max = wr_mas->pivots[offset];
+ if (index <= max)
+ goto done;
+ else if (unlikely(!max))
+ break;
+
+ min = max + 1;
+ }
+
+max:
+ max = mas->max;
+done:
+ wr_mas->r_max = max;
+ wr_mas->r_min = min;
+ wr_mas->offset_end = mas->offset = offset;
+}
+
+/*
+ * mas_topiary_range() - Add a range of slots to the topiary.
+ * @mas: The maple state
+ * @destroy: The topiary to add the slots (usually destroy)
+ * @start: The starting slot inclusively
+ * @end: The end slot inclusively
+ */
+static inline void mas_topiary_range(struct ma_state *mas,
+ struct ma_topiary *destroy, unsigned char start, unsigned char end)
+{
+ void __rcu **slots;
+ unsigned char offset;
+
+ MT_BUG_ON(mas->tree, mte_is_leaf(mas->node));
+ slots = ma_slots(mas_mn(mas), mte_node_type(mas->node));
+ for (offset = start; offset <= end; offset++) {
+ struct maple_enode *enode = mas_slot_locked(mas, slots, offset);
+
+ if (mte_dead_node(enode))
+ continue;
+
+ mat_add(destroy, enode);
+ }
+}
+
+/*
+ * mast_topiary() - Add the portions of the tree to the removal list; either to
+ * be freed or discarded (destroy walk).
+ * @mast: The maple_subtree_state.
+ */
+static inline void mast_topiary(struct maple_subtree_state *mast)
+{
+ MA_WR_STATE(wr_mas, mast->orig_l, NULL);
+ unsigned char r_start, r_end;
+ unsigned char l_start, l_end;
+ void __rcu **l_slots, **r_slots;
+
+ wr_mas.type = mte_node_type(mast->orig_l->node);
+ mast->orig_l->index = mast->orig_l->last;
+ mas_wr_node_walk(&wr_mas);
+ l_start = mast->orig_l->offset + 1;
+ l_end = mas_data_end(mast->orig_l);
+ r_start = 0;
+ r_end = mast->orig_r->offset;
+
+ if (r_end)
+ r_end--;
+
+ l_slots = ma_slots(mas_mn(mast->orig_l),
+ mte_node_type(mast->orig_l->node));
+
+ r_slots = ma_slots(mas_mn(mast->orig_r),
+ mte_node_type(mast->orig_r->node));
+
+ if ((l_start < l_end) &&
+ mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_start))) {
+ l_start++;
+ }
+
+ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_end))) {
+ if (r_end)
+ r_end--;
+ }
+
+ if ((l_start > r_end) && (mast->orig_l->node == mast->orig_r->node))
+ return;
+
+ /* At the node where left and right sides meet, add the parts between */
+ if (mast->orig_l->node == mast->orig_r->node) {
+ return mas_topiary_range(mast->orig_l, mast->destroy,
+ l_start, r_end);
+ }
+
+ /* mast->orig_r is different and consumed. */
+ if (mte_is_leaf(mast->orig_r->node))
+ return;
+
+ if (mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_end)))
+ l_end--;
+
+
+ if (l_start <= l_end)
+ mas_topiary_range(mast->orig_l, mast->destroy, l_start, l_end);
+
+ if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_start)))
+ r_start++;
+
+ if (r_start <= r_end)
+ mas_topiary_range(mast->orig_r, mast->destroy, 0, r_end);
+}
+
+/*
+ * mast_rebalance_next() - Rebalance against the next node
+ * @mast: The maple subtree state
+ * @old_r: The encoded maple node to the right (next node).
+ */
+static inline void mast_rebalance_next(struct maple_subtree_state *mast)
+{
+ unsigned char b_end = mast->bn->b_end;
+
+ mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node),
+ mast->bn, b_end);
+ mast->orig_r->last = mast->orig_r->max;
+}
+
+/*
+ * mast_rebalance_prev() - Rebalance against the previous node
+ * @mast: The maple subtree state
+ * @old_l: The encoded maple node to the left (previous node)
+ */
+static inline void mast_rebalance_prev(struct maple_subtree_state *mast)
+{
+ unsigned char end = mas_data_end(mast->orig_l) + 1;
+ unsigned char b_end = mast->bn->b_end;
+
+ mab_shift_right(mast->bn, end);
+ mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0);
+ mast->l->min = mast->orig_l->min;
+ mast->orig_l->index = mast->orig_l->min;
+ mast->bn->b_end = end + b_end;
+ mast->l->offset += end;
+}
+
+/*
+ * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring
+ * the node to the right. Checking the nodes to the right then the left at each
+ * level upwards until root is reached. Free and destroy as needed.
+ * Data is copied into the @mast->bn.
+ * @mast: The maple_subtree_state.
+ */
+static inline
+bool mast_spanning_rebalance(struct maple_subtree_state *mast)
+{
+ struct ma_state r_tmp = *mast->orig_r;
+ struct ma_state l_tmp = *mast->orig_l;
+ struct maple_enode *ancestor = NULL;
+ unsigned char start, end;
+ unsigned char depth = 0;
+
+ r_tmp = *mast->orig_r;
+ l_tmp = *mast->orig_l;
+ do {
+ mas_ascend(mast->orig_r);
+ mas_ascend(mast->orig_l);
+ depth++;
+ if (!ancestor &&
+ (mast->orig_r->node == mast->orig_l->node)) {
+ ancestor = mast->orig_r->node;
+ end = mast->orig_r->offset - 1;
+ start = mast->orig_l->offset + 1;
+ }
+
+ if (mast->orig_r->offset < mas_data_end(mast->orig_r)) {
+ if (!ancestor) {
+ ancestor = mast->orig_r->node;
+ start = 0;
+ }
+
+ mast->orig_r->offset++;
+ do {
+ mas_descend(mast->orig_r);
+ mast->orig_r->offset = 0;
+ depth--;
+ } while (depth);
+
+ mast_rebalance_next(mast);
+ do {
+ unsigned char l_off = 0;
+ struct maple_enode *child = r_tmp.node;
+
+ mas_ascend(&r_tmp);
+ if (ancestor == r_tmp.node)
+ l_off = start;
+
+ if (r_tmp.offset)
+ r_tmp.offset--;
+
+ if (l_off < r_tmp.offset)
+ mas_topiary_range(&r_tmp, mast->destroy,
+ l_off, r_tmp.offset);
+
+ if (l_tmp.node != child)
+ mat_add(mast->free, child);
+
+ } while (r_tmp.node != ancestor);
+
+ *mast->orig_l = l_tmp;
+ return true;
+
+ } else if (mast->orig_l->offset != 0) {
+ if (!ancestor) {
+ ancestor = mast->orig_l->node;
+ end = mas_data_end(mast->orig_l);
+ }
+
+ mast->orig_l->offset--;
+ do {
+ mas_descend(mast->orig_l);
+ mast->orig_l->offset =
+ mas_data_end(mast->orig_l);
+ depth--;
+ } while (depth);
+
+ mast_rebalance_prev(mast);
+ do {
+ unsigned char r_off;
+ struct maple_enode *child = l_tmp.node;
+
+ mas_ascend(&l_tmp);
+ if (ancestor == l_tmp.node)
+ r_off = end;
+ else
+ r_off = mas_data_end(&l_tmp);
+
+ if (l_tmp.offset < r_off)
+ l_tmp.offset++;
+
+ if (l_tmp.offset < r_off)
+ mas_topiary_range(&l_tmp, mast->destroy,
+ l_tmp.offset, r_off);
+
+ if (r_tmp.node != child)
+ mat_add(mast->free, child);
+
+ } while (l_tmp.node != ancestor);
+
+ *mast->orig_r = r_tmp;
+ return true;
+ }
+ } while (!mte_is_root(mast->orig_r->node));
+
+ *mast->orig_r = r_tmp;
+ *mast->orig_l = l_tmp;
+ return false;
+}
+
+/*
+ * mast_ascend_free() - Add current original maple state nodes to the free list
+ * and ascend.
+ * @mast: the maple subtree state.
+ *
+ * Ascend the original left and right sides and add the previous nodes to the
+ * free list. Set the slots to point to the correct location in the new nodes.
+ */
+static inline void
+mast_ascend_free(struct maple_subtree_state *mast)
+{
+ MA_WR_STATE(wr_mas, mast->orig_r, NULL);
+ struct maple_enode *left = mast->orig_l->node;
+ struct maple_enode *right = mast->orig_r->node;
+
+ mas_ascend(mast->orig_l);
+ mas_ascend(mast->orig_r);
+ mat_add(mast->free, left);
+
+ if (left != right)
+ mat_add(mast->free, right);
+
+ mast->orig_r->offset = 0;
+ mast->orig_r->index = mast->r->max;
+ /* last should be larger than or equal to index */
+ if (mast->orig_r->last < mast->orig_r->index)
+ mast->orig_r->last = mast->orig_r->index;
+ /*
+ * The node may not contain the value so set slot to ensure all
+ * of the nodes contents are freed or destroyed.
+ */
+ wr_mas.type = mte_node_type(mast->orig_r->node);
+ mas_wr_node_walk(&wr_mas);
+ /* Set up the left side of things */
+ mast->orig_l->offset = 0;
+ mast->orig_l->index = mast->l->min;
+ wr_mas.mas = mast->orig_l;
+ wr_mas.type = mte_node_type(mast->orig_l->node);
+ mas_wr_node_walk(&wr_mas);
+
+ mast->bn->type = wr_mas.type;
+}
+
+/*
+ * mas_new_ma_node() - Create and return a new maple node. Helper function.
+ * @mas: the maple state with the allocations.
+ * @b_node: the maple_big_node with the type encoding.
+ *
+ * Use the node type from the maple_big_node to allocate a new node from the
+ * ma_state. This function exists mainly for code readability.
+ *
+ * Return: A new maple encoded node
+ */
+static inline struct maple_enode
+*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node)
+{
+ return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type);
+}
+
+/*
+ * mas_mab_to_node() - Set up right and middle nodes
+ *
+ * @mas: the maple state that contains the allocations.
+ * @b_node: the node which contains the data.
+ * @left: The pointer which will have the left node
+ * @right: The pointer which may have the right node
+ * @middle: the pointer which may have the middle node (rare)
+ * @mid_split: the split location for the middle node
+ *
+ * Return: the split of left.
+ */
+static inline unsigned char mas_mab_to_node(struct ma_state *mas,
+ struct maple_big_node *b_node, struct maple_enode **left,
+ struct maple_enode **right, struct maple_enode **middle,
+ unsigned char *mid_split, unsigned long min)
+{
+ unsigned char split = 0;
+ unsigned char slot_count = mt_slots[b_node->type];
+
+ *left = mas_new_ma_node(mas, b_node);
+ *right = NULL;
+ *middle = NULL;
+ *mid_split = 0;
+
+ if (b_node->b_end < slot_count) {
+ split = b_node->b_end;
+ } else {
+ split = mab_calc_split(mas, b_node, mid_split, min);
+ *right = mas_new_ma_node(mas, b_node);
+ }
+
+ if (*mid_split)
+ *middle = mas_new_ma_node(mas, b_node);
+
+ return split;
+
+}
+
+/*
+ * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end
+ * pointer.
+ * @b_node - the big node to add the entry
+ * @mas - the maple state to get the pivot (mas->max)
+ * @entry - the entry to add, if NULL nothing happens.
+ */
+static inline void mab_set_b_end(struct maple_big_node *b_node,
+ struct ma_state *mas,
+ void *entry)
+{
+ if (!entry)
+ return;
+
+ b_node->slot[b_node->b_end] = entry;
+ if (mt_is_alloc(mas->tree))
+ b_node->gap[b_node->b_end] = mas_max_gap(mas);
+ b_node->pivot[b_node->b_end++] = mas->max;
+}
+
+/*
+ * mas_set_split_parent() - combine_then_separate helper function. Sets the parent
+ * of @mas->node to either @left or @right, depending on @slot and @split
+ *
+ * @mas - the maple state with the node that needs a parent
+ * @left - possible parent 1
+ * @right - possible parent 2
+ * @slot - the slot the mas->node was placed
+ * @split - the split location between @left and @right
+ */
+static inline void mas_set_split_parent(struct ma_state *mas,
+ struct maple_enode *left,
+ struct maple_enode *right,
+ unsigned char *slot, unsigned char split)
+{
+ if (mas_is_none(mas))
+ return;
+
+ if ((*slot) <= split)
+ mte_set_parent(mas->node, left, *slot);
+ else if (right)
+ mte_set_parent(mas->node, right, (*slot) - split - 1);
+
+ (*slot)++;
+}
+
+/*
+ * mte_mid_split_check() - Check if the next node passes the mid-split
+ * @**l: Pointer to left encoded maple node.
+ * @**m: Pointer to middle encoded maple node.
+ * @**r: Pointer to right encoded maple node.
+ * @slot: The offset
+ * @*split: The split location.
+ * @mid_split: The middle split.
+ */
+static inline void mte_mid_split_check(struct maple_enode **l,
+ struct maple_enode **r,
+ struct maple_enode *right,
+ unsigned char slot,
+ unsigned char *split,
+ unsigned char mid_split)
+{
+ if (*r == right)
+ return;
+
+ if (slot < mid_split)
+ return;
+
+ *l = *r;
+ *r = right;
+ *split = mid_split;
+}
+
+/*
+ * mast_set_split_parents() - Helper function to set three nodes parents. Slot
+ * is taken from @mast->l.
+ * @mast - the maple subtree state
+ * @left - the left node
+ * @right - the right node
+ * @split - the split location.
+ */
+static inline void mast_set_split_parents(struct maple_subtree_state *mast,
+ struct maple_enode *left,
+ struct maple_enode *middle,
+ struct maple_enode *right,
+ unsigned char split,
+ unsigned char mid_split)
+{
+ unsigned char slot;
+ struct maple_enode *l = left;
+ struct maple_enode *r = right;
+
+ if (mas_is_none(mast->l))
+ return;
+
+ if (middle)
+ r = middle;
+
+ slot = mast->l->offset;
+
+ mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
+ mas_set_split_parent(mast->l, l, r, &slot, split);
+
+ mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
+ mas_set_split_parent(mast->m, l, r, &slot, split);
+
+ mte_mid_split_check(&l, &r, right, slot, &split, mid_split);
+ mas_set_split_parent(mast->r, l, r, &slot, split);
+}
+
+/*
+ * mas_wmb_replace() - Write memory barrier and replace
+ * @mas: The maple state
+ * @free: the maple topiary list of nodes to free
+ * @destroy: The maple topiary list of nodes to destroy (walk and free)
+ *
+ * Updates gap as necessary.
+ */
+static inline void mas_wmb_replace(struct ma_state *mas,
+ struct ma_topiary *free,
+ struct ma_topiary *destroy)
+{
+ /* All nodes must see old data as dead prior to replacing that data */
+ smp_wmb(); /* Needed for RCU */
+
+ /* Insert the new data in the tree */
+ mas_replace(mas, true);
+
+ if (!mte_is_leaf(mas->node))
+ mas_descend_adopt(mas);
+
+ mas_mat_free(mas, free);
+
+ if (destroy)
+ mas_mat_destroy(mas, destroy);
+
+ if (mte_is_leaf(mas->node))
+ return;
+
+ mas_update_gap(mas);
+}
+
+/*
+ * mast_new_root() - Set a new tree root during subtree creation
+ * @mast: The maple subtree state
+ * @mas: The maple state
+ */
+static inline void mast_new_root(struct maple_subtree_state *mast,
+ struct ma_state *mas)
+{
+ mas_mn(mast->l)->parent =
+ ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT));
+ if (!mte_dead_node(mast->orig_l->node) &&
+ !mte_is_root(mast->orig_l->node)) {
+ do {
+ mast_ascend_free(mast);
+ mast_topiary(mast);
+ } while (!mte_is_root(mast->orig_l->node));
+ }
+ if ((mast->orig_l->node != mas->node) &&
+ (mast->l->depth > mas_mt_height(mas))) {
+ mat_add(mast->free, mas->node);
+ }
+}
+
+/*
+ * mast_cp_to_nodes() - Copy data out to nodes.
+ * @mast: The maple subtree state
+ * @left: The left encoded maple node
+ * @middle: The middle encoded maple node
+ * @right: The right encoded maple node
+ * @split: The location to split between left and (middle ? middle : right)
+ * @mid_split: The location to split between middle and right.
+ */
+static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
+ struct maple_enode *left, struct maple_enode *middle,
+ struct maple_enode *right, unsigned char split, unsigned char mid_split)
+{
+ bool new_lmax = true;
+
+ mast->l->node = mte_node_or_none(left);
+ mast->m->node = mte_node_or_none(middle);
+ mast->r->node = mte_node_or_none(right);
+
+ mast->l->min = mast->orig_l->min;
+ if (split == mast->bn->b_end) {
+ mast->l->max = mast->orig_r->max;
+ new_lmax = false;
+ }
+
+ mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax);
+
+ if (middle) {
+ mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true);
+ mast->m->min = mast->bn->pivot[split] + 1;
+ split = mid_split;
+ }
+
+ mast->r->max = mast->orig_r->max;
+ if (right) {
+ mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false);
+ mast->r->min = mast->bn->pivot[split] + 1;
+ }
+}
+
+/*
+ * mast_combine_cp_left - Copy in the original left side of the tree into the
+ * combined data set in the maple subtree state big node.
+ * @mast: The maple subtree state
+ */
+static inline void mast_combine_cp_left(struct maple_subtree_state *mast)
+{
+ unsigned char l_slot = mast->orig_l->offset;
+
+ if (!l_slot)
+ return;
+
+ mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0);
+}
+
+/*
+ * mast_combine_cp_right: Copy in the original right side of the tree into the
+ * combined data set in the maple subtree state big node.
+ * @mast: The maple subtree state
+ */
+static inline void mast_combine_cp_right(struct maple_subtree_state *mast)
+{
+ if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max)
+ return;
+
+ mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1,
+ mt_slot_count(mast->orig_r->node), mast->bn,
+ mast->bn->b_end);
+ mast->orig_r->last = mast->orig_r->max;
+}
+
+/*
+ * mast_sufficient: Check if the maple subtree state has enough data in the big
+ * node to create at least one sufficient node
+ * @mast: the maple subtree state
+ */
+static inline bool mast_sufficient(struct maple_subtree_state *mast)
+{
+ if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node))
+ return true;
+
+ return false;
+}
+
+/*
+ * mast_overflow: Check if there is too much data in the subtree state for a
+ * single node.
+ * @mast: The maple subtree state
+ */
+static inline bool mast_overflow(struct maple_subtree_state *mast)
+{
+ if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node))
+ return true;
+
+ return false;
+}
+
+static inline void *mtree_range_walk(struct ma_state *mas)
+{
+ unsigned long *pivots;
+ unsigned char offset;
+ struct maple_node *node;
+ struct maple_enode *next, *last;
+ enum maple_type type;
+ void __rcu **slots;
+ unsigned char end;
+ unsigned long max, min;
+ unsigned long prev_max, prev_min;
+
+ last = next = mas->node;
+ prev_min = min = mas->min;
+ max = mas->max;
+ do {
+ offset = 0;
+ last = next;
+ node = mte_to_node(next);
+ type = mte_node_type(next);
+ pivots = ma_pivots(node, type);
+ end = ma_data_end(node, type, pivots, max);
+ if (unlikely(ma_dead_node(node)))
+ goto dead_node;
+
+ if (pivots[offset] >= mas->index) {
+ prev_max = max;
+ prev_min = min;
+ max = pivots[offset];
+ goto next;
+ }
+
+ do {
+ offset++;
+ } while ((offset < end) && (pivots[offset] < mas->index));
+
+ prev_min = min;
+ min = pivots[offset - 1] + 1;
+ prev_max = max;
+ if (likely(offset < end && pivots[offset]))
+ max = pivots[offset];
+
+next:
+ slots = ma_slots(node, type);
+ next = mt_slot(mas->tree, slots, offset);
+ if (unlikely(ma_dead_node(node)))
+ goto dead_node;
+ } while (!ma_is_leaf(type));
+
+ mas->offset = offset;
+ mas->index = min;
+ mas->last = max;
+ mas->min = prev_min;
+ mas->max = prev_max;
+ mas->node = last;
+ return (void *) next;
+
+dead_node:
+ mas_reset(mas);
+ return NULL;
+}
+
+/*
+ * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers.
+ * @mas: The starting maple state
+ * @mast: The maple_subtree_state, keeps track of 4 maple states.
+ * @count: The estimated count of iterations needed.
+ *
+ * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root
+ * is hit. First @b_node is split into two entries which are inserted into the
+ * next iteration of the loop. @b_node is returned populated with the final
+ * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the
+ * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last
+ * to account of what has been copied into the new sub-tree. The update of
+ * orig_l_mas->last is used in mas_consume to find the slots that will need to
+ * be either freed or destroyed. orig_l_mas->depth keeps track of the height of
+ * the new sub-tree in case the sub-tree becomes the full tree.
+ *
+ * Return: the number of elements in b_node during the last loop.
+ */
+static int mas_spanning_rebalance(struct ma_state *mas,
+ struct maple_subtree_state *mast, unsigned char count)
+{
+ unsigned char split, mid_split;
+ unsigned char slot = 0;
+ struct maple_enode *left = NULL, *middle = NULL, *right = NULL;
+
+ MA_STATE(l_mas, mas->tree, mas->index, mas->index);
+ MA_STATE(r_mas, mas->tree, mas->index, mas->last);
+ MA_STATE(m_mas, mas->tree, mas->index, mas->index);
+ MA_TOPIARY(free, mas->tree);
+ MA_TOPIARY(destroy, mas->tree);
+
+ /*
+ * The tree needs to be rebalanced and leaves need to be kept at the same level.
+ * Rebalancing is done by use of the ``struct maple_topiary``.
+ */
+ mast->l = &l_mas;
+ mast->m = &m_mas;
+ mast->r = &r_mas;
+ mast->free = &free;
+ mast->destroy = &destroy;
+ l_mas.node = r_mas.node = m_mas.node = MAS_NONE;
+ if (!(mast->orig_l->min && mast->orig_r->max == ULONG_MAX) &&
+ unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type]))
+ mast_spanning_rebalance(mast);
+
+ mast->orig_l->depth = 0;
+
+ /*
+ * Each level of the tree is examined and balanced, pushing data to the left or
+ * right, or rebalancing against left or right nodes is employed to avoid
+ * rippling up the tree to limit the amount of churn. Once a new sub-section of
+ * the tree is created, there may be a mix of new and old nodes. The old nodes
+ * will have the incorrect parent pointers and currently be in two trees: the
+ * original tree and the partially new tree. To remedy the parent pointers in
+ * the old tree, the new data is swapped into the active tree and a walk down
+ * the tree is performed and the parent pointers are updated.
+ * See mas_descend_adopt() for more information..
+ */
+ while (count--) {
+ mast->bn->b_end--;
+ mast->bn->type = mte_node_type(mast->orig_l->node);
+ split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle,
+ &mid_split, mast->orig_l->min);
+ mast_set_split_parents(mast, left, middle, right, split,
+ mid_split);
+ mast_cp_to_nodes(mast, left, middle, right, split, mid_split);
+
+ /*
+ * Copy data from next level in the tree to mast->bn from next
+ * iteration
+ */
+ memset(mast->bn, 0, sizeof(struct maple_big_node));
+ mast->bn->type = mte_node_type(left);
+ mast->orig_l->depth++;
+
+ /* Root already stored in l->node. */
+ if (mas_is_root_limits(mast->l))
+ goto new_root;
+
+ mast_ascend_free(mast);
+ mast_combine_cp_left(mast);
+ l_mas.offset = mast->bn->b_end;
+ mab_set_b_end(mast->bn, &l_mas, left);
+ mab_set_b_end(mast->bn, &m_mas, middle);
+ mab_set_b_end(mast->bn, &r_mas, right);
+
+ /* Copy anything necessary out of the right node. */
+ mast_combine_cp_right(mast);
+ mast_topiary(mast);
+ mast->orig_l->last = mast->orig_l->max;
+
+ if (mast_sufficient(mast))
+ continue;
+
+ if (mast_overflow(mast))
+ continue;
+
+ /* May be a new root stored in mast->bn */
+ if (mas_is_root_limits(mast->orig_l))
+ break;
+
+ mast_spanning_rebalance(mast);
+
+ /* rebalancing from other nodes may require another loop. */
+ if (!count)
+ count++;
+ }
+
+ l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)),
+ mte_node_type(mast->orig_l->node));
+ mast->orig_l->depth++;
+ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true);
+ mte_set_parent(left, l_mas.node, slot);
+ if (middle)
+ mte_set_parent(middle, l_mas.node, ++slot);
+
+ if (right)
+ mte_set_parent(right, l_mas.node, ++slot);
+
+ if (mas_is_root_limits(mast->l)) {
+new_root:
+ mast_new_root(mast, mas);
+ } else {
+ mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent;
+ }
+
+ if (!mte_dead_node(mast->orig_l->node))
+ mat_add(&free, mast->orig_l->node);
+
+ mas->depth = mast->orig_l->depth;
+ *mast->orig_l = l_mas;
+ mte_set_node_dead(mas->node);
+
+ /* Set up mas for insertion. */
+ mast->orig_l->depth = mas->depth;
+ mast->orig_l->alloc = mas->alloc;
+ *mas = *mast->orig_l;
+ mas_wmb_replace(mas, &free, &destroy);
+ mtree_range_walk(mas);
+ return mast->bn->b_end;
+}
+
+/*
+ * mas_rebalance() - Rebalance a given node.
+ * @mas: The maple state
+ * @b_node: The big maple node.
+ *
+ * Rebalance two nodes into a single node or two new nodes that are sufficient.
+ * Continue upwards until tree is sufficient.
+ *
+ * Return: the number of elements in b_node during the last loop.
+ */
+static inline int mas_rebalance(struct ma_state *mas,
+ struct maple_big_node *b_node)
+{
+ char empty_count = mas_mt_height(mas);
+ struct maple_subtree_state mast;
+ unsigned char shift, b_end = ++b_node->b_end;
+
+ MA_STATE(l_mas, mas->tree, mas->index, mas->last);
+ MA_STATE(r_mas, mas->tree, mas->index, mas->last);
+
+ trace_ma_op(__func__, mas);
+
+ /*
+ * Rebalancing occurs if a node is insufficient. Data is rebalanced
+ * against the node to the right if it exists, otherwise the node to the
+ * left of this node is rebalanced against this node. If rebalancing
+ * causes just one node to be produced instead of two, then the parent
+ * is also examined and rebalanced if it is insufficient. Every level
+ * tries to combine the data in the same way. If one node contains the
+ * entire range of the tree, then that node is used as a new root node.
+ */
+ mas_node_count(mas, 1 + empty_count * 3);
+ if (mas_is_err(mas))
+ return 0;
+
+ mast.orig_l = &l_mas;
+ mast.orig_r = &r_mas;
+ mast.bn = b_node;
+ mast.bn->type = mte_node_type(mas->node);
+
+ l_mas = r_mas = *mas;
+
+ if (mas_next_sibling(&r_mas)) {
+ mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end);
+ r_mas.last = r_mas.index = r_mas.max;
+ } else {
+ mas_prev_sibling(&l_mas);
+ shift = mas_data_end(&l_mas) + 1;
+ mab_shift_right(b_node, shift);
+ mas->offset += shift;
+ mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0);
+ b_node->b_end = shift + b_end;
+ l_mas.index = l_mas.last = l_mas.min;
+ }
+
+ return mas_spanning_rebalance(mas, &mast, empty_count);
+}
+
+/*
+ * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple
+ * state.
+ * @mas: The maple state
+ * @end: The end of the left-most node.
+ *
+ * During a mass-insert event (such as forking), it may be necessary to
+ * rebalance the left-most node when it is not sufficient.
+ */
+static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end)
+{
+ enum maple_type mt = mte_node_type(mas->node);
+ struct maple_node reuse, *newnode, *parent, *new_left, *left, *node;
+ struct maple_enode *eparent;
+ unsigned char offset, tmp, split = mt_slots[mt] / 2;
+ void __rcu **l_slots, **slots;
+ unsigned long *l_pivs, *pivs, gap;
+ bool in_rcu = mt_in_rcu(mas->tree);
+
+ MA_STATE(l_mas, mas->tree, mas->index, mas->last);
+
+ l_mas = *mas;
+ mas_prev_sibling(&l_mas);
+
+ /* set up node. */
+ if (in_rcu) {
+ /* Allocate for both left and right as well as parent. */
+ mas_node_count(mas, 3);
+ if (mas_is_err(mas))
+ return;
+
+ newnode = mas_pop_node(mas);
+ } else {
+ newnode = &reuse;
+ }
+
+ node = mas_mn(mas);
+ newnode->parent = node->parent;
+ slots = ma_slots(newnode, mt);
+ pivs = ma_pivots(newnode, mt);
+ left = mas_mn(&l_mas);
+ l_slots = ma_slots(left, mt);
+ l_pivs = ma_pivots(left, mt);
+ if (!l_slots[split])
+ split++;
+ tmp = mas_data_end(&l_mas) - split;
+
+ memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp);
+ memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp);
+ pivs[tmp] = l_mas.max;
+ memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end);
+ memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end);
+
+ l_mas.max = l_pivs[split];
+ mas->min = l_mas.max + 1;
+ eparent = mt_mk_node(mte_parent(l_mas.node),
+ mas_parent_enum(&l_mas, l_mas.node));
+ tmp += end;
+ if (!in_rcu) {
+ unsigned char max_p = mt_pivots[mt];
+ unsigned char max_s = mt_slots[mt];
+
+ if (tmp < max_p)
+ memset(pivs + tmp, 0,
+ sizeof(unsigned long *) * (max_p - tmp));
+
+ if (tmp < mt_slots[mt])
+ memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp));
+
+ memcpy(node, newnode, sizeof(struct maple_node));
+ ma_set_meta(node, mt, 0, tmp - 1);
+ mte_set_pivot(eparent, mte_parent_slot(l_mas.node),
+ l_pivs[split]);
+
+ /* Remove data from l_pivs. */
+ tmp = split + 1;
+ memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp));
+ memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp));
+ ma_set_meta(left, mt, 0, split);
+
+ goto done;
+ }
+
+ /* RCU requires replacing both l_mas, mas, and parent. */
+ mas->node = mt_mk_node(newnode, mt);
+ ma_set_meta(newnode, mt, 0, tmp);
+
+ new_left = mas_pop_node(mas);
+ new_left->parent = left->parent;
+ mt = mte_node_type(l_mas.node);
+ slots = ma_slots(new_left, mt);
+ pivs = ma_pivots(new_left, mt);
+ memcpy(slots, l_slots, sizeof(void *) * split);
+ memcpy(pivs, l_pivs, sizeof(unsigned long) * split);
+ ma_set_meta(new_left, mt, 0, split);
+ l_mas.node = mt_mk_node(new_left, mt);
+
+ /* replace parent. */
+ offset = mte_parent_slot(mas->node);
+ mt = mas_parent_enum(&l_mas, l_mas.node);
+ parent = mas_pop_node(mas);
+ slots = ma_slots(parent, mt);
+ pivs = ma_pivots(parent, mt);
+ memcpy(parent, mte_to_node(eparent), sizeof(struct maple_node));
+ rcu_assign_pointer(slots[offset], mas->node);
+ rcu_assign_pointer(slots[offset - 1], l_mas.node);
+ pivs[offset - 1] = l_mas.max;
+ eparent = mt_mk_node(parent, mt);
+done:
+ gap = mas_leaf_max_gap(mas);
+ mte_set_gap(eparent, mte_parent_slot(mas->node), gap);
+ gap = mas_leaf_max_gap(&l_mas);
+ mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap);
+ mas_ascend(mas);
+
+ if (in_rcu)
+ mas_replace(mas, false);
+
+ mas_update_gap(mas);
+}
+
+/*
+ * mas_split_final_node() - Split the final node in a subtree operation.
+ * @mast: the maple subtree state
+ * @mas: The maple state
+ * @height: The height of the tree in case it's a new root.
+ */
+static inline bool mas_split_final_node(struct maple_subtree_state *mast,
+ struct ma_state *mas, int height)
+{
+ struct maple_enode *ancestor;
+
+ if (mte_is_root(mas->node)) {
+ if (mt_is_alloc(mas->tree))
+ mast->bn->type = maple_arange_64;
+ else
+ mast->bn->type = maple_range_64;
+ mas->depth = height;
+ }
+ /*
+ * Only a single node is used here, could be root.
+ * The Big_node data should just fit in a single node.
+ */
+ ancestor = mas_new_ma_node(mas, mast->bn);
+ mte_set_parent(mast->l->node, ancestor, mast->l->offset);
+ mte_set_parent(mast->r->node, ancestor, mast->r->offset);
+ mte_to_node(ancestor)->parent = mas_mn(mas)->parent;
+
+ mast->l->node = ancestor;
+ mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true);
+ mas->offset = mast->bn->b_end - 1;
+ return true;
+}
+
+/*
+ * mast_fill_bnode() - Copy data into the big node in the subtree state
+ * @mast: The maple subtree state
+ * @mas: the maple state
+ * @skip: The number of entries to skip for new nodes insertion.
+ */
+static inline void mast_fill_bnode(struct maple_subtree_state *mast,
+ struct ma_state *mas,
+ unsigned char skip)
+{
+ bool cp = true;
+ struct maple_enode *old = mas->node;
+ unsigned char split;
+
+ memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap));
+ memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot));
+ memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot));
+ mast->bn->b_end = 0;
+
+ if (mte_is_root(mas->node)) {
+ cp = false;
+ } else {
+ mas_ascend(mas);
+ mat_add(mast->free, old);
+ mas->offset = mte_parent_slot(mas->node);
+ }
+
+ if (cp && mast->l->offset)
+ mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0);
+
+ split = mast->bn->b_end;
+ mab_set_b_end(mast->bn, mast->l, mast->l->node);
+ mast->r->offset = mast->bn->b_end;
+ mab_set_b_end(mast->bn, mast->r, mast->r->node);
+ if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max)
+ cp = false;
+
+ if (cp)
+ mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1,
+ mast->bn, mast->bn->b_end);
+
+ mast->bn->b_end--;
+ mast->bn->type = mte_node_type(mas->node);
+}
+
+/*
+ * mast_split_data() - Split the data in the subtree state big node into regular
+ * nodes.
+ * @mast: The maple subtree state
+ * @mas: The maple state
+ * @split: The location to split the big node
+ */
+static inline void mast_split_data(struct maple_subtree_state *mast,
+ struct ma_state *mas, unsigned char split)
+{
+ unsigned char p_slot;
+
+ mab_mas_cp(mast->bn, 0, split, mast->l, true);
+ mte_set_pivot(mast->r->node, 0, mast->r->max);
+ mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false);
+ mast->l->offset = mte_parent_slot(mas->node);
+ mast->l->max = mast->bn->pivot[split];
+ mast->r->min = mast->l->max + 1;
+ if (mte_is_leaf(mas->node))
+ return;
+
+ p_slot = mast->orig_l->offset;
+ mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node,
+ &p_slot, split);
+ mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node,
+ &p_slot, split);
+}
+
+/*
+ * mas_push_data() - Instead of splitting a node, it is beneficial to push the
+ * data to the right or left node if there is room.
+ * @mas: The maple state
+ * @height: The current height of the maple state
+ * @mast: The maple subtree state
+ * @left: Push left or not.
+ *
+ * Keeping the height of the tree low means faster lookups.
+ *
+ * Return: True if pushed, false otherwise.
+ */
+static inline bool mas_push_data(struct ma_state *mas, int height,
+ struct maple_subtree_state *mast, bool left)
+{
+ unsigned char slot_total = mast->bn->b_end;
+ unsigned char end, space, split;
+
+ MA_STATE(tmp_mas, mas->tree, mas->index, mas->last);
+ tmp_mas = *mas;
+ tmp_mas.depth = mast->l->depth;
+
+ if (left && !mas_prev_sibling(&tmp_mas))
+ return false;
+ else if (!left && !mas_next_sibling(&tmp_mas))
+ return false;
+
+ end = mas_data_end(&tmp_mas);
+ slot_total += end;
+ space = 2 * mt_slot_count(mas->node) - 2;
+ /* -2 instead of -1 to ensure there isn't a triple split */
+ if (ma_is_leaf(mast->bn->type))
+ space--;
+
+ if (mas->max == ULONG_MAX)
+ space--;
+
+ if (slot_total >= space)
+ return false;
+
+ /* Get the data; Fill mast->bn */
+ mast->bn->b_end++;
+ if (left) {
+ mab_shift_right(mast->bn, end + 1);
+ mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0);
+ mast->bn->b_end = slot_total + 1;
+ } else {
+ mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end);
+ }
+
+ /* Configure mast for splitting of mast->bn */
+ split = mt_slots[mast->bn->type] - 2;
+ if (left) {
+ /* Switch mas to prev node */
+ mat_add(mast->free, mas->node);
+ *mas = tmp_mas;
+ /* Start using mast->l for the left side. */
+ tmp_mas.node = mast->l->node;
+ *mast->l = tmp_mas;
+ } else {
+ mat_add(mast->free, tmp_mas.node);
+ tmp_mas.node = mast->r->node;
+ *mast->r = tmp_mas;
+ split = slot_total - split;
+ }
+ split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]);
+ /* Update parent slot for split calculation. */
+ if (left)
+ mast->orig_l->offset += end + 1;
+
+ mast_split_data(mast, mas, split);
+ mast_fill_bnode(mast, mas, 2);
+ mas_split_final_node(mast, mas, height + 1);
+ return true;
+}
+
+/*
+ * mas_split() - Split data that is too big for one node into two.
+ * @mas: The maple state
+ * @b_node: The maple big node
+ * Return: 1 on success, 0 on failure.
+ */
+static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
+{
+
+ struct maple_subtree_state mast;
+ int height = 0;
+ unsigned char mid_split, split = 0;
+
+ /*
+ * Splitting is handled differently from any other B-tree; the Maple
+ * Tree splits upwards. Splitting up means that the split operation
+ * occurs when the walk of the tree hits the leaves and not on the way
+ * down. The reason for splitting up is that it is impossible to know
+ * how much space will be needed until the leaf is (or leaves are)
+ * reached. Since overwriting data is allowed and a range could
+ * overwrite more than one range or result in changing one entry into 3
+ * entries, it is impossible to know if a split is required until the
+ * data is examined.
+ *
+ * Splitting is a balancing act between keeping allocations to a minimum
+ * and avoiding a 'jitter' event where a tree is expanded to make room
+ * for an entry followed by a contraction when the entry is removed. To
+ * accomplish the balance, there are empty slots remaining in both left
+ * and right nodes after a split.
+ */
+ MA_STATE(l_mas, mas->tree, mas->index, mas->last);
+ MA_STATE(r_mas, mas->tree, mas->index, mas->last);
+ MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last);
+ MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last);
+ MA_TOPIARY(mat, mas->tree);
+
+ trace_ma_op(__func__, mas);
+ mas->depth = mas_mt_height(mas);
+ /* Allocation failures will happen early. */
+ mas_node_count(mas, 1 + mas->depth * 2);
+ if (mas_is_err(mas))
+ return 0;
+
+ mast.l = &l_mas;
+ mast.r = &r_mas;
+ mast.orig_l = &prev_l_mas;
+ mast.orig_r = &prev_r_mas;
+ mast.free = &mat;
+ mast.bn = b_node;
+
+ while (height++ <= mas->depth) {
+ if (mt_slots[b_node->type] > b_node->b_end) {
+ mas_split_final_node(&mast, mas, height);
+ break;
+ }
+
+ l_mas = r_mas = *mas;
+ l_mas.node = mas_new_ma_node(mas, b_node);
+ r_mas.node = mas_new_ma_node(mas, b_node);
+ /*
+ * Another way that 'jitter' is avoided is to terminate a split up early if the
+ * left or right node has space to spare. This is referred to as "pushing left"
+ * or "pushing right" and is similar to the B* tree, except the nodes left or
+ * right can rarely be reused due to RCU, but the ripple upwards is halted which
+ * is a significant savings.
+ */
+ /* Try to push left. */
+ if (mas_push_data(mas, height, &mast, true))
+ break;
+
+ /* Try to push right. */
+ if (mas_push_data(mas, height, &mast, false))
+ break;
+
+ split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min);
+ mast_split_data(&mast, mas, split);
+ /*
+ * Usually correct, mab_mas_cp in the above call overwrites
+ * r->max.
+ */
+ mast.r->max = mas->max;
+ mast_fill_bnode(&mast, mas, 1);
+ prev_l_mas = *mast.l;
+ prev_r_mas = *mast.r;
+ }
+
+ /* Set the original node as dead */
+ mat_add(mast.free, mas->node);
+ mas->node = l_mas.node;
+ mas_wmb_replace(mas, mast.free, NULL);
+ mtree_range_walk(mas);
+ return 1;
+}
+
+/*
+ * mas_reuse_node() - Reuse the node to store the data.
+ * @wr_mas: The maple write state
+ * @bn: The maple big node
+ * @end: The end of the data.
+ *
+ * Will always return false in RCU mode.
+ *
+ * Return: True if node was reused, false otherwise.
+ */
+static inline bool mas_reuse_node(struct ma_wr_state *wr_mas,
+ struct maple_big_node *bn, unsigned char end)
+{
+ /* Need to be rcu safe. */
+ if (mt_in_rcu(wr_mas->mas->tree))
+ return false;
+
+ if (end > bn->b_end) {
+ int clear = mt_slots[wr_mas->type] - bn->b_end;
+
+ memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--);
+ memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear);
+ }
+ mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false);
+ return true;
+}
+
+/*
+ * mas_commit_b_node() - Commit the big node into the tree.
+ * @wr_mas: The maple write state
+ * @b_node: The maple big node
+ * @end: The end of the data.
+ */
+static inline int mas_commit_b_node(struct ma_wr_state *wr_mas,
+ struct maple_big_node *b_node, unsigned char end)
+{
+ struct maple_node *node;
+ unsigned char b_end = b_node->b_end;
+ enum maple_type b_type = b_node->type;
+
+ if ((b_end < mt_min_slots[b_type]) &&
+ (!mte_is_root(wr_mas->mas->node)) &&
+ (mas_mt_height(wr_mas->mas) > 1))
+ return mas_rebalance(wr_mas->mas, b_node);
+
+ if (b_end >= mt_slots[b_type])
+ return mas_split(wr_mas->mas, b_node);
+
+ if (mas_reuse_node(wr_mas, b_node, end))
+ goto reuse_node;
+
+ mas_node_count(wr_mas->mas, 1);
+ if (mas_is_err(wr_mas->mas))
+ return 0;
+
+ node = mas_pop_node(wr_mas->mas);
+ node->parent = mas_mn(wr_mas->mas)->parent;
+ wr_mas->mas->node = mt_mk_node(node, b_type);
+ mab_mas_cp(b_node, 0, b_end, wr_mas->mas, true);
+
+ mas_replace(wr_mas->mas, false);
+reuse_node:
+ mas_update_gap(wr_mas->mas);
+ return 1;
+}
+
+/*
+ * mas_root_expand() - Expand a root to a node
+ * @mas: The maple state
+ * @entry: The entry to store into the tree
+ */
+static inline int mas_root_expand(struct ma_state *mas, void *entry)
+{
+ void *contents = mas_root_locked(mas);
+ enum maple_type type = maple_leaf_64;
+ struct maple_node *node;
+ void __rcu **slots;
+ unsigned long *pivots;
+ int slot = 0;
+
+ mas_node_count(mas, 1);
+ if (unlikely(mas_is_err(mas)))
+ return 0;
+
+ node = mas_pop_node(mas);
+ pivots = ma_pivots(node, type);
+ slots = ma_slots(node, type);
+ node->parent = ma_parent_ptr(
+ ((unsigned long)mas->tree | MA_ROOT_PARENT));
+ mas->node = mt_mk_node(node, type);
+
+ if (mas->index) {
+ if (contents) {
+ rcu_assign_pointer(slots[slot], contents);
+ if (likely(mas->index > 1))
+ slot++;
+ }
+ pivots[slot++] = mas->index - 1;
+ }
+
+ rcu_assign_pointer(slots[slot], entry);
+ mas->offset = slot;
+ pivots[slot] = mas->last;
+ if (mas->last != ULONG_MAX)
+ slot++;
+ mas->depth = 1;
+ mas_set_height(mas);
+
+ /* swap the new root into the tree */
+ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
+ ma_set_meta(node, maple_leaf_64, 0, slot);
+ return slot;
+}
+
+static inline void mas_store_root(struct ma_state *mas, void *entry)
+{
+ if (likely((mas->last != 0) || (mas->index != 0)))
+ mas_root_expand(mas, entry);
+ else if (((unsigned long) (entry) & 3) == 2)
+ mas_root_expand(mas, entry);
+ else {
+ rcu_assign_pointer(mas->tree->ma_root, entry);
+ mas->node = MAS_START;
+ }
+}
+
+/*
+ * mas_is_span_wr() - Check if the write needs to be treated as a write that
+ * spans the node.
+ * @mas: The maple state
+ * @piv: The pivot value being written
+ * @type: The maple node type
+ * @entry: The data to write
+ *
+ * Spanning writes are writes that start in one node and end in another OR if
+ * the write of a %NULL will cause the node to end with a %NULL.
+ *
+ * Return: True if this is a spanning write, false otherwise.
+ */
+static bool mas_is_span_wr(struct ma_wr_state *wr_mas)
+{
+ unsigned long max;
+ unsigned long last = wr_mas->mas->last;
+ unsigned long piv = wr_mas->r_max;
+ enum maple_type type = wr_mas->type;
+ void *entry = wr_mas->entry;
+
+ /* Contained in this pivot */
+ if (piv > last)
+ return false;
+
+ max = wr_mas->mas->max;
+ if (unlikely(ma_is_leaf(type))) {
+ /* Fits in the node, but may span slots. */
+ if (last < max)
+ return false;
+
+ /* Writes to the end of the node but not null. */
+ if ((last == max) && entry)
+ return false;
+
+ /*
+ * Writing ULONG_MAX is not a spanning write regardless of the
+ * value being written as long as the range fits in the node.
+ */
+ if ((last == ULONG_MAX) && (last == max))
+ return false;
+ } else if (piv == last) {
+ if (entry)
+ return false;
+
+ /* Detect spanning store wr walk */
+ if (last == ULONG_MAX)
+ return false;
+ }
+
+ trace_ma_write(__func__, wr_mas->mas, piv, entry);
+
+ return true;
+}
+
+static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas)
+{
+ wr_mas->mas->depth++;
+ wr_mas->type = mte_node_type(wr_mas->mas->node);
+ mas_wr_node_walk(wr_mas);
+ wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type);
+}
+
+static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas)
+{
+ wr_mas->mas->max = wr_mas->r_max;
+ wr_mas->mas->min = wr_mas->r_min;
+ wr_mas->mas->node = wr_mas->content;
+ wr_mas->mas->offset = 0;
+}
+/*
+ * mas_wr_walk() - Walk the tree for a write.
+ * @wr_mas: The maple write state
+ *
+ * Uses mas_slot_locked() and does not need to worry about dead nodes.
+ *
+ * Return: True if it's contained in a node, false on spanning write.
+ */
+static bool mas_wr_walk(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+
+ while (true) {
+ mas_wr_walk_descend(wr_mas);
+ if (unlikely(mas_is_span_wr(wr_mas)))
+ return false;
+
+ wr_mas->content = mas_slot_locked(mas, wr_mas->slots,
+ mas->offset);
+ if (ma_is_leaf(wr_mas->type))
+ return true;
+
+ mas_wr_walk_traverse(wr_mas);
+ }
+
+ return true;
+}
+
+static bool mas_wr_walk_index(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+
+ while (true) {
+ mas_wr_walk_descend(wr_mas);
+ wr_mas->content = mas_slot_locked(mas, wr_mas->slots,
+ mas->offset);
+ if (ma_is_leaf(wr_mas->type))
+ return true;
+ mas_wr_walk_traverse(wr_mas);
+
+ }
+ return true;
+}
+/*
+ * mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs.
+ * @l_wr_mas: The left maple write state
+ * @r_wr_mas: The right maple write state
+ */
+static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
+ struct ma_wr_state *r_wr_mas)
+{
+ struct ma_state *r_mas = r_wr_mas->mas;
+ struct ma_state *l_mas = l_wr_mas->mas;
+ unsigned char l_slot;
+
+ l_slot = l_mas->offset;
+ if (!l_wr_mas->content)
+ l_mas->index = l_wr_mas->r_min;
+
+ if ((l_mas->index == l_wr_mas->r_min) &&
+ (l_slot &&
+ !mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) {
+ if (l_slot > 1)
+ l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1;
+ else
+ l_mas->index = l_mas->min;
+
+ l_mas->offset = l_slot - 1;
+ }
+
+ if (!r_wr_mas->content) {
+ if (r_mas->last < r_wr_mas->r_max)
+ r_mas->last = r_wr_mas->r_max;
+ r_mas->offset++;
+ } else if ((r_mas->last == r_wr_mas->r_max) &&
+ (r_mas->last < r_mas->max) &&
+ !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
+ r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots,
+ r_wr_mas->type, r_mas->offset + 1);
+ r_mas->offset++;
+ }
+}
+
+static inline void *mas_state_walk(struct ma_state *mas)
+{
+ void *entry;
+
+ entry = mas_start(mas);
+ if (mas_is_none(mas))
+ return NULL;
+
+ if (mas_is_ptr(mas))
+ return entry;
+
+ return mtree_range_walk(mas);
+}
+
+/*
+ * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up
+ * to date.
+ *
+ * @mas: The maple state.
+ *
+ * Note: Leaves mas in undesirable state.
+ * Return: The entry for @mas->index or %NULL on dead node.
+ */
+static inline void *mtree_lookup_walk(struct ma_state *mas)
+{
+ unsigned long *pivots;
+ unsigned char offset;
+ struct maple_node *node;
+ struct maple_enode *next;
+ enum maple_type type;
+ void __rcu **slots;
+ unsigned char end;
+ unsigned long max;
+
+ next = mas->node;
+ max = ULONG_MAX;
+ do {
+ offset = 0;
+ node = mte_to_node(next);
+ type = mte_node_type(next);
+ pivots = ma_pivots(node, type);
+ end = ma_data_end(node, type, pivots, max);
+ if (unlikely(ma_dead_node(node)))
+ goto dead_node;
+
+ if (pivots[offset] >= mas->index)
+ goto next;
+
+ do {
+ offset++;
+ } while ((offset < end) && (pivots[offset] < mas->index));
+
+ if (likely(offset > end))
+ max = pivots[offset];
+
+next:
+ slots = ma_slots(node, type);
+ next = mt_slot(mas->tree, slots, offset);
+ if (unlikely(ma_dead_node(node)))
+ goto dead_node;
+ } while (!ma_is_leaf(type));
+
+ return (void *) next;
+
+dead_node:
+ mas_reset(mas);
+ return NULL;
+}
+
+/*
+ * mas_new_root() - Create a new root node that only contains the entry passed
+ * in.
+ * @mas: The maple state
+ * @entry: The entry to store.
+ *
+ * Only valid when the index == 0 and the last == ULONG_MAX
+ *
+ * Return 0 on error, 1 on success.
+ */
+static inline int mas_new_root(struct ma_state *mas, void *entry)
+{
+ struct maple_enode *root = mas_root_locked(mas);
+ enum maple_type type = maple_leaf_64;
+ struct maple_node *node;
+ void __rcu **slots;
+ unsigned long *pivots;
+
+ if (!entry && !mas->index && mas->last == ULONG_MAX) {
+ mas->depth = 0;
+ mas_set_height(mas);
+ rcu_assign_pointer(mas->tree->ma_root, entry);
+ mas->node = MAS_START;
+ goto done;
+ }
+
+ mas_node_count(mas, 1);
+ if (mas_is_err(mas))
+ return 0;
+
+ node = mas_pop_node(mas);
+ pivots = ma_pivots(node, type);
+ slots = ma_slots(node, type);
+ node->parent = ma_parent_ptr(
+ ((unsigned long)mas->tree | MA_ROOT_PARENT));
+ mas->node = mt_mk_node(node, type);
+ rcu_assign_pointer(slots[0], entry);
+ pivots[0] = mas->last;
+ mas->depth = 1;
+ mas_set_height(mas);
+ rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node));
+
+done:
+ if (xa_is_node(root))
+ mte_destroy_walk(root, mas->tree);
+
+ return 1;
+}
+/*
+ * mas_wr_spanning_store() - Create a subtree with the store operation completed
+ * and new nodes where necessary, then place the sub-tree in the actual tree.
+ * Note that mas is expected to point to the node which caused the store to
+ * span.
+ * @wr_mas: The maple write state
+ *
+ * Return: 0 on error, positive on success.
+ */
+static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas)
+{
+ struct maple_subtree_state mast;
+ struct maple_big_node b_node;
+ struct ma_state *mas;
+ unsigned char height;
+
+ /* Left and Right side of spanning store */
+ MA_STATE(l_mas, NULL, 0, 0);
+ MA_STATE(r_mas, NULL, 0, 0);
+
+ MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry);
+ MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry);
+
+ /*
+ * A store operation that spans multiple nodes is called a spanning
+ * store and is handled early in the store call stack by the function
+ * mas_is_span_wr(). When a spanning store is identified, the maple
+ * state is duplicated. The first maple state walks the left tree path
+ * to ``index``, the duplicate walks the right tree path to ``last``.
+ * The data in the two nodes are combined into a single node, two nodes,
+ * or possibly three nodes (see the 3-way split above). A ``NULL``
+ * written to the last entry of a node is considered a spanning store as
+ * a rebalance is required for the operation to complete and an overflow
+ * of data may happen.
+ */
+ mas = wr_mas->mas;
+ trace_ma_op(__func__, mas);
+
+ if (unlikely(!mas->index && mas->last == ULONG_MAX))
+ return mas_new_root(mas, wr_mas->entry);
+ /*
+ * Node rebalancing may occur due to this store, so there may be three new
+ * entries per level plus a new root.
+ */
+ height = mas_mt_height(mas);
+ mas_node_count(mas, 1 + height * 3);
+ if (mas_is_err(mas))
+ return 0;
+
+ /*
+ * Set up right side. Need to get to the next offset after the spanning
+ * store to ensure it's not NULL and to combine both the next node and
+ * the node with the start together.
+ */
+ r_mas = *mas;
+ /* Avoid overflow, walk to next slot in the tree. */
+ if (r_mas.last + 1)
+ r_mas.last++;
+
+ r_mas.index = r_mas.last;
+ mas_wr_walk_index(&r_wr_mas);
+ r_mas.last = r_mas.index = mas->last;
+
+ /* Set up left side. */
+ l_mas = *mas;
+ mas_wr_walk_index(&l_wr_mas);
+
+ if (!wr_mas->entry) {
+ mas_extend_spanning_null(&l_wr_mas, &r_wr_mas);
+ mas->offset = l_mas.offset;
+ mas->index = l_mas.index;
+ mas->last = l_mas.last = r_mas.last;
+ }
+
+ /* expanding NULLs may make this cover the entire range */
+ if (!l_mas.index && r_mas.last == ULONG_MAX) {
+ mas_set_range(mas, 0, ULONG_MAX);
+ return mas_new_root(mas, wr_mas->entry);
+ }
+
+ memset(&b_node, 0, sizeof(struct maple_big_node));
+ /* Copy l_mas and store the value in b_node. */
+ mas_store_b_node(&l_wr_mas, &b_node, l_wr_mas.node_end);
+ /* Copy r_mas into b_node. */
+ if (r_mas.offset <= r_wr_mas.node_end)
+ mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end,
+ &b_node, b_node.b_end + 1);
+ else
+ b_node.b_end++;
+
+ /* Stop spanning searches by searching for just index. */
+ l_mas.index = l_mas.last = mas->index;
+
+ mast.bn = &b_node;
+ mast.orig_l = &l_mas;
+ mast.orig_r = &r_mas;
+ /* Combine l_mas and r_mas and split them up evenly again. */
+ return mas_spanning_rebalance(mas, &mast, height + 1);
+}
+
+/*
+ * mas_wr_node_store() - Attempt to store the value in a node
+ * @wr_mas: The maple write state
+ *
+ * Attempts to reuse the node, but may allocate.
+ *
+ * Return: True if stored, false otherwise
+ */
+static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+ void __rcu **dst_slots;
+ unsigned long *dst_pivots;
+ unsigned char dst_offset;
+ unsigned char new_end = wr_mas->node_end;
+ unsigned char offset;
+ unsigned char node_slots = mt_slots[wr_mas->type];
+ struct maple_node reuse, *newnode;
+ unsigned char copy_size, max_piv = mt_pivots[wr_mas->type];
+ bool in_rcu = mt_in_rcu(mas->tree);
+
+ offset = mas->offset;
+ if (mas->last == wr_mas->r_max) {
+ /* runs right to the end of the node */
+ if (mas->last == mas->max)
+ new_end = offset;
+ /* don't copy this offset */
+ wr_mas->offset_end++;
+ } else if (mas->last < wr_mas->r_max) {
+ /* new range ends in this range */
+ if (unlikely(wr_mas->r_max == ULONG_MAX))
+ mas_bulk_rebalance(mas, wr_mas->node_end, wr_mas->type);
+
+ new_end++;
+ } else {
+ if (wr_mas->end_piv == mas->last)
+ wr_mas->offset_end++;
+
+ new_end -= wr_mas->offset_end - offset - 1;
+ }
+
+ /* new range starts within a range */
+ if (wr_mas->r_min < mas->index)
+ new_end++;
+
+ /* Not enough room */
+ if (new_end >= node_slots)
+ return false;
+
+ /* Not enough data. */
+ if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) &&
+ !(mas->mas_flags & MA_STATE_BULK))
+ return false;
+
+ /* set up node. */
+ if (in_rcu) {
+ mas_node_count(mas, 1);
+ if (mas_is_err(mas))
+ return false;
+
+ newnode = mas_pop_node(mas);
+ } else {
+ memset(&reuse, 0, sizeof(struct maple_node));
+ newnode = &reuse;
+ }
+
+ newnode->parent = mas_mn(mas)->parent;
+ dst_pivots = ma_pivots(newnode, wr_mas->type);
+ dst_slots = ma_slots(newnode, wr_mas->type);
+ /* Copy from start to insert point */
+ memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * (offset + 1));
+ memcpy(dst_slots, wr_mas->slots, sizeof(void *) * (offset + 1));
+ dst_offset = offset;
+
+ /* Handle insert of new range starting after old range */
+ if (wr_mas->r_min < mas->index) {
+ mas->offset++;
+ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->content);
+ dst_pivots[dst_offset++] = mas->index - 1;
+ }
+
+ /* Store the new entry and range end. */
+ if (dst_offset < max_piv)
+ dst_pivots[dst_offset] = mas->last;
+ mas->offset = dst_offset;
+ rcu_assign_pointer(dst_slots[dst_offset], wr_mas->entry);
+
+ /*
+ * this range wrote to the end of the node or it overwrote the rest of
+ * the data
+ */
+ if (wr_mas->offset_end > wr_mas->node_end || mas->last >= mas->max) {
+ new_end = dst_offset;
+ goto done;
+ }
+
+ dst_offset++;
+ /* Copy to the end of node if necessary. */
+ copy_size = wr_mas->node_end - wr_mas->offset_end + 1;
+ memcpy(dst_slots + dst_offset, wr_mas->slots + wr_mas->offset_end,
+ sizeof(void *) * copy_size);
+ if (dst_offset < max_piv) {
+ if (copy_size > max_piv - dst_offset)
+ copy_size = max_piv - dst_offset;
+
+ memcpy(dst_pivots + dst_offset,
+ wr_mas->pivots + wr_mas->offset_end,
+ sizeof(unsigned long) * copy_size);
+ }
+
+ if ((wr_mas->node_end == node_slots - 1) && (new_end < node_slots - 1))
+ dst_pivots[new_end] = mas->max;
+
+done:
+ mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end);
+ if (in_rcu) {
+ mas->node = mt_mk_node(newnode, wr_mas->type);
+ mas_replace(mas, false);
+ } else {
+ memcpy(wr_mas->node, newnode, sizeof(struct maple_node));
+ }
+ trace_ma_write(__func__, mas, 0, wr_mas->entry);
+ mas_update_gap(mas);
+ return true;
+}
+
+/*
+ * mas_wr_slot_store: Attempt to store a value in a slot.
+ * @wr_mas: the maple write state
+ *
+ * Return: True if stored, false otherwise
+ */
+static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+ unsigned long lmax; /* Logical max. */
+ unsigned char offset = mas->offset;
+
+ if ((wr_mas->r_max > mas->last) && ((wr_mas->r_min != mas->index) ||
+ (offset != wr_mas->node_end)))
+ return false;
+
+ if (offset == wr_mas->node_end - 1)
+ lmax = mas->max;
+ else
+ lmax = wr_mas->pivots[offset + 1];
+
+ /* going to overwrite too many slots. */
+ if (lmax < mas->last)
+ return false;
+
+ if (wr_mas->r_min == mas->index) {
+ /* overwriting two or more ranges with one. */
+ if (lmax == mas->last)
+ return false;
+
+ /* Overwriting all of offset and a portion of offset + 1. */
+ rcu_assign_pointer(wr_mas->slots[offset], wr_mas->entry);
+ wr_mas->pivots[offset] = mas->last;
+ goto done;
+ }
+
+ /* Doesn't end on the next range end. */
+ if (lmax != mas->last)
+ return false;
+
+ /* Overwriting a portion of offset and all of offset + 1 */
+ if ((offset + 1 < mt_pivots[wr_mas->type]) &&
+ (wr_mas->entry || wr_mas->pivots[offset + 1]))
+ wr_mas->pivots[offset + 1] = mas->last;
+
+ rcu_assign_pointer(wr_mas->slots[offset + 1], wr_mas->entry);
+ wr_mas->pivots[offset] = mas->index - 1;
+ mas->offset++; /* Keep mas accurate. */
+
+done:
+ trace_ma_write(__func__, mas, 0, wr_mas->entry);
+ mas_update_gap(mas);
+ return true;
+}
+
+static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas)
+{
+ while ((wr_mas->mas->last > wr_mas->end_piv) &&
+ (wr_mas->offset_end < wr_mas->node_end))
+ wr_mas->end_piv = wr_mas->pivots[++wr_mas->offset_end];
+
+ if (wr_mas->mas->last > wr_mas->end_piv)
+ wr_mas->end_piv = wr_mas->mas->max;
+}
+
+static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+
+ if (mas->last < wr_mas->end_piv && !wr_mas->slots[wr_mas->offset_end])
+ mas->last = wr_mas->end_piv;
+
+ /* Check next slot(s) if we are overwriting the end */
+ if ((mas->last == wr_mas->end_piv) &&
+ (wr_mas->node_end != wr_mas->offset_end) &&
+ !wr_mas->slots[wr_mas->offset_end + 1]) {
+ wr_mas->offset_end++;
+ if (wr_mas->offset_end == wr_mas->node_end)
+ mas->last = mas->max;
+ else
+ mas->last = wr_mas->pivots[wr_mas->offset_end];
+ wr_mas->end_piv = mas->last;
+ }
+
+ if (!wr_mas->content) {
+ /* If this one is null, the next and prev are not */
+ mas->index = wr_mas->r_min;
+ } else {
+ /* Check prev slot if we are overwriting the start */
+ if (mas->index == wr_mas->r_min && mas->offset &&
+ !wr_mas->slots[mas->offset - 1]) {
+ mas->offset--;
+ wr_mas->r_min = mas->index =
+ mas_safe_min(mas, wr_mas->pivots, mas->offset);
+ wr_mas->r_max = wr_mas->pivots[mas->offset];
+ }
+ }
+}
+
+static inline bool mas_wr_append(struct ma_wr_state *wr_mas)
+{
+ unsigned char end = wr_mas->node_end;
+ unsigned char new_end = end + 1;
+ struct ma_state *mas = wr_mas->mas;
+ unsigned char node_pivots = mt_pivots[wr_mas->type];
+
+ if ((mas->index != wr_mas->r_min) && (mas->last == wr_mas->r_max)) {
+ if (new_end < node_pivots)
+ wr_mas->pivots[new_end] = wr_mas->pivots[end];
+
+ if (new_end < node_pivots)
+ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end);
+
+ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->entry);
+ mas->offset = new_end;
+ wr_mas->pivots[end] = mas->index - 1;
+
+ return true;
+ }
+
+ if ((mas->index == wr_mas->r_min) && (mas->last < wr_mas->r_max)) {
+ if (new_end < node_pivots)
+ wr_mas->pivots[new_end] = wr_mas->pivots[end];
+
+ rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->content);
+ if (new_end < node_pivots)
+ ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end);
+
+ wr_mas->pivots[end] = mas->last;
+ rcu_assign_pointer(wr_mas->slots[end], wr_mas->entry);
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * mas_wr_bnode() - Slow path for a modification.
+ * @wr_mas: The write maple state
+ *
+ * This is where split, rebalance end up.
+ */
+static void mas_wr_bnode(struct ma_wr_state *wr_mas)
+{
+ struct maple_big_node b_node;
+
+ trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry);
+ memset(&b_node, 0, sizeof(struct maple_big_node));
+ mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end);
+ mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end);
+}
+
+static inline void mas_wr_modify(struct ma_wr_state *wr_mas)
+{
+ unsigned char node_slots;
+ unsigned char node_size;
+ struct ma_state *mas = wr_mas->mas;
+
+ /* Direct replacement */
+ if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) {
+ rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry);
+ if (!!wr_mas->entry ^ !!wr_mas->content)
+ mas_update_gap(mas);
+ return;
+ }
+
+ /* Attempt to append */
+ node_slots = mt_slots[wr_mas->type];
+ node_size = wr_mas->node_end - wr_mas->offset_end + mas->offset + 2;
+ if (mas->max == ULONG_MAX)
+ node_size++;
+
+ /* slot and node store will not fit, go to the slow path */
+ if (unlikely(node_size >= node_slots))
+ goto slow_path;
+
+ if (wr_mas->entry && (wr_mas->node_end < node_slots - 1) &&
+ (mas->offset == wr_mas->node_end) && mas_wr_append(wr_mas)) {
+ if (!wr_mas->content || !wr_mas->entry)
+ mas_update_gap(mas);
+ return;
+ }
+
+ if ((wr_mas->offset_end - mas->offset <= 1) && mas_wr_slot_store(wr_mas))
+ return;
+ else if (mas_wr_node_store(wr_mas))
+ return;
+
+ if (mas_is_err(mas))
+ return;
+
+slow_path:
+ mas_wr_bnode(wr_mas);
+}
+
+/*
+ * mas_wr_store_entry() - Internal call to store a value
+ * @mas: The maple state
+ * @entry: The entry to store.
+ *
+ * Return: The contents that was stored at the index.
+ */
+static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas)
+{
+ struct ma_state *mas = wr_mas->mas;
+
+ wr_mas->content = mas_start(mas);
+ if (mas_is_none(mas) || mas_is_ptr(mas)) {
+ mas_store_root(mas, wr_mas->entry);
+ return wr_mas->content;
+ }
+
+ if (unlikely(!mas_wr_walk(wr_mas))) {
+ mas_wr_spanning_store(wr_mas);
+ return wr_mas->content;
+ }
+
+ /* At this point, we are at the leaf node that needs to be altered. */
+ wr_mas->end_piv = wr_mas->r_max;
+ mas_wr_end_piv(wr_mas);
+
+ if (!wr_mas->entry)
+ mas_wr_extend_null(wr_mas);
+
+ /* New root for a single pointer */
+ if (unlikely(!mas->index && mas->last == ULONG_MAX)) {
+ mas_new_root(mas, wr_mas->entry);
+ return wr_mas->content;
+ }
+
+ mas_wr_modify(wr_mas);
+ return wr_mas->content;
+}
+
+/**
+ * mas_insert() - Internal call to insert a value
+ * @mas: The maple state
+ * @entry: The entry to store
+ *
+ * Return: %NULL or the contents that already exists at the requested index
+ * otherwise. The maple state needs to be checked for error conditions.
+ */
+static inline void *mas_insert(struct ma_state *mas, void *entry)
+{
+ MA_WR_STATE(wr_mas, mas, entry);
+
+ /*
+ * Inserting a new range inserts either 0, 1, or 2 pivots within the
+ * tree. If the insert fits exactly into an existing gap with a value
+ * of NULL, then the slot only needs to be written with the new value.
+ * If the range being inserted is adjacent to another range, then only a
+ * single pivot needs to be inserted (as well as writing the entry). If
+ * the new range is within a gap but does not touch any other ranges,
+ * then two pivots need to be inserted: the start - 1, and the end. As
+ * usual, the entry must be written. Most operations require a new node
+ * to be allocated and replace an existing node to ensure RCU safety,
+ * when in RCU mode. The exception to requiring a newly allocated node
+ * is when inserting at the end of a node (appending). When done
+ * carefully, appending can reuse the node in place.
+ */
+ wr_mas.content = mas_start(mas);
+ if (wr_mas.content)
+ goto exists;
+
+ if (mas_is_none(mas) || mas_is_ptr(mas)) {
+ mas_store_root(mas, entry);
+ return NULL;
+ }
+
+ /* spanning writes always overwrite something */
+ if (!mas_wr_walk(&wr_mas))
+ goto exists;
+
+ /* At this point, we are at the leaf node that needs to be altered. */
+ wr_mas.offset_end = mas->offset;
+ wr_mas.end_piv = wr_mas.r_max;
+
+ if (wr_mas.content || (mas->last > wr_mas.r_max))
+ goto exists;
+
+ if (!entry)
+ return NULL;
+
+ mas_wr_modify(&wr_mas);
+ return wr_mas.content;
+
+exists:
+ mas_set_err(mas, -EEXIST);
+ return wr_mas.content;
+
+}
+
+/*
+ * mas_prev_node() - Find the prev non-null entry at the same level in the
+ * tree. The prev value will be mas->node[mas->offset] or MAS_NONE.
+ * @mas: The maple state
+ * @min: The lower limit to search
+ *
+ * The prev node value will be mas->node[mas->offset] or MAS_NONE.
+ * Return: 1 if the node is dead, 0 otherwise.
+ */
+static inline int mas_prev_node(struct ma_state *mas, unsigned long min)
+{
+ enum maple_type mt;
+ int offset, level;
+ void __rcu **slots;
+ struct maple_node *node;
+ struct maple_enode *enode;
+ unsigned long *pivots;
+
+ if (mas_is_none(mas))
+ return 0;
+
+ level = 0;
+ do {
+ node = mas_mn(mas);
+ if (ma_is_root(node))
+ goto no_entry;
+
+ /* Walk up. */
+ if (unlikely(mas_ascend(mas)))
+ return 1;
+ offset = mas->offset;
+ level++;
+ } while (!offset);
+
+ offset--;
+ mt = mte_node_type(mas->node);
+ node = mas_mn(mas);
+ slots = ma_slots(node, mt);
+ pivots = ma_pivots(node, mt);
+ mas->max = pivots[offset];
+ if (offset)
+ mas->min = pivots[offset - 1] + 1;
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ if (mas->max < min)
+ goto no_entry_min;
+
+ while (level > 1) {
+ level--;
+ enode = mas_slot(mas, slots, offset);
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ mas->node = enode;
+ mt = mte_node_type(mas->node);
+ node = mas_mn(mas);
+ slots = ma_slots(node, mt);
+ pivots = ma_pivots(node, mt);
+ offset = ma_data_end(node, mt, pivots, mas->max);
+ if (offset)
+ mas->min = pivots[offset - 1] + 1;
+
+ if (offset < mt_pivots[mt])
+ mas->max = pivots[offset];
+
+ if (mas->max < min)
+ goto no_entry;
+ }
+
+ mas->node = mas_slot(mas, slots, offset);
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ mas->offset = mas_data_end(mas);
+ if (unlikely(mte_dead_node(mas->node)))
+ return 1;
+
+ return 0;
+
+no_entry_min:
+ mas->offset = offset;
+ if (offset)
+ mas->min = pivots[offset - 1] + 1;
+no_entry:
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ mas->node = MAS_NONE;
+ return 0;
+}
+
+/*
+ * mas_next_node() - Get the next node at the same level in the tree.
+ * @mas: The maple state
+ * @max: The maximum pivot value to check.
+ *
+ * The next value will be mas->node[mas->offset] or MAS_NONE.
+ * Return: 1 on dead node, 0 otherwise.
+ */
+static inline int mas_next_node(struct ma_state *mas, struct maple_node *node,
+ unsigned long max)
+{
+ unsigned long min, pivot;
+ unsigned long *pivots;
+ struct maple_enode *enode;
+ int level = 0;
+ unsigned char offset;
+ enum maple_type mt;
+ void __rcu **slots;
+
+ if (mas->max >= max)
+ goto no_entry;
+
+ level = 0;
+ do {
+ if (ma_is_root(node))
+ goto no_entry;
+
+ min = mas->max + 1;
+ if (min > max)
+ goto no_entry;
+
+ if (unlikely(mas_ascend(mas)))
+ return 1;
+
+ offset = mas->offset;
+ level++;
+ node = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ pivots = ma_pivots(node, mt);
+ } while (unlikely(offset == ma_data_end(node, mt, pivots, mas->max)));
+
+ slots = ma_slots(node, mt);
+ pivot = mas_safe_pivot(mas, pivots, ++offset, mt);
+ while (unlikely(level > 1)) {
+ /* Descend, if necessary */
+ enode = mas_slot(mas, slots, offset);
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ mas->node = enode;
+ level--;
+ node = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ slots = ma_slots(node, mt);
+ pivots = ma_pivots(node, mt);
+ offset = 0;
+ pivot = pivots[0];
+ }
+
+ enode = mas_slot(mas, slots, offset);
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ mas->node = enode;
+ mas->min = min;
+ mas->max = pivot;
+ return 0;
+
+no_entry:
+ if (unlikely(ma_dead_node(node)))
+ return 1;
+
+ mas->node = MAS_NONE;
+ return 0;
+}
+
+/*
+ * mas_next_nentry() - Get the next node entry
+ * @mas: The maple state
+ * @max: The maximum value to check
+ * @*range_start: Pointer to store the start of the range.
+ *
+ * Sets @mas->offset to the offset of the next node entry, @mas->last to the
+ * pivot of the entry.
+ *
+ * Return: The next entry, %NULL otherwise
+ */
+static inline void *mas_next_nentry(struct ma_state *mas,
+ struct maple_node *node, unsigned long max, enum maple_type type)
+{
+ unsigned char count;
+ unsigned long pivot;
+ unsigned long *pivots;
+ void __rcu **slots;
+ void *entry;
+
+ if (mas->last == mas->max) {
+ mas->index = mas->max;
+ return NULL;
+ }
+
+ pivots = ma_pivots(node, type);
+ slots = ma_slots(node, type);
+ mas->index = mas_safe_min(mas, pivots, mas->offset);
+ if (ma_dead_node(node))
+ return NULL;
+
+ if (mas->index > max)
+ return NULL;
+
+ count = ma_data_end(node, type, pivots, mas->max);
+ if (mas->offset > count)
+ return NULL;
+
+ while (mas->offset < count) {
+ pivot = pivots[mas->offset];
+ entry = mas_slot(mas, slots, mas->offset);
+ if (ma_dead_node(node))
+ return NULL;
+
+ if (entry)
+ goto found;
+
+ if (pivot >= max)
+ return NULL;
+
+ mas->index = pivot + 1;
+ mas->offset++;
+ }
+
+ if (mas->index > mas->max) {
+ mas->index = mas->last;
+ return NULL;
+ }
+
+ pivot = mas_safe_pivot(mas, pivots, mas->offset, type);
+ entry = mas_slot(mas, slots, mas->offset);
+ if (ma_dead_node(node))
+ return NULL;
+
+ if (!pivot)
+ return NULL;
+
+ if (!entry)
+ return NULL;
+
+found:
+ mas->last = pivot;
+ return entry;
+}
+
+static inline void mas_rewalk(struct ma_state *mas, unsigned long index)
+{
+
+retry:
+ mas_set(mas, index);
+ mas_state_walk(mas);
+ if (mas_is_start(mas))
+ goto retry;
+
+ return;
+
+}
+
+/*
+ * mas_next_entry() - Internal function to get the next entry.
+ * @mas: The maple state
+ * @limit: The maximum range start.
+ *
+ * Set the @mas->node to the next entry and the range_start to
+ * the beginning value for the entry. Does not check beyond @limit.
+ * Sets @mas->index and @mas->last to the limit if it is hit.
+ * Restarts on dead nodes.
+ *
+ * Return: the next entry or %NULL.
+ */
+static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit)
+{
+ void *entry = NULL;
+ struct maple_enode *prev_node;
+ struct maple_node *node;
+ unsigned char offset;
+ unsigned long last;
+ enum maple_type mt;
+
+ last = mas->last;
+retry:
+ offset = mas->offset;
+ prev_node = mas->node;
+ node = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ mas->offset++;
+ if (unlikely(mas->offset >= mt_slots[mt])) {
+ mas->offset = mt_slots[mt] - 1;
+ goto next_node;
+ }
+
+ while (!mas_is_none(mas)) {
+ entry = mas_next_nentry(mas, node, limit, mt);
+ if (unlikely(ma_dead_node(node))) {
+ mas_rewalk(mas, last);
+ goto retry;
+ }
+
+ if (likely(entry))
+ return entry;
+
+ if (unlikely((mas->index > limit)))
+ break;
+
+next_node:
+ prev_node = mas->node;
+ offset = mas->offset;
+ if (unlikely(mas_next_node(mas, node, limit))) {
+ mas_rewalk(mas, last);
+ goto retry;
+ }
+ mas->offset = 0;
+ node = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ }
+
+ mas->index = mas->last = limit;
+ mas->offset = offset;
+ mas->node = prev_node;
+ return NULL;
+}
+
+/*
+ * mas_prev_nentry() - Get the previous node entry.
+ * @mas: The maple state.
+ * @limit: The lower limit to check for a value.
+ *
+ * Return: the entry, %NULL otherwise.
+ */
+static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit,
+ unsigned long index)
+{
+ unsigned long pivot, min;
+ unsigned char offset;
+ struct maple_node *mn;
+ enum maple_type mt;
+ unsigned long *pivots;
+ void __rcu **slots;
+ void *entry;
+
+retry:
+ if (!mas->offset)
+ return NULL;
+
+ mn = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ offset = mas->offset - 1;
+ if (offset >= mt_slots[mt])
+ offset = mt_slots[mt] - 1;
+
+ slots = ma_slots(mn, mt);
+ pivots = ma_pivots(mn, mt);
+ if (offset == mt_pivots[mt])
+ pivot = mas->max;
+ else
+ pivot = pivots[offset];
+
+ if (unlikely(ma_dead_node(mn))) {
+ mas_rewalk(mas, index);
+ goto retry;
+ }
+
+ while (offset && ((!mas_slot(mas, slots, offset) && pivot >= limit) ||
+ !pivot))
+ pivot = pivots[--offset];
+
+ min = mas_safe_min(mas, pivots, offset);
+ entry = mas_slot(mas, slots, offset);
+ if (unlikely(ma_dead_node(mn))) {
+ mas_rewalk(mas, index);
+ goto retry;
+ }
+
+ if (likely(entry)) {
+ mas->offset = offset;
+ mas->last = pivot;
+ mas->index = min;
+ }
+ return entry;
+}
+
+static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min)
+{
+ void *entry;
+
+retry:
+ while (likely(!mas_is_none(mas))) {
+ entry = mas_prev_nentry(mas, min, mas->index);
+ if (unlikely(mas->last < min))
+ goto not_found;
+
+ if (likely(entry))
+ return entry;
+
+ if (unlikely(mas_prev_node(mas, min))) {
+ mas_rewalk(mas, mas->index);
+ goto retry;
+ }
+
+ mas->offset++;
+ }
+
+ mas->offset--;
+not_found:
+ mas->index = mas->last = min;
+ return NULL;
+}
+
+/*
+ * mas_rev_awalk() - Internal function. Reverse allocation walk. Find the
+ * highest gap address of a given size in a given node and descend.
+ * @mas: The maple state
+ * @size: The needed size.
+ *
+ * Return: True if found in a leaf, false otherwise.
+ *
+ */
+static bool mas_rev_awalk(struct ma_state *mas, unsigned long size)
+{
+ enum maple_type type = mte_node_type(mas->node);
+ struct maple_node *node = mas_mn(mas);
+ unsigned long *pivots, *gaps;
+ void __rcu **slots;
+ unsigned long gap = 0;
+ unsigned long max, min, index;
+ unsigned char offset;
+
+ if (unlikely(mas_is_err(mas)))
+ return true;
+
+ if (ma_is_dense(type)) {
+ /* dense nodes. */
+ mas->offset = (unsigned char)(mas->index - mas->min);
+ return true;
+ }
+
+ pivots = ma_pivots(node, type);
+ slots = ma_slots(node, type);
+ gaps = ma_gaps(node, type);
+ offset = mas->offset;
+ min = mas_safe_min(mas, pivots, offset);
+ /* Skip out of bounds. */
+ while (mas->last < min)
+ min = mas_safe_min(mas, pivots, --offset);
+
+ max = mas_safe_pivot(mas, pivots, offset, type);
+ index = mas->index;
+ while (index <= max) {
+ gap = 0;
+ if (gaps)
+ gap = gaps[offset];
+ else if (!mas_slot(mas, slots, offset))
+ gap = max - min + 1;
+
+ if (gap) {
+ if ((size <= gap) && (size <= mas->last - min + 1))
+ break;
+
+ if (!gaps) {
+ /* Skip the next slot, it cannot be a gap. */
+ if (offset < 2)
+ goto ascend;
+
+ offset -= 2;
+ max = pivots[offset];
+ min = mas_safe_min(mas, pivots, offset);
+ continue;
+ }
+ }
+
+ if (!offset)
+ goto ascend;
+
+ offset--;
+ max = min - 1;
+ min = mas_safe_min(mas, pivots, offset);
+ }
+
+ if (unlikely(index > max)) {
+ mas_set_err(mas, -EBUSY);
+ return false;
+ }
+
+ if (unlikely(ma_is_leaf(type))) {
+ mas->offset = offset;
+ mas->min = min;
+ mas->max = min + gap - 1;
+ return true;
+ }
+
+ /* descend, only happens under lock. */
+ mas->node = mas_slot(mas, slots, offset);
+ mas->min = min;
+ mas->max = max;
+ mas->offset = mas_data_end(mas);
+ return false;
+
+ascend:
+ if (mte_is_root(mas->node))
+ mas_set_err(mas, -EBUSY);
+
+ return false;
+}
+
+static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size)
+{
+ enum maple_type type = mte_node_type(mas->node);
+ unsigned long pivot, min, gap = 0;
+ unsigned char count, offset;
+ unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type);
+ void __rcu **slots = ma_slots(mas_mn(mas), type);
+ bool found = false;
+
+ if (ma_is_dense(type)) {
+ mas->offset = (unsigned char)(mas->index - mas->min);
+ return true;
+ }
+
+ gaps = ma_gaps(mte_to_node(mas->node), type);
+ offset = mas->offset;
+ count = mt_slots[type];
+ min = mas_safe_min(mas, pivots, offset);
+ for (; offset < count; offset++) {
+ pivot = mas_safe_pivot(mas, pivots, offset, type);
+ if (offset && !pivot)
+ break;
+
+ /* Not within lower bounds */
+ if (mas->index > pivot)
+ goto next_slot;
+
+ if (gaps)
+ gap = gaps[offset];
+ else if (!mas_slot(mas, slots, offset))
+ gap = min(pivot, mas->last) - max(mas->index, min) + 1;
+ else
+ goto next_slot;
+
+ if (gap >= size) {
+ if (ma_is_leaf(type)) {
+ found = true;
+ goto done;
+ }
+ if (mas->index <= pivot) {
+ mas->node = mas_slot(mas, slots, offset);
+ mas->min = min;
+ mas->max = pivot;
+ offset = 0;
+ type = mte_node_type(mas->node);
+ count = mt_slots[type];
+ break;
+ }
+ }
+next_slot:
+ min = pivot + 1;
+ if (mas->last <= pivot) {
+ mas_set_err(mas, -EBUSY);
+ return true;
+ }
+ }
+
+ if (mte_is_root(mas->node))
+ found = true;
+done:
+ mas->offset = offset;
+ return found;
+}
+
+/**
+ * mas_walk() - Search for @mas->index in the tree.
+ * @mas: The maple state.
+ *
+ * mas->index and mas->last will be set to the range if there is a value. If
+ * mas->node is MAS_NONE, reset to MAS_START.
+ *
+ * Return: the entry at the location or %NULL.
+ */
+void *mas_walk(struct ma_state *mas)
+{
+ void *entry;
+
+retry:
+ entry = mas_state_walk(mas);
+ if (mas_is_start(mas))
+ goto retry;
+
+ if (mas_is_ptr(mas)) {
+ if (!mas->index) {
+ mas->last = 0;
+ } else {
+ mas->index = 1;
+ mas->last = ULONG_MAX;
+ }
+ return entry;
+ }
+
+ if (mas_is_none(mas)) {
+ mas->index = 0;
+ mas->last = ULONG_MAX;
+ }
+
+ return entry;
+}
+
+static inline bool mas_rewind_node(struct ma_state *mas)
+{
+ unsigned char slot;
+
+ do {
+ if (mte_is_root(mas->node)) {
+ slot = mas->offset;
+ if (!slot)
+ return false;
+ } else {
+ mas_ascend(mas);
+ slot = mas->offset;
+ }
+ } while (!slot);
+
+ mas->offset = --slot;
+ return true;
+}
+
+/*
+ * mas_skip_node() - Internal function. Skip over a node.
+ * @mas: The maple state.
+ *
+ * Return: true if there is another node, false otherwise.
+ */
+static inline bool mas_skip_node(struct ma_state *mas)
+{
+ unsigned char slot, slot_count;
+ unsigned long *pivots;
+ enum maple_type mt;
+
+ mt = mte_node_type(mas->node);
+ slot_count = mt_slots[mt] - 1;
+ do {
+ if (mte_is_root(mas->node)) {
+ slot = mas->offset;
+ if (slot > slot_count) {
+ mas_set_err(mas, -EBUSY);
+ return false;
+ }
+ } else {
+ mas_ascend(mas);
+ slot = mas->offset;
+ mt = mte_node_type(mas->node);
+ slot_count = mt_slots[mt] - 1;
+ }
+ } while (slot > slot_count);
+
+ mas->offset = ++slot;
+ pivots = ma_pivots(mas_mn(mas), mt);
+ if (slot > 0)
+ mas->min = pivots[slot - 1] + 1;
+
+ if (slot <= slot_count)
+ mas->max = pivots[slot];
+
+ return true;
+}
+
+/*
+ * mas_awalk() - Allocation walk. Search from low address to high, for a gap of
+ * @size
+ * @mas: The maple state
+ * @size: The size of the gap required
+ *
+ * Search between @mas->index and @mas->last for a gap of @size.
+ */
+static inline void mas_awalk(struct ma_state *mas, unsigned long size)
+{
+ struct maple_enode *last = NULL;
+
+ /*
+ * There are 4 options:
+ * go to child (descend)
+ * go back to parent (ascend)
+ * no gap found. (return, slot == MAPLE_NODE_SLOTS)
+ * found the gap. (return, slot != MAPLE_NODE_SLOTS)
+ */
+ while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) {
+ if (last == mas->node)
+ mas_skip_node(mas);
+ else
+ last = mas->node;
+ }
+}
+
+/*
+ * mas_fill_gap() - Fill a located gap with @entry.
+ * @mas: The maple state
+ * @entry: The value to store
+ * @slot: The offset into the node to store the @entry
+ * @size: The size of the entry
+ * @index: The start location
+ */
+static inline void mas_fill_gap(struct ma_state *mas, void *entry,
+ unsigned char slot, unsigned long size, unsigned long *index)
+{
+ MA_WR_STATE(wr_mas, mas, entry);
+ unsigned char pslot = mte_parent_slot(mas->node);
+ struct maple_enode *mn = mas->node;
+ unsigned long *pivots;
+ enum maple_type ptype;
+ /*
+ * mas->index is the start address for the search
+ * which may no longer be needed.
+ * mas->last is the end address for the search
+ */
+
+ *index = mas->index;
+ mas->last = mas->index + size - 1;
+
+ /*
+ * It is possible that using mas->max and mas->min to correctly
+ * calculate the index and last will cause an issue in the gap
+ * calculation, so fix the ma_state here
+ */
+ mas_ascend(mas);
+ ptype = mte_node_type(mas->node);
+ pivots = ma_pivots(mas_mn(mas), ptype);
+ mas->max = mas_safe_pivot(mas, pivots, pslot, ptype);
+ mas->min = mas_safe_min(mas, pivots, pslot);
+ mas->node = mn;
+ mas->offset = slot;
+ mas_wr_store_entry(&wr_mas);
+}
+
+/*
+ * mas_sparse_area() - Internal function. Return upper or lower limit when
+ * searching for a gap in an empty tree.
+ * @mas: The maple state
+ * @min: the minimum range
+ * @max: The maximum range
+ * @size: The size of the gap
+ * @fwd: Searching forward or back
+ */
+static inline void mas_sparse_area(struct ma_state *mas, unsigned long min,
+ unsigned long max, unsigned long size, bool fwd)
+{
+ unsigned long start = 0;
+
+ if (!unlikely(mas_is_none(mas)))
+ start++;
+ /* mas_is_ptr */
+
+ if (start < min)
+ start = min;
+
+ if (fwd) {
+ mas->index = start;
+ mas->last = start + size - 1;
+ return;
+ }
+
+ mas->index = max;
+}
+
+/*
+ * mas_empty_area() - Get the lowest address within the range that is
+ * sufficient for the size requested.
+ * @mas: The maple state
+ * @min: The lowest value of the range
+ * @max: The highest value of the range
+ * @size: The size needed
+ */
+int mas_empty_area(struct ma_state *mas, unsigned long min,
+ unsigned long max, unsigned long size)
+{
+ unsigned char offset;
+ unsigned long *pivots;
+ enum maple_type mt;
+
+ if (mas_is_start(mas))
+ mas_start(mas);
+ else if (mas->offset >= 2)
+ mas->offset -= 2;
+ else if (!mas_skip_node(mas))
+ return -EBUSY;
+
+ /* Empty set */
+ if (mas_is_none(mas) || mas_is_ptr(mas)) {
+ mas_sparse_area(mas, min, max, size, true);
+ return 0;
+ }
+
+ /* The start of the window can only be within these values */
+ mas->index = min;
+ mas->last = max;
+ mas_awalk(mas, size);
+
+ if (unlikely(mas_is_err(mas)))
+ return xa_err(mas->node);
+
+ offset = mas->offset;
+ if (unlikely(offset == MAPLE_NODE_SLOTS))
+ return -EBUSY;
+
+ mt = mte_node_type(mas->node);
+ pivots = ma_pivots(mas_mn(mas), mt);
+ if (offset)
+ mas->min = pivots[offset - 1] + 1;
+
+ if (offset < mt_pivots[mt])
+ mas->max = pivots[offset];
+
+ if (mas->index < mas->min)
+ mas->index = mas->min;
+
+ mas->last = mas->index + size - 1;
+ return 0;
+}
+
+/*
+ * mas_empty_area_rev() - Get the highest address within the range that is
+ * sufficient for the size requested.
+ * @mas: The maple state
+ * @min: The lowest value of the range
+ * @max: The highest value of the range
+ * @size: The size needed
+ */
+int mas_empty_area_rev(struct ma_state *mas, unsigned long min,
+ unsigned long max, unsigned long size)
+{
+ struct maple_enode *last = mas->node;
+
+ if (mas_is_start(mas)) {
+ mas_start(mas);
+ mas->offset = mas_data_end(mas);
+ } else if (mas->offset >= 2) {
+ mas->offset -= 2;
+ } else if (!mas_rewind_node(mas)) {
+ return -EBUSY;
+ }
+
+ /* Empty set. */
+ if (mas_is_none(mas) || mas_is_ptr(mas)) {
+ mas_sparse_area(mas, min, max, size, false);
+ return 0;
+ }
+
+ /* The start of the window can only be within these values. */
+ mas->index = min;
+ mas->last = max;
+
+ while (!mas_rev_awalk(mas, size)) {
+ if (last == mas->node) {
+ if (!mas_rewind_node(mas))
+ return -EBUSY;
+ } else {
+ last = mas->node;
+ }
+ }
+
+ if (mas_is_err(mas))
+ return xa_err(mas->node);
+
+ if (unlikely(mas->offset == MAPLE_NODE_SLOTS))
+ return -EBUSY;
+
+ /*
+ * mas_rev_awalk() has set mas->min and mas->max to the gap values. If
+ * the maximum is outside the window we are searching, then use the last
+ * location in the search.
+ * mas->max and mas->min is the range of the gap.
+ * mas->index and mas->last are currently set to the search range.
+ */
+
+ /* Trim the upper limit to the max. */
+ if (mas->max <= mas->last)
+ mas->last = mas->max;
+
+ mas->index = mas->last - size + 1;
+ return 0;
+}
+
+static inline int mas_alloc(struct ma_state *mas, void *entry,
+ unsigned long size, unsigned long *index)
+{
+ unsigned long min;
+
+ mas_start(mas);
+ if (mas_is_none(mas) || mas_is_ptr(mas)) {
+ mas_root_expand(mas, entry);
+ if (mas_is_err(mas))
+ return xa_err(mas->node);
+
+ if (!mas->index)
+ return mte_pivot(mas->node, 0);
+ return mte_pivot(mas->node, 1);
+ }
+
+ /* Must be walking a tree. */
+ mas_awalk(mas, size);
+ if (mas_is_err(mas))
+ return xa_err(mas->node);
+
+ if (mas->offset == MAPLE_NODE_SLOTS)
+ goto no_gap;
+
+ /*
+ * At this point, mas->node points to the right node and we have an
+ * offset that has a sufficient gap.
+ */
+ min = mas->min;
+ if (mas->offset)
+ min = mte_pivot(mas->node, mas->offset - 1) + 1;
+
+ if (mas->index < min)
+ mas->index = min;
+
+ mas_fill_gap(mas, entry, mas->offset, size, index);
+ return 0;
+
+no_gap:
+ return -EBUSY;
+}
+
+static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min,
+ unsigned long max, void *entry,
+ unsigned long size, unsigned long *index)
+{
+ int ret = 0;
+
+ ret = mas_empty_area_rev(mas, min, max, size);
+ if (ret)
+ return ret;
+
+ if (mas_is_err(mas))
+ return xa_err(mas->node);
+
+ if (mas->offset == MAPLE_NODE_SLOTS)
+ goto no_gap;
+
+ mas_fill_gap(mas, entry, mas->offset, size, index);
+ return 0;
+
+no_gap:
+ return -EBUSY;
+}
+
+/*
+ * mas_dead_leaves() - Mark all leaves of a node as dead.
+ * @mas: The maple state
+ * @slots: Pointer to the slot array
+ *
+ * Must hold the write lock.
+ *
+ * Return: The number of leaves marked as dead.
+ */
+static inline
+unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots)
+{
+ struct maple_node *node;
+ enum maple_type type;
+ void *entry;
+ int offset;
+
+ for (offset = 0; offset < mt_slot_count(mas->node); offset++) {
+ entry = mas_slot_locked(mas, slots, offset);
+ type = mte_node_type(entry);
+ node = mte_to_node(entry);
+ /* Use both node and type to catch LE & BE metadata */
+ if (!node || !type)
+ break;
+
+ mte_set_node_dead(entry);
+ smp_wmb(); /* Needed for RCU */
+ node->type = type;
+ rcu_assign_pointer(slots[offset], node);
+ }
+
+ return offset;
+}
+
+static void __rcu **mas_dead_walk(struct ma_state *mas, unsigned char offset)
+{
+ struct maple_node *node, *next;
+ void __rcu **slots = NULL;
+
+ next = mas_mn(mas);
+ do {
+ mas->node = ma_enode_ptr(next);
+ node = mas_mn(mas);
+ slots = ma_slots(node, node->type);
+ next = mas_slot_locked(mas, slots, offset);
+ offset = 0;
+ } while (!ma_is_leaf(next->type));
+
+ return slots;
+}
+
+static void mt_free_walk(struct rcu_head *head)
+{
+ void __rcu **slots;
+ struct maple_node *node, *start;
+ struct maple_tree mt;
+ unsigned char offset;
+ enum maple_type type;
+ MA_STATE(mas, &mt, 0, 0);
+
+ node = container_of(head, struct maple_node, rcu);
+
+ if (ma_is_leaf(node->type))
+ goto free_leaf;
+
+ mt_init_flags(&mt, node->ma_flags);
+ mas_lock(&mas);
+ start = node;
+ mas.node = mt_mk_node(node, node->type);
+ slots = mas_dead_walk(&mas, 0);
+ node = mas_mn(&mas);
+ do {
+ mt_free_bulk(node->slot_len, slots);
+ offset = node->parent_slot + 1;
+ mas.node = node->piv_parent;
+ if (mas_mn(&mas) == node)
+ goto start_slots_free;
+
+ type = mte_node_type(mas.node);
+ slots = ma_slots(mte_to_node(mas.node), type);
+ if ((offset < mt_slots[type]) && (slots[offset]))
+ slots = mas_dead_walk(&mas, offset);
+
+ node = mas_mn(&mas);
+ } while ((node != start) || (node->slot_len < offset));
+
+ slots = ma_slots(node, node->type);
+ mt_free_bulk(node->slot_len, slots);
+
+start_slots_free:
+ mas_unlock(&mas);
+free_leaf:
+ mt_free_rcu(&node->rcu);
+}
+
+static inline void __rcu **mas_destroy_descend(struct ma_state *mas,
+ struct maple_enode *prev, unsigned char offset)
+{
+ struct maple_node *node;
+ struct maple_enode *next = mas->node;
+ void __rcu **slots = NULL;
+
+ do {
+ mas->node = next;
+ node = mas_mn(mas);
+ slots = ma_slots(node, mte_node_type(mas->node));
+ next = mas_slot_locked(mas, slots, 0);
+ if ((mte_dead_node(next)))
+ next = mas_slot_locked(mas, slots, 1);
+
+ mte_set_node_dead(mas->node);
+ node->type = mte_node_type(mas->node);
+ node->piv_parent = prev;
+ node->parent_slot = offset;
+ offset = 0;
+ prev = mas->node;
+ } while (!mte_is_leaf(next));
+
+ return slots;
+}
+
+static void mt_destroy_walk(struct maple_enode *enode, unsigned char ma_flags,
+ bool free)
+{
+ void __rcu **slots;
+ struct maple_node *node = mte_to_node(enode);
+ struct maple_enode *start;
+ struct maple_tree mt;
+
+ MA_STATE(mas, &mt, 0, 0);
+
+ if (mte_is_leaf(enode))
+ goto free_leaf;
+
+ mt_init_flags(&mt, ma_flags);
+ mas_lock(&mas);
+
+ mas.node = start = enode;
+ slots = mas_destroy_descend(&mas, start, 0);
+ node = mas_mn(&mas);
+ do {
+ enum maple_type type;
+ unsigned char offset;
+ struct maple_enode *parent, *tmp;
+
+ node->slot_len = mas_dead_leaves(&mas, slots);
+ if (free)
+ mt_free_bulk(node->slot_len, slots);
+ offset = node->parent_slot + 1;
+ mas.node = node->piv_parent;
+ if (mas_mn(&mas) == node)
+ goto start_slots_free;
+
+ type = mte_node_type(mas.node);
+ slots = ma_slots(mte_to_node(mas.node), type);
+ if (offset >= mt_slots[type])
+ goto next;
+
+ tmp = mas_slot_locked(&mas, slots, offset);
+ if (mte_node_type(tmp) && mte_to_node(tmp)) {
+ parent = mas.node;
+ mas.node = tmp;
+ slots = mas_destroy_descend(&mas, parent, offset);
+ }
+next:
+ node = mas_mn(&mas);
+ } while (start != mas.node);
+
+ node = mas_mn(&mas);
+ node->slot_len = mas_dead_leaves(&mas, slots);
+ if (free)
+ mt_free_bulk(node->slot_len, slots);
+
+start_slots_free:
+ mas_unlock(&mas);
+
+free_leaf:
+ if (free)
+ mt_free_rcu(&node->rcu);
+}
+
+/*
+ * mte_destroy_walk() - Free a tree or sub-tree.
+ * @enode - the encoded maple node (maple_enode) to start
+ * @mn - the tree to free - needed for node types.
+ *
+ * Must hold the write lock.
+ */
+static inline void mte_destroy_walk(struct maple_enode *enode,
+ struct maple_tree *mt)
+{
+ struct maple_node *node = mte_to_node(enode);
+
+ if (mt_in_rcu(mt)) {
+ mt_destroy_walk(enode, mt->ma_flags, false);
+ call_rcu(&node->rcu, mt_free_walk);
+ } else {
+ mt_destroy_walk(enode, mt->ma_flags, true);
+ }
+}
+
+static void mas_wr_store_setup(struct ma_wr_state *wr_mas)
+{
+ if (!mas_is_start(wr_mas->mas)) {
+ if (mas_is_none(wr_mas->mas)) {
+ mas_reset(wr_mas->mas);
+ } else {
+ wr_mas->r_max = wr_mas->mas->max;
+ wr_mas->type = mte_node_type(wr_mas->mas->node);
+ if (mas_is_span_wr(wr_mas))
+ mas_reset(wr_mas->mas);
+ }
+ }
+
+}
+
+/* Interface */
+
+/**
+ * mas_store() - Store an @entry.
+ * @mas: The maple state.
+ * @entry: The entry to store.
+ *
+ * The @mas->index and @mas->last is used to set the range for the @entry.
+ * Note: The @mas should have pre-allocated entries to ensure there is memory to
+ * store the entry. Please see mas_expected_entries()/mas_destroy() for more details.
+ *
+ * Return: the first entry between mas->index and mas->last or %NULL.
+ */
+void *mas_store(struct ma_state *mas, void *entry)
+{
+ MA_WR_STATE(wr_mas, mas, entry);
+
+ trace_ma_write(__func__, mas, 0, entry);
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+ if (mas->index > mas->last)
+ pr_err("Error %lu > %lu %p\n", mas->index, mas->last, entry);
+ MT_BUG_ON(mas->tree, mas->index > mas->last);
+ if (mas->index > mas->last) {
+ mas_set_err(mas, -EINVAL);
+ return NULL;
+ }
+
+#endif
+
+ /*
+ * Storing is the same operation as insert with the added caveat that it
+ * can overwrite entries. Although this seems simple enough, one may
+ * want to examine what happens if a single store operation was to
+ * overwrite multiple entries within a self-balancing B-Tree.
+ */
+ mas_wr_store_setup(&wr_mas);
+ mas_wr_store_entry(&wr_mas);
+ return wr_mas.content;
+}
+
+/**
+ * mas_store_gfp() - Store a value into the tree.
+ * @mas: The maple state
+ * @entry: The entry to store
+ * @gfp: The GFP_FLAGS to use for allocations if necessary.
+ *
+ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not
+ * be allocated.
+ */
+int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp)
+{
+ MA_WR_STATE(wr_mas, mas, entry);
+
+ mas_wr_store_setup(&wr_mas);
+ trace_ma_write(__func__, mas, 0, entry);
+retry:
+ mas_wr_store_entry(&wr_mas);
+ if (unlikely(mas_nomem(mas, gfp)))
+ goto retry;
+
+ if (unlikely(mas_is_err(mas)))
+ return xa_err(mas->node);
+
+ return 0;
+}
+
+/**
+ * mas_store_prealloc() - Store a value into the tree using memory
+ * preallocated in the maple state.
+ * @mas: The maple state
+ * @entry: The entry to store.
+ */
+void mas_store_prealloc(struct ma_state *mas, void *entry)
+{
+ MA_WR_STATE(wr_mas, mas, entry);
+
+ mas_wr_store_setup(&wr_mas);
+ trace_ma_write(__func__, mas, 0, entry);
+ mas_wr_store_entry(&wr_mas);
+ BUG_ON(mas_is_err(mas));
+ mas_destroy(mas);
+}
+
+/**
+ * mas_preallocate() - Preallocate enough nodes for a store operation
+ * @mas: The maple state
+ * @entry: The entry that will be stored
+ * @gfp: The GFP_FLAGS to use for allocations.
+ *
+ * Return: 0 on success, -ENOMEM if memory could not be allocated.
+ */
+int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp)
+{
+ int ret;
+
+ mas_node_count_gfp(mas, 1 + mas_mt_height(mas) * 3, gfp);
+ mas->mas_flags |= MA_STATE_PREALLOC;
+ if (likely(!mas_is_err(mas)))
+ return 0;
+
+ mas_set_alloc_req(mas, 0);
+ ret = xa_err(mas->node);
+ mas_reset(mas);
+ mas_destroy(mas);
+ mas_reset(mas);
+ return ret;
+}
+
+/*
+ * mas_destroy() - destroy a maple state.
+ * @mas: The maple state
+ *
+ * Upon completion, check the left-most node and rebalance against the node to
+ * the right if necessary. Frees any allocated nodes associated with this maple
+ * state.
+ */
+void mas_destroy(struct ma_state *mas)
+{
+ struct maple_alloc *node;
+
+ /*
+ * When using mas_for_each() to insert an expected number of elements,
+ * it is possible that the number inserted is less than the expected
+ * number. To fix an invalid final node, a check is performed here to
+ * rebalance the previous node with the final node.
+ */
+ if (mas->mas_flags & MA_STATE_REBALANCE) {
+ unsigned char end;
+
+ if (mas_is_start(mas))
+ mas_start(mas);
+
+ mtree_range_walk(mas);
+ end = mas_data_end(mas) + 1;
+ if (end < mt_min_slot_count(mas->node) - 1)
+ mas_destroy_rebalance(mas, end);
+
+ mas->mas_flags &= ~MA_STATE_REBALANCE;
+ }
+ mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC);
+
+ while (mas->alloc && !((unsigned long)mas->alloc & 0x1)) {
+ node = mas->alloc;
+ mas->alloc = node->slot[0];
+ if (node->node_count > 0)
+ mt_free_bulk(node->node_count,
+ (void __rcu **)&node->slot[1]);
+ kmem_cache_free(maple_node_cache, node);
+ }
+ mas->alloc = NULL;
+}
+
+/*
+ * mas_expected_entries() - Set the expected number of entries that will be inserted.
+ * @mas: The maple state
+ * @nr_entries: The number of expected entries.
+ *
+ * This will attempt to pre-allocate enough nodes to store the expected number
+ * of entries. The allocations will occur using the bulk allocator interface
+ * for speed. Please call mas_destroy() on the @mas after inserting the entries
+ * to ensure any unused nodes are freed.
+ *
+ * Return: 0 on success, -ENOMEM if memory could not be allocated.
+ */
+int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries)
+{
+ int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2;
+ struct maple_enode *enode = mas->node;
+ int nr_nodes;
+ int ret;
+
+ /*
+ * Sometimes it is necessary to duplicate a tree to a new tree, such as
+ * forking a process and duplicating the VMAs from one tree to a new
+ * tree. When such a situation arises, it is known that the new tree is
+ * not going to be used until the entire tree is populated. For
+ * performance reasons, it is best to use a bulk load with RCU disabled.
+ * This allows for optimistic splitting that favours the left and reuse
+ * of nodes during the operation.
+ */
+
+ /* Optimize splitting for bulk insert in-order */
+ mas->mas_flags |= MA_STATE_BULK;
+
+ /*
+ * Avoid overflow, assume a gap between each entry and a trailing null.
+ * If this is wrong, it just means allocation can happen during
+ * insertion of entries.
+ */
+ nr_nodes = max(nr_entries, nr_entries * 2 + 1);
+ if (!mt_is_alloc(mas->tree))
+ nonleaf_cap = MAPLE_RANGE64_SLOTS - 2;
+
+ /* Leaves; reduce slots to keep space for expansion */
+ nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2);
+ /* Internal nodes */
+ nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap);
+ /* Add working room for split (2 nodes) + new parents */
+ mas_node_count(mas, nr_nodes + 3);
+
+ /* Detect if allocations run out */
+ mas->mas_flags |= MA_STATE_PREALLOC;
+
+ if (!mas_is_err(mas))
+ return 0;
+
+ ret = xa_err(mas->node);
+ mas->node = enode;
+ mas_destroy(mas);
+ return ret;
+
+}
+
+/**
+ * mas_next() - Get the next entry.
+ * @mas: The maple state
+ * @max: The maximum index to check.
+ *
+ * Returns the next entry after @mas->index.
+ * Must hold rcu_read_lock or the write lock.
+ * Can return the zero entry.
+ *
+ * Return: The next entry or %NULL
+ */
+void *mas_next(struct ma_state *mas, unsigned long max)
+{
+ if (mas_is_none(mas) || mas_is_paused(mas))
+ mas->node = MAS_START;
+
+ if (mas_is_start(mas))
+ mas_walk(mas); /* Retries on dead nodes handled by mas_walk */
+
+ if (mas_is_ptr(mas)) {
+ if (!mas->index) {
+ mas->index = 1;
+ mas->last = ULONG_MAX;
+ }
+ return NULL;
+ }
+
+ if (mas->last == ULONG_MAX)
+ return NULL;
+
+ /* Retries on dead nodes handled by mas_next_entry */
+ return mas_next_entry(mas, max);
+}
+EXPORT_SYMBOL_GPL(mas_next);
+
+/**
+ * mt_next() - get the next value in the maple tree
+ * @mt: The maple tree
+ * @index: The start index
+ * @max: The maximum index to check
+ *
+ * Return: The entry at @index or higher, or %NULL if nothing is found.
+ */
+void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max)
+{
+ void *entry = NULL;
+ MA_STATE(mas, mt, index, index);
+
+ rcu_read_lock();
+ entry = mas_next(&mas, max);
+ rcu_read_unlock();
+ return entry;
+}
+EXPORT_SYMBOL_GPL(mt_next);
+
+/**
+ * mas_prev() - Get the previous entry
+ * @mas: The maple state
+ * @min: The minimum value to check.
+ *
+ * Must hold rcu_read_lock or the write lock.
+ * Will reset mas to MAS_START if the node is MAS_NONE. Will stop on not
+ * searchable nodes.
+ *
+ * Return: the previous value or %NULL.
+ */
+void *mas_prev(struct ma_state *mas, unsigned long min)
+{
+ if (!mas->index) {
+ /* Nothing comes before 0 */
+ mas->last = 0;
+ return NULL;
+ }
+
+ if (unlikely(mas_is_ptr(mas)))
+ return NULL;
+
+ if (mas_is_none(mas) || mas_is_paused(mas))
+ mas->node = MAS_START;
+
+ if (mas_is_start(mas)) {
+ mas_walk(mas);
+ if (!mas->index)
+ return NULL;
+ }
+
+ if (mas_is_ptr(mas)) {
+ if (!mas->index) {
+ mas->last = 0;
+ return NULL;
+ }
+
+ mas->index = mas->last = 0;
+ return mas_root_locked(mas);
+ }
+ return mas_prev_entry(mas, min);
+}
+EXPORT_SYMBOL_GPL(mas_prev);
+
+/**
+ * mt_prev() - get the previous value in the maple tree
+ * @mt: The maple tree
+ * @index: The start index
+ * @min: The minimum index to check
+ *
+ * Return: The entry at @index or lower, or %NULL if nothing is found.
+ */
+void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min)
+{
+ void *entry = NULL;
+ MA_STATE(mas, mt, index, index);
+
+ rcu_read_lock();
+ entry = mas_prev(&mas, min);
+ rcu_read_unlock();
+ return entry;
+}
+EXPORT_SYMBOL_GPL(mt_prev);
+
+/**
+ * mas_pause() - Pause a mas_find/mas_for_each to drop the lock.
+ * @mas: The maple state to pause
+ *
+ * Some users need to pause a walk and drop the lock they're holding in
+ * order to yield to a higher priority thread or carry out an operation
+ * on an entry. Those users should call this function before they drop
+ * the lock. It resets the @mas to be suitable for the next iteration
+ * of the loop after the user has reacquired the lock. If most entries
+ * found during a walk require you to call mas_pause(), the mt_for_each()
+ * iterator may be more appropriate.
+ *
+ */
+void mas_pause(struct ma_state *mas)
+{
+ mas->node = MAS_PAUSE;
+}
+EXPORT_SYMBOL_GPL(mas_pause);
+
+/**
+ * mas_find() - On the first call, find the entry at or after mas->index up to
+ * %max. Otherwise, find the entry after mas->index.
+ * @mas: The maple state
+ * @max: The maximum value to check.
+ *
+ * Must hold rcu_read_lock or the write lock.
+ * If an entry exists, last and index are updated accordingly.
+ * May set @mas->node to MAS_NONE.
+ *
+ * Return: The entry or %NULL.
+ */
+void *mas_find(struct ma_state *mas, unsigned long max)
+{
+ if (unlikely(mas_is_paused(mas))) {
+ if (unlikely(mas->last == ULONG_MAX)) {
+ mas->node = MAS_NONE;
+ return NULL;
+ }
+ mas->node = MAS_START;
+ mas->index = ++mas->last;
+ }
+
+ if (unlikely(mas_is_start(mas))) {
+ /* First run or continue */
+ void *entry;
+
+ if (mas->index > max)
+ return NULL;
+
+ entry = mas_walk(mas);
+ if (entry)
+ return entry;
+ }
+
+ if (unlikely(!mas_searchable(mas)))
+ return NULL;
+
+ /* Retries on dead nodes handled by mas_next_entry */
+ return mas_next_entry(mas, max);
+}
+
+/**
+ * mas_find_rev: On the first call, find the first non-null entry at or below
+ * mas->index down to %min. Otherwise find the first non-null entry below
+ * mas->index down to %min.
+ * @mas: The maple state
+ * @min: The minimum value to check.
+ *
+ * Must hold rcu_read_lock or the write lock.
+ * If an entry exists, last and index are updated accordingly.
+ * May set @mas->node to MAS_NONE.
+ *
+ * Return: The entry or %NULL.
+ */
+void *mas_find_rev(struct ma_state *mas, unsigned long min)
+{
+ if (unlikely(mas_is_paused(mas))) {
+ if (unlikely(mas->last == ULONG_MAX)) {
+ mas->node = MAS_NONE;
+ return NULL;
+ }
+ mas->node = MAS_START;
+ mas->last = --mas->index;
+ }
+
+ if (unlikely(mas_is_start(mas))) {
+ /* First run or continue */
+ void *entry;
+
+ if (mas->index < min)
+ return NULL;
+
+ entry = mas_walk(mas);
+ if (entry)
+ return entry;
+ }
+
+ if (unlikely(!mas_searchable(mas)))
+ return NULL;
+
+ if (mas->index < min)
+ return NULL;
+
+ /* Retries on dead nodes handled by mas_next_entry */
+ return mas_prev_entry(mas, min);
+}
+EXPORT_SYMBOL_GPL(mas_find);
+
+/**
+ * mas_erase() - Find the range in which index resides and erase the entire
+ * range.
+ * @mas: The maple state
+ *
+ * Must hold the write lock.
+ * Searches for @mas->index, sets @mas->index and @mas->last to the range and
+ * erases that range.
+ *
+ * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated.
+ */
+void *mas_erase(struct ma_state *mas)
+{
+ void *entry;
+ MA_WR_STATE(wr_mas, mas, NULL);
+
+ if (mas_is_none(mas) || mas_is_paused(mas))
+ mas->node = MAS_START;
+
+ /* Retry unnecessary when holding the write lock. */
+ entry = mas_state_walk(mas);
+ if (!entry)
+ return NULL;
+
+write_retry:
+ /* Must reset to ensure spanning writes of last slot are detected */
+ mas_reset(mas);
+ mas_wr_store_setup(&wr_mas);
+ mas_wr_store_entry(&wr_mas);
+ if (mas_nomem(mas, GFP_KERNEL))
+ goto write_retry;
+
+ return entry;
+}
+EXPORT_SYMBOL_GPL(mas_erase);
+
+/**
+ * mas_nomem() - Check if there was an error allocating and do the allocation
+ * if necessary If there are allocations, then free them.
+ * @mas: The maple state
+ * @gfp: The GFP_FLAGS to use for allocations
+ * Return: true on allocation, false otherwise.
+ */
+bool mas_nomem(struct ma_state *mas, gfp_t gfp)
+ __must_hold(mas->tree->lock)
+{
+ if (likely(mas->node != MA_ERROR(-ENOMEM))) {
+ mas_destroy(mas);
+ return false;
+ }
+
+ if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) {
+ mtree_unlock(mas->tree);
+ mas_alloc_nodes(mas, gfp);
+ mtree_lock(mas->tree);
+ } else {
+ mas_alloc_nodes(mas, gfp);
+ }
+
+ if (!mas_allocated(mas))
+ return false;
+
+ mas->node = MAS_START;
+ return true;
+}
+
+void __init maple_tree_init(void)
+{
+ maple_node_cache = kmem_cache_create("maple_node",
+ sizeof(struct maple_node), sizeof(struct maple_node),
+ SLAB_PANIC, NULL);
+}
+
+/**
+ * mtree_load() - Load a value stored in a maple tree
+ * @mt: The maple tree
+ * @index: The index to load
+ *
+ * Return: the entry or %NULL
+ */
+void *mtree_load(struct maple_tree *mt, unsigned long index)
+{
+ MA_STATE(mas, mt, index, index);
+ void *entry;
+
+ trace_ma_read(__func__, &mas);
+ rcu_read_lock();
+retry:
+ entry = mas_start(&mas);
+ if (unlikely(mas_is_none(&mas)))
+ goto unlock;
+
+ if (unlikely(mas_is_ptr(&mas))) {
+ if (index)
+ entry = NULL;
+
+ goto unlock;
+ }
+
+ entry = mtree_lookup_walk(&mas);
+ if (!entry && unlikely(mas_is_start(&mas)))
+ goto retry;
+unlock:
+ rcu_read_unlock();
+ if (xa_is_zero(entry))
+ return NULL;
+
+ return entry;
+}
+EXPORT_SYMBOL(mtree_load);
+
+/**
+ * mtree_store_range() - Store an entry at a given range.
+ * @mt: The maple tree
+ * @index: The start of the range
+ * @last: The end of the range
+ * @entry: The entry to store
+ * @gfp: The GFP_FLAGS to use for allocations
+ *
+ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not
+ * be allocated.
+ */
+int mtree_store_range(struct maple_tree *mt, unsigned long index,
+ unsigned long last, void *entry, gfp_t gfp)
+{
+ MA_STATE(mas, mt, index, last);
+ MA_WR_STATE(wr_mas, &mas, entry);
+
+ trace_ma_write(__func__, &mas, 0, entry);
+ if (WARN_ON_ONCE(xa_is_advanced(entry)))
+ return -EINVAL;
+
+ if (index > last)
+ return -EINVAL;
+
+ mtree_lock(mt);
+retry:
+ mas_wr_store_entry(&wr_mas);
+ if (mas_nomem(&mas, gfp))
+ goto retry;
+
+ mtree_unlock(mt);
+ if (mas_is_err(&mas))
+ return xa_err(mas.node);
+
+ return 0;
+}
+EXPORT_SYMBOL(mtree_store_range);
+
+/**
+ * mtree_store() - Store an entry at a given index.
+ * @mt: The maple tree
+ * @index: The index to store the value
+ * @entry: The entry to store
+ * @gfp: The GFP_FLAGS to use for allocations
+ *
+ * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not
+ * be allocated.
+ */
+int mtree_store(struct maple_tree *mt, unsigned long index, void *entry,
+ gfp_t gfp)
+{
+ return mtree_store_range(mt, index, index, entry, gfp);
+}
+EXPORT_SYMBOL(mtree_store);
+
+/**
+ * mtree_insert_range() - Insert an entry at a give range if there is no value.
+ * @mt: The maple tree
+ * @first: The start of the range
+ * @last: The end of the range
+ * @entry: The entry to store
+ * @gfp: The GFP_FLAGS to use for allocations.
+ *
+ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid
+ * request, -ENOMEM if memory could not be allocated.
+ */
+int mtree_insert_range(struct maple_tree *mt, unsigned long first,
+ unsigned long last, void *entry, gfp_t gfp)
+{
+ MA_STATE(ms, mt, first, last);
+
+ if (WARN_ON_ONCE(xa_is_advanced(entry)))
+ return -EINVAL;
+
+ if (first > last)
+ return -EINVAL;
+
+ mtree_lock(mt);
+retry:
+ mas_insert(&ms, entry);
+ if (mas_nomem(&ms, gfp))
+ goto retry;
+
+ mtree_unlock(mt);
+ if (mas_is_err(&ms))
+ return xa_err(ms.node);
+
+ return 0;
+}
+EXPORT_SYMBOL(mtree_insert_range);
+
+/**
+ * mtree_insert() - Insert an entry at a give index if there is no value.
+ * @mt: The maple tree
+ * @index : The index to store the value
+ * @entry: The entry to store
+ * @gfp: The FGP_FLAGS to use for allocations.
+ *
+ * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid
+ * request, -ENOMEM if memory could not be allocated.
+ */
+int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry,
+ gfp_t gfp)
+{
+ return mtree_insert_range(mt, index, index, entry, gfp);
+}
+EXPORT_SYMBOL(mtree_insert);
+
+int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp,
+ void *entry, unsigned long size, unsigned long min,
+ unsigned long max, gfp_t gfp)
+{
+ int ret = 0;
+
+ MA_STATE(mas, mt, min, max - size);
+ if (!mt_is_alloc(mt))
+ return -EINVAL;
+
+ if (WARN_ON_ONCE(mt_is_reserved(entry)))
+ return -EINVAL;
+
+ if (min > max)
+ return -EINVAL;
+
+ if (max < size)
+ return -EINVAL;
+
+ if (!size)
+ return -EINVAL;
+
+ mtree_lock(mt);
+retry:
+ mas.offset = 0;
+ mas.index = min;
+ mas.last = max - size;
+ ret = mas_alloc(&mas, entry, size, startp);
+ if (mas_nomem(&mas, gfp))
+ goto retry;
+
+ mtree_unlock(mt);
+ return ret;
+}
+EXPORT_SYMBOL(mtree_alloc_range);
+
+int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp,
+ void *entry, unsigned long size, unsigned long min,
+ unsigned long max, gfp_t gfp)
+{
+ int ret = 0;
+
+ MA_STATE(mas, mt, min, max - size);
+ if (!mt_is_alloc(mt))
+ return -EINVAL;
+
+ if (WARN_ON_ONCE(mt_is_reserved(entry)))
+ return -EINVAL;
+
+ if (min >= max)
+ return -EINVAL;
+
+ if (max < size - 1)
+ return -EINVAL;
+
+ if (!size)
+ return -EINVAL;
+
+ mtree_lock(mt);
+retry:
+ ret = mas_rev_alloc(&mas, min, max, entry, size, startp);
+ if (mas_nomem(&mas, gfp))
+ goto retry;
+
+ mtree_unlock(mt);
+ return ret;
+}
+EXPORT_SYMBOL(mtree_alloc_rrange);
+
+/**
+ * mtree_erase() - Find an index and erase the entire range.
+ * @mt: The maple tree
+ * @index: The index to erase
+ *
+ * Erasing is the same as a walk to an entry then a store of a NULL to that
+ * ENTIRE range. In fact, it is implemented as such using the advanced API.
+ *
+ * Return: The entry stored at the @index or %NULL
+ */
+void *mtree_erase(struct maple_tree *mt, unsigned long index)
+{
+ void *entry = NULL;
+
+ MA_STATE(mas, mt, index, index);
+ trace_ma_op(__func__, &mas);
+
+ mtree_lock(mt);
+ entry = mas_erase(&mas);
+ mtree_unlock(mt);
+
+ return entry;
+}
+EXPORT_SYMBOL(mtree_erase);
+
+/**
+ * __mt_destroy() - Walk and free all nodes of a locked maple tree.
+ * @mt: The maple tree
+ *
+ * Note: Does not handle locking.
+ */
+void __mt_destroy(struct maple_tree *mt)
+{
+ void *root = mt_root_locked(mt);
+
+ rcu_assign_pointer(mt->ma_root, NULL);
+ if (xa_is_node(root))
+ mte_destroy_walk(root, mt);
+
+ mt->ma_flags = 0;
+}
+EXPORT_SYMBOL_GPL(__mt_destroy);
+
+/**
+ * mtree_destroy() - Destroy a maple tree
+ * @mt: The maple tree
+ *
+ * Frees all resources used by the tree. Handles locking.
+ */
+void mtree_destroy(struct maple_tree *mt)
+{
+ mtree_lock(mt);
+ __mt_destroy(mt);
+ mtree_unlock(mt);
+}
+EXPORT_SYMBOL(mtree_destroy);
+
+/**
+ * mt_find() - Search from the start up until an entry is found.
+ * @mt: The maple tree
+ * @index: Pointer which contains the start location of the search
+ * @max: The maximum value to check
+ *
+ * Handles locking. @index will be incremented to one beyond the range.
+ *
+ * Return: The entry at or after the @index or %NULL
+ */
+void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max)
+{
+ MA_STATE(mas, mt, *index, *index);
+ void *entry;
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+ unsigned long copy = *index;
+#endif
+
+ trace_ma_read(__func__, &mas);
+
+ if ((*index) > max)
+ return NULL;
+
+ rcu_read_lock();
+retry:
+ entry = mas_state_walk(&mas);
+ if (mas_is_start(&mas))
+ goto retry;
+
+ if (unlikely(xa_is_zero(entry)))
+ entry = NULL;
+
+ if (entry)
+ goto unlock;
+
+ while (mas_searchable(&mas) && (mas.index < max)) {
+ entry = mas_next_entry(&mas, max);
+ if (likely(entry && !xa_is_zero(entry)))
+ break;
+ }
+
+ if (unlikely(xa_is_zero(entry)))
+ entry = NULL;
+unlock:
+ rcu_read_unlock();
+ if (likely(entry)) {
+ *index = mas.last + 1;
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+ if ((*index) && (*index) <= copy)
+ pr_err("index not increased! %lx <= %lx\n",
+ *index, copy);
+ MT_BUG_ON(mt, (*index) && ((*index) <= copy));
+#endif
+ }
+
+ return entry;
+}
+EXPORT_SYMBOL(mt_find);
+
+/**
+ * mt_find_after() - Search from the start up until an entry is found.
+ * @mt: The maple tree
+ * @index: Pointer which contains the start location of the search
+ * @max: The maximum value to check
+ *
+ * Handles locking, detects wrapping on index == 0
+ *
+ * Return: The entry at or after the @index or %NULL
+ */
+void *mt_find_after(struct maple_tree *mt, unsigned long *index,
+ unsigned long max)
+{
+ if (!(*index))
+ return NULL;
+
+ return mt_find(mt, index, max);
+}
+EXPORT_SYMBOL(mt_find_after);
+
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+atomic_t maple_tree_tests_run;
+EXPORT_SYMBOL_GPL(maple_tree_tests_run);
+atomic_t maple_tree_tests_passed;
+EXPORT_SYMBOL_GPL(maple_tree_tests_passed);
+
+#ifndef __KERNEL__
+extern void kmem_cache_set_non_kernel(struct kmem_cache *, unsigned int);
+void mt_set_non_kernel(unsigned int val)
+{
+ kmem_cache_set_non_kernel(maple_node_cache, val);
+}
+
+extern unsigned long kmem_cache_get_alloc(struct kmem_cache *);
+unsigned long mt_get_alloc_size(void)
+{
+ return kmem_cache_get_alloc(maple_node_cache);
+}
+
+extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *);
+void mt_zero_nr_tallocated(void)
+{
+ kmem_cache_zero_nr_tallocated(maple_node_cache);
+}
+
+extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *);
+unsigned int mt_nr_tallocated(void)
+{
+ return kmem_cache_nr_tallocated(maple_node_cache);
+}
+
+extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *);
+unsigned int mt_nr_allocated(void)
+{
+ return kmem_cache_nr_allocated(maple_node_cache);
+}
+
+/*
+ * mas_dead_node() - Check if the maple state is pointing to a dead node.
+ * @mas: The maple state
+ * @index: The index to restore in @mas.
+ *
+ * Used in test code.
+ * Return: 1 if @mas has been reset to MAS_START, 0 otherwise.
+ */
+static inline int mas_dead_node(struct ma_state *mas, unsigned long index)
+{
+ if (unlikely(!mas_searchable(mas) || mas_is_start(mas)))
+ return 0;
+
+ if (likely(!mte_dead_node(mas->node)))
+ return 0;
+
+ mas_rewalk(mas, index);
+ return 1;
+}
+#endif /* not defined __KERNEL__ */
+
+/*
+ * mas_get_slot() - Get the entry in the maple state node stored at @offset.
+ * @mas: The maple state
+ * @offset: The offset into the slot array to fetch.
+ *
+ * Return: The entry stored at @offset.
+ */
+static inline struct maple_enode *mas_get_slot(struct ma_state *mas,
+ unsigned char offset)
+{
+ return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)),
+ offset);
+}
+
+
+/*
+ * mas_first_entry() - Go the first leaf and find the first entry.
+ * @mas: the maple state.
+ * @limit: the maximum index to check.
+ * @*r_start: Pointer to set to the range start.
+ *
+ * Sets mas->offset to the offset of the entry, r_start to the range minimum.
+ *
+ * Return: The first entry or MAS_NONE.
+ */
+static inline void *mas_first_entry(struct ma_state *mas, struct maple_node *mn,
+ unsigned long limit, enum maple_type mt)
+
+{
+ unsigned long max;
+ unsigned long *pivots;
+ void __rcu **slots;
+ void *entry = NULL;
+
+ mas->index = mas->min;
+ if (mas->index > limit)
+ goto none;
+
+ max = mas->max;
+ mas->offset = 0;
+ while (likely(!ma_is_leaf(mt))) {
+ MT_BUG_ON(mas->tree, mte_dead_node(mas->node));
+ slots = ma_slots(mn, mt);
+ pivots = ma_pivots(mn, mt);
+ max = pivots[0];
+ entry = mas_slot(mas, slots, 0);
+ if (unlikely(ma_dead_node(mn)))
+ return NULL;
+ mas->node = entry;
+ mn = mas_mn(mas);
+ mt = mte_node_type(mas->node);
+ }
+ MT_BUG_ON(mas->tree, mte_dead_node(mas->node));
+
+ mas->max = max;
+ slots = ma_slots(mn, mt);
+ entry = mas_slot(mas, slots, 0);
+ if (unlikely(ma_dead_node(mn)))
+ return NULL;
+
+ /* Slot 0 or 1 must be set */
+ if (mas->index > limit)
+ goto none;
+
+ if (likely(entry))
+ return entry;
+
+ pivots = ma_pivots(mn, mt);
+ mas->index = pivots[0] + 1;
+ mas->offset = 1;
+ entry = mas_slot(mas, slots, 1);
+ if (unlikely(ma_dead_node(mn)))
+ return NULL;
+
+ if (mas->index > limit)
+ goto none;
+
+ if (likely(entry))
+ return entry;
+
+none:
+ if (likely(!ma_dead_node(mn)))
+ mas->node = MAS_NONE;
+ return NULL;
+}
+
+/* Depth first search, post-order */
+static void mas_dfs_postorder(struct ma_state *mas, unsigned long max)
+{
+
+ struct maple_enode *p = MAS_NONE, *mn = mas->node;
+ unsigned long p_min, p_max;
+
+ mas_next_node(mas, mas_mn(mas), max);
+ if (!mas_is_none(mas))
+ return;
+
+ if (mte_is_root(mn))
+ return;
+
+ mas->node = mn;
+ mas_ascend(mas);
+ while (mas->node != MAS_NONE) {
+ p = mas->node;
+ p_min = mas->min;
+ p_max = mas->max;
+ mas_prev_node(mas, 0);
+ }
+
+ if (p == MAS_NONE)
+ return;
+
+ mas->node = p;
+ mas->max = p_max;
+ mas->min = p_min;
+}
+
+/* Tree validations */
+static void mt_dump_node(const struct maple_tree *mt, void *entry,
+ unsigned long min, unsigned long max, unsigned int depth);
+static void mt_dump_range(unsigned long min, unsigned long max,
+ unsigned int depth)
+{
+ static const char spaces[] = " ";
+
+ if (min == max)
+ pr_info("%.*s%lu: ", depth * 2, spaces, min);
+ else
+ pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max);
+}
+
+static void mt_dump_entry(void *entry, unsigned long min, unsigned long max,
+ unsigned int depth)
+{
+ mt_dump_range(min, max, depth);
+
+ if (xa_is_value(entry))
+ pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry),
+ xa_to_value(entry), entry);
+ else if (xa_is_zero(entry))
+ pr_cont("zero (%ld)\n", xa_to_internal(entry));
+ else if (mt_is_reserved(entry))
+ pr_cont("UNKNOWN ENTRY (%p)\n", entry);
+ else
+ pr_cont("%p\n", entry);
+}
+
+static void mt_dump_range64(const struct maple_tree *mt, void *entry,
+ unsigned long min, unsigned long max, unsigned int depth)
+{
+ struct maple_range_64 *node = &mte_to_node(entry)->mr64;
+ bool leaf = mte_is_leaf(entry);
+ unsigned long first = min;
+ int i;
+
+ pr_cont(" contents: ");
+ for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++)
+ pr_cont("%p %lu ", node->slot[i], node->pivot[i]);
+ pr_cont("%p\n", node->slot[i]);
+ for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) {
+ unsigned long last = max;
+
+ if (i < (MAPLE_RANGE64_SLOTS - 1))
+ last = node->pivot[i];
+ else if (!node->slot[i] && max != mt_max[mte_node_type(entry)])
+ break;
+ if (last == 0 && i > 0)
+ break;
+ if (leaf)
+ mt_dump_entry(mt_slot(mt, node->slot, i),
+ first, last, depth + 1);
+ else if (node->slot[i])
+ mt_dump_node(mt, mt_slot(mt, node->slot, i),
+ first, last, depth + 1);
+
+ if (last == max)
+ break;
+ if (last > max) {
+ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n",
+ node, last, max, i);
+ break;
+ }
+ first = last + 1;
+ }
+}
+
+static void mt_dump_arange64(const struct maple_tree *mt, void *entry,
+ unsigned long min, unsigned long max, unsigned int depth)
+{
+ struct maple_arange_64 *node = &mte_to_node(entry)->ma64;
+ bool leaf = mte_is_leaf(entry);
+ unsigned long first = min;
+ int i;
+
+ pr_cont(" contents: ");
+ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++)
+ pr_cont("%lu ", node->gap[i]);
+ pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap);
+ for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++)
+ pr_cont("%p %lu ", node->slot[i], node->pivot[i]);
+ pr_cont("%p\n", node->slot[i]);
+ for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) {
+ unsigned long last = max;
+
+ if (i < (MAPLE_ARANGE64_SLOTS - 1))
+ last = node->pivot[i];
+ else if (!node->slot[i])
+ break;
+ if (last == 0 && i > 0)
+ break;
+ if (leaf)
+ mt_dump_entry(mt_slot(mt, node->slot, i),
+ first, last, depth + 1);
+ else if (node->slot[i])
+ mt_dump_node(mt, mt_slot(mt, node->slot, i),
+ first, last, depth + 1);
+
+ if (last == max)
+ break;
+ if (last > max) {
+ pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n",
+ node, last, max, i);
+ break;
+ }
+ first = last + 1;
+ }
+}
+
+static void mt_dump_node(const struct maple_tree *mt, void *entry,
+ unsigned long min, unsigned long max, unsigned int depth)
+{
+ struct maple_node *node = mte_to_node(entry);
+ unsigned int type = mte_node_type(entry);
+ unsigned int i;
+
+ mt_dump_range(min, max, depth);
+
+ pr_cont("node %p depth %d type %d parent %p", node, depth, type,
+ node ? node->parent : NULL);
+ switch (type) {
+ case maple_dense:
+ pr_cont("\n");
+ for (i = 0; i < MAPLE_NODE_SLOTS; i++) {
+ if (min + i > max)
+ pr_cont("OUT OF RANGE: ");
+ mt_dump_entry(mt_slot(mt, node->slot, i),
+ min + i, min + i, depth);
+ }
+ break;
+ case maple_leaf_64:
+ case maple_range_64:
+ mt_dump_range64(mt, entry, min, max, depth);
+ break;
+ case maple_arange_64:
+ mt_dump_arange64(mt, entry, min, max, depth);
+ break;
+
+ default:
+ pr_cont(" UNKNOWN TYPE\n");
+ }
+}
+
+void mt_dump(const struct maple_tree *mt)
+{
+ void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt));
+
+ pr_info("maple_tree(%p) flags %X, height %u root %p\n",
+ mt, mt->ma_flags, mt_height(mt), entry);
+ if (!xa_is_node(entry))
+ mt_dump_entry(entry, 0, 0, 0);
+ else if (entry)
+ mt_dump_node(mt, entry, 0, mt_max[mte_node_type(entry)], 0);
+}
+
+/*
+ * Calculate the maximum gap in a node and check if that's what is reported in
+ * the parent (unless root).
+ */
+static void mas_validate_gaps(struct ma_state *mas)
+{
+ struct maple_enode *mte = mas->node;
+ struct maple_node *p_mn;
+ unsigned long gap = 0, max_gap = 0;
+ unsigned long p_end, p_start = mas->min;
+ unsigned char p_slot;
+ unsigned long *gaps = NULL;
+ unsigned long *pivots = ma_pivots(mte_to_node(mte), mte_node_type(mte));
+ int i;
+
+ if (ma_is_dense(mte_node_type(mte))) {
+ for (i = 0; i < mt_slot_count(mte); i++) {
+ if (mas_get_slot(mas, i)) {
+ if (gap > max_gap)
+ max_gap = gap;
+ gap = 0;
+ continue;
+ }
+ gap++;
+ }
+ goto counted;
+ }
+
+ gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte));
+ for (i = 0; i < mt_slot_count(mte); i++) {
+ p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte));
+
+ if (!gaps) {
+ if (mas_get_slot(mas, i)) {
+ gap = 0;
+ goto not_empty;
+ }
+
+ gap += p_end - p_start + 1;
+ } else {
+ void *entry = mas_get_slot(mas, i);
+
+ gap = gaps[i];
+ if (!entry) {
+ if (gap != p_end - p_start + 1) {
+ pr_err("%p[%u] -> %p %lu != %lu - %lu + 1\n",
+ mas_mn(mas), i,
+ mas_get_slot(mas, i), gap,
+ p_end, p_start);
+ mt_dump(mas->tree);
+
+ MT_BUG_ON(mas->tree,
+ gap != p_end - p_start + 1);
+ }
+ } else {
+ if (gap > p_end - p_start + 1) {
+ pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n",
+ mas_mn(mas), i, gap, p_end, p_start,
+ p_end - p_start + 1);
+ MT_BUG_ON(mas->tree,
+ gap > p_end - p_start + 1);
+ }
+ }
+ }
+
+ if (gap > max_gap)
+ max_gap = gap;
+not_empty:
+ p_start = p_end + 1;
+ if (p_end >= mas->max)
+ break;
+ }
+
+counted:
+ if (mte_is_root(mte))
+ return;
+
+ p_slot = mte_parent_slot(mas->node);
+ p_mn = mte_parent(mte);
+ MT_BUG_ON(mas->tree, max_gap > mas->max);
+ if (ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap) {
+ pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap);
+ mt_dump(mas->tree);
+ }
+
+ MT_BUG_ON(mas->tree,
+ ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap);
+}
+
+static void mas_validate_parent_slot(struct ma_state *mas)
+{
+ struct maple_node *parent;
+ struct maple_enode *node;
+ enum maple_type p_type = mas_parent_enum(mas, mas->node);
+ unsigned char p_slot = mte_parent_slot(mas->node);
+ void __rcu **slots;
+ int i;
+
+ if (mte_is_root(mas->node))
+ return;
+
+ parent = mte_parent(mas->node);
+ slots = ma_slots(parent, p_type);
+ MT_BUG_ON(mas->tree, mas_mn(mas) == parent);
+
+ /* Check prev/next parent slot for duplicate node entry */
+
+ for (i = 0; i < mt_slots[p_type]; i++) {
+ node = mas_slot(mas, slots, i);
+ if (i == p_slot) {
+ if (node != mas->node)
+ pr_err("parent %p[%u] does not have %p\n",
+ parent, i, mas_mn(mas));
+ MT_BUG_ON(mas->tree, node != mas->node);
+ } else if (node == mas->node) {
+ pr_err("Invalid child %p at parent %p[%u] p_slot %u\n",
+ mas_mn(mas), parent, i, p_slot);
+ MT_BUG_ON(mas->tree, node == mas->node);
+ }
+ }
+}
+
+static void mas_validate_child_slot(struct ma_state *mas)
+{
+ enum maple_type type = mte_node_type(mas->node);
+ void __rcu **slots = ma_slots(mte_to_node(mas->node), type);
+ unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type);
+ struct maple_enode *child;
+ unsigned char i;
+
+ if (mte_is_leaf(mas->node))
+ return;
+
+ for (i = 0; i < mt_slots[type]; i++) {
+ child = mas_slot(mas, slots, i);
+ if (!pivots[i] || pivots[i] == mas->max)
+ break;
+
+ if (!child)
+ break;
+
+ if (mte_parent_slot(child) != i) {
+ pr_err("Slot error at %p[%u]: child %p has pslot %u\n",
+ mas_mn(mas), i, mte_to_node(child),
+ mte_parent_slot(child));
+ MT_BUG_ON(mas->tree, 1);
+ }
+
+ if (mte_parent(child) != mte_to_node(mas->node)) {
+ pr_err("child %p has parent %p not %p\n",
+ mte_to_node(child), mte_parent(child),
+ mte_to_node(mas->node));
+ MT_BUG_ON(mas->tree, 1);
+ }
+ }
+}
+
+/*
+ * Validate all pivots are within mas->min and mas->max.
+ */
+static void mas_validate_limits(struct ma_state *mas)
+{
+ int i;
+ unsigned long prev_piv = 0;
+ enum maple_type type = mte_node_type(mas->node);
+ void __rcu **slots = ma_slots(mte_to_node(mas->node), type);
+ unsigned long *pivots = ma_pivots(mas_mn(mas), type);
+
+ /* all limits are fine here. */
+ if (mte_is_root(mas->node))
+ return;
+
+ for (i = 0; i < mt_slots[type]; i++) {
+ unsigned long piv;
+
+ piv = mas_safe_pivot(mas, pivots, i, type);
+
+ if (!piv && (i != 0))
+ break;
+
+ if (!mte_is_leaf(mas->node)) {
+ void *entry = mas_slot(mas, slots, i);
+
+ if (!entry)
+ pr_err("%p[%u] cannot be null\n",
+ mas_mn(mas), i);
+
+ MT_BUG_ON(mas->tree, !entry);
+ }
+
+ if (prev_piv > piv) {
+ pr_err("%p[%u] piv %lu < prev_piv %lu\n",
+ mas_mn(mas), i, piv, prev_piv);
+ MT_BUG_ON(mas->tree, piv < prev_piv);
+ }
+
+ if (piv < mas->min) {
+ pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i,
+ piv, mas->min);
+ MT_BUG_ON(mas->tree, piv < mas->min);
+ }
+ if (piv > mas->max) {
+ pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i,
+ piv, mas->max);
+ MT_BUG_ON(mas->tree, piv > mas->max);
+ }
+ prev_piv = piv;
+ if (piv == mas->max)
+ break;
+ }
+ for (i += 1; i < mt_slots[type]; i++) {
+ void *entry = mas_slot(mas, slots, i);
+
+ if (entry && (i != mt_slots[type] - 1)) {
+ pr_err("%p[%u] should not have entry %p\n", mas_mn(mas),
+ i, entry);
+ MT_BUG_ON(mas->tree, entry != NULL);
+ }
+
+ if (i < mt_pivots[type]) {
+ unsigned long piv = pivots[i];
+
+ if (!piv)
+ continue;
+
+ pr_err("%p[%u] should not have piv %lu\n",
+ mas_mn(mas), i, piv);
+ MT_BUG_ON(mas->tree, i < mt_pivots[type] - 1);
+ }
+ }
+}
+
+static void mt_validate_nulls(struct maple_tree *mt)
+{
+ void *entry, *last = (void *)1;
+ unsigned char offset = 0;
+ void __rcu **slots;
+ MA_STATE(mas, mt, 0, 0);
+
+ mas_start(&mas);
+ if (mas_is_none(&mas) || (mas.node == MAS_ROOT))
+ return;
+
+ while (!mte_is_leaf(mas.node))
+ mas_descend(&mas);
+
+ slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node));
+ do {
+ entry = mas_slot(&mas, slots, offset);
+ if (!last && !entry) {
+ pr_err("Sequential nulls end at %p[%u]\n",
+ mas_mn(&mas), offset);
+ }
+ MT_BUG_ON(mt, !last && !entry);
+ last = entry;
+ if (offset == mas_data_end(&mas)) {
+ mas_next_node(&mas, mas_mn(&mas), ULONG_MAX);
+ if (mas_is_none(&mas))
+ return;
+ offset = 0;
+ slots = ma_slots(mte_to_node(mas.node),
+ mte_node_type(mas.node));
+ } else {
+ offset++;
+ }
+
+ } while (!mas_is_none(&mas));
+}
+
+/*
+ * validate a maple tree by checking:
+ * 1. The limits (pivots are within mas->min to mas->max)
+ * 2. The gap is correctly set in the parents
+ */
+void mt_validate(struct maple_tree *mt)
+{
+ unsigned char end;
+
+ MA_STATE(mas, mt, 0, 0);
+ rcu_read_lock();
+ mas_start(&mas);
+ if (!mas_searchable(&mas))
+ goto done;
+
+ mas_first_entry(&mas, mas_mn(&mas), ULONG_MAX, mte_node_type(mas.node));
+ while (!mas_is_none(&mas)) {
+ MT_BUG_ON(mas.tree, mte_dead_node(mas.node));
+ if (!mte_is_root(mas.node)) {
+ end = mas_data_end(&mas);
+ if ((end < mt_min_slot_count(mas.node)) &&
+ (mas.max != ULONG_MAX)) {
+ pr_err("Invalid size %u of %p\n", end,
+ mas_mn(&mas));
+ MT_BUG_ON(mas.tree, 1);
+ }
+
+ }
+ mas_validate_parent_slot(&mas);
+ mas_validate_child_slot(&mas);
+ mas_validate_limits(&mas);
+ if (mt_is_alloc(mt))
+ mas_validate_gaps(&mas);
+ mas_dfs_postorder(&mas, ULONG_MAX);
+ }
+ mt_validate_nulls(mt);
+done:
+ rcu_read_unlock();
+
+}
+
+#endif /* CONFIG_DEBUG_MAPLE_TREE */
* @state: pointer to state structure holding seeded state.
*
* This is used for pseudo-randomness with no outside seeding.
- * For more random results, use prandom_u32().
+ * For more random results, use get_random_u32().
*/
u32 prandom_u32_state(struct rnd_state *state)
{
* @bytes: the requested number of bytes
*
* This is used for pseudo-randomness with no outside seeding.
- * For more random results, use prandom_bytes().
+ * For more random results, use get_random_bytes().
*/
void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes)
{
/* Load c with random data and encode */
for (i = 0; i < dlen; i++)
- c[i] = prandom_u32() & nn;
+ c[i] = get_random_u32() & nn;
memset(c + dlen, 0, nroots * sizeof(*c));
encode_rs16(rs, c, dlen, c + dlen, 0);
for (i = 0; i < errs; i++) {
do {
/* Error value must be nonzero */
- errval = prandom_u32() & nn;
+ errval = get_random_u32() & nn;
} while (errval == 0);
do {
/* Must not choose the same location twice */
- errloc = prandom_u32() % len;
+ errloc = prandom_u32_max(len);
} while (errlocs[errloc] != 0);
errlocs[errloc] = 1;
for (i = 0; i < eras; i++) {
do {
/* Must not choose the same location twice */
- errloc = prandom_u32() % len;
+ errloc = prandom_u32_max(len);
} while (errlocs[errloc] != 0);
derrlocs[i] = errloc;
- if (ewsc && (prandom_u32() & 1)) {
+ if (ewsc && prandom_u32_max(2)) {
/* Erasure with the symbol intact */
errlocs[errloc] = 2;
} else {
/* Erasure with corrupted symbol */
do {
/* Error value must be nonzero */
- errval = prandom_u32() & nn;
+ errval = get_random_u32() & nn;
} while (errval == 0);
errlocs[errloc] = 1;
int i;
for_each_possible_cpu(i)
- *per_cpu_ptr(sb->alloc_hint, i) = prandom_u32() % depth;
+ *per_cpu_ptr(sb->alloc_hint, i) = prandom_u32_max(depth);
}
return 0;
}
hint = this_cpu_read(*sb->alloc_hint);
if (unlikely(hint >= depth)) {
- hint = depth ? prandom_u32() % depth : 0;
+ hint = depth ? prandom_u32_max(depth) : 0;
this_cpu_write(*sb->alloc_hint, hint);
}
#include <linux/mm.h>
#include <linux/cma.h>
-void show_mem(unsigned int filter, nodemask_t *nodemask)
+void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx)
{
pg_data_t *pgdat;
unsigned long total = 0, reserved = 0, highmem = 0;
printk("Mem-Info:\n");
- show_free_areas(filter, nodemask);
+ __show_free_areas(filter, nodemask, max_zone_idx);
for_each_online_pgdat(pgdat) {
int zoneid;
#define STACK_ALLOC_OFFSET_BITS (STACK_ALLOC_ORDER + PAGE_SHIFT - \
STACK_ALLOC_ALIGN)
#define STACK_ALLOC_INDEX_BITS (DEPOT_STACK_BITS - \
- STACK_ALLOC_NULL_PROTECTION_BITS - STACK_ALLOC_OFFSET_BITS)
+ STACK_ALLOC_NULL_PROTECTION_BITS - \
+ STACK_ALLOC_OFFSET_BITS - STACK_DEPOT_EXTRA_BITS)
#define STACK_ALLOC_SLABS_CAP 8192
#define STACK_ALLOC_MAX_SLABS \
(((1LL << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \
u32 slabindex : STACK_ALLOC_INDEX_BITS;
u32 offset : STACK_ALLOC_OFFSET_BITS;
u32 valid : STACK_ALLOC_NULL_PROTECTION_BITS;
+ u32 extra : STACK_DEPOT_EXTRA_BITS;
};
};
static size_t depot_offset;
static DEFINE_RAW_SPINLOCK(depot_lock);
+unsigned int stack_depot_get_extra_bits(depot_stack_handle_t handle)
+{
+ union handle_parts parts = { .handle = handle };
+
+ return parts.extra;
+}
+EXPORT_SYMBOL(stack_depot_get_extra_bits);
+
static bool init_stack_slab(void **prealloc)
{
if (!*prealloc)
stack->handle.slabindex = depot_index;
stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN;
stack->handle.valid = 1;
+ stack->handle.extra = 0;
memcpy(stack->entries, entries, flex_array_size(stack, entries, size));
depot_offset += required_size;
*
* @entries: Pointer to storage array
* @nr_entries: Size of the storage array
+ * @extra_bits: Flags to store in unused bits of depot_stack_handle_t
* @alloc_flags: Allocation gfp flags
* @can_alloc: Allocate stack slabs (increased chance of failure if false)
*
* If the stack trace in @entries is from an interrupt, only the portion up to
* interrupt entry is saved.
*
+ * Additional opaque flags can be passed in @extra_bits, stored in the unused
+ * bits of the stack handle, and retrieved using stack_depot_get_extra_bits()
+ * without calling stack_depot_fetch().
+ *
* Context: Any context, but setting @can_alloc to %false is required if
* alloc_pages() cannot be used from the current context. Currently
* this is the case from contexts where neither %GFP_ATOMIC nor
*/
depot_stack_handle_t __stack_depot_save(unsigned long *entries,
unsigned int nr_entries,
+ unsigned int extra_bits,
gfp_t alloc_flags, bool can_alloc)
{
struct stack_record *found = NULL, **bucket;
- depot_stack_handle_t retval = 0;
+ union handle_parts retval = { .handle = 0 };
struct page *page = NULL;
void *prealloc = NULL;
unsigned long flags;
free_pages((unsigned long)prealloc, STACK_ALLOC_ORDER);
}
if (found)
- retval = found->handle.handle;
+ retval.handle = found->handle.handle;
fast_exit:
- return retval;
+ retval.extra = extra_bits;
+
+ return retval.handle;
}
EXPORT_SYMBOL_GPL(__stack_depot_save);
unsigned int nr_entries,
gfp_t alloc_flags)
{
- return __stack_depot_save(entries, nr_entries, alloc_flags, true);
+ return __stack_depot_save(entries, nr_entries, 0, alloc_flags, true);
}
EXPORT_SYMBOL_GPL(stack_depot_save);
max = 0;
#endif
+ /*
+ * read_word_at_a_time() below may read uninitialized bytes after the
+ * trailing zero and use them in comparisons. Disable this optimization
+ * under KMSAN to prevent false positive reports.
+ */
+ if (IS_ENABLED(CONFIG_KMSAN))
+ max = 0;
+
while (max >= sizeof(unsigned long)) {
unsigned long c, data;
for (i = 0; i < UNESCAPE_ALL_MASK + 1; i++)
test_string_unescape("unescape", i, false);
test_string_unescape("unescape inplace",
- get_random_int() % (UNESCAPE_ANY + 1), true);
+ prandom_u32_max(UNESCAPE_ANY + 1), true);
/* Without dictionary */
for (i = 0; i < ESCAPE_ALL_MASK + 1; i++)
static int fprobe_test_init(struct kunit *test)
{
do {
- rand1 = prandom_u32();
+ rand1 = get_random_u32();
} while (rand1 <= div_factor);
target = fprobe_selftest_target;
static void __init test_hexdump_set(int rowsize, bool ascii)
{
size_t d = min_t(size_t, sizeof(data_b), rowsize);
- size_t len = get_random_int() % d + 1;
+ size_t len = prandom_u32_max(d) + 1;
test_hexdump(len, rowsize, 4, ascii);
test_hexdump(len, rowsize, 2, ascii);
static void __init test_hexdump_overflow_set(size_t buflen, bool ascii)
{
unsigned int i = 0;
- int rs = (get_random_int() % 2 + 1) * 16;
+ int rs = (prandom_u32_max(2) + 1) * 16;
do {
int gs = 1 << i;
- size_t len = get_random_int() % rs + gs;
+ size_t len = prandom_u32_max(rs) + gs;
test_hexdump_overflow(buflen, rounddown(len, gs), rs, gs, ascii);
} while (i++ < 3);
unsigned int i;
int rowsize;
- rowsize = (get_random_int() % 2 + 1) * 16;
+ rowsize = (prandom_u32_max(2) + 1) * 16;
for (i = 0; i < 16; i++)
test_hexdump_set(rowsize, false);
- rowsize = (get_random_int() % 2 + 1) * 16;
+ rowsize = (prandom_u32_max(2) + 1) * 16;
for (i = 0; i < 16; i++)
test_hexdump_set(rowsize, true);
struct dmirror_chunk {
struct dev_pagemap pagemap;
struct dmirror_device *mdevice;
+ bool remove;
};
/*
*/
struct dmirror_device {
struct cdev cdevice;
- struct hmm_devmem *devmem;
unsigned int zone_device_type;
+ struct device device;
unsigned int devmem_capacity;
unsigned int devmem_count;
return 0;
}
+static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page)
+{
+ return container_of(page->pgmap, struct dmirror_chunk, pagemap);
+}
+
static struct dmirror_device *dmirror_page_to_device(struct page *page)
{
- return container_of(page->pgmap, struct dmirror_chunk,
- pagemap)->mdevice;
+ return dmirror_page_to_chunk(page)->mdevice;
}
static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
goto error;
}
+ zone_device_page_init(dpage);
dpage->zone_device_data = rpage;
- lock_page(dpage);
return dpage;
error:
struct vm_area_struct *vma;
unsigned long src_pfns[64] = { 0 };
unsigned long dst_pfns[64] = { 0 };
- struct migrate_vma args;
+ struct migrate_vma args = { 0 };
unsigned long next;
int ret;
unsigned long src_pfns[64] = { 0 };
unsigned long dst_pfns[64] = { 0 };
struct dmirror_bounce bounce;
- struct migrate_vma args;
+ struct migrate_vma args = { 0 };
unsigned long next;
int ret;
return ret;
}
+static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
+{
+ unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT;
+ unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT;
+ unsigned long npages = end_pfn - start_pfn + 1;
+ unsigned long i;
+ unsigned long *src_pfns;
+ unsigned long *dst_pfns;
+
+ src_pfns = kcalloc(npages, sizeof(*src_pfns), GFP_KERNEL);
+ dst_pfns = kcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL);
+
+ migrate_device_range(src_pfns, start_pfn, npages);
+ for (i = 0; i < npages; i++) {
+ struct page *dpage, *spage;
+
+ spage = migrate_pfn_to_page(src_pfns[i]);
+ if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
+ continue;
+
+ if (WARN_ON(!is_device_private_page(spage) &&
+ !is_device_coherent_page(spage)))
+ continue;
+ spage = BACKING_PAGE(spage);
+ dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
+ lock_page(dpage);
+ copy_highpage(dpage, spage);
+ dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
+ if (src_pfns[i] & MIGRATE_PFN_WRITE)
+ dst_pfns[i] |= MIGRATE_PFN_WRITE;
+ }
+ migrate_device_pages(src_pfns, dst_pfns, npages);
+ migrate_device_finalize(src_pfns, dst_pfns, npages);
+ kfree(src_pfns);
+ kfree(dst_pfns);
+}
+
+/* Removes free pages from the free list so they can't be re-allocated */
+static void dmirror_remove_free_pages(struct dmirror_chunk *devmem)
+{
+ struct dmirror_device *mdevice = devmem->mdevice;
+ struct page *page;
+
+ for (page = mdevice->free_pages; page; page = page->zone_device_data)
+ if (dmirror_page_to_chunk(page) == devmem)
+ mdevice->free_pages = page->zone_device_data;
+}
+
+static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)
+{
+ unsigned int i;
+
+ mutex_lock(&mdevice->devmem_lock);
+ if (mdevice->devmem_chunks) {
+ for (i = 0; i < mdevice->devmem_count; i++) {
+ struct dmirror_chunk *devmem =
+ mdevice->devmem_chunks[i];
+
+ spin_lock(&mdevice->lock);
+ devmem->remove = true;
+ dmirror_remove_free_pages(devmem);
+ spin_unlock(&mdevice->lock);
+
+ dmirror_device_evict_chunk(devmem);
+ memunmap_pages(&devmem->pagemap);
+ if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)
+ release_mem_region(devmem->pagemap.range.start,
+ range_len(&devmem->pagemap.range));
+ kfree(devmem);
+ }
+ mdevice->devmem_count = 0;
+ mdevice->devmem_capacity = 0;
+ mdevice->free_pages = NULL;
+ kfree(mdevice->devmem_chunks);
+ mdevice->devmem_chunks = NULL;
+ }
+ mutex_unlock(&mdevice->devmem_lock);
+}
+
static long dmirror_fops_unlocked_ioctl(struct file *filp,
unsigned int command,
unsigned long arg)
ret = dmirror_snapshot(dmirror, &cmd);
break;
+ case HMM_DMIRROR_RELEASE:
+ dmirror_device_remove_chunks(dmirror->mdevice);
+ ret = 0;
+ break;
+
default:
return -EINVAL;
}
mdevice = dmirror_page_to_device(page);
spin_lock(&mdevice->lock);
- mdevice->cfree++;
- page->zone_device_data = mdevice->free_pages;
- mdevice->free_pages = page;
+
+ /* Return page to our allocator if not freeing the chunk */
+ if (!dmirror_page_to_chunk(page)->remove) {
+ mdevice->cfree++;
+ page->zone_device_data = mdevice->free_pages;
+ mdevice->free_pages = page;
+ }
spin_unlock(&mdevice->lock);
}
static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
{
- struct migrate_vma args;
+ struct migrate_vma args = { 0 };
unsigned long src_pfns = 0;
unsigned long dst_pfns = 0;
struct page *rpage;
args.dst = &dst_pfns;
args.pgmap_owner = dmirror->mdevice;
args.flags = dmirror_select_device(dmirror);
+ args.fault_page = vmf->page;
if (migrate_vma_setup(&args))
return VM_FAULT_SIGBUS;
cdev_init(&mdevice->cdevice, &dmirror_fops);
mdevice->cdevice.owner = THIS_MODULE;
- ret = cdev_add(&mdevice->cdevice, dev, 1);
+ device_initialize(&mdevice->device);
+ mdevice->device.devt = dev;
+
+ ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);
+ if (ret)
+ return ret;
+
+ ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);
if (ret)
return ret;
static void dmirror_device_remove(struct dmirror_device *mdevice)
{
- unsigned int i;
-
- if (mdevice->devmem_chunks) {
- for (i = 0; i < mdevice->devmem_count; i++) {
- struct dmirror_chunk *devmem =
- mdevice->devmem_chunks[i];
-
- memunmap_pages(&devmem->pagemap);
- if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)
- release_mem_region(devmem->pagemap.range.start,
- range_len(&devmem->pagemap.range));
- kfree(devmem);
- }
- kfree(mdevice->devmem_chunks);
- }
-
- cdev_del(&mdevice->cdevice);
+ dmirror_device_remove_chunks(mdevice);
+ cdev_device_del(&mdevice->cdevice, &mdevice->device);
}
static int __init hmm_dmirror_init(void)
#define HMM_DMIRROR_SNAPSHOT _IOWR('H', 0x04, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_EXCLUSIVE _IOWR('H', 0x05, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd)
+#define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd)
/*
* Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT.
stacktrace_driver = kprobe_stacktrace_driver;
do {
- rand1 = prandom_u32();
+ rand1 = get_random_u32();
} while (rand1 <= div_factor);
return 0;
}
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, el);
/* force some equivalencies */
- el->value = prandom_u32() % (TEST_LIST_LEN / 3);
+ el->value = prandom_u32_max(TEST_LIST_LEN / 3);
el->serial = i;
el->poison1 = TEST_POISON1;
el->poison2 = TEST_POISON2;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * test_maple_tree.c: Test the maple tree API
+ * Copyright (c) 2018 Liam R. Howlett
+ * Author: Liam R. Howlett <Liam.Howlett@Oracle.com>
+ */
+
+#include <linux/maple_tree.h>
+#include <linux/module.h>
+#include <stdlib.h>
+#include <time.h>
+
+#define MTREE_ALLOC_MAX 0x2000000000000Ul
+#define CONFIG_DEBUG_MAPLE_TREE
+#define CONFIG_MAPLE_SEARCH
+/* #define BENCH_SLOT_STORE */
+/* #define BENCH_NODE_STORE */
+/* #define BENCH_AWALK */
+/* #define BENCH_WALK */
+/* #define BENCH_MT_FOR_EACH */
+/* #define BENCH_FORK */
+static
+int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp)
+{
+ return mtree_insert(mt, index, xa_mk_value(index & LONG_MAX), gfp);
+}
+
+static void mtree_erase_index(struct maple_tree *mt, unsigned long index)
+{
+ MT_BUG_ON(mt, mtree_erase(mt, index) != xa_mk_value(index & LONG_MAX));
+ MT_BUG_ON(mt, mtree_load(mt, index) != NULL);
+}
+
+static int mtree_test_insert(struct maple_tree *mt, unsigned long index,
+ void *ptr)
+{
+ return mtree_insert(mt, index, ptr, GFP_KERNEL);
+}
+
+static int mtree_test_store_range(struct maple_tree *mt, unsigned long start,
+ unsigned long end, void *ptr)
+{
+ return mtree_store_range(mt, start, end, ptr, GFP_KERNEL);
+}
+
+static int mtree_test_store(struct maple_tree *mt, unsigned long start,
+ void *ptr)
+{
+ return mtree_test_store_range(mt, start, start, ptr);
+}
+
+static int mtree_test_insert_range(struct maple_tree *mt, unsigned long start,
+ unsigned long end, void *ptr)
+{
+ return mtree_insert_range(mt, start, end, ptr, GFP_KERNEL);
+}
+
+static void *mtree_test_load(struct maple_tree *mt, unsigned long index)
+{
+ return mtree_load(mt, index);
+}
+
+static void *mtree_test_erase(struct maple_tree *mt, unsigned long index)
+{
+ return mtree_erase(mt, index);
+}
+
+static noinline void check_mtree_alloc_range(struct maple_tree *mt,
+ unsigned long start, unsigned long end, unsigned long size,
+ unsigned long expected, int eret, void *ptr)
+{
+
+ unsigned long result = expected + 1;
+ int ret;
+
+ ret = mtree_alloc_range(mt, &result, ptr, size, start, end,
+ GFP_KERNEL);
+ MT_BUG_ON(mt, ret != eret);
+ if (ret)
+ return;
+
+ MT_BUG_ON(mt, result != expected);
+}
+
+static noinline void check_mtree_alloc_rrange(struct maple_tree *mt,
+ unsigned long start, unsigned long end, unsigned long size,
+ unsigned long expected, int eret, void *ptr)
+{
+
+ unsigned long result = expected + 1;
+ int ret;
+
+ ret = mtree_alloc_rrange(mt, &result, ptr, size, start, end - 1,
+ GFP_KERNEL);
+ MT_BUG_ON(mt, ret != eret);
+ if (ret)
+ return;
+
+ MT_BUG_ON(mt, result != expected);
+}
+
+static noinline void check_load(struct maple_tree *mt, unsigned long index,
+ void *ptr)
+{
+ void *ret = mtree_test_load(mt, index);
+
+ if (ret != ptr)
+ pr_err("Load %lu returned %p expect %p\n", index, ret, ptr);
+ MT_BUG_ON(mt, ret != ptr);
+}
+
+static noinline void check_store_range(struct maple_tree *mt,
+ unsigned long start, unsigned long end, void *ptr, int expected)
+{
+ int ret = -EINVAL;
+ unsigned long i;
+
+ ret = mtree_test_store_range(mt, start, end, ptr);
+ MT_BUG_ON(mt, ret != expected);
+
+ if (ret)
+ return;
+
+ for (i = start; i <= end; i++)
+ check_load(mt, i, ptr);
+}
+
+static noinline void check_insert_range(struct maple_tree *mt,
+ unsigned long start, unsigned long end, void *ptr, int expected)
+{
+ int ret = -EINVAL;
+ unsigned long i;
+
+ ret = mtree_test_insert_range(mt, start, end, ptr);
+ MT_BUG_ON(mt, ret != expected);
+
+ if (ret)
+ return;
+
+ for (i = start; i <= end; i++)
+ check_load(mt, i, ptr);
+}
+
+static noinline void check_insert(struct maple_tree *mt, unsigned long index,
+ void *ptr)
+{
+ int ret = -EINVAL;
+
+ ret = mtree_test_insert(mt, index, ptr);
+ MT_BUG_ON(mt, ret != 0);
+}
+
+static noinline void check_erase(struct maple_tree *mt, unsigned long index,
+ void *ptr)
+{
+ MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr);
+}
+
+static noinline void check_dup_insert(struct maple_tree *mt,
+ unsigned long index, void *ptr)
+{
+ int ret = -EINVAL;
+
+ ret = mtree_test_insert(mt, index, ptr);
+ MT_BUG_ON(mt, ret != -EEXIST);
+}
+
+
+static noinline
+void check_index_load(struct maple_tree *mt, unsigned long index)
+{
+ return check_load(mt, index, xa_mk_value(index & LONG_MAX));
+}
+
+static noinline void check_nomem(struct maple_tree *mt)
+{
+ MA_STATE(ms, mt, 1, 1);
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+ /* Ensure no bypassing of allocation failures */
+ mt_set_non_kernel(0);
+
+ /* Storing something at 1 requires memory allocation */
+ MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM);
+ /* Storing something at 0 does not */
+ MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0);
+
+ /*
+ * Simulate two threads racing; the first one fails to allocate
+ * memory to insert an entry at 1, then the second one succeeds
+ * in allocating memory to insert an entry at 2. The first one
+ * then needs to free the node it allocated. LeakSanitizer will
+ * notice this, as will the 'nr_allocated' debugging aid in the
+ * userspace test suite.
+ */
+ mtree_lock(mt);
+ mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */
+ MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM));
+ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */
+ MT_BUG_ON(mt, ms.node != MAS_START);
+ mtree_unlock(mt);
+ MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0);
+ mtree_lock(mt);
+ mas_store(&ms, &ms); /* insert 1 -> &ms */
+ mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+}
+
+static inline int not_empty(struct maple_node *node)
+{
+ int i;
+
+ if (node->parent)
+ return 1;
+
+ for (i = 0; i < ARRAY_SIZE(node->slot); i++)
+ if (node->slot[i])
+ return 1;
+
+ return 0;
+}
+
+static noinline void check_new_node(struct maple_tree *mt)
+{
+
+ struct maple_node *mn, *mn2, *mn3;
+ struct maple_alloc *smn;
+ struct maple_node *nodes[100];
+ int i, j, total;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ /* Try allocating 3 nodes */
+ mtree_lock(mt);
+ /* request 3 nodes to be allocated. */
+ mas_node_count(&mas, 3);
+ /* Allocation request of 3. */
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 3);
+ /* Allocate failed. */
+ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM));
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+
+ MT_BUG_ON(mt, mas_allocated(&mas) != 3);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, mas.alloc == NULL);
+ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL);
+ mas_push_node(&mas, mn);
+ mas_nomem(&mas, GFP_KERNEL); /* free */
+ mtree_unlock(mt);
+
+
+ /* Try allocating 1 node, then 2 more */
+ mtree_lock(mt);
+ /* Set allocation request to 1. */
+ mas_set_alloc_req(&mas, 1);
+ /* Check Allocation request of 1. */
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1);
+ mas_set_err(&mas, -ENOMEM);
+ /* Validate allocation request. */
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ /* Eat the requested node. */
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, mn->slot[0] != NULL);
+ MT_BUG_ON(mt, mn->slot[1] != NULL);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+
+ ma_free_rcu(mn);
+ mas.node = MAS_START;
+ mas_nomem(&mas, GFP_KERNEL);
+ /* Allocate 3 nodes, will fail. */
+ mas_node_count(&mas, 3);
+ /* Drop the lock and allocate 3 nodes. */
+ mas_nomem(&mas, GFP_KERNEL);
+ /* Ensure 3 are allocated. */
+ MT_BUG_ON(mt, mas_allocated(&mas) != 3);
+ /* Allocation request of 0. */
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 0);
+
+ MT_BUG_ON(mt, mas.alloc == NULL);
+ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL);
+ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL);
+ /* Ensure we counted 3. */
+ MT_BUG_ON(mt, mas_allocated(&mas) != 3);
+ /* Free. */
+ mas_nomem(&mas, GFP_KERNEL);
+
+ /* Set allocation request to 1. */
+ mas_set_alloc_req(&mas, 1);
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1);
+ mas_set_err(&mas, -ENOMEM);
+ /* Validate allocation request. */
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ MT_BUG_ON(mt, mas_allocated(&mas) != 1);
+ /* Check the node is only one node. */
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, mn->slot[0] != NULL);
+ MT_BUG_ON(mt, mn->slot[1] != NULL);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ mas_push_node(&mas, mn);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 1);
+ MT_BUG_ON(mt, mas.alloc->node_count);
+
+ mas_set_alloc_req(&mas, 2); /* request 2 more. */
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 2);
+ mas_set_err(&mas, -ENOMEM);
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ MT_BUG_ON(mt, mas_allocated(&mas) != 3);
+ MT_BUG_ON(mt, mas.alloc == NULL);
+ MT_BUG_ON(mt, mas.alloc->slot[0] == NULL);
+ MT_BUG_ON(mt, mas.alloc->slot[1] == NULL);
+ for (i = 2; i >= 0; i--) {
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, mas_allocated(&mas) != i);
+ MT_BUG_ON(mt, !mn);
+ MT_BUG_ON(mt, not_empty(mn));
+ ma_free_rcu(mn);
+ }
+
+ total = 64;
+ mas_set_alloc_req(&mas, total); /* request 2 more. */
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != total);
+ mas_set_err(&mas, -ENOMEM);
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ for (i = total; i > 0; i--) {
+ unsigned int e = 0; /* expected node_count */
+
+ if (i >= 35)
+ e = i - 35;
+ else if (i >= 5)
+ e = i - 5;
+ else if (i >= 2)
+ e = i - 2;
+ MT_BUG_ON(mt, mas.alloc->node_count != e);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mas_allocated(&mas) != i - 1);
+ MT_BUG_ON(mt, !mn);
+ ma_free_rcu(mn);
+ }
+
+ total = 100;
+ for (i = 1; i < total; i++) {
+ mas_set_alloc_req(&mas, i);
+ mas_set_err(&mas, -ENOMEM);
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ for (j = i; j > 0; j--) {
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1);
+ MT_BUG_ON(mt, !mn);
+ MT_BUG_ON(mt, not_empty(mn));
+ mas_push_node(&mas, mn);
+ MT_BUG_ON(mt, mas_allocated(&mas) != j);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mas_allocated(&mas) != j - 1);
+ ma_free_rcu(mn);
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+
+ mas_set_alloc_req(&mas, i);
+ mas_set_err(&mas, -ENOMEM);
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ for (j = 0; j <= i/2; j++) {
+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j);
+ nodes[j] = mas_pop_node(&mas);
+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1);
+ }
+
+ while (j) {
+ j--;
+ mas_push_node(&mas, nodes[j]);
+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j);
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas) != i);
+ for (j = 0; j <= i/2; j++) {
+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ ma_free_rcu(mn);
+ MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1);
+ }
+ MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL));
+
+ }
+
+ /* Set allocation request. */
+ total = 500;
+ mas_node_count(&mas, total);
+ /* Drop the lock and allocate the nodes. */
+ mas_nomem(&mas, GFP_KERNEL);
+ MT_BUG_ON(mt, !mas.alloc);
+ i = 1;
+ smn = mas.alloc;
+ while (i < total) {
+ for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) {
+ i++;
+ MT_BUG_ON(mt, !smn->slot[j]);
+ if (i == total)
+ break;
+ }
+ smn = smn->slot[0]; /* next. */
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas) != total);
+ mas_nomem(&mas, GFP_KERNEL); /* Free. */
+
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ for (i = 1; i < 128; i++) {
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */
+ for (j = i; j > 0; j--) { /*Free the requests */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, not_empty(mn));
+ ma_free_rcu(mn);
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ }
+
+ for (i = 1; i < MAPLE_NODE_MASK + 1; i++) {
+ MA_STATE(mas2, mt, 0, 0);
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */
+ for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, not_empty(mn));
+ mas_push_node(&mas2, mn);
+ MT_BUG_ON(mt, mas_allocated(&mas2) != j);
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ MT_BUG_ON(mt, mas_allocated(&mas2) != i);
+
+ for (j = i; j > 0; j--) { /*Free the requests */
+ MT_BUG_ON(mt, mas_allocated(&mas2) != j);
+ mn = mas_pop_node(&mas2); /* get the next node. */
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, not_empty(mn));
+ ma_free_rcu(mn);
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas2) != 0);
+ }
+
+
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */
+ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM));
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1);
+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1);
+
+ mn = mas_pop_node(&mas); /* get the next node. */
+ MT_BUG_ON(mt, mn == NULL);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS);
+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2);
+
+ mas_push_node(&mas, mn);
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1);
+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1);
+
+ /* Check the limit of pop/push/pop */
+ mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */
+ MT_BUG_ON(mt, mas_alloc_req(&mas) != 1);
+ MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM));
+ MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL));
+ MT_BUG_ON(mt, mas_alloc_req(&mas));
+ MT_BUG_ON(mt, mas.alloc->node_count);
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1);
+ MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1);
+ mas_push_node(&mas, mn);
+ MT_BUG_ON(mt, mas.alloc->node_count);
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ ma_free_rcu(mn);
+ for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) {
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, not_empty(mn));
+ ma_free_rcu(mn);
+ }
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+
+
+ for (i = 3; i < MAPLE_NODE_MASK * 3; i++) {
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ mas_push_node(&mas, mn); /* put it back */
+ mas_destroy(&mas);
+
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ mn2 = mas_pop_node(&mas); /* get the next node. */
+ mas_push_node(&mas, mn); /* put them back */
+ mas_push_node(&mas, mn2);
+ mas_destroy(&mas);
+
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ mn2 = mas_pop_node(&mas); /* get the next node. */
+ mn3 = mas_pop_node(&mas); /* get the next node. */
+ mas_push_node(&mas, mn); /* put them back */
+ mas_push_node(&mas, mn2);
+ mas_push_node(&mas, mn3);
+ mas_destroy(&mas);
+
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ ma_free_rcu(mn);
+ mas_destroy(&mas);
+
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, i); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mn = mas_pop_node(&mas); /* get the next node. */
+ ma_free_rcu(mn);
+ mn = mas_pop_node(&mas); /* get the next node. */
+ ma_free_rcu(mn);
+ mn = mas_pop_node(&mas); /* get the next node. */
+ ma_free_rcu(mn);
+ mas_destroy(&mas);
+ }
+
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, 5); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ MT_BUG_ON(mt, mas_allocated(&mas) != 5);
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, 10); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mas.node = MAS_START;
+ MT_BUG_ON(mt, mas_allocated(&mas) != 10);
+ mas_destroy(&mas);
+
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1);
+ mas.node = MA_ERROR(-ENOMEM);
+ mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */
+ mas_nomem(&mas, GFP_KERNEL); /* Fill request */
+ mas.node = MAS_START;
+ MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1);
+ mas_destroy(&mas);
+
+ mtree_unlock(mt);
+}
+
+static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max,
+ bool verbose)
+{
+ unsigned long i = max, j;
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+
+ mt_zero_nr_tallocated();
+ while (i) {
+ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL));
+ for (j = i; j <= max; j++)
+ check_index_load(mt, j);
+
+ check_load(mt, i - 1, NULL);
+ mt_set_in_rcu(mt);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mt_clear_in_rcu(mt);
+ MT_BUG_ON(mt, !mt_height(mt));
+ i--;
+ }
+ check_load(mt, max + 1, NULL);
+
+ if (verbose) {
+ rcu_barrier();
+ mt_dump(mt);
+ pr_info(" %s test of 0-%lu %luK in %d active (%d total)\n",
+ __func__, max, mt_get_alloc_size()/1024, mt_nr_allocated(),
+ mt_nr_tallocated());
+ }
+}
+
+static noinline void check_seq(struct maple_tree *mt, unsigned long max,
+ bool verbose)
+{
+ unsigned long i, j;
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+
+ mt_zero_nr_tallocated();
+ for (i = 0; i <= max; i++) {
+ MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL));
+ for (j = 0; j <= i; j++)
+ check_index_load(mt, j);
+
+ if (i)
+ MT_BUG_ON(mt, !mt_height(mt));
+ check_load(mt, i + 1, NULL);
+ }
+ if (verbose) {
+ rcu_barrier();
+ mt_dump(mt);
+ pr_info(" seq test of 0-%lu %luK in %d active (%d total)\n",
+ max, mt_get_alloc_size()/1024, mt_nr_allocated(),
+ mt_nr_tallocated());
+ }
+}
+
+static noinline void check_lb_not_empty(struct maple_tree *mt)
+{
+ unsigned long i, j;
+ unsigned long huge = 4000UL * 1000 * 1000;
+
+
+ i = huge;
+ while (i > 4096) {
+ check_insert(mt, i, (void *) i);
+ for (j = huge; j >= i; j /= 2) {
+ check_load(mt, j-1, NULL);
+ check_load(mt, j, (void *) j);
+ check_load(mt, j+1, NULL);
+ }
+ i /= 2;
+ }
+ mtree_destroy(mt);
+}
+
+static noinline void check_lower_bound_split(struct maple_tree *mt)
+{
+ MT_BUG_ON(mt, !mtree_empty(mt));
+ check_lb_not_empty(mt);
+}
+
+static noinline void check_upper_bound_split(struct maple_tree *mt)
+{
+ unsigned long i, j;
+ unsigned long huge = 4000UL * 1000 * 1000;
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+
+ i = 4096;
+ while (i < huge) {
+ check_insert(mt, i, (void *) i);
+ for (j = i; j >= huge; j *= 2) {
+ check_load(mt, j-1, NULL);
+ check_load(mt, j, (void *) j);
+ check_load(mt, j+1, NULL);
+ }
+ i *= 2;
+ }
+ mtree_destroy(mt);
+}
+
+static noinline void check_mid_split(struct maple_tree *mt)
+{
+ unsigned long huge = 8000UL * 1000 * 1000;
+
+ check_insert(mt, huge, (void *) huge);
+ check_insert(mt, 0, xa_mk_value(0));
+ check_lb_not_empty(mt);
+}
+
+static noinline void check_rev_find(struct maple_tree *mt)
+{
+ int i, nr_entries = 200;
+ void *val;
+ MA_STATE(mas, mt, 0, 0);
+
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ mas_set(&mas, 1000);
+ val = mas_find_rev(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(100));
+ val = mas_find_rev(&mas, 1000);
+ MT_BUG_ON(mt, val != NULL);
+
+ mas_set(&mas, 999);
+ val = mas_find_rev(&mas, 997);
+ MT_BUG_ON(mt, val != NULL);
+
+ mas_set(&mas, 1000);
+ val = mas_find_rev(&mas, 900);
+ MT_BUG_ON(mt, val != xa_mk_value(100));
+ val = mas_find_rev(&mas, 900);
+ MT_BUG_ON(mt, val != xa_mk_value(99));
+
+ mas_set(&mas, 20);
+ val = mas_find_rev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(2));
+ val = mas_find_rev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(1));
+ val = mas_find_rev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(0));
+ val = mas_find_rev(&mas, 0);
+ MT_BUG_ON(mt, val != NULL);
+}
+
+static noinline void check_find(struct maple_tree *mt)
+{
+ unsigned long val = 0;
+ unsigned long count = 20;
+ unsigned long max;
+ unsigned long last = 0, index = 0;
+ void *entry, *entry2;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ /* Insert 0. */
+ MT_BUG_ON(mt, mtree_insert_index(mt, val++, GFP_KERNEL));
+
+ for (int i = 0; i <= count; i++) {
+ if (val != 64)
+ MT_BUG_ON(mt, mtree_insert_index(mt, val, GFP_KERNEL));
+ else
+ MT_BUG_ON(mt, mtree_insert(mt, val,
+ XA_ZERO_ENTRY, GFP_KERNEL));
+
+ val <<= 2;
+ }
+
+ val = 0;
+ mas_set(&mas, val);
+ mas_lock(&mas);
+ while ((entry = mas_find(&mas, 268435456)) != NULL) {
+ if (val != 64)
+ MT_BUG_ON(mt, xa_mk_value(val) != entry);
+ else
+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY);
+
+ val <<= 2;
+ /* For zero check. */
+ if (!val)
+ val = 1;
+ }
+ mas_unlock(&mas);
+
+ val = 0;
+ mas_set(&mas, val);
+ mas_lock(&mas);
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ if (val != 64)
+ MT_BUG_ON(mt, xa_mk_value(val) != entry);
+ else
+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY);
+ val <<= 2;
+ /* For zero check. */
+ if (!val)
+ val = 1;
+ }
+ mas_unlock(&mas);
+
+ /* Test mas_pause */
+ val = 0;
+ mas_set(&mas, val);
+ mas_lock(&mas);
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ if (val != 64)
+ MT_BUG_ON(mt, xa_mk_value(val) != entry);
+ else
+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY);
+ val <<= 2;
+ /* For zero check. */
+ if (!val)
+ val = 1;
+
+ mas_pause(&mas);
+ mas_unlock(&mas);
+ mas_lock(&mas);
+ }
+ mas_unlock(&mas);
+
+ val = 0;
+ max = 300; /* A value big enough to include XA_ZERO_ENTRY at 64. */
+ mt_for_each(mt, entry, index, max) {
+ MT_BUG_ON(mt, xa_mk_value(val) != entry);
+ val <<= 2;
+ if (val == 64) /* Skip zero entry. */
+ val <<= 2;
+ /* For zero check. */
+ if (!val)
+ val = 1;
+ }
+
+ val = 0;
+ max = 0;
+ index = 0;
+ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL));
+ mt_for_each(mt, entry, index, ULONG_MAX) {
+ if (val == 4398046511104)
+ MT_BUG_ON(mt, entry !=
+ xa_mk_value(ULONG_MAX & LONG_MAX));
+ else
+ MT_BUG_ON(mt, xa_mk_value(val) != entry);
+ val <<= 2;
+ if (val == 64) /* Skip zero entry. */
+ val <<= 2;
+ /* For zero check. */
+ if (!val)
+ val = 1;
+ max++;
+ MT_BUG_ON(mt, max > 25);
+ }
+ mtree_erase_index(mt, ULONG_MAX);
+
+ mas_reset(&mas);
+ index = 17;
+ entry = mt_find(mt, &index, 512);
+ MT_BUG_ON(mt, xa_mk_value(256) != entry);
+
+ mas_reset(&mas);
+ index = 17;
+ entry = mt_find(mt, &index, 20);
+ MT_BUG_ON(mt, entry != NULL);
+
+
+ /* Range check.. */
+ /* Insert ULONG_MAX */
+ MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL));
+
+ val = 0;
+ mas_set(&mas, 0);
+ mas_lock(&mas);
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ if (val == 64)
+ MT_BUG_ON(mt, entry != XA_ZERO_ENTRY);
+ else if (val == 4398046511104)
+ MT_BUG_ON(mt, entry != xa_mk_value(ULONG_MAX & LONG_MAX));
+ else
+ MT_BUG_ON(mt, xa_mk_value(val) != entry);
+ val <<= 2;
+
+ /* For zero check. */
+ if (!val)
+ val = 1;
+ mas_pause(&mas);
+ mas_unlock(&mas);
+ mas_lock(&mas);
+ }
+ mas_unlock(&mas);
+
+ mas_set(&mas, 1048576);
+ mas_lock(&mas);
+ entry = mas_find(&mas, 1048576);
+ mas_unlock(&mas);
+ MT_BUG_ON(mas.tree, entry == NULL);
+
+ /*
+ * Find last value.
+ * 1. get the expected value, leveraging the existence of an end entry
+ * 2. delete end entry
+ * 3. find the last value but searching for ULONG_MAX and then using
+ * prev
+ */
+ /* First, get the expected result. */
+ mas_lock(&mas);
+ mas_reset(&mas);
+ mas.index = ULONG_MAX; /* start at max.. */
+ entry = mas_find(&mas, ULONG_MAX);
+ entry = mas_prev(&mas, 0);
+ index = mas.index;
+ last = mas.last;
+
+ /* Erase the last entry. */
+ mas_reset(&mas);
+ mas.index = ULONG_MAX;
+ mas.last = ULONG_MAX;
+ mas_erase(&mas);
+
+ /* Get the previous value from MAS_START */
+ mas_reset(&mas);
+ entry2 = mas_prev(&mas, 0);
+
+ /* Check results. */
+ MT_BUG_ON(mt, entry != entry2);
+ MT_BUG_ON(mt, index != mas.index);
+ MT_BUG_ON(mt, last != mas.last);
+
+
+ mas.node = MAS_NONE;
+ mas.index = ULONG_MAX;
+ mas.last = ULONG_MAX;
+ entry2 = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, entry != entry2);
+
+ mas_set(&mas, 0);
+ MT_BUG_ON(mt, mas_prev(&mas, 0) != NULL);
+
+ mas_unlock(&mas);
+ mtree_destroy(mt);
+}
+
+static noinline void check_find_2(struct maple_tree *mt)
+{
+ unsigned long i, j;
+ void *entry;
+
+ MA_STATE(mas, mt, 0, 0);
+ rcu_read_lock();
+ mas_for_each(&mas, entry, ULONG_MAX)
+ MT_BUG_ON(mt, true);
+ rcu_read_unlock();
+
+ for (i = 0; i < 256; i++) {
+ mtree_insert_index(mt, i, GFP_KERNEL);
+ j = 0;
+ mas_set(&mas, 0);
+ rcu_read_lock();
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ MT_BUG_ON(mt, entry != xa_mk_value(j));
+ j++;
+ }
+ rcu_read_unlock();
+ MT_BUG_ON(mt, j != i + 1);
+ }
+
+ for (i = 0; i < 256; i++) {
+ mtree_erase_index(mt, i);
+ j = i + 1;
+ mas_set(&mas, 0);
+ rcu_read_lock();
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ if (xa_is_zero(entry))
+ continue;
+
+ MT_BUG_ON(mt, entry != xa_mk_value(j));
+ j++;
+ }
+ rcu_read_unlock();
+ MT_BUG_ON(mt, j != 256);
+ }
+
+ /*MT_BUG_ON(mt, !mtree_empty(mt)); */
+}
+
+#define erase_ptr(i) entry[i%2]
+#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2])
+#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2])
+#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2])
+
+static noinline void check_erase_testset(struct maple_tree *mt)
+{
+ unsigned long set[] = { 5015, 5014, 5017, 25, 1000,
+ 1001, 1002, 1003, 1005, 0,
+ 6003, 6002, 6008, 6012, 6015,
+ 7003, 7002, 7008, 7012, 7015,
+ 8003, 8002, 8008, 8012, 8015,
+ 9003, 9002, 9008, 9012, 9015,
+ 10003, 10002, 10008, 10012, 10015,
+ 11003, 11002, 11008, 11012, 11015,
+ 12003, 12002, 12008, 12012, 12015,
+ 13003, 13002, 13008, 13012, 13015,
+ 14003, 14002, 14008, 14012, 14015,
+ 15003, 15002, 15008, 15012, 15015,
+ };
+
+
+ void *ptr = &set;
+ void *entry[2] = { ptr, mt };
+ void *root_node;
+
+
+ rcu_register_thread();
+ mt_set_in_rcu(mt);
+ for (int i = 0; i < 4; i++)
+ erase_check_insert(mt, i);
+ for (int i = 0; i < 4; i++)
+ erase_check_load(mt, i);
+
+ mt_set_non_kernel(2);
+ erase_check_erase(mt, 1);
+ erase_check_load(mt, 0);
+ check_load(mt, set[1], NULL);
+ for (int i = 2; i < 4; i++)
+ erase_check_load(mt, i);
+
+
+ erase_check_erase(mt, 2);
+ erase_check_load(mt, 0);
+ check_load(mt, set[1], NULL);
+ check_load(mt, set[2], NULL);
+
+ erase_check_insert(mt, 1);
+ erase_check_insert(mt, 2);
+
+ for (int i = 0; i < 4; i++)
+ erase_check_load(mt, i);
+
+ /* Check erase and load without an allocation. */
+ erase_check_load(mt, 3);
+ erase_check_erase(mt, 1);
+ erase_check_load(mt, 0);
+ check_load(mt, set[1], NULL);
+ for (int i = 2; i < 4; i++)
+ erase_check_load(mt, i);
+
+ /*
+ * Set the newly erased node. This will produce a different allocated
+ * node to avoid busy slots.
+ */
+ root_node = mt->ma_root;
+ erase_check_insert(mt, 1);
+
+ erase_check_load(mt, 0);
+ check_load(mt, 5016, NULL);
+ erase_check_load(mt, 1);
+ check_load(mt, 5013, NULL);
+ erase_check_load(mt, 2);
+ check_load(mt, 5018, NULL);
+ erase_check_load(mt, 3);
+
+ erase_check_erase(mt, 2); /* erase 5017 to check append */
+ erase_check_load(mt, 0);
+ check_load(mt, 5016, NULL);
+ erase_check_load(mt, 1);
+ check_load(mt, 5013, NULL);
+ check_load(mt, set[2], NULL);
+ check_load(mt, 5018, NULL);
+
+ erase_check_load(mt, 3);
+
+ root_node = mt->ma_root;
+ erase_check_insert(mt, 2);
+
+ erase_check_load(mt, 0);
+ check_load(mt, 5016, NULL);
+ erase_check_load(mt, 1);
+ check_load(mt, 5013, NULL);
+ erase_check_load(mt, 2);
+ check_load(mt, 5018, NULL);
+ erase_check_load(mt, 3);
+
+ mt_set_non_kernel(1);
+ erase_check_erase(mt, 2); /* erase 5017 to check append */
+ erase_check_load(mt, 0);
+ check_load(mt, 5016, NULL);
+ check_load(mt, set[2], NULL);
+ erase_check_erase(mt, 0); /* erase 5015 to check append */
+ check_load(mt, set[0], NULL);
+ check_load(mt, 5016, NULL);
+ erase_check_insert(mt, 4); /* 1000 < Should not split. */
+ check_load(mt, set[0], NULL);
+ check_load(mt, 5016, NULL);
+ erase_check_load(mt, 1);
+ check_load(mt, 5013, NULL);
+ check_load(mt, set[2], NULL);
+ check_load(mt, 5018, NULL);
+ erase_check_load(mt, 4);
+ check_load(mt, 999, NULL);
+ check_load(mt, 1001, NULL);
+ erase_check_load(mt, 4);
+ if (mt_in_rcu(mt))
+ MT_BUG_ON(mt, root_node == mt->ma_root);
+ else
+ MT_BUG_ON(mt, root_node != mt->ma_root);
+
+ /* Should not have split. */
+ MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root));
+
+
+ /* Coalesce testing */
+ erase_check_insert(mt, 0);
+ erase_check_insert(mt, 2);
+
+ for (int i = 5; i < 25; i++) {
+ erase_check_insert(mt, i);
+ for (int j = i; j >= 0; j--)
+ erase_check_load(mt, j);
+ }
+
+ erase_check_erase(mt, 14); /*6015 */
+ for (int i = 0; i < 25; i++) {
+ if (i == 14)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+ erase_check_erase(mt, 16); /*7002 */
+ for (int i = 0; i < 25; i++) {
+ if (i == 16 || i == 14)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+
+ mt_set_non_kernel(1);
+ erase_check_erase(mt, 13); /*6012 */
+ for (int i = 0; i < 25; i++) {
+ if (i == 16 || i == 14 || i == 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ erase_check_erase(mt, 15); /*7003 */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 16 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ mt_set_non_kernel(2);
+ erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 17 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ erase_check_erase(mt, 18); /*7012 */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 18 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ mt_set_non_kernel(2);
+ erase_check_erase(mt, 19); /*7015 */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 19 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ erase_check_erase(mt, 20); /*8003 */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 20 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ erase_check_erase(mt, 21); /*8002 */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 21 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ mt_set_non_kernel(2);
+ erase_check_erase(mt, 22); /*8008 */
+ for (int i = 0; i < 25; i++) {
+ if (i <= 22 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+ for (int i = 23; i < 25; i++)
+ erase_check_erase(mt, i);
+
+ for (int i = 0; i < 25; i++) {
+ if (i <= 25 && i >= 13)
+ check_load(mt, set[i], NULL);
+ else
+ erase_check_load(mt, i);
+ }
+
+ /* Shrinking tree test. */
+
+ for (int i = 13; i < ARRAY_SIZE(set); i++)
+ erase_check_insert(mt, i);
+
+ mt_set_non_kernel(99);
+ for (int i = 18; i < ARRAY_SIZE(set); i++) {
+ erase_check_erase(mt, i);
+ for (int j = 0; j < ARRAY_SIZE(set); j++) {
+ if (j < 18 || j > i)
+ erase_check_load(mt, j);
+ else
+ check_load(mt, set[j], NULL);
+ }
+ }
+ mt_set_non_kernel(35);
+ for (int i = 0; i < 18; i++) {
+ erase_check_erase(mt, i);
+ for (int j = 0; j < ARRAY_SIZE(set); j++) {
+ if (j < 18 && j > i)
+ erase_check_load(mt, j);
+ else
+ check_load(mt, set[j], NULL);
+ }
+ }
+ erase_check_insert(mt, 8);
+ erase_check_insert(mt, 9);
+ erase_check_erase(mt, 8);
+ rcu_unregister_thread();
+}
+
+#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \
+ a[(i)], a[(i + 1)], ptr)
+#define STORE 1
+#define SNULL 2
+#define ERASE 3
+#define ec_type_str(x) \
+ (((x) == STORE) ? \
+ "STORE" : \
+ (((x) == SNULL) ? \
+ "SNULL" : "ERASE") \
+ )
+#define check_erase2_debug 0
+void *mas_next(struct ma_state *mas, unsigned long max);
+
+/* Calculate the overwritten entries. */
+int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end,
+ void *s_entry, unsigned long s_min,
+ void *e_entry, unsigned long e_max,
+ unsigned long *set, int i, bool null_entry)
+{
+ int count = 0, span = 0;
+ unsigned long retry = 0;
+ void *entry;
+ struct ma_state tmp;
+
+
+ /* count slots */
+ memcpy(&tmp, mas_start, sizeof(tmp));
+ entry = mas_next(&tmp, mas_end->last);
+ while (entry) {
+ BUG_ON(retry > 50); /* stop infinite retry on testing. */
+ if (xa_is_zero(s_entry)) {
+ retry++;
+ continue;
+ }
+ count++;
+ span++;
+ entry = mas_next(&tmp, mas_end->last);
+ }
+
+ if (null_entry) {
+ /* Check splitting end. */
+ if (e_entry && (e_max > mas_end->last))
+ count--;
+
+ /* check overwrite of entire start */
+ if (s_entry && (s_min == mas_start->index))
+ count++;
+ } else { /* !null_entry (store) */
+ bool esplit = e_max > mas_end->last;
+ bool ssplit = s_min != mas_start->index;
+
+ if (s_entry && e_entry) {
+ if (esplit && ssplit)
+ count--;
+ else if (ssplit)
+ count--;
+ else if (esplit) {
+ if (span)
+ count--;
+ }
+ } else if (s_entry && !e_entry) {
+ if (ssplit)
+ count--;
+ } else if (!s_entry && e_entry) {
+ if (esplit)
+ count--;
+ count--;
+ } else {
+ count--;
+ }
+ }
+ return count;
+}
+
+/*
+ * mas_node_walk() - Walk a maple node to offset of the index.
+ * @mas: The maple state
+ * @type: The maple node type
+ * @*range_min: Pointer to store the minimum range of the offset
+ * @*range_max: Pointer to store the maximum range of the offset
+ *
+ * The offset will be stored in the maple state.
+ *
+ */
+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
+ enum maple_type type, unsigned long *range_min,
+ unsigned long *range_max)
+
+{
+ unsigned long *pivots;
+ unsigned char count;
+ unsigned long prev, max;
+ unsigned char offset;
+ unsigned long index;
+
+ if (unlikely(ma_is_dense(type))) {
+ (*range_max) = (*range_min) = mas->index;
+ if (unlikely(ma_dead_node(node)))
+ return;
+
+ mas->offset = mas->index = mas->min;
+ return;
+ }
+
+ pivots = ma_pivots(node, type);
+ max = pivots[0];
+ if (unlikely(ma_dead_node(node)))
+ return;
+
+ offset = 0;
+ prev = mas->min;
+ index = mas->index;
+ if (unlikely(index <= max))
+ goto offset_zero;
+
+ count = mt_pivots[type];
+ while (++offset < count) {
+ prev = max;
+ max = pivots[offset];
+ if (unlikely(ma_dead_node(node)))
+ return;
+
+ if (index <= max)
+ goto offset_found;
+ else if (unlikely(!max))
+ goto mas_max;
+ }
+
+ prev = max;
+mas_max:
+ max = mas->max;
+offset_found:
+ prev++;
+offset_zero:
+ mas->offset = offset;
+ if (ma_is_leaf(type)) {
+ *range_max = max;
+ *range_min = prev;
+ } else {
+ mas->max = max;
+ mas->min = prev;
+ }
+}
+
+/*
+ * mas_descend_walk(): Locates a value and sets the mas->node and slot
+ * accordingly. range_min and range_max are set to the range which the entry is
+ * valid.
+ * @mas: The maple state
+ * @*range_min: A pointer to store the minimum of the range
+ * @*range_max: A pointer to store the maximum of the range
+ *
+ * Check mas->node is still valid on return of any value.
+ *
+ * Return: true if pointing to a valid node and offset. False otherwise.
+ */
+static inline bool mas_descend_walk(struct ma_state *mas,
+ unsigned long *range_min, unsigned long *range_max)
+{
+ struct maple_enode *next;
+ struct maple_node *node;
+ enum maple_type type;
+
+ next = mas->node;
+ while (true) {
+ node = mte_to_node(next);
+ type = mte_node_type(next);
+ mas_node_walk(mas, node, type, range_min, range_max);
+ next = mas_slot(mas, ma_slots(node, type), mas->offset);
+ if (unlikely(ma_dead_node(node)))
+ return false;
+
+ if (unlikely(ma_is_leaf(type)))
+ return true;
+
+ /* Descend. */
+ mas->node = next;
+ }
+ return false;
+}
+
+/*
+ * mas_tree_walk() - Walk to @mas->index and set the range values.
+ * @mas: The maple state.
+ * @*range_min: The minimum range to be set.
+ * @*range_max: The maximum range to be set.
+ *
+ * Ranges are only valid if there is a valid entry at @mas->index.
+ *
+ * Return: True if a value exists, false otherwise.
+ */
+static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min,
+ unsigned long *range_max)
+{
+ bool ret;
+
+retry:
+ ret = false;
+ mas_start(mas);
+ if (mas_is_none(mas))
+ goto not_found;
+
+ if (mas_is_ptr(mas)) {
+ *range_min = *range_max = 0;
+ if (!mas->index)
+ return true;
+
+ goto not_found;
+ }
+
+ ret = mas_descend_walk(mas, range_min, range_max);
+ if (unlikely(mte_dead_node(mas->node))) {
+ mas->node = MAS_START;
+ goto retry;
+ }
+
+ return ret;
+
+not_found:
+ mas->offset = MAPLE_NODE_SLOTS;
+ return false;
+}
+
+static inline void *mas_range_load(struct ma_state *mas,
+ unsigned long *range_min, unsigned long *range_max)
+
+{
+ void *entry = NULL;
+ unsigned long index = mas->index;
+
+ if (mas_is_none(mas) || mas_is_paused(mas))
+ mas->node = MAS_START;
+retry:
+ if (mas_tree_walk(mas, range_min, range_max))
+ if (unlikely(mas->node == MAS_ROOT))
+ return mas_root(mas);
+
+ if (likely(mas->offset != MAPLE_NODE_SLOTS))
+ entry = mas_get_slot(mas, mas->offset);
+
+ if (mas_dead_node(mas, index))
+ goto retry;
+
+ return entry;
+}
+static noinline void check_erase2_testset(struct maple_tree *mt,
+ unsigned long *set, unsigned long size)
+{
+ int entry_count = 0;
+ int check = 0;
+ void *foo;
+ unsigned long addr = 0;
+ void *s_entry = NULL, *e_entry = NULL;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ for (int i = 0; i < size; i += 3) {
+ unsigned long s_min, s_max;
+ unsigned long e_min, e_max;
+ void *value = NULL;
+
+ MA_STATE(mas_start, mt, set[i+1], set[i+1]);
+ MA_STATE(mas_end, mt, set[i+2], set[i+2]);
+ mt_set_non_kernel(127);
+#if check_erase2_debug
+ pr_err("%s: %d %s %lu - %lu\n", __func__, i,
+ ec_type_str(set[i]),
+ set[i+1], set[i+2]);
+#endif
+ s_entry = mas_range_load(&mas_start, &s_min, &s_max);
+ e_entry = mas_range_load(&mas_end, &e_min, &e_max);
+
+ switch (set[i]) {
+ case SNULL:
+ if ((s_min == set[i+1]) && (s_max == set[i+2])) {
+ if (s_entry)
+ entry_count--;
+ } else if ((s_min != set[i+1]) && (s_max != set[i+2])) {
+ entry_count++;
+ } else if ((mas_start.node != mas_end.node) ||
+ (mas_start.offset != mas_end.offset)) {
+ entry_count -=
+ mas_ce2_over_count(&mas_start, &mas_end,
+ s_entry, s_min,
+ e_entry, e_max, set, i,
+ true);
+ }
+
+
+ erase_check_store_range(mt, set, i + 1, value);
+ break;
+ case STORE:
+ value = xa_mk_value(set[i + 1]);
+ if (mas_start.offset > mt_slot_count(mas_start.node)) {
+ entry_count++; /* appending an entry. */
+ } else if ((s_min == e_min) && (s_max == e_max)) {
+ if (!entry_count)
+ entry_count++;
+
+ else if (s_entry) {
+ if (e_max > mas_end.last)
+ entry_count++;
+
+ if (s_min < mas_start.index)
+ entry_count++;
+
+ } else {
+ entry_count++;
+ }
+ } else {
+ entry_count -=
+ mas_ce2_over_count(&mas_start, &mas_end,
+ s_entry, s_min,
+ e_entry, e_max, set, i,
+ false);
+ }
+
+ erase_check_store_range(mt, set, i + 1, value);
+ break;
+ case ERASE:
+ if (!s_entry)
+ break;
+ check_erase(mt, set[i+1], xa_mk_value(set[i+1]));
+ entry_count--;
+ break;
+ }
+ mt_validate(mt);
+ if (entry_count)
+ MT_BUG_ON(mt, !mt_height(mt));
+#if check_erase2_debug > 1
+ mt_dump(mt);
+#endif
+#if check_erase2_debug
+ pr_err("Done\n");
+#endif
+
+ check = 0;
+ addr = 0;
+ mt_for_each(mt, foo, addr, ULONG_MAX) {
+ check++;
+#if check_erase2_debug > 2
+ pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check);
+#endif
+ if (check > entry_count)
+ break;
+ }
+
+#if check_erase2_debug > 2
+ pr_err("mt_for_each %d and count %d\n", check, entry_count);
+#endif
+
+ MT_BUG_ON(mt, check != entry_count);
+
+ check = 0;
+ addr = 0;
+ mas_reset(&mas);
+ mas.index = 0;
+ rcu_read_lock();
+ mas_for_each(&mas, foo, ULONG_MAX) {
+ if (xa_is_zero(foo)) {
+ if (addr == mas.index) {
+ mt_dump(mas.tree);
+ pr_err("retry failed %lu - %lu\n",
+ mas.index, mas.last);
+ MT_BUG_ON(mt, 1);
+ }
+ addr = mas.index;
+ continue;
+ }
+#if check_erase2_debug > 2
+ pr_err("mas: %lu -> %p\n", mas.index, foo);
+#endif
+ check++;
+ if (check > entry_count)
+ break;
+ }
+ rcu_read_unlock();
+#if check_erase2_debug > 2
+ pr_err("mas_for_each %d and count %d\n", check, entry_count);
+ mt_validate(mt);
+#endif
+
+ MT_BUG_ON(mt, check != entry_count);
+
+ MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL);
+ }
+}
+
+
+/* These tests were pulled from kvm tests. */
+static noinline void check_erase2_sets(struct maple_tree *mt)
+{
+ void *entry;
+ unsigned long start = 0;
+ unsigned long set[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140721266458624, 140737488351231,
+ERASE, 140721266458624, 140737488351231,
+STORE, 140721266458624, 140721266462719,
+STORE, 94735788949504, 94735789121535,
+ERASE, 94735788949504, 94735789121535,
+STORE, 94735788949504, 94735788965887,
+STORE, 94735788965888, 94735789121535,
+ERASE, 94735788965888, 94735789121535,
+STORE, 94735788965888, 94735789068287,
+STORE, 94735789068288, 94735789109247,
+STORE, 94735789109248, 94735789121535,
+STORE, 140253902692352, 140253902864383,
+ERASE, 140253902692352, 140253902864383,
+STORE, 140253902692352, 140253902696447,
+STORE, 140253902696448, 140253902864383,
+ };
+ unsigned long set2[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140735933583360, 140737488351231,
+ERASE, 140735933583360, 140737488351231,
+STORE, 140735933583360, 140735933587455,
+STORE, 94811003260928, 94811003432959,
+ERASE, 94811003260928, 94811003432959,
+STORE, 94811003260928, 94811003277311,
+STORE, 94811003277312, 94811003432959,
+ERASE, 94811003277312, 94811003432959,
+STORE, 94811003277312, 94811003379711,
+STORE, 94811003379712, 94811003420671,
+STORE, 94811003420672, 94811003432959,
+STORE, 140277094653952, 140277094825983,
+ERASE, 140277094653952, 140277094825983,
+STORE, 140277094653952, 140277094658047,
+STORE, 140277094658048, 140277094825983,
+ERASE, 140277094658048, 140277094825983,
+STORE, 140277094658048, 140277094780927,
+STORE, 140277094780928, 140277094813695,
+STORE, 140277094813696, 140277094821887,
+STORE, 140277094821888, 140277094825983,
+STORE, 140735933906944, 140735933911039,
+ };
+ unsigned long set3[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140735790264320, 140737488351231,
+ERASE, 140735790264320, 140737488351231,
+STORE, 140735790264320, 140735790268415,
+STORE, 94016597282816, 94016597454847,
+ERASE, 94016597282816, 94016597454847,
+STORE, 94016597282816, 94016597299199,
+STORE, 94016597299200, 94016597454847,
+ERASE, 94016597299200, 94016597454847,
+STORE, 94016597299200, 94016597401599,
+STORE, 94016597401600, 94016597442559,
+STORE, 94016597442560, 94016597454847,
+STORE, 140496959283200, 140496959455231,
+ERASE, 140496959283200, 140496959455231,
+STORE, 140496959283200, 140496959287295,
+STORE, 140496959287296, 140496959455231,
+ERASE, 140496959287296, 140496959455231,
+STORE, 140496959287296, 140496959410175,
+STORE, 140496959410176, 140496959442943,
+STORE, 140496959442944, 140496959451135,
+STORE, 140496959451136, 140496959455231,
+STORE, 140735791718400, 140735791722495,
+STORE, 140735791706112, 140735791718399,
+STORE, 47135835713536, 47135835721727,
+STORE, 47135835721728, 47135835729919,
+STORE, 47135835729920, 47135835893759,
+ERASE, 47135835729920, 47135835893759,
+STORE, 47135835729920, 47135835742207,
+STORE, 47135835742208, 47135835893759,
+STORE, 47135835840512, 47135835893759,
+STORE, 47135835742208, 47135835840511,
+ERASE, 47135835742208, 47135835840511,
+STORE, 47135835742208, 47135835840511,
+STORE, 47135835885568, 47135835893759,
+STORE, 47135835840512, 47135835885567,
+ERASE, 47135835840512, 47135835885567,
+STORE, 47135835840512, 47135835893759,
+ERASE, 47135835840512, 47135835893759,
+STORE, 47135835840512, 47135835885567,
+STORE, 47135835885568, 47135835893759,
+ };
+
+ unsigned long set4[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140728251703296, 140737488351231,
+ERASE, 140728251703296, 140737488351231,
+STORE, 140728251703296, 140728251707391,
+STORE, 94668429205504, 94668429377535,
+ERASE, 94668429205504, 94668429377535,
+STORE, 94668429205504, 94668429221887,
+STORE, 94668429221888, 94668429377535,
+ERASE, 94668429221888, 94668429377535,
+STORE, 94668429221888, 94668429324287,
+STORE, 94668429324288, 94668429365247,
+STORE, 94668429365248, 94668429377535,
+STORE, 47646523273216, 47646523445247,
+ERASE, 47646523273216, 47646523445247,
+STORE, 47646523273216, 47646523277311,
+STORE, 47646523277312, 47646523445247,
+ERASE, 47646523277312, 47646523445247,
+STORE, 47646523277312, 47646523400191,
+ };
+
+ unsigned long set5[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140726874062848, 140737488351231,
+ERASE, 140726874062848, 140737488351231,
+STORE, 140726874062848, 140726874066943,
+STORE, 94248892870656, 94248893042687,
+ERASE, 94248892870656, 94248893042687,
+STORE, 94248892870656, 94248892887039,
+STORE, 94248892887040, 94248893042687,
+ERASE, 94248892887040, 94248893042687,
+STORE, 94248892887040, 94248892989439,
+STORE, 94248892989440, 94248893030399,
+STORE, 94248893030400, 94248893042687,
+STORE, 47884786266112, 47884786438143,
+ERASE, 47884786266112, 47884786438143,
+STORE, 47884786266112, 47884786270207,
+STORE, 47884786270208, 47884786438143,
+ERASE, 47884786270208, 47884786438143,
+STORE, 47884786270208, 47884786393087,
+STORE, 47884786393088, 47884786425855,
+STORE, 47884786425856, 47884786434047,
+STORE, 47884786434048, 47884786438143,
+STORE, 140726874513408, 140726874517503,
+STORE, 140726874501120, 140726874513407,
+STORE, 47884786438144, 47884786446335,
+STORE, 47884786446336, 47884786454527,
+STORE, 47884786454528, 47884786618367,
+ERASE, 47884786454528, 47884786618367,
+STORE, 47884786454528, 47884786466815,
+STORE, 47884786466816, 47884786618367,
+STORE, 47884786565120, 47884786618367,
+STORE, 47884786466816, 47884786565119,
+ERASE, 47884786466816, 47884786565119,
+STORE, 47884786466816, 47884786565119,
+STORE, 47884786610176, 47884786618367,
+STORE, 47884786565120, 47884786610175,
+ERASE, 47884786565120, 47884786610175,
+STORE, 47884786565120, 47884786618367,
+ERASE, 47884786565120, 47884786618367,
+STORE, 47884786565120, 47884786610175,
+STORE, 47884786610176, 47884786618367,
+ERASE, 47884786610176, 47884786618367,
+STORE, 47884786610176, 47884786618367,
+STORE, 47884786618368, 47884789669887,
+STORE, 47884787163136, 47884789669887,
+STORE, 47884786618368, 47884787163135,
+ERASE, 47884787163136, 47884789669887,
+STORE, 47884787163136, 47884789448703,
+STORE, 47884789448704, 47884789669887,
+STORE, 47884788858880, 47884789448703,
+STORE, 47884787163136, 47884788858879,
+ERASE, 47884787163136, 47884788858879,
+STORE, 47884787163136, 47884788858879,
+STORE, 47884789444608, 47884789448703,
+STORE, 47884788858880, 47884789444607,
+ERASE, 47884788858880, 47884789444607,
+STORE, 47884788858880, 47884789444607,
+STORE, 47884789653504, 47884789669887,
+STORE, 47884789448704, 47884789653503,
+ERASE, 47884789448704, 47884789653503,
+STORE, 47884789448704, 47884789653503,
+ERASE, 47884789653504, 47884789669887,
+STORE, 47884789653504, 47884789669887,
+STORE, 47884789669888, 47884791508991,
+STORE, 47884789809152, 47884791508991,
+STORE, 47884789669888, 47884789809151,
+ERASE, 47884789809152, 47884791508991,
+STORE, 47884789809152, 47884791468031,
+STORE, 47884791468032, 47884791508991,
+STORE, 47884791152640, 47884791468031,
+STORE, 47884789809152, 47884791152639,
+ERASE, 47884789809152, 47884791152639,
+STORE, 47884789809152, 47884791152639,
+STORE, 47884791463936, 47884791468031,
+STORE, 47884791152640, 47884791463935,
+ERASE, 47884791152640, 47884791463935,
+STORE, 47884791152640, 47884791463935,
+STORE, 47884791492608, 47884791508991,
+STORE, 47884791468032, 47884791492607,
+ERASE, 47884791468032, 47884791492607,
+STORE, 47884791468032, 47884791492607,
+ERASE, 47884791492608, 47884791508991,
+STORE, 47884791492608, 47884791508991,
+STORE, 47884791508992, 47884791644159,
+ERASE, 47884791508992, 47884791644159,
+STORE, 47884791508992, 47884791533567,
+STORE, 47884791533568, 47884791644159,
+STORE, 47884791595008, 47884791644159,
+STORE, 47884791533568, 47884791595007,
+ERASE, 47884791533568, 47884791595007,
+STORE, 47884791533568, 47884791595007,
+STORE, 47884791619584, 47884791644159,
+STORE, 47884791595008, 47884791619583,
+ERASE, 47884791595008, 47884791619583,
+STORE, 47884791595008, 47884791644159,
+ERASE, 47884791595008, 47884791644159,
+STORE, 47884791595008, 47884791619583,
+STORE, 47884791619584, 47884791644159,
+STORE, 47884791627776, 47884791644159,
+STORE, 47884791619584, 47884791627775,
+ERASE, 47884791619584, 47884791627775,
+STORE, 47884791619584, 47884791627775,
+ERASE, 47884791627776, 47884791644159,
+STORE, 47884791627776, 47884791644159,
+STORE, 47884791644160, 47884791664639,
+ERASE, 47884791644160, 47884791664639,
+STORE, 47884791644160, 47884791648255,
+STORE, 47884791648256, 47884791664639,
+STORE, 47884791652352, 47884791664639,
+STORE, 47884791648256, 47884791652351,
+ERASE, 47884791648256, 47884791652351,
+STORE, 47884791648256, 47884791652351,
+STORE, 47884791656448, 47884791664639,
+STORE, 47884791652352, 47884791656447,
+ERASE, 47884791652352, 47884791656447,
+STORE, 47884791652352, 47884791664639,
+ERASE, 47884791652352, 47884791664639,
+STORE, 47884791652352, 47884791656447,
+STORE, 47884791656448, 47884791664639,
+ERASE, 47884791656448, 47884791664639,
+STORE, 47884791656448, 47884791664639,
+STORE, 47884791664640, 47884791672831,
+ERASE, 47884791468032, 47884791492607,
+STORE, 47884791468032, 47884791484415,
+STORE, 47884791484416, 47884791492607,
+ERASE, 47884791656448, 47884791664639,
+STORE, 47884791656448, 47884791660543,
+STORE, 47884791660544, 47884791664639,
+ERASE, 47884791619584, 47884791627775,
+STORE, 47884791619584, 47884791623679,
+STORE, 47884791623680, 47884791627775,
+ };
+
+ unsigned long set6[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140722999021568, 140737488351231,
+ERASE, 140722999021568, 140737488351231,
+STORE, 140722999021568, 140722999025663,
+STORE, 94901500268544, 94901500440575,
+ERASE, 94901500268544, 94901500440575,
+STORE, 94901500268544, 94901500284927,
+STORE, 94901500284928, 94901500440575,
+ERASE, 94901500284928, 94901500440575,
+STORE, 94901500284928, 94901500387327,
+STORE, 94901500387328, 94901500428287,
+STORE, 94901500428288, 94901500440575,
+STORE, 47430426660864, 47430426832895,
+ERASE, 47430426660864, 47430426832895,
+STORE, 47430426660864, 47430426664959,
+STORE, 47430426664960, 47430426832895,
+ERASE, 47430426664960, 47430426832895,
+STORE, 47430426664960, 47430426787839,
+STORE, 47430426787840, 47430426820607,
+STORE, 47430426820608, 47430426828799,
+STORE, 47430426828800, 47430426832895,
+STORE, 140722999115776, 140722999119871,
+STORE, 140722999103488, 140722999115775,
+STORE, 47430426832896, 47430426841087,
+STORE, 47430426841088, 47430426849279,
+STORE, 47430426849280, 47430427013119,
+ERASE, 47430426849280, 47430427013119,
+STORE, 47430426849280, 47430426861567,
+STORE, 47430426861568, 47430427013119,
+STORE, 47430426959872, 47430427013119,
+STORE, 47430426861568, 47430426959871,
+ERASE, 47430426861568, 47430426959871,
+STORE, 47430426861568, 47430426959871,
+STORE, 47430427004928, 47430427013119,
+STORE, 47430426959872, 47430427004927,
+ERASE, 47430426959872, 47430427004927,
+STORE, 47430426959872, 47430427013119,
+ERASE, 47430426959872, 47430427013119,
+STORE, 47430426959872, 47430427004927,
+STORE, 47430427004928, 47430427013119,
+ERASE, 47430427004928, 47430427013119,
+STORE, 47430427004928, 47430427013119,
+STORE, 47430427013120, 47430430064639,
+STORE, 47430427557888, 47430430064639,
+STORE, 47430427013120, 47430427557887,
+ERASE, 47430427557888, 47430430064639,
+STORE, 47430427557888, 47430429843455,
+STORE, 47430429843456, 47430430064639,
+STORE, 47430429253632, 47430429843455,
+STORE, 47430427557888, 47430429253631,
+ERASE, 47430427557888, 47430429253631,
+STORE, 47430427557888, 47430429253631,
+STORE, 47430429839360, 47430429843455,
+STORE, 47430429253632, 47430429839359,
+ERASE, 47430429253632, 47430429839359,
+STORE, 47430429253632, 47430429839359,
+STORE, 47430430048256, 47430430064639,
+STORE, 47430429843456, 47430430048255,
+ERASE, 47430429843456, 47430430048255,
+STORE, 47430429843456, 47430430048255,
+ERASE, 47430430048256, 47430430064639,
+STORE, 47430430048256, 47430430064639,
+STORE, 47430430064640, 47430431903743,
+STORE, 47430430203904, 47430431903743,
+STORE, 47430430064640, 47430430203903,
+ERASE, 47430430203904, 47430431903743,
+STORE, 47430430203904, 47430431862783,
+STORE, 47430431862784, 47430431903743,
+STORE, 47430431547392, 47430431862783,
+STORE, 47430430203904, 47430431547391,
+ERASE, 47430430203904, 47430431547391,
+STORE, 47430430203904, 47430431547391,
+STORE, 47430431858688, 47430431862783,
+STORE, 47430431547392, 47430431858687,
+ERASE, 47430431547392, 47430431858687,
+STORE, 47430431547392, 47430431858687,
+STORE, 47430431887360, 47430431903743,
+STORE, 47430431862784, 47430431887359,
+ERASE, 47430431862784, 47430431887359,
+STORE, 47430431862784, 47430431887359,
+ERASE, 47430431887360, 47430431903743,
+STORE, 47430431887360, 47430431903743,
+STORE, 47430431903744, 47430432038911,
+ERASE, 47430431903744, 47430432038911,
+STORE, 47430431903744, 47430431928319,
+STORE, 47430431928320, 47430432038911,
+STORE, 47430431989760, 47430432038911,
+STORE, 47430431928320, 47430431989759,
+ERASE, 47430431928320, 47430431989759,
+STORE, 47430431928320, 47430431989759,
+STORE, 47430432014336, 47430432038911,
+STORE, 47430431989760, 47430432014335,
+ERASE, 47430431989760, 47430432014335,
+STORE, 47430431989760, 47430432038911,
+ERASE, 47430431989760, 47430432038911,
+STORE, 47430431989760, 47430432014335,
+STORE, 47430432014336, 47430432038911,
+STORE, 47430432022528, 47430432038911,
+STORE, 47430432014336, 47430432022527,
+ERASE, 47430432014336, 47430432022527,
+STORE, 47430432014336, 47430432022527,
+ERASE, 47430432022528, 47430432038911,
+STORE, 47430432022528, 47430432038911,
+STORE, 47430432038912, 47430432059391,
+ERASE, 47430432038912, 47430432059391,
+STORE, 47430432038912, 47430432043007,
+STORE, 47430432043008, 47430432059391,
+STORE, 47430432047104, 47430432059391,
+STORE, 47430432043008, 47430432047103,
+ERASE, 47430432043008, 47430432047103,
+STORE, 47430432043008, 47430432047103,
+STORE, 47430432051200, 47430432059391,
+STORE, 47430432047104, 47430432051199,
+ERASE, 47430432047104, 47430432051199,
+STORE, 47430432047104, 47430432059391,
+ERASE, 47430432047104, 47430432059391,
+STORE, 47430432047104, 47430432051199,
+STORE, 47430432051200, 47430432059391,
+ERASE, 47430432051200, 47430432059391,
+STORE, 47430432051200, 47430432059391,
+STORE, 47430432059392, 47430432067583,
+ERASE, 47430431862784, 47430431887359,
+STORE, 47430431862784, 47430431879167,
+STORE, 47430431879168, 47430431887359,
+ERASE, 47430432051200, 47430432059391,
+STORE, 47430432051200, 47430432055295,
+STORE, 47430432055296, 47430432059391,
+ERASE, 47430432014336, 47430432022527,
+STORE, 47430432014336, 47430432018431,
+STORE, 47430432018432, 47430432022527,
+ };
+ unsigned long set7[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140729808330752, 140737488351231,
+ERASE, 140729808330752, 140737488351231,
+STORE, 140729808330752, 140729808334847,
+STORE, 94629632020480, 94629632192511,
+ERASE, 94629632020480, 94629632192511,
+STORE, 94629632020480, 94629632036863,
+STORE, 94629632036864, 94629632192511,
+ERASE, 94629632036864, 94629632192511,
+STORE, 94629632036864, 94629632139263,
+STORE, 94629632139264, 94629632180223,
+STORE, 94629632180224, 94629632192511,
+STORE, 47439981776896, 47439981948927,
+ERASE, 47439981776896, 47439981948927,
+STORE, 47439981776896, 47439981780991,
+STORE, 47439981780992, 47439981948927,
+ERASE, 47439981780992, 47439981948927,
+STORE, 47439981780992, 47439981903871,
+STORE, 47439981903872, 47439981936639,
+STORE, 47439981936640, 47439981944831,
+STORE, 47439981944832, 47439981948927,
+STORE, 140729808474112, 140729808478207,
+STORE, 140729808461824, 140729808474111,
+STORE, 47439981948928, 47439981957119,
+STORE, 47439981957120, 47439981965311,
+STORE, 47439981965312, 47439982129151,
+ERASE, 47439981965312, 47439982129151,
+STORE, 47439981965312, 47439981977599,
+STORE, 47439981977600, 47439982129151,
+STORE, 47439982075904, 47439982129151,
+STORE, 47439981977600, 47439982075903,
+ERASE, 47439981977600, 47439982075903,
+STORE, 47439981977600, 47439982075903,
+STORE, 47439982120960, 47439982129151,
+STORE, 47439982075904, 47439982120959,
+ERASE, 47439982075904, 47439982120959,
+STORE, 47439982075904, 47439982129151,
+ERASE, 47439982075904, 47439982129151,
+STORE, 47439982075904, 47439982120959,
+STORE, 47439982120960, 47439982129151,
+ERASE, 47439982120960, 47439982129151,
+STORE, 47439982120960, 47439982129151,
+STORE, 47439982129152, 47439985180671,
+STORE, 47439982673920, 47439985180671,
+STORE, 47439982129152, 47439982673919,
+ERASE, 47439982673920, 47439985180671,
+STORE, 47439982673920, 47439984959487,
+STORE, 47439984959488, 47439985180671,
+STORE, 47439984369664, 47439984959487,
+STORE, 47439982673920, 47439984369663,
+ERASE, 47439982673920, 47439984369663,
+STORE, 47439982673920, 47439984369663,
+STORE, 47439984955392, 47439984959487,
+STORE, 47439984369664, 47439984955391,
+ERASE, 47439984369664, 47439984955391,
+STORE, 47439984369664, 47439984955391,
+STORE, 47439985164288, 47439985180671,
+STORE, 47439984959488, 47439985164287,
+ERASE, 47439984959488, 47439985164287,
+STORE, 47439984959488, 47439985164287,
+ERASE, 47439985164288, 47439985180671,
+STORE, 47439985164288, 47439985180671,
+STORE, 47439985180672, 47439987019775,
+STORE, 47439985319936, 47439987019775,
+STORE, 47439985180672, 47439985319935,
+ERASE, 47439985319936, 47439987019775,
+STORE, 47439985319936, 47439986978815,
+STORE, 47439986978816, 47439987019775,
+STORE, 47439986663424, 47439986978815,
+STORE, 47439985319936, 47439986663423,
+ERASE, 47439985319936, 47439986663423,
+STORE, 47439985319936, 47439986663423,
+STORE, 47439986974720, 47439986978815,
+STORE, 47439986663424, 47439986974719,
+ERASE, 47439986663424, 47439986974719,
+STORE, 47439986663424, 47439986974719,
+STORE, 47439987003392, 47439987019775,
+STORE, 47439986978816, 47439987003391,
+ERASE, 47439986978816, 47439987003391,
+STORE, 47439986978816, 47439987003391,
+ERASE, 47439987003392, 47439987019775,
+STORE, 47439987003392, 47439987019775,
+STORE, 47439987019776, 47439987154943,
+ERASE, 47439987019776, 47439987154943,
+STORE, 47439987019776, 47439987044351,
+STORE, 47439987044352, 47439987154943,
+STORE, 47439987105792, 47439987154943,
+STORE, 47439987044352, 47439987105791,
+ERASE, 47439987044352, 47439987105791,
+STORE, 47439987044352, 47439987105791,
+STORE, 47439987130368, 47439987154943,
+STORE, 47439987105792, 47439987130367,
+ERASE, 47439987105792, 47439987130367,
+STORE, 47439987105792, 47439987154943,
+ERASE, 47439987105792, 47439987154943,
+STORE, 47439987105792, 47439987130367,
+STORE, 47439987130368, 47439987154943,
+STORE, 47439987138560, 47439987154943,
+STORE, 47439987130368, 47439987138559,
+ERASE, 47439987130368, 47439987138559,
+STORE, 47439987130368, 47439987138559,
+ERASE, 47439987138560, 47439987154943,
+STORE, 47439987138560, 47439987154943,
+STORE, 47439987154944, 47439987175423,
+ERASE, 47439987154944, 47439987175423,
+STORE, 47439987154944, 47439987159039,
+STORE, 47439987159040, 47439987175423,
+STORE, 47439987163136, 47439987175423,
+STORE, 47439987159040, 47439987163135,
+ERASE, 47439987159040, 47439987163135,
+STORE, 47439987159040, 47439987163135,
+STORE, 47439987167232, 47439987175423,
+STORE, 47439987163136, 47439987167231,
+ERASE, 47439987163136, 47439987167231,
+STORE, 47439987163136, 47439987175423,
+ERASE, 47439987163136, 47439987175423,
+STORE, 47439987163136, 47439987167231,
+STORE, 47439987167232, 47439987175423,
+ERASE, 47439987167232, 47439987175423,
+STORE, 47439987167232, 47439987175423,
+STORE, 47439987175424, 47439987183615,
+ERASE, 47439986978816, 47439987003391,
+STORE, 47439986978816, 47439986995199,
+STORE, 47439986995200, 47439987003391,
+ERASE, 47439987167232, 47439987175423,
+STORE, 47439987167232, 47439987171327,
+STORE, 47439987171328, 47439987175423,
+ERASE, 47439987130368, 47439987138559,
+STORE, 47439987130368, 47439987134463,
+STORE, 47439987134464, 47439987138559,
+ };
+ unsigned long set8[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140722482974720, 140737488351231,
+ERASE, 140722482974720, 140737488351231,
+STORE, 140722482974720, 140722482978815,
+STORE, 94121505034240, 94121505206271,
+ERASE, 94121505034240, 94121505206271,
+STORE, 94121505034240, 94121505050623,
+STORE, 94121505050624, 94121505206271,
+ERASE, 94121505050624, 94121505206271,
+STORE, 94121505050624, 94121505153023,
+STORE, 94121505153024, 94121505193983,
+STORE, 94121505193984, 94121505206271,
+STORE, 47708483284992, 47708483457023,
+ERASE, 47708483284992, 47708483457023,
+STORE, 47708483284992, 47708483289087,
+STORE, 47708483289088, 47708483457023,
+ERASE, 47708483289088, 47708483457023,
+STORE, 47708483289088, 47708483411967,
+STORE, 47708483411968, 47708483444735,
+STORE, 47708483444736, 47708483452927,
+STORE, 47708483452928, 47708483457023,
+STORE, 140722483142656, 140722483146751,
+STORE, 140722483130368, 140722483142655,
+STORE, 47708483457024, 47708483465215,
+STORE, 47708483465216, 47708483473407,
+STORE, 47708483473408, 47708483637247,
+ERASE, 47708483473408, 47708483637247,
+STORE, 47708483473408, 47708483485695,
+STORE, 47708483485696, 47708483637247,
+STORE, 47708483584000, 47708483637247,
+STORE, 47708483485696, 47708483583999,
+ERASE, 47708483485696, 47708483583999,
+STORE, 47708483485696, 47708483583999,
+STORE, 47708483629056, 47708483637247,
+STORE, 47708483584000, 47708483629055,
+ERASE, 47708483584000, 47708483629055,
+STORE, 47708483584000, 47708483637247,
+ERASE, 47708483584000, 47708483637247,
+STORE, 47708483584000, 47708483629055,
+STORE, 47708483629056, 47708483637247,
+ERASE, 47708483629056, 47708483637247,
+STORE, 47708483629056, 47708483637247,
+STORE, 47708483637248, 47708486688767,
+STORE, 47708484182016, 47708486688767,
+STORE, 47708483637248, 47708484182015,
+ERASE, 47708484182016, 47708486688767,
+STORE, 47708484182016, 47708486467583,
+STORE, 47708486467584, 47708486688767,
+STORE, 47708485877760, 47708486467583,
+STORE, 47708484182016, 47708485877759,
+ERASE, 47708484182016, 47708485877759,
+STORE, 47708484182016, 47708485877759,
+STORE, 47708486463488, 47708486467583,
+STORE, 47708485877760, 47708486463487,
+ERASE, 47708485877760, 47708486463487,
+STORE, 47708485877760, 47708486463487,
+STORE, 47708486672384, 47708486688767,
+STORE, 47708486467584, 47708486672383,
+ERASE, 47708486467584, 47708486672383,
+STORE, 47708486467584, 47708486672383,
+ERASE, 47708486672384, 47708486688767,
+STORE, 47708486672384, 47708486688767,
+STORE, 47708486688768, 47708488527871,
+STORE, 47708486828032, 47708488527871,
+STORE, 47708486688768, 47708486828031,
+ERASE, 47708486828032, 47708488527871,
+STORE, 47708486828032, 47708488486911,
+STORE, 47708488486912, 47708488527871,
+STORE, 47708488171520, 47708488486911,
+STORE, 47708486828032, 47708488171519,
+ERASE, 47708486828032, 47708488171519,
+STORE, 47708486828032, 47708488171519,
+STORE, 47708488482816, 47708488486911,
+STORE, 47708488171520, 47708488482815,
+ERASE, 47708488171520, 47708488482815,
+STORE, 47708488171520, 47708488482815,
+STORE, 47708488511488, 47708488527871,
+STORE, 47708488486912, 47708488511487,
+ERASE, 47708488486912, 47708488511487,
+STORE, 47708488486912, 47708488511487,
+ERASE, 47708488511488, 47708488527871,
+STORE, 47708488511488, 47708488527871,
+STORE, 47708488527872, 47708488663039,
+ERASE, 47708488527872, 47708488663039,
+STORE, 47708488527872, 47708488552447,
+STORE, 47708488552448, 47708488663039,
+STORE, 47708488613888, 47708488663039,
+STORE, 47708488552448, 47708488613887,
+ERASE, 47708488552448, 47708488613887,
+STORE, 47708488552448, 47708488613887,
+STORE, 47708488638464, 47708488663039,
+STORE, 47708488613888, 47708488638463,
+ERASE, 47708488613888, 47708488638463,
+STORE, 47708488613888, 47708488663039,
+ERASE, 47708488613888, 47708488663039,
+STORE, 47708488613888, 47708488638463,
+STORE, 47708488638464, 47708488663039,
+STORE, 47708488646656, 47708488663039,
+STORE, 47708488638464, 47708488646655,
+ERASE, 47708488638464, 47708488646655,
+STORE, 47708488638464, 47708488646655,
+ERASE, 47708488646656, 47708488663039,
+STORE, 47708488646656, 47708488663039,
+STORE, 47708488663040, 47708488683519,
+ERASE, 47708488663040, 47708488683519,
+STORE, 47708488663040, 47708488667135,
+STORE, 47708488667136, 47708488683519,
+STORE, 47708488671232, 47708488683519,
+STORE, 47708488667136, 47708488671231,
+ERASE, 47708488667136, 47708488671231,
+STORE, 47708488667136, 47708488671231,
+STORE, 47708488675328, 47708488683519,
+STORE, 47708488671232, 47708488675327,
+ERASE, 47708488671232, 47708488675327,
+STORE, 47708488671232, 47708488683519,
+ERASE, 47708488671232, 47708488683519,
+STORE, 47708488671232, 47708488675327,
+STORE, 47708488675328, 47708488683519,
+ERASE, 47708488675328, 47708488683519,
+STORE, 47708488675328, 47708488683519,
+STORE, 47708488683520, 47708488691711,
+ERASE, 47708488486912, 47708488511487,
+STORE, 47708488486912, 47708488503295,
+STORE, 47708488503296, 47708488511487,
+ERASE, 47708488675328, 47708488683519,
+STORE, 47708488675328, 47708488679423,
+STORE, 47708488679424, 47708488683519,
+ERASE, 47708488638464, 47708488646655,
+STORE, 47708488638464, 47708488642559,
+STORE, 47708488642560, 47708488646655,
+ };
+
+ unsigned long set9[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140736427839488, 140737488351231,
+ERASE, 140736427839488, 140736427839488,
+STORE, 140736427839488, 140736427843583,
+STORE, 94071213395968, 94071213567999,
+ERASE, 94071213395968, 94071213395968,
+STORE, 94071213395968, 94071213412351,
+STORE, 94071213412352, 94071213567999,
+ERASE, 94071213412352, 94071213412352,
+STORE, 94071213412352, 94071213514751,
+STORE, 94071213514752, 94071213555711,
+STORE, 94071213555712, 94071213567999,
+STORE, 139968410644480, 139968410816511,
+ERASE, 139968410644480, 139968410644480,
+STORE, 139968410644480, 139968410648575,
+STORE, 139968410648576, 139968410816511,
+ERASE, 139968410648576, 139968410648576,
+STORE, 139968410648576, 139968410771455,
+STORE, 139968410771456, 139968410804223,
+STORE, 139968410804224, 139968410812415,
+STORE, 139968410812416, 139968410816511,
+STORE, 140736429277184, 140736429281279,
+STORE, 140736429264896, 140736429277183,
+STORE, 47664384352256, 47664384360447,
+STORE, 47664384360448, 47664384368639,
+STORE, 47664384368640, 47664384532479,
+ERASE, 47664384368640, 47664384368640,
+STORE, 47664384368640, 47664384380927,
+STORE, 47664384380928, 47664384532479,
+STORE, 47664384479232, 47664384532479,
+STORE, 47664384380928, 47664384479231,
+ERASE, 47664384380928, 47664384380928,
+STORE, 47664384380928, 47664384479231,
+STORE, 47664384524288, 47664384532479,
+STORE, 47664384479232, 47664384524287,
+ERASE, 47664384479232, 47664384479232,
+STORE, 47664384479232, 47664384532479,
+ERASE, 47664384479232, 47664384479232,
+STORE, 47664384479232, 47664384524287,
+STORE, 47664384524288, 47664384532479,
+ERASE, 47664384524288, 47664384524288,
+STORE, 47664384524288, 47664384532479,
+STORE, 47664384532480, 47664387583999,
+STORE, 47664385077248, 47664387583999,
+STORE, 47664384532480, 47664385077247,
+ERASE, 47664385077248, 47664385077248,
+STORE, 47664385077248, 47664387362815,
+STORE, 47664387362816, 47664387583999,
+STORE, 47664386772992, 47664387362815,
+STORE, 47664385077248, 47664386772991,
+ERASE, 47664385077248, 47664385077248,
+STORE, 47664385077248, 47664386772991,
+STORE, 47664387358720, 47664387362815,
+STORE, 47664386772992, 47664387358719,
+ERASE, 47664386772992, 47664386772992,
+STORE, 47664386772992, 47664387358719,
+STORE, 47664387567616, 47664387583999,
+STORE, 47664387362816, 47664387567615,
+ERASE, 47664387362816, 47664387362816,
+STORE, 47664387362816, 47664387567615,
+ERASE, 47664387567616, 47664387567616,
+STORE, 47664387567616, 47664387583999,
+STORE, 47664387584000, 47664389423103,
+STORE, 47664387723264, 47664389423103,
+STORE, 47664387584000, 47664387723263,
+ERASE, 47664387723264, 47664387723264,
+STORE, 47664387723264, 47664389382143,
+STORE, 47664389382144, 47664389423103,
+STORE, 47664389066752, 47664389382143,
+STORE, 47664387723264, 47664389066751,
+ERASE, 47664387723264, 47664387723264,
+STORE, 47664387723264, 47664389066751,
+STORE, 47664389378048, 47664389382143,
+STORE, 47664389066752, 47664389378047,
+ERASE, 47664389066752, 47664389066752,
+STORE, 47664389066752, 47664389378047,
+STORE, 47664389406720, 47664389423103,
+STORE, 47664389382144, 47664389406719,
+ERASE, 47664389382144, 47664389382144,
+STORE, 47664389382144, 47664389406719,
+ERASE, 47664389406720, 47664389406720,
+STORE, 47664389406720, 47664389423103,
+STORE, 47664389423104, 47664389558271,
+ERASE, 47664389423104, 47664389423104,
+STORE, 47664389423104, 47664389447679,
+STORE, 47664389447680, 47664389558271,
+STORE, 47664389509120, 47664389558271,
+STORE, 47664389447680, 47664389509119,
+ERASE, 47664389447680, 47664389447680,
+STORE, 47664389447680, 47664389509119,
+STORE, 47664389533696, 47664389558271,
+STORE, 47664389509120, 47664389533695,
+ERASE, 47664389509120, 47664389509120,
+STORE, 47664389509120, 47664389558271,
+ERASE, 47664389509120, 47664389509120,
+STORE, 47664389509120, 47664389533695,
+STORE, 47664389533696, 47664389558271,
+STORE, 47664389541888, 47664389558271,
+STORE, 47664389533696, 47664389541887,
+ERASE, 47664389533696, 47664389533696,
+STORE, 47664389533696, 47664389541887,
+ERASE, 47664389541888, 47664389541888,
+STORE, 47664389541888, 47664389558271,
+STORE, 47664389558272, 47664389578751,
+ERASE, 47664389558272, 47664389558272,
+STORE, 47664389558272, 47664389562367,
+STORE, 47664389562368, 47664389578751,
+STORE, 47664389566464, 47664389578751,
+STORE, 47664389562368, 47664389566463,
+ERASE, 47664389562368, 47664389562368,
+STORE, 47664389562368, 47664389566463,
+STORE, 47664389570560, 47664389578751,
+STORE, 47664389566464, 47664389570559,
+ERASE, 47664389566464, 47664389566464,
+STORE, 47664389566464, 47664389578751,
+ERASE, 47664389566464, 47664389566464,
+STORE, 47664389566464, 47664389570559,
+STORE, 47664389570560, 47664389578751,
+ERASE, 47664389570560, 47664389570560,
+STORE, 47664389570560, 47664389578751,
+STORE, 47664389578752, 47664389586943,
+ERASE, 47664389382144, 47664389382144,
+STORE, 47664389382144, 47664389398527,
+STORE, 47664389398528, 47664389406719,
+ERASE, 47664389570560, 47664389570560,
+STORE, 47664389570560, 47664389574655,
+STORE, 47664389574656, 47664389578751,
+ERASE, 47664389533696, 47664389533696,
+STORE, 47664389533696, 47664389537791,
+STORE, 47664389537792, 47664389541887,
+ERASE, 47664387362816, 47664387362816,
+STORE, 47664387362816, 47664387559423,
+STORE, 47664387559424, 47664387567615,
+ERASE, 47664384524288, 47664384524288,
+STORE, 47664384524288, 47664384528383,
+STORE, 47664384528384, 47664384532479,
+ERASE, 94071213555712, 94071213555712,
+STORE, 94071213555712, 94071213563903,
+STORE, 94071213563904, 94071213567999,
+ERASE, 139968410804224, 139968410804224,
+STORE, 139968410804224, 139968410808319,
+STORE, 139968410808320, 139968410812415,
+ERASE, 47664384352256, 47664384352256,
+STORE, 94071244402688, 94071244537855,
+STORE, 140737488347136, 140737488351231,
+STORE, 140728271503360, 140737488351231,
+ERASE, 140728271503360, 140728271503360,
+STORE, 140728271503360, 140728271507455,
+STORE, 94410361982976, 94410362155007,
+ERASE, 94410361982976, 94410361982976,
+STORE, 94410361982976, 94410361999359,
+STORE, 94410361999360, 94410362155007,
+ERASE, 94410361999360, 94410361999360,
+STORE, 94410361999360, 94410362101759,
+STORE, 94410362101760, 94410362142719,
+STORE, 94410362142720, 94410362155007,
+STORE, 140351953997824, 140351954169855,
+ERASE, 140351953997824, 140351953997824,
+STORE, 140351953997824, 140351954001919,
+STORE, 140351954001920, 140351954169855,
+ERASE, 140351954001920, 140351954001920,
+STORE, 140351954001920, 140351954124799,
+STORE, 140351954124800, 140351954157567,
+STORE, 140351954157568, 140351954165759,
+STORE, 140351954165760, 140351954169855,
+STORE, 140728272429056, 140728272433151,
+STORE, 140728272416768, 140728272429055,
+STORE, 47280840998912, 47280841007103,
+STORE, 47280841007104, 47280841015295,
+STORE, 47280841015296, 47280841179135,
+ERASE, 47280841015296, 47280841015296,
+STORE, 47280841015296, 47280841027583,
+STORE, 47280841027584, 47280841179135,
+STORE, 47280841125888, 47280841179135,
+STORE, 47280841027584, 47280841125887,
+ERASE, 47280841027584, 47280841027584,
+STORE, 47280841027584, 47280841125887,
+STORE, 47280841170944, 47280841179135,
+STORE, 47280841125888, 47280841170943,
+ERASE, 47280841125888, 47280841125888,
+STORE, 47280841125888, 47280841179135,
+ERASE, 47280841125888, 47280841125888,
+STORE, 47280841125888, 47280841170943,
+STORE, 47280841170944, 47280841179135,
+ERASE, 47280841170944, 47280841170944,
+STORE, 47280841170944, 47280841179135,
+STORE, 47280841179136, 47280844230655,
+STORE, 47280841723904, 47280844230655,
+STORE, 47280841179136, 47280841723903,
+ERASE, 47280841723904, 47280841723904,
+STORE, 47280841723904, 47280844009471,
+STORE, 47280844009472, 47280844230655,
+STORE, 47280843419648, 47280844009471,
+STORE, 47280841723904, 47280843419647,
+ERASE, 47280841723904, 47280841723904,
+STORE, 47280841723904, 47280843419647,
+STORE, 47280844005376, 47280844009471,
+STORE, 47280843419648, 47280844005375,
+ERASE, 47280843419648, 47280843419648,
+STORE, 47280843419648, 47280844005375,
+STORE, 47280844214272, 47280844230655,
+STORE, 47280844009472, 47280844214271,
+ERASE, 47280844009472, 47280844009472,
+STORE, 47280844009472, 47280844214271,
+ERASE, 47280844214272, 47280844214272,
+STORE, 47280844214272, 47280844230655,
+STORE, 47280844230656, 47280846069759,
+STORE, 47280844369920, 47280846069759,
+STORE, 47280844230656, 47280844369919,
+ERASE, 47280844369920, 47280844369920,
+STORE, 47280844369920, 47280846028799,
+STORE, 47280846028800, 47280846069759,
+STORE, 47280845713408, 47280846028799,
+STORE, 47280844369920, 47280845713407,
+ERASE, 47280844369920, 47280844369920,
+STORE, 47280844369920, 47280845713407,
+STORE, 47280846024704, 47280846028799,
+STORE, 47280845713408, 47280846024703,
+ERASE, 47280845713408, 47280845713408,
+STORE, 47280845713408, 47280846024703,
+STORE, 47280846053376, 47280846069759,
+STORE, 47280846028800, 47280846053375,
+ERASE, 47280846028800, 47280846028800,
+STORE, 47280846028800, 47280846053375,
+ERASE, 47280846053376, 47280846053376,
+STORE, 47280846053376, 47280846069759,
+STORE, 47280846069760, 47280846204927,
+ERASE, 47280846069760, 47280846069760,
+STORE, 47280846069760, 47280846094335,
+STORE, 47280846094336, 47280846204927,
+STORE, 47280846155776, 47280846204927,
+STORE, 47280846094336, 47280846155775,
+ERASE, 47280846094336, 47280846094336,
+STORE, 47280846094336, 47280846155775,
+STORE, 47280846180352, 47280846204927,
+STORE, 47280846155776, 47280846180351,
+ERASE, 47280846155776, 47280846155776,
+STORE, 47280846155776, 47280846204927,
+ERASE, 47280846155776, 47280846155776,
+STORE, 47280846155776, 47280846180351,
+STORE, 47280846180352, 47280846204927,
+STORE, 47280846188544, 47280846204927,
+STORE, 47280846180352, 47280846188543,
+ERASE, 47280846180352, 47280846180352,
+STORE, 47280846180352, 47280846188543,
+ERASE, 47280846188544, 47280846188544,
+STORE, 47280846188544, 47280846204927,
+STORE, 47280846204928, 47280846225407,
+ERASE, 47280846204928, 47280846204928,
+STORE, 47280846204928, 47280846209023,
+STORE, 47280846209024, 47280846225407,
+STORE, 47280846213120, 47280846225407,
+STORE, 47280846209024, 47280846213119,
+ERASE, 47280846209024, 47280846209024,
+STORE, 47280846209024, 47280846213119,
+STORE, 47280846217216, 47280846225407,
+STORE, 47280846213120, 47280846217215,
+ERASE, 47280846213120, 47280846213120,
+STORE, 47280846213120, 47280846225407,
+ERASE, 47280846213120, 47280846213120,
+STORE, 47280846213120, 47280846217215,
+STORE, 47280846217216, 47280846225407,
+ERASE, 47280846217216, 47280846217216,
+STORE, 47280846217216, 47280846225407,
+STORE, 47280846225408, 47280846233599,
+ERASE, 47280846028800, 47280846028800,
+STORE, 47280846028800, 47280846045183,
+STORE, 47280846045184, 47280846053375,
+ERASE, 47280846217216, 47280846217216,
+STORE, 47280846217216, 47280846221311,
+STORE, 47280846221312, 47280846225407,
+ERASE, 47280846180352, 47280846180352,
+STORE, 47280846180352, 47280846184447,
+STORE, 47280846184448, 47280846188543,
+ERASE, 47280844009472, 47280844009472,
+STORE, 47280844009472, 47280844206079,
+STORE, 47280844206080, 47280844214271,
+ERASE, 47280841170944, 47280841170944,
+STORE, 47280841170944, 47280841175039,
+STORE, 47280841175040, 47280841179135,
+ERASE, 94410362142720, 94410362142720,
+STORE, 94410362142720, 94410362150911,
+STORE, 94410362150912, 94410362155007,
+ERASE, 140351954157568, 140351954157568,
+STORE, 140351954157568, 140351954161663,
+STORE, 140351954161664, 140351954165759,
+ERASE, 47280840998912, 47280840998912,
+STORE, 94410379456512, 94410379591679,
+STORE, 140737488347136, 140737488351231,
+STORE, 140732946362368, 140737488351231,
+ERASE, 140732946362368, 140732946362368,
+STORE, 140732946362368, 140732946366463,
+STORE, 94352937934848, 94352938106879,
+ERASE, 94352937934848, 94352937934848,
+STORE, 94352937934848, 94352937951231,
+STORE, 94352937951232, 94352938106879,
+ERASE, 94352937951232, 94352937951232,
+STORE, 94352937951232, 94352938053631,
+STORE, 94352938053632, 94352938094591,
+STORE, 94352938094592, 94352938106879,
+STORE, 140595518742528, 140595518914559,
+ERASE, 140595518742528, 140595518742528,
+STORE, 140595518742528, 140595518746623,
+STORE, 140595518746624, 140595518914559,
+ERASE, 140595518746624, 140595518746624,
+STORE, 140595518746624, 140595518869503,
+STORE, 140595518869504, 140595518902271,
+STORE, 140595518902272, 140595518910463,
+STORE, 140595518910464, 140595518914559,
+STORE, 140732947468288, 140732947472383,
+STORE, 140732947456000, 140732947468287,
+STORE, 47037276254208, 47037276262399,
+STORE, 47037276262400, 47037276270591,
+STORE, 47037276270592, 47037276434431,
+ERASE, 47037276270592, 47037276270592,
+STORE, 47037276270592, 47037276282879,
+STORE, 47037276282880, 47037276434431,
+STORE, 47037276381184, 47037276434431,
+STORE, 47037276282880, 47037276381183,
+ERASE, 47037276282880, 47037276282880,
+STORE, 47037276282880, 47037276381183,
+STORE, 47037276426240, 47037276434431,
+STORE, 47037276381184, 47037276426239,
+ERASE, 47037276381184, 47037276381184,
+STORE, 47037276381184, 47037276434431,
+ERASE, 47037276381184, 47037276381184,
+STORE, 47037276381184, 47037276426239,
+STORE, 47037276426240, 47037276434431,
+ERASE, 47037276426240, 47037276426240,
+STORE, 47037276426240, 47037276434431,
+STORE, 47037276434432, 47037279485951,
+STORE, 47037276979200, 47037279485951,
+STORE, 47037276434432, 47037276979199,
+ERASE, 47037276979200, 47037276979200,
+STORE, 47037276979200, 47037279264767,
+STORE, 47037279264768, 47037279485951,
+STORE, 47037278674944, 47037279264767,
+STORE, 47037276979200, 47037278674943,
+ERASE, 47037276979200, 47037276979200,
+STORE, 47037276979200, 47037278674943,
+STORE, 47037279260672, 47037279264767,
+STORE, 47037278674944, 47037279260671,
+ERASE, 47037278674944, 47037278674944,
+STORE, 47037278674944, 47037279260671,
+STORE, 47037279469568, 47037279485951,
+STORE, 47037279264768, 47037279469567,
+ERASE, 47037279264768, 47037279264768,
+STORE, 47037279264768, 47037279469567,
+ERASE, 47037279469568, 47037279469568,
+STORE, 47037279469568, 47037279485951,
+STORE, 47037279485952, 47037281325055,
+STORE, 47037279625216, 47037281325055,
+STORE, 47037279485952, 47037279625215,
+ERASE, 47037279625216, 47037279625216,
+STORE, 47037279625216, 47037281284095,
+STORE, 47037281284096, 47037281325055,
+STORE, 47037280968704, 47037281284095,
+STORE, 47037279625216, 47037280968703,
+ERASE, 47037279625216, 47037279625216,
+STORE, 47037279625216, 47037280968703,
+STORE, 47037281280000, 47037281284095,
+STORE, 47037280968704, 47037281279999,
+ERASE, 47037280968704, 47037280968704,
+STORE, 47037280968704, 47037281279999,
+STORE, 47037281308672, 47037281325055,
+STORE, 47037281284096, 47037281308671,
+ERASE, 47037281284096, 47037281284096,
+STORE, 47037281284096, 47037281308671,
+ERASE, 47037281308672, 47037281308672,
+STORE, 47037281308672, 47037281325055,
+STORE, 47037281325056, 47037281460223,
+ERASE, 47037281325056, 47037281325056,
+STORE, 47037281325056, 47037281349631,
+STORE, 47037281349632, 47037281460223,
+STORE, 47037281411072, 47037281460223,
+STORE, 47037281349632, 47037281411071,
+ERASE, 47037281349632, 47037281349632,
+STORE, 47037281349632, 47037281411071,
+STORE, 47037281435648, 47037281460223,
+STORE, 47037281411072, 47037281435647,
+ERASE, 47037281411072, 47037281411072,
+STORE, 47037281411072, 47037281460223,
+ERASE, 47037281411072, 47037281411072,
+STORE, 47037281411072, 47037281435647,
+STORE, 47037281435648, 47037281460223,
+STORE, 47037281443840, 47037281460223,
+STORE, 47037281435648, 47037281443839,
+ERASE, 47037281435648, 47037281435648,
+STORE, 47037281435648, 47037281443839,
+ERASE, 47037281443840, 47037281443840,
+STORE, 47037281443840, 47037281460223,
+STORE, 47037281460224, 47037281480703,
+ERASE, 47037281460224, 47037281460224,
+STORE, 47037281460224, 47037281464319,
+STORE, 47037281464320, 47037281480703,
+STORE, 47037281468416, 47037281480703,
+STORE, 47037281464320, 47037281468415,
+ERASE, 47037281464320, 47037281464320,
+STORE, 47037281464320, 47037281468415,
+STORE, 47037281472512, 47037281480703,
+STORE, 47037281468416, 47037281472511,
+ERASE, 47037281468416, 47037281468416,
+STORE, 47037281468416, 47037281480703,
+ERASE, 47037281468416, 47037281468416,
+STORE, 47037281468416, 47037281472511,
+STORE, 47037281472512, 47037281480703,
+ERASE, 47037281472512, 47037281472512,
+STORE, 47037281472512, 47037281480703,
+STORE, 47037281480704, 47037281488895,
+ERASE, 47037281284096, 47037281284096,
+STORE, 47037281284096, 47037281300479,
+STORE, 47037281300480, 47037281308671,
+ERASE, 47037281472512, 47037281472512,
+STORE, 47037281472512, 47037281476607,
+STORE, 47037281476608, 47037281480703,
+ERASE, 47037281435648, 47037281435648,
+STORE, 47037281435648, 47037281439743,
+STORE, 47037281439744, 47037281443839,
+ERASE, 47037279264768, 47037279264768,
+STORE, 47037279264768, 47037279461375,
+STORE, 47037279461376, 47037279469567,
+ERASE, 47037276426240, 47037276426240,
+STORE, 47037276426240, 47037276430335,
+STORE, 47037276430336, 47037276434431,
+ERASE, 94352938094592, 94352938094592,
+STORE, 94352938094592, 94352938102783,
+STORE, 94352938102784, 94352938106879,
+ERASE, 140595518902272, 140595518902272,
+STORE, 140595518902272, 140595518906367,
+STORE, 140595518906368, 140595518910463,
+ERASE, 47037276254208, 47037276254208,
+STORE, 94352938438656, 94352938573823,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733506027520, 140737488351231,
+ERASE, 140733506027520, 140733506027520,
+STORE, 140733506027520, 140733506031615,
+STORE, 94150123073536, 94150123245567,
+ERASE, 94150123073536, 94150123073536,
+STORE, 94150123073536, 94150123089919,
+STORE, 94150123089920, 94150123245567,
+ERASE, 94150123089920, 94150123089920,
+STORE, 94150123089920, 94150123192319,
+STORE, 94150123192320, 94150123233279,
+STORE, 94150123233280, 94150123245567,
+STORE, 140081290375168, 140081290547199,
+ERASE, 140081290375168, 140081290375168,
+STORE, 140081290375168, 140081290379263,
+STORE, 140081290379264, 140081290547199,
+ERASE, 140081290379264, 140081290379264,
+STORE, 140081290379264, 140081290502143,
+STORE, 140081290502144, 140081290534911,
+STORE, 140081290534912, 140081290543103,
+STORE, 140081290543104, 140081290547199,
+STORE, 140733506707456, 140733506711551,
+STORE, 140733506695168, 140733506707455,
+STORE, 47551504621568, 47551504629759,
+STORE, 47551504629760, 47551504637951,
+STORE, 47551504637952, 47551504801791,
+ERASE, 47551504637952, 47551504637952,
+STORE, 47551504637952, 47551504650239,
+STORE, 47551504650240, 47551504801791,
+STORE, 47551504748544, 47551504801791,
+STORE, 47551504650240, 47551504748543,
+ERASE, 47551504650240, 47551504650240,
+STORE, 47551504650240, 47551504748543,
+STORE, 47551504793600, 47551504801791,
+STORE, 47551504748544, 47551504793599,
+ERASE, 47551504748544, 47551504748544,
+STORE, 47551504748544, 47551504801791,
+ERASE, 47551504748544, 47551504748544,
+STORE, 47551504748544, 47551504793599,
+STORE, 47551504793600, 47551504801791,
+ERASE, 47551504793600, 47551504793600,
+STORE, 47551504793600, 47551504801791,
+STORE, 47551504801792, 47551507853311,
+STORE, 47551505346560, 47551507853311,
+STORE, 47551504801792, 47551505346559,
+ERASE, 47551505346560, 47551505346560,
+STORE, 47551505346560, 47551507632127,
+STORE, 47551507632128, 47551507853311,
+STORE, 47551507042304, 47551507632127,
+STORE, 47551505346560, 47551507042303,
+ERASE, 47551505346560, 47551505346560,
+STORE, 47551505346560, 47551507042303,
+STORE, 47551507628032, 47551507632127,
+STORE, 47551507042304, 47551507628031,
+ERASE, 47551507042304, 47551507042304,
+STORE, 47551507042304, 47551507628031,
+STORE, 47551507836928, 47551507853311,
+STORE, 47551507632128, 47551507836927,
+ERASE, 47551507632128, 47551507632128,
+STORE, 47551507632128, 47551507836927,
+ERASE, 47551507836928, 47551507836928,
+STORE, 47551507836928, 47551507853311,
+STORE, 47551507853312, 47551509692415,
+STORE, 47551507992576, 47551509692415,
+STORE, 47551507853312, 47551507992575,
+ERASE, 47551507992576, 47551507992576,
+STORE, 47551507992576, 47551509651455,
+STORE, 47551509651456, 47551509692415,
+STORE, 47551509336064, 47551509651455,
+STORE, 47551507992576, 47551509336063,
+ERASE, 47551507992576, 47551507992576,
+STORE, 47551507992576, 47551509336063,
+STORE, 47551509647360, 47551509651455,
+STORE, 47551509336064, 47551509647359,
+ERASE, 47551509336064, 47551509336064,
+STORE, 47551509336064, 47551509647359,
+STORE, 47551509676032, 47551509692415,
+STORE, 47551509651456, 47551509676031,
+ERASE, 47551509651456, 47551509651456,
+STORE, 47551509651456, 47551509676031,
+ERASE, 47551509676032, 47551509676032,
+STORE, 47551509676032, 47551509692415,
+STORE, 47551509692416, 47551509827583,
+ERASE, 47551509692416, 47551509692416,
+STORE, 47551509692416, 47551509716991,
+STORE, 47551509716992, 47551509827583,
+STORE, 47551509778432, 47551509827583,
+STORE, 47551509716992, 47551509778431,
+ERASE, 47551509716992, 47551509716992,
+STORE, 47551509716992, 47551509778431,
+STORE, 47551509803008, 47551509827583,
+STORE, 47551509778432, 47551509803007,
+ERASE, 47551509778432, 47551509778432,
+STORE, 47551509778432, 47551509827583,
+ERASE, 47551509778432, 47551509778432,
+STORE, 47551509778432, 47551509803007,
+STORE, 47551509803008, 47551509827583,
+STORE, 47551509811200, 47551509827583,
+STORE, 47551509803008, 47551509811199,
+ERASE, 47551509803008, 47551509803008,
+STORE, 47551509803008, 47551509811199,
+ERASE, 47551509811200, 47551509811200,
+STORE, 47551509811200, 47551509827583,
+STORE, 47551509827584, 47551509848063,
+ERASE, 47551509827584, 47551509827584,
+STORE, 47551509827584, 47551509831679,
+STORE, 47551509831680, 47551509848063,
+STORE, 47551509835776, 47551509848063,
+STORE, 47551509831680, 47551509835775,
+ERASE, 47551509831680, 47551509831680,
+STORE, 47551509831680, 47551509835775,
+STORE, 47551509839872, 47551509848063,
+STORE, 47551509835776, 47551509839871,
+ERASE, 47551509835776, 47551509835776,
+STORE, 47551509835776, 47551509848063,
+ERASE, 47551509835776, 47551509835776,
+STORE, 47551509835776, 47551509839871,
+STORE, 47551509839872, 47551509848063,
+ERASE, 47551509839872, 47551509839872,
+STORE, 47551509839872, 47551509848063,
+STORE, 47551509848064, 47551509856255,
+ERASE, 47551509651456, 47551509651456,
+STORE, 47551509651456, 47551509667839,
+STORE, 47551509667840, 47551509676031,
+ERASE, 47551509839872, 47551509839872,
+STORE, 47551509839872, 47551509843967,
+STORE, 47551509843968, 47551509848063,
+ERASE, 47551509803008, 47551509803008,
+STORE, 47551509803008, 47551509807103,
+STORE, 47551509807104, 47551509811199,
+ERASE, 47551507632128, 47551507632128,
+STORE, 47551507632128, 47551507828735,
+STORE, 47551507828736, 47551507836927,
+ERASE, 47551504793600, 47551504793600,
+STORE, 47551504793600, 47551504797695,
+STORE, 47551504797696, 47551504801791,
+ERASE, 94150123233280, 94150123233280,
+STORE, 94150123233280, 94150123241471,
+STORE, 94150123241472, 94150123245567,
+ERASE, 140081290534912, 140081290534912,
+STORE, 140081290534912, 140081290539007,
+STORE, 140081290539008, 140081290543103,
+ERASE, 47551504621568, 47551504621568,
+STORE, 94150148112384, 94150148247551,
+STORE, 140737488347136, 140737488351231,
+STORE, 140734389334016, 140737488351231,
+ERASE, 140734389334016, 140734389334016,
+STORE, 140734389334016, 140734389338111,
+STORE, 94844636606464, 94844636778495,
+ERASE, 94844636606464, 94844636606464,
+STORE, 94844636606464, 94844636622847,
+STORE, 94844636622848, 94844636778495,
+ERASE, 94844636622848, 94844636622848,
+STORE, 94844636622848, 94844636725247,
+STORE, 94844636725248, 94844636766207,
+STORE, 94844636766208, 94844636778495,
+STORE, 139922765217792, 139922765389823,
+ERASE, 139922765217792, 139922765217792,
+STORE, 139922765217792, 139922765221887,
+STORE, 139922765221888, 139922765389823,
+ERASE, 139922765221888, 139922765221888,
+STORE, 139922765221888, 139922765344767,
+STORE, 139922765344768, 139922765377535,
+STORE, 139922765377536, 139922765385727,
+STORE, 139922765385728, 139922765389823,
+STORE, 140734389678080, 140734389682175,
+STORE, 140734389665792, 140734389678079,
+STORE, 47710029778944, 47710029787135,
+STORE, 47710029787136, 47710029795327,
+STORE, 47710029795328, 47710029959167,
+ERASE, 47710029795328, 47710029795328,
+STORE, 47710029795328, 47710029807615,
+STORE, 47710029807616, 47710029959167,
+STORE, 47710029905920, 47710029959167,
+STORE, 47710029807616, 47710029905919,
+ERASE, 47710029807616, 47710029807616,
+STORE, 47710029807616, 47710029905919,
+STORE, 47710029950976, 47710029959167,
+STORE, 47710029905920, 47710029950975,
+ERASE, 47710029905920, 47710029905920,
+STORE, 47710029905920, 47710029959167,
+ERASE, 47710029905920, 47710029905920,
+STORE, 47710029905920, 47710029950975,
+STORE, 47710029950976, 47710029959167,
+ERASE, 47710029950976, 47710029950976,
+STORE, 47710029950976, 47710029959167,
+STORE, 47710029959168, 47710033010687,
+STORE, 47710030503936, 47710033010687,
+STORE, 47710029959168, 47710030503935,
+ERASE, 47710030503936, 47710030503936,
+STORE, 47710030503936, 47710032789503,
+STORE, 47710032789504, 47710033010687,
+STORE, 47710032199680, 47710032789503,
+STORE, 47710030503936, 47710032199679,
+ERASE, 47710030503936, 47710030503936,
+STORE, 47710030503936, 47710032199679,
+STORE, 47710032785408, 47710032789503,
+STORE, 47710032199680, 47710032785407,
+ERASE, 47710032199680, 47710032199680,
+STORE, 47710032199680, 47710032785407,
+STORE, 47710032994304, 47710033010687,
+STORE, 47710032789504, 47710032994303,
+ERASE, 47710032789504, 47710032789504,
+STORE, 47710032789504, 47710032994303,
+ERASE, 47710032994304, 47710032994304,
+STORE, 47710032994304, 47710033010687,
+STORE, 47710033010688, 47710034849791,
+STORE, 47710033149952, 47710034849791,
+STORE, 47710033010688, 47710033149951,
+ERASE, 47710033149952, 47710033149952,
+STORE, 47710033149952, 47710034808831,
+STORE, 47710034808832, 47710034849791,
+STORE, 47710034493440, 47710034808831,
+STORE, 47710033149952, 47710034493439,
+ERASE, 47710033149952, 47710033149952,
+STORE, 47710033149952, 47710034493439,
+STORE, 47710034804736, 47710034808831,
+STORE, 47710034493440, 47710034804735,
+ERASE, 47710034493440, 47710034493440,
+STORE, 47710034493440, 47710034804735,
+STORE, 47710034833408, 47710034849791,
+STORE, 47710034808832, 47710034833407,
+ERASE, 47710034808832, 47710034808832,
+STORE, 47710034808832, 47710034833407,
+ERASE, 47710034833408, 47710034833408,
+STORE, 47710034833408, 47710034849791,
+STORE, 47710034849792, 47710034984959,
+ERASE, 47710034849792, 47710034849792,
+STORE, 47710034849792, 47710034874367,
+STORE, 47710034874368, 47710034984959,
+STORE, 47710034935808, 47710034984959,
+STORE, 47710034874368, 47710034935807,
+ERASE, 47710034874368, 47710034874368,
+STORE, 47710034874368, 47710034935807,
+STORE, 47710034960384, 47710034984959,
+STORE, 47710034935808, 47710034960383,
+ERASE, 47710034935808, 47710034935808,
+STORE, 47710034935808, 47710034984959,
+ERASE, 47710034935808, 47710034935808,
+STORE, 47710034935808, 47710034960383,
+STORE, 47710034960384, 47710034984959,
+STORE, 47710034968576, 47710034984959,
+STORE, 47710034960384, 47710034968575,
+ERASE, 47710034960384, 47710034960384,
+STORE, 47710034960384, 47710034968575,
+ERASE, 47710034968576, 47710034968576,
+STORE, 47710034968576, 47710034984959,
+STORE, 47710034984960, 47710035005439,
+ERASE, 47710034984960, 47710034984960,
+STORE, 47710034984960, 47710034989055,
+STORE, 47710034989056, 47710035005439,
+STORE, 47710034993152, 47710035005439,
+STORE, 47710034989056, 47710034993151,
+ERASE, 47710034989056, 47710034989056,
+STORE, 47710034989056, 47710034993151,
+STORE, 47710034997248, 47710035005439,
+STORE, 47710034993152, 47710034997247,
+ERASE, 47710034993152, 47710034993152,
+STORE, 47710034993152, 47710035005439,
+ERASE, 47710034993152, 47710034993152,
+STORE, 47710034993152, 47710034997247,
+STORE, 47710034997248, 47710035005439,
+ERASE, 47710034997248, 47710034997248,
+STORE, 47710034997248, 47710035005439,
+STORE, 47710035005440, 47710035013631,
+ERASE, 47710034808832, 47710034808832,
+STORE, 47710034808832, 47710034825215,
+STORE, 47710034825216, 47710034833407,
+ERASE, 47710034997248, 47710034997248,
+STORE, 47710034997248, 47710035001343,
+STORE, 47710035001344, 47710035005439,
+ERASE, 47710034960384, 47710034960384,
+STORE, 47710034960384, 47710034964479,
+STORE, 47710034964480, 47710034968575,
+ERASE, 47710032789504, 47710032789504,
+STORE, 47710032789504, 47710032986111,
+STORE, 47710032986112, 47710032994303,
+ERASE, 47710029950976, 47710029950976,
+STORE, 47710029950976, 47710029955071,
+STORE, 47710029955072, 47710029959167,
+ERASE, 94844636766208, 94844636766208,
+STORE, 94844636766208, 94844636774399,
+STORE, 94844636774400, 94844636778495,
+ERASE, 139922765377536, 139922765377536,
+STORE, 139922765377536, 139922765381631,
+STORE, 139922765381632, 139922765385727,
+ERASE, 47710029778944, 47710029778944,
+STORE, 94844641775616, 94844641910783,
+STORE, 140737488347136, 140737488351231,
+STORE, 140732213886976, 140737488351231,
+ERASE, 140732213886976, 140732213886976,
+STORE, 140732213886976, 140732213891071,
+STORE, 94240508887040, 94240509059071,
+ERASE, 94240508887040, 94240508887040,
+STORE, 94240508887040, 94240508903423,
+STORE, 94240508903424, 94240509059071,
+ERASE, 94240508903424, 94240508903424,
+STORE, 94240508903424, 94240509005823,
+STORE, 94240509005824, 94240509046783,
+STORE, 94240509046784, 94240509059071,
+STORE, 140275106516992, 140275106689023,
+ERASE, 140275106516992, 140275106516992,
+STORE, 140275106516992, 140275106521087,
+STORE, 140275106521088, 140275106689023,
+ERASE, 140275106521088, 140275106521088,
+STORE, 140275106521088, 140275106643967,
+STORE, 140275106643968, 140275106676735,
+STORE, 140275106676736, 140275106684927,
+STORE, 140275106684928, 140275106689023,
+STORE, 140732213977088, 140732213981183,
+STORE, 140732213964800, 140732213977087,
+STORE, 47357688479744, 47357688487935,
+STORE, 47357688487936, 47357688496127,
+STORE, 47357688496128, 47357688659967,
+ERASE, 47357688496128, 47357688496128,
+STORE, 47357688496128, 47357688508415,
+STORE, 47357688508416, 47357688659967,
+STORE, 47357688606720, 47357688659967,
+STORE, 47357688508416, 47357688606719,
+ERASE, 47357688508416, 47357688508416,
+STORE, 47357688508416, 47357688606719,
+STORE, 47357688651776, 47357688659967,
+STORE, 47357688606720, 47357688651775,
+ERASE, 47357688606720, 47357688606720,
+STORE, 47357688606720, 47357688659967,
+ERASE, 47357688606720, 47357688606720,
+STORE, 47357688606720, 47357688651775,
+STORE, 47357688651776, 47357688659967,
+ERASE, 47357688651776, 47357688651776,
+STORE, 47357688651776, 47357688659967,
+STORE, 47357688659968, 47357691711487,
+STORE, 47357689204736, 47357691711487,
+STORE, 47357688659968, 47357689204735,
+ERASE, 47357689204736, 47357689204736,
+STORE, 47357689204736, 47357691490303,
+STORE, 47357691490304, 47357691711487,
+STORE, 47357690900480, 47357691490303,
+STORE, 47357689204736, 47357690900479,
+ERASE, 47357689204736, 47357689204736,
+STORE, 47357689204736, 47357690900479,
+STORE, 47357691486208, 47357691490303,
+STORE, 47357690900480, 47357691486207,
+ERASE, 47357690900480, 47357690900480,
+STORE, 47357690900480, 47357691486207,
+STORE, 47357691695104, 47357691711487,
+STORE, 47357691490304, 47357691695103,
+ERASE, 47357691490304, 47357691490304,
+STORE, 47357691490304, 47357691695103,
+ERASE, 47357691695104, 47357691695104,
+STORE, 47357691695104, 47357691711487,
+STORE, 47357691711488, 47357693550591,
+STORE, 47357691850752, 47357693550591,
+STORE, 47357691711488, 47357691850751,
+ERASE, 47357691850752, 47357691850752,
+STORE, 47357691850752, 47357693509631,
+STORE, 47357693509632, 47357693550591,
+STORE, 47357693194240, 47357693509631,
+STORE, 47357691850752, 47357693194239,
+ERASE, 47357691850752, 47357691850752,
+STORE, 47357691850752, 47357693194239,
+STORE, 47357693505536, 47357693509631,
+STORE, 47357693194240, 47357693505535,
+ERASE, 47357693194240, 47357693194240,
+STORE, 47357693194240, 47357693505535,
+STORE, 47357693534208, 47357693550591,
+STORE, 47357693509632, 47357693534207,
+ERASE, 47357693509632, 47357693509632,
+STORE, 47357693509632, 47357693534207,
+ERASE, 47357693534208, 47357693534208,
+STORE, 47357693534208, 47357693550591,
+STORE, 47357693550592, 47357693685759,
+ERASE, 47357693550592, 47357693550592,
+STORE, 47357693550592, 47357693575167,
+STORE, 47357693575168, 47357693685759,
+STORE, 47357693636608, 47357693685759,
+STORE, 47357693575168, 47357693636607,
+ERASE, 47357693575168, 47357693575168,
+STORE, 47357693575168, 47357693636607,
+STORE, 47357693661184, 47357693685759,
+STORE, 47357693636608, 47357693661183,
+ERASE, 47357693636608, 47357693636608,
+STORE, 47357693636608, 47357693685759,
+ERASE, 47357693636608, 47357693636608,
+STORE, 47357693636608, 47357693661183,
+STORE, 47357693661184, 47357693685759,
+STORE, 47357693669376, 47357693685759,
+STORE, 47357693661184, 47357693669375,
+ERASE, 47357693661184, 47357693661184,
+STORE, 47357693661184, 47357693669375,
+ERASE, 47357693669376, 47357693669376,
+STORE, 47357693669376, 47357693685759,
+STORE, 47357693685760, 47357693706239,
+ERASE, 47357693685760, 47357693685760,
+STORE, 47357693685760, 47357693689855,
+STORE, 47357693689856, 47357693706239,
+STORE, 47357693693952, 47357693706239,
+STORE, 47357693689856, 47357693693951,
+ERASE, 47357693689856, 47357693689856,
+STORE, 47357693689856, 47357693693951,
+STORE, 47357693698048, 47357693706239,
+STORE, 47357693693952, 47357693698047,
+ERASE, 47357693693952, 47357693693952,
+STORE, 47357693693952, 47357693706239,
+ERASE, 47357693693952, 47357693693952,
+STORE, 47357693693952, 47357693698047,
+STORE, 47357693698048, 47357693706239,
+ERASE, 47357693698048, 47357693698048,
+STORE, 47357693698048, 47357693706239,
+STORE, 47357693706240, 47357693714431,
+ERASE, 47357693509632, 47357693509632,
+STORE, 47357693509632, 47357693526015,
+STORE, 47357693526016, 47357693534207,
+ERASE, 47357693698048, 47357693698048,
+STORE, 47357693698048, 47357693702143,
+STORE, 47357693702144, 47357693706239,
+ERASE, 47357693661184, 47357693661184,
+STORE, 47357693661184, 47357693665279,
+STORE, 47357693665280, 47357693669375,
+ERASE, 47357691490304, 47357691490304,
+STORE, 47357691490304, 47357691686911,
+STORE, 47357691686912, 47357691695103,
+ERASE, 47357688651776, 47357688651776,
+STORE, 47357688651776, 47357688655871,
+STORE, 47357688655872, 47357688659967,
+ERASE, 94240509046784, 94240509046784,
+STORE, 94240509046784, 94240509054975,
+STORE, 94240509054976, 94240509059071,
+ERASE, 140275106676736, 140275106676736,
+STORE, 140275106676736, 140275106680831,
+STORE, 140275106680832, 140275106684927,
+ERASE, 47357688479744, 47357688479744,
+STORE, 94240518361088, 94240518496255,
+STORE, 140737488347136, 140737488351231,
+STORE, 140732688277504, 140737488351231,
+ERASE, 140732688277504, 140732688277504,
+STORE, 140732688277504, 140732688281599,
+STORE, 94629171351552, 94629172064255,
+ERASE, 94629171351552, 94629171351552,
+STORE, 94629171351552, 94629171400703,
+STORE, 94629171400704, 94629172064255,
+ERASE, 94629171400704, 94629171400704,
+STORE, 94629171400704, 94629171945471,
+STORE, 94629171945472, 94629172043775,
+STORE, 94629172043776, 94629172064255,
+STORE, 139770707644416, 139770707816447,
+ERASE, 139770707644416, 139770707644416,
+STORE, 139770707644416, 139770707648511,
+STORE, 139770707648512, 139770707816447,
+ERASE, 139770707648512, 139770707648512,
+STORE, 139770707648512, 139770707771391,
+STORE, 139770707771392, 139770707804159,
+STORE, 139770707804160, 139770707812351,
+STORE, 139770707812352, 139770707816447,
+STORE, 140732689121280, 140732689125375,
+STORE, 140732689108992, 140732689121279,
+STORE, 47862087352320, 47862087360511,
+STORE, 47862087360512, 47862087368703,
+STORE, 47862087368704, 47862087475199,
+STORE, 47862087385088, 47862087475199,
+STORE, 47862087368704, 47862087385087,
+ERASE, 47862087385088, 47862087385088,
+STORE, 47862087385088, 47862087458815,
+STORE, 47862087458816, 47862087475199,
+STORE, 47862087438336, 47862087458815,
+STORE, 47862087385088, 47862087438335,
+ERASE, 47862087385088, 47862087385088,
+STORE, 47862087385088, 47862087438335,
+STORE, 47862087454720, 47862087458815,
+STORE, 47862087438336, 47862087454719,
+ERASE, 47862087438336, 47862087438336,
+STORE, 47862087438336, 47862087454719,
+STORE, 47862087467008, 47862087475199,
+STORE, 47862087458816, 47862087467007,
+ERASE, 47862087458816, 47862087458816,
+STORE, 47862087458816, 47862087467007,
+ERASE, 47862087467008, 47862087467008,
+STORE, 47862087467008, 47862087475199,
+STORE, 47862087475200, 47862089314303,
+STORE, 47862087614464, 47862089314303,
+STORE, 47862087475200, 47862087614463,
+ERASE, 47862087614464, 47862087614464,
+STORE, 47862087614464, 47862089273343,
+STORE, 47862089273344, 47862089314303,
+STORE, 47862088957952, 47862089273343,
+STORE, 47862087614464, 47862088957951,
+ERASE, 47862087614464, 47862087614464,
+STORE, 47862087614464, 47862088957951,
+STORE, 47862089269248, 47862089273343,
+STORE, 47862088957952, 47862089269247,
+ERASE, 47862088957952, 47862088957952,
+STORE, 47862088957952, 47862089269247,
+STORE, 47862089297920, 47862089314303,
+STORE, 47862089273344, 47862089297919,
+ERASE, 47862089273344, 47862089273344,
+STORE, 47862089273344, 47862089297919,
+ERASE, 47862089297920, 47862089297920,
+STORE, 47862089297920, 47862089314303,
+STORE, 47862089297920, 47862089326591,
+ERASE, 47862089273344, 47862089273344,
+STORE, 47862089273344, 47862089289727,
+STORE, 47862089289728, 47862089297919,
+ERASE, 47862087458816, 47862087458816,
+STORE, 47862087458816, 47862087462911,
+STORE, 47862087462912, 47862087467007,
+ERASE, 94629172043776, 94629172043776,
+STORE, 94629172043776, 94629172060159,
+STORE, 94629172060160, 94629172064255,
+ERASE, 139770707804160, 139770707804160,
+STORE, 139770707804160, 139770707808255,
+STORE, 139770707808256, 139770707812351,
+ERASE, 47862087352320, 47862087352320,
+STORE, 94629197533184, 94629197668351,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727540711424, 140737488351231,
+ERASE, 140727540711424, 140727540711424,
+STORE, 140727540711424, 140727540715519,
+STORE, 94299865313280, 94299866025983,
+ERASE, 94299865313280, 94299865313280,
+STORE, 94299865313280, 94299865362431,
+STORE, 94299865362432, 94299866025983,
+ERASE, 94299865362432, 94299865362432,
+STORE, 94299865362432, 94299865907199,
+STORE, 94299865907200, 94299866005503,
+STORE, 94299866005504, 94299866025983,
+STORE, 140680268763136, 140680268935167,
+ERASE, 140680268763136, 140680268763136,
+STORE, 140680268763136, 140680268767231,
+STORE, 140680268767232, 140680268935167,
+ERASE, 140680268767232, 140680268767232,
+STORE, 140680268767232, 140680268890111,
+STORE, 140680268890112, 140680268922879,
+STORE, 140680268922880, 140680268931071,
+STORE, 140680268931072, 140680268935167,
+STORE, 140727541424128, 140727541428223,
+STORE, 140727541411840, 140727541424127,
+STORE, 46952526233600, 46952526241791,
+STORE, 46952526241792, 46952526249983,
+STORE, 46952526249984, 46952526356479,
+STORE, 46952526266368, 46952526356479,
+STORE, 46952526249984, 46952526266367,
+ERASE, 46952526266368, 46952526266368,
+STORE, 46952526266368, 46952526340095,
+STORE, 46952526340096, 46952526356479,
+STORE, 46952526319616, 46952526340095,
+STORE, 46952526266368, 46952526319615,
+ERASE, 46952526266368, 46952526266368,
+STORE, 46952526266368, 46952526319615,
+STORE, 46952526336000, 46952526340095,
+STORE, 46952526319616, 46952526335999,
+ERASE, 46952526319616, 46952526319616,
+STORE, 46952526319616, 46952526335999,
+STORE, 46952526348288, 46952526356479,
+STORE, 46952526340096, 46952526348287,
+ERASE, 46952526340096, 46952526340096,
+STORE, 46952526340096, 46952526348287,
+ERASE, 46952526348288, 46952526348288,
+STORE, 46952526348288, 46952526356479,
+STORE, 46952526356480, 46952528195583,
+STORE, 46952526495744, 46952528195583,
+STORE, 46952526356480, 46952526495743,
+ERASE, 46952526495744, 46952526495744,
+STORE, 46952526495744, 46952528154623,
+STORE, 46952528154624, 46952528195583,
+STORE, 46952527839232, 46952528154623,
+STORE, 46952526495744, 46952527839231,
+ERASE, 46952526495744, 46952526495744,
+STORE, 46952526495744, 46952527839231,
+STORE, 46952528150528, 46952528154623,
+STORE, 46952527839232, 46952528150527,
+ERASE, 46952527839232, 46952527839232,
+STORE, 46952527839232, 46952528150527,
+STORE, 46952528179200, 46952528195583,
+STORE, 46952528154624, 46952528179199,
+ERASE, 46952528154624, 46952528154624,
+STORE, 46952528154624, 46952528179199,
+ERASE, 46952528179200, 46952528179200,
+STORE, 46952528179200, 46952528195583,
+STORE, 46952528179200, 46952528207871,
+ERASE, 46952528154624, 46952528154624,
+STORE, 46952528154624, 46952528171007,
+STORE, 46952528171008, 46952528179199,
+ERASE, 46952526340096, 46952526340096,
+STORE, 46952526340096, 46952526344191,
+STORE, 46952526344192, 46952526348287,
+ERASE, 94299866005504, 94299866005504,
+STORE, 94299866005504, 94299866021887,
+STORE, 94299866021888, 94299866025983,
+ERASE, 140680268922880, 140680268922880,
+STORE, 140680268922880, 140680268926975,
+STORE, 140680268926976, 140680268931071,
+ERASE, 46952526233600, 46952526233600,
+STORE, 140737488347136, 140737488351231,
+STORE, 140722874793984, 140737488351231,
+ERASE, 140722874793984, 140722874793984,
+STORE, 140722874793984, 140722874798079,
+STORE, 94448916213760, 94448916926463,
+ERASE, 94448916213760, 94448916213760,
+STORE, 94448916213760, 94448916262911,
+STORE, 94448916262912, 94448916926463,
+ERASE, 94448916262912, 94448916262912,
+STORE, 94448916262912, 94448916807679,
+STORE, 94448916807680, 94448916905983,
+STORE, 94448916905984, 94448916926463,
+STORE, 140389117046784, 140389117218815,
+ERASE, 140389117046784, 140389117046784,
+STORE, 140389117046784, 140389117050879,
+STORE, 140389117050880, 140389117218815,
+ERASE, 140389117050880, 140389117050880,
+STORE, 140389117050880, 140389117173759,
+STORE, 140389117173760, 140389117206527,
+STORE, 140389117206528, 140389117214719,
+STORE, 140389117214720, 140389117218815,
+STORE, 140722875297792, 140722875301887,
+STORE, 140722875285504, 140722875297791,
+STORE, 47243677949952, 47243677958143,
+STORE, 47243677958144, 47243677966335,
+STORE, 47243677966336, 47243678072831,
+STORE, 47243677982720, 47243678072831,
+STORE, 47243677966336, 47243677982719,
+ERASE, 47243677982720, 47243677982720,
+STORE, 47243677982720, 47243678056447,
+STORE, 47243678056448, 47243678072831,
+STORE, 47243678035968, 47243678056447,
+STORE, 47243677982720, 47243678035967,
+ERASE, 47243677982720, 47243677982720,
+STORE, 47243677982720, 47243678035967,
+STORE, 47243678052352, 47243678056447,
+STORE, 47243678035968, 47243678052351,
+ERASE, 47243678035968, 47243678035968,
+STORE, 47243678035968, 47243678052351,
+STORE, 47243678064640, 47243678072831,
+STORE, 47243678056448, 47243678064639,
+ERASE, 47243678056448, 47243678056448,
+STORE, 47243678056448, 47243678064639,
+ERASE, 47243678064640, 47243678064640,
+STORE, 47243678064640, 47243678072831,
+STORE, 47243678072832, 47243679911935,
+STORE, 47243678212096, 47243679911935,
+STORE, 47243678072832, 47243678212095,
+ERASE, 47243678212096, 47243678212096,
+STORE, 47243678212096, 47243679870975,
+STORE, 47243679870976, 47243679911935,
+STORE, 47243679555584, 47243679870975,
+STORE, 47243678212096, 47243679555583,
+ERASE, 47243678212096, 47243678212096,
+STORE, 47243678212096, 47243679555583,
+STORE, 47243679866880, 47243679870975,
+STORE, 47243679555584, 47243679866879,
+ERASE, 47243679555584, 47243679555584,
+STORE, 47243679555584, 47243679866879,
+STORE, 47243679895552, 47243679911935,
+STORE, 47243679870976, 47243679895551,
+ERASE, 47243679870976, 47243679870976,
+STORE, 47243679870976, 47243679895551,
+ERASE, 47243679895552, 47243679895552,
+STORE, 47243679895552, 47243679911935,
+STORE, 47243679895552, 47243679924223,
+ERASE, 47243679870976, 47243679870976,
+STORE, 47243679870976, 47243679887359,
+STORE, 47243679887360, 47243679895551,
+ERASE, 47243678056448, 47243678056448,
+STORE, 47243678056448, 47243678060543,
+STORE, 47243678060544, 47243678064639,
+ERASE, 94448916905984, 94448916905984,
+STORE, 94448916905984, 94448916922367,
+STORE, 94448916922368, 94448916926463,
+ERASE, 140389117206528, 140389117206528,
+STORE, 140389117206528, 140389117210623,
+STORE, 140389117210624, 140389117214719,
+ERASE, 47243677949952, 47243677949952,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733068505088, 140737488351231,
+ERASE, 140733068505088, 140733068505088,
+STORE, 140733068505088, 140733068509183,
+STORE, 94207145750528, 94207146463231,
+ERASE, 94207145750528, 94207145750528,
+STORE, 94207145750528, 94207145799679,
+STORE, 94207145799680, 94207146463231,
+ERASE, 94207145799680, 94207145799680,
+STORE, 94207145799680, 94207146344447,
+STORE, 94207146344448, 94207146442751,
+STORE, 94207146442752, 94207146463231,
+STORE, 140684504911872, 140684505083903,
+ERASE, 140684504911872, 140684504911872,
+STORE, 140684504911872, 140684504915967,
+STORE, 140684504915968, 140684505083903,
+ERASE, 140684504915968, 140684504915968,
+STORE, 140684504915968, 140684505038847,
+STORE, 140684505038848, 140684505071615,
+STORE, 140684505071616, 140684505079807,
+STORE, 140684505079808, 140684505083903,
+STORE, 140733068607488, 140733068611583,
+STORE, 140733068595200, 140733068607487,
+STORE, 46948290084864, 46948290093055,
+STORE, 46948290093056, 46948290101247,
+STORE, 46948290101248, 46948290207743,
+STORE, 46948290117632, 46948290207743,
+STORE, 46948290101248, 46948290117631,
+ERASE, 46948290117632, 46948290117632,
+STORE, 46948290117632, 46948290191359,
+STORE, 46948290191360, 46948290207743,
+STORE, 46948290170880, 46948290191359,
+STORE, 46948290117632, 46948290170879,
+ERASE, 46948290117632, 46948290117632,
+STORE, 46948290117632, 46948290170879,
+STORE, 46948290187264, 46948290191359,
+STORE, 46948290170880, 46948290187263,
+ERASE, 46948290170880, 46948290170880,
+STORE, 46948290170880, 46948290187263,
+STORE, 46948290199552, 46948290207743,
+STORE, 46948290191360, 46948290199551,
+ERASE, 46948290191360, 46948290191360,
+STORE, 46948290191360, 46948290199551,
+ERASE, 46948290199552, 46948290199552,
+STORE, 46948290199552, 46948290207743,
+STORE, 46948290207744, 46948292046847,
+STORE, 46948290347008, 46948292046847,
+STORE, 46948290207744, 46948290347007,
+ERASE, 46948290347008, 46948290347008,
+STORE, 46948290347008, 46948292005887,
+STORE, 46948292005888, 46948292046847,
+STORE, 46948291690496, 46948292005887,
+STORE, 46948290347008, 46948291690495,
+ERASE, 46948290347008, 46948290347008,
+STORE, 46948290347008, 46948291690495,
+STORE, 46948292001792, 46948292005887,
+STORE, 46948291690496, 46948292001791,
+ERASE, 46948291690496, 46948291690496,
+STORE, 46948291690496, 46948292001791,
+STORE, 46948292030464, 46948292046847,
+STORE, 46948292005888, 46948292030463,
+ERASE, 46948292005888, 46948292005888,
+STORE, 46948292005888, 46948292030463,
+ERASE, 46948292030464, 46948292030464,
+STORE, 46948292030464, 46948292046847,
+STORE, 46948292030464, 46948292059135,
+ERASE, 46948292005888, 46948292005888,
+STORE, 46948292005888, 46948292022271,
+STORE, 46948292022272, 46948292030463,
+ERASE, 46948290191360, 46948290191360,
+STORE, 46948290191360, 46948290195455,
+STORE, 46948290195456, 46948290199551,
+ERASE, 94207146442752, 94207146442752,
+STORE, 94207146442752, 94207146459135,
+STORE, 94207146459136, 94207146463231,
+ERASE, 140684505071616, 140684505071616,
+STORE, 140684505071616, 140684505075711,
+STORE, 140684505075712, 140684505079807,
+ERASE, 46948290084864, 46948290084864,
+STORE, 140737488347136, 140737488351231,
+STORE, 140726367158272, 140737488351231,
+ERASE, 140726367158272, 140726367158272,
+STORE, 140726367158272, 140726367162367,
+STORE, 94436124106752, 94436124819455,
+ERASE, 94436124106752, 94436124106752,
+STORE, 94436124106752, 94436124155903,
+STORE, 94436124155904, 94436124819455,
+ERASE, 94436124155904, 94436124155904,
+STORE, 94436124155904, 94436124700671,
+STORE, 94436124700672, 94436124798975,
+STORE, 94436124798976, 94436124819455,
+STORE, 140049025044480, 140049025216511,
+ERASE, 140049025044480, 140049025044480,
+STORE, 140049025044480, 140049025048575,
+STORE, 140049025048576, 140049025216511,
+ERASE, 140049025048576, 140049025048576,
+STORE, 140049025048576, 140049025171455,
+STORE, 140049025171456, 140049025204223,
+STORE, 140049025204224, 140049025212415,
+STORE, 140049025212416, 140049025216511,
+STORE, 140726367256576, 140726367260671,
+STORE, 140726367244288, 140726367256575,
+STORE, 47583769952256, 47583769960447,
+STORE, 47583769960448, 47583769968639,
+STORE, 47583769968640, 47583770075135,
+STORE, 47583769985024, 47583770075135,
+STORE, 47583769968640, 47583769985023,
+ERASE, 47583769985024, 47583769985024,
+STORE, 47583769985024, 47583770058751,
+STORE, 47583770058752, 47583770075135,
+STORE, 47583770038272, 47583770058751,
+STORE, 47583769985024, 47583770038271,
+ERASE, 47583769985024, 47583769985024,
+STORE, 47583769985024, 47583770038271,
+STORE, 47583770054656, 47583770058751,
+STORE, 47583770038272, 47583770054655,
+ERASE, 47583770038272, 47583770038272,
+STORE, 47583770038272, 47583770054655,
+STORE, 47583770066944, 47583770075135,
+STORE, 47583770058752, 47583770066943,
+ERASE, 47583770058752, 47583770058752,
+STORE, 47583770058752, 47583770066943,
+ERASE, 47583770066944, 47583770066944,
+STORE, 47583770066944, 47583770075135,
+STORE, 47583770075136, 47583771914239,
+STORE, 47583770214400, 47583771914239,
+STORE, 47583770075136, 47583770214399,
+ERASE, 47583770214400, 47583770214400,
+STORE, 47583770214400, 47583771873279,
+STORE, 47583771873280, 47583771914239,
+STORE, 47583771557888, 47583771873279,
+STORE, 47583770214400, 47583771557887,
+ERASE, 47583770214400, 47583770214400,
+STORE, 47583770214400, 47583771557887,
+STORE, 47583771869184, 47583771873279,
+STORE, 47583771557888, 47583771869183,
+ERASE, 47583771557888, 47583771557888,
+STORE, 47583771557888, 47583771869183,
+STORE, 47583771897856, 47583771914239,
+STORE, 47583771873280, 47583771897855,
+ERASE, 47583771873280, 47583771873280,
+STORE, 47583771873280, 47583771897855,
+ERASE, 47583771897856, 47583771897856,
+STORE, 47583771897856, 47583771914239,
+STORE, 47583771897856, 47583771926527,
+ERASE, 47583771873280, 47583771873280,
+STORE, 47583771873280, 47583771889663,
+STORE, 47583771889664, 47583771897855,
+ERASE, 47583770058752, 47583770058752,
+STORE, 47583770058752, 47583770062847,
+STORE, 47583770062848, 47583770066943,
+ERASE, 94436124798976, 94436124798976,
+STORE, 94436124798976, 94436124815359,
+STORE, 94436124815360, 94436124819455,
+ERASE, 140049025204224, 140049025204224,
+STORE, 140049025204224, 140049025208319,
+STORE, 140049025208320, 140049025212415,
+ERASE, 47583769952256, 47583769952256,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727116099584, 140737488351231,
+ERASE, 140727116099584, 140727116099584,
+STORE, 140727116099584, 140727116103679,
+STORE, 94166319734784, 94166320447487,
+ERASE, 94166319734784, 94166319734784,
+STORE, 94166319734784, 94166319783935,
+STORE, 94166319783936, 94166320447487,
+ERASE, 94166319783936, 94166319783936,
+STORE, 94166319783936, 94166320328703,
+STORE, 94166320328704, 94166320427007,
+STORE, 94166320427008, 94166320447487,
+STORE, 139976559542272, 139976559714303,
+ERASE, 139976559542272, 139976559542272,
+STORE, 139976559542272, 139976559546367,
+STORE, 139976559546368, 139976559714303,
+ERASE, 139976559546368, 139976559546368,
+STORE, 139976559546368, 139976559669247,
+STORE, 139976559669248, 139976559702015,
+STORE, 139976559702016, 139976559710207,
+STORE, 139976559710208, 139976559714303,
+STORE, 140727116222464, 140727116226559,
+STORE, 140727116210176, 140727116222463,
+STORE, 47656235454464, 47656235462655,
+STORE, 47656235462656, 47656235470847,
+STORE, 47656235470848, 47656235577343,
+STORE, 47656235487232, 47656235577343,
+STORE, 47656235470848, 47656235487231,
+ERASE, 47656235487232, 47656235487232,
+STORE, 47656235487232, 47656235560959,
+STORE, 47656235560960, 47656235577343,
+STORE, 47656235540480, 47656235560959,
+STORE, 47656235487232, 47656235540479,
+ERASE, 47656235487232, 47656235487232,
+STORE, 47656235487232, 47656235540479,
+STORE, 47656235556864, 47656235560959,
+STORE, 47656235540480, 47656235556863,
+ERASE, 47656235540480, 47656235540480,
+STORE, 47656235540480, 47656235556863,
+STORE, 47656235569152, 47656235577343,
+STORE, 47656235560960, 47656235569151,
+ERASE, 47656235560960, 47656235560960,
+STORE, 47656235560960, 47656235569151,
+ERASE, 47656235569152, 47656235569152,
+STORE, 47656235569152, 47656235577343,
+STORE, 47656235577344, 47656237416447,
+STORE, 47656235716608, 47656237416447,
+STORE, 47656235577344, 47656235716607,
+ERASE, 47656235716608, 47656235716608,
+STORE, 47656235716608, 47656237375487,
+STORE, 47656237375488, 47656237416447,
+STORE, 47656237060096, 47656237375487,
+STORE, 47656235716608, 47656237060095,
+ERASE, 47656235716608, 47656235716608,
+STORE, 47656235716608, 47656237060095,
+STORE, 47656237371392, 47656237375487,
+STORE, 47656237060096, 47656237371391,
+ERASE, 47656237060096, 47656237060096,
+STORE, 47656237060096, 47656237371391,
+STORE, 47656237400064, 47656237416447,
+STORE, 47656237375488, 47656237400063,
+ERASE, 47656237375488, 47656237375488,
+STORE, 47656237375488, 47656237400063,
+ERASE, 47656237400064, 47656237400064,
+STORE, 47656237400064, 47656237416447,
+STORE, 47656237400064, 47656237428735,
+ERASE, 47656237375488, 47656237375488,
+STORE, 47656237375488, 47656237391871,
+STORE, 47656237391872, 47656237400063,
+ERASE, 47656235560960, 47656235560960,
+STORE, 47656235560960, 47656235565055,
+STORE, 47656235565056, 47656235569151,
+ERASE, 94166320427008, 94166320427008,
+STORE, 94166320427008, 94166320443391,
+STORE, 94166320443392, 94166320447487,
+ERASE, 139976559702016, 139976559702016,
+STORE, 139976559702016, 139976559706111,
+STORE, 139976559706112, 139976559710207,
+ERASE, 47656235454464, 47656235454464,
+STORE, 94166332153856, 94166332289023,
+STORE, 140737488347136, 140737488351231,
+STORE, 140726412816384, 140737488351231,
+ERASE, 140726412816384, 140726412816384,
+STORE, 140726412816384, 140726412820479,
+STORE, 94094884507648, 94094885220351,
+ERASE, 94094884507648, 94094884507648,
+STORE, 94094884507648, 94094884556799,
+STORE, 94094884556800, 94094885220351,
+ERASE, 94094884556800, 94094884556800,
+STORE, 94094884556800, 94094885101567,
+STORE, 94094885101568, 94094885199871,
+STORE, 94094885199872, 94094885220351,
+STORE, 139773773938688, 139773774110719,
+ERASE, 139773773938688, 139773773938688,
+STORE, 139773773938688, 139773773942783,
+STORE, 139773773942784, 139773774110719,
+ERASE, 139773773942784, 139773773942784,
+STORE, 139773773942784, 139773774065663,
+STORE, 139773774065664, 139773774098431,
+STORE, 139773774098432, 139773774106623,
+STORE, 139773774106624, 139773774110719,
+STORE, 140726412963840, 140726412967935,
+STORE, 140726412951552, 140726412963839,
+STORE, 47859021058048, 47859021066239,
+STORE, 47859021066240, 47859021074431,
+STORE, 47859021074432, 47859021180927,
+STORE, 47859021090816, 47859021180927,
+STORE, 47859021074432, 47859021090815,
+ERASE, 47859021090816, 47859021090816,
+STORE, 47859021090816, 47859021164543,
+STORE, 47859021164544, 47859021180927,
+STORE, 47859021144064, 47859021164543,
+STORE, 47859021090816, 47859021144063,
+ERASE, 47859021090816, 47859021090816,
+STORE, 47859021090816, 47859021144063,
+STORE, 47859021160448, 47859021164543,
+STORE, 47859021144064, 47859021160447,
+ERASE, 47859021144064, 47859021144064,
+STORE, 47859021144064, 47859021160447,
+STORE, 47859021172736, 47859021180927,
+STORE, 47859021164544, 47859021172735,
+ERASE, 47859021164544, 47859021164544,
+STORE, 47859021164544, 47859021172735,
+ERASE, 47859021172736, 47859021172736,
+STORE, 47859021172736, 47859021180927,
+STORE, 47859021180928, 47859023020031,
+STORE, 47859021320192, 47859023020031,
+STORE, 47859021180928, 47859021320191,
+ERASE, 47859021320192, 47859021320192,
+STORE, 47859021320192, 47859022979071,
+STORE, 47859022979072, 47859023020031,
+STORE, 47859022663680, 47859022979071,
+STORE, 47859021320192, 47859022663679,
+ERASE, 47859021320192, 47859021320192,
+STORE, 47859021320192, 47859022663679,
+STORE, 47859022974976, 47859022979071,
+STORE, 47859022663680, 47859022974975,
+ERASE, 47859022663680, 47859022663680,
+STORE, 47859022663680, 47859022974975,
+STORE, 47859023003648, 47859023020031,
+STORE, 47859022979072, 47859023003647,
+ERASE, 47859022979072, 47859022979072,
+STORE, 47859022979072, 47859023003647,
+ERASE, 47859023003648, 47859023003648,
+STORE, 47859023003648, 47859023020031,
+STORE, 47859023003648, 47859023032319,
+ERASE, 47859022979072, 47859022979072,
+STORE, 47859022979072, 47859022995455,
+STORE, 47859022995456, 47859023003647,
+ERASE, 47859021164544, 47859021164544,
+STORE, 47859021164544, 47859021168639,
+STORE, 47859021168640, 47859021172735,
+ERASE, 94094885199872, 94094885199872,
+STORE, 94094885199872, 94094885216255,
+STORE, 94094885216256, 94094885220351,
+ERASE, 139773774098432, 139773774098432,
+STORE, 139773774098432, 139773774102527,
+STORE, 139773774102528, 139773774106623,
+ERASE, 47859021058048, 47859021058048,
+STORE, 94094901108736, 94094901243903,
+STORE, 140737488347136, 140737488351231,
+STORE, 140736567963648, 140737488351231,
+ERASE, 140736567963648, 140736567963648,
+STORE, 140736567963648, 140736567967743,
+STORE, 94924425748480, 94924426461183,
+ERASE, 94924425748480, 94924425748480,
+STORE, 94924425748480, 94924425797631,
+STORE, 94924425797632, 94924426461183,
+ERASE, 94924425797632, 94924425797632,
+STORE, 94924425797632, 94924426342399,
+STORE, 94924426342400, 94924426440703,
+STORE, 94924426440704, 94924426461183,
+STORE, 140042126319616, 140042126491647,
+ERASE, 140042126319616, 140042126319616,
+STORE, 140042126319616, 140042126323711,
+STORE, 140042126323712, 140042126491647,
+ERASE, 140042126323712, 140042126323712,
+STORE, 140042126323712, 140042126446591,
+STORE, 140042126446592, 140042126479359,
+STORE, 140042126479360, 140042126487551,
+STORE, 140042126487552, 140042126491647,
+STORE, 140736568672256, 140736568676351,
+STORE, 140736568659968, 140736568672255,
+STORE, 47590668677120, 47590668685311,
+STORE, 47590668685312, 47590668693503,
+STORE, 47590668693504, 47590668799999,
+STORE, 47590668709888, 47590668799999,
+STORE, 47590668693504, 47590668709887,
+ERASE, 47590668709888, 47590668709888,
+STORE, 47590668709888, 47590668783615,
+STORE, 47590668783616, 47590668799999,
+STORE, 47590668763136, 47590668783615,
+STORE, 47590668709888, 47590668763135,
+ERASE, 47590668709888, 47590668709888,
+STORE, 47590668709888, 47590668763135,
+STORE, 47590668779520, 47590668783615,
+STORE, 47590668763136, 47590668779519,
+ERASE, 47590668763136, 47590668763136,
+STORE, 47590668763136, 47590668779519,
+STORE, 47590668791808, 47590668799999,
+STORE, 47590668783616, 47590668791807,
+ERASE, 47590668783616, 47590668783616,
+STORE, 47590668783616, 47590668791807,
+ERASE, 47590668791808, 47590668791808,
+STORE, 47590668791808, 47590668799999,
+STORE, 47590668800000, 47590670639103,
+STORE, 47590668939264, 47590670639103,
+STORE, 47590668800000, 47590668939263,
+ERASE, 47590668939264, 47590668939264,
+STORE, 47590668939264, 47590670598143,
+STORE, 47590670598144, 47590670639103,
+STORE, 47590670282752, 47590670598143,
+STORE, 47590668939264, 47590670282751,
+ERASE, 47590668939264, 47590668939264,
+STORE, 47590668939264, 47590670282751,
+STORE, 47590670594048, 47590670598143,
+STORE, 47590670282752, 47590670594047,
+ERASE, 47590670282752, 47590670282752,
+STORE, 47590670282752, 47590670594047,
+STORE, 47590670622720, 47590670639103,
+STORE, 47590670598144, 47590670622719,
+ERASE, 47590670598144, 47590670598144,
+STORE, 47590670598144, 47590670622719,
+ERASE, 47590670622720, 47590670622720,
+STORE, 47590670622720, 47590670639103,
+STORE, 47590670622720, 47590670651391,
+ERASE, 47590670598144, 47590670598144,
+STORE, 47590670598144, 47590670614527,
+STORE, 47590670614528, 47590670622719,
+ERASE, 47590668783616, 47590668783616,
+STORE, 47590668783616, 47590668787711,
+STORE, 47590668787712, 47590668791807,
+ERASE, 94924426440704, 94924426440704,
+STORE, 94924426440704, 94924426457087,
+STORE, 94924426457088, 94924426461183,
+ERASE, 140042126479360, 140042126479360,
+STORE, 140042126479360, 140042126483455,
+STORE, 140042126483456, 140042126487551,
+ERASE, 47590668677120, 47590668677120,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733281439744, 140737488351231,
+ERASE, 140733281439744, 140733281439744,
+STORE, 140733281439744, 140733281443839,
+STORE, 94490667069440, 94490667782143,
+ERASE, 94490667069440, 94490667069440,
+STORE, 94490667069440, 94490667118591,
+STORE, 94490667118592, 94490667782143,
+ERASE, 94490667118592, 94490667118592,
+STORE, 94490667118592, 94490667663359,
+STORE, 94490667663360, 94490667761663,
+STORE, 94490667761664, 94490667782143,
+STORE, 139878215118848, 139878215290879,
+ERASE, 139878215118848, 139878215118848,
+STORE, 139878215118848, 139878215122943,
+STORE, 139878215122944, 139878215290879,
+ERASE, 139878215122944, 139878215122944,
+STORE, 139878215122944, 139878215245823,
+STORE, 139878215245824, 139878215278591,
+STORE, 139878215278592, 139878215286783,
+STORE, 139878215286784, 139878215290879,
+STORE, 140733281464320, 140733281468415,
+STORE, 140733281452032, 140733281464319,
+STORE, 47754579877888, 47754579886079,
+STORE, 47754579886080, 47754579894271,
+STORE, 47754579894272, 47754580000767,
+STORE, 47754579910656, 47754580000767,
+STORE, 47754579894272, 47754579910655,
+ERASE, 47754579910656, 47754579910656,
+STORE, 47754579910656, 47754579984383,
+STORE, 47754579984384, 47754580000767,
+STORE, 47754579963904, 47754579984383,
+STORE, 47754579910656, 47754579963903,
+ERASE, 47754579910656, 47754579910656,
+STORE, 47754579910656, 47754579963903,
+STORE, 47754579980288, 47754579984383,
+STORE, 47754579963904, 47754579980287,
+ERASE, 47754579963904, 47754579963904,
+STORE, 47754579963904, 47754579980287,
+STORE, 47754579992576, 47754580000767,
+STORE, 47754579984384, 47754579992575,
+ERASE, 47754579984384, 47754579984384,
+STORE, 47754579984384, 47754579992575,
+ERASE, 47754579992576, 47754579992576,
+STORE, 47754579992576, 47754580000767,
+STORE, 47754580000768, 47754581839871,
+STORE, 47754580140032, 47754581839871,
+STORE, 47754580000768, 47754580140031,
+ERASE, 47754580140032, 47754580140032,
+STORE, 47754580140032, 47754581798911,
+STORE, 47754581798912, 47754581839871,
+STORE, 47754581483520, 47754581798911,
+STORE, 47754580140032, 47754581483519,
+ERASE, 47754580140032, 47754580140032,
+STORE, 47754580140032, 47754581483519,
+STORE, 47754581794816, 47754581798911,
+STORE, 47754581483520, 47754581794815,
+ERASE, 47754581483520, 47754581483520,
+STORE, 47754581483520, 47754581794815,
+STORE, 47754581823488, 47754581839871,
+STORE, 47754581798912, 47754581823487,
+ERASE, 47754581798912, 47754581798912,
+STORE, 47754581798912, 47754581823487,
+ERASE, 47754581823488, 47754581823488,
+STORE, 47754581823488, 47754581839871,
+STORE, 47754581823488, 47754581852159,
+ERASE, 47754581798912, 47754581798912,
+STORE, 47754581798912, 47754581815295,
+STORE, 47754581815296, 47754581823487,
+ERASE, 47754579984384, 47754579984384,
+STORE, 47754579984384, 47754579988479,
+STORE, 47754579988480, 47754579992575,
+ERASE, 94490667761664, 94490667761664,
+STORE, 94490667761664, 94490667778047,
+STORE, 94490667778048, 94490667782143,
+ERASE, 139878215278592, 139878215278592,
+STORE, 139878215278592, 139878215282687,
+STORE, 139878215282688, 139878215286783,
+ERASE, 47754579877888, 47754579877888,
+STORE, 94490669649920, 94490669785087,
+STORE, 140737488347136, 140737488351231,
+STORE, 140735382188032, 140737488351231,
+ERASE, 140735382188032, 140735382188032,
+STORE, 140735382188032, 140735382192127,
+STORE, 94150181302272, 94150182014975,
+ERASE, 94150181302272, 94150181302272,
+STORE, 94150181302272, 94150181351423,
+STORE, 94150181351424, 94150182014975,
+ERASE, 94150181351424, 94150181351424,
+STORE, 94150181351424, 94150181896191,
+STORE, 94150181896192, 94150181994495,
+STORE, 94150181994496, 94150182014975,
+STORE, 139679752458240, 139679752630271,
+ERASE, 139679752458240, 139679752458240,
+STORE, 139679752458240, 139679752462335,
+STORE, 139679752462336, 139679752630271,
+ERASE, 139679752462336, 139679752462336,
+STORE, 139679752462336, 139679752585215,
+STORE, 139679752585216, 139679752617983,
+STORE, 139679752617984, 139679752626175,
+STORE, 139679752626176, 139679752630271,
+STORE, 140735382536192, 140735382540287,
+STORE, 140735382523904, 140735382536191,
+STORE, 47953042538496, 47953042546687,
+STORE, 47953042546688, 47953042554879,
+STORE, 47953042554880, 47953042661375,
+STORE, 47953042571264, 47953042661375,
+STORE, 47953042554880, 47953042571263,
+ERASE, 47953042571264, 47953042571264,
+STORE, 47953042571264, 47953042644991,
+STORE, 47953042644992, 47953042661375,
+STORE, 47953042624512, 47953042644991,
+STORE, 47953042571264, 47953042624511,
+ERASE, 47953042571264, 47953042571264,
+STORE, 47953042571264, 47953042624511,
+STORE, 47953042640896, 47953042644991,
+STORE, 47953042624512, 47953042640895,
+ERASE, 47953042624512, 47953042624512,
+STORE, 47953042624512, 47953042640895,
+STORE, 47953042653184, 47953042661375,
+STORE, 47953042644992, 47953042653183,
+ERASE, 47953042644992, 47953042644992,
+STORE, 47953042644992, 47953042653183,
+ERASE, 47953042653184, 47953042653184,
+STORE, 47953042653184, 47953042661375,
+STORE, 47953042661376, 47953044500479,
+STORE, 47953042800640, 47953044500479,
+STORE, 47953042661376, 47953042800639,
+ERASE, 47953042800640, 47953042800640,
+STORE, 47953042800640, 47953044459519,
+STORE, 47953044459520, 47953044500479,
+STORE, 47953044144128, 47953044459519,
+STORE, 47953042800640, 47953044144127,
+ERASE, 47953042800640, 47953042800640,
+STORE, 47953042800640, 47953044144127,
+STORE, 47953044455424, 47953044459519,
+STORE, 47953044144128, 47953044455423,
+ERASE, 47953044144128, 47953044144128,
+STORE, 47953044144128, 47953044455423,
+STORE, 47953044484096, 47953044500479,
+STORE, 47953044459520, 47953044484095,
+ERASE, 47953044459520, 47953044459520,
+STORE, 47953044459520, 47953044484095,
+ERASE, 47953044484096, 47953044484096,
+STORE, 47953044484096, 47953044500479,
+STORE, 47953044484096, 47953044512767,
+ERASE, 47953044459520, 47953044459520,
+STORE, 47953044459520, 47953044475903,
+STORE, 47953044475904, 47953044484095,
+ERASE, 47953042644992, 47953042644992,
+STORE, 47953042644992, 47953042649087,
+STORE, 47953042649088, 47953042653183,
+ERASE, 94150181994496, 94150181994496,
+STORE, 94150181994496, 94150182010879,
+STORE, 94150182010880, 94150182014975,
+ERASE, 139679752617984, 139679752617984,
+STORE, 139679752617984, 139679752622079,
+STORE, 139679752622080, 139679752626175,
+ERASE, 47953042538496, 47953042538496,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737044123648, 140737488351231,
+ERASE, 140737044123648, 140737044123648,
+STORE, 140737044123648, 140737044127743,
+STORE, 94425324294144, 94425325006847,
+ERASE, 94425324294144, 94425324294144,
+STORE, 94425324294144, 94425324343295,
+STORE, 94425324343296, 94425325006847,
+ERASE, 94425324343296, 94425324343296,
+STORE, 94425324343296, 94425324888063,
+STORE, 94425324888064, 94425324986367,
+STORE, 94425324986368, 94425325006847,
+STORE, 140382015016960, 140382015188991,
+ERASE, 140382015016960, 140382015016960,
+STORE, 140382015016960, 140382015021055,
+STORE, 140382015021056, 140382015188991,
+ERASE, 140382015021056, 140382015021056,
+STORE, 140382015021056, 140382015143935,
+STORE, 140382015143936, 140382015176703,
+STORE, 140382015176704, 140382015184895,
+STORE, 140382015184896, 140382015188991,
+STORE, 140737045585920, 140737045590015,
+STORE, 140737045573632, 140737045585919,
+STORE, 47250779979776, 47250779987967,
+STORE, 47250779987968, 47250779996159,
+STORE, 47250779996160, 47250780102655,
+STORE, 47250780012544, 47250780102655,
+STORE, 47250779996160, 47250780012543,
+ERASE, 47250780012544, 47250780012544,
+STORE, 47250780012544, 47250780086271,
+STORE, 47250780086272, 47250780102655,
+STORE, 47250780065792, 47250780086271,
+STORE, 47250780012544, 47250780065791,
+ERASE, 47250780012544, 47250780012544,
+STORE, 47250780012544, 47250780065791,
+STORE, 47250780082176, 47250780086271,
+STORE, 47250780065792, 47250780082175,
+ERASE, 47250780065792, 47250780065792,
+STORE, 47250780065792, 47250780082175,
+STORE, 47250780094464, 47250780102655,
+STORE, 47250780086272, 47250780094463,
+ERASE, 47250780086272, 47250780086272,
+STORE, 47250780086272, 47250780094463,
+ERASE, 47250780094464, 47250780094464,
+STORE, 47250780094464, 47250780102655,
+STORE, 47250780102656, 47250781941759,
+STORE, 47250780241920, 47250781941759,
+STORE, 47250780102656, 47250780241919,
+ERASE, 47250780241920, 47250780241920,
+STORE, 47250780241920, 47250781900799,
+STORE, 47250781900800, 47250781941759,
+STORE, 47250781585408, 47250781900799,
+STORE, 47250780241920, 47250781585407,
+ERASE, 47250780241920, 47250780241920,
+STORE, 47250780241920, 47250781585407,
+STORE, 47250781896704, 47250781900799,
+STORE, 47250781585408, 47250781896703,
+ERASE, 47250781585408, 47250781585408,
+STORE, 47250781585408, 47250781896703,
+STORE, 47250781925376, 47250781941759,
+STORE, 47250781900800, 47250781925375,
+ERASE, 47250781900800, 47250781900800,
+STORE, 47250781900800, 47250781925375,
+ERASE, 47250781925376, 47250781925376,
+STORE, 47250781925376, 47250781941759,
+STORE, 47250781925376, 47250781954047,
+ERASE, 47250781900800, 47250781900800,
+STORE, 47250781900800, 47250781917183,
+STORE, 47250781917184, 47250781925375,
+ERASE, 47250780086272, 47250780086272,
+STORE, 47250780086272, 47250780090367,
+STORE, 47250780090368, 47250780094463,
+ERASE, 94425324986368, 94425324986368,
+STORE, 94425324986368, 94425325002751,
+STORE, 94425325002752, 94425325006847,
+ERASE, 140382015176704, 140382015176704,
+STORE, 140382015176704, 140382015180799,
+STORE, 140382015180800, 140382015184895,
+ERASE, 47250779979776, 47250779979776,
+STORE, 94425351438336, 94425351573503,
+STORE, 140737488347136, 140737488351231,
+STORE, 140736801144832, 140737488351231,
+ERASE, 140736801144832, 140736801144832,
+STORE, 140736801144832, 140736801148927,
+STORE, 94629429358592, 94629430071295,
+ERASE, 94629429358592, 94629429358592,
+STORE, 94629429358592, 94629429407743,
+STORE, 94629429407744, 94629430071295,
+ERASE, 94629429407744, 94629429407744,
+STORE, 94629429407744, 94629429952511,
+STORE, 94629429952512, 94629430050815,
+STORE, 94629430050816, 94629430071295,
+STORE, 139801685483520, 139801685655551,
+ERASE, 139801685483520, 139801685483520,
+STORE, 139801685483520, 139801685487615,
+STORE, 139801685487616, 139801685655551,
+ERASE, 139801685487616, 139801685487616,
+STORE, 139801685487616, 139801685610495,
+STORE, 139801685610496, 139801685643263,
+STORE, 139801685643264, 139801685651455,
+STORE, 139801685651456, 139801685655551,
+STORE, 140736801198080, 140736801202175,
+STORE, 140736801185792, 140736801198079,
+STORE, 47831109513216, 47831109521407,
+STORE, 47831109521408, 47831109529599,
+STORE, 47831109529600, 47831109636095,
+STORE, 47831109545984, 47831109636095,
+STORE, 47831109529600, 47831109545983,
+ERASE, 47831109545984, 47831109545984,
+STORE, 47831109545984, 47831109619711,
+STORE, 47831109619712, 47831109636095,
+STORE, 47831109599232, 47831109619711,
+STORE, 47831109545984, 47831109599231,
+ERASE, 47831109545984, 47831109545984,
+STORE, 47831109545984, 47831109599231,
+STORE, 47831109615616, 47831109619711,
+STORE, 47831109599232, 47831109615615,
+ERASE, 47831109599232, 47831109599232,
+STORE, 47831109599232, 47831109615615,
+STORE, 47831109627904, 47831109636095,
+STORE, 47831109619712, 47831109627903,
+ERASE, 47831109619712, 47831109619712,
+STORE, 47831109619712, 47831109627903,
+ERASE, 47831109627904, 47831109627904,
+STORE, 47831109627904, 47831109636095,
+STORE, 47831109636096, 47831111475199,
+STORE, 47831109775360, 47831111475199,
+STORE, 47831109636096, 47831109775359,
+ERASE, 47831109775360, 47831109775360,
+STORE, 47831109775360, 47831111434239,
+STORE, 47831111434240, 47831111475199,
+STORE, 47831111118848, 47831111434239,
+STORE, 47831109775360, 47831111118847,
+ERASE, 47831109775360, 47831109775360,
+STORE, 47831109775360, 47831111118847,
+STORE, 47831111430144, 47831111434239,
+STORE, 47831111118848, 47831111430143,
+ERASE, 47831111118848, 47831111118848,
+STORE, 47831111118848, 47831111430143,
+STORE, 47831111458816, 47831111475199,
+STORE, 47831111434240, 47831111458815,
+ERASE, 47831111434240, 47831111434240,
+STORE, 47831111434240, 47831111458815,
+ERASE, 47831111458816, 47831111458816,
+STORE, 47831111458816, 47831111475199,
+STORE, 47831111458816, 47831111487487,
+ERASE, 47831111434240, 47831111434240,
+STORE, 47831111434240, 47831111450623,
+STORE, 47831111450624, 47831111458815,
+ERASE, 47831109619712, 47831109619712,
+STORE, 47831109619712, 47831109623807,
+STORE, 47831109623808, 47831109627903,
+ERASE, 94629430050816, 94629430050816,
+STORE, 94629430050816, 94629430067199,
+STORE, 94629430067200, 94629430071295,
+ERASE, 139801685643264, 139801685643264,
+STORE, 139801685643264, 139801685647359,
+STORE, 139801685647360, 139801685651455,
+ERASE, 47831109513216, 47831109513216,
+STORE, 140737488347136, 140737488351231,
+STORE, 140729419612160, 140737488351231,
+ERASE, 140729419612160, 140729419612160,
+STORE, 140729419612160, 140729419616255,
+STORE, 94443354148864, 94443354861567,
+ERASE, 94443354148864, 94443354148864,
+STORE, 94443354148864, 94443354198015,
+STORE, 94443354198016, 94443354861567,
+ERASE, 94443354198016, 94443354198016,
+STORE, 94443354198016, 94443354742783,
+STORE, 94443354742784, 94443354841087,
+STORE, 94443354841088, 94443354861567,
+STORE, 139741700038656, 139741700210687,
+ERASE, 139741700038656, 139741700038656,
+STORE, 139741700038656, 139741700042751,
+STORE, 139741700042752, 139741700210687,
+ERASE, 139741700042752, 139741700042752,
+STORE, 139741700042752, 139741700165631,
+STORE, 139741700165632, 139741700198399,
+STORE, 139741700198400, 139741700206591,
+STORE, 139741700206592, 139741700210687,
+STORE, 140729420574720, 140729420578815,
+STORE, 140729420562432, 140729420574719,
+STORE, 47891094958080, 47891094966271,
+STORE, 47891094966272, 47891094974463,
+STORE, 47891094974464, 47891095080959,
+STORE, 47891094990848, 47891095080959,
+STORE, 47891094974464, 47891094990847,
+ERASE, 47891094990848, 47891094990848,
+STORE, 47891094990848, 47891095064575,
+STORE, 47891095064576, 47891095080959,
+STORE, 47891095044096, 47891095064575,
+STORE, 47891094990848, 47891095044095,
+ERASE, 47891094990848, 47891094990848,
+STORE, 47891094990848, 47891095044095,
+STORE, 47891095060480, 47891095064575,
+STORE, 47891095044096, 47891095060479,
+ERASE, 47891095044096, 47891095044096,
+STORE, 47891095044096, 47891095060479,
+STORE, 47891095072768, 47891095080959,
+STORE, 47891095064576, 47891095072767,
+ERASE, 47891095064576, 47891095064576,
+STORE, 47891095064576, 47891095072767,
+ERASE, 47891095072768, 47891095072768,
+STORE, 47891095072768, 47891095080959,
+STORE, 47891095080960, 47891096920063,
+STORE, 47891095220224, 47891096920063,
+STORE, 47891095080960, 47891095220223,
+ERASE, 47891095220224, 47891095220224,
+STORE, 47891095220224, 47891096879103,
+STORE, 47891096879104, 47891096920063,
+STORE, 47891096563712, 47891096879103,
+STORE, 47891095220224, 47891096563711,
+ERASE, 47891095220224, 47891095220224,
+STORE, 47891095220224, 47891096563711,
+STORE, 47891096875008, 47891096879103,
+STORE, 47891096563712, 47891096875007,
+ERASE, 47891096563712, 47891096563712,
+STORE, 47891096563712, 47891096875007,
+STORE, 47891096903680, 47891096920063,
+STORE, 47891096879104, 47891096903679,
+ERASE, 47891096879104, 47891096879104,
+STORE, 47891096879104, 47891096903679,
+ERASE, 47891096903680, 47891096903680,
+STORE, 47891096903680, 47891096920063,
+STORE, 47891096903680, 47891096932351,
+ERASE, 47891096879104, 47891096879104,
+STORE, 47891096879104, 47891096895487,
+STORE, 47891096895488, 47891096903679,
+ERASE, 47891095064576, 47891095064576,
+STORE, 47891095064576, 47891095068671,
+STORE, 47891095068672, 47891095072767,
+ERASE, 94443354841088, 94443354841088,
+STORE, 94443354841088, 94443354857471,
+STORE, 94443354857472, 94443354861567,
+ERASE, 139741700198400, 139741700198400,
+STORE, 139741700198400, 139741700202495,
+STORE, 139741700202496, 139741700206591,
+ERASE, 47891094958080, 47891094958080,
+STORE, 94443360825344, 94443360960511,
+STORE, 140737488347136, 140737488351231,
+STORE, 140722961661952, 140737488351231,
+ERASE, 140722961661952, 140722961661952,
+STORE, 140722961661952, 140722961666047,
+STORE, 94878388944896, 94878389657599,
+ERASE, 94878388944896, 94878388944896,
+STORE, 94878388944896, 94878388994047,
+STORE, 94878388994048, 94878389657599,
+ERASE, 94878388994048, 94878388994048,
+STORE, 94878388994048, 94878389538815,
+STORE, 94878389538816, 94878389637119,
+STORE, 94878389637120, 94878389657599,
+STORE, 140210690056192, 140210690228223,
+ERASE, 140210690056192, 140210690056192,
+STORE, 140210690056192, 140210690060287,
+STORE, 140210690060288, 140210690228223,
+ERASE, 140210690060288, 140210690060288,
+STORE, 140210690060288, 140210690183167,
+STORE, 140210690183168, 140210690215935,
+STORE, 140210690215936, 140210690224127,
+STORE, 140210690224128, 140210690228223,
+STORE, 140722963148800, 140722963152895,
+STORE, 140722963136512, 140722963148799,
+STORE, 47422104940544, 47422104948735,
+STORE, 47422104948736, 47422104956927,
+STORE, 47422104956928, 47422105063423,
+STORE, 47422104973312, 47422105063423,
+STORE, 47422104956928, 47422104973311,
+ERASE, 47422104973312, 47422104973312,
+STORE, 47422104973312, 47422105047039,
+STORE, 47422105047040, 47422105063423,
+STORE, 47422105026560, 47422105047039,
+STORE, 47422104973312, 47422105026559,
+ERASE, 47422104973312, 47422104973312,
+STORE, 47422104973312, 47422105026559,
+STORE, 47422105042944, 47422105047039,
+STORE, 47422105026560, 47422105042943,
+ERASE, 47422105026560, 47422105026560,
+STORE, 47422105026560, 47422105042943,
+STORE, 47422105055232, 47422105063423,
+STORE, 47422105047040, 47422105055231,
+ERASE, 47422105047040, 47422105047040,
+STORE, 47422105047040, 47422105055231,
+ERASE, 47422105055232, 47422105055232,
+STORE, 47422105055232, 47422105063423,
+STORE, 47422105063424, 47422106902527,
+STORE, 47422105202688, 47422106902527,
+STORE, 47422105063424, 47422105202687,
+ERASE, 47422105202688, 47422105202688,
+STORE, 47422105202688, 47422106861567,
+STORE, 47422106861568, 47422106902527,
+STORE, 47422106546176, 47422106861567,
+STORE, 47422105202688, 47422106546175,
+ERASE, 47422105202688, 47422105202688,
+STORE, 47422105202688, 47422106546175,
+STORE, 47422106857472, 47422106861567,
+STORE, 47422106546176, 47422106857471,
+ERASE, 47422106546176, 47422106546176,
+STORE, 47422106546176, 47422106857471,
+STORE, 47422106886144, 47422106902527,
+STORE, 47422106861568, 47422106886143,
+ERASE, 47422106861568, 47422106861568,
+STORE, 47422106861568, 47422106886143,
+ERASE, 47422106886144, 47422106886144,
+STORE, 47422106886144, 47422106902527,
+STORE, 47422106886144, 47422106914815,
+ERASE, 47422106861568, 47422106861568,
+STORE, 47422106861568, 47422106877951,
+STORE, 47422106877952, 47422106886143,
+ERASE, 47422105047040, 47422105047040,
+STORE, 47422105047040, 47422105051135,
+STORE, 47422105051136, 47422105055231,
+ERASE, 94878389637120, 94878389637120,
+STORE, 94878389637120, 94878389653503,
+STORE, 94878389653504, 94878389657599,
+ERASE, 140210690215936, 140210690215936,
+STORE, 140210690215936, 140210690220031,
+STORE, 140210690220032, 140210690224127,
+ERASE, 47422104940544, 47422104940544,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727690309632, 140737488351231,
+ERASE, 140727690309632, 140727690309632,
+STORE, 140727690309632, 140727690313727,
+STORE, 94121892208640, 94121892921343,
+ERASE, 94121892208640, 94121892208640,
+STORE, 94121892208640, 94121892257791,
+STORE, 94121892257792, 94121892921343,
+ERASE, 94121892257792, 94121892257792,
+STORE, 94121892257792, 94121892802559,
+STORE, 94121892802560, 94121892900863,
+STORE, 94121892900864, 94121892921343,
+STORE, 140662438326272, 140662438498303,
+ERASE, 140662438326272, 140662438326272,
+STORE, 140662438326272, 140662438330367,
+STORE, 140662438330368, 140662438498303,
+ERASE, 140662438330368, 140662438330368,
+STORE, 140662438330368, 140662438453247,
+STORE, 140662438453248, 140662438486015,
+STORE, 140662438486016, 140662438494207,
+STORE, 140662438494208, 140662438498303,
+STORE, 140727690379264, 140727690383359,
+STORE, 140727690366976, 140727690379263,
+STORE, 46970356670464, 46970356678655,
+STORE, 46970356678656, 46970356686847,
+STORE, 46970356686848, 46970356793343,
+STORE, 46970356703232, 46970356793343,
+STORE, 46970356686848, 46970356703231,
+ERASE, 46970356703232, 46970356703232,
+STORE, 46970356703232, 46970356776959,
+STORE, 46970356776960, 46970356793343,
+STORE, 46970356756480, 46970356776959,
+STORE, 46970356703232, 46970356756479,
+ERASE, 46970356703232, 46970356703232,
+STORE, 46970356703232, 46970356756479,
+STORE, 46970356772864, 46970356776959,
+STORE, 46970356756480, 46970356772863,
+ERASE, 46970356756480, 46970356756480,
+STORE, 46970356756480, 46970356772863,
+STORE, 46970356785152, 46970356793343,
+STORE, 46970356776960, 46970356785151,
+ERASE, 46970356776960, 46970356776960,
+STORE, 46970356776960, 46970356785151,
+ERASE, 46970356785152, 46970356785152,
+STORE, 46970356785152, 46970356793343,
+STORE, 46970356793344, 46970358632447,
+STORE, 46970356932608, 46970358632447,
+STORE, 46970356793344, 46970356932607,
+ERASE, 46970356932608, 46970356932608,
+STORE, 46970356932608, 46970358591487,
+STORE, 46970358591488, 46970358632447,
+STORE, 46970358276096, 46970358591487,
+STORE, 46970356932608, 46970358276095,
+ERASE, 46970356932608, 46970356932608,
+STORE, 46970356932608, 46970358276095,
+STORE, 46970358587392, 46970358591487,
+STORE, 46970358276096, 46970358587391,
+ERASE, 46970358276096, 46970358276096,
+STORE, 46970358276096, 46970358587391,
+STORE, 46970358616064, 46970358632447,
+STORE, 46970358591488, 46970358616063,
+ERASE, 46970358591488, 46970358591488,
+STORE, 46970358591488, 46970358616063,
+ERASE, 46970358616064, 46970358616064,
+STORE, 46970358616064, 46970358632447,
+STORE, 46970358616064, 46970358644735,
+ERASE, 46970358591488, 46970358591488,
+STORE, 46970358591488, 46970358607871,
+STORE, 46970358607872, 46970358616063,
+ERASE, 46970356776960, 46970356776960,
+STORE, 46970356776960, 46970356781055,
+STORE, 46970356781056, 46970356785151,
+ERASE, 94121892900864, 94121892900864,
+STORE, 94121892900864, 94121892917247,
+STORE, 94121892917248, 94121892921343,
+ERASE, 140662438486016, 140662438486016,
+STORE, 140662438486016, 140662438490111,
+STORE, 140662438490112, 140662438494207,
+ERASE, 46970356670464, 46970356670464,
+STORE, 94121898610688, 94121898745855,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737189351424, 140737488351231,
+ERASE, 140737189351424, 140737189351424,
+STORE, 140737189351424, 140737189355519,
+STORE, 93847948832768, 93847949545471,
+ERASE, 93847948832768, 93847948832768,
+STORE, 93847948832768, 93847948881919,
+STORE, 93847948881920, 93847949545471,
+ERASE, 93847948881920, 93847948881920,
+STORE, 93847948881920, 93847949426687,
+STORE, 93847949426688, 93847949524991,
+STORE, 93847949524992, 93847949545471,
+STORE, 139698989985792, 139698990157823,
+ERASE, 139698989985792, 139698989985792,
+STORE, 139698989985792, 139698989989887,
+STORE, 139698989989888, 139698990157823,
+ERASE, 139698989989888, 139698989989888,
+STORE, 139698989989888, 139698990112767,
+STORE, 139698990112768, 139698990145535,
+STORE, 139698990145536, 139698990153727,
+STORE, 139698990153728, 139698990157823,
+STORE, 140737189744640, 140737189748735,
+STORE, 140737189732352, 140737189744639,
+STORE, 47933805010944, 47933805019135,
+STORE, 47933805019136, 47933805027327,
+STORE, 47933805027328, 47933805133823,
+STORE, 47933805043712, 47933805133823,
+STORE, 47933805027328, 47933805043711,
+ERASE, 47933805043712, 47933805043712,
+STORE, 47933805043712, 47933805117439,
+STORE, 47933805117440, 47933805133823,
+STORE, 47933805096960, 47933805117439,
+STORE, 47933805043712, 47933805096959,
+ERASE, 47933805043712, 47933805043712,
+STORE, 47933805043712, 47933805096959,
+STORE, 47933805113344, 47933805117439,
+STORE, 47933805096960, 47933805113343,
+ERASE, 47933805096960, 47933805096960,
+STORE, 47933805096960, 47933805113343,
+STORE, 47933805125632, 47933805133823,
+STORE, 47933805117440, 47933805125631,
+ERASE, 47933805117440, 47933805117440,
+STORE, 47933805117440, 47933805125631,
+ERASE, 47933805125632, 47933805125632,
+STORE, 47933805125632, 47933805133823,
+STORE, 47933805133824, 47933806972927,
+STORE, 47933805273088, 47933806972927,
+STORE, 47933805133824, 47933805273087,
+ERASE, 47933805273088, 47933805273088,
+STORE, 47933805273088, 47933806931967,
+STORE, 47933806931968, 47933806972927,
+STORE, 47933806616576, 47933806931967,
+STORE, 47933805273088, 47933806616575,
+ERASE, 47933805273088, 47933805273088,
+STORE, 47933805273088, 47933806616575,
+STORE, 47933806927872, 47933806931967,
+STORE, 47933806616576, 47933806927871,
+ERASE, 47933806616576, 47933806616576,
+STORE, 47933806616576, 47933806927871,
+STORE, 47933806956544, 47933806972927,
+STORE, 47933806931968, 47933806956543,
+ERASE, 47933806931968, 47933806931968,
+STORE, 47933806931968, 47933806956543,
+ERASE, 47933806956544, 47933806956544,
+STORE, 47933806956544, 47933806972927,
+STORE, 47933806956544, 47933806985215,
+ERASE, 47933806931968, 47933806931968,
+STORE, 47933806931968, 47933806948351,
+STORE, 47933806948352, 47933806956543,
+ERASE, 47933805117440, 47933805117440,
+STORE, 47933805117440, 47933805121535,
+STORE, 47933805121536, 47933805125631,
+ERASE, 93847949524992, 93847949524992,
+STORE, 93847949524992, 93847949541375,
+STORE, 93847949541376, 93847949545471,
+ERASE, 139698990145536, 139698990145536,
+STORE, 139698990145536, 139698990149631,
+STORE, 139698990149632, 139698990153727,
+ERASE, 47933805010944, 47933805010944,
+STORE, 140737488347136, 140737488351231,
+STORE, 140725553991680, 140737488351231,
+ERASE, 140725553991680, 140725553991680,
+STORE, 140725553991680, 140725553995775,
+STORE, 93980056248320, 93980056961023,
+ERASE, 93980056248320, 93980056248320,
+STORE, 93980056248320, 93980056297471,
+STORE, 93980056297472, 93980056961023,
+ERASE, 93980056297472, 93980056297472,
+STORE, 93980056297472, 93980056842239,
+STORE, 93980056842240, 93980056940543,
+STORE, 93980056940544, 93980056961023,
+STORE, 140146588971008, 140146589143039,
+ERASE, 140146588971008, 140146588971008,
+STORE, 140146588971008, 140146588975103,
+STORE, 140146588975104, 140146589143039,
+ERASE, 140146588975104, 140146588975104,
+STORE, 140146588975104, 140146589097983,
+STORE, 140146589097984, 140146589130751,
+STORE, 140146589130752, 140146589138943,
+STORE, 140146589138944, 140146589143039,
+STORE, 140725554860032, 140725554864127,
+STORE, 140725554847744, 140725554860031,
+STORE, 47486206025728, 47486206033919,
+STORE, 47486206033920, 47486206042111,
+STORE, 47486206042112, 47486206148607,
+STORE, 47486206058496, 47486206148607,
+STORE, 47486206042112, 47486206058495,
+ERASE, 47486206058496, 47486206058496,
+STORE, 47486206058496, 47486206132223,
+STORE, 47486206132224, 47486206148607,
+STORE, 47486206111744, 47486206132223,
+STORE, 47486206058496, 47486206111743,
+ERASE, 47486206058496, 47486206058496,
+STORE, 47486206058496, 47486206111743,
+STORE, 47486206128128, 47486206132223,
+STORE, 47486206111744, 47486206128127,
+ERASE, 47486206111744, 47486206111744,
+STORE, 47486206111744, 47486206128127,
+STORE, 47486206140416, 47486206148607,
+STORE, 47486206132224, 47486206140415,
+ERASE, 47486206132224, 47486206132224,
+STORE, 47486206132224, 47486206140415,
+ERASE, 47486206140416, 47486206140416,
+STORE, 47486206140416, 47486206148607,
+STORE, 47486206148608, 47486207987711,
+STORE, 47486206287872, 47486207987711,
+STORE, 47486206148608, 47486206287871,
+ERASE, 47486206287872, 47486206287872,
+STORE, 47486206287872, 47486207946751,
+STORE, 47486207946752, 47486207987711,
+STORE, 47486207631360, 47486207946751,
+STORE, 47486206287872, 47486207631359,
+ERASE, 47486206287872, 47486206287872,
+STORE, 47486206287872, 47486207631359,
+STORE, 47486207942656, 47486207946751,
+STORE, 47486207631360, 47486207942655,
+ERASE, 47486207631360, 47486207631360,
+STORE, 47486207631360, 47486207942655,
+STORE, 47486207971328, 47486207987711,
+STORE, 47486207946752, 47486207971327,
+ERASE, 47486207946752, 47486207946752,
+STORE, 47486207946752, 47486207971327,
+ERASE, 47486207971328, 47486207971328,
+STORE, 47486207971328, 47486207987711,
+STORE, 47486207971328, 47486207999999,
+ERASE, 47486207946752, 47486207946752,
+STORE, 47486207946752, 47486207963135,
+STORE, 47486207963136, 47486207971327,
+ERASE, 47486206132224, 47486206132224,
+STORE, 47486206132224, 47486206136319,
+STORE, 47486206136320, 47486206140415,
+ERASE, 93980056940544, 93980056940544,
+STORE, 93980056940544, 93980056956927,
+STORE, 93980056956928, 93980056961023,
+ERASE, 140146589130752, 140146589130752,
+STORE, 140146589130752, 140146589134847,
+STORE, 140146589134848, 140146589138943,
+ERASE, 47486206025728, 47486206025728,
+STORE, 93980070006784, 93980070141951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727334776832, 140737488351231,
+ERASE, 140727334776832, 140727334776832,
+STORE, 140727334776832, 140727334780927,
+STORE, 94049747247104, 94049747959807,
+ERASE, 94049747247104, 94049747247104,
+STORE, 94049747247104, 94049747296255,
+STORE, 94049747296256, 94049747959807,
+ERASE, 94049747296256, 94049747296256,
+STORE, 94049747296256, 94049747841023,
+STORE, 94049747841024, 94049747939327,
+STORE, 94049747939328, 94049747959807,
+STORE, 140227307216896, 140227307388927,
+ERASE, 140227307216896, 140227307216896,
+STORE, 140227307216896, 140227307220991,
+STORE, 140227307220992, 140227307388927,
+ERASE, 140227307220992, 140227307220992,
+STORE, 140227307220992, 140227307343871,
+STORE, 140227307343872, 140227307376639,
+STORE, 140227307376640, 140227307384831,
+STORE, 140227307384832, 140227307388927,
+STORE, 140727335337984, 140727335342079,
+STORE, 140727335325696, 140727335337983,
+STORE, 47405487779840, 47405487788031,
+STORE, 47405487788032, 47405487796223,
+STORE, 47405487796224, 47405487902719,
+STORE, 47405487812608, 47405487902719,
+STORE, 47405487796224, 47405487812607,
+ERASE, 47405487812608, 47405487812608,
+STORE, 47405487812608, 47405487886335,
+STORE, 47405487886336, 47405487902719,
+STORE, 47405487865856, 47405487886335,
+STORE, 47405487812608, 47405487865855,
+ERASE, 47405487812608, 47405487812608,
+STORE, 47405487812608, 47405487865855,
+STORE, 47405487882240, 47405487886335,
+STORE, 47405487865856, 47405487882239,
+ERASE, 47405487865856, 47405487865856,
+STORE, 47405487865856, 47405487882239,
+STORE, 47405487894528, 47405487902719,
+STORE, 47405487886336, 47405487894527,
+ERASE, 47405487886336, 47405487886336,
+STORE, 47405487886336, 47405487894527,
+ERASE, 47405487894528, 47405487894528,
+STORE, 47405487894528, 47405487902719,
+STORE, 47405487902720, 47405489741823,
+STORE, 47405488041984, 47405489741823,
+STORE, 47405487902720, 47405488041983,
+ERASE, 47405488041984, 47405488041984,
+STORE, 47405488041984, 47405489700863,
+STORE, 47405489700864, 47405489741823,
+STORE, 47405489385472, 47405489700863,
+STORE, 47405488041984, 47405489385471,
+ERASE, 47405488041984, 47405488041984,
+STORE, 47405488041984, 47405489385471,
+STORE, 47405489696768, 47405489700863,
+STORE, 47405489385472, 47405489696767,
+ERASE, 47405489385472, 47405489385472,
+STORE, 47405489385472, 47405489696767,
+STORE, 47405489725440, 47405489741823,
+STORE, 47405489700864, 47405489725439,
+ERASE, 47405489700864, 47405489700864,
+STORE, 47405489700864, 47405489725439,
+ERASE, 47405489725440, 47405489725440,
+STORE, 47405489725440, 47405489741823,
+STORE, 47405489725440, 47405489754111,
+ERASE, 47405489700864, 47405489700864,
+STORE, 47405489700864, 47405489717247,
+STORE, 47405489717248, 47405489725439,
+ERASE, 47405487886336, 47405487886336,
+STORE, 47405487886336, 47405487890431,
+STORE, 47405487890432, 47405487894527,
+ERASE, 94049747939328, 94049747939328,
+STORE, 94049747939328, 94049747955711,
+STORE, 94049747955712, 94049747959807,
+ERASE, 140227307376640, 140227307376640,
+STORE, 140227307376640, 140227307380735,
+STORE, 140227307380736, 140227307384831,
+ERASE, 47405487779840, 47405487779840,
+STORE, 94049758810112, 94049758945279,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727079718912, 140737488351231,
+ERASE, 140727079718912, 140727079718912,
+STORE, 140727079718912, 140727079723007,
+STORE, 94250996527104, 94250997239807,
+ERASE, 94250996527104, 94250996527104,
+STORE, 94250996527104, 94250996576255,
+STORE, 94250996576256, 94250997239807,
+ERASE, 94250996576256, 94250996576256,
+STORE, 94250996576256, 94250997121023,
+STORE, 94250997121024, 94250997219327,
+STORE, 94250997219328, 94250997239807,
+STORE, 140060022587392, 140060022759423,
+ERASE, 140060022587392, 140060022587392,
+STORE, 140060022587392, 140060022591487,
+STORE, 140060022591488, 140060022759423,
+ERASE, 140060022591488, 140060022591488,
+STORE, 140060022591488, 140060022714367,
+STORE, 140060022714368, 140060022747135,
+STORE, 140060022747136, 140060022755327,
+STORE, 140060022755328, 140060022759423,
+STORE, 140727079788544, 140727079792639,
+STORE, 140727079776256, 140727079788543,
+/* this next one caused issues when lowering the efficiency */
+STORE, 47572772409344, 47572772417535,
+STORE, 47572772417536, 47572772425727,
+STORE, 47572772425728, 47572772532223,
+STORE, 47572772442112, 47572772532223,
+STORE, 47572772425728, 47572772442111,
+ERASE, 47572772442112, 47572772442112,
+STORE, 47572772442112, 47572772515839,
+STORE, 47572772515840, 47572772532223,
+STORE, 47572772495360, 47572772515839,
+STORE, 47572772442112, 47572772495359,
+ERASE, 47572772442112, 47572772442112,
+STORE, 47572772442112, 47572772495359,
+STORE, 47572772511744, 47572772515839,
+STORE, 47572772495360, 47572772511743,
+ERASE, 47572772495360, 47572772495360,
+STORE, 47572772495360, 47572772511743,
+STORE, 47572772524032, 47572772532223,
+STORE, 47572772515840, 47572772524031,
+ERASE, 47572772515840, 47572772515840,
+STORE, 47572772515840, 47572772524031,
+ERASE, 47572772524032, 47572772524032,
+STORE, 47572772524032, 47572772532223,
+STORE, 47572772532224, 47572774371327,
+STORE, 47572772671488, 47572774371327,
+STORE, 47572772532224, 47572772671487,
+ERASE, 47572772671488, 47572772671488,
+STORE, 47572772671488, 47572774330367,
+STORE, 47572774330368, 47572774371327,
+STORE, 47572774014976, 47572774330367,
+STORE, 47572772671488, 47572774014975,
+ERASE, 47572772671488, 47572772671488,
+STORE, 47572772671488, 47572774014975,
+STORE, 47572774326272, 47572774330367,
+STORE, 47572774014976, 47572774326271,
+ERASE, 47572774014976, 47572774014976,
+STORE, 47572774014976, 47572774326271,
+STORE, 47572774354944, 47572774371327,
+STORE, 47572774330368, 47572774354943,
+ERASE, 47572774330368, 47572774330368,
+STORE, 47572774330368, 47572774354943,
+ERASE, 47572774354944, 47572774354944,
+STORE, 47572774354944, 47572774371327,
+STORE, 47572774354944, 47572774383615,
+ERASE, 47572774330368, 47572774330368,
+STORE, 47572774330368, 47572774346751,
+STORE, 47572774346752, 47572774354943,
+ERASE, 47572772515840, 47572772515840,
+STORE, 47572772515840, 47572772519935,
+STORE, 47572772519936, 47572772524031,
+ERASE, 94250997219328, 94250997219328,
+STORE, 94250997219328, 94250997235711,
+STORE, 94250997235712, 94250997239807,
+ERASE, 140060022747136, 140060022747136,
+STORE, 140060022747136, 140060022751231,
+STORE, 140060022751232, 140060022755327,
+ERASE, 47572772409344, 47572772409344,
+STORE, 94251018305536, 94251018440703,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730012389376, 140737488351231,
+ERASE, 140730012389376, 140730012389376,
+STORE, 140730012389376, 140730012393471,
+STORE, 94382607675392, 94382607695871,
+ERASE, 94382607675392, 94382607675392,
+STORE, 94382607675392, 94382607679487,
+STORE, 94382607679488, 94382607695871,
+ERASE, 94382607679488, 94382607679488,
+STORE, 94382607679488, 94382607683583,
+STORE, 94382607683584, 94382607687679,
+STORE, 94382607687680, 94382607695871,
+STORE, 140252451454976, 140252451627007,
+ERASE, 140252451454976, 140252451454976,
+STORE, 140252451454976, 140252451459071,
+STORE, 140252451459072, 140252451627007,
+ERASE, 140252451459072, 140252451459072,
+STORE, 140252451459072, 140252451581951,
+STORE, 140252451581952, 140252451614719,
+STORE, 140252451614720, 140252451622911,
+STORE, 140252451622912, 140252451627007,
+STORE, 140730013548544, 140730013552639,
+STORE, 140730013536256, 140730013548543,
+STORE, 47380343541760, 47380343549951,
+STORE, 47380343549952, 47380343558143,
+STORE, 47380343558144, 47380345397247,
+STORE, 47380343697408, 47380345397247,
+STORE, 47380343558144, 47380343697407,
+ERASE, 47380343697408, 47380343697408,
+STORE, 47380343697408, 47380345356287,
+STORE, 47380345356288, 47380345397247,
+STORE, 47380345040896, 47380345356287,
+STORE, 47380343697408, 47380345040895,
+ERASE, 47380343697408, 47380343697408,
+STORE, 47380343697408, 47380345040895,
+STORE, 47380345352192, 47380345356287,
+STORE, 47380345040896, 47380345352191,
+ERASE, 47380345040896, 47380345040896,
+STORE, 47380345040896, 47380345352191,
+STORE, 47380345380864, 47380345397247,
+STORE, 47380345356288, 47380345380863,
+ERASE, 47380345356288, 47380345356288,
+STORE, 47380345356288, 47380345380863,
+ERASE, 47380345380864, 47380345380864,
+STORE, 47380345380864, 47380345397247,
+ERASE, 47380345356288, 47380345356288,
+STORE, 47380345356288, 47380345372671,
+STORE, 47380345372672, 47380345380863,
+ERASE, 94382607687680, 94382607687680,
+STORE, 94382607687680, 94382607691775,
+STORE, 94382607691776, 94382607695871,
+ERASE, 140252451614720, 140252451614720,
+STORE, 140252451614720, 140252451618815,
+STORE, 140252451618816, 140252451622911,
+ERASE, 47380343541760, 47380343541760,
+STORE, 94382626803712, 94382626938879,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730900271104, 140737488351231,
+ERASE, 140730900271104, 140730900271104,
+STORE, 140730900271104, 140730900275199,
+STORE, 93855478120448, 93855478337535,
+ERASE, 93855478120448, 93855478120448,
+STORE, 93855478120448, 93855478198271,
+STORE, 93855478198272, 93855478337535,
+ERASE, 93855478198272, 93855478198272,
+STORE, 93855478198272, 93855478243327,
+STORE, 93855478243328, 93855478288383,
+STORE, 93855478288384, 93855478337535,
+STORE, 140092686573568, 140092686745599,
+ERASE, 140092686573568, 140092686573568,
+STORE, 140092686573568, 140092686577663,
+STORE, 140092686577664, 140092686745599,
+ERASE, 140092686577664, 140092686577664,
+STORE, 140092686577664, 140092686700543,
+STORE, 140092686700544, 140092686733311,
+STORE, 140092686733312, 140092686741503,
+STORE, 140092686741504, 140092686745599,
+STORE, 140730900537344, 140730900541439,
+STORE, 140730900525056, 140730900537343,
+STORE, 47540108423168, 47540108431359,
+STORE, 47540108431360, 47540108439551,
+STORE, 47540108439552, 47540110278655,
+STORE, 47540108578816, 47540110278655,
+STORE, 47540108439552, 47540108578815,
+ERASE, 47540108578816, 47540108578816,
+STORE, 47540108578816, 47540110237695,
+STORE, 47540110237696, 47540110278655,
+STORE, 47540109922304, 47540110237695,
+STORE, 47540108578816, 47540109922303,
+ERASE, 47540108578816, 47540108578816,
+STORE, 47540108578816, 47540109922303,
+STORE, 47540110233600, 47540110237695,
+STORE, 47540109922304, 47540110233599,
+ERASE, 47540109922304, 47540109922304,
+STORE, 47540109922304, 47540110233599,
+STORE, 47540110262272, 47540110278655,
+STORE, 47540110237696, 47540110262271,
+ERASE, 47540110237696, 47540110237696,
+STORE, 47540110237696, 47540110262271,
+ERASE, 47540110262272, 47540110262272,
+STORE, 47540110262272, 47540110278655,
+ERASE, 47540110237696, 47540110237696,
+STORE, 47540110237696, 47540110254079,
+STORE, 47540110254080, 47540110262271,
+ERASE, 93855478288384, 93855478288384,
+STORE, 93855478288384, 93855478333439,
+STORE, 93855478333440, 93855478337535,
+ERASE, 140092686733312, 140092686733312,
+STORE, 140092686733312, 140092686737407,
+STORE, 140092686737408, 140092686741503,
+ERASE, 47540108423168, 47540108423168,
+STORE, 93855492222976, 93855492358143,
+STORE, 93855492222976, 93855492493311,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733498146816, 140737488351231,
+ERASE, 140733498146816, 140733498146816,
+STORE, 140733498146816, 140733498150911,
+STORE, 94170739654656, 94170740367359,
+ERASE, 94170739654656, 94170739654656,
+STORE, 94170739654656, 94170739703807,
+STORE, 94170739703808, 94170740367359,
+ERASE, 94170739703808, 94170739703808,
+STORE, 94170739703808, 94170740248575,
+STORE, 94170740248576, 94170740346879,
+STORE, 94170740346880, 94170740367359,
+STORE, 140024788877312, 140024789049343,
+ERASE, 140024788877312, 140024788877312,
+STORE, 140024788877312, 140024788881407,
+STORE, 140024788881408, 140024789049343,
+ERASE, 140024788881408, 140024788881408,
+STORE, 140024788881408, 140024789004287,
+STORE, 140024789004288, 140024789037055,
+STORE, 140024789037056, 140024789045247,
+STORE, 140024789045248, 140024789049343,
+STORE, 140733499023360, 140733499027455,
+STORE, 140733499011072, 140733499023359,
+STORE, 47608006119424, 47608006127615,
+STORE, 47608006127616, 47608006135807,
+STORE, 47608006135808, 47608006242303,
+STORE, 47608006152192, 47608006242303,
+STORE, 47608006135808, 47608006152191,
+ERASE, 47608006152192, 47608006152192,
+STORE, 47608006152192, 47608006225919,
+STORE, 47608006225920, 47608006242303,
+STORE, 47608006205440, 47608006225919,
+STORE, 47608006152192, 47608006205439,
+ERASE, 47608006152192, 47608006152192,
+STORE, 47608006152192, 47608006205439,
+STORE, 47608006221824, 47608006225919,
+STORE, 47608006205440, 47608006221823,
+ERASE, 47608006205440, 47608006205440,
+STORE, 47608006205440, 47608006221823,
+STORE, 47608006234112, 47608006242303,
+STORE, 47608006225920, 47608006234111,
+ERASE, 47608006225920, 47608006225920,
+STORE, 47608006225920, 47608006234111,
+ERASE, 47608006234112, 47608006234112,
+STORE, 47608006234112, 47608006242303,
+STORE, 47608006242304, 47608008081407,
+STORE, 47608006381568, 47608008081407,
+STORE, 47608006242304, 47608006381567,
+ERASE, 47608006381568, 47608006381568,
+STORE, 47608006381568, 47608008040447,
+STORE, 47608008040448, 47608008081407,
+STORE, 47608007725056, 47608008040447,
+STORE, 47608006381568, 47608007725055,
+ERASE, 47608006381568, 47608006381568,
+STORE, 47608006381568, 47608007725055,
+STORE, 47608008036352, 47608008040447,
+STORE, 47608007725056, 47608008036351,
+ERASE, 47608007725056, 47608007725056,
+STORE, 47608007725056, 47608008036351,
+STORE, 47608008065024, 47608008081407,
+STORE, 47608008040448, 47608008065023,
+ERASE, 47608008040448, 47608008040448,
+STORE, 47608008040448, 47608008065023,
+ERASE, 47608008065024, 47608008065024,
+STORE, 47608008065024, 47608008081407,
+STORE, 47608008065024, 47608008093695,
+ERASE, 47608008040448, 47608008040448,
+STORE, 47608008040448, 47608008056831,
+STORE, 47608008056832, 47608008065023,
+ERASE, 47608006225920, 47608006225920,
+STORE, 47608006225920, 47608006230015,
+STORE, 47608006230016, 47608006234111,
+ERASE, 94170740346880, 94170740346880,
+STORE, 94170740346880, 94170740363263,
+STORE, 94170740363264, 94170740367359,
+ERASE, 140024789037056, 140024789037056,
+STORE, 140024789037056, 140024789041151,
+STORE, 140024789041152, 140024789045247,
+ERASE, 47608006119424, 47608006119424,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730264326144, 140737488351231,
+ERASE, 140730264326144, 140730264326144,
+STORE, 140730264326144, 140730264330239,
+STORE, 94653216407552, 94653217120255,
+ERASE, 94653216407552, 94653216407552,
+STORE, 94653216407552, 94653216456703,
+STORE, 94653216456704, 94653217120255,
+ERASE, 94653216456704, 94653216456704,
+STORE, 94653216456704, 94653217001471,
+STORE, 94653217001472, 94653217099775,
+STORE, 94653217099776, 94653217120255,
+STORE, 140103617011712, 140103617183743,
+ERASE, 140103617011712, 140103617011712,
+STORE, 140103617011712, 140103617015807,
+STORE, 140103617015808, 140103617183743,
+ERASE, 140103617015808, 140103617015808,
+STORE, 140103617015808, 140103617138687,
+STORE, 140103617138688, 140103617171455,
+STORE, 140103617171456, 140103617179647,
+STORE, 140103617179648, 140103617183743,
+STORE, 140730265427968, 140730265432063,
+STORE, 140730265415680, 140730265427967,
+STORE, 47529177985024, 47529177993215,
+STORE, 47529177993216, 47529178001407,
+STORE, 47529178001408, 47529178107903,
+STORE, 47529178017792, 47529178107903,
+STORE, 47529178001408, 47529178017791,
+ERASE, 47529178017792, 47529178017792,
+STORE, 47529178017792, 47529178091519,
+STORE, 47529178091520, 47529178107903,
+STORE, 47529178071040, 47529178091519,
+STORE, 47529178017792, 47529178071039,
+ERASE, 47529178017792, 47529178017792,
+STORE, 47529178017792, 47529178071039,
+STORE, 47529178087424, 47529178091519,
+STORE, 47529178071040, 47529178087423,
+ERASE, 47529178071040, 47529178071040,
+STORE, 47529178071040, 47529178087423,
+STORE, 47529178099712, 47529178107903,
+STORE, 47529178091520, 47529178099711,
+ERASE, 47529178091520, 47529178091520,
+STORE, 47529178091520, 47529178099711,
+ERASE, 47529178099712, 47529178099712,
+STORE, 47529178099712, 47529178107903,
+STORE, 47529178107904, 47529179947007,
+STORE, 47529178247168, 47529179947007,
+STORE, 47529178107904, 47529178247167,
+ERASE, 47529178247168, 47529178247168,
+STORE, 47529178247168, 47529179906047,
+STORE, 47529179906048, 47529179947007,
+STORE, 47529179590656, 47529179906047,
+STORE, 47529178247168, 47529179590655,
+ERASE, 47529178247168, 47529178247168,
+STORE, 47529178247168, 47529179590655,
+STORE, 47529179901952, 47529179906047,
+STORE, 47529179590656, 47529179901951,
+ERASE, 47529179590656, 47529179590656,
+STORE, 47529179590656, 47529179901951,
+STORE, 47529179930624, 47529179947007,
+STORE, 47529179906048, 47529179930623,
+ERASE, 47529179906048, 47529179906048,
+STORE, 47529179906048, 47529179930623,
+ERASE, 47529179930624, 47529179930624,
+STORE, 47529179930624, 47529179947007,
+STORE, 47529179930624, 47529179959295,
+ERASE, 47529179906048, 47529179906048,
+STORE, 47529179906048, 47529179922431,
+STORE, 47529179922432, 47529179930623,
+ERASE, 47529178091520, 47529178091520,
+STORE, 47529178091520, 47529178095615,
+STORE, 47529178095616, 47529178099711,
+ERASE, 94653217099776, 94653217099776,
+STORE, 94653217099776, 94653217116159,
+STORE, 94653217116160, 94653217120255,
+ERASE, 140103617171456, 140103617171456,
+STORE, 140103617171456, 140103617175551,
+STORE, 140103617175552, 140103617179647,
+ERASE, 47529177985024, 47529177985024,
+STORE, 94653241135104, 94653241270271,
+STORE, 140737488347136, 140737488351231,
+STORE, 140736284549120, 140737488351231,
+ERASE, 140736284549120, 140736284549120,
+STORE, 140736284549120, 140736284553215,
+STORE, 93963663822848, 93963664506879,
+ERASE, 93963663822848, 93963663822848,
+STORE, 93963663822848, 93963663884287,
+STORE, 93963663884288, 93963664506879,
+ERASE, 93963663884288, 93963663884288,
+STORE, 93963663884288, 93963664240639,
+STORE, 93963664240640, 93963664379903,
+STORE, 93963664379904, 93963664506879,
+STORE, 140450188439552, 140450188611583,
+ERASE, 140450188439552, 140450188439552,
+STORE, 140450188439552, 140450188443647,
+STORE, 140450188443648, 140450188611583,
+ERASE, 140450188443648, 140450188443648,
+STORE, 140450188443648, 140450188566527,
+STORE, 140450188566528, 140450188599295,
+STORE, 140450188599296, 140450188607487,
+STORE, 140450188607488, 140450188611583,
+STORE, 140736284577792, 140736284581887,
+STORE, 140736284565504, 140736284577791,
+STORE, 47182606557184, 47182606565375,
+STORE, 47182606565376, 47182606573567,
+STORE, 47182606573568, 47182608412671,
+STORE, 47182606712832, 47182608412671,
+STORE, 47182606573568, 47182606712831,
+ERASE, 47182606712832, 47182606712832,
+STORE, 47182606712832, 47182608371711,
+STORE, 47182608371712, 47182608412671,
+STORE, 47182608056320, 47182608371711,
+STORE, 47182606712832, 47182608056319,
+ERASE, 47182606712832, 47182606712832,
+STORE, 47182606712832, 47182608056319,
+STORE, 47182608367616, 47182608371711,
+STORE, 47182608056320, 47182608367615,
+ERASE, 47182608056320, 47182608056320,
+STORE, 47182608056320, 47182608367615,
+STORE, 47182608396288, 47182608412671,
+STORE, 47182608371712, 47182608396287,
+ERASE, 47182608371712, 47182608371712,
+STORE, 47182608371712, 47182608396287,
+ERASE, 47182608396288, 47182608396288,
+STORE, 47182608396288, 47182608412671,
+STORE, 47182608412672, 47182608523263,
+STORE, 47182608429056, 47182608523263,
+STORE, 47182608412672, 47182608429055,
+ERASE, 47182608429056, 47182608429056,
+STORE, 47182608429056, 47182608515071,
+STORE, 47182608515072, 47182608523263,
+STORE, 47182608490496, 47182608515071,
+STORE, 47182608429056, 47182608490495,
+ERASE, 47182608429056, 47182608429056,
+STORE, 47182608429056, 47182608490495,
+STORE, 47182608510976, 47182608515071,
+STORE, 47182608490496, 47182608510975,
+ERASE, 47182608490496, 47182608490496,
+STORE, 47182608490496, 47182608510975,
+ERASE, 47182608515072, 47182608515072,
+STORE, 47182608515072, 47182608523263,
+STORE, 47182608523264, 47182608568319,
+ERASE, 47182608523264, 47182608523264,
+STORE, 47182608523264, 47182608531455,
+STORE, 47182608531456, 47182608568319,
+STORE, 47182608551936, 47182608568319,
+STORE, 47182608531456, 47182608551935,
+ERASE, 47182608531456, 47182608531456,
+STORE, 47182608531456, 47182608551935,
+STORE, 47182608560128, 47182608568319,
+STORE, 47182608551936, 47182608560127,
+ERASE, 47182608551936, 47182608551936,
+STORE, 47182608551936, 47182608568319,
+ERASE, 47182608551936, 47182608551936,
+STORE, 47182608551936, 47182608560127,
+STORE, 47182608560128, 47182608568319,
+ERASE, 47182608560128, 47182608560128,
+STORE, 47182608560128, 47182608568319,
+STORE, 47182608568320, 47182608916479,
+STORE, 47182608609280, 47182608916479,
+STORE, 47182608568320, 47182608609279,
+ERASE, 47182608609280, 47182608609280,
+STORE, 47182608609280, 47182608891903,
+STORE, 47182608891904, 47182608916479,
+STORE, 47182608822272, 47182608891903,
+STORE, 47182608609280, 47182608822271,
+ERASE, 47182608609280, 47182608609280,
+STORE, 47182608609280, 47182608822271,
+STORE, 47182608887808, 47182608891903,
+STORE, 47182608822272, 47182608887807,
+ERASE, 47182608822272, 47182608822272,
+STORE, 47182608822272, 47182608887807,
+ERASE, 47182608891904, 47182608891904,
+STORE, 47182608891904, 47182608916479,
+STORE, 47182608916480, 47182611177471,
+STORE, 47182609068032, 47182611177471,
+STORE, 47182608916480, 47182609068031,
+ERASE, 47182609068032, 47182609068032,
+STORE, 47182609068032, 47182611161087,
+STORE, 47182611161088, 47182611177471,
+STORE, 47182611169280, 47182611177471,
+STORE, 47182611161088, 47182611169279,
+ERASE, 47182611161088, 47182611161088,
+STORE, 47182611161088, 47182611169279,
+ERASE, 47182611169280, 47182611169280,
+STORE, 47182611169280, 47182611177471,
+STORE, 47182611177472, 47182611312639,
+ERASE, 47182611177472, 47182611177472,
+STORE, 47182611177472, 47182611202047,
+STORE, 47182611202048, 47182611312639,
+STORE, 47182611263488, 47182611312639,
+STORE, 47182611202048, 47182611263487,
+ERASE, 47182611202048, 47182611202048,
+STORE, 47182611202048, 47182611263487,
+STORE, 47182611288064, 47182611312639,
+STORE, 47182611263488, 47182611288063,
+ERASE, 47182611263488, 47182611263488,
+STORE, 47182611263488, 47182611312639,
+ERASE, 47182611263488, 47182611263488,
+STORE, 47182611263488, 47182611288063,
+STORE, 47182611288064, 47182611312639,
+STORE, 47182611296256, 47182611312639,
+STORE, 47182611288064, 47182611296255,
+ERASE, 47182611288064, 47182611288064,
+STORE, 47182611288064, 47182611296255,
+ERASE, 47182611296256, 47182611296256,
+STORE, 47182611296256, 47182611312639,
+STORE, 47182611296256, 47182611320831,
+STORE, 47182611320832, 47182611484671,
+ERASE, 47182611320832, 47182611320832,
+STORE, 47182611320832, 47182611333119,
+STORE, 47182611333120, 47182611484671,
+STORE, 47182611431424, 47182611484671,
+STORE, 47182611333120, 47182611431423,
+ERASE, 47182611333120, 47182611333120,
+STORE, 47182611333120, 47182611431423,
+STORE, 47182611476480, 47182611484671,
+STORE, 47182611431424, 47182611476479,
+ERASE, 47182611431424, 47182611431424,
+STORE, 47182611431424, 47182611484671,
+ERASE, 47182611431424, 47182611431424,
+STORE, 47182611431424, 47182611476479,
+STORE, 47182611476480, 47182611484671,
+ERASE, 47182611476480, 47182611476480,
+STORE, 47182611476480, 47182611484671,
+STORE, 47182611484672, 47182612082687,
+STORE, 47182611603456, 47182612082687,
+STORE, 47182611484672, 47182611603455,
+ERASE, 47182611603456, 47182611603456,
+STORE, 47182611603456, 47182612029439,
+STORE, 47182612029440, 47182612082687,
+STORE, 47182611918848, 47182612029439,
+STORE, 47182611603456, 47182611918847,
+ERASE, 47182611603456, 47182611603456,
+STORE, 47182611603456, 47182611918847,
+STORE, 47182612025344, 47182612029439,
+STORE, 47182611918848, 47182612025343,
+ERASE, 47182611918848, 47182611918848,
+STORE, 47182611918848, 47182612025343,
+ERASE, 47182612029440, 47182612029440,
+STORE, 47182612029440, 47182612082687,
+STORE, 47182612082688, 47182615134207,
+STORE, 47182612627456, 47182615134207,
+STORE, 47182612082688, 47182612627455,
+ERASE, 47182612627456, 47182612627456,
+STORE, 47182612627456, 47182614913023,
+STORE, 47182614913024, 47182615134207,
+STORE, 47182614323200, 47182614913023,
+STORE, 47182612627456, 47182614323199,
+ERASE, 47182612627456, 47182612627456,
+STORE, 47182612627456, 47182614323199,
+STORE, 47182614908928, 47182614913023,
+STORE, 47182614323200, 47182614908927,
+ERASE, 47182614323200, 47182614323200,
+STORE, 47182614323200, 47182614908927,
+STORE, 47182615117824, 47182615134207,
+STORE, 47182614913024, 47182615117823,
+ERASE, 47182614913024, 47182614913024,
+STORE, 47182614913024, 47182615117823,
+ERASE, 47182615117824, 47182615117824,
+STORE, 47182615117824, 47182615134207,
+STORE, 47182615134208, 47182615166975,
+ERASE, 47182615134208, 47182615134208,
+STORE, 47182615134208, 47182615142399,
+STORE, 47182615142400, 47182615166975,
+STORE, 47182615154688, 47182615166975,
+STORE, 47182615142400, 47182615154687,
+ERASE, 47182615142400, 47182615142400,
+STORE, 47182615142400, 47182615154687,
+STORE, 47182615158784, 47182615166975,
+STORE, 47182615154688, 47182615158783,
+ERASE, 47182615154688, 47182615154688,
+STORE, 47182615154688, 47182615166975,
+ERASE, 47182615154688, 47182615154688,
+STORE, 47182615154688, 47182615158783,
+STORE, 47182615158784, 47182615166975,
+ERASE, 47182615158784, 47182615158784,
+STORE, 47182615158784, 47182615166975,
+STORE, 47182615166976, 47182615203839,
+ERASE, 47182615166976, 47182615166976,
+STORE, 47182615166976, 47182615175167,
+STORE, 47182615175168, 47182615203839,
+STORE, 47182615191552, 47182615203839,
+STORE, 47182615175168, 47182615191551,
+ERASE, 47182615175168, 47182615175168,
+STORE, 47182615175168, 47182615191551,
+STORE, 47182615195648, 47182615203839,
+STORE, 47182615191552, 47182615195647,
+ERASE, 47182615191552, 47182615191552,
+STORE, 47182615191552, 47182615203839,
+ERASE, 47182615191552, 47182615191552,
+STORE, 47182615191552, 47182615195647,
+STORE, 47182615195648, 47182615203839,
+ERASE, 47182615195648, 47182615195648,
+STORE, 47182615195648, 47182615203839,
+STORE, 47182615203840, 47182615678975,
+ERASE, 47182615203840, 47182615203840,
+STORE, 47182615203840, 47182615212031,
+STORE, 47182615212032, 47182615678975,
+STORE, 47182615547904, 47182615678975,
+STORE, 47182615212032, 47182615547903,
+ERASE, 47182615212032, 47182615212032,
+STORE, 47182615212032, 47182615547903,
+STORE, 47182615670784, 47182615678975,
+STORE, 47182615547904, 47182615670783,
+ERASE, 47182615547904, 47182615547904,
+STORE, 47182615547904, 47182615678975,
+ERASE, 47182615547904, 47182615547904,
+STORE, 47182615547904, 47182615670783,
+STORE, 47182615670784, 47182615678975,
+ERASE, 47182615670784, 47182615670784,
+STORE, 47182615670784, 47182615678975,
+STORE, 47182615678976, 47182615687167,
+STORE, 47182615687168, 47182615707647,
+ERASE, 47182615687168, 47182615687168,
+STORE, 47182615687168, 47182615691263,
+STORE, 47182615691264, 47182615707647,
+STORE, 47182615695360, 47182615707647,
+STORE, 47182615691264, 47182615695359,
+ERASE, 47182615691264, 47182615691264,
+STORE, 47182615691264, 47182615695359,
+STORE, 47182615699456, 47182615707647,
+STORE, 47182615695360, 47182615699455,
+ERASE, 47182615695360, 47182615695360,
+STORE, 47182615695360, 47182615707647,
+ERASE, 47182615695360, 47182615695360,
+STORE, 47182615695360, 47182615699455,
+STORE, 47182615699456, 47182615707647,
+ERASE, 47182615699456, 47182615699456,
+STORE, 47182615699456, 47182615707647,
+STORE, 47182615707648, 47182615715839,
+ERASE, 47182608371712, 47182608371712,
+STORE, 47182608371712, 47182608388095,
+STORE, 47182608388096, 47182608396287,
+ERASE, 47182615699456, 47182615699456,
+STORE, 47182615699456, 47182615703551,
+STORE, 47182615703552, 47182615707647,
+ERASE, 47182611288064, 47182611288064,
+STORE, 47182611288064, 47182611292159,
+STORE, 47182611292160, 47182611296255,
+ERASE, 47182615670784, 47182615670784,
+STORE, 47182615670784, 47182615674879,
+STORE, 47182615674880, 47182615678975,
+ERASE, 47182615195648, 47182615195648,
+STORE, 47182615195648, 47182615199743,
+STORE, 47182615199744, 47182615203839,
+ERASE, 47182615158784, 47182615158784,
+STORE, 47182615158784, 47182615162879,
+STORE, 47182615162880, 47182615166975,
+ERASE, 47182614913024, 47182614913024,
+STORE, 47182614913024, 47182615109631,
+STORE, 47182615109632, 47182615117823,
+ERASE, 47182612029440, 47182612029440,
+STORE, 47182612029440, 47182612066303,
+STORE, 47182612066304, 47182612082687,
+ERASE, 47182611476480, 47182611476480,
+STORE, 47182611476480, 47182611480575,
+STORE, 47182611480576, 47182611484671,
+ERASE, 47182611161088, 47182611161088,
+STORE, 47182611161088, 47182611165183,
+STORE, 47182611165184, 47182611169279,
+ERASE, 47182608891904, 47182608891904,
+STORE, 47182608891904, 47182608912383,
+STORE, 47182608912384, 47182608916479,
+ERASE, 47182608560128, 47182608560128,
+STORE, 47182608560128, 47182608564223,
+STORE, 47182608564224, 47182608568319,
+ERASE, 47182608515072, 47182608515072,
+STORE, 47182608515072, 47182608519167,
+STORE, 47182608519168, 47182608523263,
+ERASE, 93963664379904, 93963664379904,
+STORE, 93963664379904, 93963664502783,
+STORE, 93963664502784, 93963664506879,
+ERASE, 140450188599296, 140450188599296,
+STORE, 140450188599296, 140450188603391,
+STORE, 140450188603392, 140450188607487,
+ERASE, 47182606557184, 47182606557184,
+STORE, 93963694723072, 93963694858239,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730313261056, 140737488351231,
+ERASE, 140730313261056, 140730313261056,
+STORE, 140730313261056, 140730313265151,
+STORE, 94386579017728, 94386579697663,
+ERASE, 94386579017728, 94386579017728,
+STORE, 94386579017728, 94386579083263,
+STORE, 94386579083264, 94386579697663,
+ERASE, 94386579083264, 94386579083264,
+STORE, 94386579083264, 94386579431423,
+STORE, 94386579431424, 94386579570687,
+STORE, 94386579570688, 94386579697663,
+STORE, 140124810838016, 140124811010047,
+ERASE, 140124810838016, 140124810838016,
+STORE, 140124810838016, 140124810842111,
+STORE, 140124810842112, 140124811010047,
+ERASE, 140124810842112, 140124810842112,
+STORE, 140124810842112, 140124810964991,
+STORE, 140124810964992, 140124810997759,
+STORE, 140124810997760, 140124811005951,
+STORE, 140124811005952, 140124811010047,
+STORE, 140730313601024, 140730313605119,
+STORE, 140730313588736, 140730313601023,
+STORE, 47507984158720, 47507984166911,
+STORE, 47507984166912, 47507984175103,
+STORE, 47507984175104, 47507986014207,
+STORE, 47507984314368, 47507986014207,
+STORE, 47507984175104, 47507984314367,
+ERASE, 47507984314368, 47507984314368,
+STORE, 47507984314368, 47507985973247,
+STORE, 47507985973248, 47507986014207,
+STORE, 47507985657856, 47507985973247,
+STORE, 47507984314368, 47507985657855,
+ERASE, 47507984314368, 47507984314368,
+STORE, 47507984314368, 47507985657855,
+STORE, 47507985969152, 47507985973247,
+STORE, 47507985657856, 47507985969151,
+ERASE, 47507985657856, 47507985657856,
+STORE, 47507985657856, 47507985969151,
+STORE, 47507985997824, 47507986014207,
+STORE, 47507985973248, 47507985997823,
+ERASE, 47507985973248, 47507985973248,
+STORE, 47507985973248, 47507985997823,
+ERASE, 47507985997824, 47507985997824,
+STORE, 47507985997824, 47507986014207,
+STORE, 47507986014208, 47507986124799,
+STORE, 47507986030592, 47507986124799,
+STORE, 47507986014208, 47507986030591,
+ERASE, 47507986030592, 47507986030592,
+STORE, 47507986030592, 47507986116607,
+STORE, 47507986116608, 47507986124799,
+STORE, 47507986092032, 47507986116607,
+STORE, 47507986030592, 47507986092031,
+ERASE, 47507986030592, 47507986030592,
+STORE, 47507986030592, 47507986092031,
+STORE, 47507986112512, 47507986116607,
+STORE, 47507986092032, 47507986112511,
+ERASE, 47507986092032, 47507986092032,
+STORE, 47507986092032, 47507986112511,
+ERASE, 47507986116608, 47507986116608,
+STORE, 47507986116608, 47507986124799,
+STORE, 47507986124800, 47507986169855,
+ERASE, 47507986124800, 47507986124800,
+STORE, 47507986124800, 47507986132991,
+STORE, 47507986132992, 47507986169855,
+STORE, 47507986153472, 47507986169855,
+STORE, 47507986132992, 47507986153471,
+ERASE, 47507986132992, 47507986132992,
+STORE, 47507986132992, 47507986153471,
+STORE, 47507986161664, 47507986169855,
+STORE, 47507986153472, 47507986161663,
+ERASE, 47507986153472, 47507986153472,
+STORE, 47507986153472, 47507986169855,
+ERASE, 47507986153472, 47507986153472,
+STORE, 47507986153472, 47507986161663,
+STORE, 47507986161664, 47507986169855,
+ERASE, 47507986161664, 47507986161664,
+STORE, 47507986161664, 47507986169855,
+STORE, 47507986169856, 47507986518015,
+STORE, 47507986210816, 47507986518015,
+STORE, 47507986169856, 47507986210815,
+ERASE, 47507986210816, 47507986210816,
+STORE, 47507986210816, 47507986493439,
+STORE, 47507986493440, 47507986518015,
+STORE, 47507986423808, 47507986493439,
+STORE, 47507986210816, 47507986423807,
+ERASE, 47507986210816, 47507986210816,
+STORE, 47507986210816, 47507986423807,
+STORE, 47507986489344, 47507986493439,
+STORE, 47507986423808, 47507986489343,
+ERASE, 47507986423808, 47507986423808,
+STORE, 47507986423808, 47507986489343,
+ERASE, 47507986493440, 47507986493440,
+STORE, 47507986493440, 47507986518015,
+STORE, 47507986518016, 47507988779007,
+STORE, 47507986669568, 47507988779007,
+STORE, 47507986518016, 47507986669567,
+ERASE, 47507986669568, 47507986669568,
+STORE, 47507986669568, 47507988762623,
+STORE, 47507988762624, 47507988779007,
+STORE, 47507988770816, 47507988779007,
+STORE, 47507988762624, 47507988770815,
+ERASE, 47507988762624, 47507988762624,
+STORE, 47507988762624, 47507988770815,
+ERASE, 47507988770816, 47507988770816,
+STORE, 47507988770816, 47507988779007,
+STORE, 47507988779008, 47507988914175,
+ERASE, 47507988779008, 47507988779008,
+STORE, 47507988779008, 47507988803583,
+STORE, 47507988803584, 47507988914175,
+STORE, 47507988865024, 47507988914175,
+STORE, 47507988803584, 47507988865023,
+ERASE, 47507988803584, 47507988803584,
+STORE, 47507988803584, 47507988865023,
+STORE, 47507988889600, 47507988914175,
+STORE, 47507988865024, 47507988889599,
+ERASE, 47507988865024, 47507988865024,
+STORE, 47507988865024, 47507988914175,
+ERASE, 47507988865024, 47507988865024,
+STORE, 47507988865024, 47507988889599,
+STORE, 47507988889600, 47507988914175,
+STORE, 47507988897792, 47507988914175,
+STORE, 47507988889600, 47507988897791,
+ERASE, 47507988889600, 47507988889600,
+STORE, 47507988889600, 47507988897791,
+ERASE, 47507988897792, 47507988897792,
+STORE, 47507988897792, 47507988914175,
+STORE, 47507988897792, 47507988922367,
+STORE, 47507988922368, 47507989086207,
+ERASE, 47507988922368, 47507988922368,
+STORE, 47507988922368, 47507988934655,
+STORE, 47507988934656, 47507989086207,
+STORE, 47507989032960, 47507989086207,
+STORE, 47507988934656, 47507989032959,
+ERASE, 47507988934656, 47507988934656,
+STORE, 47507988934656, 47507989032959,
+STORE, 47507989078016, 47507989086207,
+STORE, 47507989032960, 47507989078015,
+ERASE, 47507989032960, 47507989032960,
+STORE, 47507989032960, 47507989086207,
+ERASE, 47507989032960, 47507989032960,
+STORE, 47507989032960, 47507989078015,
+STORE, 47507989078016, 47507989086207,
+ERASE, 47507989078016, 47507989078016,
+STORE, 47507989078016, 47507989086207,
+STORE, 47507989086208, 47507989684223,
+STORE, 47507989204992, 47507989684223,
+STORE, 47507989086208, 47507989204991,
+ERASE, 47507989204992, 47507989204992,
+STORE, 47507989204992, 47507989630975,
+STORE, 47507989630976, 47507989684223,
+STORE, 47507989520384, 47507989630975,
+STORE, 47507989204992, 47507989520383,
+ERASE, 47507989204992, 47507989204992,
+STORE, 47507989204992, 47507989520383,
+STORE, 47507989626880, 47507989630975,
+STORE, 47507989520384, 47507989626879,
+ERASE, 47507989520384, 47507989520384,
+STORE, 47507989520384, 47507989626879,
+ERASE, 47507989630976, 47507989630976,
+STORE, 47507989630976, 47507989684223,
+STORE, 47507989684224, 47507992735743,
+STORE, 47507990228992, 47507992735743,
+STORE, 47507989684224, 47507990228991,
+ERASE, 47507990228992, 47507990228992,
+STORE, 47507990228992, 47507992514559,
+STORE, 47507992514560, 47507992735743,
+STORE, 47507991924736, 47507992514559,
+STORE, 47507990228992, 47507991924735,
+ERASE, 47507990228992, 47507990228992,
+STORE, 47507990228992, 47507991924735,
+STORE, 47507992510464, 47507992514559,
+STORE, 47507991924736, 47507992510463,
+ERASE, 47507991924736, 47507991924736,
+STORE, 47507991924736, 47507992510463,
+STORE, 47507992719360, 47507992735743,
+STORE, 47507992514560, 47507992719359,
+ERASE, 47507992514560, 47507992514560,
+STORE, 47507992514560, 47507992719359,
+ERASE, 47507992719360, 47507992719360,
+STORE, 47507992719360, 47507992735743,
+STORE, 47507992735744, 47507992768511,
+ERASE, 47507992735744, 47507992735744,
+STORE, 47507992735744, 47507992743935,
+STORE, 47507992743936, 47507992768511,
+STORE, 47507992756224, 47507992768511,
+STORE, 47507992743936, 47507992756223,
+ERASE, 47507992743936, 47507992743936,
+STORE, 47507992743936, 47507992756223,
+STORE, 47507992760320, 47507992768511,
+STORE, 47507992756224, 47507992760319,
+ERASE, 47507992756224, 47507992756224,
+STORE, 47507992756224, 47507992768511,
+ERASE, 47507992756224, 47507992756224,
+STORE, 47507992756224, 47507992760319,
+STORE, 47507992760320, 47507992768511,
+ERASE, 47507992760320, 47507992760320,
+STORE, 47507992760320, 47507992768511,
+STORE, 47507992768512, 47507992805375,
+ERASE, 47507992768512, 47507992768512,
+STORE, 47507992768512, 47507992776703,
+STORE, 47507992776704, 47507992805375,
+STORE, 47507992793088, 47507992805375,
+STORE, 47507992776704, 47507992793087,
+ERASE, 47507992776704, 47507992776704,
+STORE, 47507992776704, 47507992793087,
+STORE, 47507992797184, 47507992805375,
+STORE, 47507992793088, 47507992797183,
+ERASE, 47507992793088, 47507992793088,
+STORE, 47507992793088, 47507992805375,
+ERASE, 47507992793088, 47507992793088,
+STORE, 47507992793088, 47507992797183,
+STORE, 47507992797184, 47507992805375,
+ERASE, 47507992797184, 47507992797184,
+STORE, 47507992797184, 47507992805375,
+STORE, 47507992805376, 47507993280511,
+ERASE, 47507992805376, 47507992805376,
+STORE, 47507992805376, 47507992813567,
+STORE, 47507992813568, 47507993280511,
+STORE, 47507993149440, 47507993280511,
+STORE, 47507992813568, 47507993149439,
+ERASE, 47507992813568, 47507992813568,
+STORE, 47507992813568, 47507993149439,
+STORE, 47507993272320, 47507993280511,
+STORE, 47507993149440, 47507993272319,
+ERASE, 47507993149440, 47507993149440,
+STORE, 47507993149440, 47507993280511,
+ERASE, 47507993149440, 47507993149440,
+STORE, 47507993149440, 47507993272319,
+STORE, 47507993272320, 47507993280511,
+ERASE, 47507993272320, 47507993272320,
+STORE, 47507993272320, 47507993280511,
+STORE, 47507993280512, 47507993288703,
+STORE, 47507993288704, 47507993309183,
+ERASE, 47507993288704, 47507993288704,
+STORE, 47507993288704, 47507993292799,
+STORE, 47507993292800, 47507993309183,
+STORE, 47507993296896, 47507993309183,
+STORE, 47507993292800, 47507993296895,
+ERASE, 47507993292800, 47507993292800,
+STORE, 47507993292800, 47507993296895,
+STORE, 47507993300992, 47507993309183,
+STORE, 47507993296896, 47507993300991,
+ERASE, 47507993296896, 47507993296896,
+STORE, 47507993296896, 47507993309183,
+ERASE, 47507993296896, 47507993296896,
+STORE, 47507993296896, 47507993300991,
+STORE, 47507993300992, 47507993309183,
+ERASE, 47507993300992, 47507993300992,
+STORE, 47507993300992, 47507993309183,
+STORE, 47507993309184, 47507993317375,
+ERASE, 47507985973248, 47507985973248,
+STORE, 47507985973248, 47507985989631,
+STORE, 47507985989632, 47507985997823,
+ERASE, 47507993300992, 47507993300992,
+STORE, 47507993300992, 47507993305087,
+STORE, 47507993305088, 47507993309183,
+ERASE, 47507988889600, 47507988889600,
+STORE, 47507988889600, 47507988893695,
+STORE, 47507988893696, 47507988897791,
+ERASE, 47507993272320, 47507993272320,
+STORE, 47507993272320, 47507993276415,
+STORE, 47507993276416, 47507993280511,
+ERASE, 47507992797184, 47507992797184,
+STORE, 47507992797184, 47507992801279,
+STORE, 47507992801280, 47507992805375,
+ERASE, 47507992760320, 47507992760320,
+STORE, 47507992760320, 47507992764415,
+STORE, 47507992764416, 47507992768511,
+ERASE, 47507992514560, 47507992514560,
+STORE, 47507992514560, 47507992711167,
+STORE, 47507992711168, 47507992719359,
+ERASE, 47507989630976, 47507989630976,
+STORE, 47507989630976, 47507989667839,
+STORE, 47507989667840, 47507989684223,
+ERASE, 47507989078016, 47507989078016,
+STORE, 47507989078016, 47507989082111,
+STORE, 47507989082112, 47507989086207,
+ERASE, 47507988762624, 47507988762624,
+STORE, 47507988762624, 47507988766719,
+STORE, 47507988766720, 47507988770815,
+ERASE, 47507986493440, 47507986493440,
+STORE, 47507986493440, 47507986513919,
+STORE, 47507986513920, 47507986518015,
+ERASE, 47507986161664, 47507986161664,
+STORE, 47507986161664, 47507986165759,
+STORE, 47507986165760, 47507986169855,
+ERASE, 47507986116608, 47507986116608,
+STORE, 47507986116608, 47507986120703,
+STORE, 47507986120704, 47507986124799,
+ERASE, 94386579570688, 94386579570688,
+STORE, 94386579570688, 94386579693567,
+STORE, 94386579693568, 94386579697663,
+ERASE, 140124810997760, 140124810997760,
+STORE, 140124810997760, 140124811001855,
+STORE, 140124811001856, 140124811005951,
+ERASE, 47507984158720, 47507984158720,
+STORE, 94386583982080, 94386584117247,
+STORE, 94386583982080, 94386584256511,
+ERASE, 94386583982080, 94386583982080,
+STORE, 94386583982080, 94386584223743,
+STORE, 94386584223744, 94386584256511,
+ERASE, 94386584223744, 94386584223744,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733763395584, 140737488351231,
+ERASE, 140733763395584, 140733763395584,
+STORE, 140733763395584, 140733763399679,
+STORE, 94011546472448, 94011547152383,
+ERASE, 94011546472448, 94011546472448,
+STORE, 94011546472448, 94011546537983,
+STORE, 94011546537984, 94011547152383,
+ERASE, 94011546537984, 94011546537984,
+STORE, 94011546537984, 94011546886143,
+STORE, 94011546886144, 94011547025407,
+STORE, 94011547025408, 94011547152383,
+STORE, 139757597949952, 139757598121983,
+ERASE, 139757597949952, 139757597949952,
+STORE, 139757597949952, 139757597954047,
+STORE, 139757597954048, 139757598121983,
+ERASE, 139757597954048, 139757597954048,
+STORE, 139757597954048, 139757598076927,
+STORE, 139757598076928, 139757598109695,
+STORE, 139757598109696, 139757598117887,
+STORE, 139757598117888, 139757598121983,
+STORE, 140733763596288, 140733763600383,
+STORE, 140733763584000, 140733763596287,
+STORE, 47875197046784, 47875197054975,
+STORE, 47875197054976, 47875197063167,
+STORE, 47875197063168, 47875198902271,
+STORE, 47875197202432, 47875198902271,
+STORE, 47875197063168, 47875197202431,
+ERASE, 47875197202432, 47875197202432,
+STORE, 47875197202432, 47875198861311,
+STORE, 47875198861312, 47875198902271,
+STORE, 47875198545920, 47875198861311,
+STORE, 47875197202432, 47875198545919,
+ERASE, 47875197202432, 47875197202432,
+STORE, 47875197202432, 47875198545919,
+STORE, 47875198857216, 47875198861311,
+STORE, 47875198545920, 47875198857215,
+ERASE, 47875198545920, 47875198545920,
+STORE, 47875198545920, 47875198857215,
+STORE, 47875198885888, 47875198902271,
+STORE, 47875198861312, 47875198885887,
+ERASE, 47875198861312, 47875198861312,
+STORE, 47875198861312, 47875198885887,
+ERASE, 47875198885888, 47875198885888,
+STORE, 47875198885888, 47875198902271,
+STORE, 47875198902272, 47875199012863,
+STORE, 47875198918656, 47875199012863,
+STORE, 47875198902272, 47875198918655,
+ERASE, 47875198918656, 47875198918656,
+STORE, 47875198918656, 47875199004671,
+STORE, 47875199004672, 47875199012863,
+STORE, 47875198980096, 47875199004671,
+STORE, 47875198918656, 47875198980095,
+ERASE, 47875198918656, 47875198918656,
+STORE, 47875198918656, 47875198980095,
+STORE, 47875199000576, 47875199004671,
+STORE, 47875198980096, 47875199000575,
+ERASE, 47875198980096, 47875198980096,
+STORE, 47875198980096, 47875199000575,
+ERASE, 47875199004672, 47875199004672,
+STORE, 47875199004672, 47875199012863,
+STORE, 47875199012864, 47875199057919,
+ERASE, 47875199012864, 47875199012864,
+STORE, 47875199012864, 47875199021055,
+STORE, 47875199021056, 47875199057919,
+STORE, 47875199041536, 47875199057919,
+STORE, 47875199021056, 47875199041535,
+ERASE, 47875199021056, 47875199021056,
+STORE, 47875199021056, 47875199041535,
+STORE, 47875199049728, 47875199057919,
+STORE, 47875199041536, 47875199049727,
+ERASE, 47875199041536, 47875199041536,
+STORE, 47875199041536, 47875199057919,
+ERASE, 47875199041536, 47875199041536,
+STORE, 47875199041536, 47875199049727,
+STORE, 47875199049728, 47875199057919,
+ERASE, 47875199049728, 47875199049728,
+STORE, 47875199049728, 47875199057919,
+STORE, 47875199057920, 47875199406079,
+STORE, 47875199098880, 47875199406079,
+STORE, 47875199057920, 47875199098879,
+ERASE, 47875199098880, 47875199098880,
+STORE, 47875199098880, 47875199381503,
+STORE, 47875199381504, 47875199406079,
+STORE, 47875199311872, 47875199381503,
+STORE, 47875199098880, 47875199311871,
+ERASE, 47875199098880, 47875199098880,
+STORE, 47875199098880, 47875199311871,
+STORE, 47875199377408, 47875199381503,
+STORE, 47875199311872, 47875199377407,
+ERASE, 47875199311872, 47875199311872,
+STORE, 47875199311872, 47875199377407,
+ERASE, 47875199381504, 47875199381504,
+STORE, 47875199381504, 47875199406079,
+STORE, 47875199406080, 47875201667071,
+STORE, 47875199557632, 47875201667071,
+STORE, 47875199406080, 47875199557631,
+ERASE, 47875199557632, 47875199557632,
+STORE, 47875199557632, 47875201650687,
+STORE, 47875201650688, 47875201667071,
+STORE, 47875201658880, 47875201667071,
+STORE, 47875201650688, 47875201658879,
+ERASE, 47875201650688, 47875201650688,
+STORE, 47875201650688, 47875201658879,
+ERASE, 47875201658880, 47875201658880,
+STORE, 47875201658880, 47875201667071,
+STORE, 47875201667072, 47875201802239,
+ERASE, 47875201667072, 47875201667072,
+STORE, 47875201667072, 47875201691647,
+STORE, 47875201691648, 47875201802239,
+STORE, 47875201753088, 47875201802239,
+STORE, 47875201691648, 47875201753087,
+ERASE, 47875201691648, 47875201691648,
+STORE, 47875201691648, 47875201753087,
+STORE, 47875201777664, 47875201802239,
+STORE, 47875201753088, 47875201777663,
+ERASE, 47875201753088, 47875201753088,
+STORE, 47875201753088, 47875201802239,
+ERASE, 47875201753088, 47875201753088,
+STORE, 47875201753088, 47875201777663,
+STORE, 47875201777664, 47875201802239,
+STORE, 47875201785856, 47875201802239,
+STORE, 47875201777664, 47875201785855,
+ERASE, 47875201777664, 47875201777664,
+STORE, 47875201777664, 47875201785855,
+ERASE, 47875201785856, 47875201785856,
+STORE, 47875201785856, 47875201802239,
+STORE, 47875201785856, 47875201810431,
+STORE, 47875201810432, 47875201974271,
+ERASE, 47875201810432, 47875201810432,
+STORE, 47875201810432, 47875201822719,
+STORE, 47875201822720, 47875201974271,
+STORE, 47875201921024, 47875201974271,
+STORE, 47875201822720, 47875201921023,
+ERASE, 47875201822720, 47875201822720,
+STORE, 47875201822720, 47875201921023,
+STORE, 47875201966080, 47875201974271,
+STORE, 47875201921024, 47875201966079,
+ERASE, 47875201921024, 47875201921024,
+STORE, 47875201921024, 47875201974271,
+ERASE, 47875201921024, 47875201921024,
+STORE, 47875201921024, 47875201966079,
+STORE, 47875201966080, 47875201974271,
+ERASE, 47875201966080, 47875201966080,
+STORE, 47875201966080, 47875201974271,
+STORE, 47875201974272, 47875202572287,
+STORE, 47875202093056, 47875202572287,
+STORE, 47875201974272, 47875202093055,
+ERASE, 47875202093056, 47875202093056,
+STORE, 47875202093056, 47875202519039,
+STORE, 47875202519040, 47875202572287,
+STORE, 47875202408448, 47875202519039,
+STORE, 47875202093056, 47875202408447,
+ERASE, 47875202093056, 47875202093056,
+STORE, 47875202093056, 47875202408447,
+STORE, 47875202514944, 47875202519039,
+STORE, 47875202408448, 47875202514943,
+ERASE, 47875202408448, 47875202408448,
+STORE, 47875202408448, 47875202514943,
+ERASE, 47875202519040, 47875202519040,
+STORE, 47875202519040, 47875202572287,
+STORE, 47875202572288, 47875205623807,
+STORE, 47875203117056, 47875205623807,
+STORE, 47875202572288, 47875203117055,
+ERASE, 47875203117056, 47875203117056,
+STORE, 47875203117056, 47875205402623,
+STORE, 47875205402624, 47875205623807,
+STORE, 47875204812800, 47875205402623,
+STORE, 47875203117056, 47875204812799,
+ERASE, 47875203117056, 47875203117056,
+STORE, 47875203117056, 47875204812799,
+STORE, 47875205398528, 47875205402623,
+STORE, 47875204812800, 47875205398527,
+ERASE, 47875204812800, 47875204812800,
+STORE, 47875204812800, 47875205398527,
+STORE, 47875205607424, 47875205623807,
+STORE, 47875205402624, 47875205607423,
+ERASE, 47875205402624, 47875205402624,
+STORE, 47875205402624, 47875205607423,
+ERASE, 47875205607424, 47875205607424,
+STORE, 47875205607424, 47875205623807,
+STORE, 47875205623808, 47875205656575,
+ERASE, 47875205623808, 47875205623808,
+STORE, 47875205623808, 47875205631999,
+STORE, 47875205632000, 47875205656575,
+STORE, 47875205644288, 47875205656575,
+STORE, 47875205632000, 47875205644287,
+ERASE, 47875205632000, 47875205632000,
+STORE, 47875205632000, 47875205644287,
+STORE, 47875205648384, 47875205656575,
+STORE, 47875205644288, 47875205648383,
+ERASE, 47875205644288, 47875205644288,
+STORE, 47875205644288, 47875205656575,
+ERASE, 47875205644288, 47875205644288,
+STORE, 47875205644288, 47875205648383,
+STORE, 47875205648384, 47875205656575,
+ERASE, 47875205648384, 47875205648384,
+STORE, 47875205648384, 47875205656575,
+STORE, 47875205656576, 47875205693439,
+ERASE, 47875205656576, 47875205656576,
+STORE, 47875205656576, 47875205664767,
+STORE, 47875205664768, 47875205693439,
+STORE, 47875205681152, 47875205693439,
+STORE, 47875205664768, 47875205681151,
+ERASE, 47875205664768, 47875205664768,
+STORE, 47875205664768, 47875205681151,
+STORE, 47875205685248, 47875205693439,
+STORE, 47875205681152, 47875205685247,
+ERASE, 47875205681152, 47875205681152,
+STORE, 47875205681152, 47875205693439,
+ERASE, 47875205681152, 47875205681152,
+STORE, 47875205681152, 47875205685247,
+STORE, 47875205685248, 47875205693439,
+ERASE, 47875205685248, 47875205685248,
+STORE, 47875205685248, 47875205693439,
+STORE, 47875205693440, 47875206168575,
+ERASE, 47875205693440, 47875205693440,
+STORE, 47875205693440, 47875205701631,
+STORE, 47875205701632, 47875206168575,
+STORE, 47875206037504, 47875206168575,
+STORE, 47875205701632, 47875206037503,
+ERASE, 47875205701632, 47875205701632,
+STORE, 47875205701632, 47875206037503,
+STORE, 47875206160384, 47875206168575,
+STORE, 47875206037504, 47875206160383,
+ERASE, 47875206037504, 47875206037504,
+STORE, 47875206037504, 47875206168575,
+ERASE, 47875206037504, 47875206037504,
+STORE, 47875206037504, 47875206160383,
+STORE, 47875206160384, 47875206168575,
+ERASE, 47875206160384, 47875206160384,
+STORE, 47875206160384, 47875206168575,
+STORE, 47875206168576, 47875206176767,
+STORE, 47875206176768, 47875206197247,
+ERASE, 47875206176768, 47875206176768,
+STORE, 47875206176768, 47875206180863,
+STORE, 47875206180864, 47875206197247,
+STORE, 47875206184960, 47875206197247,
+STORE, 47875206180864, 47875206184959,
+ERASE, 47875206180864, 47875206180864,
+STORE, 47875206180864, 47875206184959,
+STORE, 47875206189056, 47875206197247,
+STORE, 47875206184960, 47875206189055,
+ERASE, 47875206184960, 47875206184960,
+STORE, 47875206184960, 47875206197247,
+ERASE, 47875206184960, 47875206184960,
+STORE, 47875206184960, 47875206189055,
+STORE, 47875206189056, 47875206197247,
+ERASE, 47875206189056, 47875206189056,
+STORE, 47875206189056, 47875206197247,
+STORE, 47875206197248, 47875206205439,
+ERASE, 47875198861312, 47875198861312,
+STORE, 47875198861312, 47875198877695,
+STORE, 47875198877696, 47875198885887,
+ERASE, 47875206189056, 47875206189056,
+STORE, 47875206189056, 47875206193151,
+STORE, 47875206193152, 47875206197247,
+ERASE, 47875201777664, 47875201777664,
+STORE, 47875201777664, 47875201781759,
+STORE, 47875201781760, 47875201785855,
+ERASE, 47875206160384, 47875206160384,
+STORE, 47875206160384, 47875206164479,
+STORE, 47875206164480, 47875206168575,
+ERASE, 47875205685248, 47875205685248,
+STORE, 47875205685248, 47875205689343,
+STORE, 47875205689344, 47875205693439,
+ERASE, 47875205648384, 47875205648384,
+STORE, 47875205648384, 47875205652479,
+STORE, 47875205652480, 47875205656575,
+ERASE, 47875205402624, 47875205402624,
+STORE, 47875205402624, 47875205599231,
+STORE, 47875205599232, 47875205607423,
+ERASE, 47875202519040, 47875202519040,
+STORE, 47875202519040, 47875202555903,
+STORE, 47875202555904, 47875202572287,
+ERASE, 47875201966080, 47875201966080,
+STORE, 47875201966080, 47875201970175,
+STORE, 47875201970176, 47875201974271,
+ERASE, 47875201650688, 47875201650688,
+STORE, 47875201650688, 47875201654783,
+STORE, 47875201654784, 47875201658879,
+ERASE, 47875199381504, 47875199381504,
+STORE, 47875199381504, 47875199401983,
+STORE, 47875199401984, 47875199406079,
+ERASE, 47875199049728, 47875199049728,
+STORE, 47875199049728, 47875199053823,
+STORE, 47875199053824, 47875199057919,
+ERASE, 47875199004672, 47875199004672,
+STORE, 47875199004672, 47875199008767,
+STORE, 47875199008768, 47875199012863,
+ERASE, 94011547025408, 94011547025408,
+STORE, 94011547025408, 94011547148287,
+STORE, 94011547148288, 94011547152383,
+ERASE, 139757598109696, 139757598109696,
+STORE, 139757598109696, 139757598113791,
+STORE, 139757598113792, 139757598117887,
+ERASE, 47875197046784, 47875197046784,
+STORE, 94011557584896, 94011557720063,
+STORE, 94011557584896, 94011557855231,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557851135,
+STORE, 94011557851136, 94011557855231,
+ERASE, 94011557851136, 94011557851136,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557847039,
+STORE, 94011557847040, 94011557851135,
+ERASE, 94011557847040, 94011557847040,
+STORE, 94011557584896, 94011557982207,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557978111,
+STORE, 94011557978112, 94011557982207,
+ERASE, 94011557978112, 94011557978112,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557974015,
+STORE, 94011557974016, 94011557978111,
+ERASE, 94011557974016, 94011557974016,
+STORE, 140737488347136, 140737488351231,
+STORE, 140734130360320, 140737488351231,
+ERASE, 140734130360320, 140734130360320,
+STORE, 140734130360320, 140734130364415,
+STORE, 94641232105472, 94641232785407,
+ERASE, 94641232105472, 94641232105472,
+STORE, 94641232105472, 94641232171007,
+STORE, 94641232171008, 94641232785407,
+ERASE, 94641232171008, 94641232171008,
+STORE, 94641232171008, 94641232519167,
+STORE, 94641232519168, 94641232658431,
+STORE, 94641232658432, 94641232785407,
+STORE, 139726599516160, 139726599688191,
+ERASE, 139726599516160, 139726599516160,
+STORE, 139726599516160, 139726599520255,
+STORE, 139726599520256, 139726599688191,
+ERASE, 139726599520256, 139726599520256,
+STORE, 139726599520256, 139726599643135,
+STORE, 139726599643136, 139726599675903,
+STORE, 139726599675904, 139726599684095,
+STORE, 139726599684096, 139726599688191,
+STORE, 140734130446336, 140734130450431,
+STORE, 140734130434048, 140734130446335,
+STORE, 47906195480576, 47906195488767,
+STORE, 47906195488768, 47906195496959,
+STORE, 47906195496960, 47906197336063,
+STORE, 47906195636224, 47906197336063,
+STORE, 47906195496960, 47906195636223,
+ERASE, 47906195636224, 47906195636224,
+STORE, 47906195636224, 47906197295103,
+STORE, 47906197295104, 47906197336063,
+STORE, 47906196979712, 47906197295103,
+STORE, 47906195636224, 47906196979711,
+ERASE, 47906195636224, 47906195636224,
+STORE, 47906195636224, 47906196979711,
+STORE, 47906197291008, 47906197295103,
+STORE, 47906196979712, 47906197291007,
+ERASE, 47906196979712, 47906196979712,
+STORE, 47906196979712, 47906197291007,
+STORE, 47906197319680, 47906197336063,
+STORE, 47906197295104, 47906197319679,
+ERASE, 47906197295104, 47906197295104,
+STORE, 47906197295104, 47906197319679,
+ERASE, 47906197319680, 47906197319680,
+STORE, 47906197319680, 47906197336063,
+STORE, 47906197336064, 47906197446655,
+STORE, 47906197352448, 47906197446655,
+STORE, 47906197336064, 47906197352447,
+ERASE, 47906197352448, 47906197352448,
+STORE, 47906197352448, 47906197438463,
+STORE, 47906197438464, 47906197446655,
+STORE, 47906197413888, 47906197438463,
+STORE, 47906197352448, 47906197413887,
+ERASE, 47906197352448, 47906197352448,
+STORE, 47906197352448, 47906197413887,
+STORE, 47906197434368, 47906197438463,
+STORE, 47906197413888, 47906197434367,
+ERASE, 47906197413888, 47906197413888,
+STORE, 47906197413888, 47906197434367,
+ERASE, 47906197438464, 47906197438464,
+STORE, 47906197438464, 47906197446655,
+STORE, 47906197446656, 47906197491711,
+ERASE, 47906197446656, 47906197446656,
+STORE, 47906197446656, 47906197454847,
+STORE, 47906197454848, 47906197491711,
+STORE, 47906197475328, 47906197491711,
+STORE, 47906197454848, 47906197475327,
+ERASE, 47906197454848, 47906197454848,
+STORE, 47906197454848, 47906197475327,
+STORE, 47906197483520, 47906197491711,
+STORE, 47906197475328, 47906197483519,
+ERASE, 47906197475328, 47906197475328,
+STORE, 47906197475328, 47906197491711,
+ERASE, 47906197475328, 47906197475328,
+STORE, 47906197475328, 47906197483519,
+STORE, 47906197483520, 47906197491711,
+ERASE, 47906197483520, 47906197483520,
+STORE, 47906197483520, 47906197491711,
+STORE, 47906197491712, 47906197839871,
+STORE, 47906197532672, 47906197839871,
+STORE, 47906197491712, 47906197532671,
+ERASE, 47906197532672, 47906197532672,
+STORE, 47906197532672, 47906197815295,
+STORE, 47906197815296, 47906197839871,
+STORE, 47906197745664, 47906197815295,
+STORE, 47906197532672, 47906197745663,
+ERASE, 47906197532672, 47906197532672,
+STORE, 47906197532672, 47906197745663,
+STORE, 47906197811200, 47906197815295,
+STORE, 47906197745664, 47906197811199,
+ERASE, 47906197745664, 47906197745664,
+STORE, 47906197745664, 47906197811199,
+ERASE, 47906197815296, 47906197815296,
+STORE, 47906197815296, 47906197839871,
+STORE, 47906197839872, 47906200100863,
+STORE, 47906197991424, 47906200100863,
+STORE, 47906197839872, 47906197991423,
+ERASE, 47906197991424, 47906197991424,
+STORE, 47906197991424, 47906200084479,
+STORE, 47906200084480, 47906200100863,
+STORE, 47906200092672, 47906200100863,
+STORE, 47906200084480, 47906200092671,
+ERASE, 47906200084480, 47906200084480,
+STORE, 47906200084480, 47906200092671,
+ERASE, 47906200092672, 47906200092672,
+STORE, 47906200092672, 47906200100863,
+STORE, 47906200100864, 47906200236031,
+ERASE, 47906200100864, 47906200100864,
+STORE, 47906200100864, 47906200125439,
+STORE, 47906200125440, 47906200236031,
+STORE, 47906200186880, 47906200236031,
+STORE, 47906200125440, 47906200186879,
+ERASE, 47906200125440, 47906200125440,
+STORE, 47906200125440, 47906200186879,
+STORE, 47906200211456, 47906200236031,
+STORE, 47906200186880, 47906200211455,
+ERASE, 47906200186880, 47906200186880,
+STORE, 47906200186880, 47906200236031,
+ERASE, 47906200186880, 47906200186880,
+STORE, 47906200186880, 47906200211455,
+STORE, 47906200211456, 47906200236031,
+STORE, 47906200219648, 47906200236031,
+STORE, 47906200211456, 47906200219647,
+ERASE, 47906200211456, 47906200211456,
+STORE, 47906200211456, 47906200219647,
+ERASE, 47906200219648, 47906200219648,
+STORE, 47906200219648, 47906200236031,
+STORE, 47906200219648, 47906200244223,
+STORE, 47906200244224, 47906200408063,
+ERASE, 47906200244224, 47906200244224,
+STORE, 47906200244224, 47906200256511,
+STORE, 47906200256512, 47906200408063,
+STORE, 47906200354816, 47906200408063,
+STORE, 47906200256512, 47906200354815,
+ERASE, 47906200256512, 47906200256512,
+STORE, 47906200256512, 47906200354815,
+STORE, 47906200399872, 47906200408063,
+STORE, 47906200354816, 47906200399871,
+ERASE, 47906200354816, 47906200354816,
+STORE, 47906200354816, 47906200408063,
+ERASE, 47906200354816, 47906200354816,
+STORE, 47906200354816, 47906200399871,
+STORE, 47906200399872, 47906200408063,
+ERASE, 47906200399872, 47906200399872,
+STORE, 47906200399872, 47906200408063,
+STORE, 47906200408064, 47906201006079,
+STORE, 47906200526848, 47906201006079,
+STORE, 47906200408064, 47906200526847,
+ERASE, 47906200526848, 47906200526848,
+STORE, 47906200526848, 47906200952831,
+STORE, 47906200952832, 47906201006079,
+STORE, 47906200842240, 47906200952831,
+STORE, 47906200526848, 47906200842239,
+ERASE, 47906200526848, 47906200526848,
+STORE, 47906200526848, 47906200842239,
+STORE, 47906200948736, 47906200952831,
+STORE, 47906200842240, 47906200948735,
+ERASE, 47906200842240, 47906200842240,
+STORE, 47906200842240, 47906200948735,
+ERASE, 47906200952832, 47906200952832,
+STORE, 47906200952832, 47906201006079,
+STORE, 47906201006080, 47906204057599,
+STORE, 47906201550848, 47906204057599,
+STORE, 47906201006080, 47906201550847,
+ERASE, 47906201550848, 47906201550848,
+STORE, 47906201550848, 47906203836415,
+STORE, 47906203836416, 47906204057599,
+STORE, 47906203246592, 47906203836415,
+STORE, 47906201550848, 47906203246591,
+ERASE, 47906201550848, 47906201550848,
+STORE, 47906201550848, 47906203246591,
+STORE, 47906203832320, 47906203836415,
+STORE, 47906203246592, 47906203832319,
+ERASE, 47906203246592, 47906203246592,
+STORE, 47906203246592, 47906203832319,
+STORE, 47906204041216, 47906204057599,
+STORE, 47906203836416, 47906204041215,
+ERASE, 47906203836416, 47906203836416,
+STORE, 47906203836416, 47906204041215,
+ERASE, 47906204041216, 47906204041216,
+STORE, 47906204041216, 47906204057599,
+STORE, 47906204057600, 47906204090367,
+ERASE, 47906204057600, 47906204057600,
+STORE, 47906204057600, 47906204065791,
+STORE, 47906204065792, 47906204090367,
+STORE, 47906204078080, 47906204090367,
+STORE, 47906204065792, 47906204078079,
+ERASE, 47906204065792, 47906204065792,
+STORE, 47906204065792, 47906204078079,
+STORE, 47906204082176, 47906204090367,
+STORE, 47906204078080, 47906204082175,
+ERASE, 47906204078080, 47906204078080,
+STORE, 47906204078080, 47906204090367,
+ERASE, 47906204078080, 47906204078080,
+STORE, 47906204078080, 47906204082175,
+STORE, 47906204082176, 47906204090367,
+ERASE, 47906204082176, 47906204082176,
+STORE, 47906204082176, 47906204090367,
+STORE, 47906204090368, 47906204127231,
+ERASE, 47906204090368, 47906204090368,
+STORE, 47906204090368, 47906204098559,
+STORE, 47906204098560, 47906204127231,
+STORE, 47906204114944, 47906204127231,
+STORE, 47906204098560, 47906204114943,
+ERASE, 47906204098560, 47906204098560,
+STORE, 47906204098560, 47906204114943,
+STORE, 47906204119040, 47906204127231,
+STORE, 47906204114944, 47906204119039,
+ERASE, 47906204114944, 47906204114944,
+STORE, 47906204114944, 47906204127231,
+ERASE, 47906204114944, 47906204114944,
+STORE, 47906204114944, 47906204119039,
+STORE, 47906204119040, 47906204127231,
+ERASE, 47906204119040, 47906204119040,
+STORE, 47906204119040, 47906204127231,
+STORE, 47906204127232, 47906204602367,
+ERASE, 47906204127232, 47906204127232,
+STORE, 47906204127232, 47906204135423,
+STORE, 47906204135424, 47906204602367,
+STORE, 47906204471296, 47906204602367,
+STORE, 47906204135424, 47906204471295,
+ERASE, 47906204135424, 47906204135424,
+STORE, 47906204135424, 47906204471295,
+STORE, 47906204594176, 47906204602367,
+STORE, 47906204471296, 47906204594175,
+ERASE, 47906204471296, 47906204471296,
+STORE, 47906204471296, 47906204602367,
+ERASE, 47906204471296, 47906204471296,
+STORE, 47906204471296, 47906204594175,
+STORE, 47906204594176, 47906204602367,
+ERASE, 47906204594176, 47906204594176,
+STORE, 47906204594176, 47906204602367,
+STORE, 47906204602368, 47906204610559,
+STORE, 47906204610560, 47906204631039,
+ERASE, 47906204610560, 47906204610560,
+STORE, 47906204610560, 47906204614655,
+STORE, 47906204614656, 47906204631039,
+STORE, 47906204618752, 47906204631039,
+STORE, 47906204614656, 47906204618751,
+ERASE, 47906204614656, 47906204614656,
+STORE, 47906204614656, 47906204618751,
+STORE, 47906204622848, 47906204631039,
+STORE, 47906204618752, 47906204622847,
+ERASE, 47906204618752, 47906204618752,
+STORE, 47906204618752, 47906204631039,
+ERASE, 47906204618752, 47906204618752,
+STORE, 47906204618752, 47906204622847,
+STORE, 47906204622848, 47906204631039,
+ERASE, 47906204622848, 47906204622848,
+STORE, 47906204622848, 47906204631039,
+STORE, 47906204631040, 47906204639231,
+ERASE, 47906197295104, 47906197295104,
+STORE, 47906197295104, 47906197311487,
+STORE, 47906197311488, 47906197319679,
+ERASE, 47906204622848, 47906204622848,
+STORE, 47906204622848, 47906204626943,
+STORE, 47906204626944, 47906204631039,
+ERASE, 47906200211456, 47906200211456,
+STORE, 47906200211456, 47906200215551,
+STORE, 47906200215552, 47906200219647,
+ERASE, 47906204594176, 47906204594176,
+STORE, 47906204594176, 47906204598271,
+STORE, 47906204598272, 47906204602367,
+ERASE, 47906204119040, 47906204119040,
+STORE, 47906204119040, 47906204123135,
+STORE, 47906204123136, 47906204127231,
+ERASE, 47906204082176, 47906204082176,
+STORE, 47906204082176, 47906204086271,
+STORE, 47906204086272, 47906204090367,
+ERASE, 47906203836416, 47906203836416,
+STORE, 47906203836416, 47906204033023,
+STORE, 47906204033024, 47906204041215,
+ERASE, 47906200952832, 47906200952832,
+STORE, 47906200952832, 47906200989695,
+STORE, 47906200989696, 47906201006079,
+ERASE, 47906200399872, 47906200399872,
+STORE, 47906200399872, 47906200403967,
+STORE, 47906200403968, 47906200408063,
+ERASE, 47906200084480, 47906200084480,
+STORE, 47906200084480, 47906200088575,
+STORE, 47906200088576, 47906200092671,
+ERASE, 47906197815296, 47906197815296,
+STORE, 47906197815296, 47906197835775,
+STORE, 47906197835776, 47906197839871,
+ERASE, 47906197483520, 47906197483520,
+STORE, 47906197483520, 47906197487615,
+STORE, 47906197487616, 47906197491711,
+ERASE, 47906197438464, 47906197438464,
+STORE, 47906197438464, 47906197442559,
+STORE, 47906197442560, 47906197446655,
+ERASE, 94641232658432, 94641232658432,
+STORE, 94641232658432, 94641232781311,
+STORE, 94641232781312, 94641232785407,
+ERASE, 139726599675904, 139726599675904,
+STORE, 139726599675904, 139726599679999,
+STORE, 139726599680000, 139726599684095,
+ERASE, 47906195480576, 47906195480576,
+STORE, 94641242615808, 94641242750975,
+ };
+
+ unsigned long set10[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140736427839488, 140737488351231,
+ERASE, 140736427839488, 140736427839488,
+STORE, 140736427839488, 140736427843583,
+STORE, 94071213395968, 94071213567999,
+ERASE, 94071213395968, 94071213395968,
+STORE, 94071213395968, 94071213412351,
+STORE, 94071213412352, 94071213567999,
+ERASE, 94071213412352, 94071213412352,
+STORE, 94071213412352, 94071213514751,
+STORE, 94071213514752, 94071213555711,
+STORE, 94071213555712, 94071213567999,
+STORE, 139968410644480, 139968410816511,
+ERASE, 139968410644480, 139968410644480,
+STORE, 139968410644480, 139968410648575,
+STORE, 139968410648576, 139968410816511,
+ERASE, 139968410648576, 139968410648576,
+STORE, 139968410648576, 139968410771455,
+STORE, 139968410771456, 139968410804223,
+STORE, 139968410804224, 139968410812415,
+STORE, 139968410812416, 139968410816511,
+STORE, 140736429277184, 140736429281279,
+STORE, 140736429264896, 140736429277183,
+STORE, 47664384352256, 47664384360447,
+STORE, 47664384360448, 47664384368639,
+STORE, 47664384368640, 47664384532479,
+ERASE, 47664384368640, 47664384368640,
+STORE, 47664384368640, 47664384380927,
+STORE, 47664384380928, 47664384532479,
+STORE, 47664384479232, 47664384532479,
+STORE, 47664384380928, 47664384479231,
+ERASE, 47664384380928, 47664384380928,
+STORE, 47664384380928, 47664384479231,
+STORE, 47664384524288, 47664384532479,
+STORE, 47664384479232, 47664384524287,
+ERASE, 47664384479232, 47664384479232,
+STORE, 47664384479232, 47664384532479,
+ERASE, 47664384479232, 47664384479232,
+STORE, 47664384479232, 47664384524287,
+STORE, 47664384524288, 47664384532479,
+ERASE, 47664384524288, 47664384524288,
+STORE, 47664384524288, 47664384532479,
+STORE, 47664384532480, 47664387583999,
+STORE, 47664385077248, 47664387583999,
+STORE, 47664384532480, 47664385077247,
+ERASE, 47664385077248, 47664385077248,
+STORE, 47664385077248, 47664387362815,
+STORE, 47664387362816, 47664387583999,
+STORE, 47664386772992, 47664387362815,
+STORE, 47664385077248, 47664386772991,
+ERASE, 47664385077248, 47664385077248,
+STORE, 47664385077248, 47664386772991,
+STORE, 47664387358720, 47664387362815,
+STORE, 47664386772992, 47664387358719,
+ERASE, 47664386772992, 47664386772992,
+STORE, 47664386772992, 47664387358719,
+STORE, 47664387567616, 47664387583999,
+STORE, 47664387362816, 47664387567615,
+ERASE, 47664387362816, 47664387362816,
+STORE, 47664387362816, 47664387567615,
+ERASE, 47664387567616, 47664387567616,
+STORE, 47664387567616, 47664387583999,
+STORE, 47664387584000, 47664389423103,
+STORE, 47664387723264, 47664389423103,
+STORE, 47664387584000, 47664387723263,
+ERASE, 47664387723264, 47664387723264,
+STORE, 47664387723264, 47664389382143,
+STORE, 47664389382144, 47664389423103,
+STORE, 47664389066752, 47664389382143,
+STORE, 47664387723264, 47664389066751,
+ERASE, 47664387723264, 47664387723264,
+STORE, 47664387723264, 47664389066751,
+STORE, 47664389378048, 47664389382143,
+STORE, 47664389066752, 47664389378047,
+ERASE, 47664389066752, 47664389066752,
+STORE, 47664389066752, 47664389378047,
+STORE, 47664389406720, 47664389423103,
+STORE, 47664389382144, 47664389406719,
+ERASE, 47664389382144, 47664389382144,
+STORE, 47664389382144, 47664389406719,
+ERASE, 47664389406720, 47664389406720,
+STORE, 47664389406720, 47664389423103,
+STORE, 47664389423104, 47664389558271,
+ERASE, 47664389423104, 47664389423104,
+STORE, 47664389423104, 47664389447679,
+STORE, 47664389447680, 47664389558271,
+STORE, 47664389509120, 47664389558271,
+STORE, 47664389447680, 47664389509119,
+ERASE, 47664389447680, 47664389447680,
+STORE, 47664389447680, 47664389509119,
+STORE, 47664389533696, 47664389558271,
+STORE, 47664389509120, 47664389533695,
+ERASE, 47664389509120, 47664389509120,
+STORE, 47664389509120, 47664389558271,
+ERASE, 47664389509120, 47664389509120,
+STORE, 47664389509120, 47664389533695,
+STORE, 47664389533696, 47664389558271,
+STORE, 47664389541888, 47664389558271,
+STORE, 47664389533696, 47664389541887,
+ERASE, 47664389533696, 47664389533696,
+STORE, 47664389533696, 47664389541887,
+ERASE, 47664389541888, 47664389541888,
+STORE, 47664389541888, 47664389558271,
+STORE, 47664389558272, 47664389578751,
+ERASE, 47664389558272, 47664389558272,
+STORE, 47664389558272, 47664389562367,
+STORE, 47664389562368, 47664389578751,
+STORE, 47664389566464, 47664389578751,
+STORE, 47664389562368, 47664389566463,
+ERASE, 47664389562368, 47664389562368,
+STORE, 47664389562368, 47664389566463,
+STORE, 47664389570560, 47664389578751,
+STORE, 47664389566464, 47664389570559,
+ERASE, 47664389566464, 47664389566464,
+STORE, 47664389566464, 47664389578751,
+ERASE, 47664389566464, 47664389566464,
+STORE, 47664389566464, 47664389570559,
+STORE, 47664389570560, 47664389578751,
+ERASE, 47664389570560, 47664389570560,
+STORE, 47664389570560, 47664389578751,
+STORE, 47664389578752, 47664389586943,
+ERASE, 47664389382144, 47664389382144,
+STORE, 47664389382144, 47664389398527,
+STORE, 47664389398528, 47664389406719,
+ERASE, 47664389570560, 47664389570560,
+STORE, 47664389570560, 47664389574655,
+STORE, 47664389574656, 47664389578751,
+ERASE, 47664389533696, 47664389533696,
+STORE, 47664389533696, 47664389537791,
+STORE, 47664389537792, 47664389541887,
+ERASE, 47664387362816, 47664387362816,
+STORE, 47664387362816, 47664387559423,
+STORE, 47664387559424, 47664387567615,
+ERASE, 47664384524288, 47664384524288,
+STORE, 47664384524288, 47664384528383,
+STORE, 47664384528384, 47664384532479,
+ERASE, 94071213555712, 94071213555712,
+STORE, 94071213555712, 94071213563903,
+STORE, 94071213563904, 94071213567999,
+ERASE, 139968410804224, 139968410804224,
+STORE, 139968410804224, 139968410808319,
+STORE, 139968410808320, 139968410812415,
+ERASE, 47664384352256, 47664384352256,
+STORE, 94071244402688, 94071244537855,
+STORE, 140737488347136, 140737488351231,
+STORE, 140728271503360, 140737488351231,
+ERASE, 140728271503360, 140728271503360,
+STORE, 140728271503360, 140728271507455,
+STORE, 94410361982976, 94410362155007,
+ERASE, 94410361982976, 94410361982976,
+STORE, 94410361982976, 94410361999359,
+STORE, 94410361999360, 94410362155007,
+ERASE, 94410361999360, 94410361999360,
+STORE, 94410361999360, 94410362101759,
+STORE, 94410362101760, 94410362142719,
+STORE, 94410362142720, 94410362155007,
+STORE, 140351953997824, 140351954169855,
+ERASE, 140351953997824, 140351953997824,
+STORE, 140351953997824, 140351954001919,
+STORE, 140351954001920, 140351954169855,
+ERASE, 140351954001920, 140351954001920,
+STORE, 140351954001920, 140351954124799,
+STORE, 140351954124800, 140351954157567,
+STORE, 140351954157568, 140351954165759,
+STORE, 140351954165760, 140351954169855,
+STORE, 140728272429056, 140728272433151,
+STORE, 140728272416768, 140728272429055,
+STORE, 47280840998912, 47280841007103,
+STORE, 47280841007104, 47280841015295,
+STORE, 47280841015296, 47280841179135,
+ERASE, 47280841015296, 47280841015296,
+STORE, 47280841015296, 47280841027583,
+STORE, 47280841027584, 47280841179135,
+STORE, 47280841125888, 47280841179135,
+STORE, 47280841027584, 47280841125887,
+ERASE, 47280841027584, 47280841027584,
+STORE, 47280841027584, 47280841125887,
+STORE, 47280841170944, 47280841179135,
+STORE, 47280841125888, 47280841170943,
+ERASE, 47280841125888, 47280841125888,
+STORE, 47280841125888, 47280841179135,
+ERASE, 47280841125888, 47280841125888,
+STORE, 47280841125888, 47280841170943,
+STORE, 47280841170944, 47280841179135,
+ERASE, 47280841170944, 47280841170944,
+STORE, 47280841170944, 47280841179135,
+STORE, 47280841179136, 47280844230655,
+STORE, 47280841723904, 47280844230655,
+STORE, 47280841179136, 47280841723903,
+ERASE, 47280841723904, 47280841723904,
+STORE, 47280841723904, 47280844009471,
+STORE, 47280844009472, 47280844230655,
+STORE, 47280843419648, 47280844009471,
+STORE, 47280841723904, 47280843419647,
+ERASE, 47280841723904, 47280841723904,
+STORE, 47280841723904, 47280843419647,
+STORE, 47280844005376, 47280844009471,
+STORE, 47280843419648, 47280844005375,
+ERASE, 47280843419648, 47280843419648,
+STORE, 47280843419648, 47280844005375,
+STORE, 47280844214272, 47280844230655,
+STORE, 47280844009472, 47280844214271,
+ERASE, 47280844009472, 47280844009472,
+STORE, 47280844009472, 47280844214271,
+ERASE, 47280844214272, 47280844214272,
+STORE, 47280844214272, 47280844230655,
+STORE, 47280844230656, 47280846069759,
+STORE, 47280844369920, 47280846069759,
+STORE, 47280844230656, 47280844369919,
+ERASE, 47280844369920, 47280844369920,
+STORE, 47280844369920, 47280846028799,
+STORE, 47280846028800, 47280846069759,
+STORE, 47280845713408, 47280846028799,
+STORE, 47280844369920, 47280845713407,
+ERASE, 47280844369920, 47280844369920,
+STORE, 47280844369920, 47280845713407,
+STORE, 47280846024704, 47280846028799,
+STORE, 47280845713408, 47280846024703,
+ERASE, 47280845713408, 47280845713408,
+STORE, 47280845713408, 47280846024703,
+STORE, 47280846053376, 47280846069759,
+STORE, 47280846028800, 47280846053375,
+ERASE, 47280846028800, 47280846028800,
+STORE, 47280846028800, 47280846053375,
+ERASE, 47280846053376, 47280846053376,
+STORE, 47280846053376, 47280846069759,
+STORE, 47280846069760, 47280846204927,
+ERASE, 47280846069760, 47280846069760,
+STORE, 47280846069760, 47280846094335,
+STORE, 47280846094336, 47280846204927,
+STORE, 47280846155776, 47280846204927,
+STORE, 47280846094336, 47280846155775,
+ERASE, 47280846094336, 47280846094336,
+STORE, 47280846094336, 47280846155775,
+STORE, 47280846180352, 47280846204927,
+STORE, 47280846155776, 47280846180351,
+ERASE, 47280846155776, 47280846155776,
+STORE, 47280846155776, 47280846204927,
+ERASE, 47280846155776, 47280846155776,
+STORE, 47280846155776, 47280846180351,
+STORE, 47280846180352, 47280846204927,
+STORE, 47280846188544, 47280846204927,
+STORE, 47280846180352, 47280846188543,
+ERASE, 47280846180352, 47280846180352,
+STORE, 47280846180352, 47280846188543,
+ERASE, 47280846188544, 47280846188544,
+STORE, 47280846188544, 47280846204927,
+STORE, 47280846204928, 47280846225407,
+ERASE, 47280846204928, 47280846204928,
+STORE, 47280846204928, 47280846209023,
+STORE, 47280846209024, 47280846225407,
+STORE, 47280846213120, 47280846225407,
+STORE, 47280846209024, 47280846213119,
+ERASE, 47280846209024, 47280846209024,
+STORE, 47280846209024, 47280846213119,
+STORE, 47280846217216, 47280846225407,
+STORE, 47280846213120, 47280846217215,
+ERASE, 47280846213120, 47280846213120,
+STORE, 47280846213120, 47280846225407,
+ERASE, 47280846213120, 47280846213120,
+STORE, 47280846213120, 47280846217215,
+STORE, 47280846217216, 47280846225407,
+ERASE, 47280846217216, 47280846217216,
+STORE, 47280846217216, 47280846225407,
+STORE, 47280846225408, 47280846233599,
+ERASE, 47280846028800, 47280846028800,
+STORE, 47280846028800, 47280846045183,
+STORE, 47280846045184, 47280846053375,
+ERASE, 47280846217216, 47280846217216,
+STORE, 47280846217216, 47280846221311,
+STORE, 47280846221312, 47280846225407,
+ERASE, 47280846180352, 47280846180352,
+STORE, 47280846180352, 47280846184447,
+STORE, 47280846184448, 47280846188543,
+ERASE, 47280844009472, 47280844009472,
+STORE, 47280844009472, 47280844206079,
+STORE, 47280844206080, 47280844214271,
+ERASE, 47280841170944, 47280841170944,
+STORE, 47280841170944, 47280841175039,
+STORE, 47280841175040, 47280841179135,
+ERASE, 94410362142720, 94410362142720,
+STORE, 94410362142720, 94410362150911,
+STORE, 94410362150912, 94410362155007,
+ERASE, 140351954157568, 140351954157568,
+STORE, 140351954157568, 140351954161663,
+STORE, 140351954161664, 140351954165759,
+ERASE, 47280840998912, 47280840998912,
+STORE, 94410379456512, 94410379591679,
+STORE, 140737488347136, 140737488351231,
+STORE, 140732946362368, 140737488351231,
+ERASE, 140732946362368, 140732946362368,
+STORE, 140732946362368, 140732946366463,
+STORE, 94352937934848, 94352938106879,
+ERASE, 94352937934848, 94352937934848,
+STORE, 94352937934848, 94352937951231,
+STORE, 94352937951232, 94352938106879,
+ERASE, 94352937951232, 94352937951232,
+STORE, 94352937951232, 94352938053631,
+STORE, 94352938053632, 94352938094591,
+STORE, 94352938094592, 94352938106879,
+STORE, 140595518742528, 140595518914559,
+ERASE, 140595518742528, 140595518742528,
+STORE, 140595518742528, 140595518746623,
+STORE, 140595518746624, 140595518914559,
+ERASE, 140595518746624, 140595518746624,
+STORE, 140595518746624, 140595518869503,
+STORE, 140595518869504, 140595518902271,
+STORE, 140595518902272, 140595518910463,
+STORE, 140595518910464, 140595518914559,
+STORE, 140732947468288, 140732947472383,
+STORE, 140732947456000, 140732947468287,
+STORE, 47037276254208, 47037276262399,
+STORE, 47037276262400, 47037276270591,
+STORE, 47037276270592, 47037276434431,
+ERASE, 47037276270592, 47037276270592,
+STORE, 47037276270592, 47037276282879,
+STORE, 47037276282880, 47037276434431,
+STORE, 47037276381184, 47037276434431,
+STORE, 47037276282880, 47037276381183,
+ERASE, 47037276282880, 47037276282880,
+STORE, 47037276282880, 47037276381183,
+STORE, 47037276426240, 47037276434431,
+STORE, 47037276381184, 47037276426239,
+ERASE, 47037276381184, 47037276381184,
+STORE, 47037276381184, 47037276434431,
+ERASE, 47037276381184, 47037276381184,
+STORE, 47037276381184, 47037276426239,
+STORE, 47037276426240, 47037276434431,
+ERASE, 47037276426240, 47037276426240,
+STORE, 47037276426240, 47037276434431,
+STORE, 47037276434432, 47037279485951,
+STORE, 47037276979200, 47037279485951,
+STORE, 47037276434432, 47037276979199,
+ERASE, 47037276979200, 47037276979200,
+STORE, 47037276979200, 47037279264767,
+STORE, 47037279264768, 47037279485951,
+STORE, 47037278674944, 47037279264767,
+STORE, 47037276979200, 47037278674943,
+ERASE, 47037276979200, 47037276979200,
+STORE, 47037276979200, 47037278674943,
+STORE, 47037279260672, 47037279264767,
+STORE, 47037278674944, 47037279260671,
+ERASE, 47037278674944, 47037278674944,
+STORE, 47037278674944, 47037279260671,
+STORE, 47037279469568, 47037279485951,
+STORE, 47037279264768, 47037279469567,
+ERASE, 47037279264768, 47037279264768,
+STORE, 47037279264768, 47037279469567,
+ERASE, 47037279469568, 47037279469568,
+STORE, 47037279469568, 47037279485951,
+STORE, 47037279485952, 47037281325055,
+STORE, 47037279625216, 47037281325055,
+STORE, 47037279485952, 47037279625215,
+ERASE, 47037279625216, 47037279625216,
+STORE, 47037279625216, 47037281284095,
+STORE, 47037281284096, 47037281325055,
+STORE, 47037280968704, 47037281284095,
+STORE, 47037279625216, 47037280968703,
+ERASE, 47037279625216, 47037279625216,
+STORE, 47037279625216, 47037280968703,
+STORE, 47037281280000, 47037281284095,
+STORE, 47037280968704, 47037281279999,
+ERASE, 47037280968704, 47037280968704,
+STORE, 47037280968704, 47037281279999,
+STORE, 47037281308672, 47037281325055,
+STORE, 47037281284096, 47037281308671,
+ERASE, 47037281284096, 47037281284096,
+STORE, 47037281284096, 47037281308671,
+ERASE, 47037281308672, 47037281308672,
+STORE, 47037281308672, 47037281325055,
+STORE, 47037281325056, 47037281460223,
+ERASE, 47037281325056, 47037281325056,
+STORE, 47037281325056, 47037281349631,
+STORE, 47037281349632, 47037281460223,
+STORE, 47037281411072, 47037281460223,
+STORE, 47037281349632, 47037281411071,
+ERASE, 47037281349632, 47037281349632,
+STORE, 47037281349632, 47037281411071,
+STORE, 47037281435648, 47037281460223,
+STORE, 47037281411072, 47037281435647,
+ERASE, 47037281411072, 47037281411072,
+STORE, 47037281411072, 47037281460223,
+ERASE, 47037281411072, 47037281411072,
+STORE, 47037281411072, 47037281435647,
+STORE, 47037281435648, 47037281460223,
+STORE, 47037281443840, 47037281460223,
+STORE, 47037281435648, 47037281443839,
+ERASE, 47037281435648, 47037281435648,
+STORE, 47037281435648, 47037281443839,
+ERASE, 47037281443840, 47037281443840,
+STORE, 47037281443840, 47037281460223,
+STORE, 47037281460224, 47037281480703,
+ERASE, 47037281460224, 47037281460224,
+STORE, 47037281460224, 47037281464319,
+STORE, 47037281464320, 47037281480703,
+STORE, 47037281468416, 47037281480703,
+STORE, 47037281464320, 47037281468415,
+ERASE, 47037281464320, 47037281464320,
+STORE, 47037281464320, 47037281468415,
+STORE, 47037281472512, 47037281480703,
+STORE, 47037281468416, 47037281472511,
+ERASE, 47037281468416, 47037281468416,
+STORE, 47037281468416, 47037281480703,
+ERASE, 47037281468416, 47037281468416,
+STORE, 47037281468416, 47037281472511,
+STORE, 47037281472512, 47037281480703,
+ERASE, 47037281472512, 47037281472512,
+STORE, 47037281472512, 47037281480703,
+STORE, 47037281480704, 47037281488895,
+ERASE, 47037281284096, 47037281284096,
+STORE, 47037281284096, 47037281300479,
+STORE, 47037281300480, 47037281308671,
+ERASE, 47037281472512, 47037281472512,
+STORE, 47037281472512, 47037281476607,
+STORE, 47037281476608, 47037281480703,
+ERASE, 47037281435648, 47037281435648,
+STORE, 47037281435648, 47037281439743,
+STORE, 47037281439744, 47037281443839,
+ERASE, 47037279264768, 47037279264768,
+STORE, 47037279264768, 47037279461375,
+STORE, 47037279461376, 47037279469567,
+ERASE, 47037276426240, 47037276426240,
+STORE, 47037276426240, 47037276430335,
+STORE, 47037276430336, 47037276434431,
+ERASE, 94352938094592, 94352938094592,
+STORE, 94352938094592, 94352938102783,
+STORE, 94352938102784, 94352938106879,
+ERASE, 140595518902272, 140595518902272,
+STORE, 140595518902272, 140595518906367,
+STORE, 140595518906368, 140595518910463,
+ERASE, 47037276254208, 47037276254208,
+STORE, 94352938438656, 94352938573823,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733506027520, 140737488351231,
+ERASE, 140733506027520, 140733506027520,
+STORE, 140733506027520, 140733506031615,
+STORE, 94150123073536, 94150123245567,
+ERASE, 94150123073536, 94150123073536,
+STORE, 94150123073536, 94150123089919,
+STORE, 94150123089920, 94150123245567,
+ERASE, 94150123089920, 94150123089920,
+STORE, 94150123089920, 94150123192319,
+STORE, 94150123192320, 94150123233279,
+STORE, 94150123233280, 94150123245567,
+STORE, 140081290375168, 140081290547199,
+ERASE, 140081290375168, 140081290375168,
+STORE, 140081290375168, 140081290379263,
+STORE, 140081290379264, 140081290547199,
+ERASE, 140081290379264, 140081290379264,
+STORE, 140081290379264, 140081290502143,
+STORE, 140081290502144, 140081290534911,
+STORE, 140081290534912, 140081290543103,
+STORE, 140081290543104, 140081290547199,
+STORE, 140733506707456, 140733506711551,
+STORE, 140733506695168, 140733506707455,
+STORE, 47551504621568, 47551504629759,
+STORE, 47551504629760, 47551504637951,
+STORE, 47551504637952, 47551504801791,
+ERASE, 47551504637952, 47551504637952,
+STORE, 47551504637952, 47551504650239,
+STORE, 47551504650240, 47551504801791,
+STORE, 47551504748544, 47551504801791,
+STORE, 47551504650240, 47551504748543,
+ERASE, 47551504650240, 47551504650240,
+STORE, 47551504650240, 47551504748543,
+STORE, 47551504793600, 47551504801791,
+STORE, 47551504748544, 47551504793599,
+ERASE, 47551504748544, 47551504748544,
+STORE, 47551504748544, 47551504801791,
+ERASE, 47551504748544, 47551504748544,
+STORE, 47551504748544, 47551504793599,
+STORE, 47551504793600, 47551504801791,
+ERASE, 47551504793600, 47551504793600,
+STORE, 47551504793600, 47551504801791,
+STORE, 47551504801792, 47551507853311,
+STORE, 47551505346560, 47551507853311,
+STORE, 47551504801792, 47551505346559,
+ERASE, 47551505346560, 47551505346560,
+STORE, 47551505346560, 47551507632127,
+STORE, 47551507632128, 47551507853311,
+STORE, 47551507042304, 47551507632127,
+STORE, 47551505346560, 47551507042303,
+ERASE, 47551505346560, 47551505346560,
+STORE, 47551505346560, 47551507042303,
+STORE, 47551507628032, 47551507632127,
+STORE, 47551507042304, 47551507628031,
+ERASE, 47551507042304, 47551507042304,
+STORE, 47551507042304, 47551507628031,
+STORE, 47551507836928, 47551507853311,
+STORE, 47551507632128, 47551507836927,
+ERASE, 47551507632128, 47551507632128,
+STORE, 47551507632128, 47551507836927,
+ERASE, 47551507836928, 47551507836928,
+STORE, 47551507836928, 47551507853311,
+STORE, 47551507853312, 47551509692415,
+STORE, 47551507992576, 47551509692415,
+STORE, 47551507853312, 47551507992575,
+ERASE, 47551507992576, 47551507992576,
+STORE, 47551507992576, 47551509651455,
+STORE, 47551509651456, 47551509692415,
+STORE, 47551509336064, 47551509651455,
+STORE, 47551507992576, 47551509336063,
+ERASE, 47551507992576, 47551507992576,
+STORE, 47551507992576, 47551509336063,
+STORE, 47551509647360, 47551509651455,
+STORE, 47551509336064, 47551509647359,
+ERASE, 47551509336064, 47551509336064,
+STORE, 47551509336064, 47551509647359,
+STORE, 47551509676032, 47551509692415,
+STORE, 47551509651456, 47551509676031,
+ERASE, 47551509651456, 47551509651456,
+STORE, 47551509651456, 47551509676031,
+ERASE, 47551509676032, 47551509676032,
+STORE, 47551509676032, 47551509692415,
+STORE, 47551509692416, 47551509827583,
+ERASE, 47551509692416, 47551509692416,
+STORE, 47551509692416, 47551509716991,
+STORE, 47551509716992, 47551509827583,
+STORE, 47551509778432, 47551509827583,
+STORE, 47551509716992, 47551509778431,
+ERASE, 47551509716992, 47551509716992,
+STORE, 47551509716992, 47551509778431,
+STORE, 47551509803008, 47551509827583,
+STORE, 47551509778432, 47551509803007,
+ERASE, 47551509778432, 47551509778432,
+STORE, 47551509778432, 47551509827583,
+ERASE, 47551509778432, 47551509778432,
+STORE, 47551509778432, 47551509803007,
+STORE, 47551509803008, 47551509827583,
+STORE, 47551509811200, 47551509827583,
+STORE, 47551509803008, 47551509811199,
+ERASE, 47551509803008, 47551509803008,
+STORE, 47551509803008, 47551509811199,
+ERASE, 47551509811200, 47551509811200,
+STORE, 47551509811200, 47551509827583,
+STORE, 47551509827584, 47551509848063,
+ERASE, 47551509827584, 47551509827584,
+STORE, 47551509827584, 47551509831679,
+STORE, 47551509831680, 47551509848063,
+STORE, 47551509835776, 47551509848063,
+STORE, 47551509831680, 47551509835775,
+ERASE, 47551509831680, 47551509831680,
+STORE, 47551509831680, 47551509835775,
+STORE, 47551509839872, 47551509848063,
+STORE, 47551509835776, 47551509839871,
+ERASE, 47551509835776, 47551509835776,
+STORE, 47551509835776, 47551509848063,
+ERASE, 47551509835776, 47551509835776,
+STORE, 47551509835776, 47551509839871,
+STORE, 47551509839872, 47551509848063,
+ERASE, 47551509839872, 47551509839872,
+STORE, 47551509839872, 47551509848063,
+STORE, 47551509848064, 47551509856255,
+ERASE, 47551509651456, 47551509651456,
+STORE, 47551509651456, 47551509667839,
+STORE, 47551509667840, 47551509676031,
+ERASE, 47551509839872, 47551509839872,
+STORE, 47551509839872, 47551509843967,
+STORE, 47551509843968, 47551509848063,
+ERASE, 47551509803008, 47551509803008,
+STORE, 47551509803008, 47551509807103,
+STORE, 47551509807104, 47551509811199,
+ERASE, 47551507632128, 47551507632128,
+STORE, 47551507632128, 47551507828735,
+STORE, 47551507828736, 47551507836927,
+ERASE, 47551504793600, 47551504793600,
+STORE, 47551504793600, 47551504797695,
+STORE, 47551504797696, 47551504801791,
+ERASE, 94150123233280, 94150123233280,
+STORE, 94150123233280, 94150123241471,
+STORE, 94150123241472, 94150123245567,
+ERASE, 140081290534912, 140081290534912,
+STORE, 140081290534912, 140081290539007,
+STORE, 140081290539008, 140081290543103,
+ERASE, 47551504621568, 47551504621568,
+STORE, 94150148112384, 94150148247551,
+STORE, 140737488347136, 140737488351231,
+STORE, 140734389334016, 140737488351231,
+ERASE, 140734389334016, 140734389334016,
+STORE, 140734389334016, 140734389338111,
+STORE, 94844636606464, 94844636778495,
+ERASE, 94844636606464, 94844636606464,
+STORE, 94844636606464, 94844636622847,
+STORE, 94844636622848, 94844636778495,
+ERASE, 94844636622848, 94844636622848,
+STORE, 94844636622848, 94844636725247,
+STORE, 94844636725248, 94844636766207,
+STORE, 94844636766208, 94844636778495,
+STORE, 139922765217792, 139922765389823,
+ERASE, 139922765217792, 139922765217792,
+STORE, 139922765217792, 139922765221887,
+STORE, 139922765221888, 139922765389823,
+ERASE, 139922765221888, 139922765221888,
+STORE, 139922765221888, 139922765344767,
+STORE, 139922765344768, 139922765377535,
+STORE, 139922765377536, 139922765385727,
+STORE, 139922765385728, 139922765389823,
+STORE, 140734389678080, 140734389682175,
+STORE, 140734389665792, 140734389678079,
+STORE, 47710029778944, 47710029787135,
+STORE, 47710029787136, 47710029795327,
+STORE, 47710029795328, 47710029959167,
+ERASE, 47710029795328, 47710029795328,
+STORE, 47710029795328, 47710029807615,
+STORE, 47710029807616, 47710029959167,
+STORE, 47710029905920, 47710029959167,
+STORE, 47710029807616, 47710029905919,
+ERASE, 47710029807616, 47710029807616,
+STORE, 47710029807616, 47710029905919,
+STORE, 47710029950976, 47710029959167,
+STORE, 47710029905920, 47710029950975,
+ERASE, 47710029905920, 47710029905920,
+STORE, 47710029905920, 47710029959167,
+ERASE, 47710029905920, 47710029905920,
+STORE, 47710029905920, 47710029950975,
+STORE, 47710029950976, 47710029959167,
+ERASE, 47710029950976, 47710029950976,
+STORE, 47710029950976, 47710029959167,
+STORE, 47710029959168, 47710033010687,
+STORE, 47710030503936, 47710033010687,
+STORE, 47710029959168, 47710030503935,
+ERASE, 47710030503936, 47710030503936,
+STORE, 47710030503936, 47710032789503,
+STORE, 47710032789504, 47710033010687,
+STORE, 47710032199680, 47710032789503,
+STORE, 47710030503936, 47710032199679,
+ERASE, 47710030503936, 47710030503936,
+STORE, 47710030503936, 47710032199679,
+STORE, 47710032785408, 47710032789503,
+STORE, 47710032199680, 47710032785407,
+ERASE, 47710032199680, 47710032199680,
+STORE, 47710032199680, 47710032785407,
+STORE, 47710032994304, 47710033010687,
+STORE, 47710032789504, 47710032994303,
+ERASE, 47710032789504, 47710032789504,
+STORE, 47710032789504, 47710032994303,
+ERASE, 47710032994304, 47710032994304,
+STORE, 47710032994304, 47710033010687,
+STORE, 47710033010688, 47710034849791,
+STORE, 47710033149952, 47710034849791,
+STORE, 47710033010688, 47710033149951,
+ERASE, 47710033149952, 47710033149952,
+STORE, 47710033149952, 47710034808831,
+STORE, 47710034808832, 47710034849791,
+STORE, 47710034493440, 47710034808831,
+STORE, 47710033149952, 47710034493439,
+ERASE, 47710033149952, 47710033149952,
+STORE, 47710033149952, 47710034493439,
+STORE, 47710034804736, 47710034808831,
+STORE, 47710034493440, 47710034804735,
+ERASE, 47710034493440, 47710034493440,
+STORE, 47710034493440, 47710034804735,
+STORE, 47710034833408, 47710034849791,
+STORE, 47710034808832, 47710034833407,
+ERASE, 47710034808832, 47710034808832,
+STORE, 47710034808832, 47710034833407,
+ERASE, 47710034833408, 47710034833408,
+STORE, 47710034833408, 47710034849791,
+STORE, 47710034849792, 47710034984959,
+ERASE, 47710034849792, 47710034849792,
+STORE, 47710034849792, 47710034874367,
+STORE, 47710034874368, 47710034984959,
+STORE, 47710034935808, 47710034984959,
+STORE, 47710034874368, 47710034935807,
+ERASE, 47710034874368, 47710034874368,
+STORE, 47710034874368, 47710034935807,
+STORE, 47710034960384, 47710034984959,
+STORE, 47710034935808, 47710034960383,
+ERASE, 47710034935808, 47710034935808,
+STORE, 47710034935808, 47710034984959,
+ERASE, 47710034935808, 47710034935808,
+STORE, 47710034935808, 47710034960383,
+STORE, 47710034960384, 47710034984959,
+STORE, 47710034968576, 47710034984959,
+STORE, 47710034960384, 47710034968575,
+ERASE, 47710034960384, 47710034960384,
+STORE, 47710034960384, 47710034968575,
+ERASE, 47710034968576, 47710034968576,
+STORE, 47710034968576, 47710034984959,
+STORE, 47710034984960, 47710035005439,
+ERASE, 47710034984960, 47710034984960,
+STORE, 47710034984960, 47710034989055,
+STORE, 47710034989056, 47710035005439,
+STORE, 47710034993152, 47710035005439,
+STORE, 47710034989056, 47710034993151,
+ERASE, 47710034989056, 47710034989056,
+STORE, 47710034989056, 47710034993151,
+STORE, 47710034997248, 47710035005439,
+STORE, 47710034993152, 47710034997247,
+ERASE, 47710034993152, 47710034993152,
+STORE, 47710034993152, 47710035005439,
+ERASE, 47710034993152, 47710034993152,
+STORE, 47710034993152, 47710034997247,
+STORE, 47710034997248, 47710035005439,
+ERASE, 47710034997248, 47710034997248,
+STORE, 47710034997248, 47710035005439,
+STORE, 47710035005440, 47710035013631,
+ERASE, 47710034808832, 47710034808832,
+STORE, 47710034808832, 47710034825215,
+STORE, 47710034825216, 47710034833407,
+ERASE, 47710034997248, 47710034997248,
+STORE, 47710034997248, 47710035001343,
+STORE, 47710035001344, 47710035005439,
+ERASE, 47710034960384, 47710034960384,
+STORE, 47710034960384, 47710034964479,
+STORE, 47710034964480, 47710034968575,
+ERASE, 47710032789504, 47710032789504,
+STORE, 47710032789504, 47710032986111,
+STORE, 47710032986112, 47710032994303,
+ERASE, 47710029950976, 47710029950976,
+STORE, 47710029950976, 47710029955071,
+STORE, 47710029955072, 47710029959167,
+ERASE, 94844636766208, 94844636766208,
+STORE, 94844636766208, 94844636774399,
+STORE, 94844636774400, 94844636778495,
+ERASE, 139922765377536, 139922765377536,
+STORE, 139922765377536, 139922765381631,
+STORE, 139922765381632, 139922765385727,
+ERASE, 47710029778944, 47710029778944,
+STORE, 94844641775616, 94844641910783,
+STORE, 140737488347136, 140737488351231,
+STORE, 140732213886976, 140737488351231,
+ERASE, 140732213886976, 140732213886976,
+STORE, 140732213886976, 140732213891071,
+STORE, 94240508887040, 94240509059071,
+ERASE, 94240508887040, 94240508887040,
+STORE, 94240508887040, 94240508903423,
+STORE, 94240508903424, 94240509059071,
+ERASE, 94240508903424, 94240508903424,
+STORE, 94240508903424, 94240509005823,
+STORE, 94240509005824, 94240509046783,
+STORE, 94240509046784, 94240509059071,
+STORE, 140275106516992, 140275106689023,
+ERASE, 140275106516992, 140275106516992,
+STORE, 140275106516992, 140275106521087,
+STORE, 140275106521088, 140275106689023,
+ERASE, 140275106521088, 140275106521088,
+STORE, 140275106521088, 140275106643967,
+STORE, 140275106643968, 140275106676735,
+STORE, 140275106676736, 140275106684927,
+STORE, 140275106684928, 140275106689023,
+STORE, 140732213977088, 140732213981183,
+STORE, 140732213964800, 140732213977087,
+STORE, 47357688479744, 47357688487935,
+STORE, 47357688487936, 47357688496127,
+STORE, 47357688496128, 47357688659967,
+ERASE, 47357688496128, 47357688496128,
+STORE, 47357688496128, 47357688508415,
+STORE, 47357688508416, 47357688659967,
+STORE, 47357688606720, 47357688659967,
+STORE, 47357688508416, 47357688606719,
+ERASE, 47357688508416, 47357688508416,
+STORE, 47357688508416, 47357688606719,
+STORE, 47357688651776, 47357688659967,
+STORE, 47357688606720, 47357688651775,
+ERASE, 47357688606720, 47357688606720,
+STORE, 47357688606720, 47357688659967,
+ERASE, 47357688606720, 47357688606720,
+STORE, 47357688606720, 47357688651775,
+STORE, 47357688651776, 47357688659967,
+ERASE, 47357688651776, 47357688651776,
+STORE, 47357688651776, 47357688659967,
+STORE, 47357688659968, 47357691711487,
+STORE, 47357689204736, 47357691711487,
+STORE, 47357688659968, 47357689204735,
+ERASE, 47357689204736, 47357689204736,
+STORE, 47357689204736, 47357691490303,
+STORE, 47357691490304, 47357691711487,
+STORE, 47357690900480, 47357691490303,
+STORE, 47357689204736, 47357690900479,
+ERASE, 47357689204736, 47357689204736,
+STORE, 47357689204736, 47357690900479,
+STORE, 47357691486208, 47357691490303,
+STORE, 47357690900480, 47357691486207,
+ERASE, 47357690900480, 47357690900480,
+STORE, 47357690900480, 47357691486207,
+STORE, 47357691695104, 47357691711487,
+STORE, 47357691490304, 47357691695103,
+ERASE, 47357691490304, 47357691490304,
+STORE, 47357691490304, 47357691695103,
+ERASE, 47357691695104, 47357691695104,
+STORE, 47357691695104, 47357691711487,
+STORE, 47357691711488, 47357693550591,
+STORE, 47357691850752, 47357693550591,
+STORE, 47357691711488, 47357691850751,
+ERASE, 47357691850752, 47357691850752,
+STORE, 47357691850752, 47357693509631,
+STORE, 47357693509632, 47357693550591,
+STORE, 47357693194240, 47357693509631,
+STORE, 47357691850752, 47357693194239,
+ERASE, 47357691850752, 47357691850752,
+STORE, 47357691850752, 47357693194239,
+STORE, 47357693505536, 47357693509631,
+STORE, 47357693194240, 47357693505535,
+ERASE, 47357693194240, 47357693194240,
+STORE, 47357693194240, 47357693505535,
+STORE, 47357693534208, 47357693550591,
+STORE, 47357693509632, 47357693534207,
+ERASE, 47357693509632, 47357693509632,
+STORE, 47357693509632, 47357693534207,
+ERASE, 47357693534208, 47357693534208,
+STORE, 47357693534208, 47357693550591,
+STORE, 47357693550592, 47357693685759,
+ERASE, 47357693550592, 47357693550592,
+STORE, 47357693550592, 47357693575167,
+STORE, 47357693575168, 47357693685759,
+STORE, 47357693636608, 47357693685759,
+STORE, 47357693575168, 47357693636607,
+ERASE, 47357693575168, 47357693575168,
+STORE, 47357693575168, 47357693636607,
+STORE, 47357693661184, 47357693685759,
+STORE, 47357693636608, 47357693661183,
+ERASE, 47357693636608, 47357693636608,
+STORE, 47357693636608, 47357693685759,
+ERASE, 47357693636608, 47357693636608,
+STORE, 47357693636608, 47357693661183,
+STORE, 47357693661184, 47357693685759,
+STORE, 47357693669376, 47357693685759,
+STORE, 47357693661184, 47357693669375,
+ERASE, 47357693661184, 47357693661184,
+STORE, 47357693661184, 47357693669375,
+ERASE, 47357693669376, 47357693669376,
+STORE, 47357693669376, 47357693685759,
+STORE, 47357693685760, 47357693706239,
+ERASE, 47357693685760, 47357693685760,
+STORE, 47357693685760, 47357693689855,
+STORE, 47357693689856, 47357693706239,
+STORE, 47357693693952, 47357693706239,
+STORE, 47357693689856, 47357693693951,
+ERASE, 47357693689856, 47357693689856,
+STORE, 47357693689856, 47357693693951,
+STORE, 47357693698048, 47357693706239,
+STORE, 47357693693952, 47357693698047,
+ERASE, 47357693693952, 47357693693952,
+STORE, 47357693693952, 47357693706239,
+ERASE, 47357693693952, 47357693693952,
+STORE, 47357693693952, 47357693698047,
+STORE, 47357693698048, 47357693706239,
+ERASE, 47357693698048, 47357693698048,
+STORE, 47357693698048, 47357693706239,
+STORE, 47357693706240, 47357693714431,
+ERASE, 47357693509632, 47357693509632,
+STORE, 47357693509632, 47357693526015,
+STORE, 47357693526016, 47357693534207,
+ERASE, 47357693698048, 47357693698048,
+STORE, 47357693698048, 47357693702143,
+STORE, 47357693702144, 47357693706239,
+ERASE, 47357693661184, 47357693661184,
+STORE, 47357693661184, 47357693665279,
+STORE, 47357693665280, 47357693669375,
+ERASE, 47357691490304, 47357691490304,
+STORE, 47357691490304, 47357691686911,
+STORE, 47357691686912, 47357691695103,
+ERASE, 47357688651776, 47357688651776,
+STORE, 47357688651776, 47357688655871,
+STORE, 47357688655872, 47357688659967,
+ERASE, 94240509046784, 94240509046784,
+STORE, 94240509046784, 94240509054975,
+STORE, 94240509054976, 94240509059071,
+ERASE, 140275106676736, 140275106676736,
+STORE, 140275106676736, 140275106680831,
+STORE, 140275106680832, 140275106684927,
+ERASE, 47357688479744, 47357688479744,
+STORE, 94240518361088, 94240518496255,
+STORE, 140737488347136, 140737488351231,
+STORE, 140732688277504, 140737488351231,
+ERASE, 140732688277504, 140732688277504,
+STORE, 140732688277504, 140732688281599,
+STORE, 94629171351552, 94629172064255,
+ERASE, 94629171351552, 94629171351552,
+STORE, 94629171351552, 94629171400703,
+STORE, 94629171400704, 94629172064255,
+ERASE, 94629171400704, 94629171400704,
+STORE, 94629171400704, 94629171945471,
+STORE, 94629171945472, 94629172043775,
+STORE, 94629172043776, 94629172064255,
+STORE, 139770707644416, 139770707816447,
+ERASE, 139770707644416, 139770707644416,
+STORE, 139770707644416, 139770707648511,
+STORE, 139770707648512, 139770707816447,
+ERASE, 139770707648512, 139770707648512,
+STORE, 139770707648512, 139770707771391,
+STORE, 139770707771392, 139770707804159,
+STORE, 139770707804160, 139770707812351,
+STORE, 139770707812352, 139770707816447,
+STORE, 140732689121280, 140732689125375,
+STORE, 140732689108992, 140732689121279,
+STORE, 47862087352320, 47862087360511,
+STORE, 47862087360512, 47862087368703,
+STORE, 47862087368704, 47862087475199,
+STORE, 47862087385088, 47862087475199,
+STORE, 47862087368704, 47862087385087,
+ERASE, 47862087385088, 47862087385088,
+STORE, 47862087385088, 47862087458815,
+STORE, 47862087458816, 47862087475199,
+STORE, 47862087438336, 47862087458815,
+STORE, 47862087385088, 47862087438335,
+ERASE, 47862087385088, 47862087385088,
+STORE, 47862087385088, 47862087438335,
+STORE, 47862087454720, 47862087458815,
+STORE, 47862087438336, 47862087454719,
+ERASE, 47862087438336, 47862087438336,
+STORE, 47862087438336, 47862087454719,
+STORE, 47862087467008, 47862087475199,
+STORE, 47862087458816, 47862087467007,
+ERASE, 47862087458816, 47862087458816,
+STORE, 47862087458816, 47862087467007,
+ERASE, 47862087467008, 47862087467008,
+STORE, 47862087467008, 47862087475199,
+STORE, 47862087475200, 47862089314303,
+STORE, 47862087614464, 47862089314303,
+STORE, 47862087475200, 47862087614463,
+ERASE, 47862087614464, 47862087614464,
+STORE, 47862087614464, 47862089273343,
+STORE, 47862089273344, 47862089314303,
+STORE, 47862088957952, 47862089273343,
+STORE, 47862087614464, 47862088957951,
+ERASE, 47862087614464, 47862087614464,
+STORE, 47862087614464, 47862088957951,
+STORE, 47862089269248, 47862089273343,
+STORE, 47862088957952, 47862089269247,
+ERASE, 47862088957952, 47862088957952,
+STORE, 47862088957952, 47862089269247,
+STORE, 47862089297920, 47862089314303,
+STORE, 47862089273344, 47862089297919,
+ERASE, 47862089273344, 47862089273344,
+STORE, 47862089273344, 47862089297919,
+ERASE, 47862089297920, 47862089297920,
+STORE, 47862089297920, 47862089314303,
+STORE, 47862089297920, 47862089326591,
+ERASE, 47862089273344, 47862089273344,
+STORE, 47862089273344, 47862089289727,
+STORE, 47862089289728, 47862089297919,
+ERASE, 47862087458816, 47862087458816,
+STORE, 47862087458816, 47862087462911,
+STORE, 47862087462912, 47862087467007,
+ERASE, 94629172043776, 94629172043776,
+STORE, 94629172043776, 94629172060159,
+STORE, 94629172060160, 94629172064255,
+ERASE, 139770707804160, 139770707804160,
+STORE, 139770707804160, 139770707808255,
+STORE, 139770707808256, 139770707812351,
+ERASE, 47862087352320, 47862087352320,
+STORE, 94629197533184, 94629197668351,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727540711424, 140737488351231,
+ERASE, 140727540711424, 140727540711424,
+STORE, 140727540711424, 140727540715519,
+STORE, 94299865313280, 94299866025983,
+ERASE, 94299865313280, 94299865313280,
+STORE, 94299865313280, 94299865362431,
+STORE, 94299865362432, 94299866025983,
+ERASE, 94299865362432, 94299865362432,
+STORE, 94299865362432, 94299865907199,
+STORE, 94299865907200, 94299866005503,
+STORE, 94299866005504, 94299866025983,
+STORE, 140680268763136, 140680268935167,
+ERASE, 140680268763136, 140680268763136,
+STORE, 140680268763136, 140680268767231,
+STORE, 140680268767232, 140680268935167,
+ERASE, 140680268767232, 140680268767232,
+STORE, 140680268767232, 140680268890111,
+STORE, 140680268890112, 140680268922879,
+STORE, 140680268922880, 140680268931071,
+STORE, 140680268931072, 140680268935167,
+STORE, 140727541424128, 140727541428223,
+STORE, 140727541411840, 140727541424127,
+STORE, 46952526233600, 46952526241791,
+STORE, 46952526241792, 46952526249983,
+STORE, 46952526249984, 46952526356479,
+STORE, 46952526266368, 46952526356479,
+STORE, 46952526249984, 46952526266367,
+ERASE, 46952526266368, 46952526266368,
+STORE, 46952526266368, 46952526340095,
+STORE, 46952526340096, 46952526356479,
+STORE, 46952526319616, 46952526340095,
+STORE, 46952526266368, 46952526319615,
+ERASE, 46952526266368, 46952526266368,
+STORE, 46952526266368, 46952526319615,
+STORE, 46952526336000, 46952526340095,
+STORE, 46952526319616, 46952526335999,
+ERASE, 46952526319616, 46952526319616,
+STORE, 46952526319616, 46952526335999,
+STORE, 46952526348288, 46952526356479,
+STORE, 46952526340096, 46952526348287,
+ERASE, 46952526340096, 46952526340096,
+STORE, 46952526340096, 46952526348287,
+ERASE, 46952526348288, 46952526348288,
+STORE, 46952526348288, 46952526356479,
+STORE, 46952526356480, 46952528195583,
+STORE, 46952526495744, 46952528195583,
+STORE, 46952526356480, 46952526495743,
+ERASE, 46952526495744, 46952526495744,
+STORE, 46952526495744, 46952528154623,
+STORE, 46952528154624, 46952528195583,
+STORE, 46952527839232, 46952528154623,
+STORE, 46952526495744, 46952527839231,
+ERASE, 46952526495744, 46952526495744,
+STORE, 46952526495744, 46952527839231,
+STORE, 46952528150528, 46952528154623,
+STORE, 46952527839232, 46952528150527,
+ERASE, 46952527839232, 46952527839232,
+STORE, 46952527839232, 46952528150527,
+STORE, 46952528179200, 46952528195583,
+STORE, 46952528154624, 46952528179199,
+ERASE, 46952528154624, 46952528154624,
+STORE, 46952528154624, 46952528179199,
+ERASE, 46952528179200, 46952528179200,
+STORE, 46952528179200, 46952528195583,
+STORE, 46952528179200, 46952528207871,
+ERASE, 46952528154624, 46952528154624,
+STORE, 46952528154624, 46952528171007,
+STORE, 46952528171008, 46952528179199,
+ERASE, 46952526340096, 46952526340096,
+STORE, 46952526340096, 46952526344191,
+STORE, 46952526344192, 46952526348287,
+ERASE, 94299866005504, 94299866005504,
+STORE, 94299866005504, 94299866021887,
+STORE, 94299866021888, 94299866025983,
+ERASE, 140680268922880, 140680268922880,
+STORE, 140680268922880, 140680268926975,
+STORE, 140680268926976, 140680268931071,
+ERASE, 46952526233600, 46952526233600,
+STORE, 140737488347136, 140737488351231,
+STORE, 140722874793984, 140737488351231,
+ERASE, 140722874793984, 140722874793984,
+STORE, 140722874793984, 140722874798079,
+STORE, 94448916213760, 94448916926463,
+ERASE, 94448916213760, 94448916213760,
+STORE, 94448916213760, 94448916262911,
+STORE, 94448916262912, 94448916926463,
+ERASE, 94448916262912, 94448916262912,
+STORE, 94448916262912, 94448916807679,
+STORE, 94448916807680, 94448916905983,
+STORE, 94448916905984, 94448916926463,
+STORE, 140389117046784, 140389117218815,
+ERASE, 140389117046784, 140389117046784,
+STORE, 140389117046784, 140389117050879,
+STORE, 140389117050880, 140389117218815,
+ERASE, 140389117050880, 140389117050880,
+STORE, 140389117050880, 140389117173759,
+STORE, 140389117173760, 140389117206527,
+STORE, 140389117206528, 140389117214719,
+STORE, 140389117214720, 140389117218815,
+STORE, 140722875297792, 140722875301887,
+STORE, 140722875285504, 140722875297791,
+STORE, 47243677949952, 47243677958143,
+STORE, 47243677958144, 47243677966335,
+STORE, 47243677966336, 47243678072831,
+STORE, 47243677982720, 47243678072831,
+STORE, 47243677966336, 47243677982719,
+ERASE, 47243677982720, 47243677982720,
+STORE, 47243677982720, 47243678056447,
+STORE, 47243678056448, 47243678072831,
+STORE, 47243678035968, 47243678056447,
+STORE, 47243677982720, 47243678035967,
+ERASE, 47243677982720, 47243677982720,
+STORE, 47243677982720, 47243678035967,
+STORE, 47243678052352, 47243678056447,
+STORE, 47243678035968, 47243678052351,
+ERASE, 47243678035968, 47243678035968,
+STORE, 47243678035968, 47243678052351,
+STORE, 47243678064640, 47243678072831,
+STORE, 47243678056448, 47243678064639,
+ERASE, 47243678056448, 47243678056448,
+STORE, 47243678056448, 47243678064639,
+ERASE, 47243678064640, 47243678064640,
+STORE, 47243678064640, 47243678072831,
+STORE, 47243678072832, 47243679911935,
+STORE, 47243678212096, 47243679911935,
+STORE, 47243678072832, 47243678212095,
+ERASE, 47243678212096, 47243678212096,
+STORE, 47243678212096, 47243679870975,
+STORE, 47243679870976, 47243679911935,
+STORE, 47243679555584, 47243679870975,
+STORE, 47243678212096, 47243679555583,
+ERASE, 47243678212096, 47243678212096,
+STORE, 47243678212096, 47243679555583,
+STORE, 47243679866880, 47243679870975,
+STORE, 47243679555584, 47243679866879,
+ERASE, 47243679555584, 47243679555584,
+STORE, 47243679555584, 47243679866879,
+STORE, 47243679895552, 47243679911935,
+STORE, 47243679870976, 47243679895551,
+ERASE, 47243679870976, 47243679870976,
+STORE, 47243679870976, 47243679895551,
+ERASE, 47243679895552, 47243679895552,
+STORE, 47243679895552, 47243679911935,
+STORE, 47243679895552, 47243679924223,
+ERASE, 47243679870976, 47243679870976,
+STORE, 47243679870976, 47243679887359,
+STORE, 47243679887360, 47243679895551,
+ERASE, 47243678056448, 47243678056448,
+STORE, 47243678056448, 47243678060543,
+STORE, 47243678060544, 47243678064639,
+ERASE, 94448916905984, 94448916905984,
+STORE, 94448916905984, 94448916922367,
+STORE, 94448916922368, 94448916926463,
+ERASE, 140389117206528, 140389117206528,
+STORE, 140389117206528, 140389117210623,
+STORE, 140389117210624, 140389117214719,
+ERASE, 47243677949952, 47243677949952,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733068505088, 140737488351231,
+ERASE, 140733068505088, 140733068505088,
+STORE, 140733068505088, 140733068509183,
+STORE, 94207145750528, 94207146463231,
+ERASE, 94207145750528, 94207145750528,
+STORE, 94207145750528, 94207145799679,
+STORE, 94207145799680, 94207146463231,
+ERASE, 94207145799680, 94207145799680,
+STORE, 94207145799680, 94207146344447,
+STORE, 94207146344448, 94207146442751,
+STORE, 94207146442752, 94207146463231,
+STORE, 140684504911872, 140684505083903,
+ERASE, 140684504911872, 140684504911872,
+STORE, 140684504911872, 140684504915967,
+STORE, 140684504915968, 140684505083903,
+ERASE, 140684504915968, 140684504915968,
+STORE, 140684504915968, 140684505038847,
+STORE, 140684505038848, 140684505071615,
+STORE, 140684505071616, 140684505079807,
+STORE, 140684505079808, 140684505083903,
+STORE, 140733068607488, 140733068611583,
+STORE, 140733068595200, 140733068607487,
+STORE, 46948290084864, 46948290093055,
+STORE, 46948290093056, 46948290101247,
+STORE, 46948290101248, 46948290207743,
+STORE, 46948290117632, 46948290207743,
+STORE, 46948290101248, 46948290117631,
+ERASE, 46948290117632, 46948290117632,
+STORE, 46948290117632, 46948290191359,
+STORE, 46948290191360, 46948290207743,
+STORE, 46948290170880, 46948290191359,
+STORE, 46948290117632, 46948290170879,
+ERASE, 46948290117632, 46948290117632,
+STORE, 46948290117632, 46948290170879,
+STORE, 46948290187264, 46948290191359,
+STORE, 46948290170880, 46948290187263,
+ERASE, 46948290170880, 46948290170880,
+STORE, 46948290170880, 46948290187263,
+STORE, 46948290199552, 46948290207743,
+STORE, 46948290191360, 46948290199551,
+ERASE, 46948290191360, 46948290191360,
+STORE, 46948290191360, 46948290199551,
+ERASE, 46948290199552, 46948290199552,
+STORE, 46948290199552, 46948290207743,
+STORE, 46948290207744, 46948292046847,
+STORE, 46948290347008, 46948292046847,
+STORE, 46948290207744, 46948290347007,
+ERASE, 46948290347008, 46948290347008,
+STORE, 46948290347008, 46948292005887,
+STORE, 46948292005888, 46948292046847,
+STORE, 46948291690496, 46948292005887,
+STORE, 46948290347008, 46948291690495,
+ERASE, 46948290347008, 46948290347008,
+STORE, 46948290347008, 46948291690495,
+STORE, 46948292001792, 46948292005887,
+STORE, 46948291690496, 46948292001791,
+ERASE, 46948291690496, 46948291690496,
+STORE, 46948291690496, 46948292001791,
+STORE, 46948292030464, 46948292046847,
+STORE, 46948292005888, 46948292030463,
+ERASE, 46948292005888, 46948292005888,
+STORE, 46948292005888, 46948292030463,
+ERASE, 46948292030464, 46948292030464,
+STORE, 46948292030464, 46948292046847,
+STORE, 46948292030464, 46948292059135,
+ERASE, 46948292005888, 46948292005888,
+STORE, 46948292005888, 46948292022271,
+STORE, 46948292022272, 46948292030463,
+ERASE, 46948290191360, 46948290191360,
+STORE, 46948290191360, 46948290195455,
+STORE, 46948290195456, 46948290199551,
+ERASE, 94207146442752, 94207146442752,
+STORE, 94207146442752, 94207146459135,
+STORE, 94207146459136, 94207146463231,
+ERASE, 140684505071616, 140684505071616,
+STORE, 140684505071616, 140684505075711,
+STORE, 140684505075712, 140684505079807,
+ERASE, 46948290084864, 46948290084864,
+STORE, 140737488347136, 140737488351231,
+STORE, 140726367158272, 140737488351231,
+ERASE, 140726367158272, 140726367158272,
+STORE, 140726367158272, 140726367162367,
+STORE, 94436124106752, 94436124819455,
+ERASE, 94436124106752, 94436124106752,
+STORE, 94436124106752, 94436124155903,
+STORE, 94436124155904, 94436124819455,
+ERASE, 94436124155904, 94436124155904,
+STORE, 94436124155904, 94436124700671,
+STORE, 94436124700672, 94436124798975,
+STORE, 94436124798976, 94436124819455,
+STORE, 140049025044480, 140049025216511,
+ERASE, 140049025044480, 140049025044480,
+STORE, 140049025044480, 140049025048575,
+STORE, 140049025048576, 140049025216511,
+ERASE, 140049025048576, 140049025048576,
+STORE, 140049025048576, 140049025171455,
+STORE, 140049025171456, 140049025204223,
+STORE, 140049025204224, 140049025212415,
+STORE, 140049025212416, 140049025216511,
+STORE, 140726367256576, 140726367260671,
+STORE, 140726367244288, 140726367256575,
+STORE, 47583769952256, 47583769960447,
+STORE, 47583769960448, 47583769968639,
+STORE, 47583769968640, 47583770075135,
+STORE, 47583769985024, 47583770075135,
+STORE, 47583769968640, 47583769985023,
+ERASE, 47583769985024, 47583769985024,
+STORE, 47583769985024, 47583770058751,
+STORE, 47583770058752, 47583770075135,
+STORE, 47583770038272, 47583770058751,
+STORE, 47583769985024, 47583770038271,
+ERASE, 47583769985024, 47583769985024,
+STORE, 47583769985024, 47583770038271,
+STORE, 47583770054656, 47583770058751,
+STORE, 47583770038272, 47583770054655,
+ERASE, 47583770038272, 47583770038272,
+STORE, 47583770038272, 47583770054655,
+STORE, 47583770066944, 47583770075135,
+STORE, 47583770058752, 47583770066943,
+ERASE, 47583770058752, 47583770058752,
+STORE, 47583770058752, 47583770066943,
+ERASE, 47583770066944, 47583770066944,
+STORE, 47583770066944, 47583770075135,
+STORE, 47583770075136, 47583771914239,
+STORE, 47583770214400, 47583771914239,
+STORE, 47583770075136, 47583770214399,
+ERASE, 47583770214400, 47583770214400,
+STORE, 47583770214400, 47583771873279,
+STORE, 47583771873280, 47583771914239,
+STORE, 47583771557888, 47583771873279,
+STORE, 47583770214400, 47583771557887,
+ERASE, 47583770214400, 47583770214400,
+STORE, 47583770214400, 47583771557887,
+STORE, 47583771869184, 47583771873279,
+STORE, 47583771557888, 47583771869183,
+ERASE, 47583771557888, 47583771557888,
+STORE, 47583771557888, 47583771869183,
+STORE, 47583771897856, 47583771914239,
+STORE, 47583771873280, 47583771897855,
+ERASE, 47583771873280, 47583771873280,
+STORE, 47583771873280, 47583771897855,
+ERASE, 47583771897856, 47583771897856,
+STORE, 47583771897856, 47583771914239,
+STORE, 47583771897856, 47583771926527,
+ERASE, 47583771873280, 47583771873280,
+STORE, 47583771873280, 47583771889663,
+STORE, 47583771889664, 47583771897855,
+ERASE, 47583770058752, 47583770058752,
+STORE, 47583770058752, 47583770062847,
+STORE, 47583770062848, 47583770066943,
+ERASE, 94436124798976, 94436124798976,
+STORE, 94436124798976, 94436124815359,
+STORE, 94436124815360, 94436124819455,
+ERASE, 140049025204224, 140049025204224,
+STORE, 140049025204224, 140049025208319,
+STORE, 140049025208320, 140049025212415,
+ERASE, 47583769952256, 47583769952256,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727116099584, 140737488351231,
+ERASE, 140727116099584, 140727116099584,
+STORE, 140727116099584, 140727116103679,
+STORE, 94166319734784, 94166320447487,
+ERASE, 94166319734784, 94166319734784,
+STORE, 94166319734784, 94166319783935,
+STORE, 94166319783936, 94166320447487,
+ERASE, 94166319783936, 94166319783936,
+STORE, 94166319783936, 94166320328703,
+STORE, 94166320328704, 94166320427007,
+STORE, 94166320427008, 94166320447487,
+STORE, 139976559542272, 139976559714303,
+ERASE, 139976559542272, 139976559542272,
+STORE, 139976559542272, 139976559546367,
+STORE, 139976559546368, 139976559714303,
+ERASE, 139976559546368, 139976559546368,
+STORE, 139976559546368, 139976559669247,
+STORE, 139976559669248, 139976559702015,
+STORE, 139976559702016, 139976559710207,
+STORE, 139976559710208, 139976559714303,
+STORE, 140727116222464, 140727116226559,
+STORE, 140727116210176, 140727116222463,
+STORE, 47656235454464, 47656235462655,
+STORE, 47656235462656, 47656235470847,
+STORE, 47656235470848, 47656235577343,
+STORE, 47656235487232, 47656235577343,
+STORE, 47656235470848, 47656235487231,
+ERASE, 47656235487232, 47656235487232,
+STORE, 47656235487232, 47656235560959,
+STORE, 47656235560960, 47656235577343,
+STORE, 47656235540480, 47656235560959,
+STORE, 47656235487232, 47656235540479,
+ERASE, 47656235487232, 47656235487232,
+STORE, 47656235487232, 47656235540479,
+STORE, 47656235556864, 47656235560959,
+STORE, 47656235540480, 47656235556863,
+ERASE, 47656235540480, 47656235540480,
+STORE, 47656235540480, 47656235556863,
+STORE, 47656235569152, 47656235577343,
+STORE, 47656235560960, 47656235569151,
+ERASE, 47656235560960, 47656235560960,
+STORE, 47656235560960, 47656235569151,
+ERASE, 47656235569152, 47656235569152,
+STORE, 47656235569152, 47656235577343,
+STORE, 47656235577344, 47656237416447,
+STORE, 47656235716608, 47656237416447,
+STORE, 47656235577344, 47656235716607,
+ERASE, 47656235716608, 47656235716608,
+STORE, 47656235716608, 47656237375487,
+STORE, 47656237375488, 47656237416447,
+STORE, 47656237060096, 47656237375487,
+STORE, 47656235716608, 47656237060095,
+ERASE, 47656235716608, 47656235716608,
+STORE, 47656235716608, 47656237060095,
+STORE, 47656237371392, 47656237375487,
+STORE, 47656237060096, 47656237371391,
+ERASE, 47656237060096, 47656237060096,
+STORE, 47656237060096, 47656237371391,
+STORE, 47656237400064, 47656237416447,
+STORE, 47656237375488, 47656237400063,
+ERASE, 47656237375488, 47656237375488,
+STORE, 47656237375488, 47656237400063,
+ERASE, 47656237400064, 47656237400064,
+STORE, 47656237400064, 47656237416447,
+STORE, 47656237400064, 47656237428735,
+ERASE, 47656237375488, 47656237375488,
+STORE, 47656237375488, 47656237391871,
+STORE, 47656237391872, 47656237400063,
+ERASE, 47656235560960, 47656235560960,
+STORE, 47656235560960, 47656235565055,
+STORE, 47656235565056, 47656235569151,
+ERASE, 94166320427008, 94166320427008,
+STORE, 94166320427008, 94166320443391,
+STORE, 94166320443392, 94166320447487,
+ERASE, 139976559702016, 139976559702016,
+STORE, 139976559702016, 139976559706111,
+STORE, 139976559706112, 139976559710207,
+ERASE, 47656235454464, 47656235454464,
+STORE, 94166332153856, 94166332289023,
+STORE, 140737488347136, 140737488351231,
+STORE, 140726412816384, 140737488351231,
+ERASE, 140726412816384, 140726412816384,
+STORE, 140726412816384, 140726412820479,
+STORE, 94094884507648, 94094885220351,
+ERASE, 94094884507648, 94094884507648,
+STORE, 94094884507648, 94094884556799,
+STORE, 94094884556800, 94094885220351,
+ERASE, 94094884556800, 94094884556800,
+STORE, 94094884556800, 94094885101567,
+STORE, 94094885101568, 94094885199871,
+STORE, 94094885199872, 94094885220351,
+STORE, 139773773938688, 139773774110719,
+ERASE, 139773773938688, 139773773938688,
+STORE, 139773773938688, 139773773942783,
+STORE, 139773773942784, 139773774110719,
+ERASE, 139773773942784, 139773773942784,
+STORE, 139773773942784, 139773774065663,
+STORE, 139773774065664, 139773774098431,
+STORE, 139773774098432, 139773774106623,
+STORE, 139773774106624, 139773774110719,
+STORE, 140726412963840, 140726412967935,
+STORE, 140726412951552, 140726412963839,
+STORE, 47859021058048, 47859021066239,
+STORE, 47859021066240, 47859021074431,
+STORE, 47859021074432, 47859021180927,
+STORE, 47859021090816, 47859021180927,
+STORE, 47859021074432, 47859021090815,
+ERASE, 47859021090816, 47859021090816,
+STORE, 47859021090816, 47859021164543,
+STORE, 47859021164544, 47859021180927,
+STORE, 47859021144064, 47859021164543,
+STORE, 47859021090816, 47859021144063,
+ERASE, 47859021090816, 47859021090816,
+STORE, 47859021090816, 47859021144063,
+STORE, 47859021160448, 47859021164543,
+STORE, 47859021144064, 47859021160447,
+ERASE, 47859021144064, 47859021144064,
+STORE, 47859021144064, 47859021160447,
+STORE, 47859021172736, 47859021180927,
+STORE, 47859021164544, 47859021172735,
+ERASE, 47859021164544, 47859021164544,
+STORE, 47859021164544, 47859021172735,
+ERASE, 47859021172736, 47859021172736,
+STORE, 47859021172736, 47859021180927,
+STORE, 47859021180928, 47859023020031,
+STORE, 47859021320192, 47859023020031,
+STORE, 47859021180928, 47859021320191,
+ERASE, 47859021320192, 47859021320192,
+STORE, 47859021320192, 47859022979071,
+STORE, 47859022979072, 47859023020031,
+STORE, 47859022663680, 47859022979071,
+STORE, 47859021320192, 47859022663679,
+ERASE, 47859021320192, 47859021320192,
+STORE, 47859021320192, 47859022663679,
+STORE, 47859022974976, 47859022979071,
+STORE, 47859022663680, 47859022974975,
+ERASE, 47859022663680, 47859022663680,
+STORE, 47859022663680, 47859022974975,
+STORE, 47859023003648, 47859023020031,
+STORE, 47859022979072, 47859023003647,
+ERASE, 47859022979072, 47859022979072,
+STORE, 47859022979072, 47859023003647,
+ERASE, 47859023003648, 47859023003648,
+STORE, 47859023003648, 47859023020031,
+STORE, 47859023003648, 47859023032319,
+ERASE, 47859022979072, 47859022979072,
+STORE, 47859022979072, 47859022995455,
+STORE, 47859022995456, 47859023003647,
+ERASE, 47859021164544, 47859021164544,
+STORE, 47859021164544, 47859021168639,
+STORE, 47859021168640, 47859021172735,
+ERASE, 94094885199872, 94094885199872,
+STORE, 94094885199872, 94094885216255,
+STORE, 94094885216256, 94094885220351,
+ERASE, 139773774098432, 139773774098432,
+STORE, 139773774098432, 139773774102527,
+STORE, 139773774102528, 139773774106623,
+ERASE, 47859021058048, 47859021058048,
+STORE, 94094901108736, 94094901243903,
+STORE, 140737488347136, 140737488351231,
+STORE, 140736567963648, 140737488351231,
+ERASE, 140736567963648, 140736567963648,
+STORE, 140736567963648, 140736567967743,
+STORE, 94924425748480, 94924426461183,
+ERASE, 94924425748480, 94924425748480,
+STORE, 94924425748480, 94924425797631,
+STORE, 94924425797632, 94924426461183,
+ERASE, 94924425797632, 94924425797632,
+STORE, 94924425797632, 94924426342399,
+STORE, 94924426342400, 94924426440703,
+STORE, 94924426440704, 94924426461183,
+STORE, 140042126319616, 140042126491647,
+ERASE, 140042126319616, 140042126319616,
+STORE, 140042126319616, 140042126323711,
+STORE, 140042126323712, 140042126491647,
+ERASE, 140042126323712, 140042126323712,
+STORE, 140042126323712, 140042126446591,
+STORE, 140042126446592, 140042126479359,
+STORE, 140042126479360, 140042126487551,
+STORE, 140042126487552, 140042126491647,
+STORE, 140736568672256, 140736568676351,
+STORE, 140736568659968, 140736568672255,
+STORE, 47590668677120, 47590668685311,
+STORE, 47590668685312, 47590668693503,
+STORE, 47590668693504, 47590668799999,
+STORE, 47590668709888, 47590668799999,
+STORE, 47590668693504, 47590668709887,
+ERASE, 47590668709888, 47590668709888,
+STORE, 47590668709888, 47590668783615,
+STORE, 47590668783616, 47590668799999,
+STORE, 47590668763136, 47590668783615,
+STORE, 47590668709888, 47590668763135,
+ERASE, 47590668709888, 47590668709888,
+STORE, 47590668709888, 47590668763135,
+STORE, 47590668779520, 47590668783615,
+STORE, 47590668763136, 47590668779519,
+ERASE, 47590668763136, 47590668763136,
+STORE, 47590668763136, 47590668779519,
+STORE, 47590668791808, 47590668799999,
+STORE, 47590668783616, 47590668791807,
+ERASE, 47590668783616, 47590668783616,
+STORE, 47590668783616, 47590668791807,
+ERASE, 47590668791808, 47590668791808,
+STORE, 47590668791808, 47590668799999,
+STORE, 47590668800000, 47590670639103,
+STORE, 47590668939264, 47590670639103,
+STORE, 47590668800000, 47590668939263,
+ERASE, 47590668939264, 47590668939264,
+STORE, 47590668939264, 47590670598143,
+STORE, 47590670598144, 47590670639103,
+STORE, 47590670282752, 47590670598143,
+STORE, 47590668939264, 47590670282751,
+ERASE, 47590668939264, 47590668939264,
+STORE, 47590668939264, 47590670282751,
+STORE, 47590670594048, 47590670598143,
+STORE, 47590670282752, 47590670594047,
+ERASE, 47590670282752, 47590670282752,
+STORE, 47590670282752, 47590670594047,
+STORE, 47590670622720, 47590670639103,
+STORE, 47590670598144, 47590670622719,
+ERASE, 47590670598144, 47590670598144,
+STORE, 47590670598144, 47590670622719,
+ERASE, 47590670622720, 47590670622720,
+STORE, 47590670622720, 47590670639103,
+STORE, 47590670622720, 47590670651391,
+ERASE, 47590670598144, 47590670598144,
+STORE, 47590670598144, 47590670614527,
+STORE, 47590670614528, 47590670622719,
+ERASE, 47590668783616, 47590668783616,
+STORE, 47590668783616, 47590668787711,
+STORE, 47590668787712, 47590668791807,
+ERASE, 94924426440704, 94924426440704,
+STORE, 94924426440704, 94924426457087,
+STORE, 94924426457088, 94924426461183,
+ERASE, 140042126479360, 140042126479360,
+STORE, 140042126479360, 140042126483455,
+STORE, 140042126483456, 140042126487551,
+ERASE, 47590668677120, 47590668677120,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733281439744, 140737488351231,
+ERASE, 140733281439744, 140733281439744,
+STORE, 140733281439744, 140733281443839,
+STORE, 94490667069440, 94490667782143,
+ERASE, 94490667069440, 94490667069440,
+STORE, 94490667069440, 94490667118591,
+STORE, 94490667118592, 94490667782143,
+ERASE, 94490667118592, 94490667118592,
+STORE, 94490667118592, 94490667663359,
+STORE, 94490667663360, 94490667761663,
+STORE, 94490667761664, 94490667782143,
+STORE, 139878215118848, 139878215290879,
+ERASE, 139878215118848, 139878215118848,
+STORE, 139878215118848, 139878215122943,
+STORE, 139878215122944, 139878215290879,
+ERASE, 139878215122944, 139878215122944,
+STORE, 139878215122944, 139878215245823,
+STORE, 139878215245824, 139878215278591,
+STORE, 139878215278592, 139878215286783,
+STORE, 139878215286784, 139878215290879,
+STORE, 140733281464320, 140733281468415,
+STORE, 140733281452032, 140733281464319,
+STORE, 47754579877888, 47754579886079,
+STORE, 47754579886080, 47754579894271,
+STORE, 47754579894272, 47754580000767,
+STORE, 47754579910656, 47754580000767,
+STORE, 47754579894272, 47754579910655,
+ERASE, 47754579910656, 47754579910656,
+STORE, 47754579910656, 47754579984383,
+STORE, 47754579984384, 47754580000767,
+STORE, 47754579963904, 47754579984383,
+STORE, 47754579910656, 47754579963903,
+ERASE, 47754579910656, 47754579910656,
+STORE, 47754579910656, 47754579963903,
+STORE, 47754579980288, 47754579984383,
+STORE, 47754579963904, 47754579980287,
+ERASE, 47754579963904, 47754579963904,
+STORE, 47754579963904, 47754579980287,
+STORE, 47754579992576, 47754580000767,
+STORE, 47754579984384, 47754579992575,
+ERASE, 47754579984384, 47754579984384,
+STORE, 47754579984384, 47754579992575,
+ERASE, 47754579992576, 47754579992576,
+STORE, 47754579992576, 47754580000767,
+STORE, 47754580000768, 47754581839871,
+STORE, 47754580140032, 47754581839871,
+STORE, 47754580000768, 47754580140031,
+ERASE, 47754580140032, 47754580140032,
+STORE, 47754580140032, 47754581798911,
+STORE, 47754581798912, 47754581839871,
+STORE, 47754581483520, 47754581798911,
+STORE, 47754580140032, 47754581483519,
+ERASE, 47754580140032, 47754580140032,
+STORE, 47754580140032, 47754581483519,
+STORE, 47754581794816, 47754581798911,
+STORE, 47754581483520, 47754581794815,
+ERASE, 47754581483520, 47754581483520,
+STORE, 47754581483520, 47754581794815,
+STORE, 47754581823488, 47754581839871,
+STORE, 47754581798912, 47754581823487,
+ERASE, 47754581798912, 47754581798912,
+STORE, 47754581798912, 47754581823487,
+ERASE, 47754581823488, 47754581823488,
+STORE, 47754581823488, 47754581839871,
+STORE, 47754581823488, 47754581852159,
+ERASE, 47754581798912, 47754581798912,
+STORE, 47754581798912, 47754581815295,
+STORE, 47754581815296, 47754581823487,
+ERASE, 47754579984384, 47754579984384,
+STORE, 47754579984384, 47754579988479,
+STORE, 47754579988480, 47754579992575,
+ERASE, 94490667761664, 94490667761664,
+STORE, 94490667761664, 94490667778047,
+STORE, 94490667778048, 94490667782143,
+ERASE, 139878215278592, 139878215278592,
+STORE, 139878215278592, 139878215282687,
+STORE, 139878215282688, 139878215286783,
+ERASE, 47754579877888, 47754579877888,
+STORE, 94490669649920, 94490669785087,
+STORE, 140737488347136, 140737488351231,
+STORE, 140735382188032, 140737488351231,
+ERASE, 140735382188032, 140735382188032,
+STORE, 140735382188032, 140735382192127,
+STORE, 94150181302272, 94150182014975,
+ERASE, 94150181302272, 94150181302272,
+STORE, 94150181302272, 94150181351423,
+STORE, 94150181351424, 94150182014975,
+ERASE, 94150181351424, 94150181351424,
+STORE, 94150181351424, 94150181896191,
+STORE, 94150181896192, 94150181994495,
+STORE, 94150181994496, 94150182014975,
+STORE, 139679752458240, 139679752630271,
+ERASE, 139679752458240, 139679752458240,
+STORE, 139679752458240, 139679752462335,
+STORE, 139679752462336, 139679752630271,
+ERASE, 139679752462336, 139679752462336,
+STORE, 139679752462336, 139679752585215,
+STORE, 139679752585216, 139679752617983,
+STORE, 139679752617984, 139679752626175,
+STORE, 139679752626176, 139679752630271,
+STORE, 140735382536192, 140735382540287,
+STORE, 140735382523904, 140735382536191,
+STORE, 47953042538496, 47953042546687,
+STORE, 47953042546688, 47953042554879,
+STORE, 47953042554880, 47953042661375,
+STORE, 47953042571264, 47953042661375,
+STORE, 47953042554880, 47953042571263,
+ERASE, 47953042571264, 47953042571264,
+STORE, 47953042571264, 47953042644991,
+STORE, 47953042644992, 47953042661375,
+STORE, 47953042624512, 47953042644991,
+STORE, 47953042571264, 47953042624511,
+ERASE, 47953042571264, 47953042571264,
+STORE, 47953042571264, 47953042624511,
+STORE, 47953042640896, 47953042644991,
+STORE, 47953042624512, 47953042640895,
+ERASE, 47953042624512, 47953042624512,
+STORE, 47953042624512, 47953042640895,
+STORE, 47953042653184, 47953042661375,
+STORE, 47953042644992, 47953042653183,
+ERASE, 47953042644992, 47953042644992,
+STORE, 47953042644992, 47953042653183,
+ERASE, 47953042653184, 47953042653184,
+STORE, 47953042653184, 47953042661375,
+STORE, 47953042661376, 47953044500479,
+STORE, 47953042800640, 47953044500479,
+STORE, 47953042661376, 47953042800639,
+ERASE, 47953042800640, 47953042800640,
+STORE, 47953042800640, 47953044459519,
+STORE, 47953044459520, 47953044500479,
+STORE, 47953044144128, 47953044459519,
+STORE, 47953042800640, 47953044144127,
+ERASE, 47953042800640, 47953042800640,
+STORE, 47953042800640, 47953044144127,
+STORE, 47953044455424, 47953044459519,
+STORE, 47953044144128, 47953044455423,
+ERASE, 47953044144128, 47953044144128,
+STORE, 47953044144128, 47953044455423,
+STORE, 47953044484096, 47953044500479,
+STORE, 47953044459520, 47953044484095,
+ERASE, 47953044459520, 47953044459520,
+STORE, 47953044459520, 47953044484095,
+ERASE, 47953044484096, 47953044484096,
+STORE, 47953044484096, 47953044500479,
+STORE, 47953044484096, 47953044512767,
+ERASE, 47953044459520, 47953044459520,
+STORE, 47953044459520, 47953044475903,
+STORE, 47953044475904, 47953044484095,
+ERASE, 47953042644992, 47953042644992,
+STORE, 47953042644992, 47953042649087,
+STORE, 47953042649088, 47953042653183,
+ERASE, 94150181994496, 94150181994496,
+STORE, 94150181994496, 94150182010879,
+STORE, 94150182010880, 94150182014975,
+ERASE, 139679752617984, 139679752617984,
+STORE, 139679752617984, 139679752622079,
+STORE, 139679752622080, 139679752626175,
+ERASE, 47953042538496, 47953042538496,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737044123648, 140737488351231,
+ERASE, 140737044123648, 140737044123648,
+STORE, 140737044123648, 140737044127743,
+STORE, 94425324294144, 94425325006847,
+ERASE, 94425324294144, 94425324294144,
+STORE, 94425324294144, 94425324343295,
+STORE, 94425324343296, 94425325006847,
+ERASE, 94425324343296, 94425324343296,
+STORE, 94425324343296, 94425324888063,
+STORE, 94425324888064, 94425324986367,
+STORE, 94425324986368, 94425325006847,
+STORE, 140382015016960, 140382015188991,
+ERASE, 140382015016960, 140382015016960,
+STORE, 140382015016960, 140382015021055,
+STORE, 140382015021056, 140382015188991,
+ERASE, 140382015021056, 140382015021056,
+STORE, 140382015021056, 140382015143935,
+STORE, 140382015143936, 140382015176703,
+STORE, 140382015176704, 140382015184895,
+STORE, 140382015184896, 140382015188991,
+STORE, 140737045585920, 140737045590015,
+STORE, 140737045573632, 140737045585919,
+STORE, 47250779979776, 47250779987967,
+STORE, 47250779987968, 47250779996159,
+STORE, 47250779996160, 47250780102655,
+STORE, 47250780012544, 47250780102655,
+STORE, 47250779996160, 47250780012543,
+ERASE, 47250780012544, 47250780012544,
+STORE, 47250780012544, 47250780086271,
+STORE, 47250780086272, 47250780102655,
+STORE, 47250780065792, 47250780086271,
+STORE, 47250780012544, 47250780065791,
+ERASE, 47250780012544, 47250780012544,
+STORE, 47250780012544, 47250780065791,
+STORE, 47250780082176, 47250780086271,
+STORE, 47250780065792, 47250780082175,
+ERASE, 47250780065792, 47250780065792,
+STORE, 47250780065792, 47250780082175,
+STORE, 47250780094464, 47250780102655,
+STORE, 47250780086272, 47250780094463,
+ERASE, 47250780086272, 47250780086272,
+STORE, 47250780086272, 47250780094463,
+ERASE, 47250780094464, 47250780094464,
+STORE, 47250780094464, 47250780102655,
+STORE, 47250780102656, 47250781941759,
+STORE, 47250780241920, 47250781941759,
+STORE, 47250780102656, 47250780241919,
+ERASE, 47250780241920, 47250780241920,
+STORE, 47250780241920, 47250781900799,
+STORE, 47250781900800, 47250781941759,
+STORE, 47250781585408, 47250781900799,
+STORE, 47250780241920, 47250781585407,
+ERASE, 47250780241920, 47250780241920,
+STORE, 47250780241920, 47250781585407,
+STORE, 47250781896704, 47250781900799,
+STORE, 47250781585408, 47250781896703,
+ERASE, 47250781585408, 47250781585408,
+STORE, 47250781585408, 47250781896703,
+STORE, 47250781925376, 47250781941759,
+STORE, 47250781900800, 47250781925375,
+ERASE, 47250781900800, 47250781900800,
+STORE, 47250781900800, 47250781925375,
+ERASE, 47250781925376, 47250781925376,
+STORE, 47250781925376, 47250781941759,
+STORE, 47250781925376, 47250781954047,
+ERASE, 47250781900800, 47250781900800,
+STORE, 47250781900800, 47250781917183,
+STORE, 47250781917184, 47250781925375,
+ERASE, 47250780086272, 47250780086272,
+STORE, 47250780086272, 47250780090367,
+STORE, 47250780090368, 47250780094463,
+ERASE, 94425324986368, 94425324986368,
+STORE, 94425324986368, 94425325002751,
+STORE, 94425325002752, 94425325006847,
+ERASE, 140382015176704, 140382015176704,
+STORE, 140382015176704, 140382015180799,
+STORE, 140382015180800, 140382015184895,
+ERASE, 47250779979776, 47250779979776,
+STORE, 94425351438336, 94425351573503,
+STORE, 140737488347136, 140737488351231,
+STORE, 140736801144832, 140737488351231,
+ERASE, 140736801144832, 140736801144832,
+STORE, 140736801144832, 140736801148927,
+STORE, 94629429358592, 94629430071295,
+ERASE, 94629429358592, 94629429358592,
+STORE, 94629429358592, 94629429407743,
+STORE, 94629429407744, 94629430071295,
+ERASE, 94629429407744, 94629429407744,
+STORE, 94629429407744, 94629429952511,
+STORE, 94629429952512, 94629430050815,
+STORE, 94629430050816, 94629430071295,
+STORE, 139801685483520, 139801685655551,
+ERASE, 139801685483520, 139801685483520,
+STORE, 139801685483520, 139801685487615,
+STORE, 139801685487616, 139801685655551,
+ERASE, 139801685487616, 139801685487616,
+STORE, 139801685487616, 139801685610495,
+STORE, 139801685610496, 139801685643263,
+STORE, 139801685643264, 139801685651455,
+STORE, 139801685651456, 139801685655551,
+STORE, 140736801198080, 140736801202175,
+STORE, 140736801185792, 140736801198079,
+STORE, 47831109513216, 47831109521407,
+STORE, 47831109521408, 47831109529599,
+STORE, 47831109529600, 47831109636095,
+STORE, 47831109545984, 47831109636095,
+STORE, 47831109529600, 47831109545983,
+ERASE, 47831109545984, 47831109545984,
+STORE, 47831109545984, 47831109619711,
+STORE, 47831109619712, 47831109636095,
+STORE, 47831109599232, 47831109619711,
+STORE, 47831109545984, 47831109599231,
+ERASE, 47831109545984, 47831109545984,
+STORE, 47831109545984, 47831109599231,
+STORE, 47831109615616, 47831109619711,
+STORE, 47831109599232, 47831109615615,
+ERASE, 47831109599232, 47831109599232,
+STORE, 47831109599232, 47831109615615,
+STORE, 47831109627904, 47831109636095,
+STORE, 47831109619712, 47831109627903,
+ERASE, 47831109619712, 47831109619712,
+STORE, 47831109619712, 47831109627903,
+ERASE, 47831109627904, 47831109627904,
+STORE, 47831109627904, 47831109636095,
+STORE, 47831109636096, 47831111475199,
+STORE, 47831109775360, 47831111475199,
+STORE, 47831109636096, 47831109775359,
+ERASE, 47831109775360, 47831109775360,
+STORE, 47831109775360, 47831111434239,
+STORE, 47831111434240, 47831111475199,
+STORE, 47831111118848, 47831111434239,
+STORE, 47831109775360, 47831111118847,
+ERASE, 47831109775360, 47831109775360,
+STORE, 47831109775360, 47831111118847,
+STORE, 47831111430144, 47831111434239,
+STORE, 47831111118848, 47831111430143,
+ERASE, 47831111118848, 47831111118848,
+STORE, 47831111118848, 47831111430143,
+STORE, 47831111458816, 47831111475199,
+STORE, 47831111434240, 47831111458815,
+ERASE, 47831111434240, 47831111434240,
+STORE, 47831111434240, 47831111458815,
+ERASE, 47831111458816, 47831111458816,
+STORE, 47831111458816, 47831111475199,
+STORE, 47831111458816, 47831111487487,
+ERASE, 47831111434240, 47831111434240,
+STORE, 47831111434240, 47831111450623,
+STORE, 47831111450624, 47831111458815,
+ERASE, 47831109619712, 47831109619712,
+STORE, 47831109619712, 47831109623807,
+STORE, 47831109623808, 47831109627903,
+ERASE, 94629430050816, 94629430050816,
+STORE, 94629430050816, 94629430067199,
+STORE, 94629430067200, 94629430071295,
+ERASE, 139801685643264, 139801685643264,
+STORE, 139801685643264, 139801685647359,
+STORE, 139801685647360, 139801685651455,
+ERASE, 47831109513216, 47831109513216,
+STORE, 140737488347136, 140737488351231,
+STORE, 140729419612160, 140737488351231,
+ERASE, 140729419612160, 140729419612160,
+STORE, 140729419612160, 140729419616255,
+STORE, 94443354148864, 94443354861567,
+ERASE, 94443354148864, 94443354148864,
+STORE, 94443354148864, 94443354198015,
+STORE, 94443354198016, 94443354861567,
+ERASE, 94443354198016, 94443354198016,
+STORE, 94443354198016, 94443354742783,
+STORE, 94443354742784, 94443354841087,
+STORE, 94443354841088, 94443354861567,
+STORE, 139741700038656, 139741700210687,
+ERASE, 139741700038656, 139741700038656,
+STORE, 139741700038656, 139741700042751,
+STORE, 139741700042752, 139741700210687,
+ERASE, 139741700042752, 139741700042752,
+STORE, 139741700042752, 139741700165631,
+STORE, 139741700165632, 139741700198399,
+STORE, 139741700198400, 139741700206591,
+STORE, 139741700206592, 139741700210687,
+STORE, 140729420574720, 140729420578815,
+STORE, 140729420562432, 140729420574719,
+STORE, 47891094958080, 47891094966271,
+STORE, 47891094966272, 47891094974463,
+STORE, 47891094974464, 47891095080959,
+STORE, 47891094990848, 47891095080959,
+STORE, 47891094974464, 47891094990847,
+ERASE, 47891094990848, 47891094990848,
+STORE, 47891094990848, 47891095064575,
+STORE, 47891095064576, 47891095080959,
+STORE, 47891095044096, 47891095064575,
+STORE, 47891094990848, 47891095044095,
+ERASE, 47891094990848, 47891094990848,
+STORE, 47891094990848, 47891095044095,
+STORE, 47891095060480, 47891095064575,
+STORE, 47891095044096, 47891095060479,
+ERASE, 47891095044096, 47891095044096,
+STORE, 47891095044096, 47891095060479,
+STORE, 47891095072768, 47891095080959,
+STORE, 47891095064576, 47891095072767,
+ERASE, 47891095064576, 47891095064576,
+STORE, 47891095064576, 47891095072767,
+ERASE, 47891095072768, 47891095072768,
+STORE, 47891095072768, 47891095080959,
+STORE, 47891095080960, 47891096920063,
+STORE, 47891095220224, 47891096920063,
+STORE, 47891095080960, 47891095220223,
+ERASE, 47891095220224, 47891095220224,
+STORE, 47891095220224, 47891096879103,
+STORE, 47891096879104, 47891096920063,
+STORE, 47891096563712, 47891096879103,
+STORE, 47891095220224, 47891096563711,
+ERASE, 47891095220224, 47891095220224,
+STORE, 47891095220224, 47891096563711,
+STORE, 47891096875008, 47891096879103,
+STORE, 47891096563712, 47891096875007,
+ERASE, 47891096563712, 47891096563712,
+STORE, 47891096563712, 47891096875007,
+STORE, 47891096903680, 47891096920063,
+STORE, 47891096879104, 47891096903679,
+ERASE, 47891096879104, 47891096879104,
+STORE, 47891096879104, 47891096903679,
+ERASE, 47891096903680, 47891096903680,
+STORE, 47891096903680, 47891096920063,
+STORE, 47891096903680, 47891096932351,
+ERASE, 47891096879104, 47891096879104,
+STORE, 47891096879104, 47891096895487,
+STORE, 47891096895488, 47891096903679,
+ERASE, 47891095064576, 47891095064576,
+STORE, 47891095064576, 47891095068671,
+STORE, 47891095068672, 47891095072767,
+ERASE, 94443354841088, 94443354841088,
+STORE, 94443354841088, 94443354857471,
+STORE, 94443354857472, 94443354861567,
+ERASE, 139741700198400, 139741700198400,
+STORE, 139741700198400, 139741700202495,
+STORE, 139741700202496, 139741700206591,
+ERASE, 47891094958080, 47891094958080,
+STORE, 94443360825344, 94443360960511,
+STORE, 140737488347136, 140737488351231,
+STORE, 140722961661952, 140737488351231,
+ERASE, 140722961661952, 140722961661952,
+STORE, 140722961661952, 140722961666047,
+STORE, 94878388944896, 94878389657599,
+ERASE, 94878388944896, 94878388944896,
+STORE, 94878388944896, 94878388994047,
+STORE, 94878388994048, 94878389657599,
+ERASE, 94878388994048, 94878388994048,
+STORE, 94878388994048, 94878389538815,
+STORE, 94878389538816, 94878389637119,
+STORE, 94878389637120, 94878389657599,
+STORE, 140210690056192, 140210690228223,
+ERASE, 140210690056192, 140210690056192,
+STORE, 140210690056192, 140210690060287,
+STORE, 140210690060288, 140210690228223,
+ERASE, 140210690060288, 140210690060288,
+STORE, 140210690060288, 140210690183167,
+STORE, 140210690183168, 140210690215935,
+STORE, 140210690215936, 140210690224127,
+STORE, 140210690224128, 140210690228223,
+STORE, 140722963148800, 140722963152895,
+STORE, 140722963136512, 140722963148799,
+STORE, 47422104940544, 47422104948735,
+STORE, 47422104948736, 47422104956927,
+STORE, 47422104956928, 47422105063423,
+STORE, 47422104973312, 47422105063423,
+STORE, 47422104956928, 47422104973311,
+ERASE, 47422104973312, 47422104973312,
+STORE, 47422104973312, 47422105047039,
+STORE, 47422105047040, 47422105063423,
+STORE, 47422105026560, 47422105047039,
+STORE, 47422104973312, 47422105026559,
+ERASE, 47422104973312, 47422104973312,
+STORE, 47422104973312, 47422105026559,
+STORE, 47422105042944, 47422105047039,
+STORE, 47422105026560, 47422105042943,
+ERASE, 47422105026560, 47422105026560,
+STORE, 47422105026560, 47422105042943,
+STORE, 47422105055232, 47422105063423,
+STORE, 47422105047040, 47422105055231,
+ERASE, 47422105047040, 47422105047040,
+STORE, 47422105047040, 47422105055231,
+ERASE, 47422105055232, 47422105055232,
+STORE, 47422105055232, 47422105063423,
+STORE, 47422105063424, 47422106902527,
+STORE, 47422105202688, 47422106902527,
+STORE, 47422105063424, 47422105202687,
+ERASE, 47422105202688, 47422105202688,
+STORE, 47422105202688, 47422106861567,
+STORE, 47422106861568, 47422106902527,
+STORE, 47422106546176, 47422106861567,
+STORE, 47422105202688, 47422106546175,
+ERASE, 47422105202688, 47422105202688,
+STORE, 47422105202688, 47422106546175,
+STORE, 47422106857472, 47422106861567,
+STORE, 47422106546176, 47422106857471,
+ERASE, 47422106546176, 47422106546176,
+STORE, 47422106546176, 47422106857471,
+STORE, 47422106886144, 47422106902527,
+STORE, 47422106861568, 47422106886143,
+ERASE, 47422106861568, 47422106861568,
+STORE, 47422106861568, 47422106886143,
+ERASE, 47422106886144, 47422106886144,
+STORE, 47422106886144, 47422106902527,
+STORE, 47422106886144, 47422106914815,
+ERASE, 47422106861568, 47422106861568,
+STORE, 47422106861568, 47422106877951,
+STORE, 47422106877952, 47422106886143,
+ERASE, 47422105047040, 47422105047040,
+STORE, 47422105047040, 47422105051135,
+STORE, 47422105051136, 47422105055231,
+ERASE, 94878389637120, 94878389637120,
+STORE, 94878389637120, 94878389653503,
+STORE, 94878389653504, 94878389657599,
+ERASE, 140210690215936, 140210690215936,
+STORE, 140210690215936, 140210690220031,
+STORE, 140210690220032, 140210690224127,
+ERASE, 47422104940544, 47422104940544,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727690309632, 140737488351231,
+ERASE, 140727690309632, 140727690309632,
+STORE, 140727690309632, 140727690313727,
+STORE, 94121892208640, 94121892921343,
+ERASE, 94121892208640, 94121892208640,
+STORE, 94121892208640, 94121892257791,
+STORE, 94121892257792, 94121892921343,
+ERASE, 94121892257792, 94121892257792,
+STORE, 94121892257792, 94121892802559,
+STORE, 94121892802560, 94121892900863,
+STORE, 94121892900864, 94121892921343,
+STORE, 140662438326272, 140662438498303,
+ERASE, 140662438326272, 140662438326272,
+STORE, 140662438326272, 140662438330367,
+STORE, 140662438330368, 140662438498303,
+ERASE, 140662438330368, 140662438330368,
+STORE, 140662438330368, 140662438453247,
+STORE, 140662438453248, 140662438486015,
+STORE, 140662438486016, 140662438494207,
+STORE, 140662438494208, 140662438498303,
+STORE, 140727690379264, 140727690383359,
+STORE, 140727690366976, 140727690379263,
+STORE, 46970356670464, 46970356678655,
+STORE, 46970356678656, 46970356686847,
+STORE, 46970356686848, 46970356793343,
+STORE, 46970356703232, 46970356793343,
+STORE, 46970356686848, 46970356703231,
+ERASE, 46970356703232, 46970356703232,
+STORE, 46970356703232, 46970356776959,
+STORE, 46970356776960, 46970356793343,
+STORE, 46970356756480, 46970356776959,
+STORE, 46970356703232, 46970356756479,
+ERASE, 46970356703232, 46970356703232,
+STORE, 46970356703232, 46970356756479,
+STORE, 46970356772864, 46970356776959,
+STORE, 46970356756480, 46970356772863,
+ERASE, 46970356756480, 46970356756480,
+STORE, 46970356756480, 46970356772863,
+STORE, 46970356785152, 46970356793343,
+STORE, 46970356776960, 46970356785151,
+ERASE, 46970356776960, 46970356776960,
+STORE, 46970356776960, 46970356785151,
+ERASE, 46970356785152, 46970356785152,
+STORE, 46970356785152, 46970356793343,
+STORE, 46970356793344, 46970358632447,
+STORE, 46970356932608, 46970358632447,
+STORE, 46970356793344, 46970356932607,
+ERASE, 46970356932608, 46970356932608,
+STORE, 46970356932608, 46970358591487,
+STORE, 46970358591488, 46970358632447,
+STORE, 46970358276096, 46970358591487,
+STORE, 46970356932608, 46970358276095,
+ERASE, 46970356932608, 46970356932608,
+STORE, 46970356932608, 46970358276095,
+STORE, 46970358587392, 46970358591487,
+STORE, 46970358276096, 46970358587391,
+ERASE, 46970358276096, 46970358276096,
+STORE, 46970358276096, 46970358587391,
+STORE, 46970358616064, 46970358632447,
+STORE, 46970358591488, 46970358616063,
+ERASE, 46970358591488, 46970358591488,
+STORE, 46970358591488, 46970358616063,
+ERASE, 46970358616064, 46970358616064,
+STORE, 46970358616064, 46970358632447,
+STORE, 46970358616064, 46970358644735,
+ERASE, 46970358591488, 46970358591488,
+STORE, 46970358591488, 46970358607871,
+STORE, 46970358607872, 46970358616063,
+ERASE, 46970356776960, 46970356776960,
+STORE, 46970356776960, 46970356781055,
+STORE, 46970356781056, 46970356785151,
+ERASE, 94121892900864, 94121892900864,
+STORE, 94121892900864, 94121892917247,
+STORE, 94121892917248, 94121892921343,
+ERASE, 140662438486016, 140662438486016,
+STORE, 140662438486016, 140662438490111,
+STORE, 140662438490112, 140662438494207,
+ERASE, 46970356670464, 46970356670464,
+STORE, 94121898610688, 94121898745855,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737189351424, 140737488351231,
+ERASE, 140737189351424, 140737189351424,
+STORE, 140737189351424, 140737189355519,
+STORE, 93847948832768, 93847949545471,
+ERASE, 93847948832768, 93847948832768,
+STORE, 93847948832768, 93847948881919,
+STORE, 93847948881920, 93847949545471,
+ERASE, 93847948881920, 93847948881920,
+STORE, 93847948881920, 93847949426687,
+STORE, 93847949426688, 93847949524991,
+STORE, 93847949524992, 93847949545471,
+STORE, 139698989985792, 139698990157823,
+ERASE, 139698989985792, 139698989985792,
+STORE, 139698989985792, 139698989989887,
+STORE, 139698989989888, 139698990157823,
+ERASE, 139698989989888, 139698989989888,
+STORE, 139698989989888, 139698990112767,
+STORE, 139698990112768, 139698990145535,
+STORE, 139698990145536, 139698990153727,
+STORE, 139698990153728, 139698990157823,
+STORE, 140737189744640, 140737189748735,
+STORE, 140737189732352, 140737189744639,
+STORE, 47933805010944, 47933805019135,
+STORE, 47933805019136, 47933805027327,
+STORE, 47933805027328, 47933805133823,
+STORE, 47933805043712, 47933805133823,
+STORE, 47933805027328, 47933805043711,
+ERASE, 47933805043712, 47933805043712,
+STORE, 47933805043712, 47933805117439,
+STORE, 47933805117440, 47933805133823,
+STORE, 47933805096960, 47933805117439,
+STORE, 47933805043712, 47933805096959,
+ERASE, 47933805043712, 47933805043712,
+STORE, 47933805043712, 47933805096959,
+STORE, 47933805113344, 47933805117439,
+STORE, 47933805096960, 47933805113343,
+ERASE, 47933805096960, 47933805096960,
+STORE, 47933805096960, 47933805113343,
+STORE, 47933805125632, 47933805133823,
+STORE, 47933805117440, 47933805125631,
+ERASE, 47933805117440, 47933805117440,
+STORE, 47933805117440, 47933805125631,
+ERASE, 47933805125632, 47933805125632,
+STORE, 47933805125632, 47933805133823,
+STORE, 47933805133824, 47933806972927,
+STORE, 47933805273088, 47933806972927,
+STORE, 47933805133824, 47933805273087,
+ERASE, 47933805273088, 47933805273088,
+STORE, 47933805273088, 47933806931967,
+STORE, 47933806931968, 47933806972927,
+STORE, 47933806616576, 47933806931967,
+STORE, 47933805273088, 47933806616575,
+ERASE, 47933805273088, 47933805273088,
+STORE, 47933805273088, 47933806616575,
+STORE, 47933806927872, 47933806931967,
+STORE, 47933806616576, 47933806927871,
+ERASE, 47933806616576, 47933806616576,
+STORE, 47933806616576, 47933806927871,
+STORE, 47933806956544, 47933806972927,
+STORE, 47933806931968, 47933806956543,
+ERASE, 47933806931968, 47933806931968,
+STORE, 47933806931968, 47933806956543,
+ERASE, 47933806956544, 47933806956544,
+STORE, 47933806956544, 47933806972927,
+STORE, 47933806956544, 47933806985215,
+ERASE, 47933806931968, 47933806931968,
+STORE, 47933806931968, 47933806948351,
+STORE, 47933806948352, 47933806956543,
+ERASE, 47933805117440, 47933805117440,
+STORE, 47933805117440, 47933805121535,
+STORE, 47933805121536, 47933805125631,
+ERASE, 93847949524992, 93847949524992,
+STORE, 93847949524992, 93847949541375,
+STORE, 93847949541376, 93847949545471,
+ERASE, 139698990145536, 139698990145536,
+STORE, 139698990145536, 139698990149631,
+STORE, 139698990149632, 139698990153727,
+ERASE, 47933805010944, 47933805010944,
+STORE, 140737488347136, 140737488351231,
+STORE, 140725553991680, 140737488351231,
+ERASE, 140725553991680, 140725553991680,
+STORE, 140725553991680, 140725553995775,
+STORE, 93980056248320, 93980056961023,
+ERASE, 93980056248320, 93980056248320,
+STORE, 93980056248320, 93980056297471,
+STORE, 93980056297472, 93980056961023,
+ERASE, 93980056297472, 93980056297472,
+STORE, 93980056297472, 93980056842239,
+STORE, 93980056842240, 93980056940543,
+STORE, 93980056940544, 93980056961023,
+STORE, 140146588971008, 140146589143039,
+ERASE, 140146588971008, 140146588971008,
+STORE, 140146588971008, 140146588975103,
+STORE, 140146588975104, 140146589143039,
+ERASE, 140146588975104, 140146588975104,
+STORE, 140146588975104, 140146589097983,
+STORE, 140146589097984, 140146589130751,
+STORE, 140146589130752, 140146589138943,
+STORE, 140146589138944, 140146589143039,
+STORE, 140725554860032, 140725554864127,
+STORE, 140725554847744, 140725554860031,
+STORE, 47486206025728, 47486206033919,
+STORE, 47486206033920, 47486206042111,
+STORE, 47486206042112, 47486206148607,
+STORE, 47486206058496, 47486206148607,
+STORE, 47486206042112, 47486206058495,
+ERASE, 47486206058496, 47486206058496,
+STORE, 47486206058496, 47486206132223,
+STORE, 47486206132224, 47486206148607,
+STORE, 47486206111744, 47486206132223,
+STORE, 47486206058496, 47486206111743,
+ERASE, 47486206058496, 47486206058496,
+STORE, 47486206058496, 47486206111743,
+STORE, 47486206128128, 47486206132223,
+STORE, 47486206111744, 47486206128127,
+ERASE, 47486206111744, 47486206111744,
+STORE, 47486206111744, 47486206128127,
+STORE, 47486206140416, 47486206148607,
+STORE, 47486206132224, 47486206140415,
+ERASE, 47486206132224, 47486206132224,
+STORE, 47486206132224, 47486206140415,
+ERASE, 47486206140416, 47486206140416,
+STORE, 47486206140416, 47486206148607,
+STORE, 47486206148608, 47486207987711,
+STORE, 47486206287872, 47486207987711,
+STORE, 47486206148608, 47486206287871,
+ERASE, 47486206287872, 47486206287872,
+STORE, 47486206287872, 47486207946751,
+STORE, 47486207946752, 47486207987711,
+STORE, 47486207631360, 47486207946751,
+STORE, 47486206287872, 47486207631359,
+ERASE, 47486206287872, 47486206287872,
+STORE, 47486206287872, 47486207631359,
+STORE, 47486207942656, 47486207946751,
+STORE, 47486207631360, 47486207942655,
+ERASE, 47486207631360, 47486207631360,
+STORE, 47486207631360, 47486207942655,
+STORE, 47486207971328, 47486207987711,
+STORE, 47486207946752, 47486207971327,
+ERASE, 47486207946752, 47486207946752,
+STORE, 47486207946752, 47486207971327,
+ERASE, 47486207971328, 47486207971328,
+STORE, 47486207971328, 47486207987711,
+STORE, 47486207971328, 47486207999999,
+ERASE, 47486207946752, 47486207946752,
+STORE, 47486207946752, 47486207963135,
+STORE, 47486207963136, 47486207971327,
+ERASE, 47486206132224, 47486206132224,
+STORE, 47486206132224, 47486206136319,
+STORE, 47486206136320, 47486206140415,
+ERASE, 93980056940544, 93980056940544,
+STORE, 93980056940544, 93980056956927,
+STORE, 93980056956928, 93980056961023,
+ERASE, 140146589130752, 140146589130752,
+STORE, 140146589130752, 140146589134847,
+STORE, 140146589134848, 140146589138943,
+ERASE, 47486206025728, 47486206025728,
+STORE, 93980070006784, 93980070141951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727334776832, 140737488351231,
+ERASE, 140727334776832, 140727334776832,
+STORE, 140727334776832, 140727334780927,
+STORE, 94049747247104, 94049747959807,
+ERASE, 94049747247104, 94049747247104,
+STORE, 94049747247104, 94049747296255,
+STORE, 94049747296256, 94049747959807,
+ERASE, 94049747296256, 94049747296256,
+STORE, 94049747296256, 94049747841023,
+STORE, 94049747841024, 94049747939327,
+STORE, 94049747939328, 94049747959807,
+STORE, 140227307216896, 140227307388927,
+ERASE, 140227307216896, 140227307216896,
+STORE, 140227307216896, 140227307220991,
+STORE, 140227307220992, 140227307388927,
+ERASE, 140227307220992, 140227307220992,
+STORE, 140227307220992, 140227307343871,
+STORE, 140227307343872, 140227307376639,
+STORE, 140227307376640, 140227307384831,
+STORE, 140227307384832, 140227307388927,
+STORE, 140727335337984, 140727335342079,
+STORE, 140727335325696, 140727335337983,
+STORE, 47405487779840, 47405487788031,
+STORE, 47405487788032, 47405487796223,
+STORE, 47405487796224, 47405487902719,
+STORE, 47405487812608, 47405487902719,
+STORE, 47405487796224, 47405487812607,
+ERASE, 47405487812608, 47405487812608,
+STORE, 47405487812608, 47405487886335,
+STORE, 47405487886336, 47405487902719,
+STORE, 47405487865856, 47405487886335,
+STORE, 47405487812608, 47405487865855,
+ERASE, 47405487812608, 47405487812608,
+STORE, 47405487812608, 47405487865855,
+STORE, 47405487882240, 47405487886335,
+STORE, 47405487865856, 47405487882239,
+ERASE, 47405487865856, 47405487865856,
+STORE, 47405487865856, 47405487882239,
+STORE, 47405487894528, 47405487902719,
+STORE, 47405487886336, 47405487894527,
+ERASE, 47405487886336, 47405487886336,
+STORE, 47405487886336, 47405487894527,
+ERASE, 47405487894528, 47405487894528,
+STORE, 47405487894528, 47405487902719,
+STORE, 47405487902720, 47405489741823,
+STORE, 47405488041984, 47405489741823,
+STORE, 47405487902720, 47405488041983,
+ERASE, 47405488041984, 47405488041984,
+STORE, 47405488041984, 47405489700863,
+STORE, 47405489700864, 47405489741823,
+STORE, 47405489385472, 47405489700863,
+STORE, 47405488041984, 47405489385471,
+ERASE, 47405488041984, 47405488041984,
+STORE, 47405488041984, 47405489385471,
+STORE, 47405489696768, 47405489700863,
+STORE, 47405489385472, 47405489696767,
+ERASE, 47405489385472, 47405489385472,
+STORE, 47405489385472, 47405489696767,
+STORE, 47405489725440, 47405489741823,
+STORE, 47405489700864, 47405489725439,
+ERASE, 47405489700864, 47405489700864,
+STORE, 47405489700864, 47405489725439,
+ERASE, 47405489725440, 47405489725440,
+STORE, 47405489725440, 47405489741823,
+STORE, 47405489725440, 47405489754111,
+ERASE, 47405489700864, 47405489700864,
+STORE, 47405489700864, 47405489717247,
+STORE, 47405489717248, 47405489725439,
+ERASE, 47405487886336, 47405487886336,
+STORE, 47405487886336, 47405487890431,
+STORE, 47405487890432, 47405487894527,
+ERASE, 94049747939328, 94049747939328,
+STORE, 94049747939328, 94049747955711,
+STORE, 94049747955712, 94049747959807,
+ERASE, 140227307376640, 140227307376640,
+STORE, 140227307376640, 140227307380735,
+STORE, 140227307380736, 140227307384831,
+ERASE, 47405487779840, 47405487779840,
+STORE, 94049758810112, 94049758945279,
+STORE, 140737488347136, 140737488351231,
+STORE, 140727079718912, 140737488351231,
+ERASE, 140727079718912, 140727079718912,
+STORE, 140727079718912, 140727079723007,
+STORE, 94250996527104, 94250997239807,
+ERASE, 94250996527104, 94250996527104,
+STORE, 94250996527104, 94250996576255,
+STORE, 94250996576256, 94250997239807,
+ERASE, 94250996576256, 94250996576256,
+STORE, 94250996576256, 94250997121023,
+STORE, 94250997121024, 94250997219327,
+STORE, 94250997219328, 94250997239807,
+STORE, 140060022587392, 140060022759423,
+ERASE, 140060022587392, 140060022587392,
+STORE, 140060022587392, 140060022591487,
+STORE, 140060022591488, 140060022759423,
+ERASE, 140060022591488, 140060022591488,
+STORE, 140060022591488, 140060022714367,
+STORE, 140060022714368, 140060022747135,
+STORE, 140060022747136, 140060022755327,
+STORE, 140060022755328, 140060022759423,
+STORE, 140727079788544, 140727079792639,
+STORE, 140727079776256, 140727079788543,
+STORE, 47572772409344, 47572772417535,
+STORE, 47572772417536, 47572772425727,
+STORE, 47572772425728, 47572772532223,
+STORE, 47572772442112, 47572772532223,
+STORE, 47572772425728, 47572772442111,
+ERASE, 47572772442112, 47572772442112,
+STORE, 47572772442112, 47572772515839,
+STORE, 47572772515840, 47572772532223,
+STORE, 47572772495360, 47572772515839,
+STORE, 47572772442112, 47572772495359,
+ERASE, 47572772442112, 47572772442112,
+STORE, 47572772442112, 47572772495359,
+STORE, 47572772511744, 47572772515839,
+STORE, 47572772495360, 47572772511743,
+ERASE, 47572772495360, 47572772495360,
+STORE, 47572772495360, 47572772511743,
+STORE, 47572772524032, 47572772532223,
+STORE, 47572772515840, 47572772524031,
+ERASE, 47572772515840, 47572772515840,
+STORE, 47572772515840, 47572772524031,
+ERASE, 47572772524032, 47572772524032,
+STORE, 47572772524032, 47572772532223,
+STORE, 47572772532224, 47572774371327,
+STORE, 47572772671488, 47572774371327,
+STORE, 47572772532224, 47572772671487,
+ERASE, 47572772671488, 47572772671488,
+STORE, 47572772671488, 47572774330367,
+STORE, 47572774330368, 47572774371327,
+STORE, 47572774014976, 47572774330367,
+STORE, 47572772671488, 47572774014975,
+ERASE, 47572772671488, 47572772671488,
+STORE, 47572772671488, 47572774014975,
+STORE, 47572774326272, 47572774330367,
+STORE, 47572774014976, 47572774326271,
+ERASE, 47572774014976, 47572774014976,
+STORE, 47572774014976, 47572774326271,
+STORE, 47572774354944, 47572774371327,
+STORE, 47572774330368, 47572774354943,
+ERASE, 47572774330368, 47572774330368,
+STORE, 47572774330368, 47572774354943,
+ERASE, 47572774354944, 47572774354944,
+STORE, 47572774354944, 47572774371327,
+STORE, 47572774354944, 47572774383615,
+ERASE, 47572774330368, 47572774330368,
+STORE, 47572774330368, 47572774346751,
+STORE, 47572774346752, 47572774354943,
+ERASE, 47572772515840, 47572772515840,
+STORE, 47572772515840, 47572772519935,
+STORE, 47572772519936, 47572772524031,
+ERASE, 94250997219328, 94250997219328,
+STORE, 94250997219328, 94250997235711,
+STORE, 94250997235712, 94250997239807,
+ERASE, 140060022747136, 140060022747136,
+STORE, 140060022747136, 140060022751231,
+STORE, 140060022751232, 140060022755327,
+ERASE, 47572772409344, 47572772409344,
+STORE, 94251018305536, 94251018440703,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730012389376, 140737488351231,
+ERASE, 140730012389376, 140730012389376,
+STORE, 140730012389376, 140730012393471,
+STORE, 94382607675392, 94382607695871,
+ERASE, 94382607675392, 94382607675392,
+STORE, 94382607675392, 94382607679487,
+STORE, 94382607679488, 94382607695871,
+ERASE, 94382607679488, 94382607679488,
+STORE, 94382607679488, 94382607683583,
+STORE, 94382607683584, 94382607687679,
+STORE, 94382607687680, 94382607695871,
+STORE, 140252451454976, 140252451627007,
+ERASE, 140252451454976, 140252451454976,
+STORE, 140252451454976, 140252451459071,
+STORE, 140252451459072, 140252451627007,
+ERASE, 140252451459072, 140252451459072,
+STORE, 140252451459072, 140252451581951,
+STORE, 140252451581952, 140252451614719,
+STORE, 140252451614720, 140252451622911,
+STORE, 140252451622912, 140252451627007,
+STORE, 140730013548544, 140730013552639,
+STORE, 140730013536256, 140730013548543,
+STORE, 47380343541760, 47380343549951,
+STORE, 47380343549952, 47380343558143,
+STORE, 47380343558144, 47380345397247,
+STORE, 47380343697408, 47380345397247,
+STORE, 47380343558144, 47380343697407,
+ERASE, 47380343697408, 47380343697408,
+STORE, 47380343697408, 47380345356287,
+STORE, 47380345356288, 47380345397247,
+STORE, 47380345040896, 47380345356287,
+STORE, 47380343697408, 47380345040895,
+ERASE, 47380343697408, 47380343697408,
+STORE, 47380343697408, 47380345040895,
+STORE, 47380345352192, 47380345356287,
+STORE, 47380345040896, 47380345352191,
+ERASE, 47380345040896, 47380345040896,
+STORE, 47380345040896, 47380345352191,
+STORE, 47380345380864, 47380345397247,
+STORE, 47380345356288, 47380345380863,
+ERASE, 47380345356288, 47380345356288,
+STORE, 47380345356288, 47380345380863,
+ERASE, 47380345380864, 47380345380864,
+STORE, 47380345380864, 47380345397247,
+ERASE, 47380345356288, 47380345356288,
+STORE, 47380345356288, 47380345372671,
+STORE, 47380345372672, 47380345380863,
+ERASE, 94382607687680, 94382607687680,
+STORE, 94382607687680, 94382607691775,
+STORE, 94382607691776, 94382607695871,
+ERASE, 140252451614720, 140252451614720,
+STORE, 140252451614720, 140252451618815,
+STORE, 140252451618816, 140252451622911,
+ERASE, 47380343541760, 47380343541760,
+STORE, 94382626803712, 94382626938879,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730900271104, 140737488351231,
+ERASE, 140730900271104, 140730900271104,
+STORE, 140730900271104, 140730900275199,
+STORE, 93855478120448, 93855478337535,
+ERASE, 93855478120448, 93855478120448,
+STORE, 93855478120448, 93855478198271,
+STORE, 93855478198272, 93855478337535,
+ERASE, 93855478198272, 93855478198272,
+STORE, 93855478198272, 93855478243327,
+STORE, 93855478243328, 93855478288383,
+STORE, 93855478288384, 93855478337535,
+STORE, 140092686573568, 140092686745599,
+ERASE, 140092686573568, 140092686573568,
+STORE, 140092686573568, 140092686577663,
+STORE, 140092686577664, 140092686745599,
+ERASE, 140092686577664, 140092686577664,
+STORE, 140092686577664, 140092686700543,
+STORE, 140092686700544, 140092686733311,
+STORE, 140092686733312, 140092686741503,
+STORE, 140092686741504, 140092686745599,
+STORE, 140730900537344, 140730900541439,
+STORE, 140730900525056, 140730900537343,
+STORE, 47540108423168, 47540108431359,
+STORE, 47540108431360, 47540108439551,
+STORE, 47540108439552, 47540110278655,
+STORE, 47540108578816, 47540110278655,
+STORE, 47540108439552, 47540108578815,
+ERASE, 47540108578816, 47540108578816,
+STORE, 47540108578816, 47540110237695,
+STORE, 47540110237696, 47540110278655,
+STORE, 47540109922304, 47540110237695,
+STORE, 47540108578816, 47540109922303,
+ERASE, 47540108578816, 47540108578816,
+STORE, 47540108578816, 47540109922303,
+STORE, 47540110233600, 47540110237695,
+STORE, 47540109922304, 47540110233599,
+ERASE, 47540109922304, 47540109922304,
+STORE, 47540109922304, 47540110233599,
+STORE, 47540110262272, 47540110278655,
+STORE, 47540110237696, 47540110262271,
+ERASE, 47540110237696, 47540110237696,
+STORE, 47540110237696, 47540110262271,
+ERASE, 47540110262272, 47540110262272,
+STORE, 47540110262272, 47540110278655,
+ERASE, 47540110237696, 47540110237696,
+STORE, 47540110237696, 47540110254079,
+STORE, 47540110254080, 47540110262271,
+ERASE, 93855478288384, 93855478288384,
+STORE, 93855478288384, 93855478333439,
+STORE, 93855478333440, 93855478337535,
+ERASE, 140092686733312, 140092686733312,
+STORE, 140092686733312, 140092686737407,
+STORE, 140092686737408, 140092686741503,
+ERASE, 47540108423168, 47540108423168,
+STORE, 93855492222976, 93855492358143,
+STORE, 93855492222976, 93855492493311,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733498146816, 140737488351231,
+ERASE, 140733498146816, 140733498146816,
+STORE, 140733498146816, 140733498150911,
+STORE, 94170739654656, 94170740367359,
+ERASE, 94170739654656, 94170739654656,
+STORE, 94170739654656, 94170739703807,
+STORE, 94170739703808, 94170740367359,
+ERASE, 94170739703808, 94170739703808,
+STORE, 94170739703808, 94170740248575,
+STORE, 94170740248576, 94170740346879,
+STORE, 94170740346880, 94170740367359,
+STORE, 140024788877312, 140024789049343,
+ERASE, 140024788877312, 140024788877312,
+STORE, 140024788877312, 140024788881407,
+STORE, 140024788881408, 140024789049343,
+ERASE, 140024788881408, 140024788881408,
+STORE, 140024788881408, 140024789004287,
+STORE, 140024789004288, 140024789037055,
+STORE, 140024789037056, 140024789045247,
+STORE, 140024789045248, 140024789049343,
+STORE, 140733499023360, 140733499027455,
+STORE, 140733499011072, 140733499023359,
+STORE, 47608006119424, 47608006127615,
+STORE, 47608006127616, 47608006135807,
+STORE, 47608006135808, 47608006242303,
+STORE, 47608006152192, 47608006242303,
+STORE, 47608006135808, 47608006152191,
+ERASE, 47608006152192, 47608006152192,
+STORE, 47608006152192, 47608006225919,
+STORE, 47608006225920, 47608006242303,
+STORE, 47608006205440, 47608006225919,
+STORE, 47608006152192, 47608006205439,
+ERASE, 47608006152192, 47608006152192,
+STORE, 47608006152192, 47608006205439,
+STORE, 47608006221824, 47608006225919,
+STORE, 47608006205440, 47608006221823,
+ERASE, 47608006205440, 47608006205440,
+STORE, 47608006205440, 47608006221823,
+STORE, 47608006234112, 47608006242303,
+STORE, 47608006225920, 47608006234111,
+ERASE, 47608006225920, 47608006225920,
+STORE, 47608006225920, 47608006234111,
+ERASE, 47608006234112, 47608006234112,
+STORE, 47608006234112, 47608006242303,
+STORE, 47608006242304, 47608008081407,
+STORE, 47608006381568, 47608008081407,
+STORE, 47608006242304, 47608006381567,
+ERASE, 47608006381568, 47608006381568,
+STORE, 47608006381568, 47608008040447,
+STORE, 47608008040448, 47608008081407,
+STORE, 47608007725056, 47608008040447,
+STORE, 47608006381568, 47608007725055,
+ERASE, 47608006381568, 47608006381568,
+STORE, 47608006381568, 47608007725055,
+STORE, 47608008036352, 47608008040447,
+STORE, 47608007725056, 47608008036351,
+ERASE, 47608007725056, 47608007725056,
+STORE, 47608007725056, 47608008036351,
+STORE, 47608008065024, 47608008081407,
+STORE, 47608008040448, 47608008065023,
+ERASE, 47608008040448, 47608008040448,
+STORE, 47608008040448, 47608008065023,
+ERASE, 47608008065024, 47608008065024,
+STORE, 47608008065024, 47608008081407,
+STORE, 47608008065024, 47608008093695,
+ERASE, 47608008040448, 47608008040448,
+STORE, 47608008040448, 47608008056831,
+STORE, 47608008056832, 47608008065023,
+ERASE, 47608006225920, 47608006225920,
+STORE, 47608006225920, 47608006230015,
+STORE, 47608006230016, 47608006234111,
+ERASE, 94170740346880, 94170740346880,
+STORE, 94170740346880, 94170740363263,
+STORE, 94170740363264, 94170740367359,
+ERASE, 140024789037056, 140024789037056,
+STORE, 140024789037056, 140024789041151,
+STORE, 140024789041152, 140024789045247,
+ERASE, 47608006119424, 47608006119424,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730264326144, 140737488351231,
+ERASE, 140730264326144, 140730264326144,
+STORE, 140730264326144, 140730264330239,
+STORE, 94653216407552, 94653217120255,
+ERASE, 94653216407552, 94653216407552,
+STORE, 94653216407552, 94653216456703,
+STORE, 94653216456704, 94653217120255,
+ERASE, 94653216456704, 94653216456704,
+STORE, 94653216456704, 94653217001471,
+STORE, 94653217001472, 94653217099775,
+STORE, 94653217099776, 94653217120255,
+STORE, 140103617011712, 140103617183743,
+ERASE, 140103617011712, 140103617011712,
+STORE, 140103617011712, 140103617015807,
+STORE, 140103617015808, 140103617183743,
+ERASE, 140103617015808, 140103617015808,
+STORE, 140103617015808, 140103617138687,
+STORE, 140103617138688, 140103617171455,
+STORE, 140103617171456, 140103617179647,
+STORE, 140103617179648, 140103617183743,
+STORE, 140730265427968, 140730265432063,
+STORE, 140730265415680, 140730265427967,
+STORE, 47529177985024, 47529177993215,
+STORE, 47529177993216, 47529178001407,
+STORE, 47529178001408, 47529178107903,
+STORE, 47529178017792, 47529178107903,
+STORE, 47529178001408, 47529178017791,
+ERASE, 47529178017792, 47529178017792,
+STORE, 47529178017792, 47529178091519,
+STORE, 47529178091520, 47529178107903,
+STORE, 47529178071040, 47529178091519,
+STORE, 47529178017792, 47529178071039,
+ERASE, 47529178017792, 47529178017792,
+STORE, 47529178017792, 47529178071039,
+STORE, 47529178087424, 47529178091519,
+STORE, 47529178071040, 47529178087423,
+ERASE, 47529178071040, 47529178071040,
+STORE, 47529178071040, 47529178087423,
+STORE, 47529178099712, 47529178107903,
+STORE, 47529178091520, 47529178099711,
+ERASE, 47529178091520, 47529178091520,
+STORE, 47529178091520, 47529178099711,
+ERASE, 47529178099712, 47529178099712,
+STORE, 47529178099712, 47529178107903,
+STORE, 47529178107904, 47529179947007,
+STORE, 47529178247168, 47529179947007,
+STORE, 47529178107904, 47529178247167,
+ERASE, 47529178247168, 47529178247168,
+STORE, 47529178247168, 47529179906047,
+STORE, 47529179906048, 47529179947007,
+STORE, 47529179590656, 47529179906047,
+STORE, 47529178247168, 47529179590655,
+ERASE, 47529178247168, 47529178247168,
+STORE, 47529178247168, 47529179590655,
+STORE, 47529179901952, 47529179906047,
+STORE, 47529179590656, 47529179901951,
+ERASE, 47529179590656, 47529179590656,
+STORE, 47529179590656, 47529179901951,
+STORE, 47529179930624, 47529179947007,
+STORE, 47529179906048, 47529179930623,
+ERASE, 47529179906048, 47529179906048,
+STORE, 47529179906048, 47529179930623,
+ERASE, 47529179930624, 47529179930624,
+STORE, 47529179930624, 47529179947007,
+STORE, 47529179930624, 47529179959295,
+ERASE, 47529179906048, 47529179906048,
+STORE, 47529179906048, 47529179922431,
+STORE, 47529179922432, 47529179930623,
+ERASE, 47529178091520, 47529178091520,
+STORE, 47529178091520, 47529178095615,
+STORE, 47529178095616, 47529178099711,
+ERASE, 94653217099776, 94653217099776,
+STORE, 94653217099776, 94653217116159,
+STORE, 94653217116160, 94653217120255,
+ERASE, 140103617171456, 140103617171456,
+STORE, 140103617171456, 140103617175551,
+STORE, 140103617175552, 140103617179647,
+ERASE, 47529177985024, 47529177985024,
+STORE, 94653241135104, 94653241270271,
+STORE, 140737488347136, 140737488351231,
+STORE, 140736284549120, 140737488351231,
+ERASE, 140736284549120, 140736284549120,
+STORE, 140736284549120, 140736284553215,
+STORE, 93963663822848, 93963664506879,
+ERASE, 93963663822848, 93963663822848,
+STORE, 93963663822848, 93963663884287,
+STORE, 93963663884288, 93963664506879,
+ERASE, 93963663884288, 93963663884288,
+STORE, 93963663884288, 93963664240639,
+STORE, 93963664240640, 93963664379903,
+STORE, 93963664379904, 93963664506879,
+STORE, 140450188439552, 140450188611583,
+ERASE, 140450188439552, 140450188439552,
+STORE, 140450188439552, 140450188443647,
+STORE, 140450188443648, 140450188611583,
+ERASE, 140450188443648, 140450188443648,
+STORE, 140450188443648, 140450188566527,
+STORE, 140450188566528, 140450188599295,
+STORE, 140450188599296, 140450188607487,
+STORE, 140450188607488, 140450188611583,
+STORE, 140736284577792, 140736284581887,
+STORE, 140736284565504, 140736284577791,
+STORE, 47182606557184, 47182606565375,
+STORE, 47182606565376, 47182606573567,
+STORE, 47182606573568, 47182608412671,
+STORE, 47182606712832, 47182608412671,
+STORE, 47182606573568, 47182606712831,
+ERASE, 47182606712832, 47182606712832,
+STORE, 47182606712832, 47182608371711,
+STORE, 47182608371712, 47182608412671,
+STORE, 47182608056320, 47182608371711,
+STORE, 47182606712832, 47182608056319,
+ERASE, 47182606712832, 47182606712832,
+STORE, 47182606712832, 47182608056319,
+STORE, 47182608367616, 47182608371711,
+STORE, 47182608056320, 47182608367615,
+ERASE, 47182608056320, 47182608056320,
+STORE, 47182608056320, 47182608367615,
+STORE, 47182608396288, 47182608412671,
+STORE, 47182608371712, 47182608396287,
+ERASE, 47182608371712, 47182608371712,
+STORE, 47182608371712, 47182608396287,
+ERASE, 47182608396288, 47182608396288,
+STORE, 47182608396288, 47182608412671,
+STORE, 47182608412672, 47182608523263,
+STORE, 47182608429056, 47182608523263,
+STORE, 47182608412672, 47182608429055,
+ERASE, 47182608429056, 47182608429056,
+STORE, 47182608429056, 47182608515071,
+STORE, 47182608515072, 47182608523263,
+STORE, 47182608490496, 47182608515071,
+STORE, 47182608429056, 47182608490495,
+ERASE, 47182608429056, 47182608429056,
+STORE, 47182608429056, 47182608490495,
+STORE, 47182608510976, 47182608515071,
+STORE, 47182608490496, 47182608510975,
+ERASE, 47182608490496, 47182608490496,
+STORE, 47182608490496, 47182608510975,
+ERASE, 47182608515072, 47182608515072,
+STORE, 47182608515072, 47182608523263,
+STORE, 47182608523264, 47182608568319,
+ERASE, 47182608523264, 47182608523264,
+STORE, 47182608523264, 47182608531455,
+STORE, 47182608531456, 47182608568319,
+STORE, 47182608551936, 47182608568319,
+STORE, 47182608531456, 47182608551935,
+ERASE, 47182608531456, 47182608531456,
+STORE, 47182608531456, 47182608551935,
+STORE, 47182608560128, 47182608568319,
+STORE, 47182608551936, 47182608560127,
+ERASE, 47182608551936, 47182608551936,
+STORE, 47182608551936, 47182608568319,
+ERASE, 47182608551936, 47182608551936,
+STORE, 47182608551936, 47182608560127,
+STORE, 47182608560128, 47182608568319,
+ERASE, 47182608560128, 47182608560128,
+STORE, 47182608560128, 47182608568319,
+STORE, 47182608568320, 47182608916479,
+STORE, 47182608609280, 47182608916479,
+STORE, 47182608568320, 47182608609279,
+ERASE, 47182608609280, 47182608609280,
+STORE, 47182608609280, 47182608891903,
+STORE, 47182608891904, 47182608916479,
+STORE, 47182608822272, 47182608891903,
+STORE, 47182608609280, 47182608822271,
+ERASE, 47182608609280, 47182608609280,
+STORE, 47182608609280, 47182608822271,
+STORE, 47182608887808, 47182608891903,
+STORE, 47182608822272, 47182608887807,
+ERASE, 47182608822272, 47182608822272,
+STORE, 47182608822272, 47182608887807,
+ERASE, 47182608891904, 47182608891904,
+STORE, 47182608891904, 47182608916479,
+STORE, 47182608916480, 47182611177471,
+STORE, 47182609068032, 47182611177471,
+STORE, 47182608916480, 47182609068031,
+ERASE, 47182609068032, 47182609068032,
+STORE, 47182609068032, 47182611161087,
+STORE, 47182611161088, 47182611177471,
+STORE, 47182611169280, 47182611177471,
+STORE, 47182611161088, 47182611169279,
+ERASE, 47182611161088, 47182611161088,
+STORE, 47182611161088, 47182611169279,
+ERASE, 47182611169280, 47182611169280,
+STORE, 47182611169280, 47182611177471,
+STORE, 47182611177472, 47182611312639,
+ERASE, 47182611177472, 47182611177472,
+STORE, 47182611177472, 47182611202047,
+STORE, 47182611202048, 47182611312639,
+STORE, 47182611263488, 47182611312639,
+STORE, 47182611202048, 47182611263487,
+ERASE, 47182611202048, 47182611202048,
+STORE, 47182611202048, 47182611263487,
+STORE, 47182611288064, 47182611312639,
+STORE, 47182611263488, 47182611288063,
+ERASE, 47182611263488, 47182611263488,
+STORE, 47182611263488, 47182611312639,
+ERASE, 47182611263488, 47182611263488,
+STORE, 47182611263488, 47182611288063,
+STORE, 47182611288064, 47182611312639,
+STORE, 47182611296256, 47182611312639,
+STORE, 47182611288064, 47182611296255,
+ERASE, 47182611288064, 47182611288064,
+STORE, 47182611288064, 47182611296255,
+ERASE, 47182611296256, 47182611296256,
+STORE, 47182611296256, 47182611312639,
+STORE, 47182611296256, 47182611320831,
+STORE, 47182611320832, 47182611484671,
+ERASE, 47182611320832, 47182611320832,
+STORE, 47182611320832, 47182611333119,
+STORE, 47182611333120, 47182611484671,
+STORE, 47182611431424, 47182611484671,
+STORE, 47182611333120, 47182611431423,
+ERASE, 47182611333120, 47182611333120,
+STORE, 47182611333120, 47182611431423,
+STORE, 47182611476480, 47182611484671,
+STORE, 47182611431424, 47182611476479,
+ERASE, 47182611431424, 47182611431424,
+STORE, 47182611431424, 47182611484671,
+ERASE, 47182611431424, 47182611431424,
+STORE, 47182611431424, 47182611476479,
+STORE, 47182611476480, 47182611484671,
+ERASE, 47182611476480, 47182611476480,
+STORE, 47182611476480, 47182611484671,
+STORE, 47182611484672, 47182612082687,
+STORE, 47182611603456, 47182612082687,
+STORE, 47182611484672, 47182611603455,
+ERASE, 47182611603456, 47182611603456,
+STORE, 47182611603456, 47182612029439,
+STORE, 47182612029440, 47182612082687,
+STORE, 47182611918848, 47182612029439,
+STORE, 47182611603456, 47182611918847,
+ERASE, 47182611603456, 47182611603456,
+STORE, 47182611603456, 47182611918847,
+STORE, 47182612025344, 47182612029439,
+STORE, 47182611918848, 47182612025343,
+ERASE, 47182611918848, 47182611918848,
+STORE, 47182611918848, 47182612025343,
+ERASE, 47182612029440, 47182612029440,
+STORE, 47182612029440, 47182612082687,
+STORE, 47182612082688, 47182615134207,
+STORE, 47182612627456, 47182615134207,
+STORE, 47182612082688, 47182612627455,
+ERASE, 47182612627456, 47182612627456,
+STORE, 47182612627456, 47182614913023,
+STORE, 47182614913024, 47182615134207,
+STORE, 47182614323200, 47182614913023,
+STORE, 47182612627456, 47182614323199,
+ERASE, 47182612627456, 47182612627456,
+STORE, 47182612627456, 47182614323199,
+STORE, 47182614908928, 47182614913023,
+STORE, 47182614323200, 47182614908927,
+ERASE, 47182614323200, 47182614323200,
+STORE, 47182614323200, 47182614908927,
+STORE, 47182615117824, 47182615134207,
+STORE, 47182614913024, 47182615117823,
+ERASE, 47182614913024, 47182614913024,
+STORE, 47182614913024, 47182615117823,
+ERASE, 47182615117824, 47182615117824,
+STORE, 47182615117824, 47182615134207,
+STORE, 47182615134208, 47182615166975,
+ERASE, 47182615134208, 47182615134208,
+STORE, 47182615134208, 47182615142399,
+STORE, 47182615142400, 47182615166975,
+STORE, 47182615154688, 47182615166975,
+STORE, 47182615142400, 47182615154687,
+ERASE, 47182615142400, 47182615142400,
+STORE, 47182615142400, 47182615154687,
+STORE, 47182615158784, 47182615166975,
+STORE, 47182615154688, 47182615158783,
+ERASE, 47182615154688, 47182615154688,
+STORE, 47182615154688, 47182615166975,
+ERASE, 47182615154688, 47182615154688,
+STORE, 47182615154688, 47182615158783,
+STORE, 47182615158784, 47182615166975,
+ERASE, 47182615158784, 47182615158784,
+STORE, 47182615158784, 47182615166975,
+STORE, 47182615166976, 47182615203839,
+ERASE, 47182615166976, 47182615166976,
+STORE, 47182615166976, 47182615175167,
+STORE, 47182615175168, 47182615203839,
+STORE, 47182615191552, 47182615203839,
+STORE, 47182615175168, 47182615191551,
+ERASE, 47182615175168, 47182615175168,
+STORE, 47182615175168, 47182615191551,
+STORE, 47182615195648, 47182615203839,
+STORE, 47182615191552, 47182615195647,
+ERASE, 47182615191552, 47182615191552,
+STORE, 47182615191552, 47182615203839,
+ERASE, 47182615191552, 47182615191552,
+STORE, 47182615191552, 47182615195647,
+STORE, 47182615195648, 47182615203839,
+ERASE, 47182615195648, 47182615195648,
+STORE, 47182615195648, 47182615203839,
+STORE, 47182615203840, 47182615678975,
+ERASE, 47182615203840, 47182615203840,
+STORE, 47182615203840, 47182615212031,
+STORE, 47182615212032, 47182615678975,
+STORE, 47182615547904, 47182615678975,
+STORE, 47182615212032, 47182615547903,
+ERASE, 47182615212032, 47182615212032,
+STORE, 47182615212032, 47182615547903,
+STORE, 47182615670784, 47182615678975,
+STORE, 47182615547904, 47182615670783,
+ERASE, 47182615547904, 47182615547904,
+STORE, 47182615547904, 47182615678975,
+ERASE, 47182615547904, 47182615547904,
+STORE, 47182615547904, 47182615670783,
+STORE, 47182615670784, 47182615678975,
+ERASE, 47182615670784, 47182615670784,
+STORE, 47182615670784, 47182615678975,
+STORE, 47182615678976, 47182615687167,
+STORE, 47182615687168, 47182615707647,
+ERASE, 47182615687168, 47182615687168,
+STORE, 47182615687168, 47182615691263,
+STORE, 47182615691264, 47182615707647,
+STORE, 47182615695360, 47182615707647,
+STORE, 47182615691264, 47182615695359,
+ERASE, 47182615691264, 47182615691264,
+STORE, 47182615691264, 47182615695359,
+STORE, 47182615699456, 47182615707647,
+STORE, 47182615695360, 47182615699455,
+ERASE, 47182615695360, 47182615695360,
+STORE, 47182615695360, 47182615707647,
+ERASE, 47182615695360, 47182615695360,
+STORE, 47182615695360, 47182615699455,
+STORE, 47182615699456, 47182615707647,
+ERASE, 47182615699456, 47182615699456,
+STORE, 47182615699456, 47182615707647,
+STORE, 47182615707648, 47182615715839,
+ERASE, 47182608371712, 47182608371712,
+STORE, 47182608371712, 47182608388095,
+STORE, 47182608388096, 47182608396287,
+ERASE, 47182615699456, 47182615699456,
+STORE, 47182615699456, 47182615703551,
+STORE, 47182615703552, 47182615707647,
+ERASE, 47182611288064, 47182611288064,
+STORE, 47182611288064, 47182611292159,
+STORE, 47182611292160, 47182611296255,
+ERASE, 47182615670784, 47182615670784,
+STORE, 47182615670784, 47182615674879,
+STORE, 47182615674880, 47182615678975,
+ERASE, 47182615195648, 47182615195648,
+STORE, 47182615195648, 47182615199743,
+STORE, 47182615199744, 47182615203839,
+ERASE, 47182615158784, 47182615158784,
+STORE, 47182615158784, 47182615162879,
+STORE, 47182615162880, 47182615166975,
+ERASE, 47182614913024, 47182614913024,
+STORE, 47182614913024, 47182615109631,
+STORE, 47182615109632, 47182615117823,
+ERASE, 47182612029440, 47182612029440,
+STORE, 47182612029440, 47182612066303,
+STORE, 47182612066304, 47182612082687,
+ERASE, 47182611476480, 47182611476480,
+STORE, 47182611476480, 47182611480575,
+STORE, 47182611480576, 47182611484671,
+ERASE, 47182611161088, 47182611161088,
+STORE, 47182611161088, 47182611165183,
+STORE, 47182611165184, 47182611169279,
+ERASE, 47182608891904, 47182608891904,
+STORE, 47182608891904, 47182608912383,
+STORE, 47182608912384, 47182608916479,
+ERASE, 47182608560128, 47182608560128,
+STORE, 47182608560128, 47182608564223,
+STORE, 47182608564224, 47182608568319,
+ERASE, 47182608515072, 47182608515072,
+STORE, 47182608515072, 47182608519167,
+STORE, 47182608519168, 47182608523263,
+ERASE, 93963664379904, 93963664379904,
+STORE, 93963664379904, 93963664502783,
+STORE, 93963664502784, 93963664506879,
+ERASE, 140450188599296, 140450188599296,
+STORE, 140450188599296, 140450188603391,
+STORE, 140450188603392, 140450188607487,
+ERASE, 47182606557184, 47182606557184,
+STORE, 93963694723072, 93963694858239,
+STORE, 140737488347136, 140737488351231,
+STORE, 140730313261056, 140737488351231,
+ERASE, 140730313261056, 140730313261056,
+STORE, 140730313261056, 140730313265151,
+STORE, 94386579017728, 94386579697663,
+ERASE, 94386579017728, 94386579017728,
+STORE, 94386579017728, 94386579083263,
+STORE, 94386579083264, 94386579697663,
+ERASE, 94386579083264, 94386579083264,
+STORE, 94386579083264, 94386579431423,
+STORE, 94386579431424, 94386579570687,
+STORE, 94386579570688, 94386579697663,
+STORE, 140124810838016, 140124811010047,
+ERASE, 140124810838016, 140124810838016,
+STORE, 140124810838016, 140124810842111,
+STORE, 140124810842112, 140124811010047,
+ERASE, 140124810842112, 140124810842112,
+STORE, 140124810842112, 140124810964991,
+STORE, 140124810964992, 140124810997759,
+STORE, 140124810997760, 140124811005951,
+STORE, 140124811005952, 140124811010047,
+STORE, 140730313601024, 140730313605119,
+STORE, 140730313588736, 140730313601023,
+STORE, 47507984158720, 47507984166911,
+STORE, 47507984166912, 47507984175103,
+STORE, 47507984175104, 47507986014207,
+STORE, 47507984314368, 47507986014207,
+STORE, 47507984175104, 47507984314367,
+ERASE, 47507984314368, 47507984314368,
+STORE, 47507984314368, 47507985973247,
+STORE, 47507985973248, 47507986014207,
+STORE, 47507985657856, 47507985973247,
+STORE, 47507984314368, 47507985657855,
+ERASE, 47507984314368, 47507984314368,
+STORE, 47507984314368, 47507985657855,
+STORE, 47507985969152, 47507985973247,
+STORE, 47507985657856, 47507985969151,
+ERASE, 47507985657856, 47507985657856,
+STORE, 47507985657856, 47507985969151,
+STORE, 47507985997824, 47507986014207,
+STORE, 47507985973248, 47507985997823,
+ERASE, 47507985973248, 47507985973248,
+STORE, 47507985973248, 47507985997823,
+ERASE, 47507985997824, 47507985997824,
+STORE, 47507985997824, 47507986014207,
+STORE, 47507986014208, 47507986124799,
+STORE, 47507986030592, 47507986124799,
+STORE, 47507986014208, 47507986030591,
+ERASE, 47507986030592, 47507986030592,
+STORE, 47507986030592, 47507986116607,
+STORE, 47507986116608, 47507986124799,
+STORE, 47507986092032, 47507986116607,
+STORE, 47507986030592, 47507986092031,
+ERASE, 47507986030592, 47507986030592,
+STORE, 47507986030592, 47507986092031,
+STORE, 47507986112512, 47507986116607,
+STORE, 47507986092032, 47507986112511,
+ERASE, 47507986092032, 47507986092032,
+STORE, 47507986092032, 47507986112511,
+ERASE, 47507986116608, 47507986116608,
+STORE, 47507986116608, 47507986124799,
+STORE, 47507986124800, 47507986169855,
+ERASE, 47507986124800, 47507986124800,
+STORE, 47507986124800, 47507986132991,
+STORE, 47507986132992, 47507986169855,
+STORE, 47507986153472, 47507986169855,
+STORE, 47507986132992, 47507986153471,
+ERASE, 47507986132992, 47507986132992,
+STORE, 47507986132992, 47507986153471,
+STORE, 47507986161664, 47507986169855,
+STORE, 47507986153472, 47507986161663,
+ERASE, 47507986153472, 47507986153472,
+STORE, 47507986153472, 47507986169855,
+ERASE, 47507986153472, 47507986153472,
+STORE, 47507986153472, 47507986161663,
+STORE, 47507986161664, 47507986169855,
+ERASE, 47507986161664, 47507986161664,
+STORE, 47507986161664, 47507986169855,
+STORE, 47507986169856, 47507986518015,
+STORE, 47507986210816, 47507986518015,
+STORE, 47507986169856, 47507986210815,
+ERASE, 47507986210816, 47507986210816,
+STORE, 47507986210816, 47507986493439,
+STORE, 47507986493440, 47507986518015,
+STORE, 47507986423808, 47507986493439,
+STORE, 47507986210816, 47507986423807,
+ERASE, 47507986210816, 47507986210816,
+STORE, 47507986210816, 47507986423807,
+STORE, 47507986489344, 47507986493439,
+STORE, 47507986423808, 47507986489343,
+ERASE, 47507986423808, 47507986423808,
+STORE, 47507986423808, 47507986489343,
+ERASE, 47507986493440, 47507986493440,
+STORE, 47507986493440, 47507986518015,
+STORE, 47507986518016, 47507988779007,
+STORE, 47507986669568, 47507988779007,
+STORE, 47507986518016, 47507986669567,
+ERASE, 47507986669568, 47507986669568,
+STORE, 47507986669568, 47507988762623,
+STORE, 47507988762624, 47507988779007,
+STORE, 47507988770816, 47507988779007,
+STORE, 47507988762624, 47507988770815,
+ERASE, 47507988762624, 47507988762624,
+STORE, 47507988762624, 47507988770815,
+ERASE, 47507988770816, 47507988770816,
+STORE, 47507988770816, 47507988779007,
+STORE, 47507988779008, 47507988914175,
+ERASE, 47507988779008, 47507988779008,
+STORE, 47507988779008, 47507988803583,
+STORE, 47507988803584, 47507988914175,
+STORE, 47507988865024, 47507988914175,
+STORE, 47507988803584, 47507988865023,
+ERASE, 47507988803584, 47507988803584,
+STORE, 47507988803584, 47507988865023,
+STORE, 47507988889600, 47507988914175,
+STORE, 47507988865024, 47507988889599,
+ERASE, 47507988865024, 47507988865024,
+STORE, 47507988865024, 47507988914175,
+ERASE, 47507988865024, 47507988865024,
+STORE, 47507988865024, 47507988889599,
+STORE, 47507988889600, 47507988914175,
+STORE, 47507988897792, 47507988914175,
+STORE, 47507988889600, 47507988897791,
+ERASE, 47507988889600, 47507988889600,
+STORE, 47507988889600, 47507988897791,
+ERASE, 47507988897792, 47507988897792,
+STORE, 47507988897792, 47507988914175,
+STORE, 47507988897792, 47507988922367,
+STORE, 47507988922368, 47507989086207,
+ERASE, 47507988922368, 47507988922368,
+STORE, 47507988922368, 47507988934655,
+STORE, 47507988934656, 47507989086207,
+STORE, 47507989032960, 47507989086207,
+STORE, 47507988934656, 47507989032959,
+ERASE, 47507988934656, 47507988934656,
+STORE, 47507988934656, 47507989032959,
+STORE, 47507989078016, 47507989086207,
+STORE, 47507989032960, 47507989078015,
+ERASE, 47507989032960, 47507989032960,
+STORE, 47507989032960, 47507989086207,
+ERASE, 47507989032960, 47507989032960,
+STORE, 47507989032960, 47507989078015,
+STORE, 47507989078016, 47507989086207,
+ERASE, 47507989078016, 47507989078016,
+STORE, 47507989078016, 47507989086207,
+STORE, 47507989086208, 47507989684223,
+STORE, 47507989204992, 47507989684223,
+STORE, 47507989086208, 47507989204991,
+ERASE, 47507989204992, 47507989204992,
+STORE, 47507989204992, 47507989630975,
+STORE, 47507989630976, 47507989684223,
+STORE, 47507989520384, 47507989630975,
+STORE, 47507989204992, 47507989520383,
+ERASE, 47507989204992, 47507989204992,
+STORE, 47507989204992, 47507989520383,
+STORE, 47507989626880, 47507989630975,
+STORE, 47507989520384, 47507989626879,
+ERASE, 47507989520384, 47507989520384,
+STORE, 47507989520384, 47507989626879,
+ERASE, 47507989630976, 47507989630976,
+STORE, 47507989630976, 47507989684223,
+STORE, 47507989684224, 47507992735743,
+STORE, 47507990228992, 47507992735743,
+STORE, 47507989684224, 47507990228991,
+ERASE, 47507990228992, 47507990228992,
+STORE, 47507990228992, 47507992514559,
+STORE, 47507992514560, 47507992735743,
+STORE, 47507991924736, 47507992514559,
+STORE, 47507990228992, 47507991924735,
+ERASE, 47507990228992, 47507990228992,
+STORE, 47507990228992, 47507991924735,
+STORE, 47507992510464, 47507992514559,
+STORE, 47507991924736, 47507992510463,
+ERASE, 47507991924736, 47507991924736,
+STORE, 47507991924736, 47507992510463,
+STORE, 47507992719360, 47507992735743,
+STORE, 47507992514560, 47507992719359,
+ERASE, 47507992514560, 47507992514560,
+STORE, 47507992514560, 47507992719359,
+ERASE, 47507992719360, 47507992719360,
+STORE, 47507992719360, 47507992735743,
+STORE, 47507992735744, 47507992768511,
+ERASE, 47507992735744, 47507992735744,
+STORE, 47507992735744, 47507992743935,
+STORE, 47507992743936, 47507992768511,
+STORE, 47507992756224, 47507992768511,
+STORE, 47507992743936, 47507992756223,
+ERASE, 47507992743936, 47507992743936,
+STORE, 47507992743936, 47507992756223,
+STORE, 47507992760320, 47507992768511,
+STORE, 47507992756224, 47507992760319,
+ERASE, 47507992756224, 47507992756224,
+STORE, 47507992756224, 47507992768511,
+ERASE, 47507992756224, 47507992756224,
+STORE, 47507992756224, 47507992760319,
+STORE, 47507992760320, 47507992768511,
+ERASE, 47507992760320, 47507992760320,
+STORE, 47507992760320, 47507992768511,
+STORE, 47507992768512, 47507992805375,
+ERASE, 47507992768512, 47507992768512,
+STORE, 47507992768512, 47507992776703,
+STORE, 47507992776704, 47507992805375,
+STORE, 47507992793088, 47507992805375,
+STORE, 47507992776704, 47507992793087,
+ERASE, 47507992776704, 47507992776704,
+STORE, 47507992776704, 47507992793087,
+STORE, 47507992797184, 47507992805375,
+STORE, 47507992793088, 47507992797183,
+ERASE, 47507992793088, 47507992793088,
+STORE, 47507992793088, 47507992805375,
+ERASE, 47507992793088, 47507992793088,
+STORE, 47507992793088, 47507992797183,
+STORE, 47507992797184, 47507992805375,
+ERASE, 47507992797184, 47507992797184,
+STORE, 47507992797184, 47507992805375,
+STORE, 47507992805376, 47507993280511,
+ERASE, 47507992805376, 47507992805376,
+STORE, 47507992805376, 47507992813567,
+STORE, 47507992813568, 47507993280511,
+STORE, 47507993149440, 47507993280511,
+STORE, 47507992813568, 47507993149439,
+ERASE, 47507992813568, 47507992813568,
+STORE, 47507992813568, 47507993149439,
+STORE, 47507993272320, 47507993280511,
+STORE, 47507993149440, 47507993272319,
+ERASE, 47507993149440, 47507993149440,
+STORE, 47507993149440, 47507993280511,
+ERASE, 47507993149440, 47507993149440,
+STORE, 47507993149440, 47507993272319,
+STORE, 47507993272320, 47507993280511,
+ERASE, 47507993272320, 47507993272320,
+STORE, 47507993272320, 47507993280511,
+STORE, 47507993280512, 47507993288703,
+STORE, 47507993288704, 47507993309183,
+ERASE, 47507993288704, 47507993288704,
+STORE, 47507993288704, 47507993292799,
+STORE, 47507993292800, 47507993309183,
+STORE, 47507993296896, 47507993309183,
+STORE, 47507993292800, 47507993296895,
+ERASE, 47507993292800, 47507993292800,
+STORE, 47507993292800, 47507993296895,
+STORE, 47507993300992, 47507993309183,
+STORE, 47507993296896, 47507993300991,
+ERASE, 47507993296896, 47507993296896,
+STORE, 47507993296896, 47507993309183,
+ERASE, 47507993296896, 47507993296896,
+STORE, 47507993296896, 47507993300991,
+STORE, 47507993300992, 47507993309183,
+ERASE, 47507993300992, 47507993300992,
+STORE, 47507993300992, 47507993309183,
+STORE, 47507993309184, 47507993317375,
+ERASE, 47507985973248, 47507985973248,
+STORE, 47507985973248, 47507985989631,
+STORE, 47507985989632, 47507985997823,
+ERASE, 47507993300992, 47507993300992,
+STORE, 47507993300992, 47507993305087,
+STORE, 47507993305088, 47507993309183,
+ERASE, 47507988889600, 47507988889600,
+STORE, 47507988889600, 47507988893695,
+STORE, 47507988893696, 47507988897791,
+ERASE, 47507993272320, 47507993272320,
+STORE, 47507993272320, 47507993276415,
+STORE, 47507993276416, 47507993280511,
+ERASE, 47507992797184, 47507992797184,
+STORE, 47507992797184, 47507992801279,
+STORE, 47507992801280, 47507992805375,
+ERASE, 47507992760320, 47507992760320,
+STORE, 47507992760320, 47507992764415,
+STORE, 47507992764416, 47507992768511,
+ERASE, 47507992514560, 47507992514560,
+STORE, 47507992514560, 47507992711167,
+STORE, 47507992711168, 47507992719359,
+ERASE, 47507989630976, 47507989630976,
+STORE, 47507989630976, 47507989667839,
+STORE, 47507989667840, 47507989684223,
+ERASE, 47507989078016, 47507989078016,
+STORE, 47507989078016, 47507989082111,
+STORE, 47507989082112, 47507989086207,
+ERASE, 47507988762624, 47507988762624,
+STORE, 47507988762624, 47507988766719,
+STORE, 47507988766720, 47507988770815,
+ERASE, 47507986493440, 47507986493440,
+STORE, 47507986493440, 47507986513919,
+STORE, 47507986513920, 47507986518015,
+ERASE, 47507986161664, 47507986161664,
+STORE, 47507986161664, 47507986165759,
+STORE, 47507986165760, 47507986169855,
+ERASE, 47507986116608, 47507986116608,
+STORE, 47507986116608, 47507986120703,
+STORE, 47507986120704, 47507986124799,
+ERASE, 94386579570688, 94386579570688,
+STORE, 94386579570688, 94386579693567,
+STORE, 94386579693568, 94386579697663,
+ERASE, 140124810997760, 140124810997760,
+STORE, 140124810997760, 140124811001855,
+STORE, 140124811001856, 140124811005951,
+ERASE, 47507984158720, 47507984158720,
+STORE, 94386583982080, 94386584117247,
+STORE, 94386583982080, 94386584256511,
+ERASE, 94386583982080, 94386583982080,
+STORE, 94386583982080, 94386584223743,
+STORE, 94386584223744, 94386584256511,
+ERASE, 94386584223744, 94386584223744,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733763395584, 140737488351231,
+ERASE, 140733763395584, 140733763395584,
+STORE, 140733763395584, 140733763399679,
+STORE, 94011546472448, 94011547152383,
+ERASE, 94011546472448, 94011546472448,
+STORE, 94011546472448, 94011546537983,
+STORE, 94011546537984, 94011547152383,
+ERASE, 94011546537984, 94011546537984,
+STORE, 94011546537984, 94011546886143,
+STORE, 94011546886144, 94011547025407,
+STORE, 94011547025408, 94011547152383,
+STORE, 139757597949952, 139757598121983,
+ERASE, 139757597949952, 139757597949952,
+STORE, 139757597949952, 139757597954047,
+STORE, 139757597954048, 139757598121983,
+ERASE, 139757597954048, 139757597954048,
+STORE, 139757597954048, 139757598076927,
+STORE, 139757598076928, 139757598109695,
+STORE, 139757598109696, 139757598117887,
+STORE, 139757598117888, 139757598121983,
+STORE, 140733763596288, 140733763600383,
+STORE, 140733763584000, 140733763596287,
+STORE, 47875197046784, 47875197054975,
+STORE, 47875197054976, 47875197063167,
+STORE, 47875197063168, 47875198902271,
+STORE, 47875197202432, 47875198902271,
+STORE, 47875197063168, 47875197202431,
+ERASE, 47875197202432, 47875197202432,
+STORE, 47875197202432, 47875198861311,
+STORE, 47875198861312, 47875198902271,
+STORE, 47875198545920, 47875198861311,
+STORE, 47875197202432, 47875198545919,
+ERASE, 47875197202432, 47875197202432,
+STORE, 47875197202432, 47875198545919,
+STORE, 47875198857216, 47875198861311,
+STORE, 47875198545920, 47875198857215,
+ERASE, 47875198545920, 47875198545920,
+STORE, 47875198545920, 47875198857215,
+STORE, 47875198885888, 47875198902271,
+STORE, 47875198861312, 47875198885887,
+ERASE, 47875198861312, 47875198861312,
+STORE, 47875198861312, 47875198885887,
+ERASE, 47875198885888, 47875198885888,
+STORE, 47875198885888, 47875198902271,
+STORE, 47875198902272, 47875199012863,
+STORE, 47875198918656, 47875199012863,
+STORE, 47875198902272, 47875198918655,
+ERASE, 47875198918656, 47875198918656,
+STORE, 47875198918656, 47875199004671,
+STORE, 47875199004672, 47875199012863,
+STORE, 47875198980096, 47875199004671,
+STORE, 47875198918656, 47875198980095,
+ERASE, 47875198918656, 47875198918656,
+STORE, 47875198918656, 47875198980095,
+STORE, 47875199000576, 47875199004671,
+STORE, 47875198980096, 47875199000575,
+ERASE, 47875198980096, 47875198980096,
+STORE, 47875198980096, 47875199000575,
+ERASE, 47875199004672, 47875199004672,
+STORE, 47875199004672, 47875199012863,
+STORE, 47875199012864, 47875199057919,
+ERASE, 47875199012864, 47875199012864,
+STORE, 47875199012864, 47875199021055,
+STORE, 47875199021056, 47875199057919,
+STORE, 47875199041536, 47875199057919,
+STORE, 47875199021056, 47875199041535,
+ERASE, 47875199021056, 47875199021056,
+STORE, 47875199021056, 47875199041535,
+STORE, 47875199049728, 47875199057919,
+STORE, 47875199041536, 47875199049727,
+ERASE, 47875199041536, 47875199041536,
+STORE, 47875199041536, 47875199057919,
+ERASE, 47875199041536, 47875199041536,
+STORE, 47875199041536, 47875199049727,
+STORE, 47875199049728, 47875199057919,
+ERASE, 47875199049728, 47875199049728,
+STORE, 47875199049728, 47875199057919,
+STORE, 47875199057920, 47875199406079,
+STORE, 47875199098880, 47875199406079,
+STORE, 47875199057920, 47875199098879,
+ERASE, 47875199098880, 47875199098880,
+STORE, 47875199098880, 47875199381503,
+STORE, 47875199381504, 47875199406079,
+STORE, 47875199311872, 47875199381503,
+STORE, 47875199098880, 47875199311871,
+ERASE, 47875199098880, 47875199098880,
+STORE, 47875199098880, 47875199311871,
+STORE, 47875199377408, 47875199381503,
+STORE, 47875199311872, 47875199377407,
+ERASE, 47875199311872, 47875199311872,
+STORE, 47875199311872, 47875199377407,
+ERASE, 47875199381504, 47875199381504,
+STORE, 47875199381504, 47875199406079,
+STORE, 47875199406080, 47875201667071,
+STORE, 47875199557632, 47875201667071,
+STORE, 47875199406080, 47875199557631,
+ERASE, 47875199557632, 47875199557632,
+STORE, 47875199557632, 47875201650687,
+STORE, 47875201650688, 47875201667071,
+STORE, 47875201658880, 47875201667071,
+STORE, 47875201650688, 47875201658879,
+ERASE, 47875201650688, 47875201650688,
+STORE, 47875201650688, 47875201658879,
+ERASE, 47875201658880, 47875201658880,
+STORE, 47875201658880, 47875201667071,
+STORE, 47875201667072, 47875201802239,
+ERASE, 47875201667072, 47875201667072,
+STORE, 47875201667072, 47875201691647,
+STORE, 47875201691648, 47875201802239,
+STORE, 47875201753088, 47875201802239,
+STORE, 47875201691648, 47875201753087,
+ERASE, 47875201691648, 47875201691648,
+STORE, 47875201691648, 47875201753087,
+STORE, 47875201777664, 47875201802239,
+STORE, 47875201753088, 47875201777663,
+ERASE, 47875201753088, 47875201753088,
+STORE, 47875201753088, 47875201802239,
+ERASE, 47875201753088, 47875201753088,
+STORE, 47875201753088, 47875201777663,
+STORE, 47875201777664, 47875201802239,
+STORE, 47875201785856, 47875201802239,
+STORE, 47875201777664, 47875201785855,
+ERASE, 47875201777664, 47875201777664,
+STORE, 47875201777664, 47875201785855,
+ERASE, 47875201785856, 47875201785856,
+STORE, 47875201785856, 47875201802239,
+STORE, 47875201785856, 47875201810431,
+STORE, 47875201810432, 47875201974271,
+ERASE, 47875201810432, 47875201810432,
+STORE, 47875201810432, 47875201822719,
+STORE, 47875201822720, 47875201974271,
+STORE, 47875201921024, 47875201974271,
+STORE, 47875201822720, 47875201921023,
+ERASE, 47875201822720, 47875201822720,
+STORE, 47875201822720, 47875201921023,
+STORE, 47875201966080, 47875201974271,
+STORE, 47875201921024, 47875201966079,
+ERASE, 47875201921024, 47875201921024,
+STORE, 47875201921024, 47875201974271,
+ERASE, 47875201921024, 47875201921024,
+STORE, 47875201921024, 47875201966079,
+STORE, 47875201966080, 47875201974271,
+ERASE, 47875201966080, 47875201966080,
+STORE, 47875201966080, 47875201974271,
+STORE, 47875201974272, 47875202572287,
+STORE, 47875202093056, 47875202572287,
+STORE, 47875201974272, 47875202093055,
+ERASE, 47875202093056, 47875202093056,
+STORE, 47875202093056, 47875202519039,
+STORE, 47875202519040, 47875202572287,
+STORE, 47875202408448, 47875202519039,
+STORE, 47875202093056, 47875202408447,
+ERASE, 47875202093056, 47875202093056,
+STORE, 47875202093056, 47875202408447,
+STORE, 47875202514944, 47875202519039,
+STORE, 47875202408448, 47875202514943,
+ERASE, 47875202408448, 47875202408448,
+STORE, 47875202408448, 47875202514943,
+ERASE, 47875202519040, 47875202519040,
+STORE, 47875202519040, 47875202572287,
+STORE, 47875202572288, 47875205623807,
+STORE, 47875203117056, 47875205623807,
+STORE, 47875202572288, 47875203117055,
+ERASE, 47875203117056, 47875203117056,
+STORE, 47875203117056, 47875205402623,
+STORE, 47875205402624, 47875205623807,
+STORE, 47875204812800, 47875205402623,
+STORE, 47875203117056, 47875204812799,
+ERASE, 47875203117056, 47875203117056,
+STORE, 47875203117056, 47875204812799,
+STORE, 47875205398528, 47875205402623,
+STORE, 47875204812800, 47875205398527,
+ERASE, 47875204812800, 47875204812800,
+STORE, 47875204812800, 47875205398527,
+STORE, 47875205607424, 47875205623807,
+STORE, 47875205402624, 47875205607423,
+ERASE, 47875205402624, 47875205402624,
+STORE, 47875205402624, 47875205607423,
+ERASE, 47875205607424, 47875205607424,
+STORE, 47875205607424, 47875205623807,
+STORE, 47875205623808, 47875205656575,
+ERASE, 47875205623808, 47875205623808,
+STORE, 47875205623808, 47875205631999,
+STORE, 47875205632000, 47875205656575,
+STORE, 47875205644288, 47875205656575,
+STORE, 47875205632000, 47875205644287,
+ERASE, 47875205632000, 47875205632000,
+STORE, 47875205632000, 47875205644287,
+STORE, 47875205648384, 47875205656575,
+STORE, 47875205644288, 47875205648383,
+ERASE, 47875205644288, 47875205644288,
+STORE, 47875205644288, 47875205656575,
+ERASE, 47875205644288, 47875205644288,
+STORE, 47875205644288, 47875205648383,
+STORE, 47875205648384, 47875205656575,
+ERASE, 47875205648384, 47875205648384,
+STORE, 47875205648384, 47875205656575,
+STORE, 47875205656576, 47875205693439,
+ERASE, 47875205656576, 47875205656576,
+STORE, 47875205656576, 47875205664767,
+STORE, 47875205664768, 47875205693439,
+STORE, 47875205681152, 47875205693439,
+STORE, 47875205664768, 47875205681151,
+ERASE, 47875205664768, 47875205664768,
+STORE, 47875205664768, 47875205681151,
+STORE, 47875205685248, 47875205693439,
+STORE, 47875205681152, 47875205685247,
+ERASE, 47875205681152, 47875205681152,
+STORE, 47875205681152, 47875205693439,
+ERASE, 47875205681152, 47875205681152,
+STORE, 47875205681152, 47875205685247,
+STORE, 47875205685248, 47875205693439,
+ERASE, 47875205685248, 47875205685248,
+STORE, 47875205685248, 47875205693439,
+STORE, 47875205693440, 47875206168575,
+ERASE, 47875205693440, 47875205693440,
+STORE, 47875205693440, 47875205701631,
+STORE, 47875205701632, 47875206168575,
+STORE, 47875206037504, 47875206168575,
+STORE, 47875205701632, 47875206037503,
+ERASE, 47875205701632, 47875205701632,
+STORE, 47875205701632, 47875206037503,
+STORE, 47875206160384, 47875206168575,
+STORE, 47875206037504, 47875206160383,
+ERASE, 47875206037504, 47875206037504,
+STORE, 47875206037504, 47875206168575,
+ERASE, 47875206037504, 47875206037504,
+STORE, 47875206037504, 47875206160383,
+STORE, 47875206160384, 47875206168575,
+ERASE, 47875206160384, 47875206160384,
+STORE, 47875206160384, 47875206168575,
+STORE, 47875206168576, 47875206176767,
+STORE, 47875206176768, 47875206197247,
+ERASE, 47875206176768, 47875206176768,
+STORE, 47875206176768, 47875206180863,
+STORE, 47875206180864, 47875206197247,
+STORE, 47875206184960, 47875206197247,
+STORE, 47875206180864, 47875206184959,
+ERASE, 47875206180864, 47875206180864,
+STORE, 47875206180864, 47875206184959,
+STORE, 47875206189056, 47875206197247,
+STORE, 47875206184960, 47875206189055,
+ERASE, 47875206184960, 47875206184960,
+STORE, 47875206184960, 47875206197247,
+ERASE, 47875206184960, 47875206184960,
+STORE, 47875206184960, 47875206189055,
+STORE, 47875206189056, 47875206197247,
+ERASE, 47875206189056, 47875206189056,
+STORE, 47875206189056, 47875206197247,
+STORE, 47875206197248, 47875206205439,
+ERASE, 47875198861312, 47875198861312,
+STORE, 47875198861312, 47875198877695,
+STORE, 47875198877696, 47875198885887,
+ERASE, 47875206189056, 47875206189056,
+STORE, 47875206189056, 47875206193151,
+STORE, 47875206193152, 47875206197247,
+ERASE, 47875201777664, 47875201777664,
+STORE, 47875201777664, 47875201781759,
+STORE, 47875201781760, 47875201785855,
+ERASE, 47875206160384, 47875206160384,
+STORE, 47875206160384, 47875206164479,
+STORE, 47875206164480, 47875206168575,
+ERASE, 47875205685248, 47875205685248,
+STORE, 47875205685248, 47875205689343,
+STORE, 47875205689344, 47875205693439,
+ERASE, 47875205648384, 47875205648384,
+STORE, 47875205648384, 47875205652479,
+STORE, 47875205652480, 47875205656575,
+ERASE, 47875205402624, 47875205402624,
+STORE, 47875205402624, 47875205599231,
+STORE, 47875205599232, 47875205607423,
+ERASE, 47875202519040, 47875202519040,
+STORE, 47875202519040, 47875202555903,
+STORE, 47875202555904, 47875202572287,
+ERASE, 47875201966080, 47875201966080,
+STORE, 47875201966080, 47875201970175,
+STORE, 47875201970176, 47875201974271,
+ERASE, 47875201650688, 47875201650688,
+STORE, 47875201650688, 47875201654783,
+STORE, 47875201654784, 47875201658879,
+ERASE, 47875199381504, 47875199381504,
+STORE, 47875199381504, 47875199401983,
+STORE, 47875199401984, 47875199406079,
+ERASE, 47875199049728, 47875199049728,
+STORE, 47875199049728, 47875199053823,
+STORE, 47875199053824, 47875199057919,
+ERASE, 47875199004672, 47875199004672,
+STORE, 47875199004672, 47875199008767,
+STORE, 47875199008768, 47875199012863,
+ERASE, 94011547025408, 94011547025408,
+STORE, 94011547025408, 94011547148287,
+STORE, 94011547148288, 94011547152383,
+ERASE, 139757598109696, 139757598109696,
+STORE, 139757598109696, 139757598113791,
+STORE, 139757598113792, 139757598117887,
+ERASE, 47875197046784, 47875197046784,
+STORE, 94011557584896, 94011557720063,
+STORE, 94011557584896, 94011557855231,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557851135,
+STORE, 94011557851136, 94011557855231,
+ERASE, 94011557851136, 94011557851136,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557847039,
+STORE, 94011557847040, 94011557851135,
+ERASE, 94011557847040, 94011557847040,
+STORE, 94011557584896, 94011557982207,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557978111,
+STORE, 94011557978112, 94011557982207,
+ERASE, 94011557978112, 94011557978112,
+ERASE, 94011557584896, 94011557584896,
+STORE, 94011557584896, 94011557974015,
+STORE, 94011557974016, 94011557978111,
+ERASE, 94011557974016, 94011557974016,
+STORE, 140737488347136, 140737488351231,
+STORE, 140734130360320, 140737488351231,
+ERASE, 140734130360320, 140734130360320,
+STORE, 140734130360320, 140734130364415,
+STORE, 94641232105472, 94641232785407,
+ERASE, 94641232105472, 94641232105472,
+STORE, 94641232105472, 94641232171007,
+STORE, 94641232171008, 94641232785407,
+ERASE, 94641232171008, 94641232171008,
+STORE, 94641232171008, 94641232519167,
+STORE, 94641232519168, 94641232658431,
+STORE, 94641232658432, 94641232785407,
+STORE, 139726599516160, 139726599688191,
+ERASE, 139726599516160, 139726599516160,
+STORE, 139726599516160, 139726599520255,
+STORE, 139726599520256, 139726599688191,
+ERASE, 139726599520256, 139726599520256,
+STORE, 139726599520256, 139726599643135,
+STORE, 139726599643136, 139726599675903,
+STORE, 139726599675904, 139726599684095,
+STORE, 139726599684096, 139726599688191,
+STORE, 140734130446336, 140734130450431,
+STORE, 140734130434048, 140734130446335,
+STORE, 47906195480576, 47906195488767,
+STORE, 47906195488768, 47906195496959,
+STORE, 47906195496960, 47906197336063,
+STORE, 47906195636224, 47906197336063,
+STORE, 47906195496960, 47906195636223,
+ERASE, 47906195636224, 47906195636224,
+STORE, 47906195636224, 47906197295103,
+STORE, 47906197295104, 47906197336063,
+STORE, 47906196979712, 47906197295103,
+STORE, 47906195636224, 47906196979711,
+ERASE, 47906195636224, 47906195636224,
+STORE, 47906195636224, 47906196979711,
+STORE, 47906197291008, 47906197295103,
+STORE, 47906196979712, 47906197291007,
+ERASE, 47906196979712, 47906196979712,
+STORE, 47906196979712, 47906197291007,
+STORE, 47906197319680, 47906197336063,
+STORE, 47906197295104, 47906197319679,
+ERASE, 47906197295104, 47906197295104,
+STORE, 47906197295104, 47906197319679,
+ERASE, 47906197319680, 47906197319680,
+STORE, 47906197319680, 47906197336063,
+STORE, 47906197336064, 47906197446655,
+STORE, 47906197352448, 47906197446655,
+STORE, 47906197336064, 47906197352447,
+ERASE, 47906197352448, 47906197352448,
+STORE, 47906197352448, 47906197438463,
+STORE, 47906197438464, 47906197446655,
+STORE, 47906197413888, 47906197438463,
+STORE, 47906197352448, 47906197413887,
+ERASE, 47906197352448, 47906197352448,
+STORE, 47906197352448, 47906197413887,
+STORE, 47906197434368, 47906197438463,
+STORE, 47906197413888, 47906197434367,
+ERASE, 47906197413888, 47906197413888,
+STORE, 47906197413888, 47906197434367,
+ERASE, 47906197438464, 47906197438464,
+STORE, 47906197438464, 47906197446655,
+STORE, 47906197446656, 47906197491711,
+ERASE, 47906197446656, 47906197446656,
+STORE, 47906197446656, 47906197454847,
+STORE, 47906197454848, 47906197491711,
+STORE, 47906197475328, 47906197491711,
+STORE, 47906197454848, 47906197475327,
+ERASE, 47906197454848, 47906197454848,
+STORE, 47906197454848, 47906197475327,
+STORE, 47906197483520, 47906197491711,
+STORE, 47906197475328, 47906197483519,
+ERASE, 47906197475328, 47906197475328,
+STORE, 47906197475328, 47906197491711,
+ERASE, 47906197475328, 47906197475328,
+STORE, 47906197475328, 47906197483519,
+STORE, 47906197483520, 47906197491711,
+ERASE, 47906197483520, 47906197483520,
+STORE, 47906197483520, 47906197491711,
+STORE, 47906197491712, 47906197839871,
+STORE, 47906197532672, 47906197839871,
+STORE, 47906197491712, 47906197532671,
+ERASE, 47906197532672, 47906197532672,
+STORE, 47906197532672, 47906197815295,
+STORE, 47906197815296, 47906197839871,
+STORE, 47906197745664, 47906197815295,
+STORE, 47906197532672, 47906197745663,
+ERASE, 47906197532672, 47906197532672,
+STORE, 47906197532672, 47906197745663,
+STORE, 47906197811200, 47906197815295,
+STORE, 47906197745664, 47906197811199,
+ERASE, 47906197745664, 47906197745664,
+STORE, 47906197745664, 47906197811199,
+ERASE, 47906197815296, 47906197815296,
+STORE, 47906197815296, 47906197839871,
+STORE, 47906197839872, 47906200100863,
+STORE, 47906197991424, 47906200100863,
+STORE, 47906197839872, 47906197991423,
+ERASE, 47906197991424, 47906197991424,
+STORE, 47906197991424, 47906200084479,
+STORE, 47906200084480, 47906200100863,
+STORE, 47906200092672, 47906200100863,
+STORE, 47906200084480, 47906200092671,
+ERASE, 47906200084480, 47906200084480,
+STORE, 47906200084480, 47906200092671,
+ERASE, 47906200092672, 47906200092672,
+STORE, 47906200092672, 47906200100863,
+STORE, 47906200100864, 47906200236031,
+ERASE, 47906200100864, 47906200100864,
+STORE, 47906200100864, 47906200125439,
+STORE, 47906200125440, 47906200236031,
+STORE, 47906200186880, 47906200236031,
+STORE, 47906200125440, 47906200186879,
+ERASE, 47906200125440, 47906200125440,
+STORE, 47906200125440, 47906200186879,
+STORE, 47906200211456, 47906200236031,
+STORE, 47906200186880, 47906200211455,
+ERASE, 47906200186880, 47906200186880,
+STORE, 47906200186880, 47906200236031,
+ERASE, 47906200186880, 47906200186880,
+STORE, 47906200186880, 47906200211455,
+STORE, 47906200211456, 47906200236031,
+STORE, 47906200219648, 47906200236031,
+STORE, 47906200211456, 47906200219647,
+ERASE, 47906200211456, 47906200211456,
+STORE, 47906200211456, 47906200219647,
+ERASE, 47906200219648, 47906200219648,
+STORE, 47906200219648, 47906200236031,
+STORE, 47906200219648, 47906200244223,
+STORE, 47906200244224, 47906200408063,
+ERASE, 47906200244224, 47906200244224,
+STORE, 47906200244224, 47906200256511,
+STORE, 47906200256512, 47906200408063,
+STORE, 47906200354816, 47906200408063,
+STORE, 47906200256512, 47906200354815,
+ERASE, 47906200256512, 47906200256512,
+STORE, 47906200256512, 47906200354815,
+STORE, 47906200399872, 47906200408063,
+STORE, 47906200354816, 47906200399871,
+ERASE, 47906200354816, 47906200354816,
+STORE, 47906200354816, 47906200408063,
+ERASE, 47906200354816, 47906200354816,
+STORE, 47906200354816, 47906200399871,
+STORE, 47906200399872, 47906200408063,
+ERASE, 47906200399872, 47906200399872,
+STORE, 47906200399872, 47906200408063,
+STORE, 47906200408064, 47906201006079,
+STORE, 47906200526848, 47906201006079,
+STORE, 47906200408064, 47906200526847,
+ERASE, 47906200526848, 47906200526848,
+STORE, 47906200526848, 47906200952831,
+STORE, 47906200952832, 47906201006079,
+STORE, 47906200842240, 47906200952831,
+STORE, 47906200526848, 47906200842239,
+ERASE, 47906200526848, 47906200526848,
+STORE, 47906200526848, 47906200842239,
+STORE, 47906200948736, 47906200952831,
+STORE, 47906200842240, 47906200948735,
+ERASE, 47906200842240, 47906200842240,
+STORE, 47906200842240, 47906200948735,
+ERASE, 47906200952832, 47906200952832,
+STORE, 47906200952832, 47906201006079,
+STORE, 47906201006080, 47906204057599,
+STORE, 47906201550848, 47906204057599,
+STORE, 47906201006080, 47906201550847,
+ERASE, 47906201550848, 47906201550848,
+STORE, 47906201550848, 47906203836415,
+STORE, 47906203836416, 47906204057599,
+STORE, 47906203246592, 47906203836415,
+STORE, 47906201550848, 47906203246591,
+ERASE, 47906201550848, 47906201550848,
+STORE, 47906201550848, 47906203246591,
+STORE, 47906203832320, 47906203836415,
+STORE, 47906203246592, 47906203832319,
+ERASE, 47906203246592, 47906203246592,
+STORE, 47906203246592, 47906203832319,
+STORE, 47906204041216, 47906204057599,
+STORE, 47906203836416, 47906204041215,
+ERASE, 47906203836416, 47906203836416,
+STORE, 47906203836416, 47906204041215,
+ERASE, 47906204041216, 47906204041216,
+STORE, 47906204041216, 47906204057599,
+STORE, 47906204057600, 47906204090367,
+ERASE, 47906204057600, 47906204057600,
+STORE, 47906204057600, 47906204065791,
+STORE, 47906204065792, 47906204090367,
+STORE, 47906204078080, 47906204090367,
+STORE, 47906204065792, 47906204078079,
+ERASE, 47906204065792, 47906204065792,
+STORE, 47906204065792, 47906204078079,
+STORE, 47906204082176, 47906204090367,
+STORE, 47906204078080, 47906204082175,
+ERASE, 47906204078080, 47906204078080,
+STORE, 47906204078080, 47906204090367,
+ERASE, 47906204078080, 47906204078080,
+STORE, 47906204078080, 47906204082175,
+STORE, 47906204082176, 47906204090367,
+ERASE, 47906204082176, 47906204082176,
+STORE, 47906204082176, 47906204090367,
+STORE, 47906204090368, 47906204127231,
+ERASE, 47906204090368, 47906204090368,
+STORE, 47906204090368, 47906204098559,
+STORE, 47906204098560, 47906204127231,
+STORE, 47906204114944, 47906204127231,
+STORE, 47906204098560, 47906204114943,
+ERASE, 47906204098560, 47906204098560,
+STORE, 47906204098560, 47906204114943,
+STORE, 47906204119040, 47906204127231,
+STORE, 47906204114944, 47906204119039,
+ERASE, 47906204114944, 47906204114944,
+STORE, 47906204114944, 47906204127231,
+ERASE, 47906204114944, 47906204114944,
+STORE, 47906204114944, 47906204119039,
+STORE, 47906204119040, 47906204127231,
+ERASE, 47906204119040, 47906204119040,
+STORE, 47906204119040, 47906204127231,
+STORE, 47906204127232, 47906204602367,
+ERASE, 47906204127232, 47906204127232,
+STORE, 47906204127232, 47906204135423,
+STORE, 47906204135424, 47906204602367,
+STORE, 47906204471296, 47906204602367,
+STORE, 47906204135424, 47906204471295,
+ERASE, 47906204135424, 47906204135424,
+STORE, 47906204135424, 47906204471295,
+STORE, 47906204594176, 47906204602367,
+STORE, 47906204471296, 47906204594175,
+ERASE, 47906204471296, 47906204471296,
+STORE, 47906204471296, 47906204602367,
+ERASE, 47906204471296, 47906204471296,
+STORE, 47906204471296, 47906204594175,
+STORE, 47906204594176, 47906204602367,
+ERASE, 47906204594176, 47906204594176,
+STORE, 47906204594176, 47906204602367,
+STORE, 47906204602368, 47906204610559,
+STORE, 47906204610560, 47906204631039,
+ERASE, 47906204610560, 47906204610560,
+STORE, 47906204610560, 47906204614655,
+STORE, 47906204614656, 47906204631039,
+STORE, 47906204618752, 47906204631039,
+STORE, 47906204614656, 47906204618751,
+ERASE, 47906204614656, 47906204614656,
+STORE, 47906204614656, 47906204618751,
+STORE, 47906204622848, 47906204631039,
+STORE, 47906204618752, 47906204622847,
+ERASE, 47906204618752, 47906204618752,
+STORE, 47906204618752, 47906204631039,
+ERASE, 47906204618752, 47906204618752,
+STORE, 47906204618752, 47906204622847,
+STORE, 47906204622848, 47906204631039,
+ERASE, 47906204622848, 47906204622848,
+STORE, 47906204622848, 47906204631039,
+STORE, 47906204631040, 47906204639231,
+ERASE, 47906197295104, 47906197295104,
+STORE, 47906197295104, 47906197311487,
+STORE, 47906197311488, 47906197319679,
+ERASE, 47906204622848, 47906204622848,
+STORE, 47906204622848, 47906204626943,
+STORE, 47906204626944, 47906204631039,
+ERASE, 47906200211456, 47906200211456,
+STORE, 47906200211456, 47906200215551,
+STORE, 47906200215552, 47906200219647,
+ERASE, 47906204594176, 47906204594176,
+STORE, 47906204594176, 47906204598271,
+STORE, 47906204598272, 47906204602367,
+ERASE, 47906204119040, 47906204119040,
+STORE, 47906204119040, 47906204123135,
+STORE, 47906204123136, 47906204127231,
+ERASE, 47906204082176, 47906204082176,
+STORE, 47906204082176, 47906204086271,
+STORE, 47906204086272, 47906204090367,
+ERASE, 47906203836416, 47906203836416,
+STORE, 47906203836416, 47906204033023,
+STORE, 47906204033024, 47906204041215,
+ERASE, 47906200952832, 47906200952832,
+STORE, 47906200952832, 47906200989695,
+STORE, 47906200989696, 47906201006079,
+ERASE, 47906200399872, 47906200399872,
+STORE, 47906200399872, 47906200403967,
+STORE, 47906200403968, 47906200408063,
+ERASE, 47906200084480, 47906200084480,
+STORE, 47906200084480, 47906200088575,
+STORE, 47906200088576, 47906200092671,
+ERASE, 47906197815296, 47906197815296,
+STORE, 47906197815296, 47906197835775,
+STORE, 47906197835776, 47906197839871,
+ERASE, 47906197483520, 47906197483520,
+STORE, 47906197483520, 47906197487615,
+STORE, 47906197487616, 47906197491711,
+ERASE, 47906197438464, 47906197438464,
+STORE, 47906197438464, 47906197442559,
+STORE, 47906197442560, 47906197446655,
+ERASE, 94641232658432, 94641232658432,
+STORE, 94641232658432, 94641232781311,
+STORE, 94641232781312, 94641232785407,
+ERASE, 139726599675904, 139726599675904,
+STORE, 139726599675904, 139726599679999,
+STORE, 139726599680000, 139726599684095,
+ERASE, 47906195480576, 47906195480576,
+STORE, 94641242615808, 94641242750975,
+ };
+ unsigned long set11[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140732658499584, 140737488351231,
+ERASE, 140732658499584, 140732658499584,
+STORE, 140732658499584, 140732658503679,
+STORE, 94029856579584, 94029856751615,
+ERASE, 94029856579584, 94029856579584,
+STORE, 94029856579584, 94029856595967,
+STORE, 94029856595968, 94029856751615,
+ERASE, 94029856595968, 94029856595968,
+STORE, 94029856595968, 94029856698367,
+STORE, 94029856698368, 94029856739327,
+STORE, 94029856739328, 94029856751615,
+STORE, 140014592573440, 140014592745471,
+ERASE, 140014592573440, 140014592573440,
+STORE, 140014592573440, 140014592577535,
+STORE, 140014592577536, 140014592745471,
+ERASE, 140014592577536, 140014592577536,
+STORE, 140014592577536, 140014592700415,
+STORE, 140014592700416, 140014592733183,
+STORE, 140014592733184, 140014592741375,
+STORE, 140014592741376, 140014592745471,
+STORE, 140732658565120, 140732658569215,
+STORE, 140732658552832, 140732658565119,
+ };
+
+ unsigned long set12[] = { /* contains 12 values. */
+STORE, 140737488347136, 140737488351231,
+STORE, 140732658499584, 140737488351231,
+ERASE, 140732658499584, 140732658499584,
+STORE, 140732658499584, 140732658503679,
+STORE, 94029856579584, 94029856751615,
+ERASE, 94029856579584, 94029856579584,
+STORE, 94029856579584, 94029856595967,
+STORE, 94029856595968, 94029856751615,
+ERASE, 94029856595968, 94029856595968,
+STORE, 94029856595968, 94029856698367,
+STORE, 94029856698368, 94029856739327,
+STORE, 94029856739328, 94029856751615,
+STORE, 140014592573440, 140014592745471,
+ERASE, 140014592573440, 140014592573440,
+STORE, 140014592573440, 140014592577535,
+STORE, 140014592577536, 140014592745471,
+ERASE, 140014592577536, 140014592577536,
+STORE, 140014592577536, 140014592700415,
+STORE, 140014592700416, 140014592733183,
+STORE, 140014592733184, 140014592741375,
+STORE, 140014592741376, 140014592745471,
+STORE, 140732658565120, 140732658569215,
+STORE, 140732658552832, 140732658565119,
+STORE, 140014592741375, 140014592741375, /* contrived */
+STORE, 140014592733184, 140014592741376, /* creates first entry retry. */
+ };
+ unsigned long set13[] = {
+STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */
+STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */
+STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */
+STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */
+STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */
+STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */
+STORE, 140373518663680, 140373518684159,/*: deleted (257) */
+STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */
+STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */
+STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */
+STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */
+ };
+ unsigned long set14[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140731667996672, 140737488351231,
+SNULL, 140731668000767, 140737488351231,
+STORE, 140731667996672, 140731668000767,
+STORE, 140731667865600, 140731668000767,
+STORE, 94077521272832, 94077521313791,
+SNULL, 94077521301503, 94077521313791,
+STORE, 94077521272832, 94077521301503,
+STORE, 94077521301504, 94077521313791,
+ERASE, 94077521301504, 94077521313791,
+STORE, 94077521305600, 94077521313791,
+STORE, 139826134630400, 139826136883199,
+SNULL, 139826134773759, 139826136883199,
+STORE, 139826134630400, 139826134773759,
+STORE, 139826134773760, 139826136883199,
+ERASE, 139826134773760, 139826136883199,
+STORE, 139826136870912, 139826136879103,
+STORE, 139826136879104, 139826136883199,
+STORE, 140731668013056, 140731668017151,
+STORE, 140731668000768, 140731668013055,
+STORE, 139826136862720, 139826136870911,
+STORE, 139826132406272, 139826134630399,
+SNULL, 139826134056959, 139826134630399,
+STORE, 139826132406272, 139826134056959,
+STORE, 139826134056960, 139826134630399,
+SNULL, 139826134056960, 139826134626303,
+STORE, 139826134626304, 139826134630399,
+STORE, 139826134056960, 139826134626303,
+ERASE, 139826134056960, 139826134626303,
+STORE, 139826134056960, 139826134626303,
+ERASE, 139826134626304, 139826134630399,
+STORE, 139826134626304, 139826134630399,
+STORE, 139826136842240, 139826136862719,
+STORE, 139826130022400, 139826132406271,
+SNULL, 139826130022400, 139826130288639,
+STORE, 139826130288640, 139826132406271,
+STORE, 139826130022400, 139826130288639,
+SNULL, 139826132381695, 139826132406271,
+STORE, 139826130288640, 139826132381695,
+STORE, 139826132381696, 139826132406271,
+SNULL, 139826132381696, 139826132402175,
+STORE, 139826132402176, 139826132406271,
+STORE, 139826132381696, 139826132402175,
+ERASE, 139826132381696, 139826132402175,
+STORE, 139826132381696, 139826132402175,
+ERASE, 139826132402176, 139826132406271,
+STORE, 139826132402176, 139826132406271,
+STORE, 139826127806464, 139826130022399,
+SNULL, 139826127806464, 139826127904767,
+STORE, 139826127904768, 139826130022399,
+STORE, 139826127806464, 139826127904767,
+SNULL, 139826129997823, 139826130022399,
+STORE, 139826127904768, 139826129997823,
+STORE, 139826129997824, 139826130022399,
+SNULL, 139826129997824, 139826130006015,
+STORE, 139826130006016, 139826130022399,
+STORE, 139826129997824, 139826130006015,
+ERASE, 139826129997824, 139826130006015,
+STORE, 139826129997824, 139826130006015,
+ERASE, 139826130006016, 139826130022399,
+STORE, 139826130006016, 139826130022399,
+STORE, 139826124009472, 139826127806463,
+SNULL, 139826124009472, 139826125668351,
+STORE, 139826125668352, 139826127806463,
+STORE, 139826124009472, 139826125668351,
+SNULL, 139826127765503, 139826127806463,
+STORE, 139826125668352, 139826127765503,
+STORE, 139826127765504, 139826127806463,
+SNULL, 139826127765504, 139826127790079,
+STORE, 139826127790080, 139826127806463,
+STORE, 139826127765504, 139826127790079,
+ERASE, 139826127765504, 139826127790079,
+STORE, 139826127765504, 139826127790079,
+ERASE, 139826127790080, 139826127806463,
+STORE, 139826127790080, 139826127806463,
+STORE, 139826121748480, 139826124009471,
+SNULL, 139826121748480, 139826121900031,
+STORE, 139826121900032, 139826124009471,
+STORE, 139826121748480, 139826121900031,
+SNULL, 139826123993087, 139826124009471,
+STORE, 139826121900032, 139826123993087,
+STORE, 139826123993088, 139826124009471,
+SNULL, 139826123993088, 139826124001279,
+STORE, 139826124001280, 139826124009471,
+STORE, 139826123993088, 139826124001279,
+ERASE, 139826123993088, 139826124001279,
+STORE, 139826123993088, 139826124001279,
+ERASE, 139826124001280, 139826124009471,
+STORE, 139826124001280, 139826124009471,
+STORE, 139826119626752, 139826121748479,
+SNULL, 139826119626752, 139826119643135,
+STORE, 139826119643136, 139826121748479,
+STORE, 139826119626752, 139826119643135,
+SNULL, 139826121740287, 139826121748479,
+STORE, 139826119643136, 139826121740287,
+STORE, 139826121740288, 139826121748479,
+ERASE, 139826121740288, 139826121748479,
+STORE, 139826121740288, 139826121748479,
+STORE, 139826136834048, 139826136842239,
+STORE, 139826117496832, 139826119626751,
+SNULL, 139826117496832, 139826117525503,
+STORE, 139826117525504, 139826119626751,
+STORE, 139826117496832, 139826117525503,
+SNULL, 139826119618559, 139826119626751,
+STORE, 139826117525504, 139826119618559,
+STORE, 139826119618560, 139826119626751,
+ERASE, 139826119618560, 139826119626751,
+STORE, 139826119618560, 139826119626751,
+STORE, 139826115244032, 139826117496831,
+SNULL, 139826115244032, 139826115395583,
+STORE, 139826115395584, 139826117496831,
+STORE, 139826115244032, 139826115395583,
+SNULL, 139826117488639, 139826117496831,
+STORE, 139826115395584, 139826117488639,
+STORE, 139826117488640, 139826117496831,
+ERASE, 139826117488640, 139826117496831,
+STORE, 139826117488640, 139826117496831,
+STORE, 139826113073152, 139826115244031,
+SNULL, 139826113073152, 139826113142783,
+STORE, 139826113142784, 139826115244031,
+STORE, 139826113073152, 139826113142783,
+SNULL, 139826115235839, 139826115244031,
+STORE, 139826113142784, 139826115235839,
+STORE, 139826115235840, 139826115244031,
+ERASE, 139826115235840, 139826115244031,
+STORE, 139826115235840, 139826115244031,
+STORE, 139826109861888, 139826113073151,
+SNULL, 139826109861888, 139826110939135,
+STORE, 139826110939136, 139826113073151,
+STORE, 139826109861888, 139826110939135,
+SNULL, 139826113036287, 139826113073151,
+STORE, 139826110939136, 139826113036287,
+STORE, 139826113036288, 139826113073151,
+ERASE, 139826113036288, 139826113073151,
+STORE, 139826113036288, 139826113073151,
+STORE, 139826107727872, 139826109861887,
+SNULL, 139826107727872, 139826107756543,
+STORE, 139826107756544, 139826109861887,
+STORE, 139826107727872, 139826107756543,
+SNULL, 139826109853695, 139826109861887,
+STORE, 139826107756544, 139826109853695,
+STORE, 139826109853696, 139826109861887,
+ERASE, 139826109853696, 139826109861887,
+STORE, 139826109853696, 139826109861887,
+STORE, 139826105417728, 139826107727871,
+SNULL, 139826105417728, 139826105622527,
+STORE, 139826105622528, 139826107727871,
+STORE, 139826105417728, 139826105622527,
+SNULL, 139826107719679, 139826107727871,
+STORE, 139826105622528, 139826107719679,
+STORE, 139826107719680, 139826107727871,
+ERASE, 139826107719680, 139826107727871,
+STORE, 139826107719680, 139826107727871,
+STORE, 139826136825856, 139826136842239,
+STORE, 139826103033856, 139826105417727,
+SNULL, 139826103033856, 139826103226367,
+STORE, 139826103226368, 139826105417727,
+STORE, 139826103033856, 139826103226367,
+SNULL, 139826105319423, 139826105417727,
+STORE, 139826103226368, 139826105319423,
+STORE, 139826105319424, 139826105417727,
+ERASE, 139826105319424, 139826105417727,
+STORE, 139826105319424, 139826105417727,
+STORE, 139826100916224, 139826103033855,
+SNULL, 139826100916224, 139826100932607,
+STORE, 139826100932608, 139826103033855,
+STORE, 139826100916224, 139826100932607,
+SNULL, 139826103025663, 139826103033855,
+STORE, 139826100932608, 139826103025663,
+STORE, 139826103025664, 139826103033855,
+ERASE, 139826103025664, 139826103033855,
+STORE, 139826103025664, 139826103033855,
+STORE, 139826098348032, 139826100916223,
+SNULL, 139826098348032, 139826098814975,
+STORE, 139826098814976, 139826100916223,
+STORE, 139826098348032, 139826098814975,
+SNULL, 139826100908031, 139826100916223,
+STORE, 139826098814976, 139826100908031,
+STORE, 139826100908032, 139826100916223,
+ERASE, 139826100908032, 139826100916223,
+STORE, 139826100908032, 139826100916223,
+STORE, 139826096234496, 139826098348031,
+SNULL, 139826096234496, 139826096246783,
+STORE, 139826096246784, 139826098348031,
+STORE, 139826096234496, 139826096246783,
+SNULL, 139826098339839, 139826098348031,
+STORE, 139826096246784, 139826098339839,
+STORE, 139826098339840, 139826098348031,
+ERASE, 139826098339840, 139826098348031,
+STORE, 139826098339840, 139826098348031,
+STORE, 139826094055424, 139826096234495,
+SNULL, 139826094055424, 139826094133247,
+STORE, 139826094133248, 139826096234495,
+STORE, 139826094055424, 139826094133247,
+SNULL, 139826096226303, 139826096234495,
+STORE, 139826094133248, 139826096226303,
+STORE, 139826096226304, 139826096234495,
+ERASE, 139826096226304, 139826096234495,
+STORE, 139826096226304, 139826096234495,
+STORE, 139826136817664, 139826136842239,
+STORE, 139826091937792, 139826094055423,
+SNULL, 139826091937792, 139826091954175,
+STORE, 139826091954176, 139826094055423,
+STORE, 139826091937792, 139826091954175,
+SNULL, 139826094047231, 139826094055423,
+STORE, 139826091954176, 139826094047231,
+STORE, 139826094047232, 139826094055423,
+ERASE, 139826094047232, 139826094055423,
+STORE, 139826094047232, 139826094055423,
+STORE, 139826136809472, 139826136842239,
+SNULL, 139826127781887, 139826127790079,
+STORE, 139826127765504, 139826127781887,
+STORE, 139826127781888, 139826127790079,
+SNULL, 139826094051327, 139826094055423,
+STORE, 139826094047232, 139826094051327,
+STORE, 139826094051328, 139826094055423,
+SNULL, 139826096230399, 139826096234495,
+STORE, 139826096226304, 139826096230399,
+STORE, 139826096230400, 139826096234495,
+SNULL, 139826098343935, 139826098348031,
+STORE, 139826098339840, 139826098343935,
+STORE, 139826098343936, 139826098348031,
+SNULL, 139826130001919, 139826130006015,
+STORE, 139826129997824, 139826130001919,
+STORE, 139826130001920, 139826130006015,
+SNULL, 139826100912127, 139826100916223,
+STORE, 139826100908032, 139826100912127,
+STORE, 139826100912128, 139826100916223,
+SNULL, 139826103029759, 139826103033855,
+STORE, 139826103025664, 139826103029759,
+STORE, 139826103029760, 139826103033855,
+SNULL, 139826105413631, 139826105417727,
+STORE, 139826105319424, 139826105413631,
+STORE, 139826105413632, 139826105417727,
+SNULL, 139826107723775, 139826107727871,
+STORE, 139826107719680, 139826107723775,
+STORE, 139826107723776, 139826107727871,
+SNULL, 139826109857791, 139826109861887,
+STORE, 139826109853696, 139826109857791,
+STORE, 139826109857792, 139826109861887,
+SNULL, 139826113044479, 139826113073151,
+STORE, 139826113036288, 139826113044479,
+STORE, 139826113044480, 139826113073151,
+SNULL, 139826115239935, 139826115244031,
+STORE, 139826115235840, 139826115239935,
+STORE, 139826115239936, 139826115244031,
+SNULL, 139826117492735, 139826117496831,
+STORE, 139826117488640, 139826117492735,
+STORE, 139826117492736, 139826117496831,
+SNULL, 139826119622655, 139826119626751,
+STORE, 139826119618560, 139826119622655,
+STORE, 139826119622656, 139826119626751,
+SNULL, 139826121744383, 139826121748479,
+STORE, 139826121740288, 139826121744383,
+STORE, 139826121744384, 139826121748479,
+SNULL, 139826123997183, 139826124001279,
+STORE, 139826123993088, 139826123997183,
+STORE, 139826123997184, 139826124001279,
+SNULL, 139826132398079, 139826132402175,
+STORE, 139826132381696, 139826132398079,
+STORE, 139826132398080, 139826132402175,
+SNULL, 139826134622207, 139826134626303,
+STORE, 139826134056960, 139826134622207,
+STORE, 139826134622208, 139826134626303,
+SNULL, 94077521309695, 94077521313791,
+STORE, 94077521305600, 94077521309695,
+STORE, 94077521309696, 94077521313791,
+SNULL, 139826136875007, 139826136879103,
+STORE, 139826136870912, 139826136875007,
+STORE, 139826136875008, 139826136879103,
+ERASE, 139826136842240, 139826136862719,
+STORE, 94077554049024, 94077554184191,
+STORE, 139826136543232, 139826136842239,
+STORE, 139826136276992, 139826136842239,
+STORE, 139826136010752, 139826136842239,
+STORE, 139826135744512, 139826136842239,
+SNULL, 139826136543231, 139826136842239,
+STORE, 139826135744512, 139826136543231,
+STORE, 139826136543232, 139826136842239,
+SNULL, 139826136543232, 139826136809471,
+STORE, 139826136809472, 139826136842239,
+STORE, 139826136543232, 139826136809471,
+ };
+ unsigned long set15[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140722061451264, 140737488351231,
+SNULL, 140722061455359, 140737488351231,
+STORE, 140722061451264, 140722061455359,
+STORE, 140722061320192, 140722061455359,
+STORE, 94728600248320, 94728600289279,
+SNULL, 94728600276991, 94728600289279,
+STORE, 94728600248320, 94728600276991,
+STORE, 94728600276992, 94728600289279,
+ERASE, 94728600276992, 94728600289279,
+STORE, 94728600281088, 94728600289279,
+STORE, 139906806779904, 139906809032703,
+SNULL, 139906806923263, 139906809032703,
+STORE, 139906806779904, 139906806923263,
+STORE, 139906806923264, 139906809032703,
+ERASE, 139906806923264, 139906809032703,
+STORE, 139906809020416, 139906809028607,
+STORE, 139906809028608, 139906809032703,
+STORE, 140722061692928, 140722061697023,
+STORE, 140722061680640, 140722061692927,
+STORE, 139906809012224, 139906809020415,
+STORE, 139906804555776, 139906806779903,
+SNULL, 139906806206463, 139906806779903,
+STORE, 139906804555776, 139906806206463,
+STORE, 139906806206464, 139906806779903,
+SNULL, 139906806206464, 139906806775807,
+STORE, 139906806775808, 139906806779903,
+STORE, 139906806206464, 139906806775807,
+ERASE, 139906806206464, 139906806775807,
+STORE, 139906806206464, 139906806775807,
+ERASE, 139906806775808, 139906806779903,
+STORE, 139906806775808, 139906806779903,
+STORE, 139906808991744, 139906809012223,
+STORE, 139906802171904, 139906804555775,
+SNULL, 139906802171904, 139906802438143,
+STORE, 139906802438144, 139906804555775,
+STORE, 139906802171904, 139906802438143,
+SNULL, 139906804531199, 139906804555775,
+STORE, 139906802438144, 139906804531199,
+STORE, 139906804531200, 139906804555775,
+SNULL, 139906804531200, 139906804551679,
+STORE, 139906804551680, 139906804555775,
+STORE, 139906804531200, 139906804551679,
+ERASE, 139906804531200, 139906804551679,
+STORE, 139906804531200, 139906804551679,
+ERASE, 139906804551680, 139906804555775,
+STORE, 139906804551680, 139906804555775,
+STORE, 139906799955968, 139906802171903,
+SNULL, 139906799955968, 139906800054271,
+STORE, 139906800054272, 139906802171903,
+STORE, 139906799955968, 139906800054271,
+SNULL, 139906802147327, 139906802171903,
+STORE, 139906800054272, 139906802147327,
+STORE, 139906802147328, 139906802171903,
+SNULL, 139906802147328, 139906802155519,
+STORE, 139906802155520, 139906802171903,
+STORE, 139906802147328, 139906802155519,
+ERASE, 139906802147328, 139906802155519,
+STORE, 139906802147328, 139906802155519,
+ERASE, 139906802155520, 139906802171903,
+STORE, 139906802155520, 139906802171903,
+STORE, 139906796158976, 139906799955967,
+SNULL, 139906796158976, 139906797817855,
+STORE, 139906797817856, 139906799955967,
+STORE, 139906796158976, 139906797817855,
+SNULL, 139906799915007, 139906799955967,
+STORE, 139906797817856, 139906799915007,
+STORE, 139906799915008, 139906799955967,
+SNULL, 139906799915008, 139906799939583,
+STORE, 139906799939584, 139906799955967,
+STORE, 139906799915008, 139906799939583,
+ERASE, 139906799915008, 139906799939583,
+STORE, 139906799915008, 139906799939583,
+ERASE, 139906799939584, 139906799955967,
+STORE, 139906799939584, 139906799955967,
+STORE, 139906793897984, 139906796158975,
+SNULL, 139906793897984, 139906794049535,
+STORE, 139906794049536, 139906796158975,
+STORE, 139906793897984, 139906794049535,
+SNULL, 139906796142591, 139906796158975,
+STORE, 139906794049536, 139906796142591,
+STORE, 139906796142592, 139906796158975,
+SNULL, 139906796142592, 139906796150783,
+STORE, 139906796150784, 139906796158975,
+STORE, 139906796142592, 139906796150783,
+ERASE, 139906796142592, 139906796150783,
+STORE, 139906796142592, 139906796150783,
+ERASE, 139906796150784, 139906796158975,
+STORE, 139906796150784, 139906796158975,
+STORE, 139906791776256, 139906793897983,
+SNULL, 139906791776256, 139906791792639,
+STORE, 139906791792640, 139906793897983,
+STORE, 139906791776256, 139906791792639,
+SNULL, 139906793889791, 139906793897983,
+STORE, 139906791792640, 139906793889791,
+STORE, 139906793889792, 139906793897983,
+ERASE, 139906793889792, 139906793897983,
+STORE, 139906793889792, 139906793897983,
+STORE, 139906808983552, 139906808991743,
+STORE, 139906789646336, 139906791776255,
+SNULL, 139906789646336, 139906789675007,
+STORE, 139906789675008, 139906791776255,
+STORE, 139906789646336, 139906789675007,
+SNULL, 139906791768063, 139906791776255,
+STORE, 139906789675008, 139906791768063,
+STORE, 139906791768064, 139906791776255,
+ERASE, 139906791768064, 139906791776255,
+STORE, 139906791768064, 139906791776255,
+STORE, 139906787393536, 139906789646335,
+SNULL, 139906787393536, 139906787545087,
+STORE, 139906787545088, 139906789646335,
+STORE, 139906787393536, 139906787545087,
+SNULL, 139906789638143, 139906789646335,
+STORE, 139906787545088, 139906789638143,
+STORE, 139906789638144, 139906789646335,
+ERASE, 139906789638144, 139906789646335,
+STORE, 139906789638144, 139906789646335,
+STORE, 139906785222656, 139906787393535,
+SNULL, 139906785222656, 139906785292287,
+STORE, 139906785292288, 139906787393535,
+STORE, 139906785222656, 139906785292287,
+SNULL, 139906787385343, 139906787393535,
+STORE, 139906785292288, 139906787385343,
+STORE, 139906787385344, 139906787393535,
+ERASE, 139906787385344, 139906787393535,
+STORE, 139906787385344, 139906787393535,
+STORE, 139906782011392, 139906785222655,
+SNULL, 139906782011392, 139906783088639,
+STORE, 139906783088640, 139906785222655,
+STORE, 139906782011392, 139906783088639,
+SNULL, 139906785185791, 139906785222655,
+STORE, 139906783088640, 139906785185791,
+STORE, 139906785185792, 139906785222655,
+ERASE, 139906785185792, 139906785222655,
+STORE, 139906785185792, 139906785222655,
+STORE, 139906779877376, 139906782011391,
+SNULL, 139906779877376, 139906779906047,
+STORE, 139906779906048, 139906782011391,
+STORE, 139906779877376, 139906779906047,
+SNULL, 139906782003199, 139906782011391,
+STORE, 139906779906048, 139906782003199,
+STORE, 139906782003200, 139906782011391,
+ERASE, 139906782003200, 139906782011391,
+STORE, 139906782003200, 139906782011391,
+STORE, 139906777567232, 139906779877375,
+SNULL, 139906777567232, 139906777772031,
+STORE, 139906777772032, 139906779877375,
+STORE, 139906777567232, 139906777772031,
+SNULL, 139906779869183, 139906779877375,
+STORE, 139906777772032, 139906779869183,
+STORE, 139906779869184, 139906779877375,
+ERASE, 139906779869184, 139906779877375,
+STORE, 139906779869184, 139906779877375,
+STORE, 139906808975360, 139906808991743,
+STORE, 139906775183360, 139906777567231,
+SNULL, 139906775183360, 139906775375871,
+STORE, 139906775375872, 139906777567231,
+STORE, 139906775183360, 139906775375871,
+SNULL, 139906777468927, 139906777567231,
+STORE, 139906775375872, 139906777468927,
+STORE, 139906777468928, 139906777567231,
+ERASE, 139906777468928, 139906777567231,
+STORE, 139906777468928, 139906777567231,
+STORE, 139906773065728, 139906775183359,
+SNULL, 139906773065728, 139906773082111,
+STORE, 139906773082112, 139906775183359,
+STORE, 139906773065728, 139906773082111,
+SNULL, 139906775175167, 139906775183359,
+STORE, 139906773082112, 139906775175167,
+STORE, 139906775175168, 139906775183359,
+ERASE, 139906775175168, 139906775183359,
+STORE, 139906775175168, 139906775183359,
+STORE, 139906770497536, 139906773065727,
+SNULL, 139906770497536, 139906770964479,
+STORE, 139906770964480, 139906773065727,
+STORE, 139906770497536, 139906770964479,
+SNULL, 139906773057535, 139906773065727,
+STORE, 139906770964480, 139906773057535,
+STORE, 139906773057536, 139906773065727,
+ERASE, 139906773057536, 139906773065727,
+STORE, 139906773057536, 139906773065727,
+STORE, 139906768384000, 139906770497535,
+SNULL, 139906768384000, 139906768396287,
+STORE, 139906768396288, 139906770497535,
+STORE, 139906768384000, 139906768396287,
+SNULL, 139906770489343, 139906770497535,
+STORE, 139906768396288, 139906770489343,
+STORE, 139906770489344, 139906770497535,
+ERASE, 139906770489344, 139906770497535,
+STORE, 139906770489344, 139906770497535,
+STORE, 139906766204928, 139906768383999,
+SNULL, 139906766204928, 139906766282751,
+STORE, 139906766282752, 139906768383999,
+STORE, 139906766204928, 139906766282751,
+SNULL, 139906768375807, 139906768383999,
+STORE, 139906766282752, 139906768375807,
+STORE, 139906768375808, 139906768383999,
+ERASE, 139906768375808, 139906768383999,
+STORE, 139906768375808, 139906768383999,
+STORE, 139906808967168, 139906808991743,
+STORE, 139906764087296, 139906766204927,
+SNULL, 139906764087296, 139906764103679,
+STORE, 139906764103680, 139906766204927,
+STORE, 139906764087296, 139906764103679,
+SNULL, 139906766196735, 139906766204927,
+STORE, 139906764103680, 139906766196735,
+STORE, 139906766196736, 139906766204927,
+ERASE, 139906766196736, 139906766204927,
+STORE, 139906766196736, 139906766204927,
+STORE, 139906808958976, 139906808991743,
+SNULL, 139906799931391, 139906799939583,
+STORE, 139906799915008, 139906799931391,
+STORE, 139906799931392, 139906799939583,
+SNULL, 139906766200831, 139906766204927,
+STORE, 139906766196736, 139906766200831,
+STORE, 139906766200832, 139906766204927,
+SNULL, 139906768379903, 139906768383999,
+STORE, 139906768375808, 139906768379903,
+STORE, 139906768379904, 139906768383999,
+SNULL, 139906770493439, 139906770497535,
+STORE, 139906770489344, 139906770493439,
+STORE, 139906770493440, 139906770497535,
+SNULL, 139906802151423, 139906802155519,
+STORE, 139906802147328, 139906802151423,
+STORE, 139906802151424, 139906802155519,
+SNULL, 139906773061631, 139906773065727,
+STORE, 139906773057536, 139906773061631,
+STORE, 139906773061632, 139906773065727,
+SNULL, 139906775179263, 139906775183359,
+STORE, 139906775175168, 139906775179263,
+STORE, 139906775179264, 139906775183359,
+SNULL, 139906777563135, 139906777567231,
+STORE, 139906777468928, 139906777563135,
+STORE, 139906777563136, 139906777567231,
+SNULL, 139906779873279, 139906779877375,
+STORE, 139906779869184, 139906779873279,
+STORE, 139906779873280, 139906779877375,
+SNULL, 139906782007295, 139906782011391,
+STORE, 139906782003200, 139906782007295,
+STORE, 139906782007296, 139906782011391,
+SNULL, 139906785193983, 139906785222655,
+STORE, 139906785185792, 139906785193983,
+STORE, 139906785193984, 139906785222655,
+SNULL, 139906787389439, 139906787393535,
+STORE, 139906787385344, 139906787389439,
+STORE, 139906787389440, 139906787393535,
+SNULL, 139906789642239, 139906789646335,
+STORE, 139906789638144, 139906789642239,
+STORE, 139906789642240, 139906789646335,
+SNULL, 139906791772159, 139906791776255,
+STORE, 139906791768064, 139906791772159,
+STORE, 139906791772160, 139906791776255,
+SNULL, 139906793893887, 139906793897983,
+STORE, 139906793889792, 139906793893887,
+STORE, 139906793893888, 139906793897983,
+SNULL, 139906796146687, 139906796150783,
+STORE, 139906796142592, 139906796146687,
+STORE, 139906796146688, 139906796150783,
+SNULL, 139906804547583, 139906804551679,
+STORE, 139906804531200, 139906804547583,
+STORE, 139906804547584, 139906804551679,
+SNULL, 139906806771711, 139906806775807,
+STORE, 139906806206464, 139906806771711,
+STORE, 139906806771712, 139906806775807,
+SNULL, 94728600285183, 94728600289279,
+STORE, 94728600281088, 94728600285183,
+STORE, 94728600285184, 94728600289279,
+SNULL, 139906809024511, 139906809028607,
+STORE, 139906809020416, 139906809024511,
+STORE, 139906809024512, 139906809028607,
+ERASE, 139906808991744, 139906809012223,
+STORE, 94728620138496, 94728620273663,
+STORE, 139906808692736, 139906808991743,
+STORE, 139906808426496, 139906808991743,
+STORE, 139906808160256, 139906808991743,
+STORE, 139906807894016, 139906808991743,
+SNULL, 139906808692735, 139906808991743,
+STORE, 139906807894016, 139906808692735,
+STORE, 139906808692736, 139906808991743,
+SNULL, 139906808692736, 139906808958975,
+STORE, 139906808958976, 139906808991743,
+STORE, 139906808692736, 139906808958975,
+ };
+
+ unsigned long set16[] = {
+STORE, 94174808662016, 94174809321471,
+STORE, 94174811414528, 94174811426815,
+STORE, 94174811426816, 94174811430911,
+STORE, 94174811430912, 94174811443199,
+STORE, 94174841700352, 94174841835519,
+STORE, 140173257838592, 140173259497471,
+STORE, 140173259497472, 140173261594623,
+STORE, 140173261594624, 140173261611007,
+STORE, 140173261611008, 140173261619199,
+STORE, 140173261619200, 140173261635583,
+STORE, 140173261635584, 140173261778943,
+STORE, 140173263863808, 140173263871999,
+STORE, 140173263876096, 140173263880191,
+STORE, 140173263880192, 140173263884287,
+STORE, 140173263884288, 140173263888383,
+STORE, 140729801007104, 140729801142271,
+STORE, 140729801617408, 140729801629695,
+STORE, 140729801629696, 140729801633791,
+STORE, 140737488347136, 140737488351231,
+STORE, 140728166858752, 140737488351231,
+SNULL, 140728166862847, 140737488351231,
+STORE, 140728166858752, 140728166862847,
+STORE, 140728166727680, 140728166862847,
+STORE, 93912949866496, 93912950337535,
+SNULL, 93912950288383, 93912950337535,
+STORE, 93912949866496, 93912950288383,
+STORE, 93912950288384, 93912950337535,
+ERASE, 93912950288384, 93912950337535,
+STORE, 93912950292480, 93912950337535,
+STORE, 139921863385088, 139921865637887,
+SNULL, 139921863528447, 139921865637887,
+STORE, 139921863385088, 139921863528447,
+STORE, 139921863528448, 139921865637887,
+ERASE, 139921863528448, 139921865637887,
+STORE, 139921865625600, 139921865633791,
+STORE, 139921865633792, 139921865637887,
+STORE, 140728167899136, 140728167903231,
+STORE, 140728167886848, 140728167899135,
+STORE, 139921865601024, 139921865625599,
+STORE, 139921865592832, 139921865601023,
+STORE, 139921861251072, 139921863385087,
+SNULL, 139921861251072, 139921861279743,
+STORE, 139921861279744, 139921863385087,
+STORE, 139921861251072, 139921861279743,
+SNULL, 139921863376895, 139921863385087,
+STORE, 139921861279744, 139921863376895,
+STORE, 139921863376896, 139921863385087,
+ERASE, 139921863376896, 139921863385087,
+STORE, 139921863376896, 139921863385087,
+STORE, 139921858867200, 139921861251071,
+SNULL, 139921858867200, 139921859133439,
+STORE, 139921859133440, 139921861251071,
+STORE, 139921858867200, 139921859133439,
+SNULL, 139921861226495, 139921861251071,
+STORE, 139921859133440, 139921861226495,
+STORE, 139921861226496, 139921861251071,
+SNULL, 139921861226496, 139921861246975,
+STORE, 139921861246976, 139921861251071,
+STORE, 139921861226496, 139921861246975,
+ERASE, 139921861226496, 139921861246975,
+STORE, 139921861226496, 139921861246975,
+ERASE, 139921861246976, 139921861251071,
+STORE, 139921861246976, 139921861251071,
+STORE, 139921856675840, 139921858867199,
+SNULL, 139921856675840, 139921856765951,
+STORE, 139921856765952, 139921858867199,
+STORE, 139921856675840, 139921856765951,
+SNULL, 139921858859007, 139921858867199,
+STORE, 139921856765952, 139921858859007,
+STORE, 139921858859008, 139921858867199,
+ERASE, 139921858859008, 139921858867199,
+STORE, 139921858859008, 139921858867199,
+STORE, 139921854414848, 139921856675839,
+SNULL, 139921854414848, 139921854566399,
+STORE, 139921854566400, 139921856675839,
+STORE, 139921854414848, 139921854566399,
+SNULL, 139921856659455, 139921856675839,
+STORE, 139921854566400, 139921856659455,
+STORE, 139921856659456, 139921856675839,
+SNULL, 139921856659456, 139921856667647,
+STORE, 139921856667648, 139921856675839,
+STORE, 139921856659456, 139921856667647,
+ERASE, 139921856659456, 139921856667647,
+STORE, 139921856659456, 139921856667647,
+ERASE, 139921856667648, 139921856675839,
+STORE, 139921856667648, 139921856675839,
+STORE, 139921852284928, 139921854414847,
+SNULL, 139921852284928, 139921852313599,
+STORE, 139921852313600, 139921854414847,
+STORE, 139921852284928, 139921852313599,
+SNULL, 139921854406655, 139921854414847,
+STORE, 139921852313600, 139921854406655,
+STORE, 139921854406656, 139921854414847,
+ERASE, 139921854406656, 139921854414847,
+STORE, 139921854406656, 139921854414847,
+STORE, 139921850068992, 139921852284927,
+SNULL, 139921850068992, 139921850167295,
+STORE, 139921850167296, 139921852284927,
+STORE, 139921850068992, 139921850167295,
+SNULL, 139921852260351, 139921852284927,
+STORE, 139921850167296, 139921852260351,
+STORE, 139921852260352, 139921852284927,
+SNULL, 139921852260352, 139921852268543,
+STORE, 139921852268544, 139921852284927,
+STORE, 139921852260352, 139921852268543,
+ERASE, 139921852260352, 139921852268543,
+STORE, 139921852260352, 139921852268543,
+ERASE, 139921852268544, 139921852284927,
+STORE, 139921852268544, 139921852284927,
+STORE, 139921865584640, 139921865601023,
+STORE, 139921846272000, 139921850068991,
+SNULL, 139921846272000, 139921847930879,
+STORE, 139921847930880, 139921850068991,
+STORE, 139921846272000, 139921847930879,
+SNULL, 139921850028031, 139921850068991,
+STORE, 139921847930880, 139921850028031,
+STORE, 139921850028032, 139921850068991,
+SNULL, 139921850028032, 139921850052607,
+STORE, 139921850052608, 139921850068991,
+STORE, 139921850028032, 139921850052607,
+ERASE, 139921850028032, 139921850052607,
+STORE, 139921850028032, 139921850052607,
+ERASE, 139921850052608, 139921850068991,
+STORE, 139921850052608, 139921850068991,
+STORE, 139921844154368, 139921846271999,
+SNULL, 139921844154368, 139921844170751,
+STORE, 139921844170752, 139921846271999,
+STORE, 139921844154368, 139921844170751,
+SNULL, 139921846263807, 139921846271999,
+STORE, 139921844170752, 139921846263807,
+STORE, 139921846263808, 139921846271999,
+ERASE, 139921846263808, 139921846271999,
+STORE, 139921846263808, 139921846271999,
+STORE, 139921842036736, 139921844154367,
+SNULL, 139921842036736, 139921842053119,
+STORE, 139921842053120, 139921844154367,
+STORE, 139921842036736, 139921842053119,
+SNULL, 139921844146175, 139921844154367,
+STORE, 139921842053120, 139921844146175,
+STORE, 139921844146176, 139921844154367,
+ERASE, 139921844146176, 139921844154367,
+STORE, 139921844146176, 139921844154367,
+STORE, 139921839468544, 139921842036735,
+SNULL, 139921839468544, 139921839935487,
+STORE, 139921839935488, 139921842036735,
+STORE, 139921839468544, 139921839935487,
+SNULL, 139921842028543, 139921842036735,
+STORE, 139921839935488, 139921842028543,
+STORE, 139921842028544, 139921842036735,
+ERASE, 139921842028544, 139921842036735,
+STORE, 139921842028544, 139921842036735,
+STORE, 139921837355008, 139921839468543,
+SNULL, 139921837355008, 139921837367295,
+STORE, 139921837367296, 139921839468543,
+STORE, 139921837355008, 139921837367295,
+SNULL, 139921839460351, 139921839468543,
+STORE, 139921837367296, 139921839460351,
+STORE, 139921839460352, 139921839468543,
+ERASE, 139921839460352, 139921839468543,
+STORE, 139921839460352, 139921839468543,
+STORE, 139921865576448, 139921865601023,
+STORE, 139921865564160, 139921865601023,
+SNULL, 139921850044415, 139921850052607,
+STORE, 139921850028032, 139921850044415,
+STORE, 139921850044416, 139921850052607,
+SNULL, 139921839464447, 139921839468543,
+STORE, 139921839460352, 139921839464447,
+STORE, 139921839464448, 139921839468543,
+SNULL, 139921852264447, 139921852268543,
+STORE, 139921852260352, 139921852264447,
+STORE, 139921852264448, 139921852268543,
+SNULL, 139921842032639, 139921842036735,
+STORE, 139921842028544, 139921842032639,
+STORE, 139921842032640, 139921842036735,
+SNULL, 139921844150271, 139921844154367,
+STORE, 139921844146176, 139921844150271,
+STORE, 139921844150272, 139921844154367,
+SNULL, 139921846267903, 139921846271999,
+STORE, 139921846263808, 139921846267903,
+STORE, 139921846267904, 139921846271999,
+SNULL, 139921854410751, 139921854414847,
+STORE, 139921854406656, 139921854410751,
+STORE, 139921854410752, 139921854414847,
+SNULL, 139921856663551, 139921856667647,
+STORE, 139921856659456, 139921856663551,
+STORE, 139921856663552, 139921856667647,
+SNULL, 139921858863103, 139921858867199,
+STORE, 139921858859008, 139921858863103,
+STORE, 139921858863104, 139921858867199,
+SNULL, 139921861242879, 139921861246975,
+STORE, 139921861226496, 139921861242879,
+STORE, 139921861242880, 139921861246975,
+SNULL, 139921863380991, 139921863385087,
+STORE, 139921863376896, 139921863380991,
+STORE, 139921863380992, 139921863385087,
+SNULL, 93912950333439, 93912950337535,
+STORE, 93912950292480, 93912950333439,
+STORE, 93912950333440, 93912950337535,
+SNULL, 139921865629695, 139921865633791,
+STORE, 139921865625600, 139921865629695,
+STORE, 139921865629696, 139921865633791,
+ERASE, 139921865601024, 139921865625599,
+STORE, 93912968110080, 93912968245247,
+STORE, 139921828913152, 139921837355007,
+STORE, 139921865621504, 139921865625599,
+STORE, 139921865617408, 139921865621503,
+STORE, 139921865613312, 139921865617407,
+STORE, 139921865547776, 139921865564159,
+ };
+
+ unsigned long set17[] = {
+STORE, 94397057224704, 94397057646591,
+STORE, 94397057650688, 94397057691647,
+STORE, 94397057691648, 94397057695743,
+STORE, 94397075271680, 94397075406847,
+STORE, 139953169051648, 139953169063935,
+STORE, 139953169063936, 139953171156991,
+STORE, 139953171156992, 139953171161087,
+STORE, 139953171161088, 139953171165183,
+STORE, 139953171165184, 139953171632127,
+STORE, 139953171632128, 139953173725183,
+STORE, 139953173725184, 139953173729279,
+STORE, 139953173729280, 139953173733375,
+STORE, 139953173733376, 139953173749759,
+STORE, 139953173749760, 139953175842815,
+STORE, 139953175842816, 139953175846911,
+STORE, 139953175846912, 139953175851007,
+STORE, 139953175851008, 139953175867391,
+STORE, 139953175867392, 139953177960447,
+STORE, 139953177960448, 139953177964543,
+STORE, 139953177964544, 139953177968639,
+STORE, 139953177968640, 139953179627519,
+STORE, 139953179627520, 139953181724671,
+STORE, 139953181724672, 139953181741055,
+STORE, 139953181741056, 139953181749247,
+STORE, 139953181749248, 139953181765631,
+STORE, 139953181765632, 139953181863935,
+STORE, 139953181863936, 139953183956991,
+STORE, 139953183956992, 139953183961087,
+STORE, 139953183961088, 139953183965183,
+STORE, 139953183965184, 139953183981567,
+STORE, 139953183981568, 139953184010239,
+STORE, 139953184010240, 139953186103295,
+STORE, 139953186103296, 139953186107391,
+STORE, 139953186107392, 139953186111487,
+STORE, 139953186111488, 139953186263039,
+STORE, 139953186263040, 139953188356095,
+STORE, 139953188356096, 139953188360191,
+STORE, 139953188360192, 139953188364287,
+STORE, 139953188364288, 139953188372479,
+STORE, 139953188372480, 139953188462591,
+STORE, 139953188462592, 139953190555647,
+STORE, 139953190555648, 139953190559743,
+STORE, 139953190559744, 139953190563839,
+STORE, 139953190563840, 139953190830079,
+STORE, 139953190830080, 139953192923135,
+STORE, 139953192923136, 139953192939519,
+STORE, 139953192939520, 139953192943615,
+STORE, 139953192943616, 139953192947711,
+STORE, 139953192947712, 139953192976383,
+STORE, 139953192976384, 139953195073535,
+STORE, 139953195073536, 139953195077631,
+STORE, 139953195077632, 139953195081727,
+STORE, 139953195081728, 139953195225087,
+STORE, 139953197281280, 139953197318143,
+STORE, 139953197322240, 139953197326335,
+STORE, 139953197326336, 139953197330431,
+STORE, 139953197330432, 139953197334527,
+STORE, 140720477511680, 140720477646847,
+STORE, 140720478302208, 140720478314495,
+STORE, 140720478314496, 140720478318591,
+ };
+ unsigned long set18[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140724953673728, 140737488351231,
+SNULL, 140724953677823, 140737488351231,
+STORE, 140724953673728, 140724953677823,
+STORE, 140724953542656, 140724953677823,
+STORE, 94675199266816, 94675199311871,
+SNULL, 94675199303679, 94675199311871,
+STORE, 94675199266816, 94675199303679,
+STORE, 94675199303680, 94675199311871,
+ERASE, 94675199303680, 94675199311871,
+STORE, 94675199303680, 94675199311871,
+STORE, 140222970605568, 140222972858367,
+SNULL, 140222970748927, 140222972858367,
+STORE, 140222970605568, 140222970748927,
+STORE, 140222970748928, 140222972858367,
+ERASE, 140222970748928, 140222972858367,
+STORE, 140222972846080, 140222972854271,
+STORE, 140222972854272, 140222972858367,
+STORE, 140724954365952, 140724954370047,
+STORE, 140724954353664, 140724954365951,
+STORE, 140222972841984, 140222972846079,
+STORE, 140222972833792, 140222972841983,
+STORE, 140222968475648, 140222970605567,
+SNULL, 140222968475648, 140222968504319,
+STORE, 140222968504320, 140222970605567,
+STORE, 140222968475648, 140222968504319,
+SNULL, 140222970597375, 140222970605567,
+STORE, 140222968504320, 140222970597375,
+STORE, 140222970597376, 140222970605567,
+ERASE, 140222970597376, 140222970605567,
+STORE, 140222970597376, 140222970605567,
+ };
+ unsigned long set19[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140725182459904, 140737488351231,
+SNULL, 140725182463999, 140737488351231,
+STORE, 140725182459904, 140725182463999,
+STORE, 140725182328832, 140725182463999,
+STORE, 94730166636544, 94730166763519,
+SNULL, 94730166747135, 94730166763519,
+STORE, 94730166636544, 94730166747135,
+STORE, 94730166747136, 94730166763519,
+ERASE, 94730166747136, 94730166763519,
+STORE, 94730166751232, 94730166763519,
+STORE, 140656834555904, 140656836808703,
+SNULL, 140656834699263, 140656836808703,
+STORE, 140656834555904, 140656834699263,
+STORE, 140656834699264, 140656836808703,
+ERASE, 140656834699264, 140656836808703,
+STORE, 140656836796416, 140656836804607,
+STORE, 140656836804608, 140656836808703,
+STORE, 140725183389696, 140725183393791,
+STORE, 140725183377408, 140725183389695,
+STORE, 140656836788224, 140656836796415,
+STORE, 140656832331776, 140656834555903,
+SNULL, 140656833982463, 140656834555903,
+STORE, 140656832331776, 140656833982463,
+STORE, 140656833982464, 140656834555903,
+SNULL, 140656833982464, 140656834551807,
+STORE, 140656834551808, 140656834555903,
+STORE, 140656833982464, 140656834551807,
+ERASE, 140656833982464, 140656834551807,
+STORE, 140656833982464, 140656834551807,
+ERASE, 140656834551808, 140656834555903,
+STORE, 140656834551808, 140656834555903,
+STORE, 140656836763648, 140656836788223,
+STORE, 140656830070784, 140656832331775,
+SNULL, 140656830070784, 140656830222335,
+STORE, 140656830222336, 140656832331775,
+STORE, 140656830070784, 140656830222335,
+SNULL, 140656832315391, 140656832331775,
+STORE, 140656830222336, 140656832315391,
+STORE, 140656832315392, 140656832331775,
+SNULL, 140656832315392, 140656832323583,
+STORE, 140656832323584, 140656832331775,
+STORE, 140656832315392, 140656832323583,
+ERASE, 140656832315392, 140656832323583,
+STORE, 140656832315392, 140656832323583,
+ERASE, 140656832323584, 140656832331775,
+STORE, 140656832323584, 140656832331775,
+STORE, 140656827940864, 140656830070783,
+SNULL, 140656827940864, 140656827969535,
+STORE, 140656827969536, 140656830070783,
+STORE, 140656827940864, 140656827969535,
+SNULL, 140656830062591, 140656830070783,
+STORE, 140656827969536, 140656830062591,
+STORE, 140656830062592, 140656830070783,
+ERASE, 140656830062592, 140656830070783,
+STORE, 140656830062592, 140656830070783,
+STORE, 140656825724928, 140656827940863,
+SNULL, 140656825724928, 140656825823231,
+STORE, 140656825823232, 140656827940863,
+STORE, 140656825724928, 140656825823231,
+SNULL, 140656827916287, 140656827940863,
+STORE, 140656825823232, 140656827916287,
+STORE, 140656827916288, 140656827940863,
+SNULL, 140656827916288, 140656827924479,
+STORE, 140656827924480, 140656827940863,
+STORE, 140656827916288, 140656827924479,
+ERASE, 140656827916288, 140656827924479,
+STORE, 140656827916288, 140656827924479,
+ERASE, 140656827924480, 140656827940863,
+STORE, 140656827924480, 140656827940863,
+STORE, 140656821927936, 140656825724927,
+SNULL, 140656821927936, 140656823586815,
+STORE, 140656823586816, 140656825724927,
+STORE, 140656821927936, 140656823586815,
+SNULL, 140656825683967, 140656825724927,
+STORE, 140656823586816, 140656825683967,
+STORE, 140656825683968, 140656825724927,
+SNULL, 140656825683968, 140656825708543,
+STORE, 140656825708544, 140656825724927,
+STORE, 140656825683968, 140656825708543,
+ERASE, 140656825683968, 140656825708543,
+STORE, 140656825683968, 140656825708543,
+ERASE, 140656825708544, 140656825724927,
+STORE, 140656825708544, 140656825724927,
+STORE, 140656819806208, 140656821927935,
+SNULL, 140656819806208, 140656819822591,
+STORE, 140656819822592, 140656821927935,
+STORE, 140656819806208, 140656819822591,
+SNULL, 140656821919743, 140656821927935,
+STORE, 140656819822592, 140656821919743,
+STORE, 140656821919744, 140656821927935,
+ERASE, 140656821919744, 140656821927935,
+STORE, 140656821919744, 140656821927935,
+STORE, 140656836755456, 140656836763647,
+STORE, 140656817553408, 140656819806207,
+SNULL, 140656817553408, 140656817704959,
+STORE, 140656817704960, 140656819806207,
+STORE, 140656817553408, 140656817704959,
+SNULL, 140656819798015, 140656819806207,
+STORE, 140656817704960, 140656819798015,
+STORE, 140656819798016, 140656819806207,
+ERASE, 140656819798016, 140656819806207,
+STORE, 140656819798016, 140656819806207,
+STORE, 140656815382528, 140656817553407,
+SNULL, 140656815382528, 140656815452159,
+STORE, 140656815452160, 140656817553407,
+STORE, 140656815382528, 140656815452159,
+SNULL, 140656817545215, 140656817553407,
+STORE, 140656815452160, 140656817545215,
+STORE, 140656817545216, 140656817553407,
+ERASE, 140656817545216, 140656817553407,
+STORE, 140656817545216, 140656817553407,
+STORE, 140656812171264, 140656815382527,
+SNULL, 140656812171264, 140656813248511,
+STORE, 140656813248512, 140656815382527,
+STORE, 140656812171264, 140656813248511,
+SNULL, 140656815345663, 140656815382527,
+STORE, 140656813248512, 140656815345663,
+STORE, 140656815345664, 140656815382527,
+ERASE, 140656815345664, 140656815382527,
+STORE, 140656815345664, 140656815382527,
+STORE, 140656810037248, 140656812171263,
+SNULL, 140656810037248, 140656810065919,
+STORE, 140656810065920, 140656812171263,
+STORE, 140656810037248, 140656810065919,
+SNULL, 140656812163071, 140656812171263,
+STORE, 140656810065920, 140656812163071,
+STORE, 140656812163072, 140656812171263,
+ERASE, 140656812163072, 140656812171263,
+STORE, 140656812163072, 140656812171263,
+STORE, 140656807727104, 140656810037247,
+SNULL, 140656807727104, 140656807931903,
+STORE, 140656807931904, 140656810037247,
+STORE, 140656807727104, 140656807931903,
+SNULL, 140656810029055, 140656810037247,
+STORE, 140656807931904, 140656810029055,
+STORE, 140656810029056, 140656810037247,
+ERASE, 140656810029056, 140656810037247,
+STORE, 140656810029056, 140656810037247,
+STORE, 140656805343232, 140656807727103,
+SNULL, 140656805343232, 140656805535743,
+STORE, 140656805535744, 140656807727103,
+STORE, 140656805343232, 140656805535743,
+SNULL, 140656807628799, 140656807727103,
+STORE, 140656805535744, 140656807628799,
+STORE, 140656807628800, 140656807727103,
+ERASE, 140656807628800, 140656807727103,
+STORE, 140656807628800, 140656807727103,
+STORE, 140656836747264, 140656836763647,
+STORE, 140656802775040, 140656805343231,
+SNULL, 140656802775040, 140656803241983,
+STORE, 140656803241984, 140656805343231,
+STORE, 140656802775040, 140656803241983,
+SNULL, 140656805335039, 140656805343231,
+STORE, 140656803241984, 140656805335039,
+STORE, 140656805335040, 140656805343231,
+ERASE, 140656805335040, 140656805343231,
+STORE, 140656805335040, 140656805343231,
+STORE, 140656800661504, 140656802775039,
+SNULL, 140656800661504, 140656800673791,
+STORE, 140656800673792, 140656802775039,
+STORE, 140656800661504, 140656800673791,
+SNULL, 140656802766847, 140656802775039,
+STORE, 140656800673792, 140656802766847,
+STORE, 140656802766848, 140656802775039,
+ERASE, 140656802766848, 140656802775039,
+STORE, 140656802766848, 140656802775039,
+STORE, 140656798482432, 140656800661503,
+SNULL, 140656798482432, 140656798560255,
+STORE, 140656798560256, 140656800661503,
+STORE, 140656798482432, 140656798560255,
+SNULL, 140656800653311, 140656800661503,
+STORE, 140656798560256, 140656800653311,
+STORE, 140656800653312, 140656800661503,
+ERASE, 140656800653312, 140656800661503,
+STORE, 140656800653312, 140656800661503,
+STORE, 140656796364800, 140656798482431,
+SNULL, 140656796364800, 140656796381183,
+STORE, 140656796381184, 140656798482431,
+STORE, 140656796364800, 140656796381183,
+SNULL, 140656798474239, 140656798482431,
+STORE, 140656796381184, 140656798474239,
+STORE, 140656798474240, 140656798482431,
+ERASE, 140656798474240, 140656798482431,
+STORE, 140656798474240, 140656798482431,
+STORE, 140656836739072, 140656836763647,
+STORE, 140656836726784, 140656836763647,
+SNULL, 140656825700351, 140656825708543,
+STORE, 140656825683968, 140656825700351,
+STORE, 140656825700352, 140656825708543,
+SNULL, 140656798478335, 140656798482431,
+STORE, 140656798474240, 140656798478335,
+STORE, 140656798478336, 140656798482431,
+SNULL, 140656800657407, 140656800661503,
+STORE, 140656800653312, 140656800657407,
+STORE, 140656800657408, 140656800661503,
+SNULL, 140656802770943, 140656802775039,
+STORE, 140656802766848, 140656802770943,
+STORE, 140656802770944, 140656802775039,
+SNULL, 140656827920383, 140656827924479,
+STORE, 140656827916288, 140656827920383,
+STORE, 140656827920384, 140656827924479,
+SNULL, 140656805339135, 140656805343231,
+STORE, 140656805335040, 140656805339135,
+STORE, 140656805339136, 140656805343231,
+SNULL, 140656807723007, 140656807727103,
+STORE, 140656807628800, 140656807723007,
+STORE, 140656807723008, 140656807727103,
+SNULL, 140656810033151, 140656810037247,
+STORE, 140656810029056, 140656810033151,
+STORE, 140656810033152, 140656810037247,
+SNULL, 140656812167167, 140656812171263,
+STORE, 140656812163072, 140656812167167,
+STORE, 140656812167168, 140656812171263,
+SNULL, 140656815353855, 140656815382527,
+STORE, 140656815345664, 140656815353855,
+STORE, 140656815353856, 140656815382527,
+SNULL, 140656817549311, 140656817553407,
+STORE, 140656817545216, 140656817549311,
+STORE, 140656817549312, 140656817553407,
+SNULL, 140656819802111, 140656819806207,
+STORE, 140656819798016, 140656819802111,
+STORE, 140656819802112, 140656819806207,
+SNULL, 140656821923839, 140656821927935,
+STORE, 140656821919744, 140656821923839,
+STORE, 140656821923840, 140656821927935,
+SNULL, 140656830066687, 140656830070783,
+STORE, 140656830062592, 140656830066687,
+STORE, 140656830066688, 140656830070783,
+SNULL, 140656832319487, 140656832323583,
+STORE, 140656832315392, 140656832319487,
+STORE, 140656832319488, 140656832323583,
+SNULL, 140656834547711, 140656834551807,
+STORE, 140656833982464, 140656834547711,
+STORE, 140656834547712, 140656834551807,
+SNULL, 94730166759423, 94730166763519,
+STORE, 94730166751232, 94730166759423,
+STORE, 94730166759424, 94730166763519,
+SNULL, 140656836800511, 140656836804607,
+STORE, 140656836796416, 140656836800511,
+STORE, 140656836800512, 140656836804607,
+ERASE, 140656836763648, 140656836788223,
+STORE, 94730171318272, 94730171453439,
+STORE, 140656836784128, 140656836788223,
+STORE, 140656836780032, 140656836784127,
+STORE, 140656791920640, 140656796364799,
+STORE, 140656836775936, 140656836780031,
+STORE, 140656787476480, 140656791920639,
+STORE, 140656779083776, 140656787476479,
+SNULL, 140656779087871, 140656787476479,
+STORE, 140656779083776, 140656779087871,
+STORE, 140656779087872, 140656787476479,
+STORE, 140656836771840, 140656836775935,
+STORE, 140656774639616, 140656779083775,
+STORE, 140656766246912, 140656774639615,
+SNULL, 140656766251007, 140656774639615,
+STORE, 140656766246912, 140656766251007,
+STORE, 140656766251008, 140656774639615,
+ERASE, 140656791920640, 140656796364799,
+ERASE, 140656836780032, 140656836784127,
+ERASE, 140656787476480, 140656791920639,
+ERASE, 140656836775936, 140656836780031,
+STORE, 140656836780032, 140656836784127,
+STORE, 140656791920640, 140656796364799,
+STORE, 140656836775936, 140656836780031,
+STORE, 140656787476480, 140656791920639,
+ERASE, 140656774639616, 140656779083775,
+ };
+ unsigned long set20[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140735952392192, 140737488351231,
+SNULL, 140735952396287, 140737488351231,
+STORE, 140735952392192, 140735952396287,
+STORE, 140735952261120, 140735952396287,
+STORE, 94849008947200, 94849009414143,
+SNULL, 94849009364991, 94849009414143,
+STORE, 94849008947200, 94849009364991,
+STORE, 94849009364992, 94849009414143,
+ERASE, 94849009364992, 94849009414143,
+STORE, 94849009364992, 94849009414143,
+STORE, 140590397943808, 140590400196607,
+SNULL, 140590398087167, 140590400196607,
+STORE, 140590397943808, 140590398087167,
+STORE, 140590398087168, 140590400196607,
+ERASE, 140590398087168, 140590400196607,
+STORE, 140590400184320, 140590400192511,
+STORE, 140590400192512, 140590400196607,
+STORE, 140735952850944, 140735952855039,
+STORE, 140735952838656, 140735952850943,
+STORE, 140590400180224, 140590400184319,
+STORE, 140590400172032, 140590400180223,
+STORE, 140590395809792, 140590397943807,
+SNULL, 140590395809792, 140590395838463,
+STORE, 140590395838464, 140590397943807,
+STORE, 140590395809792, 140590395838463,
+SNULL, 140590397935615, 140590397943807,
+STORE, 140590395838464, 140590397935615,
+STORE, 140590397935616, 140590397943807,
+ERASE, 140590397935616, 140590397943807,
+STORE, 140590397935616, 140590397943807,
+STORE, 140590393425920, 140590395809791,
+SNULL, 140590393425920, 140590393692159,
+STORE, 140590393692160, 140590395809791,
+STORE, 140590393425920, 140590393692159,
+SNULL, 140590395785215, 140590395809791,
+STORE, 140590393692160, 140590395785215,
+STORE, 140590395785216, 140590395809791,
+SNULL, 140590395785216, 140590395805695,
+STORE, 140590395805696, 140590395809791,
+STORE, 140590395785216, 140590395805695,
+ERASE, 140590395785216, 140590395805695,
+STORE, 140590395785216, 140590395805695,
+ERASE, 140590395805696, 140590395809791,
+STORE, 140590395805696, 140590395809791,
+STORE, 140590391234560, 140590393425919,
+SNULL, 140590391234560, 140590391324671,
+STORE, 140590391324672, 140590393425919,
+STORE, 140590391234560, 140590391324671,
+SNULL, 140590393417727, 140590393425919,
+STORE, 140590391324672, 140590393417727,
+STORE, 140590393417728, 140590393425919,
+ERASE, 140590393417728, 140590393425919,
+STORE, 140590393417728, 140590393425919,
+STORE, 140590388973568, 140590391234559,
+SNULL, 140590388973568, 140590389125119,
+STORE, 140590389125120, 140590391234559,
+STORE, 140590388973568, 140590389125119,
+SNULL, 140590391218175, 140590391234559,
+STORE, 140590389125120, 140590391218175,
+STORE, 140590391218176, 140590391234559,
+SNULL, 140590391218176, 140590391226367,
+STORE, 140590391226368, 140590391234559,
+STORE, 140590391218176, 140590391226367,
+ERASE, 140590391218176, 140590391226367,
+STORE, 140590391218176, 140590391226367,
+ERASE, 140590391226368, 140590391234559,
+STORE, 140590391226368, 140590391234559,
+STORE, 140590386843648, 140590388973567,
+SNULL, 140590386843648, 140590386872319,
+STORE, 140590386872320, 140590388973567,
+STORE, 140590386843648, 140590386872319,
+SNULL, 140590388965375, 140590388973567,
+STORE, 140590386872320, 140590388965375,
+STORE, 140590388965376, 140590388973567,
+ERASE, 140590388965376, 140590388973567,
+STORE, 140590388965376, 140590388973567,
+STORE, 140590384627712, 140590386843647,
+SNULL, 140590384627712, 140590384726015,
+STORE, 140590384726016, 140590386843647,
+STORE, 140590384627712, 140590384726015,
+SNULL, 140590386819071, 140590386843647,
+STORE, 140590384726016, 140590386819071,
+STORE, 140590386819072, 140590386843647,
+SNULL, 140590386819072, 140590386827263,
+STORE, 140590386827264, 140590386843647,
+STORE, 140590386819072, 140590386827263,
+ERASE, 140590386819072, 140590386827263,
+STORE, 140590386819072, 140590386827263,
+ERASE, 140590386827264, 140590386843647,
+STORE, 140590386827264, 140590386843647,
+STORE, 140590400163840, 140590400180223,
+STORE, 140590380830720, 140590384627711,
+SNULL, 140590380830720, 140590382489599,
+STORE, 140590382489600, 140590384627711,
+STORE, 140590380830720, 140590382489599,
+SNULL, 140590384586751, 140590384627711,
+STORE, 140590382489600, 140590384586751,
+STORE, 140590384586752, 140590384627711,
+SNULL, 140590384586752, 140590384611327,
+STORE, 140590384611328, 140590384627711,
+STORE, 140590384586752, 140590384611327,
+ERASE, 140590384586752, 140590384611327,
+STORE, 140590384586752, 140590384611327,
+ERASE, 140590384611328, 140590384627711,
+STORE, 140590384611328, 140590384627711,
+STORE, 140590378713088, 140590380830719,
+SNULL, 140590378713088, 140590378729471,
+STORE, 140590378729472, 140590380830719,
+STORE, 140590378713088, 140590378729471,
+SNULL, 140590380822527, 140590380830719,
+STORE, 140590378729472, 140590380822527,
+STORE, 140590380822528, 140590380830719,
+ERASE, 140590380822528, 140590380830719,
+STORE, 140590380822528, 140590380830719,
+STORE, 140590376595456, 140590378713087,
+SNULL, 140590376595456, 140590376611839,
+STORE, 140590376611840, 140590378713087,
+STORE, 140590376595456, 140590376611839,
+SNULL, 140590378704895, 140590378713087,
+STORE, 140590376611840, 140590378704895,
+STORE, 140590378704896, 140590378713087,
+ERASE, 140590378704896, 140590378713087,
+STORE, 140590378704896, 140590378713087,
+STORE, 140590374027264, 140590376595455,
+SNULL, 140590374027264, 140590374494207,
+STORE, 140590374494208, 140590376595455,
+STORE, 140590374027264, 140590374494207,
+SNULL, 140590376587263, 140590376595455,
+STORE, 140590374494208, 140590376587263,
+STORE, 140590376587264, 140590376595455,
+ERASE, 140590376587264, 140590376595455,
+STORE, 140590376587264, 140590376595455,
+STORE, 140590371913728, 140590374027263,
+SNULL, 140590371913728, 140590371926015,
+STORE, 140590371926016, 140590374027263,
+STORE, 140590371913728, 140590371926015,
+SNULL, 140590374019071, 140590374027263,
+STORE, 140590371926016, 140590374019071,
+STORE, 140590374019072, 140590374027263,
+ERASE, 140590374019072, 140590374027263,
+STORE, 140590374019072, 140590374027263,
+STORE, 140590400155648, 140590400180223,
+STORE, 140590400143360, 140590400180223,
+SNULL, 140590384603135, 140590384611327,
+STORE, 140590384586752, 140590384603135,
+STORE, 140590384603136, 140590384611327,
+SNULL, 140590374023167, 140590374027263,
+STORE, 140590374019072, 140590374023167,
+STORE, 140590374023168, 140590374027263,
+SNULL, 140590386823167, 140590386827263,
+STORE, 140590386819072, 140590386823167,
+STORE, 140590386823168, 140590386827263,
+SNULL, 140590376591359, 140590376595455,
+ };
+ unsigned long set21[] = {
+STORE, 93874710941696, 93874711363583,
+STORE, 93874711367680, 93874711408639,
+STORE, 93874711408640, 93874711412735,
+STORE, 93874720989184, 93874721124351,
+STORE, 140708365086720, 140708365099007,
+STORE, 140708365099008, 140708367192063,
+STORE, 140708367192064, 140708367196159,
+STORE, 140708367196160, 140708367200255,
+STORE, 140708367200256, 140708367667199,
+STORE, 140708367667200, 140708369760255,
+STORE, 140708369760256, 140708369764351,
+STORE, 140708369764352, 140708369768447,
+STORE, 140708369768448, 140708369784831,
+STORE, 140708369784832, 140708371877887,
+STORE, 140708371877888, 140708371881983,
+STORE, 140708371881984, 140708371886079,
+STORE, 140708371886080, 140708371902463,
+STORE, 140708371902464, 140708373995519,
+STORE, 140708373995520, 140708373999615,
+STORE, 140708373999616, 140708374003711,
+STORE, 140708374003712, 140708375662591,
+STORE, 140708375662592, 140708377759743,
+STORE, 140708377759744, 140708377776127,
+STORE, 140708377776128, 140708377784319,
+STORE, 140708377784320, 140708377800703,
+STORE, 140708377800704, 140708377899007,
+STORE, 140708377899008, 140708379992063,
+STORE, 140708379992064, 140708379996159,
+STORE, 140708379996160, 140708380000255,
+STORE, 140708380000256, 140708380016639,
+STORE, 140708380016640, 140708380045311,
+STORE, 140708380045312, 140708382138367,
+STORE, 140708382138368, 140708382142463,
+STORE, 140708382142464, 140708382146559,
+STORE, 140708382146560, 140708382298111,
+STORE, 140708382298112, 140708384391167,
+STORE, 140708384391168, 140708384395263,
+STORE, 140708384395264, 140708384399359,
+STORE, 140708384399360, 140708384407551,
+STORE, 140708384407552, 140708384497663,
+STORE, 140708384497664, 140708386590719,
+STORE, 140708386590720, 140708386594815,
+STORE, 140708386594816, 140708386598911,
+STORE, 140708386598912, 140708386865151,
+STORE, 140708386865152, 140708388958207,
+STORE, 140708388958208, 140708388974591,
+STORE, 140708388974592, 140708388978687,
+STORE, 140708388978688, 140708388982783,
+STORE, 140708388982784, 140708389011455,
+STORE, 140708389011456, 140708391108607,
+STORE, 140708391108608, 140708391112703,
+STORE, 140708391112704, 140708391116799,
+STORE, 140708391116800, 140708391260159,
+STORE, 140708393291776, 140708393308159,
+STORE, 140708393308160, 140708393312255,
+STORE, 140708393312256, 140708393316351,
+STORE, 140708393316352, 140708393353215,
+STORE, 140708393353216, 140708393357311,
+STORE, 140708393357312, 140708393361407,
+STORE, 140708393361408, 140708393365503,
+STORE, 140708393365504, 140708393369599,
+STORE, 140730557042688, 140730557177855,
+STORE, 140730557235200, 140730557247487,
+STORE, 140730557247488, 140730557251583,
+ERASE, 140708393353216, 140708393357311,
+ERASE, 140708393312256, 140708393316351,
+ERASE, 140708393308160, 140708393312255,
+ERASE, 140708393291776, 140708393308159,
+ };
+ unsigned long set22[] = {
+STORE, 93951397134336, 93951397183487,
+STORE, 93951397183488, 93951397728255,
+STORE, 93951397728256, 93951397826559,
+STORE, 93951397826560, 93951397842943,
+STORE, 93951397842944, 93951397847039,
+STORE, 93951425974272, 93951426109439,
+STORE, 140685152665600, 140685152677887,
+STORE, 140685152677888, 140685152829439,
+STORE, 140685152829440, 140685154181119,
+STORE, 140685154181120, 140685154484223,
+STORE, 140685154484224, 140685154496511,
+STORE, 140685154496512, 140685154508799,
+STORE, 140685154508800, 140685154525183,
+STORE, 140685154525184, 140685154541567,
+STORE, 140685154541568, 140685154590719,
+STORE, 140685154590720, 140685154603007,
+STORE, 140685154603008, 140685154607103,
+STORE, 140685154607104, 140685154611199,
+STORE, 140685154611200, 140685154615295,
+STORE, 140685154615296, 140685154631679,
+STORE, 140685154639872, 140685154643967,
+STORE, 140685154643968, 140685154766847,
+STORE, 140685154766848, 140685154799615,
+STORE, 140685154803712, 140685154807807,
+STORE, 140685154807808, 140685154811903,
+STORE, 140685154811904, 140685154815999,
+STORE, 140722188902400, 140722189037567,
+STORE, 140722189512704, 140722189524991,
+STORE, 140722189524992, 140722189529087,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733429354496, 140737488351231,
+SNULL, 140733429358591, 140737488351231,
+STORE, 140733429354496, 140733429358591,
+STORE, 140733429223424, 140733429358591,
+STORE, 94526683537408, 94526683660287,
+SNULL, 94526683553791, 94526683660287,
+STORE, 94526683537408, 94526683553791,
+STORE, 94526683553792, 94526683660287,
+ERASE, 94526683553792, 94526683660287,
+STORE, 94526683553792, 94526683623423,
+STORE, 94526683623424, 94526683647999,
+STORE, 94526683652096, 94526683660287,
+STORE, 140551363747840, 140551363923967,
+SNULL, 140551363751935, 140551363923967,
+STORE, 140551363747840, 140551363751935,
+STORE, 140551363751936, 140551363923967,
+ERASE, 140551363751936, 140551363923967,
+STORE, 140551363751936, 140551363874815,
+STORE, 140551363874816, 140551363907583,
+STORE, 140551363911680, 140551363919871,
+STORE, 140551363919872, 140551363923967,
+STORE, 140733429690368, 140733429694463,
+STORE, 140733429678080, 140733429690367,
+STORE, 140551363739648, 140551363747839,
+STORE, 140551363731456, 140551363739647,
+STORE, 140551363379200, 140551363731455,
+SNULL, 140551363379200, 140551363420159,
+STORE, 140551363420160, 140551363731455,
+STORE, 140551363379200, 140551363420159,
+SNULL, 140551363706879, 140551363731455,
+STORE, 140551363420160, 140551363706879,
+STORE, 140551363706880, 140551363731455,
+SNULL, 140551363420160, 140551363637247,
+STORE, 140551363637248, 140551363706879,
+STORE, 140551363420160, 140551363637247,
+ERASE, 140551363420160, 140551363637247,
+STORE, 140551363420160, 140551363637247,
+SNULL, 140551363637248, 140551363702783,
+STORE, 140551363702784, 140551363706879,
+STORE, 140551363637248, 140551363702783,
+ERASE, 140551363637248, 140551363702783,
+STORE, 140551363637248, 140551363702783,
+ERASE, 140551363706880, 140551363731455,
+STORE, 140551363706880, 140551363731455,
+STORE, 140551361531904, 140551363379199,
+SNULL, 140551361683455, 140551363379199,
+STORE, 140551361531904, 140551361683455,
+STORE, 140551361683456, 140551363379199,
+SNULL, 140551361683456, 140551363035135,
+STORE, 140551363035136, 140551363379199,
+STORE, 140551361683456, 140551363035135,
+ERASE, 140551361683456, 140551363035135,
+STORE, 140551361683456, 140551363035135,
+SNULL, 140551363035136, 140551363338239,
+STORE, 140551363338240, 140551363379199,
+STORE, 140551363035136, 140551363338239,
+ERASE, 140551363035136, 140551363338239,
+STORE, 140551363035136, 140551363379199,
+SNULL, 140551363338239, 140551363379199,
+STORE, 140551363035136, 140551363338239,
+STORE, 140551363338240, 140551363379199,
+SNULL, 140551363338240, 140551363362815,
+STORE, 140551363362816, 140551363379199,
+STORE, 140551363338240, 140551363362815,
+ERASE, 140551363338240, 140551363362815,
+STORE, 140551363338240, 140551363362815,
+ERASE, 140551363362816, 140551363379199,
+STORE, 140551363362816, 140551363379199,
+STORE, 140551361519616, 140551361531903,
+SNULL, 140551363350527, 140551363362815,
+STORE, 140551363338240, 140551363350527,
+STORE, 140551363350528, 140551363362815,
+SNULL, 140551363727359, 140551363731455,
+STORE, 140551363706880, 140551363727359,
+STORE, 140551363727360, 140551363731455,
+SNULL, 94526683656191, 94526683660287,
+STORE, 94526683652096, 94526683656191,
+STORE, 94526683656192, 94526683660287,
+SNULL, 140551363915775, 140551363919871,
+STORE, 140551363911680, 140551363915775,
+STORE, 140551363915776, 140551363919871,
+ERASE, 140551363739648, 140551363747839,
+STORE, 94526715490304, 94526715625471,
+STORE, 140551361253376, 140551361531903,
+STORE, 140551360987136, 140551361531903,
+STORE, 140551360720896, 140551361531903,
+STORE, 140551360454656, 140551361531903,
+SNULL, 140551361253375, 140551361531903,
+STORE, 140551360454656, 140551361253375,
+STORE, 140551361253376, 140551361531903,
+SNULL, 140551361253376, 140551361519615,
+STORE, 140551361519616, 140551361531903,
+STORE, 140551361253376, 140551361519615,
+ERASE, 140551361253376, 140551361519615,
+ };
+
+ unsigned long set23[] = {
+STORE, 94014447943680, 94014448156671,
+STORE, 94014450253824, 94014450257919,
+STORE, 94014450257920, 94014450266111,
+STORE, 94014450266112, 94014450278399,
+STORE, 94014464225280, 94014464630783,
+STORE, 139761764306944, 139761765965823,
+STORE, 139761765965824, 139761768062975,
+STORE, 139761768062976, 139761768079359,
+STORE, 139761768079360, 139761768087551,
+STORE, 139761768087552, 139761768103935,
+STORE, 139761768103936, 139761768116223,
+STORE, 139761768116224, 139761770209279,
+STORE, 139761770209280, 139761770213375,
+STORE, 139761770213376, 139761770217471,
+STORE, 139761770217472, 139761770360831,
+STORE, 139761770729472, 139761772412927,
+STORE, 139761772412928, 139761772429311,
+STORE, 139761772457984, 139761772462079,
+STORE, 139761772462080, 139761772466175,
+STORE, 139761772466176, 139761772470271,
+STORE, 140724336517120, 140724336652287,
+STORE, 140724336955392, 140724336967679,
+STORE, 140724336967680, 140724336971775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140721840295936, 140737488351231,
+SNULL, 140721840300031, 140737488351231,
+STORE, 140721840295936, 140721840300031,
+STORE, 140721840164864, 140721840300031,
+STORE, 93937913667584, 93937915830271,
+SNULL, 93937913729023, 93937915830271,
+STORE, 93937913667584, 93937913729023,
+STORE, 93937913729024, 93937915830271,
+ERASE, 93937913729024, 93937915830271,
+STORE, 93937915822080, 93937915830271,
+STORE, 140598835335168, 140598837587967,
+SNULL, 140598835478527, 140598837587967,
+STORE, 140598835335168, 140598835478527,
+STORE, 140598835478528, 140598837587967,
+ERASE, 140598835478528, 140598837587967,
+STORE, 140598837575680, 140598837583871,
+STORE, 140598837583872, 140598837587967,
+STORE, 140721841086464, 140721841090559,
+STORE, 140721841074176, 140721841086463,
+STORE, 140598837547008, 140598837575679,
+STORE, 140598837538816, 140598837547007,
+STORE, 140598831538176, 140598835335167,
+SNULL, 140598831538176, 140598833197055,
+STORE, 140598833197056, 140598835335167,
+STORE, 140598831538176, 140598833197055,
+SNULL, 140598835294207, 140598835335167,
+STORE, 140598833197056, 140598835294207,
+STORE, 140598835294208, 140598835335167,
+SNULL, 140598835294208, 140598835318783,
+STORE, 140598835318784, 140598835335167,
+STORE, 140598835294208, 140598835318783,
+ERASE, 140598835294208, 140598835318783,
+STORE, 140598835294208, 140598835318783,
+ERASE, 140598835318784, 140598835335167,
+STORE, 140598835318784, 140598835335167,
+SNULL, 140598835310591, 140598835318783,
+STORE, 140598835294208, 140598835310591,
+STORE, 140598835310592, 140598835318783,
+SNULL, 93937915826175, 93937915830271,
+STORE, 93937915822080, 93937915826175,
+STORE, 93937915826176, 93937915830271,
+SNULL, 140598837579775, 140598837583871,
+STORE, 140598837575680, 140598837579775,
+STORE, 140598837579776, 140598837583871,
+ERASE, 140598837547008, 140598837575679,
+STORE, 93937929179136, 93937929314303,
+STORE, 140598835855360, 140598837538815,
+STORE, 140737488347136, 140737488351231,
+STORE, 140728187723776, 140737488351231,
+SNULL, 140728187727871, 140737488351231,
+STORE, 140728187723776, 140728187727871,
+STORE, 140728187592704, 140728187727871,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 140583951437824, 140583953690623,
+SNULL, 140583951581183, 140583953690623,
+STORE, 140583951437824, 140583951581183,
+STORE, 140583951581184, 140583953690623,
+ERASE, 140583951581184, 140583953690623,
+STORE, 140583953678336, 140583953686527,
+STORE, 140583953686528, 140583953690623,
+STORE, 140728189116416, 140728189120511,
+STORE, 140728189104128, 140728189116415,
+STORE, 140583953649664, 140583953678335,
+STORE, 140583953641472, 140583953649663,
+STORE, 140583948275712, 140583951437823,
+SNULL, 140583948275712, 140583949336575,
+STORE, 140583949336576, 140583951437823,
+STORE, 140583948275712, 140583949336575,
+SNULL, 140583951429631, 140583951437823,
+STORE, 140583949336576, 140583951429631,
+STORE, 140583951429632, 140583951437823,
+ERASE, 140583951429632, 140583951437823,
+STORE, 140583951429632, 140583951437823,
+STORE, 140583944478720, 140583948275711,
+SNULL, 140583944478720, 140583946137599,
+STORE, 140583946137600, 140583948275711,
+STORE, 140583944478720, 140583946137599,
+SNULL, 140583948234751, 140583948275711,
+STORE, 140583946137600, 140583948234751,
+STORE, 140583948234752, 140583948275711,
+SNULL, 140583948234752, 140583948259327,
+STORE, 140583948259328, 140583948275711,
+STORE, 140583948234752, 140583948259327,
+ERASE, 140583948234752, 140583948259327,
+STORE, 140583948234752, 140583948259327,
+ERASE, 140583948259328, 140583948275711,
+STORE, 140583948259328, 140583948275711,
+STORE, 140583953629184, 140583953649663,
+SNULL, 140583948251135, 140583948259327,
+STORE, 140583948234752, 140583948251135,
+STORE, 140583948251136, 140583948259327,
+SNULL, 140583951433727, 140583951437823,
+STORE, 140583951429632, 140583951433727,
+STORE, 140583951433728, 140583951437823,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 140583953682431, 140583953686527,
+STORE, 140583953678336, 140583953682431,
+STORE, 140583953682432, 140583953686527,
+ERASE, 140583953649664, 140583953678335,
+STORE, 17821696, 17956863,
+STORE, 17821696, 18104319,
+STORE, 140583951945728, 140583953629183,
+STORE, 94014447943680, 94014448156671,
+STORE, 94014450253824, 94014450257919,
+STORE, 94014450257920, 94014450266111,
+STORE, 94014450266112, 94014450278399,
+STORE, 94014464225280, 94014465196031,
+STORE, 139761764306944, 139761765965823,
+STORE, 139761765965824, 139761768062975,
+STORE, 139761768062976, 139761768079359,
+STORE, 139761768079360, 139761768087551,
+STORE, 139761768087552, 139761768103935,
+STORE, 139761768103936, 139761768116223,
+STORE, 139761768116224, 139761770209279,
+STORE, 139761770209280, 139761770213375,
+STORE, 139761770213376, 139761770217471,
+STORE, 139761770217472, 139761770360831,
+STORE, 139761770729472, 139761772412927,
+STORE, 139761772412928, 139761772429311,
+STORE, 139761772457984, 139761772462079,
+STORE, 139761772462080, 139761772466175,
+STORE, 139761772466176, 139761772470271,
+STORE, 140724336517120, 140724336652287,
+STORE, 140724336955392, 140724336967679,
+STORE, 140724336967680, 140724336971775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140726063296512, 140737488351231,
+SNULL, 140726063300607, 140737488351231,
+STORE, 140726063296512, 140726063300607,
+STORE, 140726063165440, 140726063300607,
+STORE, 94016795934720, 94016798158847,
+SNULL, 94016796045311, 94016798158847,
+STORE, 94016795934720, 94016796045311,
+STORE, 94016796045312, 94016798158847,
+ERASE, 94016796045312, 94016798158847,
+STORE, 94016798138368, 94016798150655,
+STORE, 94016798150656, 94016798158847,
+STORE, 139975915966464, 139975918219263,
+SNULL, 139975916109823, 139975918219263,
+STORE, 139975915966464, 139975916109823,
+STORE, 139975916109824, 139975918219263,
+ERASE, 139975916109824, 139975918219263,
+STORE, 139975918206976, 139975918215167,
+STORE, 139975918215168, 139975918219263,
+STORE, 140726064541696, 140726064545791,
+STORE, 140726064529408, 140726064541695,
+STORE, 139975918178304, 139975918206975,
+STORE, 139975918170112, 139975918178303,
+STORE, 139975912169472, 139975915966463,
+SNULL, 139975912169472, 139975913828351,
+STORE, 139975913828352, 139975915966463,
+STORE, 139975912169472, 139975913828351,
+SNULL, 139975915925503, 139975915966463,
+STORE, 139975913828352, 139975915925503,
+STORE, 139975915925504, 139975915966463,
+SNULL, 139975915925504, 139975915950079,
+STORE, 139975915950080, 139975915966463,
+STORE, 139975915925504, 139975915950079,
+ERASE, 139975915925504, 139975915950079,
+STORE, 139975915925504, 139975915950079,
+ERASE, 139975915950080, 139975915966463,
+STORE, 139975915950080, 139975915966463,
+SNULL, 139975915941887, 139975915950079,
+STORE, 139975915925504, 139975915941887,
+STORE, 139975915941888, 139975915950079,
+SNULL, 94016798146559, 94016798150655,
+STORE, 94016798138368, 94016798146559,
+STORE, 94016798146560, 94016798150655,
+SNULL, 139975918211071, 139975918215167,
+STORE, 139975918206976, 139975918211071,
+STORE, 139975918211072, 139975918215167,
+ERASE, 139975918178304, 139975918206975,
+STORE, 94016804925440, 94016805060607,
+STORE, 94596177661952, 94596177772543,
+STORE, 94596179865600, 94596179873791,
+STORE, 94596179873792, 94596179877887,
+STORE, 94596179877888, 94596179886079,
+STORE, 94596211597312, 94596211863551,
+STORE, 140127351840768, 140127353499647,
+STORE, 140127353499648, 140127355596799,
+STORE, 140127355596800, 140127355613183,
+STORE, 140127355613184, 140127355621375,
+STORE, 140127355621376, 140127355637759,
+STORE, 140127355637760, 140127355781119,
+STORE, 140127357841408, 140127357849599,
+STORE, 140127357878272, 140127357882367,
+STORE, 140127357882368, 140127357886463,
+STORE, 140127357886464, 140127357890559,
+STORE, 140726167252992, 140726167392255,
+STORE, 140726167838720, 140726167851007,
+STORE, 140726167851008, 140726167855103,
+STORE, 140737488347136, 140737488351231,
+STORE, 140731874017280, 140737488351231,
+SNULL, 140731874021375, 140737488351231,
+STORE, 140731874017280, 140731874021375,
+STORE, 140731873886208, 140731874021375,
+STORE, 94178682265600, 94178684489727,
+SNULL, 94178682376191, 94178684489727,
+STORE, 94178682265600, 94178682376191,
+STORE, 94178682376192, 94178684489727,
+ERASE, 94178682376192, 94178684489727,
+STORE, 94178684469248, 94178684481535,
+STORE, 94178684481536, 94178684489727,
+STORE, 140460853403648, 140460855656447,
+SNULL, 140460853547007, 140460855656447,
+STORE, 140460853403648, 140460853547007,
+STORE, 140460853547008, 140460855656447,
+ERASE, 140460853547008, 140460855656447,
+STORE, 140460855644160, 140460855652351,
+STORE, 140460855652352, 140460855656447,
+STORE, 140731874103296, 140731874107391,
+STORE, 140731874091008, 140731874103295,
+STORE, 140460855615488, 140460855644159,
+STORE, 140460855607296, 140460855615487,
+STORE, 140460849606656, 140460853403647,
+SNULL, 140460849606656, 140460851265535,
+STORE, 140460851265536, 140460853403647,
+STORE, 140460849606656, 140460851265535,
+SNULL, 140460853362687, 140460853403647,
+STORE, 140460851265536, 140460853362687,
+STORE, 140460853362688, 140460853403647,
+SNULL, 140460853362688, 140460853387263,
+STORE, 140460853387264, 140460853403647,
+STORE, 140460853362688, 140460853387263,
+ERASE, 140460853362688, 140460853387263,
+STORE, 140460853362688, 140460853387263,
+ERASE, 140460853387264, 140460853403647,
+STORE, 140460853387264, 140460853403647,
+SNULL, 140460853379071, 140460853387263,
+STORE, 140460853362688, 140460853379071,
+STORE, 140460853379072, 140460853387263,
+SNULL, 94178684477439, 94178684481535,
+STORE, 94178684469248, 94178684477439,
+STORE, 94178684477440, 94178684481535,
+SNULL, 140460855648255, 140460855652351,
+STORE, 140460855644160, 140460855648255,
+STORE, 140460855648256, 140460855652351,
+ERASE, 140460855615488, 140460855644159,
+STORE, 94178692063232, 94178692198399,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733096603648, 140737488351231,
+SNULL, 140733096611839, 140737488351231,
+STORE, 140733096603648, 140733096611839,
+STORE, 140733096472576, 140733096611839,
+STORE, 94796716122112, 94796718325759,
+SNULL, 94796716224511, 94796718325759,
+STORE, 94796716122112, 94796716224511,
+STORE, 94796716224512, 94796718325759,
+ERASE, 94796716224512, 94796718325759,
+STORE, 94796718317568, 94796718325759,
+STORE, 139667892793344, 139667895046143,
+SNULL, 139667892936703, 139667895046143,
+STORE, 139667892793344, 139667892936703,
+STORE, 139667892936704, 139667895046143,
+ERASE, 139667892936704, 139667895046143,
+STORE, 139667895033856, 139667895042047,
+STORE, 139667895042048, 139667895046143,
+STORE, 140733096857600, 140733096861695,
+STORE, 140733096845312, 140733096857599,
+STORE, 139667895005184, 139667895033855,
+STORE, 139667894996992, 139667895005183,
+STORE, 139667890532352, 139667892793343,
+SNULL, 139667890532352, 139667890683903,
+STORE, 139667890683904, 139667892793343,
+STORE, 139667890532352, 139667890683903,
+SNULL, 139667892776959, 139667892793343,
+STORE, 139667890683904, 139667892776959,
+STORE, 139667892776960, 139667892793343,
+SNULL, 139667892776960, 139667892785151,
+STORE, 139667892785152, 139667892793343,
+STORE, 139667892776960, 139667892785151,
+ERASE, 139667892776960, 139667892785151,
+STORE, 139667892776960, 139667892785151,
+ERASE, 139667892785152, 139667892793343,
+STORE, 139667892785152, 139667892793343,
+STORE, 139667886735360, 139667890532351,
+SNULL, 139667886735360, 139667888394239,
+STORE, 139667888394240, 139667890532351,
+STORE, 139667886735360, 139667888394239,
+SNULL, 139667890491391, 139667890532351,
+STORE, 139667888394240, 139667890491391,
+STORE, 139667890491392, 139667890532351,
+SNULL, 139667890491392, 139667890515967,
+STORE, 139667890515968, 139667890532351,
+STORE, 139667890491392, 139667890515967,
+ERASE, 139667890491392, 139667890515967,
+STORE, 139667890491392, 139667890515967,
+ERASE, 139667890515968, 139667890532351,
+STORE, 139667890515968, 139667890532351,
+STORE, 139667884167168, 139667886735359,
+SNULL, 139667884167168, 139667884634111,
+STORE, 139667884634112, 139667886735359,
+STORE, 139667884167168, 139667884634111,
+SNULL, 139667886727167, 139667886735359,
+STORE, 139667884634112, 139667886727167,
+STORE, 139667886727168, 139667886735359,
+ERASE, 139667886727168, 139667886735359,
+STORE, 139667886727168, 139667886735359,
+STORE, 139667882053632, 139667884167167,
+SNULL, 139667882053632, 139667882065919,
+STORE, 139667882065920, 139667884167167,
+STORE, 139667882053632, 139667882065919,
+SNULL, 139667884158975, 139667884167167,
+STORE, 139667882065920, 139667884158975,
+STORE, 139667884158976, 139667884167167,
+ERASE, 139667884158976, 139667884167167,
+STORE, 139667884158976, 139667884167167,
+STORE, 139667879837696, 139667882053631,
+SNULL, 139667879837696, 139667879935999,
+STORE, 139667879936000, 139667882053631,
+STORE, 139667879837696, 139667879935999,
+SNULL, 139667882029055, 139667882053631,
+STORE, 139667879936000, 139667882029055,
+STORE, 139667882029056, 139667882053631,
+SNULL, 139667882029056, 139667882037247,
+STORE, 139667882037248, 139667882053631,
+STORE, 139667882029056, 139667882037247,
+ERASE, 139667882029056, 139667882037247,
+STORE, 139667882029056, 139667882037247,
+ERASE, 139667882037248, 139667882053631,
+STORE, 139667882037248, 139667882053631,
+STORE, 139667894988800, 139667895005183,
+SNULL, 139667890507775, 139667890515967,
+STORE, 139667890491392, 139667890507775,
+STORE, 139667890507776, 139667890515967,
+SNULL, 139667882033151, 139667882037247,
+STORE, 139667882029056, 139667882033151,
+STORE, 139667882033152, 139667882037247,
+SNULL, 139667884163071, 139667884167167,
+STORE, 139667884158976, 139667884163071,
+STORE, 139667884163072, 139667884167167,
+SNULL, 139667886731263, 139667886735359,
+STORE, 139667886727168, 139667886731263,
+STORE, 139667886731264, 139667886735359,
+SNULL, 139667892781055, 139667892785151,
+STORE, 139667892776960, 139667892781055,
+STORE, 139667892781056, 139667892785151,
+SNULL, 94796718321663, 94796718325759,
+STORE, 94796718317568, 94796718321663,
+STORE, 94796718321664, 94796718325759,
+SNULL, 139667895037951, 139667895042047,
+STORE, 139667895033856, 139667895037951,
+STORE, 139667895037952, 139667895042047,
+ERASE, 139667895005184, 139667895033855,
+STORE, 94796726063104, 94796726198271,
+STORE, 139667893305344, 139667894988799,
+STORE, 139667895005184, 139667895033855,
+STORE, 94796726063104, 94796726333439,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722489507840, 140737488351231,
+SNULL, 140722489516031, 140737488351231,
+STORE, 140722489507840, 140722489516031,
+STORE, 140722489376768, 140722489516031,
+STORE, 93980993265664, 93980995489791,
+SNULL, 93980993376255, 93980995489791,
+STORE, 93980993265664, 93980993376255,
+STORE, 93980993376256, 93980995489791,
+ERASE, 93980993376256, 93980995489791,
+STORE, 93980995469312, 93980995481599,
+STORE, 93980995481600, 93980995489791,
+STORE, 140261313593344, 140261315846143,
+SNULL, 140261313736703, 140261315846143,
+STORE, 140261313593344, 140261313736703,
+STORE, 140261313736704, 140261315846143,
+ERASE, 140261313736704, 140261315846143,
+STORE, 140261315833856, 140261315842047,
+STORE, 140261315842048, 140261315846143,
+STORE, 140722489675776, 140722489679871,
+STORE, 140722489663488, 140722489675775,
+STORE, 140261315805184, 140261315833855,
+STORE, 140261315796992, 140261315805183,
+STORE, 140261309796352, 140261313593343,
+SNULL, 140261309796352, 140261311455231,
+STORE, 140261311455232, 140261313593343,
+STORE, 140261309796352, 140261311455231,
+SNULL, 140261313552383, 140261313593343,
+STORE, 140261311455232, 140261313552383,
+STORE, 140261313552384, 140261313593343,
+SNULL, 140261313552384, 140261313576959,
+STORE, 140261313576960, 140261313593343,
+STORE, 140261313552384, 140261313576959,
+ERASE, 140261313552384, 140261313576959,
+STORE, 140261313552384, 140261313576959,
+ERASE, 140261313576960, 140261313593343,
+STORE, 140261313576960, 140261313593343,
+SNULL, 140261313568767, 140261313576959,
+STORE, 140261313552384, 140261313568767,
+STORE, 140261313568768, 140261313576959,
+SNULL, 93980995477503, 93980995481599,
+STORE, 93980995469312, 93980995477503,
+STORE, 93980995477504, 93980995481599,
+SNULL, 140261315837951, 140261315842047,
+STORE, 140261315833856, 140261315837951,
+STORE, 140261315837952, 140261315842047,
+ERASE, 140261315805184, 140261315833855,
+STORE, 93980997443584, 93980997578751,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140737488338944, 140737488351231,
+STORE, 140734059450368, 140737488351231,
+SNULL, 140734059462655, 140737488351231,
+STORE, 140734059450368, 140734059462655,
+STORE, 140734059319296, 140734059462655,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 140307554983936, 140307557236735,
+SNULL, 140307555127295, 140307557236735,
+STORE, 140307554983936, 140307555127295,
+STORE, 140307555127296, 140307557236735,
+ERASE, 140307555127296, 140307557236735,
+STORE, 140307557224448, 140307557232639,
+STORE, 140307557232640, 140307557236735,
+STORE, 140734059483136, 140734059487231,
+STORE, 140734059470848, 140734059483135,
+STORE, 140307557195776, 140307557224447,
+STORE, 140307557187584, 140307557195775,
+STORE, 140307551821824, 140307554983935,
+SNULL, 140307551821824, 140307552882687,
+STORE, 140307552882688, 140307554983935,
+STORE, 140307551821824, 140307552882687,
+SNULL, 140307554975743, 140307554983935,
+STORE, 140307552882688, 140307554975743,
+STORE, 140307554975744, 140307554983935,
+ERASE, 140307554975744, 140307554983935,
+STORE, 140307554975744, 140307554983935,
+STORE, 140307548024832, 140307551821823,
+SNULL, 140307548024832, 140307549683711,
+STORE, 140307549683712, 140307551821823,
+STORE, 140307548024832, 140307549683711,
+SNULL, 140307551780863, 140307551821823,
+STORE, 140307549683712, 140307551780863,
+STORE, 140307551780864, 140307551821823,
+SNULL, 140307551780864, 140307551805439,
+STORE, 140307551805440, 140307551821823,
+STORE, 140307551780864, 140307551805439,
+ERASE, 140307551780864, 140307551805439,
+STORE, 140307551780864, 140307551805439,
+ERASE, 140307551805440, 140307551821823,
+STORE, 140307551805440, 140307551821823,
+STORE, 140307557175296, 140307557195775,
+SNULL, 140307551797247, 140307551805439,
+STORE, 140307551780864, 140307551797247,
+STORE, 140307551797248, 140307551805439,
+SNULL, 140307554979839, 140307554983935,
+STORE, 140307554975744, 140307554979839,
+STORE, 140307554979840, 140307554983935,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 140307557228543, 140307557232639,
+STORE, 140307557224448, 140307557228543,
+STORE, 140307557228544, 140307557232639,
+ERASE, 140307557195776, 140307557224447,
+STORE, 39698432, 39833599,
+STORE, 39698432, 39981055,
+STORE, 94306485321728, 94306485432319,
+STORE, 94306487525376, 94306487533567,
+STORE, 94306487533568, 94306487537663,
+STORE, 94306487537664, 94306487545855,
+STORE, 94306488868864, 94306489004031,
+STORE, 140497673998336, 140497675657215,
+STORE, 140497675657216, 140497677754367,
+STORE, 140497677754368, 140497677770751,
+STORE, 140497677770752, 140497677778943,
+STORE, 140497677778944, 140497677795327,
+STORE, 140497677795328, 140497677938687,
+STORE, 140497679998976, 140497680007167,
+STORE, 140497680035840, 140497680039935,
+STORE, 140497680039936, 140497680044031,
+STORE, 140497680044032, 140497680048127,
+STORE, 140732780462080, 140732780601343,
+STORE, 140732782239744, 140732782252031,
+STORE, 140732782252032, 140732782256127,
+STORE, 94236915900416, 94236916011007,
+STORE, 94236918104064, 94236918112255,
+STORE, 94236918112256, 94236918116351,
+STORE, 94236918116352, 94236918124543,
+STORE, 94236939489280, 94236939624447,
+STORE, 140046091743232, 140046093402111,
+STORE, 140046093402112, 140046095499263,
+STORE, 140046095499264, 140046095515647,
+STORE, 140046095515648, 140046095523839,
+STORE, 140046095523840, 140046095540223,
+STORE, 140046095540224, 140046095683583,
+STORE, 140046097743872, 140046097752063,
+STORE, 140046097780736, 140046097784831,
+STORE, 140046097784832, 140046097788927,
+STORE, 140046097788928, 140046097793023,
+STORE, 140726694449152, 140726694588415,
+STORE, 140726695313408, 140726695325695,
+STORE, 140726695325696, 140726695329791,
+STORE, 94894582779904, 94894582992895,
+STORE, 94894585090048, 94894585094143,
+STORE, 94894585094144, 94894585102335,
+STORE, 94894585102336, 94894585114623,
+STORE, 94894592868352, 94894594293759,
+STORE, 139733563842560, 139733565501439,
+STORE, 139733565501440, 139733567598591,
+STORE, 139733567598592, 139733567614975,
+STORE, 139733567614976, 139733567623167,
+STORE, 139733567623168, 139733567639551,
+STORE, 139733567639552, 139733567651839,
+STORE, 139733567651840, 139733569744895,
+STORE, 139733569744896, 139733569748991,
+STORE, 139733569748992, 139733569753087,
+STORE, 139733569753088, 139733569896447,
+STORE, 139733570265088, 139733571948543,
+STORE, 139733571948544, 139733571964927,
+STORE, 139733571993600, 139733571997695,
+STORE, 139733571997696, 139733572001791,
+STORE, 139733572001792, 139733572005887,
+STORE, 140726369255424, 140726369394687,
+STORE, 140726370402304, 140726370414591,
+STORE, 140726370414592, 140726370418687,
+STORE, 94899236483072, 94899236696063,
+STORE, 94899238793216, 94899238797311,
+STORE, 94899238797312, 94899238805503,
+STORE, 94899238805504, 94899238817791,
+STORE, 94899263045632, 94899263979519,
+STORE, 140040959893504, 140040961552383,
+STORE, 140040961552384, 140040963649535,
+STORE, 140040963649536, 140040963665919,
+STORE, 140040963665920, 140040963674111,
+STORE, 140040963674112, 140040963690495,
+STORE, 140040963690496, 140040963702783,
+STORE, 140040963702784, 140040965795839,
+STORE, 140040965795840, 140040965799935,
+STORE, 140040965799936, 140040965804031,
+STORE, 140040965804032, 140040965947391,
+STORE, 140040966316032, 140040967999487,
+STORE, 140040967999488, 140040968015871,
+STORE, 140040968044544, 140040968048639,
+STORE, 140040968048640, 140040968052735,
+STORE, 140040968052736, 140040968056831,
+STORE, 140729921359872, 140729921499135,
+STORE, 140729921613824, 140729921626111,
+STORE, 140729921626112, 140729921630207,
+STORE, 94818265190400, 94818265403391,
+STORE, 94818267500544, 94818267504639,
+STORE, 94818267504640, 94818267512831,
+STORE, 94818267512832, 94818267525119,
+STORE, 94818283372544, 94818285858815,
+STORE, 139818425675776, 139818427334655,
+STORE, 139818427334656, 139818429431807,
+STORE, 139818429431808, 139818429448191,
+STORE, 139818429448192, 139818429456383,
+STORE, 139818429456384, 139818429472767,
+STORE, 139818429472768, 139818429485055,
+STORE, 139818429485056, 139818431578111,
+STORE, 139818431578112, 139818431582207,
+STORE, 139818431582208, 139818431586303,
+STORE, 139818431586304, 139818431729663,
+STORE, 139818432098304, 139818433781759,
+STORE, 139818433781760, 139818433798143,
+STORE, 139818433826816, 139818433830911,
+STORE, 139818433830912, 139818433835007,
+STORE, 139818433835008, 139818433839103,
+STORE, 140726170509312, 140726170648575,
+STORE, 140726171824128, 140726171836415,
+STORE, 140726171836416, 140726171840511,
+STORE, 94611513188352, 94611513401343,
+STORE, 94611515498496, 94611515502591,
+STORE, 94611515502592, 94611515510783,
+STORE, 94611515510784, 94611515523071,
+STORE, 94611516502016, 94611516907519,
+STORE, 140596246388736, 140596248047615,
+STORE, 140596248047616, 140596250144767,
+STORE, 140596250144768, 140596250161151,
+STORE, 140596250161152, 140596250169343,
+STORE, 140596250169344, 140596250185727,
+STORE, 140596250185728, 140596250198015,
+STORE, 140596250198016, 140596252291071,
+STORE, 140596252291072, 140596252295167,
+STORE, 140596252295168, 140596252299263,
+STORE, 140596252299264, 140596252442623,
+STORE, 140596252811264, 140596254494719,
+STORE, 140596254494720, 140596254511103,
+STORE, 140596254539776, 140596254543871,
+STORE, 140596254543872, 140596254547967,
+STORE, 140596254547968, 140596254552063,
+STORE, 140731551338496, 140731551477759,
+STORE, 140731551780864, 140731551793151,
+STORE, 140731551793152, 140731551797247,
+STORE, 94313835851776, 94313836064767,
+STORE, 94313838161920, 94313838166015,
+STORE, 94313838166016, 94313838174207,
+STORE, 94313838174208, 94313838186495,
+STORE, 94313858416640, 94313861906431,
+STORE, 140693503918080, 140693505576959,
+STORE, 140693505576960, 140693507674111,
+STORE, 140693507674112, 140693507690495,
+STORE, 140693507690496, 140693507698687,
+STORE, 140693507698688, 140693507715071,
+STORE, 140693507715072, 140693507727359,
+STORE, 140693507727360, 140693509820415,
+STORE, 140693509820416, 140693509824511,
+STORE, 140693509824512, 140693509828607,
+STORE, 140693509828608, 140693509971967,
+STORE, 140693510340608, 140693512024063,
+STORE, 140693512024064, 140693512040447,
+STORE, 140693512069120, 140693512073215,
+STORE, 140693512073216, 140693512077311,
+STORE, 140693512077312, 140693512081407,
+STORE, 140721116065792, 140721116205055,
+STORE, 140721117831168, 140721117843455,
+STORE, 140721117843456, 140721117847551,
+STORE, 94843650150400, 94843650363391,
+STORE, 94843652460544, 94843652464639,
+STORE, 94843652464640, 94843652472831,
+STORE, 94843652472832, 94843652485119,
+STORE, 94843685388288, 94843686281215,
+STORE, 140484193681408, 140484195340287,
+STORE, 140484195340288, 140484197437439,
+STORE, 140484197437440, 140484197453823,
+STORE, 140484197453824, 140484197462015,
+STORE, 140484197462016, 140484197478399,
+STORE, 140484197478400, 140484197490687,
+STORE, 140484197490688, 140484199583743,
+STORE, 140484199583744, 140484199587839,
+STORE, 140484199587840, 140484199591935,
+STORE, 140484199591936, 140484199735295,
+STORE, 140484200103936, 140484201787391,
+STORE, 140484201787392, 140484201803775,
+STORE, 140484201832448, 140484201836543,
+STORE, 140484201836544, 140484201840639,
+STORE, 140484201840640, 140484201844735,
+STORE, 140726294315008, 140726294454271,
+STORE, 140726295646208, 140726295658495,
+STORE, 140726295658496, 140726295662591,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140720422371328, 140737488351231,
+SNULL, 140720422379519, 140737488351231,
+STORE, 140720422371328, 140720422379519,
+STORE, 140720422240256, 140720422379519,
+STORE, 94417967845376, 94417970180095,
+SNULL, 94417968058367, 94417970180095,
+STORE, 94417967845376, 94417968058367,
+STORE, 94417968058368, 94417970180095,
+ERASE, 94417968058368, 94417970180095,
+STORE, 94417970155520, 94417970167807,
+STORE, 94417970167808, 94417970180095,
+STORE, 140252450045952, 140252452298751,
+SNULL, 140252450189311, 140252452298751,
+STORE, 140252450045952, 140252450189311,
+STORE, 140252450189312, 140252452298751,
+ERASE, 140252450189312, 140252452298751,
+STORE, 140252452286464, 140252452294655,
+STORE, 140252452294656, 140252452298751,
+STORE, 140720422416384, 140720422420479,
+STORE, 140720422404096, 140720422416383,
+STORE, 140252452257792, 140252452286463,
+STORE, 140252452249600, 140252452257791,
+STORE, 140252447932416, 140252450045951,
+SNULL, 140252447932416, 140252447944703,
+STORE, 140252447944704, 140252450045951,
+STORE, 140252447932416, 140252447944703,
+SNULL, 140252450037759, 140252450045951,
+STORE, 140252447944704, 140252450037759,
+STORE, 140252450037760, 140252450045951,
+ERASE, 140252450037760, 140252450045951,
+STORE, 140252450037760, 140252450045951,
+STORE, 140252444135424, 140252447932415,
+SNULL, 140252444135424, 140252445794303,
+STORE, 140252445794304, 140252447932415,
+STORE, 140252444135424, 140252445794303,
+SNULL, 140252447891455, 140252447932415,
+STORE, 140252445794304, 140252447891455,
+STORE, 140252447891456, 140252447932415,
+SNULL, 140252447891456, 140252447916031,
+STORE, 140252447916032, 140252447932415,
+STORE, 140252447891456, 140252447916031,
+ERASE, 140252447891456, 140252447916031,
+STORE, 140252447891456, 140252447916031,
+ERASE, 140252447916032, 140252447932415,
+STORE, 140252447916032, 140252447932415,
+STORE, 140252452241408, 140252452257791,
+SNULL, 140252447907839, 140252447916031,
+STORE, 140252447891456, 140252447907839,
+STORE, 140252447907840, 140252447916031,
+SNULL, 140252450041855, 140252450045951,
+STORE, 140252450037760, 140252450041855,
+STORE, 140252450041856, 140252450045951,
+SNULL, 94417970159615, 94417970167807,
+STORE, 94417970155520, 94417970159615,
+STORE, 94417970159616, 94417970167807,
+SNULL, 140252452290559, 140252452294655,
+STORE, 140252452286464, 140252452290559,
+STORE, 140252452290560, 140252452294655,
+ERASE, 140252452257792, 140252452286463,
+STORE, 94417996333056, 94417996468223,
+STORE, 140252450557952, 140252452241407,
+STORE, 94417996333056, 94417996603391,
+STORE, 94417996333056, 94417996738559,
+STORE, 94417996333056, 94417996910591,
+SNULL, 94417996881919, 94417996910591,
+STORE, 94417996333056, 94417996881919,
+STORE, 94417996881920, 94417996910591,
+ERASE, 94417996881920, 94417996910591,
+STORE, 94417996333056, 94417997017087,
+STORE, 94417996333056, 94417997152255,
+SNULL, 94417997135871, 94417997152255,
+STORE, 94417996333056, 94417997135871,
+STORE, 94417997135872, 94417997152255,
+ERASE, 94417997135872, 94417997152255,
+STORE, 94417996333056, 94417997291519,
+SNULL, 94417997271039, 94417997291519,
+STORE, 94417996333056, 94417997271039,
+STORE, 94417997271040, 94417997291519,
+ERASE, 94417997271040, 94417997291519,
+STORE, 94417996333056, 94417997406207,
+SNULL, 94417997381631, 94417997406207,
+STORE, 94417996333056, 94417997381631,
+STORE, 94417997381632, 94417997406207,
+ERASE, 94417997381632, 94417997406207,
+STORE, 94417996333056, 94417997516799,
+SNULL, 94417997488127, 94417997516799,
+STORE, 94417996333056, 94417997488127,
+STORE, 94417997488128, 94417997516799,
+ERASE, 94417997488128, 94417997516799,
+STORE, 94417996333056, 94417997643775,
+SNULL, 94417997631487, 94417997643775,
+STORE, 94417996333056, 94417997631487,
+STORE, 94417997631488, 94417997643775,
+ERASE, 94417997631488, 94417997643775,
+SNULL, 94417997590527, 94417997631487,
+STORE, 94417996333056, 94417997590527,
+STORE, 94417997590528, 94417997631487,
+ERASE, 94417997590528, 94417997631487,
+STORE, 94417996333056, 94417997733887,
+STORE, 94417996333056, 94417997869055,
+STORE, 94417996333056, 94417998004223,
+SNULL, 94417998000127, 94417998004223,
+STORE, 94417996333056, 94417998000127,
+STORE, 94417998000128, 94417998004223,
+ERASE, 94417998000128, 94417998004223,
+STORE, 94049170993152, 94049171206143,
+STORE, 94049173303296, 94049173307391,
+STORE, 94049173307392, 94049173315583,
+STORE, 94049173315584, 94049173327871,
+STORE, 94049176236032, 94049183645695,
+STORE, 139807795544064, 139807797202943,
+STORE, 139807797202944, 139807799300095,
+STORE, 139807799300096, 139807799316479,
+STORE, 139807799316480, 139807799324671,
+STORE, 139807799324672, 139807799341055,
+STORE, 139807799341056, 139807799353343,
+STORE, 139807799353344, 139807801446399,
+STORE, 139807801446400, 139807801450495,
+STORE, 139807801450496, 139807801454591,
+STORE, 139807801454592, 139807801597951,
+STORE, 139807801966592, 139807803650047,
+STORE, 139807803650048, 139807803666431,
+STORE, 139807803695104, 139807803699199,
+STORE, 139807803699200, 139807803703295,
+STORE, 139807803703296, 139807803707391,
+STORE, 140727555538944, 140727555678207,
+STORE, 140727555940352, 140727555952639,
+STORE, 140727555952640, 140727555956735,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722483441664, 140737488351231,
+SNULL, 140722483449855, 140737488351231,
+STORE, 140722483441664, 140722483449855,
+STORE, 140722483310592, 140722483449855,
+STORE, 94416704921600, 94416707145727,
+SNULL, 94416705032191, 94416707145727,
+STORE, 94416704921600, 94416705032191,
+STORE, 94416705032192, 94416707145727,
+ERASE, 94416705032192, 94416707145727,
+STORE, 94416707125248, 94416707137535,
+STORE, 94416707137536, 94416707145727,
+STORE, 140555439296512, 140555441549311,
+SNULL, 140555439439871, 140555441549311,
+STORE, 140555439296512, 140555439439871,
+STORE, 140555439439872, 140555441549311,
+ERASE, 140555439439872, 140555441549311,
+STORE, 140555441537024, 140555441545215,
+STORE, 140555441545216, 140555441549311,
+STORE, 140722484781056, 140722484785151,
+STORE, 140722484768768, 140722484781055,
+STORE, 140555441508352, 140555441537023,
+STORE, 140555441500160, 140555441508351,
+STORE, 140555435499520, 140555439296511,
+SNULL, 140555435499520, 140555437158399,
+STORE, 140555437158400, 140555439296511,
+STORE, 140555435499520, 140555437158399,
+SNULL, 140555439255551, 140555439296511,
+STORE, 140555437158400, 140555439255551,
+STORE, 140555439255552, 140555439296511,
+SNULL, 140555439255552, 140555439280127,
+STORE, 140555439280128, 140555439296511,
+STORE, 140555439255552, 140555439280127,
+ERASE, 140555439255552, 140555439280127,
+STORE, 140555439255552, 140555439280127,
+ERASE, 140555439280128, 140555439296511,
+STORE, 140555439280128, 140555439296511,
+SNULL, 140555439271935, 140555439280127,
+STORE, 140555439255552, 140555439271935,
+STORE, 140555439271936, 140555439280127,
+SNULL, 94416707133439, 94416707137535,
+STORE, 94416707125248, 94416707133439,
+STORE, 94416707133440, 94416707137535,
+SNULL, 140555441541119, 140555441545215,
+STORE, 140555441537024, 140555441541119,
+STORE, 140555441541120, 140555441545215,
+ERASE, 140555441508352, 140555441537023,
+STORE, 94416724672512, 94416724807679,
+STORE, 94686636953600, 94686637166591,
+STORE, 94686639263744, 94686639267839,
+STORE, 94686639267840, 94686639276031,
+STORE, 94686639276032, 94686639288319,
+STORE, 94686662193152, 94686663163903,
+STORE, 140312944431104, 140312946089983,
+STORE, 140312946089984, 140312948187135,
+STORE, 140312948187136, 140312948203519,
+STORE, 140312948203520, 140312948211711,
+STORE, 140312948211712, 140312948228095,
+STORE, 140312948228096, 140312948240383,
+STORE, 140312948240384, 140312950333439,
+STORE, 140312950333440, 140312950337535,
+STORE, 140312950337536, 140312950341631,
+STORE, 140312950341632, 140312950484991,
+STORE, 140312950853632, 140312952537087,
+STORE, 140312952537088, 140312952553471,
+STORE, 140312952582144, 140312952586239,
+STORE, 140312952586240, 140312952590335,
+STORE, 140312952590336, 140312952594431,
+STORE, 140730598920192, 140730599059455,
+STORE, 140730599108608, 140730599120895,
+STORE, 140730599120896, 140730599124991,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140726234079232, 140737488351231,
+SNULL, 140726234087423, 140737488351231,
+STORE, 140726234079232, 140726234087423,
+STORE, 140726233948160, 140726234087423,
+STORE, 94589467578368, 94589469802495,
+SNULL, 94589467688959, 94589469802495,
+STORE, 94589467578368, 94589467688959,
+STORE, 94589467688960, 94589469802495,
+ERASE, 94589467688960, 94589469802495,
+STORE, 94589469782016, 94589469794303,
+STORE, 94589469794304, 94589469802495,
+STORE, 140587082842112, 140587085094911,
+SNULL, 140587082985471, 140587085094911,
+STORE, 140587082842112, 140587082985471,
+STORE, 140587082985472, 140587085094911,
+ERASE, 140587082985472, 140587085094911,
+STORE, 140587085082624, 140587085090815,
+STORE, 140587085090816, 140587085094911,
+STORE, 140726234103808, 140726234107903,
+STORE, 140726234091520, 140726234103807,
+STORE, 140587085053952, 140587085082623,
+STORE, 140587085045760, 140587085053951,
+STORE, 140587079045120, 140587082842111,
+SNULL, 140587079045120, 140587080703999,
+STORE, 140587080704000, 140587082842111,
+STORE, 140587079045120, 140587080703999,
+SNULL, 140587082801151, 140587082842111,
+STORE, 140587080704000, 140587082801151,
+STORE, 140587082801152, 140587082842111,
+SNULL, 140587082801152, 140587082825727,
+STORE, 140587082825728, 140587082842111,
+STORE, 140587082801152, 140587082825727,
+ERASE, 140587082801152, 140587082825727,
+STORE, 140587082801152, 140587082825727,
+ERASE, 140587082825728, 140587082842111,
+STORE, 140587082825728, 140587082842111,
+SNULL, 140587082817535, 140587082825727,
+STORE, 140587082801152, 140587082817535,
+STORE, 140587082817536, 140587082825727,
+SNULL, 94589469790207, 94589469794303,
+STORE, 94589469782016, 94589469790207,
+STORE, 94589469790208, 94589469794303,
+SNULL, 140587085086719, 140587085090815,
+STORE, 140587085082624, 140587085086719,
+STORE, 140587085086720, 140587085090815,
+ERASE, 140587085053952, 140587085082623,
+STORE, 94589477507072, 94589477642239,
+STORE, 94225448325120, 94225448538111,
+STORE, 94225450635264, 94225450639359,
+STORE, 94225450639360, 94225450647551,
+STORE, 94225450647552, 94225450659839,
+STORE, 94225470246912, 94225473548287,
+STORE, 140199245496320, 140199247155199,
+STORE, 140199247155200, 140199249252351,
+STORE, 140199249252352, 140199249268735,
+STORE, 140199249268736, 140199249276927,
+STORE, 140199249276928, 140199249293311,
+STORE, 140199249293312, 140199249305599,
+STORE, 140199249305600, 140199251398655,
+STORE, 140199251398656, 140199251402751,
+STORE, 140199251402752, 140199251406847,
+STORE, 140199251406848, 140199251550207,
+STORE, 140199251918848, 140199253602303,
+STORE, 140199253602304, 140199253618687,
+STORE, 140199253647360, 140199253651455,
+STORE, 140199253651456, 140199253655551,
+STORE, 140199253655552, 140199253659647,
+STORE, 140726264414208, 140726264553471,
+STORE, 140726265843712, 140726265855999,
+STORE, 140726265856000, 140726265860095,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733508358144, 140737488351231,
+SNULL, 140733508366335, 140737488351231,
+STORE, 140733508358144, 140733508366335,
+STORE, 140733508227072, 140733508366335,
+STORE, 94766263947264, 94766266171391,
+SNULL, 94766264057855, 94766266171391,
+STORE, 94766263947264, 94766264057855,
+STORE, 94766264057856, 94766266171391,
+ERASE, 94766264057856, 94766266171391,
+STORE, 94766266150912, 94766266163199,
+STORE, 94766266163200, 94766266171391,
+STORE, 140693985132544, 140693987385343,
+SNULL, 140693985275903, 140693987385343,
+STORE, 140693985132544, 140693985275903,
+STORE, 140693985275904, 140693987385343,
+ERASE, 140693985275904, 140693987385343,
+STORE, 140693987373056, 140693987381247,
+STORE, 140693987381248, 140693987385343,
+STORE, 140733509939200, 140733509943295,
+STORE, 140733509926912, 140733509939199,
+STORE, 140693987344384, 140693987373055,
+STORE, 140693987336192, 140693987344383,
+STORE, 140693981335552, 140693985132543,
+SNULL, 140693981335552, 140693982994431,
+STORE, 140693982994432, 140693985132543,
+STORE, 140693981335552, 140693982994431,
+SNULL, 140693985091583, 140693985132543,
+STORE, 140693982994432, 140693985091583,
+STORE, 140693985091584, 140693985132543,
+SNULL, 140693985091584, 140693985116159,
+STORE, 140693985116160, 140693985132543,
+STORE, 140693985091584, 140693985116159,
+ERASE, 140693985091584, 140693985116159,
+STORE, 140693985091584, 140693985116159,
+ERASE, 140693985116160, 140693985132543,
+STORE, 140693985116160, 140693985132543,
+SNULL, 140693985107967, 140693985116159,
+STORE, 140693985091584, 140693985107967,
+STORE, 140693985107968, 140693985116159,
+SNULL, 94766266159103, 94766266163199,
+STORE, 94766266150912, 94766266159103,
+STORE, 94766266159104, 94766266163199,
+SNULL, 140693987377151, 140693987381247,
+STORE, 140693987373056, 140693987377151,
+STORE, 140693987377152, 140693987381247,
+ERASE, 140693987344384, 140693987373055,
+STORE, 94766282035200, 94766282170367,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724769353728, 140737488351231,
+SNULL, 140724769361919, 140737488351231,
+STORE, 140724769353728, 140724769361919,
+STORE, 140724769222656, 140724769361919,
+STORE, 94710460526592, 94710462750719,
+SNULL, 94710460637183, 94710462750719,
+STORE, 94710460526592, 94710460637183,
+STORE, 94710460637184, 94710462750719,
+ERASE, 94710460637184, 94710462750719,
+STORE, 94710462730240, 94710462742527,
+STORE, 94710462742528, 94710462750719,
+STORE, 140469764395008, 140469766647807,
+SNULL, 140469764538367, 140469766647807,
+STORE, 140469764395008, 140469764538367,
+STORE, 140469764538368, 140469766647807,
+ERASE, 140469764538368, 140469766647807,
+STORE, 140469766635520, 140469766643711,
+STORE, 140469766643712, 140469766647807,
+STORE, 140724770877440, 140724770881535,
+STORE, 140724770865152, 140724770877439,
+STORE, 140469766606848, 140469766635519,
+STORE, 140469766598656, 140469766606847,
+STORE, 140469760598016, 140469764395007,
+SNULL, 140469760598016, 140469762256895,
+STORE, 140469762256896, 140469764395007,
+STORE, 140469760598016, 140469762256895,
+SNULL, 140469764354047, 140469764395007,
+STORE, 140469762256896, 140469764354047,
+STORE, 140469764354048, 140469764395007,
+SNULL, 140469764354048, 140469764378623,
+STORE, 140469764378624, 140469764395007,
+STORE, 140469764354048, 140469764378623,
+ERASE, 140469764354048, 140469764378623,
+STORE, 140469764354048, 140469764378623,
+ERASE, 140469764378624, 140469764395007,
+STORE, 140469764378624, 140469764395007,
+SNULL, 140469764370431, 140469764378623,
+STORE, 140469764354048, 140469764370431,
+STORE, 140469764370432, 140469764378623,
+SNULL, 94710462738431, 94710462742527,
+STORE, 94710462730240, 94710462738431,
+STORE, 94710462738432, 94710462742527,
+SNULL, 140469766639615, 140469766643711,
+STORE, 140469766635520, 140469766639615,
+STORE, 140469766639616, 140469766643711,
+ERASE, 140469766606848, 140469766635519,
+STORE, 94710485581824, 94710485716991,
+STORE, 94105755795456, 94105756008447,
+STORE, 94105758105600, 94105758109695,
+STORE, 94105758109696, 94105758117887,
+STORE, 94105758117888, 94105758130175,
+STORE, 94105788981248, 94105794871295,
+STORE, 140641190031360, 140641191690239,
+STORE, 140641191690240, 140641193787391,
+STORE, 140641193787392, 140641193803775,
+STORE, 140641193803776, 140641193811967,
+STORE, 140641193811968, 140641193828351,
+STORE, 140641193828352, 140641193840639,
+STORE, 140641193840640, 140641195933695,
+STORE, 140641195933696, 140641195937791,
+STORE, 140641195937792, 140641195941887,
+STORE, 140641195941888, 140641196085247,
+STORE, 140641196453888, 140641198137343,
+STORE, 140641198137344, 140641198153727,
+STORE, 140641198182400, 140641198186495,
+STORE, 140641198186496, 140641198190591,
+STORE, 140641198190592, 140641198194687,
+STORE, 140731980034048, 140731980173311,
+STORE, 140731981078528, 140731981090815,
+STORE, 140731981090816, 140731981094911,
+STORE, 93828086431744, 93828086644735,
+STORE, 93828088741888, 93828088745983,
+STORE, 93828088745984, 93828088754175,
+STORE, 93828088754176, 93828088766463,
+STORE, 93828094193664, 93828096831487,
+STORE, 139844717334528, 139844718993407,
+STORE, 139844718993408, 139844721090559,
+STORE, 139844721090560, 139844721106943,
+STORE, 139844721106944, 139844721115135,
+STORE, 139844721115136, 139844721131519,
+STORE, 139844721131520, 139844721143807,
+STORE, 139844721143808, 139844723236863,
+STORE, 139844723236864, 139844723240959,
+STORE, 139844723240960, 139844723245055,
+STORE, 139844723245056, 139844723388415,
+STORE, 139844723757056, 139844725440511,
+STORE, 139844725440512, 139844725456895,
+STORE, 139844725485568, 139844725489663,
+STORE, 139844725489664, 139844725493759,
+STORE, 139844725493760, 139844725497855,
+STORE, 140729996185600, 140729996324863,
+STORE, 140729996828672, 140729996840959,
+STORE, 140729996840960, 140729996845055,
+STORE, 140737488347136, 140737488351231,
+STORE, 140722494771200, 140737488351231,
+SNULL, 140722494775295, 140737488351231,
+STORE, 140722494771200, 140722494775295,
+STORE, 140722494640128, 140722494775295,
+STORE, 94324011311104, 94324013535231,
+SNULL, 94324011421695, 94324013535231,
+STORE, 94324011311104, 94324011421695,
+STORE, 94324011421696, 94324013535231,
+ERASE, 94324011421696, 94324013535231,
+STORE, 94324013514752, 94324013527039,
+STORE, 94324013527040, 94324013535231,
+STORE, 140151462309888, 140151464562687,
+SNULL, 140151462453247, 140151464562687,
+STORE, 140151462309888, 140151462453247,
+STORE, 140151462453248, 140151464562687,
+ERASE, 140151462453248, 140151464562687,
+STORE, 140151464550400, 140151464558591,
+STORE, 140151464558592, 140151464562687,
+STORE, 140722495467520, 140722495471615,
+STORE, 140722495455232, 140722495467519,
+STORE, 140151464521728, 140151464550399,
+STORE, 140151464513536, 140151464521727,
+STORE, 140151458512896, 140151462309887,
+SNULL, 140151458512896, 140151460171775,
+STORE, 140151460171776, 140151462309887,
+STORE, 140151458512896, 140151460171775,
+SNULL, 140151462268927, 140151462309887,
+STORE, 140151460171776, 140151462268927,
+STORE, 140151462268928, 140151462309887,
+SNULL, 140151462268928, 140151462293503,
+STORE, 140151462293504, 140151462309887,
+STORE, 140151462268928, 140151462293503,
+ERASE, 140151462268928, 140151462293503,
+STORE, 140151462268928, 140151462293503,
+ERASE, 140151462293504, 140151462309887,
+STORE, 140151462293504, 140151462309887,
+SNULL, 140151462285311, 140151462293503,
+STORE, 140151462268928, 140151462285311,
+STORE, 140151462285312, 140151462293503,
+SNULL, 94324013522943, 94324013527039,
+STORE, 94324013514752, 94324013522943,
+STORE, 94324013522944, 94324013527039,
+SNULL, 140151464554495, 140151464558591,
+STORE, 140151464550400, 140151464554495,
+STORE, 140151464554496, 140151464558591,
+ERASE, 140151464521728, 140151464550399,
+STORE, 94324024778752, 94324024913919,
+STORE, 94899262967808, 94899263180799,
+STORE, 94899265277952, 94899265282047,
+STORE, 94899265282048, 94899265290239,
+STORE, 94899265290240, 94899265302527,
+STORE, 94899295469568, 94899298689023,
+STORE, 140434388418560, 140434390077439,
+STORE, 140434390077440, 140434392174591,
+STORE, 140434392174592, 140434392190975,
+STORE, 140434392190976, 140434392199167,
+STORE, 140434392199168, 140434392215551,
+STORE, 140434392215552, 140434392227839,
+STORE, 140434392227840, 140434394320895,
+STORE, 140434394320896, 140434394324991,
+STORE, 140434394324992, 140434394329087,
+STORE, 140434394329088, 140434394472447,
+STORE, 140434394841088, 140434396524543,
+STORE, 140434396524544, 140434396540927,
+STORE, 140434396569600, 140434396573695,
+STORE, 140434396573696, 140434396577791,
+STORE, 140434396577792, 140434396581887,
+STORE, 140720618135552, 140720618274815,
+STORE, 140720618418176, 140720618430463,
+STORE, 140720618430464, 140720618434559,
+STORE, 94425529798656, 94425530011647,
+STORE, 94425532108800, 94425532112895,
+STORE, 94425532112896, 94425532121087,
+STORE, 94425532121088, 94425532133375,
+STORE, 94425557753856, 94425566576639,
+STORE, 140600528470016, 140600530128895,
+STORE, 140600530128896, 140600532226047,
+STORE, 140600532226048, 140600532242431,
+STORE, 140600532242432, 140600532250623,
+STORE, 140600532250624, 140600532267007,
+STORE, 140600532267008, 140600532279295,
+STORE, 140600532279296, 140600534372351,
+STORE, 140600534372352, 140600534376447,
+STORE, 140600534376448, 140600534380543,
+STORE, 140600534380544, 140600534523903,
+STORE, 140600534892544, 140600536575999,
+STORE, 140600536576000, 140600536592383,
+STORE, 140600536621056, 140600536625151,
+STORE, 140600536625152, 140600536629247,
+STORE, 140600536629248, 140600536633343,
+STORE, 140721857785856, 140721857925119,
+STORE, 140721858068480, 140721858080767,
+STORE, 140721858080768, 140721858084863,
+STORE, 94425529798656, 94425530011647,
+STORE, 94425532108800, 94425532112895,
+STORE, 94425532112896, 94425532121087,
+STORE, 94425532121088, 94425532133375,
+STORE, 94425557753856, 94425568772095,
+STORE, 140600528470016, 140600530128895,
+STORE, 140600530128896, 140600532226047,
+STORE, 140600532226048, 140600532242431,
+STORE, 140600532242432, 140600532250623,
+STORE, 140600532250624, 140600532267007,
+STORE, 140600532267008, 140600532279295,
+STORE, 140600532279296, 140600534372351,
+STORE, 140600534372352, 140600534376447,
+STORE, 140600534376448, 140600534380543,
+STORE, 140600534380544, 140600534523903,
+STORE, 140600534892544, 140600536575999,
+STORE, 140600536576000, 140600536592383,
+STORE, 140600536621056, 140600536625151,
+STORE, 140600536625152, 140600536629247,
+STORE, 140600536629248, 140600536633343,
+STORE, 140721857785856, 140721857925119,
+STORE, 140721858068480, 140721858080767,
+STORE, 140721858080768, 140721858084863,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735611645952, 140737488351231,
+SNULL, 140735611654143, 140737488351231,
+STORE, 140735611645952, 140735611654143,
+STORE, 140735611514880, 140735611654143,
+STORE, 94592137641984, 94592139866111,
+SNULL, 94592137752575, 94592139866111,
+STORE, 94592137641984, 94592137752575,
+STORE, 94592137752576, 94592139866111,
+ERASE, 94592137752576, 94592139866111,
+STORE, 94592139845632, 94592139857919,
+STORE, 94592139857920, 94592139866111,
+STORE, 140350425030656, 140350427283455,
+SNULL, 140350425174015, 140350427283455,
+STORE, 140350425030656, 140350425174015,
+STORE, 140350425174016, 140350427283455,
+ERASE, 140350425174016, 140350427283455,
+STORE, 140350427271168, 140350427279359,
+STORE, 140350427279360, 140350427283455,
+STORE, 140735612043264, 140735612047359,
+STORE, 140735612030976, 140735612043263,
+STORE, 140350427242496, 140350427271167,
+STORE, 140350427234304, 140350427242495,
+STORE, 140350421233664, 140350425030655,
+SNULL, 140350421233664, 140350422892543,
+STORE, 140350422892544, 140350425030655,
+STORE, 140350421233664, 140350422892543,
+SNULL, 140350424989695, 140350425030655,
+STORE, 140350422892544, 140350424989695,
+STORE, 140350424989696, 140350425030655,
+SNULL, 140350424989696, 140350425014271,
+STORE, 140350425014272, 140350425030655,
+STORE, 140350424989696, 140350425014271,
+ERASE, 140350424989696, 140350425014271,
+STORE, 140350424989696, 140350425014271,
+ERASE, 140350425014272, 140350425030655,
+STORE, 140350425014272, 140350425030655,
+SNULL, 140350425006079, 140350425014271,
+STORE, 140350424989696, 140350425006079,
+STORE, 140350425006080, 140350425014271,
+SNULL, 94592139853823, 94592139857919,
+STORE, 94592139845632, 94592139853823,
+STORE, 94592139853824, 94592139857919,
+SNULL, 140350427275263, 140350427279359,
+STORE, 140350427271168, 140350427275263,
+STORE, 140350427275264, 140350427279359,
+ERASE, 140350427242496, 140350427271167,
+STORE, 94592164823040, 94592164958207,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723500535808, 140737488351231,
+SNULL, 140723500543999, 140737488351231,
+STORE, 140723500535808, 140723500543999,
+STORE, 140723500404736, 140723500543999,
+STORE, 94458379010048, 94458381234175,
+SNULL, 94458379120639, 94458381234175,
+STORE, 94458379010048, 94458379120639,
+STORE, 94458379120640, 94458381234175,
+ERASE, 94458379120640, 94458381234175,
+STORE, 94458381213696, 94458381225983,
+STORE, 94458381225984, 94458381234175,
+STORE, 139771674230784, 139771676483583,
+SNULL, 139771674374143, 139771676483583,
+STORE, 139771674230784, 139771674374143,
+STORE, 139771674374144, 139771676483583,
+ERASE, 139771674374144, 139771676483583,
+STORE, 139771676471296, 139771676479487,
+STORE, 139771676479488, 139771676483583,
+STORE, 140723500769280, 140723500773375,
+STORE, 140723500756992, 140723500769279,
+STORE, 139771676442624, 139771676471295,
+STORE, 139771676434432, 139771676442623,
+STORE, 139771670433792, 139771674230783,
+SNULL, 139771670433792, 139771672092671,
+STORE, 139771672092672, 139771674230783,
+STORE, 139771670433792, 139771672092671,
+SNULL, 139771674189823, 139771674230783,
+STORE, 139771672092672, 139771674189823,
+STORE, 139771674189824, 139771674230783,
+SNULL, 139771674189824, 139771674214399,
+STORE, 139771674214400, 139771674230783,
+STORE, 139771674189824, 139771674214399,
+ERASE, 139771674189824, 139771674214399,
+STORE, 139771674189824, 139771674214399,
+ERASE, 139771674214400, 139771674230783,
+STORE, 139771674214400, 139771674230783,
+SNULL, 139771674206207, 139771674214399,
+STORE, 139771674189824, 139771674206207,
+STORE, 139771674206208, 139771674214399,
+SNULL, 94458381221887, 94458381225983,
+STORE, 94458381213696, 94458381221887,
+STORE, 94458381221888, 94458381225983,
+SNULL, 139771676475391, 139771676479487,
+STORE, 139771676471296, 139771676475391,
+STORE, 139771676475392, 139771676479487,
+ERASE, 139771676442624, 139771676471295,
+STORE, 94458401873920, 94458402009087,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731316264960, 140737488351231,
+SNULL, 140731316273151, 140737488351231,
+STORE, 140731316264960, 140731316273151,
+STORE, 140731316133888, 140731316273151,
+STORE, 94437830881280, 94437833215999,
+SNULL, 94437831094271, 94437833215999,
+STORE, 94437830881280, 94437831094271,
+STORE, 94437831094272, 94437833215999,
+ERASE, 94437831094272, 94437833215999,
+STORE, 94437833191424, 94437833203711,
+STORE, 94437833203712, 94437833215999,
+STORE, 140265986031616, 140265988284415,
+SNULL, 140265986174975, 140265988284415,
+STORE, 140265986031616, 140265986174975,
+STORE, 140265986174976, 140265988284415,
+ERASE, 140265986174976, 140265988284415,
+STORE, 140265988272128, 140265988280319,
+STORE, 140265988280320, 140265988284415,
+STORE, 140731316318208, 140731316322303,
+STORE, 140731316305920, 140731316318207,
+STORE, 140265988243456, 140265988272127,
+STORE, 140265988235264, 140265988243455,
+STORE, 140265983918080, 140265986031615,
+SNULL, 140265983918080, 140265983930367,
+STORE, 140265983930368, 140265986031615,
+STORE, 140265983918080, 140265983930367,
+SNULL, 140265986023423, 140265986031615,
+STORE, 140265983930368, 140265986023423,
+STORE, 140265986023424, 140265986031615,
+ERASE, 140265986023424, 140265986031615,
+STORE, 140265986023424, 140265986031615,
+STORE, 140265980121088, 140265983918079,
+SNULL, 140265980121088, 140265981779967,
+STORE, 140265981779968, 140265983918079,
+STORE, 140265980121088, 140265981779967,
+SNULL, 140265983877119, 140265983918079,
+STORE, 140265981779968, 140265983877119,
+STORE, 140265983877120, 140265983918079,
+SNULL, 140265983877120, 140265983901695,
+STORE, 140265983901696, 140265983918079,
+STORE, 140265983877120, 140265983901695,
+ERASE, 140265983877120, 140265983901695,
+STORE, 140265983877120, 140265983901695,
+ERASE, 140265983901696, 140265983918079,
+STORE, 140265983901696, 140265983918079,
+STORE, 140265988227072, 140265988243455,
+SNULL, 140265983893503, 140265983901695,
+STORE, 140265983877120, 140265983893503,
+STORE, 140265983893504, 140265983901695,
+SNULL, 140265986027519, 140265986031615,
+STORE, 140265986023424, 140265986027519,
+STORE, 140265986027520, 140265986031615,
+SNULL, 94437833195519, 94437833203711,
+STORE, 94437833191424, 94437833195519,
+STORE, 94437833195520, 94437833203711,
+SNULL, 140265988276223, 140265988280319,
+STORE, 140265988272128, 140265988276223,
+STORE, 140265988276224, 140265988280319,
+ERASE, 140265988243456, 140265988272127,
+STORE, 94437847638016, 94437847773183,
+STORE, 140265986543616, 140265988227071,
+STORE, 94437847638016, 94437847908351,
+STORE, 94437847638016, 94437848043519,
+STORE, 94437847638016, 94437848190975,
+SNULL, 94437848178687, 94437848190975,
+STORE, 94437847638016, 94437848178687,
+STORE, 94437848178688, 94437848190975,
+ERASE, 94437848178688, 94437848190975,
+STORE, 94437847638016, 94437848330239,
+STORE, 94437847638016, 94437848465407,
+SNULL, 94437848444927, 94437848465407,
+STORE, 94437847638016, 94437848444927,
+STORE, 94437848444928, 94437848465407,
+ERASE, 94437848444928, 94437848465407,
+STORE, 94437847638016, 94437848584191,
+STORE, 94437847638016, 94437848719359,
+SNULL, 94437848678399, 94437848719359,
+STORE, 94437847638016, 94437848678399,
+STORE, 94437848678400, 94437848719359,
+ERASE, 94437848678400, 94437848719359,
+STORE, 94437847638016, 94437848842239,
+SNULL, 94437848825855, 94437848842239,
+STORE, 94437847638016, 94437848825855,
+STORE, 94437848825856, 94437848842239,
+ERASE, 94437848825856, 94437848842239,
+STORE, 94437847638016, 94437848961023,
+STORE, 94437847638016, 94437849096191,
+STORE, 94661814710272, 94661814923263,
+STORE, 94661817020416, 94661817024511,
+STORE, 94661817024512, 94661817032703,
+STORE, 94661817032704, 94661817044991,
+STORE, 94661840424960, 94661841240063,
+STORE, 140582259814400, 140582261473279,
+STORE, 140582261473280, 140582263570431,
+STORE, 140582263570432, 140582263586815,
+STORE, 140582263586816, 140582263595007,
+STORE, 140582263595008, 140582263611391,
+STORE, 140582263611392, 140582263623679,
+STORE, 140582263623680, 140582265716735,
+STORE, 140582265716736, 140582265720831,
+STORE, 140582265720832, 140582265724927,
+STORE, 140582265724928, 140582265868287,
+STORE, 140582266236928, 140582267920383,
+STORE, 140582267920384, 140582267936767,
+STORE, 140582267965440, 140582267969535,
+STORE, 140582267969536, 140582267973631,
+STORE, 140582267973632, 140582267977727,
+STORE, 140735472508928, 140735472648191,
+STORE, 140735472672768, 140735472685055,
+STORE, 140735472685056, 140735472689151,
+STORE, 94440069140480, 94440069353471,
+STORE, 94440071450624, 94440071454719,
+STORE, 94440071454720, 94440071462911,
+STORE, 94440071462912, 94440071475199,
+STORE, 94440072122368, 94440079048703,
+STORE, 140112218095616, 140112219754495,
+STORE, 140112219754496, 140112221851647,
+STORE, 140112221851648, 140112221868031,
+STORE, 140112221868032, 140112221876223,
+STORE, 140112221876224, 140112221892607,
+STORE, 140112221892608, 140112221904895,
+STORE, 140112221904896, 140112223997951,
+STORE, 140112223997952, 140112224002047,
+STORE, 140112224002048, 140112224006143,
+STORE, 140112224006144, 140112224149503,
+STORE, 140112224518144, 140112226201599,
+STORE, 140112226201600, 140112226217983,
+STORE, 140112226246656, 140112226250751,
+STORE, 140112226250752, 140112226254847,
+STORE, 140112226254848, 140112226258943,
+STORE, 140737460969472, 140737461108735,
+STORE, 140737462083584, 140737462095871,
+STORE, 140737462095872, 140737462099967,
+STORE, 94257654345728, 94257654390783,
+STORE, 94257656483840, 94257656487935,
+STORE, 94257656487936, 94257656492031,
+STORE, 94257656492032, 94257656496127,
+STORE, 94257665859584, 94257665994751,
+STORE, 140507070345216, 140507070386175,
+STORE, 140507070386176, 140507072483327,
+STORE, 140507072483328, 140507072487423,
+STORE, 140507072487424, 140507072491519,
+STORE, 140507072491520, 140507072516095,
+STORE, 140507072516096, 140507072561151,
+STORE, 140507072561152, 140507074654207,
+STORE, 140507074654208, 140507074658303,
+STORE, 140507074658304, 140507074662399,
+STORE, 140507074662400, 140507074744319,
+STORE, 140507074744320, 140507076841471,
+STORE, 140507076841472, 140507076845567,
+STORE, 140507076845568, 140507076849663,
+STORE, 140507076849664, 140507076857855,
+STORE, 140507076857856, 140507076886527,
+STORE, 140507076886528, 140507078979583,
+STORE, 140507078979584, 140507078983679,
+STORE, 140507078983680, 140507078987775,
+STORE, 140507078987776, 140507079086079,
+STORE, 140507079086080, 140507081179135,
+STORE, 140507081179136, 140507081183231,
+STORE, 140507081183232, 140507081187327,
+STORE, 140507081187328, 140507081203711,
+STORE, 140507081203712, 140507081220095,
+STORE, 140507081220096, 140507083317247,
+STORE, 140507083317248, 140507083321343,
+STORE, 140507083321344, 140507083325439,
+STORE, 140507083325440, 140507083792383,
+STORE, 140507083792384, 140507085885439,
+STORE, 140507085885440, 140507085889535,
+STORE, 140507085889536, 140507085893631,
+STORE, 140507085893632, 140507085905919,
+STORE, 140507085905920, 140507087998975,
+STORE, 140507087998976, 140507088003071,
+STORE, 140507088003072, 140507088007167,
+STORE, 140507088007168, 140507088125951,
+STORE, 140507088125952, 140507090219007,
+STORE, 140507090219008, 140507090223103,
+STORE, 140507090223104, 140507090227199,
+STORE, 140507090227200, 140507090268159,
+STORE, 140507090268160, 140507091927039,
+STORE, 140507091927040, 140507094024191,
+STORE, 140507094024192, 140507094040575,
+STORE, 140507094040576, 140507094048767,
+STORE, 140507094048768, 140507094065151,
+STORE, 140507094065152, 140507094216703,
+STORE, 140507094216704, 140507096309759,
+STORE, 140507096309760, 140507096313855,
+STORE, 140507096313856, 140507096317951,
+STORE, 140507096317952, 140507096326143,
+STORE, 140507096326144, 140507096379391,
+STORE, 140507096379392, 140507098472447,
+STORE, 140507098472448, 140507098476543,
+STORE, 140507098476544, 140507098480639,
+STORE, 140507098480640, 140507098623999,
+STORE, 140507098980352, 140507100663807,
+STORE, 140507100663808, 140507100692479,
+STORE, 140507100721152, 140507100725247,
+STORE, 140507100725248, 140507100729343,
+STORE, 140507100729344, 140507100733439,
+STORE, 140728152780800, 140728152915967,
+STORE, 140728153698304, 140728153710591,
+STORE, 140728153710592, 140728153714687,
+STORE, 140507068137472, 140507070345215,
+SNULL, 140507068137472, 140507068190719,
+STORE, 140507068190720, 140507070345215,
+STORE, 140507068137472, 140507068190719,
+SNULL, 140507070287871, 140507070345215,
+STORE, 140507068190720, 140507070287871,
+STORE, 140507070287872, 140507070345215,
+SNULL, 140507070287872, 140507070296063,
+STORE, 140507070296064, 140507070345215,
+STORE, 140507070287872, 140507070296063,
+ERASE, 140507070287872, 140507070296063,
+STORE, 140507070287872, 140507070296063,
+ERASE, 140507070296064, 140507070345215,
+STORE, 140507070296064, 140507070345215,
+STORE, 140507100692480, 140507100721151,
+STORE, 140507065810944, 140507068137471,
+SNULL, 140507065810944, 140507065843711,
+STORE, 140507065843712, 140507068137471,
+STORE, 140507065810944, 140507065843711,
+SNULL, 140507067940863, 140507068137471,
+STORE, 140507065843712, 140507067940863,
+STORE, 140507067940864, 140507068137471,
+SNULL, 140507067940864, 140507067949055,
+STORE, 140507067949056, 140507068137471,
+STORE, 140507067940864, 140507067949055,
+ERASE, 140507067940864, 140507067949055,
+STORE, 140507067940864, 140507067949055,
+ERASE, 140507067949056, 140507068137471,
+STORE, 140507067949056, 140507068137471,
+SNULL, 140507067944959, 140507067949055,
+STORE, 140507067940864, 140507067944959,
+STORE, 140507067944960, 140507067949055,
+SNULL, 140507070291967, 140507070296063,
+STORE, 140507070287872, 140507070291967,
+STORE, 140507070291968, 140507070296063,
+ERASE, 140507100692480, 140507100721151,
+STORE, 140507063705600, 140507065810943,
+SNULL, 140507063705600, 140507063709695,
+STORE, 140507063709696, 140507065810943,
+STORE, 140507063705600, 140507063709695,
+SNULL, 140507065802751, 140507065810943,
+STORE, 140507063709696, 140507065802751,
+STORE, 140507065802752, 140507065810943,
+ERASE, 140507065802752, 140507065810943,
+STORE, 140507065802752, 140507065810943,
+SNULL, 140507065806847, 140507065810943,
+STORE, 140507065802752, 140507065806847,
+STORE, 140507065806848, 140507065810943,
+STORE, 140507061600256, 140507063705599,
+SNULL, 140507061600256, 140507061604351,
+STORE, 140507061604352, 140507063705599,
+STORE, 140507061600256, 140507061604351,
+SNULL, 140507063697407, 140507063705599,
+STORE, 140507061604352, 140507063697407,
+STORE, 140507063697408, 140507063705599,
+ERASE, 140507063697408, 140507063705599,
+STORE, 140507063697408, 140507063705599,
+SNULL, 140507063701503, 140507063705599,
+STORE, 140507063697408, 140507063701503,
+STORE, 140507063701504, 140507063705599,
+STORE, 140507059490816, 140507061600255,
+SNULL, 140507059490816, 140507059499007,
+STORE, 140507059499008, 140507061600255,
+STORE, 140507059490816, 140507059499007,
+SNULL, 140507061592063, 140507061600255,
+STORE, 140507059499008, 140507061592063,
+STORE, 140507061592064, 140507061600255,
+ERASE, 140507061592064, 140507061600255,
+STORE, 140507061592064, 140507061600255,
+SNULL, 140507061596159, 140507061600255,
+STORE, 140507061592064, 140507061596159,
+STORE, 140507061596160, 140507061600255,
+STORE, 140507057377280, 140507059490815,
+SNULL, 140507057377280, 140507057389567,
+STORE, 140507057389568, 140507059490815,
+STORE, 140507057377280, 140507057389567,
+SNULL, 140507059482623, 140507059490815,
+STORE, 140507057389568, 140507059482623,
+STORE, 140507059482624, 140507059490815,
+ERASE, 140507059482624, 140507059490815,
+STORE, 140507059482624, 140507059490815,
+SNULL, 140507059486719, 140507059490815,
+STORE, 140507059482624, 140507059486719,
+STORE, 140507059486720, 140507059490815,
+STORE, 140507055255552, 140507057377279,
+SNULL, 140507055255552, 140507055276031,
+STORE, 140507055276032, 140507057377279,
+STORE, 140507055255552, 140507055276031,
+SNULL, 140507057369087, 140507057377279,
+STORE, 140507055276032, 140507057369087,
+STORE, 140507057369088, 140507057377279,
+ERASE, 140507057369088, 140507057377279,
+STORE, 140507057369088, 140507057377279,
+SNULL, 140507057373183, 140507057377279,
+STORE, 140507057369088, 140507057373183,
+STORE, 140507057373184, 140507057377279,
+STORE, 140507098693632, 140507098980351,
+SNULL, 140507098959871, 140507098980351,
+STORE, 140507098693632, 140507098959871,
+STORE, 140507098959872, 140507098980351,
+SNULL, 140507098959872, 140507098976255,
+STORE, 140507098976256, 140507098980351,
+STORE, 140507098959872, 140507098976255,
+ERASE, 140507098959872, 140507098976255,
+STORE, 140507098959872, 140507098976255,
+ERASE, 140507098976256, 140507098980351,
+STORE, 140507098976256, 140507098980351,
+STORE, 140507100692480, 140507100721151,
+STORE, 140507053125632, 140507055255551,
+SNULL, 140507053125632, 140507053154303,
+STORE, 140507053154304, 140507055255551,
+STORE, 140507053125632, 140507053154303,
+SNULL, 140507055247359, 140507055255551,
+STORE, 140507053154304, 140507055247359,
+STORE, 140507055247360, 140507055255551,
+ERASE, 140507055247360, 140507055255551,
+STORE, 140507055247360, 140507055255551,
+STORE, 140507051012096, 140507053125631,
+SNULL, 140507051012096, 140507051024383,
+STORE, 140507051024384, 140507053125631,
+STORE, 140507051012096, 140507051024383,
+SNULL, 140507053117439, 140507053125631,
+STORE, 140507051024384, 140507053117439,
+STORE, 140507053117440, 140507053125631,
+ERASE, 140507053117440, 140507053125631,
+STORE, 140507053117440, 140507053125631,
+SNULL, 140507053121535, 140507053125631,
+STORE, 140507053117440, 140507053121535,
+STORE, 140507053121536, 140507053125631,
+SNULL, 140507055251455, 140507055255551,
+STORE, 140507055247360, 140507055251455,
+STORE, 140507055251456, 140507055255551,
+SNULL, 140507098972159, 140507098976255,
+STORE, 140507098959872, 140507098972159,
+STORE, 140507098972160, 140507098976255,
+ERASE, 140507100692480, 140507100721151,
+STORE, 140507100717056, 140507100721151,
+ERASE, 140507100717056, 140507100721151,
+STORE, 140507100717056, 140507100721151,
+ERASE, 140507100717056, 140507100721151,
+STORE, 140507100717056, 140507100721151,
+ERASE, 140507100717056, 140507100721151,
+STORE, 140507100717056, 140507100721151,
+ERASE, 140507100717056, 140507100721151,
+STORE, 140507100692480, 140507100721151,
+ERASE, 140507068137472, 140507068190719,
+ERASE, 140507068190720, 140507070287871,
+ERASE, 140507070287872, 140507070291967,
+ERASE, 140507070291968, 140507070296063,
+ERASE, 140507070296064, 140507070345215,
+ERASE, 140507065810944, 140507065843711,
+ERASE, 140507065843712, 140507067940863,
+ERASE, 140507067940864, 140507067944959,
+ERASE, 140507067944960, 140507067949055,
+ERASE, 140507067949056, 140507068137471,
+ERASE, 140507063705600, 140507063709695,
+ERASE, 140507063709696, 140507065802751,
+ERASE, 140507065802752, 140507065806847,
+ERASE, 140507065806848, 140507065810943,
+ERASE, 140507061600256, 140507061604351,
+ERASE, 140507061604352, 140507063697407,
+ERASE, 140507063697408, 140507063701503,
+ERASE, 140507063701504, 140507063705599,
+ERASE, 140507059490816, 140507059499007,
+ERASE, 140507059499008, 140507061592063,
+ERASE, 140507061592064, 140507061596159,
+ERASE, 140507061596160, 140507061600255,
+ERASE, 140507057377280, 140507057389567,
+ERASE, 140507057389568, 140507059482623,
+ERASE, 140507059482624, 140507059486719,
+ERASE, 140507059486720, 140507059490815,
+ERASE, 140507055255552, 140507055276031,
+ERASE, 140507055276032, 140507057369087,
+ERASE, 140507057369088, 140507057373183,
+ERASE, 140507057373184, 140507057377279,
+ERASE, 140507098693632, 140507098959871,
+ERASE, 140507098959872, 140507098972159,
+ERASE, 140507098972160, 140507098976255,
+ERASE, 140507098976256, 140507098980351,
+ERASE, 140507051012096, 140507051024383,
+ERASE, 140507051024384, 140507053117439,
+ERASE, 140507053117440, 140507053121535,
+ERASE, 140507053121536, 140507053125631,
+STORE, 94036448296960, 94036448509951,
+STORE, 94036450607104, 94036450611199,
+STORE, 94036450611200, 94036450619391,
+STORE, 94036450619392, 94036450631679,
+STORE, 94036482445312, 94036502376447,
+STORE, 140469487013888, 140469488672767,
+STORE, 140469488672768, 140469490769919,
+STORE, 140469490769920, 140469490786303,
+STORE, 140469490786304, 140469490794495,
+STORE, 140469490794496, 140469490810879,
+STORE, 140469490810880, 140469490823167,
+STORE, 140469490823168, 140469492916223,
+STORE, 140469492916224, 140469492920319,
+STORE, 140469492920320, 140469492924415,
+STORE, 140469492924416, 140469493067775,
+STORE, 140469493436416, 140469495119871,
+STORE, 140469495119872, 140469495136255,
+STORE, 140469495164928, 140469495169023,
+STORE, 140469495169024, 140469495173119,
+STORE, 140469495173120, 140469495177215,
+STORE, 140732281446400, 140732281585663,
+STORE, 140732282736640, 140732282748927,
+STORE, 140732282748928, 140732282753023,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723411931136, 140737488351231,
+SNULL, 140723411939327, 140737488351231,
+STORE, 140723411931136, 140723411939327,
+STORE, 140723411800064, 140723411939327,
+STORE, 93993768685568, 93993770909695,
+SNULL, 93993768796159, 93993770909695,
+STORE, 93993768685568, 93993768796159,
+STORE, 93993768796160, 93993770909695,
+ERASE, 93993768796160, 93993770909695,
+STORE, 93993770889216, 93993770901503,
+STORE, 93993770901504, 93993770909695,
+STORE, 140508681740288, 140508683993087,
+SNULL, 140508681883647, 140508683993087,
+STORE, 140508681740288, 140508681883647,
+STORE, 140508681883648, 140508683993087,
+ERASE, 140508681883648, 140508683993087,
+STORE, 140508683980800, 140508683988991,
+STORE, 140508683988992, 140508683993087,
+STORE, 140723412070400, 140723412074495,
+STORE, 140723412058112, 140723412070399,
+STORE, 140508683952128, 140508683980799,
+STORE, 140508683943936, 140508683952127,
+STORE, 140508677943296, 140508681740287,
+SNULL, 140508677943296, 140508679602175,
+STORE, 140508679602176, 140508681740287,
+STORE, 140508677943296, 140508679602175,
+SNULL, 140508681699327, 140508681740287,
+STORE, 140508679602176, 140508681699327,
+STORE, 140508681699328, 140508681740287,
+SNULL, 140508681699328, 140508681723903,
+STORE, 140508681723904, 140508681740287,
+STORE, 140508681699328, 140508681723903,
+ERASE, 140508681699328, 140508681723903,
+STORE, 140508681699328, 140508681723903,
+ERASE, 140508681723904, 140508681740287,
+STORE, 140508681723904, 140508681740287,
+SNULL, 140508681715711, 140508681723903,
+STORE, 140508681699328, 140508681715711,
+STORE, 140508681715712, 140508681723903,
+SNULL, 93993770897407, 93993770901503,
+STORE, 93993770889216, 93993770897407,
+STORE, 93993770897408, 93993770901503,
+SNULL, 140508683984895, 140508683988991,
+STORE, 140508683980800, 140508683984895,
+STORE, 140508683984896, 140508683988991,
+ERASE, 140508683952128, 140508683980799,
+STORE, 93993791582208, 93993791717375,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140734685458432, 140737488351231,
+SNULL, 140734685466623, 140737488351231,
+STORE, 140734685458432, 140734685466623,
+STORE, 140734685327360, 140734685466623,
+STORE, 93832321548288, 93832323772415,
+SNULL, 93832321658879, 93832323772415,
+STORE, 93832321548288, 93832321658879,
+STORE, 93832321658880, 93832323772415,
+ERASE, 93832321658880, 93832323772415,
+STORE, 93832323751936, 93832323764223,
+STORE, 93832323764224, 93832323772415,
+STORE, 140650945118208, 140650947371007,
+SNULL, 140650945261567, 140650947371007,
+STORE, 140650945118208, 140650945261567,
+STORE, 140650945261568, 140650947371007,
+ERASE, 140650945261568, 140650947371007,
+STORE, 140650947358720, 140650947366911,
+STORE, 140650947366912, 140650947371007,
+STORE, 140734686081024, 140734686085119,
+STORE, 140734686068736, 140734686081023,
+STORE, 140650947330048, 140650947358719,
+STORE, 140650947321856, 140650947330047,
+STORE, 140650941321216, 140650945118207,
+SNULL, 140650941321216, 140650942980095,
+STORE, 140650942980096, 140650945118207,
+STORE, 140650941321216, 140650942980095,
+SNULL, 140650945077247, 140650945118207,
+STORE, 140650942980096, 140650945077247,
+STORE, 140650945077248, 140650945118207,
+SNULL, 140650945077248, 140650945101823,
+STORE, 140650945101824, 140650945118207,
+STORE, 140650945077248, 140650945101823,
+ERASE, 140650945077248, 140650945101823,
+STORE, 140650945077248, 140650945101823,
+ERASE, 140650945101824, 140650945118207,
+STORE, 140650945101824, 140650945118207,
+SNULL, 140650945093631, 140650945101823,
+STORE, 140650945077248, 140650945093631,
+STORE, 140650945093632, 140650945101823,
+SNULL, 93832323760127, 93832323764223,
+STORE, 93832323751936, 93832323760127,
+STORE, 93832323760128, 93832323764223,
+SNULL, 140650947362815, 140650947366911,
+STORE, 140650947358720, 140650947362815,
+STORE, 140650947362816, 140650947366911,
+ERASE, 140650947330048, 140650947358719,
+STORE, 93832331890688, 93832332025855,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728333520896, 140737488351231,
+SNULL, 140728333529087, 140737488351231,
+STORE, 140728333520896, 140728333529087,
+STORE, 140728333389824, 140728333529087,
+STORE, 94872734732288, 94872736956415,
+SNULL, 94872734842879, 94872736956415,
+STORE, 94872734732288, 94872734842879,
+STORE, 94872734842880, 94872736956415,
+ERASE, 94872734842880, 94872736956415,
+STORE, 94872736935936, 94872736948223,
+STORE, 94872736948224, 94872736956415,
+STORE, 139755193257984, 139755195510783,
+SNULL, 139755193401343, 139755195510783,
+STORE, 139755193257984, 139755193401343,
+STORE, 139755193401344, 139755195510783,
+ERASE, 139755193401344, 139755195510783,
+STORE, 139755195498496, 139755195506687,
+STORE, 139755195506688, 139755195510783,
+STORE, 140728333926400, 140728333930495,
+STORE, 140728333914112, 140728333926399,
+STORE, 139755195469824, 139755195498495,
+STORE, 139755195461632, 139755195469823,
+STORE, 139755189460992, 139755193257983,
+SNULL, 139755189460992, 139755191119871,
+STORE, 139755191119872, 139755193257983,
+STORE, 139755189460992, 139755191119871,
+SNULL, 139755193217023, 139755193257983,
+STORE, 139755191119872, 139755193217023,
+STORE, 139755193217024, 139755193257983,
+SNULL, 139755193217024, 139755193241599,
+STORE, 139755193241600, 139755193257983,
+STORE, 139755193217024, 139755193241599,
+ERASE, 139755193217024, 139755193241599,
+STORE, 139755193217024, 139755193241599,
+ERASE, 139755193241600, 139755193257983,
+STORE, 139755193241600, 139755193257983,
+SNULL, 139755193233407, 139755193241599,
+STORE, 139755193217024, 139755193233407,
+STORE, 139755193233408, 139755193241599,
+SNULL, 94872736944127, 94872736948223,
+STORE, 94872736935936, 94872736944127,
+STORE, 94872736944128, 94872736948223,
+SNULL, 139755195502591, 139755195506687,
+STORE, 139755195498496, 139755195502591,
+STORE, 139755195502592, 139755195506687,
+ERASE, 139755195469824, 139755195498495,
+STORE, 94872749744128, 94872749879295,
+STORE, 94720243642368, 94720243855359,
+STORE, 94720245952512, 94720245956607,
+STORE, 94720245956608, 94720245964799,
+STORE, 94720245964800, 94720245977087,
+STORE, 94720277745664, 94720278151167,
+STORE, 140453174497280, 140453176156159,
+STORE, 140453176156160, 140453178253311,
+STORE, 140453178253312, 140453178269695,
+STORE, 140453178269696, 140453178277887,
+STORE, 140453178277888, 140453178294271,
+STORE, 140453178294272, 140453178306559,
+STORE, 140453178306560, 140453180399615,
+STORE, 140453180399616, 140453180403711,
+STORE, 140453180403712, 140453180407807,
+STORE, 140453180407808, 140453180551167,
+STORE, 140453180919808, 140453182603263,
+STORE, 140453182603264, 140453182619647,
+STORE, 140453182648320, 140453182652415,
+STORE, 140453182652416, 140453182656511,
+STORE, 140453182656512, 140453182660607,
+STORE, 140733223923712, 140733224062975,
+STORE, 140733224808448, 140733224820735,
+STORE, 140733224820736, 140733224824831,
+STORE, 94321091141632, 94321091354623,
+STORE, 94321093451776, 94321093455871,
+STORE, 94321093455872, 94321093464063,
+STORE, 94321093464064, 94321093476351,
+STORE, 94321115873280, 94321117229055,
+STORE, 139695978840064, 139695980498943,
+STORE, 139695980498944, 139695982596095,
+STORE, 139695982596096, 139695982612479,
+STORE, 139695982612480, 139695982620671,
+STORE, 139695982620672, 139695982637055,
+STORE, 139695982637056, 139695982649343,
+STORE, 139695982649344, 139695984742399,
+STORE, 139695984742400, 139695984746495,
+STORE, 139695984746496, 139695984750591,
+STORE, 139695984750592, 139695984893951,
+STORE, 139695985262592, 139695986946047,
+STORE, 139695986946048, 139695986962431,
+STORE, 139695986991104, 139695986995199,
+STORE, 139695986995200, 139695986999295,
+STORE, 139695986999296, 139695987003391,
+STORE, 140734650564608, 140734650703871,
+STORE, 140734650785792, 140734650798079,
+STORE, 140734650798080, 140734650802175,
+STORE, 94523438456832, 94523438669823,
+STORE, 94523440766976, 94523440771071,
+STORE, 94523440771072, 94523440779263,
+STORE, 94523440779264, 94523440791551,
+STORE, 94523464544256, 94523465842687,
+STORE, 140453231493120, 140453233151999,
+STORE, 140453233152000, 140453235249151,
+STORE, 140453235249152, 140453235265535,
+STORE, 140453235265536, 140453235273727,
+STORE, 140453235273728, 140453235290111,
+STORE, 140453235290112, 140453235302399,
+STORE, 140453235302400, 140453237395455,
+STORE, 140453237395456, 140453237399551,
+STORE, 140453237399552, 140453237403647,
+STORE, 140453237403648, 140453237547007,
+STORE, 140453237915648, 140453239599103,
+STORE, 140453239599104, 140453239615487,
+STORE, 140453239644160, 140453239648255,
+STORE, 140453239648256, 140453239652351,
+STORE, 140453239652352, 140453239656447,
+STORE, 140734679445504, 140734679584767,
+STORE, 140734680018944, 140734680031231,
+STORE, 140734680031232, 140734680035327,
+STORE, 94614776987648, 94614777200639,
+STORE, 94614779297792, 94614779301887,
+STORE, 94614779301888, 94614779310079,
+STORE, 94614779310080, 94614779322367,
+STORE, 94614798467072, 94614800699391,
+STORE, 139677037182976, 139677038841855,
+STORE, 139677038841856, 139677040939007,
+STORE, 139677040939008, 139677040955391,
+STORE, 139677040955392, 139677040963583,
+STORE, 139677040963584, 139677040979967,
+STORE, 139677040979968, 139677040992255,
+STORE, 139677040992256, 139677043085311,
+STORE, 139677043085312, 139677043089407,
+STORE, 139677043089408, 139677043093503,
+STORE, 139677043093504, 139677043236863,
+STORE, 139677043605504, 139677045288959,
+STORE, 139677045288960, 139677045305343,
+STORE, 139677045334016, 139677045338111,
+STORE, 139677045338112, 139677045342207,
+STORE, 139677045342208, 139677045346303,
+STORE, 140721604411392, 140721604550655,
+STORE, 140721606135808, 140721606148095,
+STORE, 140721606148096, 140721606152191,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140729280544768, 140737488351231,
+SNULL, 140729280552959, 140737488351231,
+STORE, 140729280544768, 140729280552959,
+STORE, 140729280413696, 140729280552959,
+STORE, 94863939334144, 94863941558271,
+SNULL, 94863939444735, 94863941558271,
+STORE, 94863939334144, 94863939444735,
+STORE, 94863939444736, 94863941558271,
+ERASE, 94863939444736, 94863941558271,
+STORE, 94863941537792, 94863941550079,
+STORE, 94863941550080, 94863941558271,
+STORE, 139691047276544, 139691049529343,
+SNULL, 139691047419903, 139691049529343,
+STORE, 139691047276544, 139691047419903,
+STORE, 139691047419904, 139691049529343,
+ERASE, 139691047419904, 139691049529343,
+STORE, 139691049517056, 139691049525247,
+STORE, 139691049525248, 139691049529343,
+STORE, 140729281679360, 140729281683455,
+STORE, 140729281667072, 140729281679359,
+STORE, 139691049488384, 139691049517055,
+STORE, 139691049480192, 139691049488383,
+STORE, 139691043479552, 139691047276543,
+SNULL, 139691043479552, 139691045138431,
+STORE, 139691045138432, 139691047276543,
+STORE, 139691043479552, 139691045138431,
+SNULL, 139691047235583, 139691047276543,
+STORE, 139691045138432, 139691047235583,
+STORE, 139691047235584, 139691047276543,
+SNULL, 139691047235584, 139691047260159,
+STORE, 139691047260160, 139691047276543,
+STORE, 139691047235584, 139691047260159,
+ERASE, 139691047235584, 139691047260159,
+STORE, 139691047235584, 139691047260159,
+ERASE, 139691047260160, 139691047276543,
+STORE, 139691047260160, 139691047276543,
+SNULL, 139691047251967, 139691047260159,
+STORE, 139691047235584, 139691047251967,
+STORE, 139691047251968, 139691047260159,
+SNULL, 94863941545983, 94863941550079,
+STORE, 94863941537792, 94863941545983,
+STORE, 94863941545984, 94863941550079,
+SNULL, 139691049521151, 139691049525247,
+STORE, 139691049517056, 139691049521151,
+STORE, 139691049521152, 139691049525247,
+ERASE, 139691049488384, 139691049517055,
+STORE, 94863951294464, 94863951429631,
+STORE, 93998209294336, 93998209507327,
+STORE, 93998211604480, 93998211608575,
+STORE, 93998211608576, 93998211616767,
+STORE, 93998211616768, 93998211629055,
+STORE, 93998227210240, 93998227615743,
+STORE, 140243029913600, 140243031572479,
+STORE, 140243031572480, 140243033669631,
+STORE, 140243033669632, 140243033686015,
+STORE, 140243033686016, 140243033694207,
+STORE, 140243033694208, 140243033710591,
+STORE, 140243033710592, 140243033722879,
+STORE, 140243033722880, 140243035815935,
+STORE, 140243035815936, 140243035820031,
+STORE, 140243035820032, 140243035824127,
+STORE, 140243035824128, 140243035967487,
+STORE, 140243036336128, 140243038019583,
+STORE, 140243038019584, 140243038035967,
+STORE, 140243038064640, 140243038068735,
+STORE, 140243038068736, 140243038072831,
+STORE, 140243038072832, 140243038076927,
+STORE, 140734976479232, 140734976618495,
+STORE, 140734977978368, 140734977990655,
+STORE, 140734977990656, 140734977994751,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722742775808, 140737488351231,
+SNULL, 140722742783999, 140737488351231,
+STORE, 140722742775808, 140722742783999,
+STORE, 140722742644736, 140722742783999,
+STORE, 93857673662464, 93857675997183,
+SNULL, 93857673875455, 93857675997183,
+STORE, 93857673662464, 93857673875455,
+STORE, 93857673875456, 93857675997183,
+ERASE, 93857673875456, 93857675997183,
+STORE, 93857675972608, 93857675984895,
+STORE, 93857675984896, 93857675997183,
+STORE, 140629677498368, 140629679751167,
+SNULL, 140629677641727, 140629679751167,
+STORE, 140629677498368, 140629677641727,
+STORE, 140629677641728, 140629679751167,
+ERASE, 140629677641728, 140629679751167,
+STORE, 140629679738880, 140629679747071,
+STORE, 140629679747072, 140629679751167,
+STORE, 140722743222272, 140722743226367,
+STORE, 140722743209984, 140722743222271,
+STORE, 140629679710208, 140629679738879,
+STORE, 140629679702016, 140629679710207,
+STORE, 140629675384832, 140629677498367,
+SNULL, 140629675384832, 140629675397119,
+STORE, 140629675397120, 140629677498367,
+STORE, 140629675384832, 140629675397119,
+SNULL, 140629677490175, 140629677498367,
+STORE, 140629675397120, 140629677490175,
+STORE, 140629677490176, 140629677498367,
+ERASE, 140629677490176, 140629677498367,
+STORE, 140629677490176, 140629677498367,
+STORE, 140629671587840, 140629675384831,
+SNULL, 140629671587840, 140629673246719,
+STORE, 140629673246720, 140629675384831,
+STORE, 140629671587840, 140629673246719,
+SNULL, 140629675343871, 140629675384831,
+STORE, 140629673246720, 140629675343871,
+STORE, 140629675343872, 140629675384831,
+SNULL, 140629675343872, 140629675368447,
+STORE, 140629675368448, 140629675384831,
+STORE, 140629675343872, 140629675368447,
+ERASE, 140629675343872, 140629675368447,
+STORE, 140629675343872, 140629675368447,
+ERASE, 140629675368448, 140629675384831,
+STORE, 140629675368448, 140629675384831,
+STORE, 140629679693824, 140629679710207,
+SNULL, 140629675360255, 140629675368447,
+STORE, 140629675343872, 140629675360255,
+STORE, 140629675360256, 140629675368447,
+SNULL, 140629677494271, 140629677498367,
+STORE, 140629677490176, 140629677494271,
+STORE, 140629677494272, 140629677498367,
+SNULL, 93857675976703, 93857675984895,
+STORE, 93857675972608, 93857675976703,
+STORE, 93857675976704, 93857675984895,
+SNULL, 140629679742975, 140629679747071,
+STORE, 140629679738880, 140629679742975,
+STORE, 140629679742976, 140629679747071,
+ERASE, 140629679710208, 140629679738879,
+STORE, 93857705832448, 93857705967615,
+STORE, 140629678010368, 140629679693823,
+STORE, 93857705832448, 93857706102783,
+STORE, 93857705832448, 93857706237951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735922421760, 140737488351231,
+SNULL, 140735922429951, 140737488351231,
+STORE, 140735922421760, 140735922429951,
+STORE, 140735922290688, 140735922429951,
+STORE, 94651136139264, 94651138363391,
+SNULL, 94651136249855, 94651138363391,
+STORE, 94651136139264, 94651136249855,
+STORE, 94651136249856, 94651138363391,
+ERASE, 94651136249856, 94651138363391,
+STORE, 94651138342912, 94651138355199,
+STORE, 94651138355200, 94651138363391,
+STORE, 140325788266496, 140325790519295,
+SNULL, 140325788409855, 140325790519295,
+STORE, 140325788266496, 140325788409855,
+STORE, 140325788409856, 140325790519295,
+ERASE, 140325788409856, 140325790519295,
+STORE, 140325790507008, 140325790515199,
+STORE, 140325790515200, 140325790519295,
+STORE, 140735923572736, 140735923576831,
+STORE, 140735923560448, 140735923572735,
+STORE, 140325790478336, 140325790507007,
+STORE, 140325790470144, 140325790478335,
+STORE, 140325784469504, 140325788266495,
+SNULL, 140325784469504, 140325786128383,
+STORE, 140325786128384, 140325788266495,
+STORE, 140325784469504, 140325786128383,
+SNULL, 140325788225535, 140325788266495,
+STORE, 140325786128384, 140325788225535,
+STORE, 140325788225536, 140325788266495,
+SNULL, 140325788225536, 140325788250111,
+STORE, 140325788250112, 140325788266495,
+STORE, 140325788225536, 140325788250111,
+ERASE, 140325788225536, 140325788250111,
+STORE, 140325788225536, 140325788250111,
+ERASE, 140325788250112, 140325788266495,
+STORE, 140325788250112, 140325788266495,
+SNULL, 140325788241919, 140325788250111,
+STORE, 140325788225536, 140325788241919,
+STORE, 140325788241920, 140325788250111,
+SNULL, 94651138351103, 94651138355199,
+STORE, 94651138342912, 94651138351103,
+STORE, 94651138351104, 94651138355199,
+SNULL, 140325790511103, 140325790515199,
+STORE, 140325790507008, 140325790511103,
+STORE, 140325790511104, 140325790515199,
+ERASE, 140325790478336, 140325790507007,
+STORE, 94651146297344, 94651146432511,
+STORE, 94212330168320, 94212330381311,
+STORE, 94212332478464, 94212332482559,
+STORE, 94212332482560, 94212332490751,
+STORE, 94212332490752, 94212332503039,
+STORE, 94212348891136, 94212349825023,
+STORE, 140611630604288, 140611632263167,
+STORE, 140611632263168, 140611634360319,
+STORE, 140611634360320, 140611634376703,
+STORE, 140611634376704, 140611634384895,
+STORE, 140611634384896, 140611634401279,
+STORE, 140611634401280, 140611634413567,
+STORE, 140611634413568, 140611636506623,
+STORE, 140611636506624, 140611636510719,
+STORE, 140611636510720, 140611636514815,
+STORE, 140611636514816, 140611636658175,
+STORE, 140611637026816, 140611638710271,
+STORE, 140611638710272, 140611638726655,
+STORE, 140611638755328, 140611638759423,
+STORE, 140611638759424, 140611638763519,
+STORE, 140611638763520, 140611638767615,
+STORE, 140726974533632, 140726974672895,
+STORE, 140726974943232, 140726974955519,
+STORE, 140726974955520, 140726974959615,
+STORE, 94572463521792, 94572463734783,
+STORE, 94572465831936, 94572465836031,
+STORE, 94572465836032, 94572465844223,
+STORE, 94572465844224, 94572465856511,
+STORE, 94572491534336, 94572492865535,
+STORE, 140644351492096, 140644353150975,
+STORE, 140644353150976, 140644355248127,
+STORE, 140644355248128, 140644355264511,
+STORE, 140644355264512, 140644355272703,
+STORE, 140644355272704, 140644355289087,
+STORE, 140644355289088, 140644355301375,
+STORE, 140644355301376, 140644357394431,
+STORE, 140644357394432, 140644357398527,
+STORE, 140644357398528, 140644357402623,
+STORE, 140644357402624, 140644357545983,
+STORE, 140644357914624, 140644359598079,
+STORE, 140644359598080, 140644359614463,
+STORE, 140644359643136, 140644359647231,
+STORE, 140644359647232, 140644359651327,
+STORE, 140644359651328, 140644359655423,
+STORE, 140727841824768, 140727841964031,
+STORE, 140727843188736, 140727843201023,
+STORE, 140727843201024, 140727843205119,
+STORE, 94144315457536, 94144315670527,
+STORE, 94144317767680, 94144317771775,
+STORE, 94144317771776, 94144317779967,
+STORE, 94144317779968, 94144317792255,
+STORE, 94144318369792, 94144320815103,
+STORE, 140316717645824, 140316719304703,
+STORE, 140316719304704, 140316721401855,
+STORE, 140316721401856, 140316721418239,
+STORE, 140316721418240, 140316721426431,
+STORE, 140316721426432, 140316721442815,
+STORE, 140316721442816, 140316721455103,
+STORE, 140316721455104, 140316723548159,
+STORE, 140316723548160, 140316723552255,
+STORE, 140316723552256, 140316723556351,
+STORE, 140316723556352, 140316723699711,
+STORE, 140316724068352, 140316725751807,
+STORE, 140316725751808, 140316725768191,
+STORE, 140316725796864, 140316725800959,
+STORE, 140316725800960, 140316725805055,
+STORE, 140316725805056, 140316725809151,
+STORE, 140725744283648, 140725744422911,
+STORE, 140725745852416, 140725745864703,
+STORE, 140725745864704, 140725745868799,
+STORE, 94646858846208, 94646859059199,
+STORE, 94646861156352, 94646861160447,
+STORE, 94646861160448, 94646861168639,
+STORE, 94646861168640, 94646861180927,
+STORE, 94646879805440, 94646881894399,
+STORE, 140435449745408, 140435451404287,
+STORE, 140435451404288, 140435453501439,
+STORE, 140435453501440, 140435453517823,
+STORE, 140435453517824, 140435453526015,
+STORE, 140435453526016, 140435453542399,
+STORE, 140435453542400, 140435453554687,
+STORE, 140435453554688, 140435455647743,
+STORE, 140435455647744, 140435455651839,
+STORE, 140435455651840, 140435455655935,
+STORE, 140435455655936, 140435455799295,
+STORE, 140435456167936, 140435457851391,
+STORE, 140435457851392, 140435457867775,
+STORE, 140435457896448, 140435457900543,
+STORE, 140435457900544, 140435457904639,
+STORE, 140435457904640, 140435457908735,
+STORE, 140721033818112, 140721033957375,
+STORE, 140721034018816, 140721034031103,
+STORE, 140721034031104, 140721034035199,
+STORE, 94872903438336, 94872903651327,
+STORE, 94872905748480, 94872905752575,
+STORE, 94872905752576, 94872905760767,
+STORE, 94872905760768, 94872905773055,
+STORE, 94872931246080, 94872931651583,
+STORE, 139771607810048, 139771609468927,
+STORE, 139771609468928, 139771611566079,
+STORE, 139771611566080, 139771611582463,
+STORE, 139771611582464, 139771611590655,
+STORE, 139771611590656, 139771611607039,
+STORE, 139771611607040, 139771611619327,
+STORE, 139771611619328, 139771613712383,
+STORE, 139771613712384, 139771613716479,
+STORE, 139771613716480, 139771613720575,
+STORE, 139771613720576, 139771613863935,
+STORE, 139771614232576, 139771615916031,
+STORE, 139771615916032, 139771615932415,
+STORE, 139771615961088, 139771615965183,
+STORE, 139771615965184, 139771615969279,
+STORE, 139771615969280, 139771615973375,
+STORE, 140725402931200, 140725403070463,
+STORE, 140725403852800, 140725403865087,
+STORE, 140725403865088, 140725403869183,
+STORE, 94740737736704, 94740737949695,
+STORE, 94740740046848, 94740740050943,
+STORE, 94740740050944, 94740740059135,
+STORE, 94740740059136, 94740740071423,
+STORE, 94740743249920, 94740744724479,
+STORE, 140640287010816, 140640288669695,
+STORE, 140640288669696, 140640290766847,
+STORE, 140640290766848, 140640290783231,
+STORE, 140640290783232, 140640290791423,
+STORE, 140640290791424, 140640290807807,
+STORE, 140640290807808, 140640290820095,
+STORE, 140640290820096, 140640292913151,
+STORE, 140640292913152, 140640292917247,
+STORE, 140640292917248, 140640292921343,
+STORE, 140640292921344, 140640293064703,
+STORE, 140640293433344, 140640295116799,
+STORE, 140640295116800, 140640295133183,
+STORE, 140640295161856, 140640295165951,
+STORE, 140640295165952, 140640295170047,
+STORE, 140640295170048, 140640295174143,
+STORE, 140725133303808, 140725133443071,
+STORE, 140725133684736, 140725133697023,
+STORE, 140725133697024, 140725133701119,
+STORE, 140737488347136, 140737488351231,
+STORE, 140722826371072, 140737488351231,
+SNULL, 140722826375167, 140737488351231,
+STORE, 140722826371072, 140722826375167,
+STORE, 140722826240000, 140722826375167,
+STORE, 94113818611712, 94113820835839,
+SNULL, 94113818722303, 94113820835839,
+STORE, 94113818611712, 94113818722303,
+STORE, 94113818722304, 94113820835839,
+ERASE, 94113818722304, 94113820835839,
+STORE, 94113820815360, 94113820827647,
+STORE, 94113820827648, 94113820835839,
+STORE, 139628194508800, 139628196761599,
+SNULL, 139628194652159, 139628196761599,
+STORE, 139628194508800, 139628194652159,
+STORE, 139628194652160, 139628196761599,
+ERASE, 139628194652160, 139628196761599,
+STORE, 139628196749312, 139628196757503,
+STORE, 139628196757504, 139628196761599,
+STORE, 140722826727424, 140722826731519,
+STORE, 140722826715136, 140722826727423,
+STORE, 139628196720640, 139628196749311,
+STORE, 139628196712448, 139628196720639,
+STORE, 139628190711808, 139628194508799,
+SNULL, 139628190711808, 139628192370687,
+STORE, 139628192370688, 139628194508799,
+STORE, 139628190711808, 139628192370687,
+SNULL, 139628194467839, 139628194508799,
+STORE, 139628192370688, 139628194467839,
+STORE, 139628194467840, 139628194508799,
+SNULL, 139628194467840, 139628194492415,
+STORE, 139628194492416, 139628194508799,
+STORE, 139628194467840, 139628194492415,
+ERASE, 139628194467840, 139628194492415,
+STORE, 139628194467840, 139628194492415,
+ERASE, 139628194492416, 139628194508799,
+STORE, 139628194492416, 139628194508799,
+SNULL, 139628194484223, 139628194492415,
+STORE, 139628194467840, 139628194484223,
+STORE, 139628194484224, 139628194492415,
+SNULL, 94113820823551, 94113820827647,
+STORE, 94113820815360, 94113820823551,
+STORE, 94113820823552, 94113820827647,
+SNULL, 139628196753407, 139628196757503,
+STORE, 139628196749312, 139628196753407,
+STORE, 139628196753408, 139628196757503,
+ERASE, 139628196720640, 139628196749311,
+STORE, 94113830850560, 94113830985727,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731865833472, 140737488351231,
+SNULL, 140731865841663, 140737488351231,
+STORE, 140731865833472, 140731865841663,
+STORE, 140731865702400, 140731865841663,
+STORE, 94763339386880, 94763341611007,
+SNULL, 94763339497471, 94763341611007,
+STORE, 94763339386880, 94763339497471,
+STORE, 94763339497472, 94763341611007,
+ERASE, 94763339497472, 94763341611007,
+STORE, 94763341590528, 94763341602815,
+STORE, 94763341602816, 94763341611007,
+STORE, 139778398486528, 139778400739327,
+SNULL, 139778398629887, 139778400739327,
+STORE, 139778398486528, 139778398629887,
+STORE, 139778398629888, 139778400739327,
+ERASE, 139778398629888, 139778400739327,
+STORE, 139778400727040, 139778400735231,
+STORE, 139778400735232, 139778400739327,
+STORE, 140731865858048, 140731865862143,
+STORE, 140731865845760, 140731865858047,
+STORE, 139778400698368, 139778400727039,
+STORE, 139778400690176, 139778400698367,
+STORE, 139778394689536, 139778398486527,
+SNULL, 139778394689536, 139778396348415,
+STORE, 139778396348416, 139778398486527,
+STORE, 139778394689536, 139778396348415,
+SNULL, 139778398445567, 139778398486527,
+STORE, 139778396348416, 139778398445567,
+STORE, 139778398445568, 139778398486527,
+SNULL, 139778398445568, 139778398470143,
+STORE, 139778398470144, 139778398486527,
+STORE, 139778398445568, 139778398470143,
+ERASE, 139778398445568, 139778398470143,
+STORE, 139778398445568, 139778398470143,
+ERASE, 139778398470144, 139778398486527,
+STORE, 139778398470144, 139778398486527,
+SNULL, 139778398461951, 139778398470143,
+STORE, 139778398445568, 139778398461951,
+STORE, 139778398461952, 139778398470143,
+SNULL, 94763341598719, 94763341602815,
+STORE, 94763341590528, 94763341598719,
+STORE, 94763341598720, 94763341602815,
+SNULL, 139778400731135, 139778400735231,
+STORE, 139778400727040, 139778400731135,
+STORE, 139778400731136, 139778400735231,
+ERASE, 139778400698368, 139778400727039,
+STORE, 94763362197504, 94763362332671,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140737488338944, 140737488351231,
+STORE, 140732053192704, 140737488351231,
+SNULL, 140732053204991, 140737488351231,
+STORE, 140732053192704, 140732053204991,
+STORE, 140732053061632, 140732053204991,
+STORE, 4194304, 26279935,
+STORE, 28372992, 28454911,
+STORE, 28454912, 29806591,
+STORE, 140176018599936, 140176020852735,
+SNULL, 140176018743295, 140176020852735,
+STORE, 140176018599936, 140176018743295,
+STORE, 140176018743296, 140176020852735,
+ERASE, 140176018743296, 140176020852735,
+STORE, 140176020840448, 140176020848639,
+STORE, 140176020848640, 140176020852735,
+STORE, 140732053381120, 140732053385215,
+STORE, 140732053368832, 140732053381119,
+STORE, 140176020811776, 140176020840447,
+STORE, 140176020803584, 140176020811775,
+STORE, 140176014766080, 140176018599935,
+SNULL, 140176014766080, 140176016474111,
+STORE, 140176016474112, 140176018599935,
+STORE, 140176014766080, 140176016474111,
+SNULL, 140176018567167, 140176018599935,
+STORE, 140176016474112, 140176018567167,
+STORE, 140176018567168, 140176018599935,
+ERASE, 140176018567168, 140176018599935,
+STORE, 140176018567168, 140176018599935,
+STORE, 140176012570624, 140176014766079,
+SNULL, 140176012570624, 140176012664831,
+STORE, 140176012664832, 140176014766079,
+STORE, 140176012570624, 140176012664831,
+SNULL, 140176014757887, 140176014766079,
+STORE, 140176012664832, 140176014757887,
+STORE, 140176014757888, 140176014766079,
+ERASE, 140176014757888, 140176014766079,
+STORE, 140176014757888, 140176014766079,
+STORE, 140176010051584, 140176012570623,
+SNULL, 140176010051584, 140176010465279,
+STORE, 140176010465280, 140176012570623,
+STORE, 140176010051584, 140176010465279,
+SNULL, 140176012558335, 140176012570623,
+STORE, 140176010465280, 140176012558335,
+STORE, 140176012558336, 140176012570623,
+ERASE, 140176012558336, 140176012570623,
+STORE, 140176012558336, 140176012570623,
+STORE, 140176007417856, 140176010051583,
+SNULL, 140176007417856, 140176007946239,
+STORE, 140176007946240, 140176010051583,
+STORE, 140176007417856, 140176007946239,
+SNULL, 140176010043391, 140176010051583,
+STORE, 140176007946240, 140176010043391,
+STORE, 140176010043392, 140176010051583,
+ERASE, 140176010043392, 140176010051583,
+STORE, 140176010043392, 140176010051583,
+STORE, 140176005304320, 140176007417855,
+SNULL, 140176005304320, 140176005316607,
+STORE, 140176005316608, 140176007417855,
+STORE, 140176005304320, 140176005316607,
+SNULL, 140176007409663, 140176007417855,
+STORE, 140176005316608, 140176007409663,
+STORE, 140176007409664, 140176007417855,
+ERASE, 140176007409664, 140176007417855,
+STORE, 140176007409664, 140176007417855,
+STORE, 140176003100672, 140176005304319,
+SNULL, 140176003100672, 140176003203071,
+STORE, 140176003203072, 140176005304319,
+STORE, 140176003100672, 140176003203071,
+SNULL, 140176005296127, 140176005304319,
+STORE, 140176003203072, 140176005296127,
+STORE, 140176005296128, 140176005304319,
+ERASE, 140176005296128, 140176005304319,
+STORE, 140176005296128, 140176005304319,
+STORE, 140176020795392, 140176020811775,
+STORE, 140175999938560, 140176003100671,
+SNULL, 140175999938560, 140176000999423,
+STORE, 140176000999424, 140176003100671,
+STORE, 140175999938560, 140176000999423,
+SNULL, 140176003092479, 140176003100671,
+STORE, 140176000999424, 140176003092479,
+STORE, 140176003092480, 140176003100671,
+ERASE, 140176003092480, 140176003100671,
+STORE, 140176003092480, 140176003100671,
+STORE, 140175996141568, 140175999938559,
+SNULL, 140175996141568, 140175997800447,
+STORE, 140175997800448, 140175999938559,
+STORE, 140175996141568, 140175997800447,
+SNULL, 140175999897599, 140175999938559,
+STORE, 140175997800448, 140175999897599,
+STORE, 140175999897600, 140175999938559,
+SNULL, 140175999897600, 140175999922175,
+STORE, 140175999922176, 140175999938559,
+STORE, 140175999897600, 140175999922175,
+ERASE, 140175999897600, 140175999922175,
+STORE, 140175999897600, 140175999922175,
+ERASE, 140175999922176, 140175999938559,
+STORE, 140175999922176, 140175999938559,
+STORE, 140176020783104, 140176020811775,
+SNULL, 140175999913983, 140175999922175,
+STORE, 140175999897600, 140175999913983,
+STORE, 140175999913984, 140175999922175,
+SNULL, 140176003096575, 140176003100671,
+STORE, 140176003092480, 140176003096575,
+STORE, 140176003096576, 140176003100671,
+SNULL, 140176005300223, 140176005304319,
+STORE, 140176005296128, 140176005300223,
+STORE, 140176005300224, 140176005304319,
+SNULL, 140176007413759, 140176007417855,
+STORE, 140176007409664, 140176007413759,
+STORE, 140176007413760, 140176007417855,
+SNULL, 140176010047487, 140176010051583,
+STORE, 140176010043392, 140176010047487,
+STORE, 140176010047488, 140176010051583,
+SNULL, 140176012566527, 140176012570623,
+STORE, 140176012558336, 140176012566527,
+STORE, 140176012566528, 140176012570623,
+SNULL, 140176014761983, 140176014766079,
+STORE, 140176014757888, 140176014761983,
+STORE, 140176014761984, 140176014766079,
+SNULL, 140176018571263, 140176018599935,
+STORE, 140176018567168, 140176018571263,
+STORE, 140176018571264, 140176018599935,
+SNULL, 28405759, 28454911,
+STORE, 28372992, 28405759,
+STORE, 28405760, 28454911,
+SNULL, 140176020844543, 140176020848639,
+STORE, 140176020840448, 140176020844543,
+STORE, 140176020844544, 140176020848639,
+ERASE, 140176020811776, 140176020840447,
+STORE, 53080064, 53215231,
+STORE, 140176019099648, 140176020783103,
+STORE, 140176020836352, 140176020840447,
+STORE, 140176018964480, 140176019099647,
+STORE, 53080064, 53358591,
+STORE, 140175994044416, 140175996141567,
+STORE, 140176020828160, 140176020840447,
+STORE, 140176020819968, 140176020840447,
+STORE, 140176020783104, 140176020819967,
+STORE, 140176018948096, 140176019099647,
+STORE, 53080064, 53493759,
+STORE, 53080064, 53649407,
+STORE, 140176018939904, 140176019099647,
+STORE, 140176018931712, 140176019099647,
+STORE, 53080064, 53784575,
+STORE, 53080064, 53919743,
+STORE, 140176018915328, 140176019099647,
+STORE, 140176018907136, 140176019099647,
+STORE, 53080064, 54059007,
+STORE, 140175993769984, 140175996141567,
+STORE, 140176018747392, 140176019099647,
+STORE, 53080064, 54198271,
+SNULL, 54190079, 54198271,
+STORE, 53080064, 54190079,
+STORE, 54190080, 54198271,
+ERASE, 54190080, 54198271,
+SNULL, 54181887, 54190079,
+STORE, 53080064, 54181887,
+STORE, 54181888, 54190079,
+ERASE, 54181888, 54190079,
+SNULL, 54173695, 54181887,
+STORE, 53080064, 54173695,
+STORE, 54173696, 54181887,
+ERASE, 54173696, 54181887,
+SNULL, 54165503, 54173695,
+STORE, 53080064, 54165503,
+STORE, 54165504, 54173695,
+ERASE, 54165504, 54173695,
+STORE, 140175993753600, 140175996141567,
+STORE, 140175993688064, 140175996141567,
+STORE, 140175993655296, 140175996141567,
+STORE, 140175991558144, 140175996141567,
+STORE, 140175991492608, 140175996141567,
+STORE, 53080064, 54312959,
+STORE, 140175991361536, 140175996141567,
+STORE, 140175991099392, 140175996141567,
+STORE, 140175991091200, 140175996141567,
+STORE, 140175991074816, 140175996141567,
+STORE, 140175991066624, 140175996141567,
+STORE, 140175991058432, 140175996141567,
+STORE, 53080064, 54448127,
+SNULL, 54439935, 54448127,
+STORE, 53080064, 54439935,
+STORE, 54439936, 54448127,
+ERASE, 54439936, 54448127,
+SNULL, 54431743, 54439935,
+STORE, 53080064, 54431743,
+STORE, 54431744, 54439935,
+ERASE, 54431744, 54439935,
+SNULL, 54419455, 54431743,
+STORE, 53080064, 54419455,
+STORE, 54419456, 54431743,
+ERASE, 54419456, 54431743,
+SNULL, 54403071, 54419455,
+STORE, 53080064, 54403071,
+STORE, 54403072, 54419455,
+ERASE, 54403072, 54419455,
+STORE, 140175991042048, 140175996141567,
+STORE, 53080064, 54538239,
+SNULL, 54534143, 54538239,
+STORE, 53080064, 54534143,
+STORE, 54534144, 54538239,
+ERASE, 54534144, 54538239,
+SNULL, 54530047, 54534143,
+STORE, 53080064, 54530047,
+STORE, 54530048, 54534143,
+ERASE, 54530048, 54534143,
+SNULL, 54525951, 54530047,
+STORE, 53080064, 54525951,
+STORE, 54525952, 54530047,
+ERASE, 54525952, 54530047,
+SNULL, 54521855, 54525951,
+STORE, 53080064, 54521855,
+STORE, 54521856, 54525951,
+ERASE, 54521856, 54525951,
+SNULL, 54517759, 54521855,
+STORE, 53080064, 54517759,
+STORE, 54517760, 54521855,
+ERASE, 54517760, 54521855,
+SNULL, 54513663, 54517759,
+STORE, 53080064, 54513663,
+STORE, 54513664, 54517759,
+ERASE, 54513664, 54517759,
+SNULL, 54509567, 54513663,
+STORE, 53080064, 54509567,
+STORE, 54509568, 54513663,
+ERASE, 54509568, 54513663,
+STORE, 140175991025664, 140175996141567,
+STORE, 140175990992896, 140175996141567,
+STORE, 53080064, 54644735,
+SNULL, 54628351, 54644735,
+STORE, 53080064, 54628351,
+STORE, 54628352, 54644735,
+ERASE, 54628352, 54644735,
+SNULL, 54616063, 54628351,
+STORE, 53080064, 54616063,
+STORE, 54616064, 54628351,
+ERASE, 54616064, 54628351,
+STORE, 140175988895744, 140175996141567,
+STORE, 53080064, 54767615,
+STORE, 140175988879360, 140175996141567,
+STORE, 140175988617216, 140175996141567,
+STORE, 140175988609024, 140175996141567,
+STORE, 140175988600832, 140175996141567,
+STORE, 53080064, 54906879,
+SNULL, 54898687, 54906879,
+STORE, 53080064, 54898687,
+STORE, 54898688, 54906879,
+ERASE, 54898688, 54906879,
+SNULL, 54853631, 54898687,
+STORE, 53080064, 54853631,
+STORE, 54853632, 54898687,
+ERASE, 54853632, 54898687,
+STORE, 140175986503680, 140175996141567,
+STORE, 53080064, 54996991,
+STORE, 140175986495488, 140175996141567,
+STORE, 140175986487296, 140175996141567,
+STORE, 140175985438720, 140175996141567,
+STORE, 53080064, 55136255,
+STORE, 140175985405952, 140175996141567,
+STORE, 140175985139712, 140175996141567,
+SNULL, 140176018964479, 140176019099647,
+STORE, 140176018747392, 140176018964479,
+STORE, 140176018964480, 140176019099647,
+ERASE, 140176018964480, 140176019099647,
+STORE, 140175983042560, 140175996141567,
+STORE, 140175982518272, 140175996141567,
+STORE, 140175980421120, 140175996141567,
+STORE, 53080064, 55287807,
+STORE, 53080064, 55427071,
+STORE, 140176019091456, 140176019099647,
+STORE, 140176019083264, 140176019099647,
+STORE, 140176019075072, 140176019099647,
+STORE, 140176019066880, 140176019099647,
+STORE, 140176019058688, 140176019099647,
+STORE, 140175980158976, 140175996141567,
+STORE, 140176019050496, 140176019099647,
+STORE, 140176019042304, 140176019099647,
+STORE, 140176019034112, 140176019099647,
+STORE, 140176019025920, 140176019099647,
+STORE, 140176019017728, 140176019099647,
+STORE, 140176019009536, 140176019099647,
+STORE, 140176019001344, 140176019099647,
+STORE, 140176018993152, 140176019099647,
+STORE, 140176018984960, 140176019099647,
+STORE, 140176018976768, 140176019099647,
+STORE, 140176018968576, 140176019099647,
+STORE, 140175978061824, 140175996141567,
+STORE, 53080064, 55603199,
+STORE, 140175978029056, 140175996141567,
+STORE, 140175977996288, 140175996141567,
+STORE, 53080064, 55738367,
+STORE, 53080064, 55881727,
+STORE, 140175977963520, 140175996141567,
+STORE, 140175977930752, 140175996141567,
+STORE, 53080064, 56041471,
+STORE, 140175977897984, 140175996141567,
+STORE, 140175977865216, 140175996141567,
+SNULL, 55881727, 56041471,
+STORE, 53080064, 55881727,
+STORE, 55881728, 56041471,
+ERASE, 55881728, 56041471,
+SNULL, 55721983, 55881727,
+STORE, 53080064, 55721983,
+STORE, 55721984, 55881727,
+ERASE, 55721984, 55881727,
+SNULL, 55570431, 55721983,
+STORE, 53080064, 55570431,
+STORE, 55570432, 55721983,
+ERASE, 55570432, 55721983,
+STORE, 140175977857024, 140175996141567,
+STORE, 140175975759872, 140175996141567,
+STORE, 53080064, 55754751,
+STORE, 53080064, 55943167,
+STORE, 140175975751680, 140175996141567,
+STORE, 140175975743488, 140175996141567,
+STORE, 140175975735296, 140175996141567,
+STORE, 140175975727104, 140175996141567,
+STORE, 140175975718912, 140175996141567,
+STORE, 140175975710720, 140175996141567,
+STORE, 140175975702528, 140175996141567,
+STORE, 140175975694336, 140175996141567,
+STORE, 140175975686144, 140175996141567,
+STORE, 140175975677952, 140175996141567,
+STORE, 140175975669760, 140175996141567,
+STORE, 140175974621184, 140175996141567,
+STORE, 140175974612992, 140175996141567,
+STORE, 53080064, 56139775,
+STORE, 140175972515840, 140175996141567,
+STORE, 53080064, 56401919,
+STORE, 140175970418688, 140175996141567,
+STORE, 140175970410496, 140175996141567,
+STORE, 140175970402304, 140175996141567,
+STORE, 140175970394112, 140175996141567,
+STORE, 53080064, 56569855,
+STORE, 140175969865728, 140175996141567,
+SNULL, 140175985139711, 140175996141567,
+STORE, 140175969865728, 140175985139711,
+STORE, 140175985139712, 140175996141567,
+SNULL, 140175985139712, 140175985405951,
+STORE, 140175985405952, 140175996141567,
+STORE, 140175985139712, 140175985405951,
+ERASE, 140175985139712, 140175985405951,
+STORE, 140175965671424, 140175985139711,
+STORE, 140175985397760, 140175996141567,
+STORE, 140175985389568, 140175996141567,
+STORE, 140175985381376, 140175996141567,
+STORE, 140175985373184, 140175996141567,
+STORE, 140175985364992, 140175996141567,
+STORE, 140175985356800, 140175996141567,
+STORE, 140175985348608, 140175996141567,
+STORE, 140175985340416, 140175996141567,
+STORE, 140175985332224, 140175996141567,
+STORE, 140175985324032, 140175996141567,
+STORE, 140175985315840, 140175996141567,
+STORE, 140175985307648, 140175996141567,
+STORE, 140175985299456, 140175996141567,
+STORE, 140175985291264, 140175996141567,
+STORE, 140175985283072, 140175996141567,
+STORE, 140175985274880, 140175996141567,
+STORE, 140175963574272, 140175985139711,
+STORE, 140175985266688, 140175996141567,
+STORE, 140175961477120, 140175985139711,
+STORE, 53080064, 56831999,
+STORE, 140175959379968, 140175985139711,
+STORE, 140175985258496, 140175996141567,
+STORE, 140175957282816, 140175985139711,
+STORE, 140175985250304, 140175996141567,
+STORE, 140175985242112, 140175996141567,
+STORE, 140175985233920, 140175996141567,
+STORE, 140175985225728, 140175996141567,
+STORE, 140175985217536, 140175996141567,
+STORE, 140175957151744, 140175985139711,
+STORE, 140175956627456, 140175985139711,
+SNULL, 140175980158975, 140175985139711,
+STORE, 140175956627456, 140175980158975,
+STORE, 140175980158976, 140175985139711,
+SNULL, 140175980158976, 140175980421119,
+STORE, 140175980421120, 140175985139711,
+STORE, 140175980158976, 140175980421119,
+ERASE, 140175980158976, 140175980421119,
+STORE, 140175954530304, 140175980158975,
+STORE, 140175985209344, 140175996141567,
+STORE, 53080064, 57094143,
+STORE, 140175952433152, 140175980158975,
+STORE, 140175985192960, 140175996141567,
+STORE, 140175985184768, 140175996141567,
+STORE, 140175985176576, 140175996141567,
+STORE, 140175985168384, 140175996141567,
+STORE, 140175985160192, 140175996141567,
+STORE, 140175985152000, 140175996141567,
+STORE, 140175985143808, 140175996141567,
+STORE, 140175980412928, 140175985139711,
+STORE, 140175980404736, 140175985139711,
+STORE, 140175980396544, 140175985139711,
+STORE, 140175980388352, 140175985139711,
+STORE, 140175980380160, 140175985139711,
+STORE, 140175980371968, 140175985139711,
+STORE, 140175980363776, 140175985139711,
+STORE, 140175980355584, 140175985139711,
+STORE, 140175980347392, 140175985139711,
+STORE, 140175980339200, 140175985139711,
+STORE, 53080064, 57356287,
+SNULL, 140176018747392, 140176018907135,
+STORE, 140176018907136, 140176018964479,
+STORE, 140176018747392, 140176018907135,
+ERASE, 140176018747392, 140176018907135,
+STORE, 140175952146432, 140175980158975,
+STORE, 140175950049280, 140175980158975,
+SNULL, 140175952146431, 140175980158975,
+STORE, 140175950049280, 140175952146431,
+STORE, 140175952146432, 140175980158975,
+SNULL, 140175952146432, 140175952433151,
+STORE, 140175952433152, 140175980158975,
+STORE, 140175952146432, 140175952433151,
+ERASE, 140175952146432, 140175952433151,
+STORE, 140176018898944, 140176018964479,
+STORE, 53080064, 57749503,
+STORE, 140175949520896, 140175952146431,
+STORE, 140175947423744, 140175952146431,
+SNULL, 140175993769983, 140175996141567,
+STORE, 140175985143808, 140175993769983,
+STORE, 140175993769984, 140175996141567,
+SNULL, 140175993769984, 140175994044415,
+STORE, 140175994044416, 140175996141567,
+STORE, 140175993769984, 140175994044415,
+ERASE, 140175993769984, 140175994044415,
+STORE, 140176018890752, 140176018964479,
+STORE, 140176018882560, 140176018964479,
+STORE, 140176018874368, 140176018964479,
+STORE, 140176018866176, 140176018964479,
+STORE, 140176018849792, 140176018964479,
+STORE, 140176018841600, 140176018964479,
+STORE, 140176018825216, 140176018964479,
+STORE, 140176018817024, 140176018964479,
+STORE, 140176018800640, 140176018964479,
+STORE, 140176018792448, 140176018964479,
+STORE, 140176018759680, 140176018964479,
+STORE, 140176018751488, 140176018964479,
+STORE, 140175994028032, 140175996141567,
+STORE, 140176018743296, 140176018964479,
+STORE, 140175994011648, 140175996141567,
+STORE, 140175994003456, 140175996141567,
+STORE, 140175993987072, 140175996141567,
+STORE, 140175993978880, 140175996141567,
+STORE, 140175993946112, 140175996141567,
+STORE, 140175993937920, 140175996141567,
+STORE, 140175993921536, 140175996141567,
+STORE, 140175993913344, 140175996141567,
+STORE, 140175993896960, 140175996141567,
+STORE, 140175993888768, 140175996141567,
+STORE, 140175993872384, 140175996141567,
+STORE, 140175993864192, 140175996141567,
+STORE, 140175993831424, 140175996141567,
+STORE, 140175993823232, 140175996141567,
+STORE, 140175993806848, 140175996141567,
+STORE, 140175993798656, 140175996141567,
+STORE, 140175993782272, 140175996141567,
+STORE, 140175993774080, 140175996141567,
+STORE, 140175980322816, 140175985139711,
+STORE, 140175980314624, 140175985139711,
+STORE, 140175980281856, 140175985139711,
+STORE, 140175980273664, 140175985139711,
+STORE, 140175980257280, 140175985139711,
+STORE, 140175945326592, 140175952146431,
+STORE, 140175980249088, 140175985139711,
+STORE, 140175980232704, 140175985139711,
+STORE, 140175980224512, 140175985139711,
+STORE, 140175980208128, 140175985139711,
+STORE, 140175980199936, 140175985139711,
+STORE, 140175980167168, 140175985139711,
+STORE, 140175952433152, 140175985139711,
+STORE, 140175952416768, 140175985139711,
+STORE, 140175952408576, 140175985139711,
+STORE, 140175952392192, 140175985139711,
+STORE, 140175952384000, 140175985139711,
+STORE, 140175952367616, 140175985139711,
+STORE, 140175943229440, 140175952146431,
+STORE, 140175952359424, 140175985139711,
+STORE, 140175952326656, 140175985139711,
+STORE, 140175952318464, 140175985139711,
+STORE, 140175952302080, 140175985139711,
+STORE, 140175952293888, 140175985139711,
+STORE, 140175952277504, 140175985139711,
+STORE, 140175952269312, 140175985139711,
+STORE, 140175952252928, 140175985139711,
+STORE, 140175952244736, 140175985139711,
+STORE, 140175952211968, 140175985139711,
+STORE, 140175952203776, 140175985139711,
+STORE, 140175952187392, 140175985139711,
+STORE, 140175952179200, 140175985139711,
+STORE, 140175952162816, 140175985139711,
+STORE, 140175952154624, 140175985139711,
+STORE, 140175943213056, 140175952146431,
+STORE, 140175943213056, 140175985139711,
+STORE, 140175943180288, 140175985139711,
+STORE, 140175943172096, 140175985139711,
+STORE, 140175943155712, 140175985139711,
+STORE, 140175943147520, 140175985139711,
+STORE, 140175943131136, 140175985139711,
+STORE, 140175943122944, 140175985139711,
+STORE, 140175943106560, 140175985139711,
+STORE, 140175943098368, 140175985139711,
+STORE, 140175943065600, 140175985139711,
+STORE, 140175943057408, 140175985139711,
+STORE, 140175943041024, 140175985139711,
+STORE, 140175943032832, 140175985139711,
+STORE, 140175943016448, 140175985139711,
+STORE, 140175943008256, 140175985139711,
+STORE, 140175942991872, 140175985139711,
+STORE, 140175942983680, 140175985139711,
+STORE, 140175942950912, 140175985139711,
+STORE, 140175942942720, 140175985139711,
+STORE, 140175942926336, 140175985139711,
+STORE, 140175942918144, 140175985139711,
+STORE, 140175942901760, 140175985139711,
+STORE, 140175942893568, 140175985139711,
+STORE, 140175942877184, 140175985139711,
+STORE, 140175942868992, 140175985139711,
+STORE, 140175942836224, 140175985139711,
+STORE, 140175942828032, 140175985139711,
+STORE, 140175942811648, 140175985139711,
+STORE, 140175942803456, 140175985139711,
+STORE, 140175942787072, 140175985139711,
+STORE, 140175942778880, 140175985139711,
+STORE, 140175942762496, 140175985139711,
+STORE, 140175942754304, 140175985139711,
+STORE, 140175942721536, 140175985139711,
+STORE, 140175942713344, 140175985139711,
+STORE, 140175942696960, 140175985139711,
+STORE, 140175942688768, 140175985139711,
+STORE, 140175942672384, 140175985139711,
+STORE, 140175942664192, 140175985139711,
+STORE, 140175942647808, 140175985139711,
+STORE, 140175942639616, 140175985139711,
+STORE, 140175942606848, 140175985139711,
+STORE, 140175942598656, 140175985139711,
+STORE, 140175942582272, 140175985139711,
+STORE, 140175942574080, 140175985139711,
+STORE, 140175942557696, 140175985139711,
+STORE, 140175942549504, 140175985139711,
+STORE, 140175942533120, 140175985139711,
+STORE, 140175942524928, 140175985139711,
+STORE, 140175942492160, 140175985139711,
+STORE, 140175942483968, 140175985139711,
+STORE, 140175942467584, 140175985139711,
+STORE, 140175942459392, 140175985139711,
+STORE, 140175942443008, 140175985139711,
+STORE, 140175942434816, 140175985139711,
+STORE, 140175942418432, 140175985139711,
+STORE, 140175942410240, 140175985139711,
+STORE, 140175942377472, 140175985139711,
+STORE, 140175942369280, 140175985139711,
+STORE, 140175942352896, 140175985139711,
+STORE, 140175942344704, 140175985139711,
+STORE, 140175942328320, 140175985139711,
+STORE, 140175942320128, 140175985139711,
+STORE, 140175942303744, 140175985139711,
+STORE, 140175942295552, 140175985139711,
+STORE, 140175942262784, 140175985139711,
+STORE, 140175942254592, 140175985139711,
+STORE, 140175942238208, 140175985139711,
+STORE, 140175942230016, 140175985139711,
+STORE, 140175942213632, 140175985139711,
+STORE, 140175942205440, 140175985139711,
+STORE, 140175942189056, 140175985139711,
+STORE, 140175942180864, 140175985139711,
+STORE, 140175942148096, 140175985139711,
+STORE, 140175942139904, 140175985139711,
+STORE, 140175942123520, 140175985139711,
+STORE, 140175942115328, 140175985139711,
+STORE, 140175942098944, 140175985139711,
+STORE, 140175942090752, 140175985139711,
+STORE, 140175942074368, 140175985139711,
+STORE, 140175942066176, 140175985139711,
+STORE, 140175942033408, 140175985139711,
+STORE, 140175942025216, 140175985139711,
+STORE, 140175942008832, 140175985139711,
+STORE, 140175942000640, 140175985139711,
+STORE, 140175941984256, 140175985139711,
+STORE, 140175941976064, 140175985139711,
+STORE, 140175941959680, 140175985139711,
+STORE, 140175939862528, 140175985139711,
+STORE, 140175939854336, 140175985139711,
+STORE, 140175939821568, 140175985139711,
+STORE, 140175939813376, 140175985139711,
+STORE, 140175939796992, 140175985139711,
+STORE, 140175939788800, 140175985139711,
+STORE, 140175939772416, 140175985139711,
+STORE, 140175939764224, 140175985139711,
+STORE, 140175939747840, 140175985139711,
+STORE, 140175939739648, 140175985139711,
+STORE, 140175939706880, 140175985139711,
+STORE, 140175939698688, 140175985139711,
+STORE, 140175939682304, 140175985139711,
+STORE, 140175939674112, 140175985139711,
+STORE, 140175939657728, 140175985139711,
+STORE, 140175939649536, 140175985139711,
+STORE, 140175939633152, 140175985139711,
+STORE, 140175939624960, 140175985139711,
+STORE, 140175939592192, 140175985139711,
+STORE, 140175939584000, 140175985139711,
+STORE, 140175939567616, 140175985139711,
+STORE, 140175939559424, 140175985139711,
+STORE, 140175939543040, 140175985139711,
+STORE, 140175939534848, 140175985139711,
+STORE, 140175939518464, 140175985139711,
+STORE, 140175939510272, 140175985139711,
+STORE, 140175939477504, 140175985139711,
+STORE, 140175939469312, 140175985139711,
+STORE, 140175939452928, 140175985139711,
+STORE, 140175939444736, 140175985139711,
+STORE, 140175939428352, 140175985139711,
+STORE, 140175939420160, 140175985139711,
+STORE, 140175939403776, 140175985139711,
+STORE, 140175939395584, 140175985139711,
+STORE, 140175939362816, 140175985139711,
+STORE, 140175939354624, 140175985139711,
+STORE, 140175939338240, 140175985139711,
+STORE, 140175939330048, 140175985139711,
+STORE, 140175939313664, 140175985139711,
+STORE, 140175939305472, 140175985139711,
+STORE, 140175939289088, 140175985139711,
+STORE, 140175939280896, 140175985139711,
+STORE, 140175939248128, 140175985139711,
+STORE, 140175939239936, 140175985139711,
+STORE, 140175939223552, 140175985139711,
+STORE, 140175939215360, 140175985139711,
+STORE, 140175939198976, 140175985139711,
+STORE, 140175939190784, 140175985139711,
+STORE, 140175939174400, 140175985139711,
+STORE, 140175939166208, 140175985139711,
+STORE, 140175939133440, 140175985139711,
+STORE, 140175939125248, 140175985139711,
+STORE, 140175939108864, 140175985139711,
+STORE, 140175939100672, 140175985139711,
+STORE, 140175939084288, 140175985139711,
+STORE, 140175939076096, 140175985139711,
+STORE, 140175939059712, 140175985139711,
+STORE, 140175939051520, 140175985139711,
+STORE, 140175939018752, 140175985139711,
+STORE, 140175939010560, 140175985139711,
+STORE, 140175938994176, 140175985139711,
+STORE, 140175938985984, 140175985139711,
+STORE, 140175938969600, 140175985139711,
+STORE, 140175938961408, 140175985139711,
+STORE, 140175938945024, 140175985139711,
+STORE, 140175938936832, 140175985139711,
+STORE, 140175938904064, 140175985139711,
+STORE, 140175938895872, 140175985139711,
+STORE, 140175938879488, 140175985139711,
+STORE, 140175938871296, 140175985139711,
+STORE, 140175938854912, 140175985139711,
+STORE, 140175938846720, 140175985139711,
+STORE, 140175938830336, 140175985139711,
+STORE, 140175938822144, 140175985139711,
+STORE, 140175938789376, 140175985139711,
+STORE, 140175938781184, 140175985139711,
+STORE, 140175938764800, 140175985139711,
+STORE, 140175938756608, 140175985139711,
+STORE, 140175938740224, 140175985139711,
+STORE, 140175938732032, 140175985139711,
+STORE, 140175938715648, 140175985139711,
+STORE, 140175938707456, 140175985139711,
+STORE, 140175938674688, 140175985139711,
+STORE, 140175938666496, 140175985139711,
+STORE, 140175938650112, 140175985139711,
+STORE, 140175938641920, 140175985139711,
+STORE, 140175938625536, 140175985139711,
+STORE, 140175938617344, 140175985139711,
+STORE, 140175938600960, 140175985139711,
+STORE, 140175938592768, 140175985139711,
+STORE, 140175938560000, 140175985139711,
+STORE, 140175938551808, 140175985139711,
+STORE, 140175938535424, 140175985139711,
+STORE, 140175938527232, 140175985139711,
+STORE, 140175938510848, 140175985139711,
+STORE, 140175938502656, 140175985139711,
+STORE, 140175938486272, 140175985139711,
+STORE, 140175938478080, 140175985139711,
+STORE, 140175938445312, 140175985139711,
+STORE, 140175938437120, 140175985139711,
+STORE, 140175938420736, 140175985139711,
+STORE, 140175938412544, 140175985139711,
+STORE, 140175938396160, 140175985139711,
+STORE, 140175938387968, 140175985139711,
+STORE, 140175938371584, 140175985139711,
+STORE, 140175938363392, 140175985139711,
+STORE, 140175938330624, 140175985139711,
+STORE, 140175938322432, 140175985139711,
+STORE, 140175938306048, 140175985139711,
+STORE, 140175938297856, 140175985139711,
+STORE, 140175938281472, 140175985139711,
+STORE, 140175938273280, 140175985139711,
+STORE, 140175938256896, 140175985139711,
+STORE, 140175938248704, 140175985139711,
+STORE, 140175938215936, 140175985139711,
+STORE, 140175938207744, 140175985139711,
+STORE, 140175938191360, 140175985139711,
+STORE, 140175938183168, 140175985139711,
+STORE, 140175938166784, 140175985139711,
+STORE, 140175938158592, 140175985139711,
+STORE, 140175938142208, 140175985139711,
+STORE, 140175936045056, 140175985139711,
+STORE, 140175936036864, 140175985139711,
+STORE, 140175936004096, 140175985139711,
+STORE, 140175935995904, 140175985139711,
+STORE, 140175935979520, 140175985139711,
+STORE, 140175935971328, 140175985139711,
+STORE, 140175935954944, 140175985139711,
+STORE, 140175935946752, 140175985139711,
+STORE, 140175935930368, 140175985139711,
+STORE, 140175935922176, 140175985139711,
+STORE, 140175935889408, 140175985139711,
+STORE, 140175935881216, 140175985139711,
+STORE, 140175935864832, 140175985139711,
+STORE, 140175935856640, 140175985139711,
+STORE, 140175935840256, 140175985139711,
+STORE, 140175935832064, 140175985139711,
+STORE, 140175935815680, 140175985139711,
+STORE, 140175935807488, 140175985139711,
+STORE, 140175935774720, 140175985139711,
+STORE, 140175935766528, 140175985139711,
+STORE, 140175935750144, 140175985139711,
+STORE, 140175935741952, 140175985139711,
+STORE, 140175935725568, 140175985139711,
+STORE, 140175935717376, 140175985139711,
+STORE, 140175935700992, 140175985139711,
+STORE, 140175935692800, 140175985139711,
+STORE, 140175935660032, 140175985139711,
+STORE, 140175935651840, 140175985139711,
+STORE, 140175935635456, 140175985139711,
+STORE, 140175935627264, 140175985139711,
+STORE, 140175935610880, 140175985139711,
+STORE, 140175935602688, 140175985139711,
+STORE, 140175935586304, 140175985139711,
+STORE, 140175935578112, 140175985139711,
+STORE, 140175935545344, 140175985139711,
+STORE, 140175935537152, 140175985139711,
+STORE, 140175935520768, 140175985139711,
+STORE, 140175935512576, 140175985139711,
+STORE, 140175935496192, 140175985139711,
+STORE, 140175935488000, 140175985139711,
+STORE, 140175935471616, 140175985139711,
+STORE, 140175935463424, 140175985139711,
+STORE, 140175935430656, 140175985139711,
+STORE, 140175935422464, 140175985139711,
+STORE, 140175935406080, 140175985139711,
+STORE, 140175935397888, 140175985139711,
+STORE, 140175935381504, 140175985139711,
+STORE, 140175935373312, 140175985139711,
+STORE, 140175935356928, 140175985139711,
+STORE, 140175935348736, 140175985139711,
+STORE, 140175935315968, 140175985139711,
+STORE, 140175935307776, 140175985139711,
+STORE, 140175935291392, 140175985139711,
+STORE, 140175935283200, 140175985139711,
+STORE, 140175935266816, 140175985139711,
+STORE, 140175935258624, 140175985139711,
+STORE, 140175935242240, 140175985139711,
+STORE, 140175935234048, 140175985139711,
+STORE, 140175935201280, 140175985139711,
+STORE, 140175935193088, 140175985139711,
+STORE, 140175935176704, 140175985139711,
+STORE, 140175935168512, 140175985139711,
+STORE, 140175935152128, 140175985139711,
+STORE, 140175935143936, 140175985139711,
+STORE, 140175935127552, 140175985139711,
+STORE, 140175935119360, 140175985139711,
+STORE, 140175935086592, 140175985139711,
+STORE, 140175935078400, 140175985139711,
+STORE, 140175935062016, 140175985139711,
+STORE, 140175935053824, 140175985139711,
+STORE, 140175935037440, 140175985139711,
+STORE, 140175935029248, 140175985139711,
+STORE, 140175935012864, 140175985139711,
+STORE, 140175935004672, 140175985139711,
+STORE, 140175934971904, 140175985139711,
+STORE, 140175934963712, 140175985139711,
+STORE, 140175934947328, 140175985139711,
+STORE, 140175934939136, 140175985139711,
+STORE, 140175934922752, 140175985139711,
+STORE, 140175934914560, 140175985139711,
+STORE, 140175934898176, 140175985139711,
+STORE, 140175934889984, 140175985139711,
+STORE, 140175934857216, 140175985139711,
+STORE, 140175934849024, 140175985139711,
+STORE, 140175934832640, 140175985139711,
+STORE, 140175934824448, 140175985139711,
+STORE, 140175934808064, 140175985139711,
+STORE, 140175934799872, 140175985139711,
+STORE, 140175934783488, 140175985139711,
+STORE, 140175934775296, 140175985139711,
+STORE, 140175934742528, 140175985139711,
+STORE, 140175934734336, 140175985139711,
+STORE, 140175934717952, 140175985139711,
+STORE, 140175934709760, 140175985139711,
+STORE, 140175934693376, 140175985139711,
+STORE, 140175934685184, 140175985139711,
+STORE, 140175934668800, 140175985139711,
+STORE, 140175934660608, 140175985139711,
+STORE, 140175934627840, 140175985139711,
+STORE, 140175934619648, 140175985139711,
+STORE, 140175934603264, 140175985139711,
+STORE, 140175934595072, 140175985139711,
+STORE, 140175934578688, 140175985139711,
+STORE, 140175934570496, 140175985139711,
+STORE, 140175934554112, 140175985139711,
+STORE, 140175934545920, 140175985139711,
+STORE, 140175934513152, 140175985139711,
+STORE, 140175934504960, 140175985139711,
+STORE, 140175934488576, 140175985139711,
+STORE, 140175934480384, 140175985139711,
+STORE, 140175934464000, 140175985139711,
+STORE, 140175934455808, 140175985139711,
+STORE, 140175934439424, 140175985139711,
+STORE, 140175934431232, 140175985139711,
+STORE, 140175934398464, 140175985139711,
+STORE, 140175934390272, 140175985139711,
+STORE, 140175934373888, 140175985139711,
+STORE, 140175934365696, 140175985139711,
+STORE, 140175934349312, 140175985139711,
+STORE, 140175934341120, 140175985139711,
+STORE, 140175934324736, 140175985139711,
+STORE, 140175932227584, 140175985139711,
+STORE, 140175932219392, 140175985139711,
+STORE, 140175932186624, 140175985139711,
+STORE, 140175932178432, 140175985139711,
+STORE, 140175932162048, 140175985139711,
+STORE, 140175932153856, 140175985139711,
+STORE, 140175932137472, 140175985139711,
+STORE, 53080064, 57884671,
+STORE, 140175932129280, 140175985139711,
+STORE, 140175932112896, 140175985139711,
+STORE, 140175932104704, 140175985139711,
+STORE, 140175932071936, 140175985139711,
+STORE, 140175932063744, 140175985139711,
+STORE, 140175932047360, 140175985139711,
+STORE, 140175932039168, 140175985139711,
+STORE, 140175932022784, 140175985139711,
+STORE, 140175932014592, 140175985139711,
+STORE, 140175931998208, 140175985139711,
+STORE, 140175931990016, 140175985139711,
+STORE, 140175931957248, 140175985139711,
+STORE, 140175931949056, 140175985139711,
+STORE, 140175931932672, 140175985139711,
+STORE, 140175931924480, 140175985139711,
+STORE, 140175931908096, 140175985139711,
+STORE, 140175931899904, 140175985139711,
+STORE, 140175931883520, 140175985139711,
+STORE, 140175931875328, 140175985139711,
+STORE, 140175931842560, 140175985139711,
+STORE, 140175931834368, 140175985139711,
+STORE, 140175931817984, 140175985139711,
+STORE, 140175931809792, 140175985139711,
+STORE, 140175931793408, 140175985139711,
+STORE, 140175931785216, 140175985139711,
+STORE, 140175931768832, 140175985139711,
+STORE, 140175931760640, 140175985139711,
+STORE, 140175931727872, 140175985139711,
+STORE, 140175931719680, 140175985139711,
+STORE, 140175931703296, 140175985139711,
+STORE, 140175931695104, 140175985139711,
+STORE, 140175931678720, 140175985139711,
+STORE, 140175931670528, 140175985139711,
+STORE, 140175931654144, 140175985139711,
+STORE, 140175931645952, 140175985139711,
+STORE, 140175931613184, 140175985139711,
+STORE, 140175931604992, 140175985139711,
+STORE, 140175931588608, 140175985139711,
+STORE, 140175931580416, 140175985139711,
+STORE, 140175931564032, 140175985139711,
+STORE, 140175931555840, 140175985139711,
+STORE, 140175931539456, 140175985139711,
+STORE, 140175931531264, 140175985139711,
+STORE, 140175931498496, 140175985139711,
+STORE, 140175931490304, 140175985139711,
+STORE, 140175931473920, 140175985139711,
+STORE, 140175931465728, 140175985139711,
+STORE, 140175931449344, 140175985139711,
+STORE, 140175931441152, 140175985139711,
+STORE, 140175931424768, 140175985139711,
+STORE, 140175931416576, 140175985139711,
+STORE, 140175931383808, 140175985139711,
+STORE, 140175931375616, 140175985139711,
+STORE, 140175931359232, 140175985139711,
+STORE, 140175931351040, 140175985139711,
+STORE, 140175931334656, 140175985139711,
+STORE, 140175931326464, 140175985139711,
+STORE, 140175931310080, 140175985139711,
+STORE, 140175931301888, 140175985139711,
+STORE, 140175931269120, 140175985139711,
+STORE, 140175931260928, 140175985139711,
+STORE, 140175931244544, 140175985139711,
+STORE, 140175931236352, 140175985139711,
+STORE, 140175931219968, 140175985139711,
+STORE, 140175931211776, 140175985139711,
+STORE, 140175931195392, 140175985139711,
+STORE, 140175931187200, 140175985139711,
+STORE, 140175931154432, 140175985139711,
+STORE, 140175931146240, 140175985139711,
+STORE, 140175931129856, 140175985139711,
+STORE, 140175931121664, 140175985139711,
+STORE, 140175931105280, 140175985139711,
+STORE, 140175931097088, 140175985139711,
+STORE, 140175931080704, 140175985139711,
+STORE, 140175931072512, 140175985139711,
+STORE, 140175931039744, 140175985139711,
+STORE, 140175931031552, 140175985139711,
+STORE, 140175931015168, 140175985139711,
+STORE, 140175931006976, 140175985139711,
+STORE, 140175930990592, 140175985139711,
+STORE, 140175930982400, 140175985139711,
+STORE, 140175930966016, 140175985139711,
+STORE, 140175930957824, 140175985139711,
+STORE, 140175930925056, 140175985139711,
+STORE, 140175930916864, 140175985139711,
+STORE, 140175930900480, 140175985139711,
+STORE, 140175930892288, 140175985139711,
+STORE, 140175930875904, 140175985139711,
+STORE, 140175930867712, 140175985139711,
+STORE, 140175930851328, 140175985139711,
+STORE, 140175930843136, 140175985139711,
+STORE, 140175930810368, 140175985139711,
+STORE, 140175930802176, 140175985139711,
+STORE, 140175930785792, 140175985139711,
+STORE, 140175930777600, 140175985139711,
+STORE, 140175930761216, 140175985139711,
+STORE, 140175930753024, 140175985139711,
+STORE, 140175930736640, 140175985139711,
+STORE, 140175930728448, 140175985139711,
+STORE, 140175930695680, 140175985139711,
+STORE, 140175930687488, 140175985139711,
+STORE, 140175930671104, 140175985139711,
+STORE, 140175930662912, 140175985139711,
+STORE, 140175930646528, 140175985139711,
+STORE, 140175930638336, 140175985139711,
+STORE, 140175930621952, 140175985139711,
+STORE, 140175930613760, 140175985139711,
+STORE, 140175930580992, 140175985139711,
+STORE, 140175930572800, 140175985139711,
+STORE, 140175930556416, 140175985139711,
+STORE, 140175930548224, 140175985139711,
+STORE, 140175930531840, 140175985139711,
+STORE, 140175930523648, 140175985139711,
+STORE, 140175930507264, 140175985139711,
+STORE, 140175928410112, 140175985139711,
+STORE, 140175928401920, 140175985139711,
+STORE, 140175928369152, 140175985139711,
+STORE, 140175928360960, 140175985139711,
+STORE, 140175928344576, 140175985139711,
+STORE, 140175928336384, 140175985139711,
+STORE, 140175928320000, 140175985139711,
+STORE, 140175928311808, 140175985139711,
+STORE, 140175928295424, 140175985139711,
+STORE, 140175927242752, 140175985139711,
+SNULL, 140175956627455, 140175985139711,
+STORE, 140175927242752, 140175956627455,
+STORE, 140175956627456, 140175985139711,
+ };
+ unsigned long set24[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140735281639424, 140737488351231,
+SNULL, 140735281643519, 140737488351231,
+STORE, 140735281639424, 140735281643519,
+STORE, 140735281508352, 140735281643519,
+STORE, 94717834911744, 94717834928127,
+SNULL, 94717834915839, 94717834928127,
+STORE, 94717834911744, 94717834915839,
+STORE, 94717834915840, 94717834928127,
+ERASE, 94717834915840, 94717834928127,
+STORE, 94717834919936, 94717834928127,
+STORE, 140428246065152, 140428248317951,
+SNULL, 140428246208511, 140428248317951,
+STORE, 140428246065152, 140428246208511,
+STORE, 140428246208512, 140428248317951,
+ERASE, 140428246208512, 140428248317951,
+STORE, 140428248305664, 140428248313855,
+STORE, 140428248313856, 140428248317951,
+STORE, 140735281811456, 140735281815551,
+STORE, 140735281799168, 140735281811455,
+STORE, 140428248297472, 140428248305663,
+STORE, 140428243841024, 140428246065151,
+SNULL, 140428245491711, 140428246065151,
+STORE, 140428243841024, 140428245491711,
+STORE, 140428245491712, 140428246065151,
+SNULL, 140428245491712, 140428246061055,
+STORE, 140428246061056, 140428246065151,
+STORE, 140428245491712, 140428246061055,
+ERASE, 140428245491712, 140428246061055,
+STORE, 140428245491712, 140428246061055,
+ERASE, 140428246061056, 140428246065151,
+STORE, 140428246061056, 140428246065151,
+STORE, 140428248268800, 140428248297471,
+STORE, 140428241625088, 140428243841023,
+SNULL, 140428241625088, 140428241723391,
+STORE, 140428241723392, 140428243841023,
+STORE, 140428241625088, 140428241723391,
+SNULL, 140428243816447, 140428243841023,
+STORE, 140428241723392, 140428243816447,
+STORE, 140428243816448, 140428243841023,
+SNULL, 140428243816448, 140428243824639,
+STORE, 140428243824640, 140428243841023,
+STORE, 140428243816448, 140428243824639,
+ERASE, 140428243816448, 140428243824639,
+STORE, 140428243816448, 140428243824639,
+ERASE, 140428243824640, 140428243841023,
+STORE, 140428243824640, 140428243841023,
+STORE, 140428237828096, 140428241625087,
+SNULL, 140428237828096, 140428239486975,
+STORE, 140428239486976, 140428241625087,
+STORE, 140428237828096, 140428239486975,
+SNULL, 140428241584127, 140428241625087,
+STORE, 140428239486976, 140428241584127,
+STORE, 140428241584128, 140428241625087,
+SNULL, 140428241584128, 140428241608703,
+STORE, 140428241608704, 140428241625087,
+STORE, 140428241584128, 140428241608703,
+ERASE, 140428241584128, 140428241608703,
+STORE, 140428241584128, 140428241608703,
+ERASE, 140428241608704, 140428241625087,
+STORE, 140428241608704, 140428241625087,
+STORE, 140428235567104, 140428237828095,
+SNULL, 140428235567104, 140428235718655,
+STORE, 140428235718656, 140428237828095,
+STORE, 140428235567104, 140428235718655,
+SNULL, 140428237811711, 140428237828095,
+STORE, 140428235718656, 140428237811711,
+STORE, 140428237811712, 140428237828095,
+SNULL, 140428237811712, 140428237819903,
+STORE, 140428237819904, 140428237828095,
+STORE, 140428237811712, 140428237819903,
+ERASE, 140428237811712, 140428237819903,
+STORE, 140428237811712, 140428237819903,
+ERASE, 140428237819904, 140428237828095,
+STORE, 140428237819904, 140428237828095,
+STORE, 140428233445376, 140428235567103,
+SNULL, 140428233445376, 140428233461759,
+STORE, 140428233461760, 140428235567103,
+STORE, 140428233445376, 140428233461759,
+SNULL, 140428235558911, 140428235567103,
+STORE, 140428233461760, 140428235558911,
+STORE, 140428235558912, 140428235567103,
+ERASE, 140428235558912, 140428235567103,
+STORE, 140428235558912, 140428235567103,
+STORE, 140428231315456, 140428233445375,
+SNULL, 140428231315456, 140428231344127,
+STORE, 140428231344128, 140428233445375,
+STORE, 140428231315456, 140428231344127,
+SNULL, 140428233437183, 140428233445375,
+STORE, 140428231344128, 140428233437183,
+STORE, 140428233437184, 140428233445375,
+ERASE, 140428233437184, 140428233445375,
+STORE, 140428233437184, 140428233445375,
+STORE, 140428248260608, 140428248268799,
+STORE, 140428229062656, 140428231315455,
+SNULL, 140428229062656, 140428229214207,
+STORE, 140428229214208, 140428231315455,
+STORE, 140428229062656, 140428229214207,
+SNULL, 140428231307263, 140428231315455,
+STORE, 140428229214208, 140428231307263,
+STORE, 140428231307264, 140428231315455,
+ERASE, 140428231307264, 140428231315455,
+STORE, 140428231307264, 140428231315455,
+STORE, 140428226891776, 140428229062655,
+SNULL, 140428226891776, 140428226961407,
+STORE, 140428226961408, 140428229062655,
+STORE, 140428226891776, 140428226961407,
+SNULL, 140428229054463, 140428229062655,
+STORE, 140428226961408, 140428229054463,
+STORE, 140428229054464, 140428229062655,
+ERASE, 140428229054464, 140428229062655,
+STORE, 140428229054464, 140428229062655,
+STORE, 140428223680512, 140428226891775,
+SNULL, 140428223680512, 140428224757759,
+STORE, 140428224757760, 140428226891775,
+STORE, 140428223680512, 140428224757759,
+SNULL, 140428226854911, 140428226891775,
+STORE, 140428224757760, 140428226854911,
+STORE, 140428226854912, 140428226891775,
+ERASE, 140428226854912, 140428226891775,
+STORE, 140428226854912, 140428226891775,
+STORE, 140428221546496, 140428223680511,
+SNULL, 140428221546496, 140428221575167,
+STORE, 140428221575168, 140428223680511,
+STORE, 140428221546496, 140428221575167,
+SNULL, 140428223672319, 140428223680511,
+STORE, 140428221575168, 140428223672319,
+STORE, 140428223672320, 140428223680511,
+ERASE, 140428223672320, 140428223680511,
+STORE, 140428223672320, 140428223680511,
+STORE, 140428219236352, 140428221546495,
+SNULL, 140428219236352, 140428219441151,
+STORE, 140428219441152, 140428221546495,
+STORE, 140428219236352, 140428219441151,
+SNULL, 140428221538303, 140428221546495,
+STORE, 140428219441152, 140428221538303,
+STORE, 140428221538304, 140428221546495,
+ERASE, 140428221538304, 140428221546495,
+STORE, 140428221538304, 140428221546495,
+STORE, 140428216852480, 140428219236351,
+SNULL, 140428216852480, 140428217044991,
+STORE, 140428217044992, 140428219236351,
+STORE, 140428216852480, 140428217044991,
+SNULL, 140428219138047, 140428219236351,
+STORE, 140428217044992, 140428219138047,
+STORE, 140428219138048, 140428219236351,
+ERASE, 140428219138048, 140428219236351,
+STORE, 140428219138048, 140428219236351,
+STORE, 140428248252416, 140428248268799,
+STORE, 140428214284288, 140428216852479,
+SNULL, 140428214284288, 140428214751231,
+STORE, 140428214751232, 140428216852479,
+STORE, 140428214284288, 140428214751231,
+SNULL, 140428216844287, 140428216852479,
+STORE, 140428214751232, 140428216844287,
+STORE, 140428216844288, 140428216852479,
+ERASE, 140428216844288, 140428216852479,
+STORE, 140428216844288, 140428216852479,
+STORE, 140428212170752, 140428214284287,
+SNULL, 140428212170752, 140428212183039,
+STORE, 140428212183040, 140428214284287,
+STORE, 140428212170752, 140428212183039,
+SNULL, 140428214276095, 140428214284287,
+STORE, 140428212183040, 140428214276095,
+STORE, 140428214276096, 140428214284287,
+ERASE, 140428214276096, 140428214284287,
+STORE, 140428214276096, 140428214284287,
+STORE, 140428209991680, 140428212170751,
+SNULL, 140428209991680, 140428210069503,
+STORE, 140428210069504, 140428212170751,
+STORE, 140428209991680, 140428210069503,
+SNULL, 140428212162559, 140428212170751,
+STORE, 140428210069504, 140428212162559,
+STORE, 140428212162560, 140428212170751,
+ERASE, 140428212162560, 140428212170751,
+STORE, 140428212162560, 140428212170751,
+STORE, 140428207874048, 140428209991679,
+SNULL, 140428207874048, 140428207890431,
+STORE, 140428207890432, 140428209991679,
+STORE, 140428207874048, 140428207890431,
+SNULL, 140428209983487, 140428209991679,
+STORE, 140428207890432, 140428209983487,
+STORE, 140428209983488, 140428209991679,
+ERASE, 140428209983488, 140428209991679,
+STORE, 140428209983488, 140428209991679,
+STORE, 140428248244224, 140428248268799,
+STORE, 140428248231936, 140428248268799,
+SNULL, 140428241600511, 140428241608703,
+STORE, 140428241584128, 140428241600511,
+STORE, 140428241600512, 140428241608703,
+SNULL, 140428209987583, 140428209991679,
+STORE, 140428209983488, 140428209987583,
+STORE, 140428209987584, 140428209991679,
+SNULL, 140428212166655, 140428212170751,
+STORE, 140428212162560, 140428212166655,
+STORE, 140428212166656, 140428212170751,
+SNULL, 140428214280191, 140428214284287,
+STORE, 140428214276096, 140428214280191,
+STORE, 140428214280192, 140428214284287,
+SNULL, 140428243820543, 140428243824639,
+STORE, 140428243816448, 140428243820543,
+STORE, 140428243820544, 140428243824639,
+SNULL, 140428216848383, 140428216852479,
+STORE, 140428216844288, 140428216848383,
+STORE, 140428216848384, 140428216852479,
+SNULL, 140428219232255, 140428219236351,
+STORE, 140428219138048, 140428219232255,
+STORE, 140428219232256, 140428219236351,
+SNULL, 140428221542399, 140428221546495,
+STORE, 140428221538304, 140428221542399,
+STORE, 140428221542400, 140428221546495,
+SNULL, 140428223676415, 140428223680511,
+STORE, 140428223672320, 140428223676415,
+STORE, 140428223676416, 140428223680511,
+SNULL, 140428226863103, 140428226891775,
+STORE, 140428226854912, 140428226863103,
+STORE, 140428226863104, 140428226891775,
+SNULL, 140428229058559, 140428229062655,
+STORE, 140428229054464, 140428229058559,
+STORE, 140428229058560, 140428229062655,
+SNULL, 140428231311359, 140428231315455,
+STORE, 140428231307264, 140428231311359,
+STORE, 140428231311360, 140428231315455,
+SNULL, 140428233441279, 140428233445375,
+STORE, 140428233437184, 140428233441279,
+STORE, 140428233441280, 140428233445375,
+SNULL, 140428235563007, 140428235567103,
+STORE, 140428235558912, 140428235563007,
+STORE, 140428235563008, 140428235567103,
+SNULL, 140428237815807, 140428237819903,
+STORE, 140428237811712, 140428237815807,
+STORE, 140428237815808, 140428237819903,
+SNULL, 140428246056959, 140428246061055,
+STORE, 140428245491712, 140428246056959,
+STORE, 140428246056960, 140428246061055,
+SNULL, 94717834924031, 94717834928127,
+STORE, 94717834919936, 94717834924031,
+STORE, 94717834924032, 94717834928127,
+SNULL, 140428248309759, 140428248313855,
+STORE, 140428248305664, 140428248309759,
+STORE, 140428248309760, 140428248313855,
+ERASE, 140428248268800, 140428248297471,
+STORE, 94717843058688, 94717843193855,
+STORE, 94749677137920, 94749677559807,
+STORE, 94749677563904, 94749677604863,
+STORE, 94749677604864, 94749677608959,
+STORE, 94749710970880, 94749711241215,
+STORE, 140490884894720, 140490884935679,
+STORE, 140490884935680, 140490887032831,
+STORE, 140490887032832, 140490887036927,
+STORE, 140490887036928, 140490887041023,
+STORE, 140490887041024, 140490887065599,
+STORE, 140490887065600, 140490887110655,
+STORE, 140490887110656, 140490889203711,
+STORE, 140490889203712, 140490889207807,
+STORE, 140490889207808, 140490889211903,
+STORE, 140490889211904, 140490889293823,
+STORE, 140490889293824, 140490891390975,
+STORE, 140490891390976, 140490891395071,
+STORE, 140490891395072, 140490891399167,
+STORE, 140490891399168, 140490891407359,
+STORE, 140490891407360, 140490891436031,
+STORE, 140490891436032, 140490893529087,
+STORE, 140490893529088, 140490893533183,
+STORE, 140490893533184, 140490893537279,
+STORE, 140490893537280, 140490901979135,
+STORE, 140490901979136, 140490901991423,
+STORE, 140490901991424, 140490904084479,
+STORE, 140490904084480, 140490904088575,
+STORE, 140490904088576, 140490904092671,
+STORE, 140490904092672, 140490904559615,
+STORE, 140490904559616, 140490906652671,
+STORE, 140490906652672, 140490906656767,
+STORE, 140490906656768, 140490906660863,
+STORE, 140490906660864, 140490906677247,
+STORE, 140490906677248, 140490908770303,
+STORE, 140490908770304, 140490908774399,
+STORE, 140490908774400, 140490908778495,
+STORE, 140490908778496, 140490908794879,
+STORE, 140490908794880, 140490910887935,
+STORE, 140490910887936, 140490910892031,
+STORE, 140490910892032, 140490910896127,
+STORE, 140490910896128, 140490912555007,
+STORE, 140490912555008, 140490914652159,
+STORE, 140490914652160, 140490914668543,
+STORE, 140490914668544, 140490914676735,
+STORE, 140490914676736, 140490914693119,
+STORE, 140490914693120, 140490914791423,
+STORE, 140490914791424, 140490916884479,
+STORE, 140490916884480, 140490916888575,
+STORE, 140490916888576, 140490916892671,
+STORE, 140490916892672, 140490916909055,
+STORE, 140490916909056, 140490916937727,
+STORE, 140490916937728, 140490919030783,
+STORE, 140490919030784, 140490919034879,
+STORE, 140490919034880, 140490919038975,
+STORE, 140490919038976, 140490919190527,
+STORE, 140490919190528, 140490921283583,
+STORE, 140490921283584, 140490921287679,
+STORE, 140490921287680, 140490921291775,
+STORE, 140490921291776, 140490921299967,
+STORE, 140490921299968, 140490921390079,
+STORE, 140490921390080, 140490923483135,
+STORE, 140490923483136, 140490923487231,
+STORE, 140490923487232, 140490923491327,
+STORE, 140490923491328, 140490923757567,
+STORE, 140490923757568, 140490925850623,
+STORE, 140490925850624, 140490925867007,
+STORE, 140490925867008, 140490925871103,
+STORE, 140490925871104, 140490925875199,
+STORE, 140490925875200, 140490925903871,
+STORE, 140490925903872, 140490928001023,
+STORE, 140490928001024, 140490928005119,
+STORE, 140490928005120, 140490928009215,
+STORE, 140490928009216, 140490928152575,
+STORE, 140490930184192, 140490930221055,
+STORE, 140490930221056, 140490930237439,
+STORE, 140490930237440, 140490930241535,
+STORE, 140490930241536, 140490930245631,
+STORE, 140490930245632, 140490930249727,
+STORE, 140490930249728, 140490930253823,
+STORE, 140490930253824, 140490930257919,
+STORE, 140490930257920, 140490930262015,
+STORE, 140724611694592, 140724611829759,
+STORE, 140724612427776, 140724612440063,
+STORE, 140724612440064, 140724612444159,
+STORE, 94103163662336, 94103163772927,
+STORE, 94103165865984, 94103165874175,
+STORE, 94103165874176, 94103165878271,
+STORE, 94103165878272, 94103165886463,
+STORE, 94103182548992, 94103182684159,
+STORE, 140092694708224, 140092696367103,
+STORE, 140092696367104, 140092698464255,
+STORE, 140092698464256, 140092698480639,
+STORE, 140092698480640, 140092698488831,
+STORE, 140092698488832, 140092698505215,
+STORE, 140092698505216, 140092698648575,
+STORE, 140092700708864, 140092700717055,
+STORE, 140092700745728, 140092700749823,
+STORE, 140092700749824, 140092700753919,
+STORE, 140092700753920, 140092700758015,
+STORE, 140736800911360, 140736801046527,
+STORE, 140736802308096, 140736802320383,
+STORE, 140736802320384, 140736802324479,
+STORE, 93948802064384, 93948802174975,
+STORE, 93948804268032, 93948804276223,
+STORE, 93948804276224, 93948804280319,
+STORE, 93948804280320, 93948804288511,
+STORE, 93948806266880, 93948806402047,
+STORE, 140222999113728, 140223000772607,
+STORE, 140223000772608, 140223002869759,
+STORE, 140223002869760, 140223002886143,
+STORE, 140223002886144, 140223002894335,
+STORE, 140223002894336, 140223002910719,
+STORE, 140223002910720, 140223003054079,
+STORE, 140223005114368, 140223005122559,
+STORE, 140223005151232, 140223005155327,
+STORE, 140223005155328, 140223005159423,
+STORE, 140223005159424, 140223005163519,
+STORE, 140720877506560, 140720877641727,
+STORE, 140720878231552, 140720878243839,
+STORE, 140720878243840, 140720878247935,
+STORE, 140737488347136, 140737488351231,
+STORE, 140733232087040, 140737488351231,
+SNULL, 140733232091135, 140737488351231,
+STORE, 140733232087040, 140733232091135,
+STORE, 140733231955968, 140733232091135,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 140161681321984, 140161683574783,
+SNULL, 140161681465343, 140161683574783,
+STORE, 140161681321984, 140161681465343,
+STORE, 140161681465344, 140161683574783,
+ERASE, 140161681465344, 140161683574783,
+STORE, 140161683562496, 140161683570687,
+STORE, 140161683570688, 140161683574783,
+STORE, 140733232214016, 140733232218111,
+STORE, 140733232201728, 140733232214015,
+STORE, 140161683533824, 140161683562495,
+STORE, 140161683525632, 140161683533823,
+STORE, 140161678159872, 140161681321983,
+SNULL, 140161678159872, 140161679220735,
+STORE, 140161679220736, 140161681321983,
+STORE, 140161678159872, 140161679220735,
+SNULL, 140161681313791, 140161681321983,
+STORE, 140161679220736, 140161681313791,
+STORE, 140161681313792, 140161681321983,
+ERASE, 140161681313792, 140161681321983,
+STORE, 140161681313792, 140161681321983,
+STORE, 140161674362880, 140161678159871,
+SNULL, 140161674362880, 140161676021759,
+STORE, 140161676021760, 140161678159871,
+STORE, 140161674362880, 140161676021759,
+SNULL, 140161678118911, 140161678159871,
+STORE, 140161676021760, 140161678118911,
+STORE, 140161678118912, 140161678159871,
+SNULL, 140161678118912, 140161678143487,
+STORE, 140161678143488, 140161678159871,
+STORE, 140161678118912, 140161678143487,
+ERASE, 140161678118912, 140161678143487,
+STORE, 140161678118912, 140161678143487,
+ERASE, 140161678143488, 140161678159871,
+STORE, 140161678143488, 140161678159871,
+STORE, 140161683513344, 140161683533823,
+SNULL, 140161678135295, 140161678143487,
+STORE, 140161678118912, 140161678135295,
+STORE, 140161678135296, 140161678143487,
+SNULL, 140161681317887, 140161681321983,
+STORE, 140161681313792, 140161681317887,
+STORE, 140161681317888, 140161681321983,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 140161683566591, 140161683570687,
+STORE, 140161683562496, 140161683566591,
+STORE, 140161683566592, 140161683570687,
+ERASE, 140161683533824, 140161683562495,
+STORE, 25477120, 25612287,
+STORE, 25477120, 25759743,
+STORE, 140161681829888, 140161683513343,
+STORE, 25477120, 25915391,
+STORE, 25477120, 26054655,
+SNULL, 25800703, 26054655,
+STORE, 25477120, 25800703,
+STORE, 25800704, 26054655,
+ERASE, 25800704, 26054655,
+STORE, 140737488347136, 140737488351231,
+STORE, 140723218452480, 140737488351231,
+SNULL, 140723218456575, 140737488351231,
+STORE, 140723218452480, 140723218456575,
+STORE, 140723218321408, 140723218456575,
+STORE, 4194304, 26279935,
+STORE, 28372992, 28454911,
+STORE, 28454912, 29806591,
+STORE, 140398872264704, 140398874517503,
+SNULL, 140398872408063, 140398874517503,
+STORE, 140398872264704, 140398872408063,
+STORE, 140398872408064, 140398874517503,
+ERASE, 140398872408064, 140398874517503,
+STORE, 140398874505216, 140398874513407,
+STORE, 140398874513408, 140398874517503,
+STORE, 140723219247104, 140723219251199,
+STORE, 140723219234816, 140723219247103,
+STORE, 140398874476544, 140398874505215,
+STORE, 140398874468352, 140398874476543,
+STORE, 140398868430848, 140398872264703,
+SNULL, 140398868430848, 140398870138879,
+STORE, 140398870138880, 140398872264703,
+STORE, 140398868430848, 140398870138879,
+SNULL, 140398872231935, 140398872264703,
+STORE, 140398870138880, 140398872231935,
+STORE, 140398872231936, 140398872264703,
+ERASE, 140398872231936, 140398872264703,
+STORE, 140398872231936, 140398872264703,
+STORE, 140398866235392, 140398868430847,
+SNULL, 140398866235392, 140398866329599,
+STORE, 140398866329600, 140398868430847,
+STORE, 140398866235392, 140398866329599,
+SNULL, 140398868422655, 140398868430847,
+STORE, 140398866329600, 140398868422655,
+STORE, 140398868422656, 140398868430847,
+ERASE, 140398868422656, 140398868430847,
+STORE, 140398868422656, 140398868430847,
+STORE, 140398863716352, 140398866235391,
+SNULL, 140398863716352, 140398864130047,
+STORE, 140398864130048, 140398866235391,
+STORE, 140398863716352, 140398864130047,
+SNULL, 140398866223103, 140398866235391,
+STORE, 140398864130048, 140398866223103,
+STORE, 140398866223104, 140398866235391,
+ERASE, 140398866223104, 140398866235391,
+STORE, 140398866223104, 140398866235391,
+STORE, 140398861082624, 140398863716351,
+SNULL, 140398861082624, 140398861611007,
+STORE, 140398861611008, 140398863716351,
+STORE, 140398861082624, 140398861611007,
+SNULL, 140398863708159, 140398863716351,
+STORE, 140398861611008, 140398863708159,
+STORE, 140398863708160, 140398863716351,
+ERASE, 140398863708160, 140398863716351,
+STORE, 140398863708160, 140398863716351,
+STORE, 140398858969088, 140398861082623,
+SNULL, 140398858969088, 140398858981375,
+STORE, 140398858981376, 140398861082623,
+STORE, 140398858969088, 140398858981375,
+SNULL, 140398861074431, 140398861082623,
+STORE, 140398858981376, 140398861074431,
+STORE, 140398861074432, 140398861082623,
+ERASE, 140398861074432, 140398861082623,
+STORE, 140398861074432, 140398861082623,
+STORE, 140398856765440, 140398858969087,
+SNULL, 140398856765440, 140398856867839,
+STORE, 140398856867840, 140398858969087,
+STORE, 140398856765440, 140398856867839,
+SNULL, 140398858960895, 140398858969087,
+STORE, 140398856867840, 140398858960895,
+STORE, 140398858960896, 140398858969087,
+ERASE, 140398858960896, 140398858969087,
+STORE, 140398858960896, 140398858969087,
+STORE, 140398874460160, 140398874476543,
+STORE, 140398853603328, 140398856765439,
+SNULL, 140398853603328, 140398854664191,
+STORE, 140398854664192, 140398856765439,
+STORE, 140398853603328, 140398854664191,
+SNULL, 140398856757247, 140398856765439,
+STORE, 140398854664192, 140398856757247,
+STORE, 140398856757248, 140398856765439,
+ERASE, 140398856757248, 140398856765439,
+STORE, 140398856757248, 140398856765439,
+STORE, 140398849806336, 140398853603327,
+SNULL, 140398849806336, 140398851465215,
+STORE, 140398851465216, 140398853603327,
+STORE, 140398849806336, 140398851465215,
+SNULL, 140398853562367, 140398853603327,
+STORE, 140398851465216, 140398853562367,
+STORE, 140398853562368, 140398853603327,
+SNULL, 140398853562368, 140398853586943,
+STORE, 140398853586944, 140398853603327,
+STORE, 140398853562368, 140398853586943,
+ERASE, 140398853562368, 140398853586943,
+STORE, 140398853562368, 140398853586943,
+ERASE, 140398853586944, 140398853603327,
+STORE, 140398853586944, 140398853603327,
+STORE, 140398874447872, 140398874476543,
+SNULL, 140398853578751, 140398853586943,
+STORE, 140398853562368, 140398853578751,
+STORE, 140398853578752, 140398853586943,
+SNULL, 140398856761343, 140398856765439,
+STORE, 140398856757248, 140398856761343,
+STORE, 140398856761344, 140398856765439,
+SNULL, 140398858964991, 140398858969087,
+STORE, 140398858960896, 140398858964991,
+STORE, 140398858964992, 140398858969087,
+SNULL, 140398861078527, 140398861082623,
+STORE, 140398861074432, 140398861078527,
+STORE, 140398861078528, 140398861082623,
+SNULL, 140398863712255, 140398863716351,
+STORE, 140398863708160, 140398863712255,
+STORE, 140398863712256, 140398863716351,
+SNULL, 140398866231295, 140398866235391,
+STORE, 140398866223104, 140398866231295,
+STORE, 140398866231296, 140398866235391,
+SNULL, 140398868426751, 140398868430847,
+STORE, 140398868422656, 140398868426751,
+STORE, 140398868426752, 140398868430847,
+SNULL, 140398872236031, 140398872264703,
+STORE, 140398872231936, 140398872236031,
+STORE, 140398872236032, 140398872264703,
+SNULL, 28405759, 28454911,
+STORE, 28372992, 28405759,
+STORE, 28405760, 28454911,
+SNULL, 140398874509311, 140398874513407,
+STORE, 140398874505216, 140398874509311,
+STORE, 140398874509312, 140398874513407,
+ERASE, 140398874476544, 140398874505215,
+STORE, 43278336, 43413503,
+STORE, 140398872764416, 140398874447871,
+STORE, 140398874501120, 140398874505215,
+STORE, 140398872629248, 140398872764415,
+STORE, 43278336, 43556863,
+STORE, 140398847709184, 140398849806335,
+STORE, 140398874492928, 140398874505215,
+STORE, 140398874484736, 140398874505215,
+STORE, 140398874447872, 140398874484735,
+STORE, 140398872612864, 140398872764415,
+STORE, 43278336, 43692031,
+STORE, 43278336, 43880447,
+STORE, 140398872604672, 140398872764415,
+STORE, 140398872596480, 140398872764415,
+STORE, 43278336, 44044287,
+STORE, 140398872580096, 140398872764415,
+STORE, 140737488347136, 140737488351231,
+STORE, 140734403092480, 140737488351231,
+SNULL, 140734403096575, 140737488351231,
+STORE, 140734403092480, 140734403096575,
+STORE, 140734402961408, 140734403096575,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 140240662380544, 140240664633343,
+SNULL, 140240662523903, 140240664633343,
+STORE, 140240662380544, 140240662523903,
+STORE, 140240662523904, 140240664633343,
+ERASE, 140240662523904, 140240664633343,
+STORE, 140240664621056, 140240664629247,
+STORE, 140240664629248, 140240664633343,
+STORE, 140734403145728, 140734403149823,
+STORE, 140734403133440, 140734403145727,
+STORE, 140240664592384, 140240664621055,
+STORE, 140240664584192, 140240664592383,
+STORE, 140240659218432, 140240662380543,
+SNULL, 140240659218432, 140240660279295,
+STORE, 140240660279296, 140240662380543,
+STORE, 140240659218432, 140240660279295,
+SNULL, 140240662372351, 140240662380543,
+STORE, 140240660279296, 140240662372351,
+STORE, 140240662372352, 140240662380543,
+ERASE, 140240662372352, 140240662380543,
+STORE, 140240662372352, 140240662380543,
+STORE, 140240655421440, 140240659218431,
+SNULL, 140240655421440, 140240657080319,
+STORE, 140240657080320, 140240659218431,
+STORE, 140240655421440, 140240657080319,
+SNULL, 140240659177471, 140240659218431,
+STORE, 140240657080320, 140240659177471,
+STORE, 140240659177472, 140240659218431,
+SNULL, 140240659177472, 140240659202047,
+STORE, 140240659202048, 140240659218431,
+STORE, 140240659177472, 140240659202047,
+ERASE, 140240659177472, 140240659202047,
+STORE, 140240659177472, 140240659202047,
+ERASE, 140240659202048, 140240659218431,
+STORE, 140240659202048, 140240659218431,
+STORE, 140240664571904, 140240664592383,
+SNULL, 140240659193855, 140240659202047,
+STORE, 140240659177472, 140240659193855,
+STORE, 140240659193856, 140240659202047,
+SNULL, 140240662376447, 140240662380543,
+STORE, 140240662372352, 140240662376447,
+STORE, 140240662376448, 140240662380543,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 140240664625151, 140240664629247,
+STORE, 140240664621056, 140240664625151,
+STORE, 140240664625152, 140240664629247,
+ERASE, 140240664592384, 140240664621055,
+STORE, 30646272, 30781439,
+STORE, 30646272, 30928895,
+STORE, 140240662888448, 140240664571903,
+STORE, 94256659468288, 94256659578879,
+STORE, 94256661671936, 94256661680127,
+STORE, 94256661680128, 94256661684223,
+STORE, 94256661684224, 94256661692415,
+STORE, 94256687980544, 94256688115711,
+STORE, 139801712504832, 139801714163711,
+STORE, 139801714163712, 139801716260863,
+STORE, 139801716260864, 139801716277247,
+STORE, 139801716277248, 139801716285439,
+STORE, 139801716285440, 139801716301823,
+STORE, 139801716301824, 139801716445183,
+STORE, 139801718505472, 139801718513663,
+STORE, 139801718542336, 139801718546431,
+STORE, 139801718546432, 139801718550527,
+STORE, 139801718550528, 139801718554623,
+STORE, 140721575538688, 140721575673855,
+STORE, 140721577013248, 140721577025535,
+STORE, 140721577025536, 140721577029631,
+STORE, 140737488347136, 140737488351231,
+STORE, 140729259393024, 140737488351231,
+SNULL, 140729259397119, 140737488351231,
+STORE, 140729259393024, 140729259397119,
+STORE, 140729259261952, 140729259397119,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 139682376638464, 139682378891263,
+SNULL, 139682376781823, 139682378891263,
+STORE, 139682376638464, 139682376781823,
+STORE, 139682376781824, 139682378891263,
+ERASE, 139682376781824, 139682378891263,
+STORE, 139682378878976, 139682378887167,
+STORE, 139682378887168, 139682378891263,
+STORE, 140729260462080, 140729260466175,
+STORE, 140729260449792, 140729260462079,
+STORE, 139682378850304, 139682378878975,
+STORE, 139682378842112, 139682378850303,
+STORE, 139682373476352, 139682376638463,
+SNULL, 139682373476352, 139682374537215,
+STORE, 139682374537216, 139682376638463,
+STORE, 139682373476352, 139682374537215,
+SNULL, 139682376630271, 139682376638463,
+STORE, 139682374537216, 139682376630271,
+STORE, 139682376630272, 139682376638463,
+ERASE, 139682376630272, 139682376638463,
+STORE, 139682376630272, 139682376638463,
+STORE, 139682369679360, 139682373476351,
+SNULL, 139682369679360, 139682371338239,
+STORE, 139682371338240, 139682373476351,
+STORE, 139682369679360, 139682371338239,
+SNULL, 139682373435391, 139682373476351,
+STORE, 139682371338240, 139682373435391,
+STORE, 139682373435392, 139682373476351,
+SNULL, 139682373435392, 139682373459967,
+STORE, 139682373459968, 139682373476351,
+STORE, 139682373435392, 139682373459967,
+ERASE, 139682373435392, 139682373459967,
+STORE, 139682373435392, 139682373459967,
+ERASE, 139682373459968, 139682373476351,
+STORE, 139682373459968, 139682373476351,
+STORE, 139682378829824, 139682378850303,
+SNULL, 139682373451775, 139682373459967,
+STORE, 139682373435392, 139682373451775,
+STORE, 139682373451776, 139682373459967,
+SNULL, 139682376634367, 139682376638463,
+STORE, 139682376630272, 139682376634367,
+STORE, 139682376634368, 139682376638463,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 139682378883071, 139682378887167,
+STORE, 139682378878976, 139682378883071,
+STORE, 139682378883072, 139682378887167,
+ERASE, 139682378850304, 139682378878975,
+STORE, 10022912, 10158079,
+STORE, 10022912, 10305535,
+STORE, 139682377146368, 139682378829823,
+STORE, 140737488347136, 140737488351231,
+STORE, 140731831926784, 140737488351231,
+SNULL, 140731831930879, 140737488351231,
+STORE, 140731831926784, 140731831930879,
+STORE, 140731831795712, 140731831930879,
+STORE, 94615305261056, 94615307485183,
+SNULL, 94615305371647, 94615307485183,
+STORE, 94615305261056, 94615305371647,
+STORE, 94615305371648, 94615307485183,
+ERASE, 94615305371648, 94615307485183,
+STORE, 94615307464704, 94615307476991,
+STORE, 94615307476992, 94615307485183,
+STORE, 140163912994816, 140163915247615,
+SNULL, 140163913138175, 140163915247615,
+STORE, 140163912994816, 140163913138175,
+STORE, 140163913138176, 140163915247615,
+ERASE, 140163913138176, 140163915247615,
+STORE, 140163915235328, 140163915243519,
+STORE, 140163915243520, 140163915247615,
+STORE, 140731832217600, 140731832221695,
+STORE, 140731832205312, 140731832217599,
+STORE, 140163915206656, 140163915235327,
+STORE, 140163915198464, 140163915206655,
+STORE, 140163909197824, 140163912994815,
+SNULL, 140163909197824, 140163910856703,
+STORE, 140163910856704, 140163912994815,
+STORE, 140163909197824, 140163910856703,
+SNULL, 140163912953855, 140163912994815,
+STORE, 140163910856704, 140163912953855,
+STORE, 140163912953856, 140163912994815,
+SNULL, 140163912953856, 140163912978431,
+STORE, 140163912978432, 140163912994815,
+STORE, 140163912953856, 140163912978431,
+ERASE, 140163912953856, 140163912978431,
+STORE, 140163912953856, 140163912978431,
+ERASE, 140163912978432, 140163912994815,
+STORE, 140163912978432, 140163912994815,
+SNULL, 140163912970239, 140163912978431,
+STORE, 140163912953856, 140163912970239,
+STORE, 140163912970240, 140163912978431,
+SNULL, 94615307472895, 94615307476991,
+STORE, 94615307464704, 94615307472895,
+STORE, 94615307472896, 94615307476991,
+SNULL, 140163915239423, 140163915243519,
+STORE, 140163915235328, 140163915239423,
+STORE, 140163915239424, 140163915243519,
+ERASE, 140163915206656, 140163915235327,
+STORE, 94615330672640, 94615330807807,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140725254479872, 140737488351231,
+SNULL, 140725254488063, 140737488351231,
+STORE, 140725254479872, 140725254488063,
+STORE, 140725254348800, 140725254488063,
+STORE, 94572781277184, 94572785741823,
+SNULL, 94572783312895, 94572785741823,
+STORE, 94572781277184, 94572783312895,
+STORE, 94572783312896, 94572785741823,
+ERASE, 94572783312896, 94572785741823,
+STORE, 94572785405952, 94572785455103,
+STORE, 94572785455104, 94572785741823,
+STORE, 139636001341440, 139636003594239,
+SNULL, 139636001484799, 139636003594239,
+STORE, 139636001341440, 139636001484799,
+STORE, 139636001484800, 139636003594239,
+ERASE, 139636001484800, 139636003594239,
+STORE, 139636003581952, 139636003590143,
+STORE, 139636003590144, 139636003594239,
+STORE, 140725255557120, 140725255561215,
+STORE, 140725255544832, 140725255557119,
+STORE, 139636003553280, 139636003581951,
+STORE, 139636003545088, 139636003553279,
+STORE, 139635998773248, 139636001341439,
+SNULL, 139635998773248, 139635999240191,
+STORE, 139635999240192, 139636001341439,
+STORE, 139635998773248, 139635999240191,
+SNULL, 139636001333247, 139636001341439,
+STORE, 139635999240192, 139636001333247,
+STORE, 139636001333248, 139636001341439,
+ERASE, 139636001333248, 139636001341439,
+STORE, 139636001333248, 139636001341439,
+STORE, 139635996569600, 139635998773247,
+SNULL, 139635996569600, 139635996671999,
+STORE, 139635996672000, 139635998773247,
+STORE, 139635996569600, 139635996671999,
+SNULL, 139635998765055, 139635998773247,
+STORE, 139635996672000, 139635998765055,
+STORE, 139635998765056, 139635998773247,
+ERASE, 139635998765056, 139635998773247,
+STORE, 139635998765056, 139635998773247,
+STORE, 139635994353664, 139635996569599,
+SNULL, 139635994353664, 139635994451967,
+STORE, 139635994451968, 139635996569599,
+STORE, 139635994353664, 139635994451967,
+SNULL, 139635996545023, 139635996569599,
+STORE, 139635994451968, 139635996545023,
+STORE, 139635996545024, 139635996569599,
+SNULL, 139635996545024, 139635996553215,
+STORE, 139635996553216, 139635996569599,
+STORE, 139635996545024, 139635996553215,
+ERASE, 139635996545024, 139635996553215,
+STORE, 139635996545024, 139635996553215,
+ERASE, 139635996553216, 139635996569599,
+STORE, 139635996553216, 139635996569599,
+STORE, 139635992223744, 139635994353663,
+SNULL, 139635992223744, 139635992252415,
+STORE, 139635992252416, 139635994353663,
+STORE, 139635992223744, 139635992252415,
+SNULL, 139635994345471, 139635994353663,
+STORE, 139635992252416, 139635994345471,
+STORE, 139635994345472, 139635994353663,
+ERASE, 139635994345472, 139635994353663,
+STORE, 139635994345472, 139635994353663,
+STORE, 139635988426752, 139635992223743,
+SNULL, 139635988426752, 139635990085631,
+STORE, 139635990085632, 139635992223743,
+STORE, 139635988426752, 139635990085631,
+SNULL, 139635992182783, 139635992223743,
+STORE, 139635990085632, 139635992182783,
+STORE, 139635992182784, 139635992223743,
+SNULL, 139635992182784, 139635992207359,
+STORE, 139635992207360, 139635992223743,
+STORE, 139635992182784, 139635992207359,
+ERASE, 139635992182784, 139635992207359,
+STORE, 139635992182784, 139635992207359,
+ERASE, 139635992207360, 139635992223743,
+STORE, 139635992207360, 139635992223743,
+STORE, 139636003536896, 139636003553279,
+SNULL, 139635992199167, 139635992207359,
+STORE, 139635992182784, 139635992199167,
+STORE, 139635992199168, 139635992207359,
+SNULL, 139635996549119, 139635996553215,
+STORE, 139635996545024, 139635996549119,
+STORE, 139635996549120, 139635996553215,
+SNULL, 139635994349567, 139635994353663,
+STORE, 139635994345472, 139635994349567,
+STORE, 139635994349568, 139635994353663,
+SNULL, 139635998769151, 139635998773247,
+STORE, 139635998765056, 139635998769151,
+STORE, 139635998769152, 139635998773247,
+SNULL, 139636001337343, 139636001341439,
+STORE, 139636001333248, 139636001337343,
+STORE, 139636001337344, 139636001341439,
+SNULL, 94572785418239, 94572785455103,
+STORE, 94572785405952, 94572785418239,
+STORE, 94572785418240, 94572785455103,
+SNULL, 139636003586047, 139636003590143,
+STORE, 139636003581952, 139636003586047,
+STORE, 139636003586048, 139636003590143,
+ERASE, 139636003553280, 139636003581951,
+STORE, 94572798435328, 94572798570495,
+STORE, 139636001853440, 139636003536895,
+STORE, 139635981426688, 139635988426751,
+STORE, 139635980615680, 139635981426687,
+STORE, 94572798435328, 94572798705663,
+STORE, 94572798435328, 94572798840831,
+STORE, 94572798435328, 94572798975999,
+STORE, 94572798435328, 94572799111167,
+STORE, 94572798435328, 94572799246335,
+STORE, 94572798435328, 94572799381503,
+STORE, 94572798435328, 94572799516671,
+STORE, 94572798435328, 94572799651839,
+STORE, 94572798435328, 94572799787007,
+STORE, 94572798435328, 94572799922175,
+STORE, 94572798435328, 94572800057343,
+STORE, 94572798435328, 94572800192511,
+STORE, 94572798435328, 94572800327679,
+STORE, 94572798435328, 94572800462847,
+STORE, 94572798435328, 94572800598015,
+STORE, 94572798435328, 94572800733183,
+STORE, 94572798435328, 94572800868351,
+STORE, 94572798435328, 94572801003519,
+STORE, 94572798435328, 94572801138687,
+STORE, 94572798435328, 94572801273855,
+STORE, 94572798435328, 94572801409023,
+STORE, 94572798435328, 94572801544191,
+STORE, 94572798435328, 94572801679359,
+STORE, 94572798435328, 94572801814527,
+STORE, 94572798435328, 94572801949695,
+STORE, 94572798435328, 94572802084863,
+STORE, 94572798435328, 94572802220031,
+STORE, 94572798435328, 94572802355199,
+STORE, 94572798435328, 94572802490367,
+STORE, 94572798435328, 94572802625535,
+STORE, 94572798435328, 94572802760703,
+STORE, 94572798435328, 94572802895871,
+STORE, 94572798435328, 94572803031039,
+STORE, 94572798435328, 94572803166207,
+STORE, 94572798435328, 94572803301375,
+STORE, 94572798435328, 94572803436543,
+STORE, 94572798435328, 94572803571711,
+STORE, 94572798435328, 94572803706879,
+STORE, 94572798435328, 94572803842047,
+STORE, 94572798435328, 94572803977215,
+STORE, 94572798435328, 94572804112383,
+STORE, 94572798435328, 94572804247551,
+STORE, 94572798435328, 94572804382719,
+STORE, 94572798435328, 94572804517887,
+STORE, 94572798435328, 94572804653055,
+STORE, 94572798435328, 94572804788223,
+STORE, 94572798435328, 94572804923391,
+STORE, 94572798435328, 94572805058559,
+STORE, 94572798435328, 94572805193727,
+STORE, 94572798435328, 94572805328895,
+STORE, 94572798435328, 94572805464063,
+STORE, 94572798435328, 94572805599231,
+STORE, 94572798435328, 94572805734399,
+STORE, 94572798435328, 94572805869567,
+STORE, 94572798435328, 94572806004735,
+STORE, 94572798435328, 94572806139903,
+STORE, 94572798435328, 94572806275071,
+STORE, 94572798435328, 94572806410239,
+STORE, 94572798435328, 94572806545407,
+STORE, 94572798435328, 94572806680575,
+STORE, 94572798435328, 94572806815743,
+STORE, 94572798435328, 94572806950911,
+STORE, 94572798435328, 94572807086079,
+STORE, 94572798435328, 94572807221247,
+STORE, 94572798435328, 94572807356415,
+STORE, 94572798435328, 94572807491583,
+STORE, 94572798435328, 94572807626751,
+STORE, 94572798435328, 94572807761919,
+STORE, 94572798435328, 94572807897087,
+STORE, 94572798435328, 94572808032255,
+STORE, 94572798435328, 94572808167423,
+STORE, 94572798435328, 94572808302591,
+STORE, 94572798435328, 94572808437759,
+STORE, 94572798435328, 94572808572927,
+ERASE, 139635981426688, 139635988426751,
+STORE, 139635985088512, 139635988426751,
+STORE, 139635778273280, 139635980615679,
+STORE, 139635567632384, 139635778273279,
+STORE, 94572798435328, 94572808716287,
+STORE, 139635984564224, 139635985088511,
+STORE, 139635559239680, 139635567632383,
+SNULL, 139635559243775, 139635567632383,
+STORE, 139635559239680, 139635559243775,
+STORE, 139635559243776, 139635567632383,
+STORE, 139635550846976, 139635559239679,
+SNULL, 139635550851071, 139635559239679,
+STORE, 139635550846976, 139635550851071,
+STORE, 139635550851072, 139635559239679,
+STORE, 139635542454272, 139635550846975,
+STORE, 139635408236544, 139635542454271,
+SNULL, 139635408236544, 139635426590719,
+STORE, 139635426590720, 139635542454271,
+STORE, 139635408236544, 139635426590719,
+ERASE, 139635408236544, 139635426590719,
+STORE, 139635292372992, 139635542454271,
+SNULL, 139635359481855, 139635542454271,
+STORE, 139635292372992, 139635359481855,
+STORE, 139635359481856, 139635542454271,
+SNULL, 139635359481856, 139635426590719,
+STORE, 139635426590720, 139635542454271,
+STORE, 139635359481856, 139635426590719,
+ERASE, 139635359481856, 139635426590719,
+SNULL, 139635542458367, 139635550846975,
+STORE, 139635542454272, 139635542458367,
+STORE, 139635542458368, 139635550846975,
+STORE, 139635418198016, 139635426590719,
+SNULL, 139635493699583, 139635542454271,
+STORE, 139635426590720, 139635493699583,
+STORE, 139635493699584, 139635542454271,
+ERASE, 139635493699584, 139635542454271,
+SNULL, 139635426725887, 139635493699583,
+STORE, 139635426590720, 139635426725887,
+STORE, 139635426725888, 139635493699583,
+SNULL, 139635292508159, 139635359481855,
+STORE, 139635292372992, 139635292508159,
+STORE, 139635292508160, 139635359481855,
+SNULL, 139635418202111, 139635426590719,
+STORE, 139635418198016, 139635418202111,
+STORE, 139635418202112, 139635426590719,
+STORE, 139635225264128, 139635292372991,
+STORE, 139635534061568, 139635542454271,
+SNULL, 139635534065663, 139635542454271,
+STORE, 139635534061568, 139635534065663,
+STORE, 139635534065664, 139635542454271,
+STORE, 139635525668864, 139635534061567,
+SNULL, 139635525672959, 139635534061567,
+STORE, 139635525668864, 139635525672959,
+STORE, 139635525672960, 139635534061567,
+SNULL, 139635225399295, 139635292372991,
+STORE, 139635225264128, 139635225399295,
+STORE, 139635225399296, 139635292372991,
+STORE, 139635091046400, 139635225264127,
+SNULL, 139635158155263, 139635225264127,
+STORE, 139635091046400, 139635158155263,
+STORE, 139635158155264, 139635225264127,
+ERASE, 139635158155264, 139635225264127,
+STORE, 139634956828672, 139635158155263,
+STORE, 139635517276160, 139635525668863,
+SNULL, 139635517280255, 139635525668863,
+STORE, 139635517276160, 139635517280255,
+STORE, 139635517280256, 139635525668863,
+SNULL, 139634956828672, 139635091046399,
+STORE, 139635091046400, 139635158155263,
+STORE, 139634956828672, 139635091046399,
+SNULL, 139635091181567, 139635158155263,
+STORE, 139635091046400, 139635091181567,
+STORE, 139635091181568, 139635158155263,
+SNULL, 139635023937535, 139635091046399,
+STORE, 139634956828672, 139635023937535,
+STORE, 139635023937536, 139635091046399,
+ERASE, 139635023937536, 139635091046399,
+STORE, 139634956828672, 139635091046399,
+SNULL, 139634956828672, 139635023937535,
+STORE, 139635023937536, 139635091046399,
+STORE, 139634956828672, 139635023937535,
+SNULL, 139635024072703, 139635091046399,
+STORE, 139635023937536, 139635024072703,
+STORE, 139635024072704, 139635091046399,
+STORE, 139635508883456, 139635517276159,
+SNULL, 139635508887551, 139635517276159,
+STORE, 139635508883456, 139635508887551,
+STORE, 139635508887552, 139635517276159,
+STORE, 139634822610944, 139635023937535,
+SNULL, 139634822610944, 139634956828671,
+STORE, 139634956828672, 139635023937535,
+STORE, 139634822610944, 139634956828671,
+SNULL, 139634956963839, 139635023937535,
+STORE, 139634956828672, 139634956963839,
+STORE, 139634956963840, 139635023937535,
+STORE, 139635500490752, 139635508883455,
+SNULL, 139634889719807, 139634956828671,
+STORE, 139634822610944, 139634889719807,
+STORE, 139634889719808, 139634956828671,
+ERASE, 139634889719808, 139634956828671,
+SNULL, 139635500494847, 139635508883455,
+STORE, 139635500490752, 139635500494847,
+STORE, 139635500494848, 139635508883455,
+SNULL, 139634822746111, 139634889719807,
+STORE, 139634822610944, 139634822746111,
+STORE, 139634822746112, 139634889719807,
+STORE, 139635409805312, 139635418198015,
+STORE, 139634822746112, 139634956828671,
+SNULL, 139634822746112, 139634889719807,
+STORE, 139634889719808, 139634956828671,
+STORE, 139634822746112, 139634889719807,
+SNULL, 139634889854975, 139634956828671,
+STORE, 139634889719808, 139634889854975,
+STORE, 139634889854976, 139634956828671,
+SNULL, 139635409809407, 139635418198015,
+STORE, 139635409805312, 139635409809407,
+STORE, 139635409809408, 139635418198015,
+STORE, 139635401412608, 139635409805311,
+STORE, 139634688393216, 139634822610943,
+SNULL, 139634755502079, 139634822610943,
+STORE, 139634688393216, 139634755502079,
+STORE, 139634755502080, 139634822610943,
+ERASE, 139634755502080, 139634822610943,
+SNULL, 139635401416703, 139635409805311,
+STORE, 139635401412608, 139635401416703,
+STORE, 139635401416704, 139635409805311,
+STORE, 139634554175488, 139634755502079,
+SNULL, 139634554175488, 139634688393215,
+STORE, 139634688393216, 139634755502079,
+STORE, 139634554175488, 139634688393215,
+SNULL, 139634688528383, 139634755502079,
+STORE, 139634688393216, 139634688528383,
+STORE, 139634688528384, 139634755502079,
+STORE, 139635393019904, 139635401412607,
+SNULL, 139634621284351, 139634688393215,
+STORE, 139634554175488, 139634621284351,
+STORE, 139634621284352, 139634688393215,
+ERASE, 139634621284352, 139634688393215,
+SNULL, 139634554310655, 139634621284351,
+STORE, 139634554175488, 139634554310655,
+STORE, 139634554310656, 139634621284351,
+STORE, 139634554310656, 139634688393215,
+SNULL, 139635393023999, 139635401412607,
+STORE, 139635393019904, 139635393023999,
+STORE, 139635393024000, 139635401412607,
+SNULL, 139634554310656, 139634621284351,
+STORE, 139634621284352, 139634688393215,
+STORE, 139634554310656, 139634621284351,
+SNULL, 139634621419519, 139634688393215,
+STORE, 139634621284352, 139634621419519,
+STORE, 139634621419520, 139634688393215,
+STORE, 139635384627200, 139635393019903,
+SNULL, 139635384631295, 139635393019903,
+STORE, 139635384627200, 139635384631295,
+STORE, 139635384631296, 139635393019903,
+STORE, 139635376234496, 139635384627199,
+SNULL, 139635376238591, 139635384627199,
+STORE, 139635376234496, 139635376238591,
+STORE, 139635376238592, 139635384627199,
+STORE, 139635367841792, 139635376234495,
+SNULL, 139635367845887, 139635376234495,
+STORE, 139635367841792, 139635367845887,
+STORE, 139635367845888, 139635376234495,
+STORE, 139634419957760, 139634554175487,
+SNULL, 139634487066623, 139634554175487,
+STORE, 139634419957760, 139634487066623,
+STORE, 139634487066624, 139634554175487,
+ERASE, 139634487066624, 139634554175487,
+STORE, 139635216871424, 139635225264127,
+SNULL, 139635216875519, 139635225264127,
+STORE, 139635216871424, 139635216875519,
+STORE, 139635216875520, 139635225264127,
+SNULL, 139634420092927, 139634487066623,
+STORE, 139634419957760, 139634420092927,
+STORE, 139634420092928, 139634487066623,
+STORE, 139635208478720, 139635216871423,
+SNULL, 139635208482815, 139635216871423,
+STORE, 139635208478720, 139635208482815,
+STORE, 139635208482816, 139635216871423,
+STORE, 139635200086016, 139635208478719,
+SNULL, 139635200090111, 139635208478719,
+STORE, 139635200086016, 139635200090111,
+STORE, 139635200090112, 139635208478719,
+STORE, 139635191693312, 139635200086015,
+SNULL, 139635191697407, 139635200086015,
+STORE, 139635191693312, 139635191697407,
+STORE, 139635191697408, 139635200086015,
+STORE, 139635183300608, 139635191693311,
+SNULL, 139635183304703, 139635191693311,
+STORE, 139635183300608, 139635183304703,
+STORE, 139635183304704, 139635191693311,
+STORE, 139634420092928, 139634554175487,
+SNULL, 139634420092928, 139634487066623,
+STORE, 139634487066624, 139634554175487,
+STORE, 139634420092928, 139634487066623,
+SNULL, 139634487201791, 139634554175487,
+STORE, 139634487066624, 139634487201791,
+STORE, 139634487201792, 139634554175487,
+ERASE, 139635559239680, 139635559243775,
+ERASE, 139635559243776, 139635567632383,
+ERASE, 139635550846976, 139635550851071,
+ERASE, 139635550851072, 139635559239679,
+ERASE, 139635542454272, 139635542458367,
+ERASE, 139635542458368, 139635550846975,
+ERASE, 139635418198016, 139635418202111,
+ERASE, 139635418202112, 139635426590719,
+ERASE, 139635534061568, 139635534065663,
+ERASE, 139635534065664, 139635542454271,
+ERASE, 139635525668864, 139635525672959,
+ERASE, 139635525672960, 139635534061567,
+ERASE, 139635517276160, 139635517280255,
+ERASE, 139635517280256, 139635525668863,
+ERASE, 139635508883456, 139635508887551,
+ERASE, 139635508887552, 139635517276159,
+ERASE, 139635500490752, 139635500494847,
+ERASE, 139635500494848, 139635508883455,
+ERASE, 139635409805312, 139635409809407,
+ERASE, 139635409809408, 139635418198015,
+ERASE, 139635401412608, 139635401416703,
+ERASE, 139635401416704, 139635409805311,
+ERASE, 139635393019904, 139635393023999,
+ERASE, 139635393024000, 139635401412607,
+ERASE, 139635384627200, 139635384631295,
+ERASE, 139635384631296, 139635393019903,
+ };
+ unsigned long set25[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722547441664, 140737488351231,
+SNULL, 140722547449855, 140737488351231,
+STORE, 140722547441664, 140722547449855,
+STORE, 140722547310592, 140722547449855,
+STORE, 94827521732608, 94827523956735,
+SNULL, 94827521843199, 94827523956735,
+STORE, 94827521732608, 94827521843199,
+STORE, 94827521843200, 94827523956735,
+ERASE, 94827521843200, 94827523956735,
+STORE, 94827523936256, 94827523948543,
+STORE, 94827523948544, 94827523956735,
+STORE, 139816136847360, 139816139100159,
+SNULL, 139816136990719, 139816139100159,
+STORE, 139816136847360, 139816136990719,
+STORE, 139816136990720, 139816139100159,
+ERASE, 139816136990720, 139816139100159,
+STORE, 139816139087872, 139816139096063,
+STORE, 139816139096064, 139816139100159,
+STORE, 140722548142080, 140722548146175,
+STORE, 140722548129792, 140722548142079,
+STORE, 139816139059200, 139816139087871,
+STORE, 139816139051008, 139816139059199,
+STORE, 139816133050368, 139816136847359,
+SNULL, 139816133050368, 139816134709247,
+STORE, 139816134709248, 139816136847359,
+STORE, 139816133050368, 139816134709247,
+SNULL, 139816136806399, 139816136847359,
+STORE, 139816134709248, 139816136806399,
+STORE, 139816136806400, 139816136847359,
+SNULL, 139816136806400, 139816136830975,
+STORE, 139816136830976, 139816136847359,
+STORE, 139816136806400, 139816136830975,
+ERASE, 139816136806400, 139816136830975,
+STORE, 139816136806400, 139816136830975,
+ERASE, 139816136830976, 139816136847359,
+STORE, 139816136830976, 139816136847359,
+SNULL, 139816136822783, 139816136830975,
+STORE, 139816136806400, 139816136822783,
+STORE, 139816136822784, 139816136830975,
+SNULL, 94827523944447, 94827523948543,
+STORE, 94827523936256, 94827523944447,
+STORE, 94827523944448, 94827523948543,
+SNULL, 139816139091967, 139816139096063,
+STORE, 139816139087872, 139816139091967,
+STORE, 139816139091968, 139816139096063,
+ERASE, 139816139059200, 139816139087871,
+STORE, 94827534970880, 94827535106047,
+STORE, 94114394132480, 94114394345471,
+STORE, 94114396442624, 94114396446719,
+STORE, 94114396446720, 94114396454911,
+STORE, 94114396454912, 94114396467199,
+STORE, 94114421575680, 94114427715583,
+STORE, 139934313955328, 139934315614207,
+STORE, 139934315614208, 139934317711359,
+STORE, 139934317711360, 139934317727743,
+STORE, 139934317727744, 139934317735935,
+STORE, 139934317735936, 139934317752319,
+STORE, 139934317752320, 139934317764607,
+STORE, 139934317764608, 139934319857663,
+STORE, 139934319857664, 139934319861759,
+STORE, 139934319861760, 139934319865855,
+STORE, 139934319865856, 139934320009215,
+STORE, 139934320377856, 139934322061311,
+STORE, 139934322061312, 139934322077695,
+STORE, 139934322106368, 139934322110463,
+STORE, 139934322110464, 139934322114559,
+STORE, 139934322114560, 139934322118655,
+STORE, 140731200376832, 140731200516095,
+STORE, 140731200929792, 140731200942079,
+STORE, 140731200942080, 140731200946175,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140734133174272, 140737488351231,
+SNULL, 140734133182463, 140737488351231,
+STORE, 140734133174272, 140734133182463,
+STORE, 140734133043200, 140734133182463,
+STORE, 94412675600384, 94412677824511,
+SNULL, 94412675710975, 94412677824511,
+STORE, 94412675600384, 94412675710975,
+STORE, 94412675710976, 94412677824511,
+ERASE, 94412675710976, 94412677824511,
+STORE, 94412677804032, 94412677816319,
+STORE, 94412677816320, 94412677824511,
+STORE, 140320087945216, 140320090198015,
+SNULL, 140320088088575, 140320090198015,
+STORE, 140320087945216, 140320088088575,
+STORE, 140320088088576, 140320090198015,
+ERASE, 140320088088576, 140320090198015,
+STORE, 140320090185728, 140320090193919,
+STORE, 140320090193920, 140320090198015,
+STORE, 140734134591488, 140734134595583,
+STORE, 140734134579200, 140734134591487,
+STORE, 140320090157056, 140320090185727,
+STORE, 140320090148864, 140320090157055,
+STORE, 140320084148224, 140320087945215,
+SNULL, 140320084148224, 140320085807103,
+STORE, 140320085807104, 140320087945215,
+STORE, 140320084148224, 140320085807103,
+SNULL, 140320087904255, 140320087945215,
+STORE, 140320085807104, 140320087904255,
+STORE, 140320087904256, 140320087945215,
+SNULL, 140320087904256, 140320087928831,
+STORE, 140320087928832, 140320087945215,
+STORE, 140320087904256, 140320087928831,
+ERASE, 140320087904256, 140320087928831,
+STORE, 140320087904256, 140320087928831,
+ERASE, 140320087928832, 140320087945215,
+STORE, 140320087928832, 140320087945215,
+SNULL, 140320087920639, 140320087928831,
+STORE, 140320087904256, 140320087920639,
+STORE, 140320087920640, 140320087928831,
+SNULL, 94412677812223, 94412677816319,
+STORE, 94412677804032, 94412677812223,
+STORE, 94412677812224, 94412677816319,
+SNULL, 140320090189823, 140320090193919,
+STORE, 140320090185728, 140320090189823,
+STORE, 140320090189824, 140320090193919,
+ERASE, 140320090157056, 140320090185727,
+STORE, 94412684546048, 94412684681215,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723005485056, 140737488351231,
+SNULL, 140723005493247, 140737488351231,
+STORE, 140723005485056, 140723005493247,
+STORE, 140723005353984, 140723005493247,
+STORE, 94387431936000, 94387434160127,
+SNULL, 94387432046591, 94387434160127,
+STORE, 94387431936000, 94387432046591,
+STORE, 94387432046592, 94387434160127,
+ERASE, 94387432046592, 94387434160127,
+STORE, 94387434139648, 94387434151935,
+STORE, 94387434151936, 94387434160127,
+STORE, 140151675392000, 140151677644799,
+SNULL, 140151675535359, 140151677644799,
+STORE, 140151675392000, 140151675535359,
+STORE, 140151675535360, 140151677644799,
+ERASE, 140151675535360, 140151677644799,
+STORE, 140151677632512, 140151677640703,
+STORE, 140151677640704, 140151677644799,
+STORE, 140723005784064, 140723005788159,
+STORE, 140723005771776, 140723005784063,
+STORE, 140151677603840, 140151677632511,
+STORE, 140151677595648, 140151677603839,
+STORE, 140151671595008, 140151675391999,
+SNULL, 140151671595008, 140151673253887,
+STORE, 140151673253888, 140151675391999,
+STORE, 140151671595008, 140151673253887,
+SNULL, 140151675351039, 140151675391999,
+STORE, 140151673253888, 140151675351039,
+STORE, 140151675351040, 140151675391999,
+SNULL, 140151675351040, 140151675375615,
+STORE, 140151675375616, 140151675391999,
+STORE, 140151675351040, 140151675375615,
+ERASE, 140151675351040, 140151675375615,
+STORE, 140151675351040, 140151675375615,
+ERASE, 140151675375616, 140151675391999,
+STORE, 140151675375616, 140151675391999,
+SNULL, 140151675367423, 140151675375615,
+STORE, 140151675351040, 140151675367423,
+STORE, 140151675367424, 140151675375615,
+SNULL, 94387434147839, 94387434151935,
+STORE, 94387434139648, 94387434147839,
+STORE, 94387434147840, 94387434151935,
+SNULL, 140151677636607, 140151677640703,
+STORE, 140151677632512, 140151677636607,
+STORE, 140151677636608, 140151677640703,
+ERASE, 140151677603840, 140151677632511,
+STORE, 94387458818048, 94387458953215,
+STORE, 94909010997248, 94909011210239,
+STORE, 94909013307392, 94909013311487,
+STORE, 94909013311488, 94909013319679,
+STORE, 94909013319680, 94909013331967,
+STORE, 94909014827008, 94909023371263,
+STORE, 140712411975680, 140712413634559,
+STORE, 140712413634560, 140712415731711,
+STORE, 140712415731712, 140712415748095,
+STORE, 140712415748096, 140712415756287,
+STORE, 140712415756288, 140712415772671,
+STORE, 140712415772672, 140712415784959,
+STORE, 140712415784960, 140712417878015,
+STORE, 140712417878016, 140712417882111,
+STORE, 140712417882112, 140712417886207,
+STORE, 140712417886208, 140712418029567,
+STORE, 140712418398208, 140712420081663,
+STORE, 140712420081664, 140712420098047,
+STORE, 140712420126720, 140712420130815,
+STORE, 140712420130816, 140712420134911,
+STORE, 140712420134912, 140712420139007,
+STORE, 140729293111296, 140729293250559,
+STORE, 140729293307904, 140729293320191,
+STORE, 140729293320192, 140729293324287,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140720541691904, 140737488351231,
+SNULL, 140720541700095, 140737488351231,
+STORE, 140720541691904, 140720541700095,
+STORE, 140720541560832, 140720541700095,
+STORE, 94203603419136, 94203605643263,
+SNULL, 94203603529727, 94203605643263,
+STORE, 94203603419136, 94203603529727,
+STORE, 94203603529728, 94203605643263,
+ERASE, 94203603529728, 94203605643263,
+STORE, 94203605622784, 94203605635071,
+STORE, 94203605635072, 94203605643263,
+STORE, 139847623081984, 139847625334783,
+SNULL, 139847623225343, 139847625334783,
+STORE, 139847623081984, 139847623225343,
+STORE, 139847623225344, 139847625334783,
+ERASE, 139847623225344, 139847625334783,
+STORE, 139847625322496, 139847625330687,
+STORE, 139847625330688, 139847625334783,
+STORE, 140720542547968, 140720542552063,
+STORE, 140720542535680, 140720542547967,
+STORE, 139847625293824, 139847625322495,
+STORE, 139847625285632, 139847625293823,
+STORE, 139847619284992, 139847623081983,
+SNULL, 139847619284992, 139847620943871,
+STORE, 139847620943872, 139847623081983,
+STORE, 139847619284992, 139847620943871,
+SNULL, 139847623041023, 139847623081983,
+STORE, 139847620943872, 139847623041023,
+STORE, 139847623041024, 139847623081983,
+SNULL, 139847623041024, 139847623065599,
+STORE, 139847623065600, 139847623081983,
+STORE, 139847623041024, 139847623065599,
+ERASE, 139847623041024, 139847623065599,
+STORE, 139847623041024, 139847623065599,
+ERASE, 139847623065600, 139847623081983,
+STORE, 139847623065600, 139847623081983,
+SNULL, 139847623057407, 139847623065599,
+STORE, 139847623041024, 139847623057407,
+STORE, 139847623057408, 139847623065599,
+SNULL, 94203605630975, 94203605635071,
+STORE, 94203605622784, 94203605630975,
+STORE, 94203605630976, 94203605635071,
+SNULL, 139847625326591, 139847625330687,
+STORE, 139847625322496, 139847625326591,
+STORE, 139847625326592, 139847625330687,
+ERASE, 139847625293824, 139847625322495,
+STORE, 94203634880512, 94203635015679,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140721428738048, 140737488351231,
+SNULL, 140721428746239, 140737488351231,
+STORE, 140721428738048, 140721428746239,
+STORE, 140721428606976, 140721428746239,
+STORE, 93968808378368, 93968810602495,
+SNULL, 93968808488959, 93968810602495,
+STORE, 93968808378368, 93968808488959,
+STORE, 93968808488960, 93968810602495,
+ERASE, 93968808488960, 93968810602495,
+STORE, 93968810582016, 93968810594303,
+STORE, 93968810594304, 93968810602495,
+STORE, 140397757026304, 140397759279103,
+SNULL, 140397757169663, 140397759279103,
+STORE, 140397757026304, 140397757169663,
+STORE, 140397757169664, 140397759279103,
+ERASE, 140397757169664, 140397759279103,
+STORE, 140397759266816, 140397759275007,
+STORE, 140397759275008, 140397759279103,
+STORE, 140721430368256, 140721430372351,
+STORE, 140721430355968, 140721430368255,
+STORE, 140397759238144, 140397759266815,
+STORE, 140397759229952, 140397759238143,
+STORE, 140397753229312, 140397757026303,
+SNULL, 140397753229312, 140397754888191,
+STORE, 140397754888192, 140397757026303,
+STORE, 140397753229312, 140397754888191,
+SNULL, 140397756985343, 140397757026303,
+STORE, 140397754888192, 140397756985343,
+STORE, 140397756985344, 140397757026303,
+SNULL, 140397756985344, 140397757009919,
+STORE, 140397757009920, 140397757026303,
+STORE, 140397756985344, 140397757009919,
+ERASE, 140397756985344, 140397757009919,
+STORE, 140397756985344, 140397757009919,
+ERASE, 140397757009920, 140397757026303,
+STORE, 140397757009920, 140397757026303,
+SNULL, 140397757001727, 140397757009919,
+STORE, 140397756985344, 140397757001727,
+STORE, 140397757001728, 140397757009919,
+SNULL, 93968810590207, 93968810594303,
+STORE, 93968810582016, 93968810590207,
+STORE, 93968810590208, 93968810594303,
+SNULL, 140397759270911, 140397759275007,
+STORE, 140397759266816, 140397759270911,
+STORE, 140397759270912, 140397759275007,
+ERASE, 140397759238144, 140397759266815,
+STORE, 93968837025792, 93968837160959,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140721751044096, 140737488351231,
+SNULL, 140721751052287, 140737488351231,
+STORE, 140721751044096, 140721751052287,
+STORE, 140721750913024, 140721751052287,
+STORE, 94426051657728, 94426053881855,
+SNULL, 94426051768319, 94426053881855,
+STORE, 94426051657728, 94426051768319,
+STORE, 94426051768320, 94426053881855,
+ERASE, 94426051768320, 94426053881855,
+STORE, 94426053861376, 94426053873663,
+STORE, 94426053873664, 94426053881855,
+STORE, 140228456181760, 140228458434559,
+SNULL, 140228456325119, 140228458434559,
+STORE, 140228456181760, 140228456325119,
+STORE, 140228456325120, 140228458434559,
+ERASE, 140228456325120, 140228458434559,
+STORE, 140228458422272, 140228458430463,
+STORE, 140228458430464, 140228458434559,
+STORE, 140721751117824, 140721751121919,
+STORE, 140721751105536, 140721751117823,
+STORE, 140228458393600, 140228458422271,
+STORE, 140228458385408, 140228458393599,
+STORE, 140228452384768, 140228456181759,
+SNULL, 140228452384768, 140228454043647,
+STORE, 140228454043648, 140228456181759,
+STORE, 140228452384768, 140228454043647,
+SNULL, 140228456140799, 140228456181759,
+STORE, 140228454043648, 140228456140799,
+STORE, 140228456140800, 140228456181759,
+SNULL, 140228456140800, 140228456165375,
+STORE, 140228456165376, 140228456181759,
+STORE, 140228456140800, 140228456165375,
+ERASE, 140228456140800, 140228456165375,
+STORE, 140228456140800, 140228456165375,
+ERASE, 140228456165376, 140228456181759,
+STORE, 140228456165376, 140228456181759,
+SNULL, 140228456157183, 140228456165375,
+STORE, 140228456140800, 140228456157183,
+STORE, 140228456157184, 140228456165375,
+SNULL, 94426053869567, 94426053873663,
+STORE, 94426053861376, 94426053869567,
+STORE, 94426053869568, 94426053873663,
+SNULL, 140228458426367, 140228458430463,
+STORE, 140228458422272, 140228458426367,
+STORE, 140228458426368, 140228458430463,
+ERASE, 140228458393600, 140228458422271,
+STORE, 94426073681920, 94426073817087,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140732727623680, 140737488351231,
+SNULL, 140732727631871, 140737488351231,
+STORE, 140732727623680, 140732727631871,
+STORE, 140732727492608, 140732727631871,
+STORE, 94537485996032, 94537488220159,
+SNULL, 94537486106623, 94537488220159,
+STORE, 94537485996032, 94537486106623,
+STORE, 94537486106624, 94537488220159,
+ERASE, 94537486106624, 94537488220159,
+STORE, 94537488199680, 94537488211967,
+STORE, 94537488211968, 94537488220159,
+STORE, 140446578036736, 140446580289535,
+SNULL, 140446578180095, 140446580289535,
+STORE, 140446578036736, 140446578180095,
+STORE, 140446578180096, 140446580289535,
+ERASE, 140446578180096, 140446580289535,
+STORE, 140446580277248, 140446580285439,
+STORE, 140446580285440, 140446580289535,
+STORE, 140732727758848, 140732727762943,
+STORE, 140732727746560, 140732727758847,
+STORE, 140446580248576, 140446580277247,
+STORE, 140446580240384, 140446580248575,
+STORE, 140446574239744, 140446578036735,
+SNULL, 140446574239744, 140446575898623,
+STORE, 140446575898624, 140446578036735,
+STORE, 140446574239744, 140446575898623,
+SNULL, 140446577995775, 140446578036735,
+STORE, 140446575898624, 140446577995775,
+STORE, 140446577995776, 140446578036735,
+SNULL, 140446577995776, 140446578020351,
+STORE, 140446578020352, 140446578036735,
+STORE, 140446577995776, 140446578020351,
+ERASE, 140446577995776, 140446578020351,
+STORE, 140446577995776, 140446578020351,
+ERASE, 140446578020352, 140446578036735,
+STORE, 140446578020352, 140446578036735,
+SNULL, 140446578012159, 140446578020351,
+STORE, 140446577995776, 140446578012159,
+STORE, 140446578012160, 140446578020351,
+SNULL, 94537488207871, 94537488211967,
+STORE, 94537488199680, 94537488207871,
+STORE, 94537488207872, 94537488211967,
+SNULL, 140446580281343, 140446580285439,
+STORE, 140446580277248, 140446580281343,
+STORE, 140446580281344, 140446580285439,
+ERASE, 140446580248576, 140446580277247,
+STORE, 94537489014784, 94537489149951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728766808064, 140737488351231,
+SNULL, 140728766816255, 140737488351231,
+STORE, 140728766808064, 140728766816255,
+STORE, 140728766676992, 140728766816255,
+STORE, 94418513866752, 94418516090879,
+SNULL, 94418513977343, 94418516090879,
+STORE, 94418513866752, 94418513977343,
+STORE, 94418513977344, 94418516090879,
+ERASE, 94418513977344, 94418516090879,
+STORE, 94418516070400, 94418516082687,
+STORE, 94418516082688, 94418516090879,
+STORE, 140556479520768, 140556481773567,
+SNULL, 140556479664127, 140556481773567,
+STORE, 140556479520768, 140556479664127,
+STORE, 140556479664128, 140556481773567,
+ERASE, 140556479664128, 140556481773567,
+STORE, 140556481761280, 140556481769471,
+STORE, 140556481769472, 140556481773567,
+STORE, 140728767148032, 140728767152127,
+STORE, 140728767135744, 140728767148031,
+STORE, 140556481732608, 140556481761279,
+STORE, 140556481724416, 140556481732607,
+STORE, 140556475723776, 140556479520767,
+SNULL, 140556475723776, 140556477382655,
+STORE, 140556477382656, 140556479520767,
+STORE, 140556475723776, 140556477382655,
+SNULL, 140556479479807, 140556479520767,
+STORE, 140556477382656, 140556479479807,
+STORE, 140556479479808, 140556479520767,
+SNULL, 140556479479808, 140556479504383,
+STORE, 140556479504384, 140556479520767,
+STORE, 140556479479808, 140556479504383,
+ERASE, 140556479479808, 140556479504383,
+STORE, 140556479479808, 140556479504383,
+ERASE, 140556479504384, 140556479520767,
+STORE, 140556479504384, 140556479520767,
+SNULL, 140556479496191, 140556479504383,
+STORE, 140556479479808, 140556479496191,
+STORE, 140556479496192, 140556479504383,
+SNULL, 94418516078591, 94418516082687,
+STORE, 94418516070400, 94418516078591,
+STORE, 94418516078592, 94418516082687,
+SNULL, 140556481765375, 140556481769471,
+STORE, 140556481761280, 140556481765375,
+STORE, 140556481765376, 140556481769471,
+ERASE, 140556481732608, 140556481761279,
+STORE, 94418541113344, 94418541248511,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723945873408, 140737488351231,
+SNULL, 140723945881599, 140737488351231,
+STORE, 140723945873408, 140723945881599,
+STORE, 140723945742336, 140723945881599,
+STORE, 94543169773568, 94543171997695,
+SNULL, 94543169884159, 94543171997695,
+STORE, 94543169773568, 94543169884159,
+STORE, 94543169884160, 94543171997695,
+ERASE, 94543169884160, 94543171997695,
+STORE, 94543171977216, 94543171989503,
+STORE, 94543171989504, 94543171997695,
+STORE, 139890420883456, 139890423136255,
+SNULL, 139890421026815, 139890423136255,
+STORE, 139890420883456, 139890421026815,
+STORE, 139890421026816, 139890423136255,
+ERASE, 139890421026816, 139890423136255,
+STORE, 139890423123968, 139890423132159,
+STORE, 139890423132160, 139890423136255,
+STORE, 140723946102784, 140723946106879,
+STORE, 140723946090496, 140723946102783,
+STORE, 139890423095296, 139890423123967,
+STORE, 139890423087104, 139890423095295,
+STORE, 139890417086464, 139890420883455,
+SNULL, 139890417086464, 139890418745343,
+STORE, 139890418745344, 139890420883455,
+STORE, 139890417086464, 139890418745343,
+SNULL, 139890420842495, 139890420883455,
+STORE, 139890418745344, 139890420842495,
+STORE, 139890420842496, 139890420883455,
+SNULL, 139890420842496, 139890420867071,
+STORE, 139890420867072, 139890420883455,
+STORE, 139890420842496, 139890420867071,
+ERASE, 139890420842496, 139890420867071,
+STORE, 139890420842496, 139890420867071,
+ERASE, 139890420867072, 139890420883455,
+STORE, 139890420867072, 139890420883455,
+SNULL, 139890420858879, 139890420867071,
+STORE, 139890420842496, 139890420858879,
+STORE, 139890420858880, 139890420867071,
+SNULL, 94543171985407, 94543171989503,
+STORE, 94543171977216, 94543171985407,
+STORE, 94543171985408, 94543171989503,
+SNULL, 139890423128063, 139890423132159,
+STORE, 139890423123968, 139890423128063,
+STORE, 139890423128064, 139890423132159,
+ERASE, 139890423095296, 139890423123967,
+STORE, 94543197097984, 94543197233151,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140736205979648, 140737488351231,
+SNULL, 140736205987839, 140737488351231,
+STORE, 140736205979648, 140736205987839,
+STORE, 140736205848576, 140736205987839,
+STORE, 94913209913344, 94913212137471,
+SNULL, 94913210023935, 94913212137471,
+STORE, 94913209913344, 94913210023935,
+STORE, 94913210023936, 94913212137471,
+ERASE, 94913210023936, 94913212137471,
+STORE, 94913212116992, 94913212129279,
+STORE, 94913212129280, 94913212137471,
+STORE, 140006323052544, 140006325305343,
+SNULL, 140006323195903, 140006325305343,
+STORE, 140006323052544, 140006323195903,
+STORE, 140006323195904, 140006325305343,
+ERASE, 140006323195904, 140006325305343,
+STORE, 140006325293056, 140006325301247,
+STORE, 140006325301248, 140006325305343,
+STORE, 140736206716928, 140736206721023,
+STORE, 140736206704640, 140736206716927,
+STORE, 140006325264384, 140006325293055,
+STORE, 140006325256192, 140006325264383,
+STORE, 140006319255552, 140006323052543,
+SNULL, 140006319255552, 140006320914431,
+STORE, 140006320914432, 140006323052543,
+STORE, 140006319255552, 140006320914431,
+SNULL, 140006323011583, 140006323052543,
+STORE, 140006320914432, 140006323011583,
+STORE, 140006323011584, 140006323052543,
+SNULL, 140006323011584, 140006323036159,
+STORE, 140006323036160, 140006323052543,
+STORE, 140006323011584, 140006323036159,
+ERASE, 140006323011584, 140006323036159,
+STORE, 140006323011584, 140006323036159,
+ERASE, 140006323036160, 140006323052543,
+STORE, 140006323036160, 140006323052543,
+SNULL, 140006323027967, 140006323036159,
+STORE, 140006323011584, 140006323027967,
+STORE, 140006323027968, 140006323036159,
+SNULL, 94913212125183, 94913212129279,
+STORE, 94913212116992, 94913212125183,
+STORE, 94913212125184, 94913212129279,
+SNULL, 140006325297151, 140006325301247,
+STORE, 140006325293056, 140006325297151,
+STORE, 140006325297152, 140006325301247,
+ERASE, 140006325264384, 140006325293055,
+STORE, 94913239932928, 94913240068095,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140726926897152, 140737488351231,
+SNULL, 140726926905343, 140737488351231,
+STORE, 140726926897152, 140726926905343,
+STORE, 140726926766080, 140726926905343,
+STORE, 94213246820352, 94213249044479,
+SNULL, 94213246930943, 94213249044479,
+STORE, 94213246820352, 94213246930943,
+STORE, 94213246930944, 94213249044479,
+ERASE, 94213246930944, 94213249044479,
+STORE, 94213249024000, 94213249036287,
+STORE, 94213249036288, 94213249044479,
+STORE, 140368830242816, 140368832495615,
+SNULL, 140368830386175, 140368832495615,
+STORE, 140368830242816, 140368830386175,
+STORE, 140368830386176, 140368832495615,
+ERASE, 140368830386176, 140368832495615,
+STORE, 140368832483328, 140368832491519,
+STORE, 140368832491520, 140368832495615,
+STORE, 140726926999552, 140726927003647,
+STORE, 140726926987264, 140726926999551,
+STORE, 140368832454656, 140368832483327,
+STORE, 140368832446464, 140368832454655,
+STORE, 140368826445824, 140368830242815,
+SNULL, 140368826445824, 140368828104703,
+STORE, 140368828104704, 140368830242815,
+STORE, 140368826445824, 140368828104703,
+SNULL, 140368830201855, 140368830242815,
+STORE, 140368828104704, 140368830201855,
+STORE, 140368830201856, 140368830242815,
+SNULL, 140368830201856, 140368830226431,
+STORE, 140368830226432, 140368830242815,
+STORE, 140368830201856, 140368830226431,
+ERASE, 140368830201856, 140368830226431,
+STORE, 140368830201856, 140368830226431,
+ERASE, 140368830226432, 140368830242815,
+STORE, 140368830226432, 140368830242815,
+SNULL, 140368830218239, 140368830226431,
+STORE, 140368830201856, 140368830218239,
+STORE, 140368830218240, 140368830226431,
+SNULL, 94213249032191, 94213249036287,
+STORE, 94213249024000, 94213249032191,
+STORE, 94213249032192, 94213249036287,
+SNULL, 140368832487423, 140368832491519,
+STORE, 140368832483328, 140368832487423,
+STORE, 140368832487424, 140368832491519,
+ERASE, 140368832454656, 140368832483327,
+STORE, 94213267435520, 94213267570687,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728954130432, 140737488351231,
+SNULL, 140728954138623, 140737488351231,
+STORE, 140728954130432, 140728954138623,
+STORE, 140728953999360, 140728954138623,
+STORE, 94672570966016, 94672573190143,
+SNULL, 94672571076607, 94672573190143,
+STORE, 94672570966016, 94672571076607,
+STORE, 94672571076608, 94672573190143,
+ERASE, 94672571076608, 94672573190143,
+STORE, 94672573169664, 94672573181951,
+STORE, 94672573181952, 94672573190143,
+STORE, 140201696735232, 140201698988031,
+SNULL, 140201696878591, 140201698988031,
+STORE, 140201696735232, 140201696878591,
+STORE, 140201696878592, 140201698988031,
+ERASE, 140201696878592, 140201698988031,
+STORE, 140201698975744, 140201698983935,
+STORE, 140201698983936, 140201698988031,
+STORE, 140728954163200, 140728954167295,
+STORE, 140728954150912, 140728954163199,
+STORE, 140201698947072, 140201698975743,
+STORE, 140201698938880, 140201698947071,
+STORE, 140201692938240, 140201696735231,
+SNULL, 140201692938240, 140201694597119,
+STORE, 140201694597120, 140201696735231,
+STORE, 140201692938240, 140201694597119,
+SNULL, 140201696694271, 140201696735231,
+STORE, 140201694597120, 140201696694271,
+STORE, 140201696694272, 140201696735231,
+SNULL, 140201696694272, 140201696718847,
+STORE, 140201696718848, 140201696735231,
+STORE, 140201696694272, 140201696718847,
+ERASE, 140201696694272, 140201696718847,
+STORE, 140201696694272, 140201696718847,
+ERASE, 140201696718848, 140201696735231,
+STORE, 140201696718848, 140201696735231,
+SNULL, 140201696710655, 140201696718847,
+STORE, 140201696694272, 140201696710655,
+STORE, 140201696710656, 140201696718847,
+SNULL, 94672573177855, 94672573181951,
+STORE, 94672573169664, 94672573177855,
+STORE, 94672573177856, 94672573181951,
+SNULL, 140201698979839, 140201698983935,
+STORE, 140201698975744, 140201698979839,
+STORE, 140201698979840, 140201698983935,
+ERASE, 140201698947072, 140201698975743,
+STORE, 94672595689472, 94672595824639,
+STORE, 94114394132480, 94114394345471,
+STORE, 94114396442624, 94114396446719,
+STORE, 94114396446720, 94114396454911,
+STORE, 94114396454912, 94114396467199,
+STORE, 94114421575680, 94114428256255,
+STORE, 139934313955328, 139934315614207,
+STORE, 139934315614208, 139934317711359,
+STORE, 139934317711360, 139934317727743,
+STORE, 139934317727744, 139934317735935,
+STORE, 139934317735936, 139934317752319,
+STORE, 139934317752320, 139934317764607,
+STORE, 139934317764608, 139934319857663,
+STORE, 139934319857664, 139934319861759,
+STORE, 139934319861760, 139934319865855,
+STORE, 139934319865856, 139934320009215,
+STORE, 139934320377856, 139934322061311,
+STORE, 139934322061312, 139934322077695,
+STORE, 139934322106368, 139934322110463,
+STORE, 139934322110464, 139934322114559,
+STORE, 139934322114560, 139934322118655,
+STORE, 140731200376832, 140731200516095,
+STORE, 140731200929792, 140731200942079,
+STORE, 140731200942080, 140731200946175,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140721532362752, 140737488351231,
+SNULL, 140721532370943, 140737488351231,
+STORE, 140721532362752, 140721532370943,
+STORE, 140721532231680, 140721532370943,
+STORE, 94467222597632, 94467224821759,
+SNULL, 94467222708223, 94467224821759,
+STORE, 94467222597632, 94467222708223,
+STORE, 94467222708224, 94467224821759,
+ERASE, 94467222708224, 94467224821759,
+STORE, 94467224801280, 94467224813567,
+STORE, 94467224813568, 94467224821759,
+STORE, 140191433543680, 140191435796479,
+SNULL, 140191433687039, 140191435796479,
+STORE, 140191433543680, 140191433687039,
+STORE, 140191433687040, 140191435796479,
+ERASE, 140191433687040, 140191435796479,
+STORE, 140191435784192, 140191435792383,
+STORE, 140191435792384, 140191435796479,
+STORE, 140721533034496, 140721533038591,
+STORE, 140721533022208, 140721533034495,
+STORE, 140191435755520, 140191435784191,
+STORE, 140191435747328, 140191435755519,
+STORE, 140191429746688, 140191433543679,
+SNULL, 140191429746688, 140191431405567,
+STORE, 140191431405568, 140191433543679,
+STORE, 140191429746688, 140191431405567,
+SNULL, 140191433502719, 140191433543679,
+STORE, 140191431405568, 140191433502719,
+STORE, 140191433502720, 140191433543679,
+SNULL, 140191433502720, 140191433527295,
+STORE, 140191433527296, 140191433543679,
+STORE, 140191433502720, 140191433527295,
+ERASE, 140191433502720, 140191433527295,
+STORE, 140191433502720, 140191433527295,
+ERASE, 140191433527296, 140191433543679,
+STORE, 140191433527296, 140191433543679,
+SNULL, 140191433519103, 140191433527295,
+STORE, 140191433502720, 140191433519103,
+STORE, 140191433519104, 140191433527295,
+SNULL, 94467224809471, 94467224813567,
+STORE, 94467224801280, 94467224809471,
+STORE, 94467224809472, 94467224813567,
+SNULL, 140191435788287, 140191435792383,
+STORE, 140191435784192, 140191435788287,
+STORE, 140191435788288, 140191435792383,
+ERASE, 140191435755520, 140191435784191,
+STORE, 94467251847168, 94467251982335,
+STORE, 94367895400448, 94367895613439,
+STORE, 94367897710592, 94367897714687,
+STORE, 94367897714688, 94367897722879,
+STORE, 94367897722880, 94367897735167,
+STORE, 94367925264384, 94367926861823,
+STORE, 139801317548032, 139801319206911,
+STORE, 139801319206912, 139801321304063,
+STORE, 139801321304064, 139801321320447,
+STORE, 139801321320448, 139801321328639,
+STORE, 139801321328640, 139801321345023,
+STORE, 139801321345024, 139801321357311,
+STORE, 139801321357312, 139801323450367,
+STORE, 139801323450368, 139801323454463,
+STORE, 139801323454464, 139801323458559,
+STORE, 139801323458560, 139801323601919,
+STORE, 139801323970560, 139801325654015,
+STORE, 139801325654016, 139801325670399,
+STORE, 139801325699072, 139801325703167,
+STORE, 139801325703168, 139801325707263,
+STORE, 139801325707264, 139801325711359,
+STORE, 140724442861568, 140724443000831,
+STORE, 140724443611136, 140724443623423,
+STORE, 140724443623424, 140724443627519,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731353149440, 140737488351231,
+SNULL, 140731353157631, 140737488351231,
+STORE, 140731353149440, 140731353157631,
+STORE, 140731353018368, 140731353157631,
+STORE, 94310379503616, 94310381838335,
+SNULL, 94310379716607, 94310381838335,
+STORE, 94310379503616, 94310379716607,
+STORE, 94310379716608, 94310381838335,
+ERASE, 94310379716608, 94310381838335,
+STORE, 94310381813760, 94310381826047,
+STORE, 94310381826048, 94310381838335,
+STORE, 140515434659840, 140515436912639,
+SNULL, 140515434803199, 140515436912639,
+STORE, 140515434659840, 140515434803199,
+STORE, 140515434803200, 140515436912639,
+ERASE, 140515434803200, 140515436912639,
+STORE, 140515436900352, 140515436908543,
+STORE, 140515436908544, 140515436912639,
+STORE, 140731353886720, 140731353890815,
+STORE, 140731353874432, 140731353886719,
+STORE, 140515436871680, 140515436900351,
+STORE, 140515436863488, 140515436871679,
+STORE, 140515432546304, 140515434659839,
+SNULL, 140515432546304, 140515432558591,
+STORE, 140515432558592, 140515434659839,
+STORE, 140515432546304, 140515432558591,
+SNULL, 140515434651647, 140515434659839,
+STORE, 140515432558592, 140515434651647,
+STORE, 140515434651648, 140515434659839,
+ERASE, 140515434651648, 140515434659839,
+STORE, 140515434651648, 140515434659839,
+STORE, 140515428749312, 140515432546303,
+SNULL, 140515428749312, 140515430408191,
+STORE, 140515430408192, 140515432546303,
+STORE, 140515428749312, 140515430408191,
+SNULL, 140515432505343, 140515432546303,
+STORE, 140515430408192, 140515432505343,
+STORE, 140515432505344, 140515432546303,
+SNULL, 140515432505344, 140515432529919,
+STORE, 140515432529920, 140515432546303,
+STORE, 140515432505344, 140515432529919,
+ERASE, 140515432505344, 140515432529919,
+STORE, 140515432505344, 140515432529919,
+ERASE, 140515432529920, 140515432546303,
+STORE, 140515432529920, 140515432546303,
+STORE, 140515436855296, 140515436871679,
+SNULL, 140515432521727, 140515432529919,
+STORE, 140515432505344, 140515432521727,
+STORE, 140515432521728, 140515432529919,
+SNULL, 140515434655743, 140515434659839,
+STORE, 140515434651648, 140515434655743,
+STORE, 140515434655744, 140515434659839,
+SNULL, 94310381817855, 94310381826047,
+STORE, 94310381813760, 94310381817855,
+STORE, 94310381817856, 94310381826047,
+SNULL, 140515436904447, 140515436908543,
+STORE, 140515436900352, 140515436904447,
+STORE, 140515436904448, 140515436908543,
+ERASE, 140515436871680, 140515436900351,
+STORE, 94310395457536, 94310395592703,
+STORE, 140515435171840, 140515436855295,
+STORE, 94310395457536, 94310395727871,
+STORE, 94310395457536, 94310395863039,
+STORE, 94310395457536, 94310396047359,
+SNULL, 94310396022783, 94310396047359,
+STORE, 94310395457536, 94310396022783,
+STORE, 94310396022784, 94310396047359,
+ERASE, 94310396022784, 94310396047359,
+STORE, 94310395457536, 94310396157951,
+STORE, 94310395457536, 94310396293119,
+SNULL, 94310396276735, 94310396293119,
+STORE, 94310395457536, 94310396276735,
+STORE, 94310396276736, 94310396293119,
+ERASE, 94310396276736, 94310396293119,
+STORE, 94310395457536, 94310396411903,
+SNULL, 94310396383231, 94310396411903,
+STORE, 94310395457536, 94310396383231,
+STORE, 94310396383232, 94310396411903,
+ERASE, 94310396383232, 94310396411903,
+STORE, 94310395457536, 94310396522495,
+STORE, 94310395457536, 94310396674047,
+SNULL, 94310396657663, 94310396674047,
+STORE, 94310395457536, 94310396657663,
+STORE, 94310396657664, 94310396674047,
+ERASE, 94310396657664, 94310396674047,
+SNULL, 94310396624895, 94310396657663,
+STORE, 94310395457536, 94310396624895,
+STORE, 94310396624896, 94310396657663,
+ERASE, 94310396624896, 94310396657663,
+STORE, 94310395457536, 94310396776447,
+SNULL, 94310396764159, 94310396776447,
+STORE, 94310395457536, 94310396764159,
+STORE, 94310396764160, 94310396776447,
+ERASE, 94310396764160, 94310396776447,
+SNULL, 94310396739583, 94310396764159,
+STORE, 94310395457536, 94310396739583,
+STORE, 94310396739584, 94310396764159,
+ERASE, 94310396739584, 94310396764159,
+STORE, 94310395457536, 94310396882943,
+STORE, 94310395457536, 94310397018111,
+STORE, 94310395457536, 94310397161471,
+STORE, 94310395457536, 94310397300735,
+SNULL, 94310397292543, 94310397300735,
+STORE, 94310395457536, 94310397292543,
+STORE, 94310397292544, 94310397300735,
+ERASE, 94310397292544, 94310397300735,
+STORE, 94359222210560, 94359222423551,
+STORE, 94359224520704, 94359224524799,
+STORE, 94359224524800, 94359224532991,
+STORE, 94359224532992, 94359224545279,
+STORE, 94359238348800, 94359239385087,
+STORE, 140675699838976, 140675701497855,
+STORE, 140675701497856, 140675703595007,
+STORE, 140675703595008, 140675703611391,
+STORE, 140675703611392, 140675703619583,
+STORE, 140675703619584, 140675703635967,
+STORE, 140675703635968, 140675703648255,
+STORE, 140675703648256, 140675705741311,
+STORE, 140675705741312, 140675705745407,
+STORE, 140675705745408, 140675705749503,
+STORE, 140675705749504, 140675705892863,
+STORE, 140675706261504, 140675707944959,
+STORE, 140675707944960, 140675707961343,
+STORE, 140675707990016, 140675707994111,
+STORE, 140675707994112, 140675707998207,
+STORE, 140675707998208, 140675708002303,
+STORE, 140721324634112, 140721324773375,
+STORE, 140721324810240, 140721324822527,
+STORE, 140721324822528, 140721324826623,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724099678208, 140737488351231,
+SNULL, 140724099686399, 140737488351231,
+STORE, 140724099678208, 140724099686399,
+STORE, 140724099547136, 140724099686399,
+STORE, 94586638516224, 94586640850943,
+SNULL, 94586638729215, 94586640850943,
+STORE, 94586638516224, 94586638729215,
+STORE, 94586638729216, 94586640850943,
+ERASE, 94586638729216, 94586640850943,
+STORE, 94586640826368, 94586640838655,
+STORE, 94586640838656, 94586640850943,
+STORE, 140371033796608, 140371036049407,
+SNULL, 140371033939967, 140371036049407,
+STORE, 140371033796608, 140371033939967,
+STORE, 140371033939968, 140371036049407,
+ERASE, 140371033939968, 140371036049407,
+STORE, 140371036037120, 140371036045311,
+STORE, 140371036045312, 140371036049407,
+STORE, 140724100001792, 140724100005887,
+STORE, 140724099989504, 140724100001791,
+STORE, 140371036008448, 140371036037119,
+STORE, 140371036000256, 140371036008447,
+STORE, 140371031683072, 140371033796607,
+SNULL, 140371031683072, 140371031695359,
+STORE, 140371031695360, 140371033796607,
+STORE, 140371031683072, 140371031695359,
+SNULL, 140371033788415, 140371033796607,
+STORE, 140371031695360, 140371033788415,
+STORE, 140371033788416, 140371033796607,
+ERASE, 140371033788416, 140371033796607,
+STORE, 140371033788416, 140371033796607,
+STORE, 140371027886080, 140371031683071,
+SNULL, 140371027886080, 140371029544959,
+STORE, 140371029544960, 140371031683071,
+STORE, 140371027886080, 140371029544959,
+SNULL, 140371031642111, 140371031683071,
+STORE, 140371029544960, 140371031642111,
+STORE, 140371031642112, 140371031683071,
+SNULL, 140371031642112, 140371031666687,
+STORE, 140371031666688, 140371031683071,
+STORE, 140371031642112, 140371031666687,
+ERASE, 140371031642112, 140371031666687,
+STORE, 140371031642112, 140371031666687,
+ERASE, 140371031666688, 140371031683071,
+STORE, 140371031666688, 140371031683071,
+STORE, 140371035992064, 140371036008447,
+SNULL, 140371031658495, 140371031666687,
+STORE, 140371031642112, 140371031658495,
+STORE, 140371031658496, 140371031666687,
+SNULL, 140371033792511, 140371033796607,
+STORE, 140371033788416, 140371033792511,
+STORE, 140371033792512, 140371033796607,
+SNULL, 94586640830463, 94586640838655,
+STORE, 94586640826368, 94586640830463,
+STORE, 94586640830464, 94586640838655,
+SNULL, 140371036041215, 140371036045311,
+STORE, 140371036037120, 140371036041215,
+STORE, 140371036041216, 140371036045311,
+ERASE, 140371036008448, 140371036037119,
+STORE, 94586663849984, 94586663985151,
+STORE, 140371034308608, 140371035992063,
+STORE, 94586663849984, 94586664120319,
+STORE, 94586663849984, 94586664255487,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140727532937216, 140737488351231,
+SNULL, 140727532945407, 140737488351231,
+STORE, 140727532937216, 140727532945407,
+STORE, 140727532806144, 140727532945407,
+STORE, 94849780191232, 94849782525951,
+SNULL, 94849780404223, 94849782525951,
+STORE, 94849780191232, 94849780404223,
+STORE, 94849780404224, 94849782525951,
+ERASE, 94849780404224, 94849782525951,
+STORE, 94849782501376, 94849782513663,
+STORE, 94849782513664, 94849782525951,
+STORE, 140382070218752, 140382072471551,
+SNULL, 140382070362111, 140382072471551,
+STORE, 140382070218752, 140382070362111,
+STORE, 140382070362112, 140382072471551,
+ERASE, 140382070362112, 140382072471551,
+STORE, 140382072459264, 140382072467455,
+STORE, 140382072467456, 140382072471551,
+STORE, 140727533092864, 140727533096959,
+STORE, 140727533080576, 140727533092863,
+STORE, 140382072430592, 140382072459263,
+STORE, 140382072422400, 140382072430591,
+STORE, 140382068105216, 140382070218751,
+SNULL, 140382068105216, 140382068117503,
+STORE, 140382068117504, 140382070218751,
+STORE, 140382068105216, 140382068117503,
+SNULL, 140382070210559, 140382070218751,
+STORE, 140382068117504, 140382070210559,
+STORE, 140382070210560, 140382070218751,
+ERASE, 140382070210560, 140382070218751,
+STORE, 140382070210560, 140382070218751,
+STORE, 140382064308224, 140382068105215,
+SNULL, 140382064308224, 140382065967103,
+STORE, 140382065967104, 140382068105215,
+STORE, 140382064308224, 140382065967103,
+SNULL, 140382068064255, 140382068105215,
+STORE, 140382065967104, 140382068064255,
+STORE, 140382068064256, 140382068105215,
+SNULL, 140382068064256, 140382068088831,
+STORE, 140382068088832, 140382068105215,
+STORE, 140382068064256, 140382068088831,
+ERASE, 140382068064256, 140382068088831,
+STORE, 140382068064256, 140382068088831,
+ERASE, 140382068088832, 140382068105215,
+STORE, 140382068088832, 140382068105215,
+STORE, 140382072414208, 140382072430591,
+SNULL, 140382068080639, 140382068088831,
+STORE, 140382068064256, 140382068080639,
+STORE, 140382068080640, 140382068088831,
+SNULL, 140382070214655, 140382070218751,
+STORE, 140382070210560, 140382070214655,
+STORE, 140382070214656, 140382070218751,
+SNULL, 94849782505471, 94849782513663,
+STORE, 94849782501376, 94849782505471,
+STORE, 94849782505472, 94849782513663,
+SNULL, 140382072463359, 140382072467455,
+STORE, 140382072459264, 140382072463359,
+STORE, 140382072463360, 140382072467455,
+ERASE, 140382072430592, 140382072459263,
+STORE, 94849782845440, 94849782980607,
+STORE, 140382070730752, 140382072414207,
+STORE, 94849782845440, 94849783115775,
+STORE, 94849782845440, 94849783250943,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722594377728, 140737488351231,
+SNULL, 140722594385919, 140737488351231,
+STORE, 140722594377728, 140722594385919,
+STORE, 140722594246656, 140722594385919,
+STORE, 94421466353664, 94421468577791,
+SNULL, 94421466464255, 94421468577791,
+STORE, 94421466353664, 94421466464255,
+STORE, 94421466464256, 94421468577791,
+ERASE, 94421466464256, 94421468577791,
+STORE, 94421468557312, 94421468569599,
+STORE, 94421468569600, 94421468577791,
+STORE, 140345458057216, 140345460310015,
+SNULL, 140345458200575, 140345460310015,
+STORE, 140345458057216, 140345458200575,
+STORE, 140345458200576, 140345460310015,
+ERASE, 140345458200576, 140345460310015,
+STORE, 140345460297728, 140345460305919,
+STORE, 140345460305920, 140345460310015,
+STORE, 140722595557376, 140722595561471,
+STORE, 140722595545088, 140722595557375,
+STORE, 140345460269056, 140345460297727,
+STORE, 140345460260864, 140345460269055,
+STORE, 140345454260224, 140345458057215,
+SNULL, 140345454260224, 140345455919103,
+STORE, 140345455919104, 140345458057215,
+STORE, 140345454260224, 140345455919103,
+SNULL, 140345458016255, 140345458057215,
+STORE, 140345455919104, 140345458016255,
+STORE, 140345458016256, 140345458057215,
+SNULL, 140345458016256, 140345458040831,
+STORE, 140345458040832, 140345458057215,
+STORE, 140345458016256, 140345458040831,
+ERASE, 140345458016256, 140345458040831,
+STORE, 140345458016256, 140345458040831,
+ERASE, 140345458040832, 140345458057215,
+STORE, 140345458040832, 140345458057215,
+SNULL, 140345458032639, 140345458040831,
+STORE, 140345458016256, 140345458032639,
+STORE, 140345458032640, 140345458040831,
+SNULL, 94421468565503, 94421468569599,
+STORE, 94421468557312, 94421468565503,
+STORE, 94421468565504, 94421468569599,
+SNULL, 140345460301823, 140345460305919,
+STORE, 140345460297728, 140345460301823,
+STORE, 140345460301824, 140345460305919,
+ERASE, 140345460269056, 140345460297727,
+STORE, 94421496004608, 94421496139775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140726096302080, 140737488351231,
+SNULL, 140726096310271, 140737488351231,
+STORE, 140726096302080, 140726096310271,
+STORE, 140726096171008, 140726096310271,
+STORE, 94101992124416, 94101994459135,
+SNULL, 94101992337407, 94101994459135,
+STORE, 94101992124416, 94101992337407,
+STORE, 94101992337408, 94101994459135,
+ERASE, 94101992337408, 94101994459135,
+STORE, 94101994434560, 94101994446847,
+STORE, 94101994446848, 94101994459135,
+STORE, 140192085594112, 140192087846911,
+SNULL, 140192085737471, 140192087846911,
+STORE, 140192085594112, 140192085737471,
+STORE, 140192085737472, 140192087846911,
+ERASE, 140192085737472, 140192087846911,
+STORE, 140192087834624, 140192087842815,
+STORE, 140192087842816, 140192087846911,
+STORE, 140726096375808, 140726096379903,
+STORE, 140726096363520, 140726096375807,
+STORE, 140192087805952, 140192087834623,
+STORE, 140192087797760, 140192087805951,
+STORE, 140192083480576, 140192085594111,
+SNULL, 140192083480576, 140192083492863,
+STORE, 140192083492864, 140192085594111,
+STORE, 140192083480576, 140192083492863,
+SNULL, 140192085585919, 140192085594111,
+STORE, 140192083492864, 140192085585919,
+STORE, 140192085585920, 140192085594111,
+ERASE, 140192085585920, 140192085594111,
+STORE, 140192085585920, 140192085594111,
+STORE, 140192079683584, 140192083480575,
+SNULL, 140192079683584, 140192081342463,
+STORE, 140192081342464, 140192083480575,
+STORE, 140192079683584, 140192081342463,
+SNULL, 140192083439615, 140192083480575,
+STORE, 140192081342464, 140192083439615,
+STORE, 140192083439616, 140192083480575,
+SNULL, 140192083439616, 140192083464191,
+STORE, 140192083464192, 140192083480575,
+STORE, 140192083439616, 140192083464191,
+ERASE, 140192083439616, 140192083464191,
+STORE, 140192083439616, 140192083464191,
+ERASE, 140192083464192, 140192083480575,
+STORE, 140192083464192, 140192083480575,
+STORE, 140192087789568, 140192087805951,
+SNULL, 140192083455999, 140192083464191,
+STORE, 140192083439616, 140192083455999,
+STORE, 140192083456000, 140192083464191,
+SNULL, 140192085590015, 140192085594111,
+STORE, 140192085585920, 140192085590015,
+STORE, 140192085590016, 140192085594111,
+SNULL, 94101994438655, 94101994446847,
+STORE, 94101994434560, 94101994438655,
+STORE, 94101994438656, 94101994446847,
+SNULL, 140192087838719, 140192087842815,
+STORE, 140192087834624, 140192087838719,
+STORE, 140192087838720, 140192087842815,
+ERASE, 140192087805952, 140192087834623,
+STORE, 94102011887616, 94102012022783,
+STORE, 140192086106112, 140192087789567,
+STORE, 94102011887616, 94102012157951,
+STORE, 94102011887616, 94102012293119,
+STORE, 94102011887616, 94102012440575,
+SNULL, 94102012428287, 94102012440575,
+STORE, 94102011887616, 94102012428287,
+STORE, 94102012428288, 94102012440575,
+ERASE, 94102012428288, 94102012440575,
+STORE, 94102011887616, 94102012579839,
+STORE, 94102011887616, 94102012715007,
+SNULL, 94102012694527, 94102012715007,
+STORE, 94102011887616, 94102012694527,
+STORE, 94102012694528, 94102012715007,
+ERASE, 94102012694528, 94102012715007,
+STORE, 94102011887616, 94102012833791,
+STORE, 94102011887616, 94102012968959,
+SNULL, 94102012927999, 94102012968959,
+STORE, 94102011887616, 94102012927999,
+STORE, 94102012928000, 94102012968959,
+ERASE, 94102012928000, 94102012968959,
+STORE, 94102011887616, 94102013091839,
+SNULL, 94102013075455, 94102013091839,
+STORE, 94102011887616, 94102013075455,
+STORE, 94102013075456, 94102013091839,
+ERASE, 94102013075456, 94102013091839,
+STORE, 94102011887616, 94102013210623,
+STORE, 94102011887616, 94102013345791,
+STORE, 93968727965696, 93968728178687,
+STORE, 93968730275840, 93968730279935,
+STORE, 93968730279936, 93968730288127,
+STORE, 93968730288128, 93968730300415,
+STORE, 93968731140096, 93968732704767,
+STORE, 140588443168768, 140588444827647,
+STORE, 140588444827648, 140588446924799,
+STORE, 140588446924800, 140588446941183,
+STORE, 140588446941184, 140588446949375,
+STORE, 140588446949376, 140588446965759,
+STORE, 140588446965760, 140588446978047,
+STORE, 140588446978048, 140588449071103,
+STORE, 140588449071104, 140588449075199,
+STORE, 140588449075200, 140588449079295,
+STORE, 140588449079296, 140588449222655,
+STORE, 140588449591296, 140588451274751,
+STORE, 140588451274752, 140588451291135,
+STORE, 140588451319808, 140588451323903,
+STORE, 140588451323904, 140588451327999,
+STORE, 140588451328000, 140588451332095,
+STORE, 140733877239808, 140733877379071,
+STORE, 140733878702080, 140733878714367,
+STORE, 140733878714368, 140733878718463,
+STORE, 93968727965696, 93968728178687,
+STORE, 93968730275840, 93968730279935,
+STORE, 93968730279936, 93968730288127,
+STORE, 93968730288128, 93968730300415,
+STORE, 93968731140096, 93968732991487,
+STORE, 140588443168768, 140588444827647,
+STORE, 140588444827648, 140588446924799,
+STORE, 140588446924800, 140588446941183,
+STORE, 140588446941184, 140588446949375,
+STORE, 140588446949376, 140588446965759,
+STORE, 140588446965760, 140588446978047,
+STORE, 140588446978048, 140588449071103,
+STORE, 140588449071104, 140588449075199,
+STORE, 140588449075200, 140588449079295,
+STORE, 140588449079296, 140588449222655,
+STORE, 140588449591296, 140588451274751,
+STORE, 140588451274752, 140588451291135,
+STORE, 140588451319808, 140588451323903,
+STORE, 140588451323904, 140588451327999,
+STORE, 140588451328000, 140588451332095,
+STORE, 140733877239808, 140733877379071,
+STORE, 140733878702080, 140733878714367,
+STORE, 140733878714368, 140733878718463,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733054472192, 140737488351231,
+SNULL, 140733054480383, 140737488351231,
+STORE, 140733054472192, 140733054480383,
+STORE, 140733054341120, 140733054480383,
+STORE, 93992873623552, 93992875847679,
+SNULL, 93992873734143, 93992875847679,
+STORE, 93992873623552, 93992873734143,
+STORE, 93992873734144, 93992875847679,
+ERASE, 93992873734144, 93992875847679,
+STORE, 93992875827200, 93992875839487,
+STORE, 93992875839488, 93992875847679,
+STORE, 139790881488896, 139790883741695,
+SNULL, 139790881632255, 139790883741695,
+STORE, 139790881488896, 139790881632255,
+STORE, 139790881632256, 139790883741695,
+ERASE, 139790881632256, 139790883741695,
+STORE, 139790883729408, 139790883737599,
+STORE, 139790883737600, 139790883741695,
+STORE, 140733054754816, 140733054758911,
+STORE, 140733054742528, 140733054754815,
+STORE, 139790883700736, 139790883729407,
+STORE, 139790883692544, 139790883700735,
+STORE, 139790877691904, 139790881488895,
+SNULL, 139790877691904, 139790879350783,
+STORE, 139790879350784, 139790881488895,
+STORE, 139790877691904, 139790879350783,
+SNULL, 139790881447935, 139790881488895,
+STORE, 139790879350784, 139790881447935,
+STORE, 139790881447936, 139790881488895,
+SNULL, 139790881447936, 139790881472511,
+STORE, 139790881472512, 139790881488895,
+STORE, 139790881447936, 139790881472511,
+ERASE, 139790881447936, 139790881472511,
+STORE, 139790881447936, 139790881472511,
+ERASE, 139790881472512, 139790881488895,
+STORE, 139790881472512, 139790881488895,
+SNULL, 139790881464319, 139790881472511,
+STORE, 139790881447936, 139790881464319,
+STORE, 139790881464320, 139790881472511,
+SNULL, 93992875835391, 93992875839487,
+STORE, 93992875827200, 93992875835391,
+STORE, 93992875835392, 93992875839487,
+SNULL, 139790883733503, 139790883737599,
+STORE, 139790883729408, 139790883733503,
+STORE, 139790883733504, 139790883737599,
+ERASE, 139790883700736, 139790883729407,
+STORE, 93992877031424, 93992877166591,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728550887424, 140737488351231,
+SNULL, 140728550895615, 140737488351231,
+STORE, 140728550887424, 140728550895615,
+STORE, 140728550756352, 140728550895615,
+STORE, 94707634077696, 94707636301823,
+SNULL, 94707634188287, 94707636301823,
+STORE, 94707634077696, 94707634188287,
+STORE, 94707634188288, 94707636301823,
+ERASE, 94707634188288, 94707636301823,
+STORE, 94707636281344, 94707636293631,
+STORE, 94707636293632, 94707636301823,
+STORE, 140553545666560, 140553547919359,
+SNULL, 140553545809919, 140553547919359,
+STORE, 140553545666560, 140553545809919,
+STORE, 140553545809920, 140553547919359,
+ERASE, 140553545809920, 140553547919359,
+STORE, 140553547907072, 140553547915263,
+STORE, 140553547915264, 140553547919359,
+STORE, 140728552374272, 140728552378367,
+STORE, 140728552361984, 140728552374271,
+STORE, 140553547878400, 140553547907071,
+STORE, 140553547870208, 140553547878399,
+STORE, 140553541869568, 140553545666559,
+SNULL, 140553541869568, 140553543528447,
+STORE, 140553543528448, 140553545666559,
+STORE, 140553541869568, 140553543528447,
+SNULL, 140553545625599, 140553545666559,
+STORE, 140553543528448, 140553545625599,
+STORE, 140553545625600, 140553545666559,
+SNULL, 140553545625600, 140553545650175,
+STORE, 140553545650176, 140553545666559,
+STORE, 140553545625600, 140553545650175,
+ERASE, 140553545625600, 140553545650175,
+STORE, 140553545625600, 140553545650175,
+ERASE, 140553545650176, 140553545666559,
+STORE, 140553545650176, 140553545666559,
+SNULL, 140553545641983, 140553545650175,
+STORE, 140553545625600, 140553545641983,
+STORE, 140553545641984, 140553545650175,
+SNULL, 94707636289535, 94707636293631,
+STORE, 94707636281344, 94707636289535,
+STORE, 94707636289536, 94707636293631,
+SNULL, 140553547911167, 140553547915263,
+STORE, 140553547907072, 140553547911167,
+STORE, 140553547911168, 140553547915263,
+ERASE, 140553547878400, 140553547907071,
+STORE, 94707651411968, 94707651547135,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140732168695808, 140737488351231,
+SNULL, 140732168703999, 140737488351231,
+STORE, 140732168695808, 140732168703999,
+STORE, 140732168564736, 140732168703999,
+STORE, 94454287859712, 94454290083839,
+SNULL, 94454287970303, 94454290083839,
+STORE, 94454287859712, 94454287970303,
+STORE, 94454287970304, 94454290083839,
+ERASE, 94454287970304, 94454290083839,
+STORE, 94454290063360, 94454290075647,
+STORE, 94454290075648, 94454290083839,
+STORE, 140564947107840, 140564949360639,
+SNULL, 140564947251199, 140564949360639,
+STORE, 140564947107840, 140564947251199,
+STORE, 140564947251200, 140564949360639,
+ERASE, 140564947251200, 140564949360639,
+STORE, 140564949348352, 140564949356543,
+STORE, 140564949356544, 140564949360639,
+STORE, 140732168843264, 140732168847359,
+STORE, 140732168830976, 140732168843263,
+STORE, 140564949319680, 140564949348351,
+STORE, 140564949311488, 140564949319679,
+STORE, 140564943310848, 140564947107839,
+SNULL, 140564943310848, 140564944969727,
+STORE, 140564944969728, 140564947107839,
+STORE, 140564943310848, 140564944969727,
+SNULL, 140564947066879, 140564947107839,
+STORE, 140564944969728, 140564947066879,
+STORE, 140564947066880, 140564947107839,
+SNULL, 140564947066880, 140564947091455,
+STORE, 140564947091456, 140564947107839,
+STORE, 140564947066880, 140564947091455,
+ERASE, 140564947066880, 140564947091455,
+STORE, 140564947066880, 140564947091455,
+ERASE, 140564947091456, 140564947107839,
+STORE, 140564947091456, 140564947107839,
+SNULL, 140564947083263, 140564947091455,
+STORE, 140564947066880, 140564947083263,
+STORE, 140564947083264, 140564947091455,
+SNULL, 94454290071551, 94454290075647,
+STORE, 94454290063360, 94454290071551,
+STORE, 94454290071552, 94454290075647,
+SNULL, 140564949352447, 140564949356543,
+STORE, 140564949348352, 140564949352447,
+STORE, 140564949352448, 140564949356543,
+ERASE, 140564949319680, 140564949348351,
+STORE, 94454316236800, 94454316371967,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735155617792, 140737488351231,
+SNULL, 140735155625983, 140737488351231,
+STORE, 140735155617792, 140735155625983,
+STORE, 140735155486720, 140735155625983,
+STORE, 93915969556480, 93915971780607,
+SNULL, 93915969667071, 93915971780607,
+STORE, 93915969556480, 93915969667071,
+STORE, 93915969667072, 93915971780607,
+ERASE, 93915969667072, 93915971780607,
+STORE, 93915971760128, 93915971772415,
+STORE, 93915971772416, 93915971780607,
+STORE, 140141164605440, 140141166858239,
+SNULL, 140141164748799, 140141166858239,
+STORE, 140141164605440, 140141164748799,
+STORE, 140141164748800, 140141166858239,
+ERASE, 140141164748800, 140141166858239,
+STORE, 140141166845952, 140141166854143,
+STORE, 140141166854144, 140141166858239,
+STORE, 140735155691520, 140735155695615,
+STORE, 140735155679232, 140735155691519,
+STORE, 140141166817280, 140141166845951,
+STORE, 140141166809088, 140141166817279,
+STORE, 140141160808448, 140141164605439,
+SNULL, 140141160808448, 140141162467327,
+STORE, 140141162467328, 140141164605439,
+STORE, 140141160808448, 140141162467327,
+SNULL, 140141164564479, 140141164605439,
+STORE, 140141162467328, 140141164564479,
+STORE, 140141164564480, 140141164605439,
+SNULL, 140141164564480, 140141164589055,
+STORE, 140141164589056, 140141164605439,
+STORE, 140141164564480, 140141164589055,
+ERASE, 140141164564480, 140141164589055,
+STORE, 140141164564480, 140141164589055,
+ERASE, 140141164589056, 140141164605439,
+STORE, 140141164589056, 140141164605439,
+SNULL, 140141164580863, 140141164589055,
+STORE, 140141164564480, 140141164580863,
+STORE, 140141164580864, 140141164589055,
+SNULL, 93915971768319, 93915971772415,
+STORE, 93915971760128, 93915971768319,
+STORE, 93915971768320, 93915971772415,
+SNULL, 140141166850047, 140141166854143,
+STORE, 140141166845952, 140141166850047,
+STORE, 140141166850048, 140141166854143,
+ERASE, 140141166817280, 140141166845951,
+STORE, 93916002775040, 93916002910207,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728988409856, 140737488351231,
+SNULL, 140728988418047, 140737488351231,
+STORE, 140728988409856, 140728988418047,
+STORE, 140728988278784, 140728988418047,
+STORE, 94021634813952, 94021637038079,
+SNULL, 94021634924543, 94021637038079,
+STORE, 94021634813952, 94021634924543,
+STORE, 94021634924544, 94021637038079,
+ERASE, 94021634924544, 94021637038079,
+STORE, 94021637017600, 94021637029887,
+STORE, 94021637029888, 94021637038079,
+STORE, 140638014038016, 140638016290815,
+SNULL, 140638014181375, 140638016290815,
+STORE, 140638014038016, 140638014181375,
+STORE, 140638014181376, 140638016290815,
+ERASE, 140638014181376, 140638016290815,
+STORE, 140638016278528, 140638016286719,
+STORE, 140638016286720, 140638016290815,
+STORE, 140728988536832, 140728988540927,
+STORE, 140728988524544, 140728988536831,
+STORE, 140638016249856, 140638016278527,
+STORE, 140638016241664, 140638016249855,
+STORE, 140638010241024, 140638014038015,
+SNULL, 140638010241024, 140638011899903,
+STORE, 140638011899904, 140638014038015,
+STORE, 140638010241024, 140638011899903,
+SNULL, 140638013997055, 140638014038015,
+STORE, 140638011899904, 140638013997055,
+STORE, 140638013997056, 140638014038015,
+SNULL, 140638013997056, 140638014021631,
+STORE, 140638014021632, 140638014038015,
+STORE, 140638013997056, 140638014021631,
+ERASE, 140638013997056, 140638014021631,
+STORE, 140638013997056, 140638014021631,
+ERASE, 140638014021632, 140638014038015,
+STORE, 140638014021632, 140638014038015,
+SNULL, 140638014013439, 140638014021631,
+STORE, 140638013997056, 140638014013439,
+STORE, 140638014013440, 140638014021631,
+SNULL, 94021637025791, 94021637029887,
+STORE, 94021637017600, 94021637025791,
+STORE, 94021637025792, 94021637029887,
+SNULL, 140638016282623, 140638016286719,
+STORE, 140638016278528, 140638016282623,
+STORE, 140638016282624, 140638016286719,
+ERASE, 140638016249856, 140638016278527,
+STORE, 94021643124736, 94021643259903,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731219275776, 140737488351231,
+SNULL, 140731219283967, 140737488351231,
+STORE, 140731219275776, 140731219283967,
+STORE, 140731219144704, 140731219283967,
+STORE, 93888803647488, 93888805871615,
+SNULL, 93888803758079, 93888805871615,
+STORE, 93888803647488, 93888803758079,
+STORE, 93888803758080, 93888805871615,
+ERASE, 93888803758080, 93888805871615,
+STORE, 93888805851136, 93888805863423,
+STORE, 93888805863424, 93888805871615,
+STORE, 139630576934912, 139630579187711,
+SNULL, 139630577078271, 139630579187711,
+STORE, 139630576934912, 139630577078271,
+STORE, 139630577078272, 139630579187711,
+ERASE, 139630577078272, 139630579187711,
+STORE, 139630579175424, 139630579183615,
+STORE, 139630579183616, 139630579187711,
+STORE, 140731219718144, 140731219722239,
+STORE, 140731219705856, 140731219718143,
+STORE, 139630579146752, 139630579175423,
+STORE, 139630579138560, 139630579146751,
+STORE, 139630573137920, 139630576934911,
+SNULL, 139630573137920, 139630574796799,
+STORE, 139630574796800, 139630576934911,
+STORE, 139630573137920, 139630574796799,
+SNULL, 139630576893951, 139630576934911,
+STORE, 139630574796800, 139630576893951,
+STORE, 139630576893952, 139630576934911,
+SNULL, 139630576893952, 139630576918527,
+STORE, 139630576918528, 139630576934911,
+STORE, 139630576893952, 139630576918527,
+ERASE, 139630576893952, 139630576918527,
+STORE, 139630576893952, 139630576918527,
+ERASE, 139630576918528, 139630576934911,
+STORE, 139630576918528, 139630576934911,
+SNULL, 139630576910335, 139630576918527,
+STORE, 139630576893952, 139630576910335,
+STORE, 139630576910336, 139630576918527,
+SNULL, 93888805859327, 93888805863423,
+STORE, 93888805851136, 93888805859327,
+STORE, 93888805859328, 93888805863423,
+SNULL, 139630579179519, 139630579183615,
+STORE, 139630579175424, 139630579179519,
+STORE, 139630579179520, 139630579183615,
+ERASE, 139630579146752, 139630579175423,
+STORE, 93888822235136, 93888822370303,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733391151104, 140737488351231,
+SNULL, 140733391159295, 140737488351231,
+STORE, 140733391151104, 140733391159295,
+STORE, 140733391020032, 140733391159295,
+STORE, 94393875324928, 94393877549055,
+SNULL, 94393875435519, 94393877549055,
+STORE, 94393875324928, 94393875435519,
+STORE, 94393875435520, 94393877549055,
+ERASE, 94393875435520, 94393877549055,
+STORE, 94393877528576, 94393877540863,
+STORE, 94393877540864, 94393877549055,
+STORE, 140292111740928, 140292113993727,
+SNULL, 140292111884287, 140292113993727,
+STORE, 140292111740928, 140292111884287,
+STORE, 140292111884288, 140292113993727,
+ERASE, 140292111884288, 140292113993727,
+STORE, 140292113981440, 140292113989631,
+STORE, 140292113989632, 140292113993727,
+STORE, 140733391532032, 140733391536127,
+STORE, 140733391519744, 140733391532031,
+STORE, 140292113952768, 140292113981439,
+STORE, 140292113944576, 140292113952767,
+STORE, 140292107943936, 140292111740927,
+SNULL, 140292107943936, 140292109602815,
+STORE, 140292109602816, 140292111740927,
+STORE, 140292107943936, 140292109602815,
+SNULL, 140292111699967, 140292111740927,
+STORE, 140292109602816, 140292111699967,
+STORE, 140292111699968, 140292111740927,
+SNULL, 140292111699968, 140292111724543,
+STORE, 140292111724544, 140292111740927,
+STORE, 140292111699968, 140292111724543,
+ERASE, 140292111699968, 140292111724543,
+STORE, 140292111699968, 140292111724543,
+ERASE, 140292111724544, 140292111740927,
+STORE, 140292111724544, 140292111740927,
+SNULL, 140292111716351, 140292111724543,
+STORE, 140292111699968, 140292111716351,
+STORE, 140292111716352, 140292111724543,
+SNULL, 94393877536767, 94393877540863,
+STORE, 94393877528576, 94393877536767,
+STORE, 94393877536768, 94393877540863,
+SNULL, 140292113985535, 140292113989631,
+STORE, 140292113981440, 140292113985535,
+STORE, 140292113985536, 140292113989631,
+ERASE, 140292113952768, 140292113981439,
+STORE, 94393909342208, 94393909477375,
+STORE, 94458367512576, 94458367725567,
+STORE, 94458369822720, 94458369826815,
+STORE, 94458369826816, 94458369835007,
+STORE, 94458369835008, 94458369847295,
+STORE, 94458393292800, 94458399666175,
+STORE, 140619773841408, 140619775500287,
+STORE, 140619775500288, 140619777597439,
+STORE, 140619777597440, 140619777613823,
+STORE, 140619777613824, 140619777622015,
+STORE, 140619777622016, 140619777638399,
+STORE, 140619777638400, 140619777650687,
+STORE, 140619777650688, 140619779743743,
+STORE, 140619779743744, 140619779747839,
+STORE, 140619779747840, 140619779751935,
+STORE, 140619779751936, 140619779895295,
+STORE, 140619780263936, 140619781947391,
+STORE, 140619781947392, 140619781963775,
+STORE, 140619781992448, 140619781996543,
+STORE, 140619781996544, 140619782000639,
+STORE, 140619782000640, 140619782004735,
+STORE, 140725811675136, 140725811814399,
+STORE, 140725812813824, 140725812826111,
+STORE, 140725812826112, 140725812830207,
+STORE, 94458367512576, 94458367725567,
+STORE, 94458369822720, 94458369826815,
+STORE, 94458369826816, 94458369835007,
+STORE, 94458369835008, 94458369847295,
+STORE, 94458393292800, 94458400366591,
+STORE, 140619773841408, 140619775500287,
+STORE, 140619775500288, 140619777597439,
+STORE, 140619777597440, 140619777613823,
+STORE, 140619777613824, 140619777622015,
+STORE, 140619777622016, 140619777638399,
+STORE, 140619777638400, 140619777650687,
+STORE, 140619777650688, 140619779743743,
+STORE, 140619779743744, 140619779747839,
+STORE, 140619779747840, 140619779751935,
+STORE, 140619779751936, 140619779895295,
+STORE, 140619780263936, 140619781947391,
+STORE, 140619781947392, 140619781963775,
+STORE, 140619781992448, 140619781996543,
+STORE, 140619781996544, 140619782000639,
+STORE, 140619782000640, 140619782004735,
+STORE, 140725811675136, 140725811814399,
+STORE, 140725812813824, 140725812826111,
+STORE, 140725812826112, 140725812830207,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728740679680, 140737488351231,
+SNULL, 140728740687871, 140737488351231,
+STORE, 140728740679680, 140728740687871,
+STORE, 140728740548608, 140728740687871,
+STORE, 94764075249664, 94764077473791,
+SNULL, 94764075360255, 94764077473791,
+STORE, 94764075249664, 94764075360255,
+STORE, 94764075360256, 94764077473791,
+ERASE, 94764075360256, 94764077473791,
+STORE, 94764077453312, 94764077465599,
+STORE, 94764077465600, 94764077473791,
+STORE, 139766406791168, 139766409043967,
+SNULL, 139766406934527, 139766409043967,
+STORE, 139766406791168, 139766406934527,
+STORE, 139766406934528, 139766409043967,
+ERASE, 139766406934528, 139766409043967,
+STORE, 139766409031680, 139766409039871,
+STORE, 139766409039872, 139766409043967,
+STORE, 140728740913152, 140728740917247,
+STORE, 140728740900864, 140728740913151,
+STORE, 139766409003008, 139766409031679,
+STORE, 139766408994816, 139766409003007,
+STORE, 139766402994176, 139766406791167,
+SNULL, 139766402994176, 139766404653055,
+STORE, 139766404653056, 139766406791167,
+STORE, 139766402994176, 139766404653055,
+SNULL, 139766406750207, 139766406791167,
+STORE, 139766404653056, 139766406750207,
+STORE, 139766406750208, 139766406791167,
+SNULL, 139766406750208, 139766406774783,
+STORE, 139766406774784, 139766406791167,
+STORE, 139766406750208, 139766406774783,
+ERASE, 139766406750208, 139766406774783,
+STORE, 139766406750208, 139766406774783,
+ERASE, 139766406774784, 139766406791167,
+STORE, 139766406774784, 139766406791167,
+SNULL, 139766406766591, 139766406774783,
+STORE, 139766406750208, 139766406766591,
+STORE, 139766406766592, 139766406774783,
+SNULL, 94764077461503, 94764077465599,
+STORE, 94764077453312, 94764077461503,
+STORE, 94764077461504, 94764077465599,
+SNULL, 139766409035775, 139766409039871,
+STORE, 139766409031680, 139766409035775,
+STORE, 139766409035776, 139766409039871,
+ERASE, 139766409003008, 139766409031679,
+STORE, 94764090458112, 94764090593279,
+STORE, 94758057480192, 94758057590783,
+STORE, 94758059683840, 94758059692031,
+STORE, 94758059692032, 94758059696127,
+STORE, 94758059696128, 94758059704319,
+STORE, 94758083215360, 94758083350527,
+STORE, 139951456772096, 139951458430975,
+STORE, 139951458430976, 139951460528127,
+STORE, 139951460528128, 139951460544511,
+STORE, 139951460544512, 139951460552703,
+STORE, 139951460552704, 139951460569087,
+STORE, 139951460569088, 139951460712447,
+STORE, 139951462772736, 139951462780927,
+STORE, 139951462809600, 139951462813695,
+STORE, 139951462813696, 139951462817791,
+STORE, 139951462817792, 139951462821887,
+STORE, 140734098313216, 140734098452479,
+STORE, 140734098911232, 140734098923519,
+STORE, 140734098923520, 140734098927615,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724904095744, 140737488351231,
+SNULL, 140724904103935, 140737488351231,
+STORE, 140724904095744, 140724904103935,
+STORE, 140724903964672, 140724904103935,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 140408497864704, 140408500117503,
+SNULL, 140408498008063, 140408500117503,
+STORE, 140408497864704, 140408498008063,
+STORE, 140408498008064, 140408500117503,
+ERASE, 140408498008064, 140408500117503,
+STORE, 140408500105216, 140408500113407,
+STORE, 140408500113408, 140408500117503,
+STORE, 140724905369600, 140724905373695,
+STORE, 140724905357312, 140724905369599,
+STORE, 140408500076544, 140408500105215,
+STORE, 140408500068352, 140408500076543,
+STORE, 140408494702592, 140408497864703,
+SNULL, 140408494702592, 140408495763455,
+STORE, 140408495763456, 140408497864703,
+STORE, 140408494702592, 140408495763455,
+SNULL, 140408497856511, 140408497864703,
+STORE, 140408495763456, 140408497856511,
+STORE, 140408497856512, 140408497864703,
+ERASE, 140408497856512, 140408497864703,
+STORE, 140408497856512, 140408497864703,
+STORE, 140408490905600, 140408494702591,
+SNULL, 140408490905600, 140408492564479,
+STORE, 140408492564480, 140408494702591,
+STORE, 140408490905600, 140408492564479,
+SNULL, 140408494661631, 140408494702591,
+STORE, 140408492564480, 140408494661631,
+STORE, 140408494661632, 140408494702591,
+SNULL, 140408494661632, 140408494686207,
+STORE, 140408494686208, 140408494702591,
+STORE, 140408494661632, 140408494686207,
+ERASE, 140408494661632, 140408494686207,
+STORE, 140408494661632, 140408494686207,
+ERASE, 140408494686208, 140408494702591,
+STORE, 140408494686208, 140408494702591,
+STORE, 140408500056064, 140408500076543,
+SNULL, 140408494678015, 140408494686207,
+STORE, 140408494661632, 140408494678015,
+STORE, 140408494678016, 140408494686207,
+SNULL, 140408497860607, 140408497864703,
+STORE, 140408497856512, 140408497860607,
+STORE, 140408497860608, 140408497864703,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 140408500109311, 140408500113407,
+STORE, 140408500105216, 140408500109311,
+STORE, 140408500109312, 140408500113407,
+ERASE, 140408500076544, 140408500105215,
+STORE, 25235456, 25370623,
+STORE, 25235456, 25518079,
+STORE, 140408498372608, 140408500056063,
+STORE, 94543937388544, 94543937499135,
+STORE, 94543939592192, 94543939600383,
+STORE, 94543939600384, 94543939604479,
+STORE, 94543939604480, 94543939612671,
+STORE, 94543941447680, 94543941582847,
+STORE, 140282621947904, 140282623606783,
+STORE, 140282623606784, 140282625703935,
+STORE, 140282625703936, 140282625720319,
+STORE, 140282625720320, 140282625728511,
+STORE, 140282625728512, 140282625744895,
+STORE, 140282625744896, 140282625888255,
+STORE, 140282627948544, 140282627956735,
+STORE, 140282627985408, 140282627989503,
+STORE, 140282627989504, 140282627993599,
+STORE, 140282627993600, 140282627997695,
+STORE, 140728295723008, 140728295862271,
+STORE, 140728296476672, 140728296488959,
+STORE, 140728296488960, 140728296493055,
+STORE, 94431504838656, 94431505051647,
+STORE, 94431507148800, 94431507152895,
+STORE, 94431507152896, 94431507161087,
+STORE, 94431507161088, 94431507173375,
+STORE, 94431510286336, 94431510691839,
+STORE, 139818797948928, 139818799607807,
+STORE, 139818799607808, 139818801704959,
+STORE, 139818801704960, 139818801721343,
+STORE, 139818801721344, 139818801729535,
+STORE, 139818801729536, 139818801745919,
+STORE, 139818801745920, 139818801758207,
+STORE, 139818801758208, 139818803851263,
+STORE, 139818803851264, 139818803855359,
+STORE, 139818803855360, 139818803859455,
+STORE, 139818803859456, 139818804002815,
+STORE, 139818804371456, 139818806054911,
+STORE, 139818806054912, 139818806071295,
+STORE, 139818806099968, 139818806104063,
+STORE, 139818806104064, 139818806108159,
+STORE, 139818806108160, 139818806112255,
+STORE, 140731430457344, 140731430596607,
+STORE, 140731431227392, 140731431239679,
+STORE, 140731431239680, 140731431243775,
+STORE, 94431504838656, 94431505051647,
+STORE, 94431507148800, 94431507152895,
+STORE, 94431507152896, 94431507161087,
+STORE, 94431507161088, 94431507173375,
+STORE, 94431510286336, 94431510691839,
+STORE, 139818797948928, 139818799607807,
+STORE, 139818799607808, 139818801704959,
+STORE, 139818801704960, 139818801721343,
+STORE, 139818801721344, 139818801729535,
+STORE, 139818801729536, 139818801745919,
+STORE, 139818801745920, 139818801758207,
+STORE, 139818801758208, 139818803851263,
+STORE, 139818803851264, 139818803855359,
+STORE, 139818803855360, 139818803859455,
+STORE, 139818803859456, 139818804002815,
+STORE, 139818804371456, 139818806054911,
+STORE, 139818806054912, 139818806071295,
+STORE, 139818806099968, 139818806104063,
+STORE, 139818806104064, 139818806108159,
+STORE, 139818806108160, 139818806112255,
+STORE, 140731430457344, 140731430596607,
+STORE, 140731431227392, 140731431239679,
+STORE, 140731431239680, 140731431243775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140737488338944, 140737488351231,
+STORE, 140736944451584, 140737488351231,
+SNULL, 140736944463871, 140737488351231,
+STORE, 140736944451584, 140736944463871,
+STORE, 140736944320512, 140736944463871,
+STORE, 4194304, 26279935,
+STORE, 28372992, 28454911,
+STORE, 28454912, 29806591,
+STORE, 139693609893888, 139693612146687,
+SNULL, 139693610037247, 139693612146687,
+STORE, 139693609893888, 139693610037247,
+STORE, 139693610037248, 139693612146687,
+ERASE, 139693610037248, 139693612146687,
+STORE, 139693612134400, 139693612142591,
+STORE, 139693612142592, 139693612146687,
+STORE, 140736945152000, 140736945156095,
+STORE, 140736945139712, 140736945151999,
+STORE, 139693612105728, 139693612134399,
+STORE, 139693612097536, 139693612105727,
+STORE, 139693606060032, 139693609893887,
+SNULL, 139693606060032, 139693607768063,
+STORE, 139693607768064, 139693609893887,
+STORE, 139693606060032, 139693607768063,
+SNULL, 139693609861119, 139693609893887,
+STORE, 139693607768064, 139693609861119,
+STORE, 139693609861120, 139693609893887,
+ERASE, 139693609861120, 139693609893887,
+STORE, 139693609861120, 139693609893887,
+STORE, 139693603864576, 139693606060031,
+SNULL, 139693603864576, 139693603958783,
+STORE, 139693603958784, 139693606060031,
+STORE, 139693603864576, 139693603958783,
+SNULL, 139693606051839, 139693606060031,
+STORE, 139693603958784, 139693606051839,
+STORE, 139693606051840, 139693606060031,
+ERASE, 139693606051840, 139693606060031,
+STORE, 139693606051840, 139693606060031,
+STORE, 139693601345536, 139693603864575,
+SNULL, 139693601345536, 139693601759231,
+STORE, 139693601759232, 139693603864575,
+STORE, 139693601345536, 139693601759231,
+SNULL, 139693603852287, 139693603864575,
+STORE, 139693601759232, 139693603852287,
+STORE, 139693603852288, 139693603864575,
+ERASE, 139693603852288, 139693603864575,
+STORE, 139693603852288, 139693603864575,
+STORE, 139693598711808, 139693601345535,
+SNULL, 139693598711808, 139693599240191,
+STORE, 139693599240192, 139693601345535,
+STORE, 139693598711808, 139693599240191,
+SNULL, 139693601337343, 139693601345535,
+STORE, 139693599240192, 139693601337343,
+STORE, 139693601337344, 139693601345535,
+ERASE, 139693601337344, 139693601345535,
+STORE, 139693601337344, 139693601345535,
+STORE, 139693596598272, 139693598711807,
+SNULL, 139693596598272, 139693596610559,
+STORE, 139693596610560, 139693598711807,
+STORE, 139693596598272, 139693596610559,
+SNULL, 139693598703615, 139693598711807,
+STORE, 139693596610560, 139693598703615,
+STORE, 139693598703616, 139693598711807,
+ERASE, 139693598703616, 139693598711807,
+STORE, 139693598703616, 139693598711807,
+STORE, 139693594394624, 139693596598271,
+SNULL, 139693594394624, 139693594497023,
+STORE, 139693594497024, 139693596598271,
+STORE, 139693594394624, 139693594497023,
+SNULL, 139693596590079, 139693596598271,
+STORE, 139693594497024, 139693596590079,
+STORE, 139693596590080, 139693596598271,
+ERASE, 139693596590080, 139693596598271,
+STORE, 139693596590080, 139693596598271,
+STORE, 139693612089344, 139693612105727,
+STORE, 139693591232512, 139693594394623,
+SNULL, 139693591232512, 139693592293375,
+STORE, 139693592293376, 139693594394623,
+STORE, 139693591232512, 139693592293375,
+SNULL, 139693594386431, 139693594394623,
+STORE, 139693592293376, 139693594386431,
+STORE, 139693594386432, 139693594394623,
+ERASE, 139693594386432, 139693594394623,
+STORE, 139693594386432, 139693594394623,
+STORE, 139693587435520, 139693591232511,
+SNULL, 139693587435520, 139693589094399,
+STORE, 139693589094400, 139693591232511,
+STORE, 139693587435520, 139693589094399,
+SNULL, 139693591191551, 139693591232511,
+STORE, 139693589094400, 139693591191551,
+STORE, 139693591191552, 139693591232511,
+SNULL, 139693591191552, 139693591216127,
+STORE, 139693591216128, 139693591232511,
+STORE, 139693591191552, 139693591216127,
+ERASE, 139693591191552, 139693591216127,
+STORE, 139693591191552, 139693591216127,
+ERASE, 139693591216128, 139693591232511,
+STORE, 139693591216128, 139693591232511,
+STORE, 139693612077056, 139693612105727,
+SNULL, 139693591207935, 139693591216127,
+STORE, 139693591191552, 139693591207935,
+STORE, 139693591207936, 139693591216127,
+SNULL, 139693594390527, 139693594394623,
+STORE, 139693594386432, 139693594390527,
+STORE, 139693594390528, 139693594394623,
+SNULL, 139693596594175, 139693596598271,
+STORE, 139693596590080, 139693596594175,
+STORE, 139693596594176, 139693596598271,
+SNULL, 139693598707711, 139693598711807,
+STORE, 139693598703616, 139693598707711,
+STORE, 139693598707712, 139693598711807,
+SNULL, 139693601341439, 139693601345535,
+STORE, 139693601337344, 139693601341439,
+STORE, 139693601341440, 139693601345535,
+SNULL, 139693603860479, 139693603864575,
+STORE, 139693603852288, 139693603860479,
+STORE, 139693603860480, 139693603864575,
+SNULL, 139693606055935, 139693606060031,
+STORE, 139693606051840, 139693606055935,
+STORE, 139693606055936, 139693606060031,
+SNULL, 139693609865215, 139693609893887,
+STORE, 139693609861120, 139693609865215,
+STORE, 139693609865216, 139693609893887,
+SNULL, 28405759, 28454911,
+STORE, 28372992, 28405759,
+STORE, 28405760, 28454911,
+SNULL, 139693612138495, 139693612142591,
+STORE, 139693612134400, 139693612138495,
+STORE, 139693612138496, 139693612142591,
+ERASE, 139693612105728, 139693612134399,
+STORE, 39976960, 40112127,
+STORE, 139693610393600, 139693612077055,
+STORE, 139693612130304, 139693612134399,
+STORE, 139693610258432, 139693610393599,
+STORE, 39976960, 40255487,
+STORE, 139693585338368, 139693587435519,
+STORE, 139693612122112, 139693612134399,
+STORE, 139693612113920, 139693612134399,
+STORE, 139693612077056, 139693612113919,
+STORE, 139693610242048, 139693610393599,
+STORE, 39976960, 40390655,
+STORE, 39976960, 40546303,
+STORE, 139693610233856, 139693610393599,
+STORE, 139693610225664, 139693610393599,
+STORE, 39976960, 40714239,
+STORE, 139693610209280, 139693610393599,
+STORE, 39976960, 40861695,
+STORE, 94431504838656, 94431505051647,
+STORE, 94431507148800, 94431507152895,
+STORE, 94431507152896, 94431507161087,
+STORE, 94431507161088, 94431507173375,
+STORE, 94431510286336, 94431528759295,
+STORE, 139818797948928, 139818799607807,
+STORE, 139818799607808, 139818801704959,
+STORE, 139818801704960, 139818801721343,
+STORE, 139818801721344, 139818801729535,
+STORE, 139818801729536, 139818801745919,
+STORE, 139818801745920, 139818801758207,
+STORE, 139818801758208, 139818803851263,
+STORE, 139818803851264, 139818803855359,
+STORE, 139818803855360, 139818803859455,
+STORE, 139818803859456, 139818804002815,
+STORE, 139818804371456, 139818806054911,
+STORE, 139818806054912, 139818806071295,
+STORE, 139818806099968, 139818806104063,
+STORE, 139818806104064, 139818806108159,
+STORE, 139818806108160, 139818806112255,
+STORE, 140731430457344, 140731430596607,
+STORE, 140731431227392, 140731431239679,
+STORE, 140731431239680, 140731431243775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140729993904128, 140737488351231,
+SNULL, 140729993912319, 140737488351231,
+STORE, 140729993904128, 140729993912319,
+STORE, 140729993773056, 140729993912319,
+STORE, 93926271991808, 93926274215935,
+SNULL, 93926272102399, 93926274215935,
+STORE, 93926271991808, 93926272102399,
+STORE, 93926272102400, 93926274215935,
+ERASE, 93926272102400, 93926274215935,
+STORE, 93926274195456, 93926274207743,
+STORE, 93926274207744, 93926274215935,
+STORE, 139962167296000, 139962169548799,
+SNULL, 139962167439359, 139962169548799,
+STORE, 139962167296000, 139962167439359,
+STORE, 139962167439360, 139962169548799,
+ERASE, 139962167439360, 139962169548799,
+STORE, 139962169536512, 139962169544703,
+STORE, 139962169544704, 139962169548799,
+STORE, 140729995096064, 140729995100159,
+STORE, 140729995083776, 140729995096063,
+STORE, 139962169507840, 139962169536511,
+STORE, 139962169499648, 139962169507839,
+STORE, 139962163499008, 139962167295999,
+SNULL, 139962163499008, 139962165157887,
+STORE, 139962165157888, 139962167295999,
+STORE, 139962163499008, 139962165157887,
+SNULL, 139962167255039, 139962167295999,
+STORE, 139962165157888, 139962167255039,
+STORE, 139962167255040, 139962167295999,
+SNULL, 139962167255040, 139962167279615,
+STORE, 139962167279616, 139962167295999,
+STORE, 139962167255040, 139962167279615,
+ERASE, 139962167255040, 139962167279615,
+STORE, 139962167255040, 139962167279615,
+ERASE, 139962167279616, 139962167295999,
+STORE, 139962167279616, 139962167295999,
+SNULL, 139962167271423, 139962167279615,
+STORE, 139962167255040, 139962167271423,
+STORE, 139962167271424, 139962167279615,
+SNULL, 93926274203647, 93926274207743,
+STORE, 93926274195456, 93926274203647,
+STORE, 93926274203648, 93926274207743,
+SNULL, 139962169540607, 139962169544703,
+STORE, 139962169536512, 139962169540607,
+STORE, 139962169540608, 139962169544703,
+ERASE, 139962169507840, 139962169536511,
+STORE, 93926291120128, 93926291255295,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724960579584, 140737488351231,
+SNULL, 140724960587775, 140737488351231,
+STORE, 140724960579584, 140724960587775,
+STORE, 140724960448512, 140724960587775,
+STORE, 94246489489408, 94246491713535,
+SNULL, 94246489599999, 94246491713535,
+STORE, 94246489489408, 94246489599999,
+STORE, 94246489600000, 94246491713535,
+ERASE, 94246489600000, 94246491713535,
+STORE, 94246491693056, 94246491705343,
+STORE, 94246491705344, 94246491713535,
+STORE, 140098174926848, 140098177179647,
+SNULL, 140098175070207, 140098177179647,
+STORE, 140098174926848, 140098175070207,
+STORE, 140098175070208, 140098177179647,
+ERASE, 140098175070208, 140098177179647,
+STORE, 140098177167360, 140098177175551,
+STORE, 140098177175552, 140098177179647,
+STORE, 140724961439744, 140724961443839,
+STORE, 140724961427456, 140724961439743,
+STORE, 140098177138688, 140098177167359,
+STORE, 140098177130496, 140098177138687,
+STORE, 140098171129856, 140098174926847,
+SNULL, 140098171129856, 140098172788735,
+STORE, 140098172788736, 140098174926847,
+STORE, 140098171129856, 140098172788735,
+SNULL, 140098174885887, 140098174926847,
+STORE, 140098172788736, 140098174885887,
+STORE, 140098174885888, 140098174926847,
+SNULL, 140098174885888, 140098174910463,
+STORE, 140098174910464, 140098174926847,
+STORE, 140098174885888, 140098174910463,
+ERASE, 140098174885888, 140098174910463,
+STORE, 140098174885888, 140098174910463,
+ERASE, 140098174910464, 140098174926847,
+STORE, 140098174910464, 140098174926847,
+SNULL, 140098174902271, 140098174910463,
+STORE, 140098174885888, 140098174902271,
+STORE, 140098174902272, 140098174910463,
+SNULL, 94246491701247, 94246491705343,
+STORE, 94246491693056, 94246491701247,
+STORE, 94246491701248, 94246491705343,
+SNULL, 140098177171455, 140098177175551,
+STORE, 140098177167360, 140098177171455,
+STORE, 140098177171456, 140098177175551,
+ERASE, 140098177138688, 140098177167359,
+STORE, 94246516998144, 94246517133311,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140730522918912, 140737488351231,
+SNULL, 140730522927103, 140737488351231,
+STORE, 140730522918912, 140730522927103,
+STORE, 140730522787840, 140730522927103,
+STORE, 94196043120640, 94196045344767,
+SNULL, 94196043231231, 94196045344767,
+STORE, 94196043120640, 94196043231231,
+STORE, 94196043231232, 94196045344767,
+ERASE, 94196043231232, 94196045344767,
+STORE, 94196045324288, 94196045336575,
+STORE, 94196045336576, 94196045344767,
+STORE, 139815918940160, 139815921192959,
+SNULL, 139815919083519, 139815921192959,
+STORE, 139815918940160, 139815919083519,
+STORE, 139815919083520, 139815921192959,
+ERASE, 139815919083520, 139815921192959,
+STORE, 139815921180672, 139815921188863,
+STORE, 139815921188864, 139815921192959,
+STORE, 140730523344896, 140730523348991,
+STORE, 140730523332608, 140730523344895,
+STORE, 139815921152000, 139815921180671,
+STORE, 139815921143808, 139815921151999,
+STORE, 139815915143168, 139815918940159,
+SNULL, 139815915143168, 139815916802047,
+STORE, 139815916802048, 139815918940159,
+STORE, 139815915143168, 139815916802047,
+SNULL, 139815918899199, 139815918940159,
+STORE, 139815916802048, 139815918899199,
+STORE, 139815918899200, 139815918940159,
+SNULL, 139815918899200, 139815918923775,
+STORE, 139815918923776, 139815918940159,
+STORE, 139815918899200, 139815918923775,
+ERASE, 139815918899200, 139815918923775,
+STORE, 139815918899200, 139815918923775,
+ERASE, 139815918923776, 139815918940159,
+STORE, 139815918923776, 139815918940159,
+SNULL, 139815918915583, 139815918923775,
+STORE, 139815918899200, 139815918915583,
+STORE, 139815918915584, 139815918923775,
+SNULL, 94196045332479, 94196045336575,
+STORE, 94196045324288, 94196045332479,
+STORE, 94196045332480, 94196045336575,
+SNULL, 139815921184767, 139815921188863,
+STORE, 139815921180672, 139815921184767,
+STORE, 139815921184768, 139815921188863,
+ERASE, 139815921152000, 139815921180671,
+STORE, 94196076183552, 94196076318719,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722460393472, 140737488351231,
+SNULL, 140722460401663, 140737488351231,
+STORE, 140722460393472, 140722460401663,
+STORE, 140722460262400, 140722460401663,
+STORE, 94569810399232, 94569812623359,
+SNULL, 94569810509823, 94569812623359,
+STORE, 94569810399232, 94569810509823,
+STORE, 94569810509824, 94569812623359,
+ERASE, 94569810509824, 94569812623359,
+STORE, 94569812602880, 94569812615167,
+STORE, 94569812615168, 94569812623359,
+STORE, 139681565450240, 139681567703039,
+SNULL, 139681565593599, 139681567703039,
+STORE, 139681565450240, 139681565593599,
+STORE, 139681565593600, 139681567703039,
+ERASE, 139681565593600, 139681567703039,
+STORE, 139681567690752, 139681567698943,
+STORE, 139681567698944, 139681567703039,
+STORE, 140722460569600, 140722460573695,
+STORE, 140722460557312, 140722460569599,
+STORE, 139681567662080, 139681567690751,
+STORE, 139681567653888, 139681567662079,
+STORE, 139681561653248, 139681565450239,
+SNULL, 139681561653248, 139681563312127,
+STORE, 139681563312128, 139681565450239,
+STORE, 139681561653248, 139681563312127,
+SNULL, 139681565409279, 139681565450239,
+STORE, 139681563312128, 139681565409279,
+STORE, 139681565409280, 139681565450239,
+SNULL, 139681565409280, 139681565433855,
+STORE, 139681565433856, 139681565450239,
+STORE, 139681565409280, 139681565433855,
+ERASE, 139681565409280, 139681565433855,
+STORE, 139681565409280, 139681565433855,
+ERASE, 139681565433856, 139681565450239,
+STORE, 139681565433856, 139681565450239,
+SNULL, 139681565425663, 139681565433855,
+STORE, 139681565409280, 139681565425663,
+STORE, 139681565425664, 139681565433855,
+SNULL, 94569812611071, 94569812615167,
+STORE, 94569812602880, 94569812611071,
+STORE, 94569812611072, 94569812615167,
+SNULL, 139681567694847, 139681567698943,
+STORE, 139681567690752, 139681567694847,
+STORE, 139681567694848, 139681567698943,
+ERASE, 139681567662080, 139681567690751,
+STORE, 94569818066944, 94569818202111,
+STORE, 94431504838656, 94431505051647,
+STORE, 94431507148800, 94431507152895,
+STORE, 94431507152896, 94431507161087,
+STORE, 94431507161088, 94431507173375,
+STORE, 94431510286336, 94431534280703,
+STORE, 139818797948928, 139818799607807,
+STORE, 139818799607808, 139818801704959,
+STORE, 139818801704960, 139818801721343,
+STORE, 139818801721344, 139818801729535,
+STORE, 139818801729536, 139818801745919,
+STORE, 139818801745920, 139818801758207,
+STORE, 139818801758208, 139818803851263,
+STORE, 139818803851264, 139818803855359,
+STORE, 139818803855360, 139818803859455,
+STORE, 139818803859456, 139818804002815,
+STORE, 139818804371456, 139818806054911,
+STORE, 139818806054912, 139818806071295,
+STORE, 139818806099968, 139818806104063,
+STORE, 139818806104064, 139818806108159,
+STORE, 139818806108160, 139818806112255,
+STORE, 140731430457344, 140731430596607,
+STORE, 140731431227392, 140731431239679,
+STORE, 140731431239680, 140731431243775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140725452365824, 140737488351231,
+SNULL, 140725452374015, 140737488351231,
+STORE, 140725452365824, 140725452374015,
+STORE, 140725452234752, 140725452374015,
+STORE, 94395067465728, 94395069689855,
+SNULL, 94395067576319, 94395069689855,
+STORE, 94395067465728, 94395067576319,
+STORE, 94395067576320, 94395069689855,
+ERASE, 94395067576320, 94395069689855,
+STORE, 94395069669376, 94395069681663,
+STORE, 94395069681664, 94395069689855,
+STORE, 140269941211136, 140269943463935,
+SNULL, 140269941354495, 140269943463935,
+STORE, 140269941211136, 140269941354495,
+STORE, 140269941354496, 140269943463935,
+ERASE, 140269941354496, 140269943463935,
+STORE, 140269943451648, 140269943459839,
+STORE, 140269943459840, 140269943463935,
+STORE, 140725452558336, 140725452562431,
+STORE, 140725452546048, 140725452558335,
+STORE, 140269943422976, 140269943451647,
+STORE, 140269943414784, 140269943422975,
+STORE, 140269937414144, 140269941211135,
+SNULL, 140269937414144, 140269939073023,
+STORE, 140269939073024, 140269941211135,
+STORE, 140269937414144, 140269939073023,
+SNULL, 140269941170175, 140269941211135,
+STORE, 140269939073024, 140269941170175,
+STORE, 140269941170176, 140269941211135,
+SNULL, 140269941170176, 140269941194751,
+STORE, 140269941194752, 140269941211135,
+STORE, 140269941170176, 140269941194751,
+ERASE, 140269941170176, 140269941194751,
+STORE, 140269941170176, 140269941194751,
+ERASE, 140269941194752, 140269941211135,
+STORE, 140269941194752, 140269941211135,
+SNULL, 140269941186559, 140269941194751,
+STORE, 140269941170176, 140269941186559,
+STORE, 140269941186560, 140269941194751,
+SNULL, 94395069677567, 94395069681663,
+STORE, 94395069669376, 94395069677567,
+STORE, 94395069677568, 94395069681663,
+SNULL, 140269943455743, 140269943459839,
+STORE, 140269943451648, 140269943455743,
+STORE, 140269943455744, 140269943459839,
+ERASE, 140269943422976, 140269943451647,
+STORE, 94395101691904, 94395101827071,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733860118528, 140737488351231,
+SNULL, 140733860126719, 140737488351231,
+STORE, 140733860118528, 140733860126719,
+STORE, 140733859987456, 140733860126719,
+STORE, 94484752990208, 94484755214335,
+SNULL, 94484753100799, 94484755214335,
+STORE, 94484752990208, 94484753100799,
+STORE, 94484753100800, 94484755214335,
+ERASE, 94484753100800, 94484755214335,
+STORE, 94484755193856, 94484755206143,
+STORE, 94484755206144, 94484755214335,
+STORE, 139958922309632, 139958924562431,
+SNULL, 139958922452991, 139958924562431,
+STORE, 139958922309632, 139958922452991,
+STORE, 139958922452992, 139958924562431,
+ERASE, 139958922452992, 139958924562431,
+STORE, 139958924550144, 139958924558335,
+STORE, 139958924558336, 139958924562431,
+STORE, 140733860253696, 140733860257791,
+STORE, 140733860241408, 140733860253695,
+STORE, 139958924521472, 139958924550143,
+STORE, 139958924513280, 139958924521471,
+STORE, 139958918512640, 139958922309631,
+SNULL, 139958918512640, 139958920171519,
+STORE, 139958920171520, 139958922309631,
+STORE, 139958918512640, 139958920171519,
+SNULL, 139958922268671, 139958922309631,
+STORE, 139958920171520, 139958922268671,
+STORE, 139958922268672, 139958922309631,
+SNULL, 139958922268672, 139958922293247,
+STORE, 139958922293248, 139958922309631,
+STORE, 139958922268672, 139958922293247,
+ERASE, 139958922268672, 139958922293247,
+STORE, 139958922268672, 139958922293247,
+ERASE, 139958922293248, 139958922309631,
+STORE, 139958922293248, 139958922309631,
+SNULL, 139958922285055, 139958922293247,
+STORE, 139958922268672, 139958922285055,
+STORE, 139958922285056, 139958922293247,
+SNULL, 94484755202047, 94484755206143,
+STORE, 94484755193856, 94484755202047,
+STORE, 94484755202048, 94484755206143,
+SNULL, 139958924554239, 139958924558335,
+STORE, 139958924550144, 139958924554239,
+STORE, 139958924554240, 139958924558335,
+ERASE, 139958924521472, 139958924550143,
+STORE, 94484777615360, 94484777750527,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731051036672, 140737488351231,
+SNULL, 140731051044863, 140737488351231,
+STORE, 140731051036672, 140731051044863,
+STORE, 140731050905600, 140731051044863,
+STORE, 93945822998528, 93945825222655,
+SNULL, 93945823109119, 93945825222655,
+STORE, 93945822998528, 93945823109119,
+STORE, 93945823109120, 93945825222655,
+ERASE, 93945823109120, 93945825222655,
+STORE, 93945825202176, 93945825214463,
+STORE, 93945825214464, 93945825222655,
+STORE, 140153503997952, 140153506250751,
+SNULL, 140153504141311, 140153506250751,
+STORE, 140153503997952, 140153504141311,
+STORE, 140153504141312, 140153506250751,
+ERASE, 140153504141312, 140153506250751,
+STORE, 140153506238464, 140153506246655,
+STORE, 140153506246656, 140153506250751,
+STORE, 140731051331584, 140731051335679,
+STORE, 140731051319296, 140731051331583,
+STORE, 140153506209792, 140153506238463,
+STORE, 140153506201600, 140153506209791,
+STORE, 140153500200960, 140153503997951,
+SNULL, 140153500200960, 140153501859839,
+STORE, 140153501859840, 140153503997951,
+STORE, 140153500200960, 140153501859839,
+SNULL, 140153503956991, 140153503997951,
+STORE, 140153501859840, 140153503956991,
+STORE, 140153503956992, 140153503997951,
+SNULL, 140153503956992, 140153503981567,
+STORE, 140153503981568, 140153503997951,
+STORE, 140153503956992, 140153503981567,
+ERASE, 140153503956992, 140153503981567,
+STORE, 140153503956992, 140153503981567,
+ERASE, 140153503981568, 140153503997951,
+STORE, 140153503981568, 140153503997951,
+SNULL, 140153503973375, 140153503981567,
+STORE, 140153503956992, 140153503973375,
+STORE, 140153503973376, 140153503981567,
+SNULL, 93945825210367, 93945825214463,
+STORE, 93945825202176, 93945825210367,
+STORE, 93945825210368, 93945825214463,
+SNULL, 140153506242559, 140153506246655,
+STORE, 140153506238464, 140153506242559,
+STORE, 140153506242560, 140153506246655,
+ERASE, 140153506209792, 140153506238463,
+STORE, 93945854537728, 93945854672895,
+STORE, 94431504838656, 94431505051647,
+STORE, 94431507148800, 94431507152895,
+STORE, 94431507152896, 94431507161087,
+STORE, 94431507161088, 94431507173375,
+STORE, 94431510286336, 94431537885183,
+STORE, 139818797948928, 139818799607807,
+STORE, 139818799607808, 139818801704959,
+STORE, 139818801704960, 139818801721343,
+STORE, 139818801721344, 139818801729535,
+STORE, 139818801729536, 139818801745919,
+STORE, 139818801745920, 139818801758207,
+STORE, 139818801758208, 139818803851263,
+STORE, 139818803851264, 139818803855359,
+STORE, 139818803855360, 139818803859455,
+STORE, 139818803859456, 139818804002815,
+STORE, 139818804371456, 139818806054911,
+STORE, 139818806054912, 139818806071295,
+STORE, 139818806099968, 139818806104063,
+STORE, 139818806104064, 139818806108159,
+STORE, 139818806108160, 139818806112255,
+STORE, 140731430457344, 140731430596607,
+STORE, 140731431227392, 140731431239679,
+STORE, 140731431239680, 140731431243775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140736025325568, 140737488351231,
+SNULL, 140736025333759, 140737488351231,
+STORE, 140736025325568, 140736025333759,
+STORE, 140736025194496, 140736025333759,
+STORE, 94809095172096, 94809097396223,
+SNULL, 94809095282687, 94809097396223,
+STORE, 94809095172096, 94809095282687,
+STORE, 94809095282688, 94809097396223,
+ERASE, 94809095282688, 94809097396223,
+STORE, 94809097375744, 94809097388031,
+STORE, 94809097388032, 94809097396223,
+STORE, 140194992517120, 140194994769919,
+SNULL, 140194992660479, 140194994769919,
+STORE, 140194992517120, 140194992660479,
+STORE, 140194992660480, 140194994769919,
+ERASE, 140194992660480, 140194994769919,
+STORE, 140194994757632, 140194994765823,
+STORE, 140194994765824, 140194994769919,
+STORE, 140736026173440, 140736026177535,
+STORE, 140736026161152, 140736026173439,
+STORE, 140194994728960, 140194994757631,
+STORE, 140194994720768, 140194994728959,
+STORE, 140194988720128, 140194992517119,
+SNULL, 140194988720128, 140194990379007,
+STORE, 140194990379008, 140194992517119,
+STORE, 140194988720128, 140194990379007,
+SNULL, 140194992476159, 140194992517119,
+STORE, 140194990379008, 140194992476159,
+STORE, 140194992476160, 140194992517119,
+SNULL, 140194992476160, 140194992500735,
+STORE, 140194992500736, 140194992517119,
+STORE, 140194992476160, 140194992500735,
+ERASE, 140194992476160, 140194992500735,
+STORE, 140194992476160, 140194992500735,
+ERASE, 140194992500736, 140194992517119,
+STORE, 140194992500736, 140194992517119,
+SNULL, 140194992492543, 140194992500735,
+STORE, 140194992476160, 140194992492543,
+STORE, 140194992492544, 140194992500735,
+SNULL, 94809097383935, 94809097388031,
+STORE, 94809097375744, 94809097383935,
+STORE, 94809097383936, 94809097388031,
+SNULL, 140194994761727, 140194994765823,
+STORE, 140194994757632, 140194994761727,
+STORE, 140194994761728, 140194994765823,
+ERASE, 140194994728960, 140194994757631,
+STORE, 94809124286464, 94809124421631,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140726342660096, 140737488351231,
+SNULL, 140726342668287, 140737488351231,
+STORE, 140726342660096, 140726342668287,
+STORE, 140726342529024, 140726342668287,
+STORE, 94140331462656, 94140333686783,
+SNULL, 94140331573247, 94140333686783,
+STORE, 94140331462656, 94140331573247,
+STORE, 94140331573248, 94140333686783,
+ERASE, 94140331573248, 94140333686783,
+STORE, 94140333666304, 94140333678591,
+STORE, 94140333678592, 94140333686783,
+STORE, 140714077208576, 140714079461375,
+SNULL, 140714077351935, 140714079461375,
+STORE, 140714077208576, 140714077351935,
+STORE, 140714077351936, 140714079461375,
+ERASE, 140714077351936, 140714079461375,
+STORE, 140714079449088, 140714079457279,
+STORE, 140714079457280, 140714079461375,
+STORE, 140726343933952, 140726343938047,
+STORE, 140726343921664, 140726343933951,
+STORE, 140714079420416, 140714079449087,
+STORE, 140714079412224, 140714079420415,
+STORE, 140714073411584, 140714077208575,
+SNULL, 140714073411584, 140714075070463,
+STORE, 140714075070464, 140714077208575,
+STORE, 140714073411584, 140714075070463,
+SNULL, 140714077167615, 140714077208575,
+STORE, 140714075070464, 140714077167615,
+STORE, 140714077167616, 140714077208575,
+SNULL, 140714077167616, 140714077192191,
+STORE, 140714077192192, 140714077208575,
+STORE, 140714077167616, 140714077192191,
+ERASE, 140714077167616, 140714077192191,
+STORE, 140714077167616, 140714077192191,
+ERASE, 140714077192192, 140714077208575,
+STORE, 140714077192192, 140714077208575,
+SNULL, 140714077183999, 140714077192191,
+STORE, 140714077167616, 140714077183999,
+STORE, 140714077184000, 140714077192191,
+SNULL, 94140333674495, 94140333678591,
+STORE, 94140333666304, 94140333674495,
+STORE, 94140333674496, 94140333678591,
+SNULL, 140714079453183, 140714079457279,
+STORE, 140714079449088, 140714079453183,
+STORE, 140714079453184, 140714079457279,
+ERASE, 140714079420416, 140714079449087,
+STORE, 94140341432320, 94140341567487,
+STORE, 94431504838656, 94431505051647,
+STORE, 94431507148800, 94431507152895,
+STORE, 94431507152896, 94431507161087,
+STORE, 94431507161088, 94431507173375,
+STORE, 94431510286336, 94431539601407,
+STORE, 139818797948928, 139818799607807,
+STORE, 139818799607808, 139818801704959,
+STORE, 139818801704960, 139818801721343,
+STORE, 139818801721344, 139818801729535,
+STORE, 139818801729536, 139818801745919,
+STORE, 139818801745920, 139818801758207,
+STORE, 139818801758208, 139818803851263,
+STORE, 139818803851264, 139818803855359,
+STORE, 139818803855360, 139818803859455,
+STORE, 139818803859456, 139818804002815,
+STORE, 139818804371456, 139818806054911,
+STORE, 139818806054912, 139818806071295,
+STORE, 139818806099968, 139818806104063,
+STORE, 139818806104064, 139818806108159,
+STORE, 139818806108160, 139818806112255,
+STORE, 140731430457344, 140731430596607,
+STORE, 140731431227392, 140731431239679,
+STORE, 140731431239680, 140731431243775,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140725843607552, 140737488351231,
+SNULL, 140725843615743, 140737488351231,
+STORE, 140725843607552, 140725843615743,
+STORE, 140725843476480, 140725843615743,
+STORE, 94889043505152, 94889045839871,
+SNULL, 94889043718143, 94889045839871,
+STORE, 94889043505152, 94889043718143,
+STORE, 94889043718144, 94889045839871,
+ERASE, 94889043718144, 94889045839871,
+STORE, 94889045815296, 94889045827583,
+STORE, 94889045827584, 94889045839871,
+STORE, 140250965946368, 140250968199167,
+SNULL, 140250966089727, 140250968199167,
+STORE, 140250965946368, 140250966089727,
+STORE, 140250966089728, 140250968199167,
+ERASE, 140250966089728, 140250968199167,
+STORE, 140250968186880, 140250968195071,
+STORE, 140250968195072, 140250968199167,
+STORE, 140725844500480, 140725844504575,
+STORE, 140725844488192, 140725844500479,
+STORE, 140250968158208, 140250968186879,
+STORE, 140250968150016, 140250968158207,
+STORE, 140250963832832, 140250965946367,
+SNULL, 140250963832832, 140250963845119,
+STORE, 140250963845120, 140250965946367,
+STORE, 140250963832832, 140250963845119,
+SNULL, 140250965938175, 140250965946367,
+STORE, 140250963845120, 140250965938175,
+STORE, 140250965938176, 140250965946367,
+ERASE, 140250965938176, 140250965946367,
+STORE, 140250965938176, 140250965946367,
+STORE, 140250960035840, 140250963832831,
+SNULL, 140250960035840, 140250961694719,
+STORE, 140250961694720, 140250963832831,
+STORE, 140250960035840, 140250961694719,
+SNULL, 140250963791871, 140250963832831,
+STORE, 140250961694720, 140250963791871,
+STORE, 140250963791872, 140250963832831,
+SNULL, 140250963791872, 140250963816447,
+STORE, 140250963816448, 140250963832831,
+STORE, 140250963791872, 140250963816447,
+ERASE, 140250963791872, 140250963816447,
+STORE, 140250963791872, 140250963816447,
+ERASE, 140250963816448, 140250963832831,
+STORE, 140250963816448, 140250963832831,
+STORE, 140250968141824, 140250968158207,
+SNULL, 140250963808255, 140250963816447,
+STORE, 140250963791872, 140250963808255,
+STORE, 140250963808256, 140250963816447,
+SNULL, 140250965942271, 140250965946367,
+STORE, 140250965938176, 140250965942271,
+STORE, 140250965942272, 140250965946367,
+SNULL, 94889045819391, 94889045827583,
+STORE, 94889045815296, 94889045819391,
+STORE, 94889045819392, 94889045827583,
+SNULL, 140250968190975, 140250968195071,
+STORE, 140250968186880, 140250968190975,
+STORE, 140250968190976, 140250968195071,
+ERASE, 140250968158208, 140250968186879,
+STORE, 94889052213248, 94889052348415,
+STORE, 140250966458368, 140250968141823,
+STORE, 94889052213248, 94889052483583,
+STORE, 94889052213248, 94889052618751,
+STORE, 94170851819520, 94170852032511,
+STORE, 94170854129664, 94170854133759,
+STORE, 94170854133760, 94170854141951,
+STORE, 94170854141952, 94170854154239,
+STORE, 94170866515968, 94170867740671,
+STORE, 140062030422016, 140062032080895,
+STORE, 140062032080896, 140062034178047,
+STORE, 140062034178048, 140062034194431,
+STORE, 140062034194432, 140062034202623,
+STORE, 140062034202624, 140062034219007,
+STORE, 140062034219008, 140062034231295,
+STORE, 140062034231296, 140062036324351,
+STORE, 140062036324352, 140062036328447,
+STORE, 140062036328448, 140062036332543,
+STORE, 140062036332544, 140062036475903,
+STORE, 140062036844544, 140062038527999,
+STORE, 140062038528000, 140062038544383,
+STORE, 140062038573056, 140062038577151,
+STORE, 140062038577152, 140062038581247,
+STORE, 140062038581248, 140062038585343,
+STORE, 140736210550784, 140736210690047,
+STORE, 140736210759680, 140736210771967,
+STORE, 140736210771968, 140736210776063,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724272365568, 140737488351231,
+SNULL, 140724272373759, 140737488351231,
+STORE, 140724272365568, 140724272373759,
+STORE, 140724272234496, 140724272373759,
+STORE, 94607711965184, 94607714189311,
+SNULL, 94607712075775, 94607714189311,
+STORE, 94607711965184, 94607712075775,
+STORE, 94607712075776, 94607714189311,
+ERASE, 94607712075776, 94607714189311,
+STORE, 94607714168832, 94607714181119,
+STORE, 94607714181120, 94607714189311,
+STORE, 140054949253120, 140054951505919,
+SNULL, 140054949396479, 140054951505919,
+STORE, 140054949253120, 140054949396479,
+STORE, 140054949396480, 140054951505919,
+ERASE, 140054949396480, 140054951505919,
+STORE, 140054951493632, 140054951501823,
+STORE, 140054951501824, 140054951505919,
+STORE, 140724272992256, 140724272996351,
+STORE, 140724272979968, 140724272992255,
+STORE, 140054951464960, 140054951493631,
+STORE, 140054951456768, 140054951464959,
+STORE, 140054945456128, 140054949253119,
+SNULL, 140054945456128, 140054947115007,
+STORE, 140054947115008, 140054949253119,
+STORE, 140054945456128, 140054947115007,
+SNULL, 140054949212159, 140054949253119,
+STORE, 140054947115008, 140054949212159,
+STORE, 140054949212160, 140054949253119,
+SNULL, 140054949212160, 140054949236735,
+STORE, 140054949236736, 140054949253119,
+STORE, 140054949212160, 140054949236735,
+ERASE, 140054949212160, 140054949236735,
+STORE, 140054949212160, 140054949236735,
+ERASE, 140054949236736, 140054949253119,
+STORE, 140054949236736, 140054949253119,
+SNULL, 140054949228543, 140054949236735,
+STORE, 140054949212160, 140054949228543,
+STORE, 140054949228544, 140054949236735,
+SNULL, 94607714177023, 94607714181119,
+STORE, 94607714168832, 94607714177023,
+STORE, 94607714177024, 94607714181119,
+SNULL, 140054951497727, 140054951501823,
+STORE, 140054951493632, 140054951497727,
+STORE, 140054951497728, 140054951501823,
+ERASE, 140054951464960, 140054951493631,
+STORE, 94607733374976, 94607733510143,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733586923520, 140737488351231,
+SNULL, 140733586931711, 140737488351231,
+STORE, 140733586923520, 140733586931711,
+STORE, 140733586792448, 140733586931711,
+STORE, 93901634904064, 93901637128191,
+SNULL, 93901635014655, 93901637128191,
+STORE, 93901634904064, 93901635014655,
+STORE, 93901635014656, 93901637128191,
+ERASE, 93901635014656, 93901637128191,
+STORE, 93901637107712, 93901637119999,
+STORE, 93901637120000, 93901637128191,
+STORE, 140086104784896, 140086107037695,
+SNULL, 140086104928255, 140086107037695,
+STORE, 140086104784896, 140086104928255,
+STORE, 140086104928256, 140086107037695,
+ERASE, 140086104928256, 140086107037695,
+STORE, 140086107025408, 140086107033599,
+STORE, 140086107033600, 140086107037695,
+STORE, 140733587263488, 140733587267583,
+STORE, 140733587251200, 140733587263487,
+STORE, 140086106996736, 140086107025407,
+STORE, 140086106988544, 140086106996735,
+STORE, 140086100987904, 140086104784895,
+SNULL, 140086100987904, 140086102646783,
+STORE, 140086102646784, 140086104784895,
+STORE, 140086100987904, 140086102646783,
+SNULL, 140086104743935, 140086104784895,
+STORE, 140086102646784, 140086104743935,
+STORE, 140086104743936, 140086104784895,
+SNULL, 140086104743936, 140086104768511,
+STORE, 140086104768512, 140086104784895,
+STORE, 140086104743936, 140086104768511,
+ERASE, 140086104743936, 140086104768511,
+STORE, 140086104743936, 140086104768511,
+ERASE, 140086104768512, 140086104784895,
+STORE, 140086104768512, 140086104784895,
+SNULL, 140086104760319, 140086104768511,
+STORE, 140086104743936, 140086104760319,
+STORE, 140086104760320, 140086104768511,
+SNULL, 93901637115903, 93901637119999,
+STORE, 93901637107712, 93901637115903,
+STORE, 93901637115904, 93901637119999,
+SNULL, 140086107029503, 140086107033599,
+STORE, 140086107025408, 140086107029503,
+STORE, 140086107029504, 140086107033599,
+ERASE, 140086106996736, 140086107025407,
+STORE, 93901662715904, 93901662851071,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723365613568, 140737488351231,
+SNULL, 140723365621759, 140737488351231,
+STORE, 140723365613568, 140723365621759,
+STORE, 140723365482496, 140723365621759,
+STORE, 94759193546752, 94759195770879,
+SNULL, 94759193657343, 94759195770879,
+STORE, 94759193546752, 94759193657343,
+STORE, 94759193657344, 94759195770879,
+ERASE, 94759193657344, 94759195770879,
+STORE, 94759195750400, 94759195762687,
+STORE, 94759195762688, 94759195770879,
+STORE, 140607636246528, 140607638499327,
+SNULL, 140607636389887, 140607638499327,
+STORE, 140607636246528, 140607636389887,
+STORE, 140607636389888, 140607638499327,
+ERASE, 140607636389888, 140607638499327,
+STORE, 140607638487040, 140607638495231,
+STORE, 140607638495232, 140607638499327,
+STORE, 140723365900288, 140723365904383,
+STORE, 140723365888000, 140723365900287,
+STORE, 140607638458368, 140607638487039,
+STORE, 140607638450176, 140607638458367,
+STORE, 140607632449536, 140607636246527,
+SNULL, 140607632449536, 140607634108415,
+STORE, 140607634108416, 140607636246527,
+STORE, 140607632449536, 140607634108415,
+SNULL, 140607636205567, 140607636246527,
+STORE, 140607634108416, 140607636205567,
+STORE, 140607636205568, 140607636246527,
+SNULL, 140607636205568, 140607636230143,
+STORE, 140607636230144, 140607636246527,
+STORE, 140607636205568, 140607636230143,
+ERASE, 140607636205568, 140607636230143,
+STORE, 140607636205568, 140607636230143,
+ERASE, 140607636230144, 140607636246527,
+STORE, 140607636230144, 140607636246527,
+SNULL, 140607636221951, 140607636230143,
+STORE, 140607636205568, 140607636221951,
+STORE, 140607636221952, 140607636230143,
+SNULL, 94759195758591, 94759195762687,
+STORE, 94759195750400, 94759195758591,
+STORE, 94759195758592, 94759195762687,
+SNULL, 140607638491135, 140607638495231,
+STORE, 140607638487040, 140607638491135,
+STORE, 140607638491136, 140607638495231,
+ERASE, 140607638458368, 140607638487039,
+STORE, 94759204995072, 94759205130239,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140732503789568, 140737488351231,
+SNULL, 140732503797759, 140737488351231,
+STORE, 140732503789568, 140732503797759,
+STORE, 140732503658496, 140732503797759,
+STORE, 94077792956416, 94077795180543,
+SNULL, 94077793067007, 94077795180543,
+STORE, 94077792956416, 94077793067007,
+STORE, 94077793067008, 94077795180543,
+ERASE, 94077793067008, 94077795180543,
+STORE, 94077795160064, 94077795172351,
+STORE, 94077795172352, 94077795180543,
+STORE, 140359874252800, 140359876505599,
+SNULL, 140359874396159, 140359876505599,
+STORE, 140359874252800, 140359874396159,
+STORE, 140359874396160, 140359876505599,
+ERASE, 140359874396160, 140359876505599,
+STORE, 140359876493312, 140359876501503,
+STORE, 140359876501504, 140359876505599,
+STORE, 140732504465408, 140732504469503,
+STORE, 140732504453120, 140732504465407,
+STORE, 140359876464640, 140359876493311,
+STORE, 140359876456448, 140359876464639,
+STORE, 140359870455808, 140359874252799,
+SNULL, 140359870455808, 140359872114687,
+STORE, 140359872114688, 140359874252799,
+STORE, 140359870455808, 140359872114687,
+SNULL, 140359874211839, 140359874252799,
+STORE, 140359872114688, 140359874211839,
+STORE, 140359874211840, 140359874252799,
+SNULL, 140359874211840, 140359874236415,
+STORE, 140359874236416, 140359874252799,
+STORE, 140359874211840, 140359874236415,
+ERASE, 140359874211840, 140359874236415,
+STORE, 140359874211840, 140359874236415,
+ERASE, 140359874236416, 140359874252799,
+STORE, 140359874236416, 140359874252799,
+SNULL, 140359874228223, 140359874236415,
+STORE, 140359874211840, 140359874228223,
+STORE, 140359874228224, 140359874236415,
+SNULL, 94077795168255, 94077795172351,
+STORE, 94077795160064, 94077795168255,
+STORE, 94077795168256, 94077795172351,
+SNULL, 140359876497407, 140359876501503,
+STORE, 140359876493312, 140359876497407,
+STORE, 140359876497408, 140359876501503,
+ERASE, 140359876464640, 140359876493311,
+STORE, 94077808717824, 94077808852991,
+STORE, 94549486252032, 94549486465023,
+STORE, 94549488562176, 94549488566271,
+STORE, 94549488566272, 94549488574463,
+STORE, 94549488574464, 94549488586751,
+STORE, 94549503492096, 94549506121727,
+STORE, 140085800894464, 140085802553343,
+STORE, 140085802553344, 140085804650495,
+STORE, 140085804650496, 140085804666879,
+STORE, 140085804666880, 140085804675071,
+STORE, 140085804675072, 140085804691455,
+STORE, 140085804691456, 140085804703743,
+STORE, 140085804703744, 140085806796799,
+STORE, 140085806796800, 140085806800895,
+STORE, 140085806800896, 140085806804991,
+STORE, 140085806804992, 140085806948351,
+STORE, 140085807316992, 140085809000447,
+STORE, 140085809000448, 140085809016831,
+STORE, 140085809045504, 140085809049599,
+STORE, 140085809049600, 140085809053695,
+STORE, 140085809053696, 140085809057791,
+STORE, 140731810545664, 140731810684927,
+STORE, 140731810967552, 140731810979839,
+STORE, 140731810979840, 140731810983935,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724752330752, 140737488351231,
+SNULL, 140724752338943, 140737488351231,
+STORE, 140724752330752, 140724752338943,
+STORE, 140724752199680, 140724752338943,
+STORE, 94656357539840, 94656359874559,
+SNULL, 94656357752831, 94656359874559,
+STORE, 94656357539840, 94656357752831,
+STORE, 94656357752832, 94656359874559,
+ERASE, 94656357752832, 94656359874559,
+STORE, 94656359849984, 94656359862271,
+STORE, 94656359862272, 94656359874559,
+STORE, 139632585203712, 139632587456511,
+SNULL, 139632585347071, 139632587456511,
+STORE, 139632585203712, 139632585347071,
+STORE, 139632585347072, 139632587456511,
+ERASE, 139632585347072, 139632587456511,
+STORE, 139632587444224, 139632587452415,
+STORE, 139632587452416, 139632587456511,
+STORE, 139632587440128, 139632587444223,
+STORE, 139632587427840, 139632587440127,
+STORE, 139632587399168, 139632587427839,
+STORE, 139632587390976, 139632587399167,
+STORE, 139632583090176, 139632585203711,
+SNULL, 139632583090176, 139632583102463,
+STORE, 139632583102464, 139632585203711,
+STORE, 139632583090176, 139632583102463,
+SNULL, 139632585195519, 139632585203711,
+STORE, 139632583102464, 139632585195519,
+STORE, 139632585195520, 139632585203711,
+ERASE, 139632585195520, 139632585203711,
+STORE, 139632585195520, 139632585203711,
+STORE, 139632579293184, 139632583090175,
+SNULL, 139632579293184, 139632580952063,
+STORE, 139632580952064, 139632583090175,
+STORE, 139632579293184, 139632580952063,
+SNULL, 139632583049215, 139632583090175,
+STORE, 139632580952064, 139632583049215,
+STORE, 139632583049216, 139632583090175,
+SNULL, 139632583049216, 139632583073791,
+STORE, 139632583073792, 139632583090175,
+STORE, 139632583049216, 139632583073791,
+ERASE, 139632583049216, 139632583073791,
+STORE, 139632583049216, 139632583073791,
+ERASE, 139632583073792, 139632583090175,
+STORE, 139632583073792, 139632583090175,
+STORE, 139632587382784, 139632587399167,
+SNULL, 139632583065599, 139632583073791,
+STORE, 139632583049216, 139632583065599,
+STORE, 139632583065600, 139632583073791,
+SNULL, 139632585199615, 139632585203711,
+STORE, 139632585195520, 139632585199615,
+STORE, 139632585199616, 139632585203711,
+SNULL, 94656359854079, 94656359862271,
+STORE, 94656359849984, 94656359854079,
+STORE, 94656359854080, 94656359862271,
+SNULL, 139632587448319, 139632587452415,
+STORE, 139632587444224, 139632587448319,
+STORE, 139632587448320, 139632587452415,
+ERASE, 139632587399168, 139632587427839,
+STORE, 94656378912768, 94656379047935,
+STORE, 139632585699328, 139632587382783,
+STORE, 94656378912768, 94656379183103,
+STORE, 94656378912768, 94656379318271,
+STORE, 94656378912768, 94656379494399,
+SNULL, 94656379469823, 94656379494399,
+STORE, 94656378912768, 94656379469823,
+STORE, 94656379469824, 94656379494399,
+ERASE, 94656379469824, 94656379494399,
+STORE, 94656378912768, 94656379621375,
+STORE, 94656378912768, 94656379756543,
+STORE, 94656378912768, 94656379912191,
+STORE, 94656378912768, 94656380055551,
+STORE, 94656378912768, 94656380190719,
+STORE, 94656378912768, 94656380338175,
+SNULL, 94656380313599, 94656380338175,
+STORE, 94656378912768, 94656380313599,
+STORE, 94656380313600, 94656380338175,
+ERASE, 94656380313600, 94656380338175,
+STORE, 94656378912768, 94656380448767,
+SNULL, 94656380432383, 94656380448767,
+STORE, 94656378912768, 94656380432383,
+STORE, 94656380432384, 94656380448767,
+ERASE, 94656380432384, 94656380448767,
+STORE, 94656378912768, 94656380567551,
+STORE, 94656378912768, 94656380719103,
+STORE, 94656378912768, 94656380858367,
+STORE, 94656378912768, 94656380997631,
+STORE, 94656378912768, 94656381132799,
+SNULL, 94656381124607, 94656381132799,
+STORE, 94656378912768, 94656381124607,
+STORE, 94656381124608, 94656381132799,
+ERASE, 94656381124608, 94656381132799,
+STORE, 94656378912768, 94656381276159,
+STORE, 94656378912768, 94656381427711,
+STORE, 94604087611392, 94604087824383,
+STORE, 94604089921536, 94604089925631,
+STORE, 94604089925632, 94604089933823,
+STORE, 94604089933824, 94604089946111,
+STORE, 94604105125888, 94604106424319,
+STORE, 140454937694208, 140454939353087,
+STORE, 140454939353088, 140454941450239,
+STORE, 140454941450240, 140454941466623,
+STORE, 140454941466624, 140454941474815,
+STORE, 140454941474816, 140454941491199,
+STORE, 140454941491200, 140454941503487,
+STORE, 140454941503488, 140454943596543,
+STORE, 140454943596544, 140454943600639,
+STORE, 140454943600640, 140454943604735,
+STORE, 140454943604736, 140454943748095,
+STORE, 140454944116736, 140454945800191,
+STORE, 140454945800192, 140454945816575,
+STORE, 140454945845248, 140454945849343,
+STORE, 140454945849344, 140454945853439,
+STORE, 140454945853440, 140454945857535,
+STORE, 140728438214656, 140728438353919,
+STORE, 140728439095296, 140728439107583,
+STORE, 140728439107584, 140728439111679,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140727821099008, 140737488351231,
+SNULL, 140727821107199, 140737488351231,
+STORE, 140727821099008, 140727821107199,
+STORE, 140727820967936, 140727821107199,
+STORE, 94088457240576, 94088459575295,
+SNULL, 94088457453567, 94088459575295,
+STORE, 94088457240576, 94088457453567,
+STORE, 94088457453568, 94088459575295,
+ERASE, 94088457453568, 94088459575295,
+STORE, 94088459550720, 94088459563007,
+STORE, 94088459563008, 94088459575295,
+STORE, 140234378989568, 140234381242367,
+SNULL, 140234379132927, 140234381242367,
+STORE, 140234378989568, 140234379132927,
+STORE, 140234379132928, 140234381242367,
+ERASE, 140234379132928, 140234381242367,
+STORE, 140234381230080, 140234381238271,
+STORE, 140234381238272, 140234381242367,
+STORE, 140727822077952, 140727822082047,
+STORE, 140727822065664, 140727822077951,
+STORE, 140234381201408, 140234381230079,
+STORE, 140234381193216, 140234381201407,
+STORE, 140234376876032, 140234378989567,
+SNULL, 140234376876032, 140234376888319,
+STORE, 140234376888320, 140234378989567,
+STORE, 140234376876032, 140234376888319,
+SNULL, 140234378981375, 140234378989567,
+STORE, 140234376888320, 140234378981375,
+STORE, 140234378981376, 140234378989567,
+ERASE, 140234378981376, 140234378989567,
+STORE, 140234378981376, 140234378989567,
+STORE, 140234373079040, 140234376876031,
+SNULL, 140234373079040, 140234374737919,
+STORE, 140234374737920, 140234376876031,
+STORE, 140234373079040, 140234374737919,
+SNULL, 140234376835071, 140234376876031,
+STORE, 140234374737920, 140234376835071,
+STORE, 140234376835072, 140234376876031,
+SNULL, 140234376835072, 140234376859647,
+STORE, 140234376859648, 140234376876031,
+STORE, 140234376835072, 140234376859647,
+ERASE, 140234376835072, 140234376859647,
+STORE, 140234376835072, 140234376859647,
+ERASE, 140234376859648, 140234376876031,
+STORE, 140234376859648, 140234376876031,
+STORE, 140234381185024, 140234381201407,
+SNULL, 140234376851455, 140234376859647,
+STORE, 140234376835072, 140234376851455,
+STORE, 140234376851456, 140234376859647,
+SNULL, 140234378985471, 140234378989567,
+STORE, 140234378981376, 140234378985471,
+STORE, 140234378985472, 140234378989567,
+SNULL, 94088459554815, 94088459563007,
+STORE, 94088459550720, 94088459554815,
+STORE, 94088459554816, 94088459563007,
+SNULL, 140234381234175, 140234381238271,
+STORE, 140234381230080, 140234381234175,
+STORE, 140234381234176, 140234381238271,
+ERASE, 140234381201408, 140234381230079,
+STORE, 94088468852736, 94088468987903,
+STORE, 140234379501568, 140234381185023,
+STORE, 94088468852736, 94088469123071,
+STORE, 94088468852736, 94088469258239,
+STORE, 94110050402304, 94110050615295,
+STORE, 94110052712448, 94110052716543,
+STORE, 94110052716544, 94110052724735,
+STORE, 94110052724736, 94110052737023,
+STORE, 94110061875200, 94110062415871,
+STORE, 140139439357952, 140139441016831,
+STORE, 140139441016832, 140139443113983,
+STORE, 140139443113984, 140139443130367,
+STORE, 140139443130368, 140139443138559,
+STORE, 140139443138560, 140139443154943,
+STORE, 140139443154944, 140139443167231,
+STORE, 140139443167232, 140139445260287,
+STORE, 140139445260288, 140139445264383,
+STORE, 140139445264384, 140139445268479,
+STORE, 140139445268480, 140139445411839,
+STORE, 140139445780480, 140139447463935,
+STORE, 140139447463936, 140139447480319,
+STORE, 140139447508992, 140139447513087,
+STORE, 140139447513088, 140139447517183,
+STORE, 140139447517184, 140139447521279,
+STORE, 140731901427712, 140731901566975,
+STORE, 140731902259200, 140731902271487,
+STORE, 140731902271488, 140731902275583,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140727282622464, 140737488351231,
+SNULL, 140727282630655, 140737488351231,
+STORE, 140727282622464, 140727282630655,
+STORE, 140727282491392, 140727282630655,
+STORE, 94266649866240, 94266652200959,
+SNULL, 94266650079231, 94266652200959,
+STORE, 94266649866240, 94266650079231,
+STORE, 94266650079232, 94266652200959,
+ERASE, 94266650079232, 94266652200959,
+STORE, 94266652176384, 94266652188671,
+STORE, 94266652188672, 94266652200959,
+STORE, 139888497991680, 139888500244479,
+SNULL, 139888498135039, 139888500244479,
+STORE, 139888497991680, 139888498135039,
+STORE, 139888498135040, 139888500244479,
+ERASE, 139888498135040, 139888500244479,
+STORE, 139888500232192, 139888500240383,
+STORE, 139888500240384, 139888500244479,
+STORE, 140727283113984, 140727283118079,
+STORE, 140727283101696, 140727283113983,
+STORE, 139888500203520, 139888500232191,
+STORE, 139888500195328, 139888500203519,
+STORE, 139888495878144, 139888497991679,
+SNULL, 139888495878144, 139888495890431,
+STORE, 139888495890432, 139888497991679,
+STORE, 139888495878144, 139888495890431,
+SNULL, 139888497983487, 139888497991679,
+STORE, 139888495890432, 139888497983487,
+STORE, 139888497983488, 139888497991679,
+ERASE, 139888497983488, 139888497991679,
+STORE, 139888497983488, 139888497991679,
+STORE, 139888492081152, 139888495878143,
+SNULL, 139888492081152, 139888493740031,
+STORE, 139888493740032, 139888495878143,
+STORE, 139888492081152, 139888493740031,
+SNULL, 139888495837183, 139888495878143,
+STORE, 139888493740032, 139888495837183,
+STORE, 139888495837184, 139888495878143,
+SNULL, 139888495837184, 139888495861759,
+STORE, 139888495861760, 139888495878143,
+STORE, 139888495837184, 139888495861759,
+ERASE, 139888495837184, 139888495861759,
+STORE, 139888495837184, 139888495861759,
+ERASE, 139888495861760, 139888495878143,
+STORE, 139888495861760, 139888495878143,
+STORE, 139888500187136, 139888500203519,
+SNULL, 139888495853567, 139888495861759,
+STORE, 139888495837184, 139888495853567,
+STORE, 139888495853568, 139888495861759,
+SNULL, 139888497987583, 139888497991679,
+STORE, 139888497983488, 139888497987583,
+STORE, 139888497987584, 139888497991679,
+SNULL, 94266652180479, 94266652188671,
+STORE, 94266652176384, 94266652180479,
+STORE, 94266652180480, 94266652188671,
+SNULL, 139888500236287, 139888500240383,
+STORE, 139888500232192, 139888500236287,
+STORE, 139888500236288, 139888500240383,
+ERASE, 139888500203520, 139888500232191,
+STORE, 94266678542336, 94266678677503,
+STORE, 139888498503680, 139888500187135,
+STORE, 94266678542336, 94266678812671,
+STORE, 94266678542336, 94266678947839,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722507702272, 140737488351231,
+SNULL, 140722507710463, 140737488351231,
+STORE, 140722507702272, 140722507710463,
+STORE, 140722507571200, 140722507710463,
+STORE, 94313981394944, 94313983729663,
+SNULL, 94313981607935, 94313983729663,
+STORE, 94313981394944, 94313981607935,
+STORE, 94313981607936, 94313983729663,
+ERASE, 94313981607936, 94313983729663,
+STORE, 94313983705088, 94313983717375,
+STORE, 94313983717376, 94313983729663,
+STORE, 140456286076928, 140456288329727,
+SNULL, 140456286220287, 140456288329727,
+STORE, 140456286076928, 140456286220287,
+STORE, 140456286220288, 140456288329727,
+ERASE, 140456286220288, 140456288329727,
+STORE, 140456288317440, 140456288325631,
+STORE, 140456288325632, 140456288329727,
+STORE, 140722507997184, 140722508001279,
+STORE, 140722507984896, 140722507997183,
+STORE, 140456288288768, 140456288317439,
+STORE, 140456288280576, 140456288288767,
+STORE, 140456283963392, 140456286076927,
+SNULL, 140456283963392, 140456283975679,
+STORE, 140456283975680, 140456286076927,
+STORE, 140456283963392, 140456283975679,
+SNULL, 140456286068735, 140456286076927,
+STORE, 140456283975680, 140456286068735,
+STORE, 140456286068736, 140456286076927,
+ERASE, 140456286068736, 140456286076927,
+STORE, 140456286068736, 140456286076927,
+STORE, 140456280166400, 140456283963391,
+SNULL, 140456280166400, 140456281825279,
+STORE, 140456281825280, 140456283963391,
+STORE, 140456280166400, 140456281825279,
+SNULL, 140456283922431, 140456283963391,
+STORE, 140456281825280, 140456283922431,
+STORE, 140456283922432, 140456283963391,
+SNULL, 140456283922432, 140456283947007,
+STORE, 140456283947008, 140456283963391,
+STORE, 140456283922432, 140456283947007,
+ERASE, 140456283922432, 140456283947007,
+STORE, 140456283922432, 140456283947007,
+ERASE, 140456283947008, 140456283963391,
+STORE, 140456283947008, 140456283963391,
+STORE, 140456288272384, 140456288288767,
+SNULL, 140456283938815, 140456283947007,
+STORE, 140456283922432, 140456283938815,
+STORE, 140456283938816, 140456283947007,
+SNULL, 140456286072831, 140456286076927,
+STORE, 140456286068736, 140456286072831,
+STORE, 140456286072832, 140456286076927,
+SNULL, 94313983709183, 94313983717375,
+STORE, 94313983705088, 94313983709183,
+STORE, 94313983709184, 94313983717375,
+SNULL, 140456288321535, 140456288325631,
+STORE, 140456288317440, 140456288321535,
+STORE, 140456288321536, 140456288325631,
+ERASE, 140456288288768, 140456288317439,
+STORE, 94314006716416, 94314006851583,
+STORE, 140456286588928, 140456288272383,
+STORE, 94314006716416, 94314006986751,
+STORE, 94314006716416, 94314007121919,
+STORE, 93948644454400, 93948644667391,
+STORE, 93948646764544, 93948646768639,
+STORE, 93948646768640, 93948646776831,
+STORE, 93948646776832, 93948646789119,
+STORE, 93948664999936, 93948667142143,
+STORE, 140187350659072, 140187352317951,
+STORE, 140187352317952, 140187354415103,
+STORE, 140187354415104, 140187354431487,
+STORE, 140187354431488, 140187354439679,
+STORE, 140187354439680, 140187354456063,
+STORE, 140187354456064, 140187354468351,
+STORE, 140187354468352, 140187356561407,
+STORE, 140187356561408, 140187356565503,
+STORE, 140187356565504, 140187356569599,
+STORE, 140187356569600, 140187356712959,
+STORE, 140187357081600, 140187358765055,
+STORE, 140187358765056, 140187358781439,
+STORE, 140187358810112, 140187358814207,
+STORE, 140187358814208, 140187358818303,
+STORE, 140187358818304, 140187358822399,
+STORE, 140730484518912, 140730484658175,
+STORE, 140730485690368, 140730485702655,
+STORE, 140730485702656, 140730485706751,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140721211551744, 140737488351231,
+SNULL, 140721211559935, 140737488351231,
+STORE, 140721211551744, 140721211559935,
+STORE, 140721211420672, 140721211559935,
+STORE, 94105221423104, 94105223757823,
+SNULL, 94105221636095, 94105223757823,
+STORE, 94105221423104, 94105221636095,
+STORE, 94105221636096, 94105223757823,
+ERASE, 94105221636096, 94105223757823,
+STORE, 94105223733248, 94105223745535,
+STORE, 94105223745536, 94105223757823,
+STORE, 140474453676032, 140474455928831,
+SNULL, 140474453819391, 140474455928831,
+STORE, 140474453676032, 140474453819391,
+STORE, 140474453819392, 140474455928831,
+ERASE, 140474453819392, 140474455928831,
+STORE, 140474455916544, 140474455924735,
+STORE, 140474455924736, 140474455928831,
+STORE, 140721211703296, 140721211707391,
+STORE, 140721211691008, 140721211703295,
+STORE, 140474455887872, 140474455916543,
+STORE, 140474455879680, 140474455887871,
+STORE, 140474451562496, 140474453676031,
+SNULL, 140474451562496, 140474451574783,
+STORE, 140474451574784, 140474453676031,
+STORE, 140474451562496, 140474451574783,
+SNULL, 140474453667839, 140474453676031,
+STORE, 140474451574784, 140474453667839,
+STORE, 140474453667840, 140474453676031,
+ERASE, 140474453667840, 140474453676031,
+STORE, 140474453667840, 140474453676031,
+STORE, 140474447765504, 140474451562495,
+SNULL, 140474447765504, 140474449424383,
+STORE, 140474449424384, 140474451562495,
+STORE, 140474447765504, 140474449424383,
+SNULL, 140474451521535, 140474451562495,
+STORE, 140474449424384, 140474451521535,
+STORE, 140474451521536, 140474451562495,
+SNULL, 140474451521536, 140474451546111,
+STORE, 140474451546112, 140474451562495,
+STORE, 140474451521536, 140474451546111,
+ERASE, 140474451521536, 140474451546111,
+STORE, 140474451521536, 140474451546111,
+ERASE, 140474451546112, 140474451562495,
+STORE, 140474451546112, 140474451562495,
+STORE, 140474455871488, 140474455887871,
+SNULL, 140474451537919, 140474451546111,
+STORE, 140474451521536, 140474451537919,
+STORE, 140474451537920, 140474451546111,
+SNULL, 140474453671935, 140474453676031,
+STORE, 140474453667840, 140474453671935,
+STORE, 140474453671936, 140474453676031,
+SNULL, 94105223737343, 94105223745535,
+STORE, 94105223733248, 94105223737343,
+STORE, 94105223737344, 94105223745535,
+SNULL, 140474455920639, 140474455924735,
+STORE, 140474455916544, 140474455920639,
+STORE, 140474455920640, 140474455924735,
+ERASE, 140474455887872, 140474455916543,
+STORE, 94105238712320, 94105238847487,
+STORE, 140474454188032, 140474455871487,
+STORE, 94105238712320, 94105238982655,
+STORE, 94105238712320, 94105239117823,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140732356354048, 140737488351231,
+SNULL, 140732356362239, 140737488351231,
+STORE, 140732356354048, 140732356362239,
+STORE, 140732356222976, 140732356362239,
+STORE, 94461165989888, 94461168324607,
+SNULL, 94461166202879, 94461168324607,
+STORE, 94461165989888, 94461166202879,
+STORE, 94461166202880, 94461168324607,
+ERASE, 94461166202880, 94461168324607,
+STORE, 94461168300032, 94461168312319,
+STORE, 94461168312320, 94461168324607,
+STORE, 140317255110656, 140317257363455,
+SNULL, 140317255254015, 140317257363455,
+STORE, 140317255110656, 140317255254015,
+STORE, 140317255254016, 140317257363455,
+ERASE, 140317255254016, 140317257363455,
+STORE, 140317257351168, 140317257359359,
+STORE, 140317257359360, 140317257363455,
+STORE, 140732356583424, 140732356587519,
+STORE, 140732356571136, 140732356583423,
+STORE, 140317257322496, 140317257351167,
+STORE, 140317257314304, 140317257322495,
+STORE, 140317252997120, 140317255110655,
+SNULL, 140317252997120, 140317253009407,
+STORE, 140317253009408, 140317255110655,
+STORE, 140317252997120, 140317253009407,
+SNULL, 140317255102463, 140317255110655,
+STORE, 140317253009408, 140317255102463,
+STORE, 140317255102464, 140317255110655,
+ERASE, 140317255102464, 140317255110655,
+STORE, 140317255102464, 140317255110655,
+STORE, 140317249200128, 140317252997119,
+SNULL, 140317249200128, 140317250859007,
+STORE, 140317250859008, 140317252997119,
+STORE, 140317249200128, 140317250859007,
+SNULL, 140317252956159, 140317252997119,
+STORE, 140317250859008, 140317252956159,
+STORE, 140317252956160, 140317252997119,
+SNULL, 140317252956160, 140317252980735,
+STORE, 140317252980736, 140317252997119,
+STORE, 140317252956160, 140317252980735,
+ERASE, 140317252956160, 140317252980735,
+STORE, 140317252956160, 140317252980735,
+ERASE, 140317252980736, 140317252997119,
+STORE, 140317252980736, 140317252997119,
+STORE, 140317257306112, 140317257322495,
+SNULL, 140317252972543, 140317252980735,
+STORE, 140317252956160, 140317252972543,
+STORE, 140317252972544, 140317252980735,
+SNULL, 140317255106559, 140317255110655,
+STORE, 140317255102464, 140317255106559,
+STORE, 140317255106560, 140317255110655,
+SNULL, 94461168304127, 94461168312319,
+STORE, 94461168300032, 94461168304127,
+STORE, 94461168304128, 94461168312319,
+SNULL, 140317257355263, 140317257359359,
+STORE, 140317257351168, 140317257355263,
+STORE, 140317257355264, 140317257359359,
+ERASE, 140317257322496, 140317257351167,
+STORE, 94461195268096, 94461195403263,
+STORE, 140317255622656, 140317257306111,
+STORE, 94461195268096, 94461195538431,
+STORE, 94461195268096, 94461195673599,
+STORE, 94110050402304, 94110050615295,
+STORE, 94110052712448, 94110052716543,
+STORE, 94110052716544, 94110052724735,
+STORE, 94110052724736, 94110052737023,
+STORE, 94110061875200, 94110062415871,
+STORE, 140139439357952, 140139441016831,
+STORE, 140139441016832, 140139443113983,
+STORE, 140139443113984, 140139443130367,
+STORE, 140139443130368, 140139443138559,
+STORE, 140139443138560, 140139443154943,
+STORE, 140139443154944, 140139443167231,
+STORE, 140139443167232, 140139445260287,
+STORE, 140139445260288, 140139445264383,
+STORE, 140139445264384, 140139445268479,
+STORE, 140139445268480, 140139445411839,
+STORE, 140139445780480, 140139447463935,
+STORE, 140139447463936, 140139447480319,
+STORE, 140139447508992, 140139447513087,
+STORE, 140139447513088, 140139447517183,
+STORE, 140139447517184, 140139447521279,
+STORE, 140731901427712, 140731901566975,
+STORE, 140731902259200, 140731902271487,
+STORE, 140731902271488, 140731902275583,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140720941613056, 140737488351231,
+SNULL, 140720941621247, 140737488351231,
+STORE, 140720941613056, 140720941621247,
+STORE, 140720941481984, 140720941621247,
+STORE, 93902377721856, 93902379945983,
+SNULL, 93902377832447, 93902379945983,
+STORE, 93902377721856, 93902377832447,
+STORE, 93902377832448, 93902379945983,
+ERASE, 93902377832448, 93902379945983,
+STORE, 93902379925504, 93902379937791,
+STORE, 93902379937792, 93902379945983,
+STORE, 139836543635456, 139836545888255,
+SNULL, 139836543778815, 139836545888255,
+STORE, 139836543635456, 139836543778815,
+STORE, 139836543778816, 139836545888255,
+ERASE, 139836543778816, 139836545888255,
+STORE, 139836545875968, 139836545884159,
+STORE, 139836545884160, 139836545888255,
+STORE, 140720941711360, 140720941715455,
+STORE, 140720941699072, 140720941711359,
+STORE, 139836545847296, 139836545875967,
+STORE, 139836545839104, 139836545847295,
+STORE, 139836539838464, 139836543635455,
+SNULL, 139836539838464, 139836541497343,
+STORE, 139836541497344, 139836543635455,
+STORE, 139836539838464, 139836541497343,
+SNULL, 139836543594495, 139836543635455,
+STORE, 139836541497344, 139836543594495,
+STORE, 139836543594496, 139836543635455,
+SNULL, 139836543594496, 139836543619071,
+STORE, 139836543619072, 139836543635455,
+STORE, 139836543594496, 139836543619071,
+ERASE, 139836543594496, 139836543619071,
+STORE, 139836543594496, 139836543619071,
+ERASE, 139836543619072, 139836543635455,
+STORE, 139836543619072, 139836543635455,
+SNULL, 139836543610879, 139836543619071,
+STORE, 139836543594496, 139836543610879,
+STORE, 139836543610880, 139836543619071,
+SNULL, 93902379933695, 93902379937791,
+STORE, 93902379925504, 93902379933695,
+STORE, 93902379933696, 93902379937791,
+SNULL, 139836545880063, 139836545884159,
+STORE, 139836545875968, 139836545880063,
+STORE, 139836545880064, 139836545884159,
+ERASE, 139836545847296, 139836545875967,
+STORE, 93902396891136, 93902397026303,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140736538206208, 140737488351231,
+SNULL, 140736538214399, 140737488351231,
+STORE, 140736538206208, 140736538214399,
+STORE, 140736538075136, 140736538214399,
+STORE, 94173471399936, 94173473734655,
+SNULL, 94173471612927, 94173473734655,
+STORE, 94173471399936, 94173471612927,
+STORE, 94173471612928, 94173473734655,
+ERASE, 94173471612928, 94173473734655,
+STORE, 94173473710080, 94173473722367,
+STORE, 94173473722368, 94173473734655,
+STORE, 140035513556992, 140035515809791,
+SNULL, 140035513700351, 140035515809791,
+STORE, 140035513556992, 140035513700351,
+STORE, 140035513700352, 140035515809791,
+ERASE, 140035513700352, 140035515809791,
+STORE, 140035515797504, 140035515805695,
+STORE, 140035515805696, 140035515809791,
+STORE, 140736538329088, 140736538333183,
+STORE, 140736538316800, 140736538329087,
+STORE, 140035515768832, 140035515797503,
+STORE, 140035515760640, 140035515768831,
+STORE, 140035511443456, 140035513556991,
+SNULL, 140035511443456, 140035511455743,
+STORE, 140035511455744, 140035513556991,
+STORE, 140035511443456, 140035511455743,
+SNULL, 140035513548799, 140035513556991,
+STORE, 140035511455744, 140035513548799,
+STORE, 140035513548800, 140035513556991,
+ERASE, 140035513548800, 140035513556991,
+STORE, 140035513548800, 140035513556991,
+STORE, 140035507646464, 140035511443455,
+SNULL, 140035507646464, 140035509305343,
+STORE, 140035509305344, 140035511443455,
+STORE, 140035507646464, 140035509305343,
+SNULL, 140035511402495, 140035511443455,
+STORE, 140035509305344, 140035511402495,
+STORE, 140035511402496, 140035511443455,
+SNULL, 140035511402496, 140035511427071,
+STORE, 140035511427072, 140035511443455,
+STORE, 140035511402496, 140035511427071,
+ERASE, 140035511402496, 140035511427071,
+STORE, 140035511402496, 140035511427071,
+ERASE, 140035511427072, 140035511443455,
+STORE, 140035511427072, 140035511443455,
+STORE, 140035515752448, 140035515768831,
+SNULL, 140035511418879, 140035511427071,
+STORE, 140035511402496, 140035511418879,
+STORE, 140035511418880, 140035511427071,
+SNULL, 140035513552895, 140035513556991,
+STORE, 140035513548800, 140035513552895,
+STORE, 140035513552896, 140035513556991,
+SNULL, 94173473714175, 94173473722367,
+STORE, 94173473710080, 94173473714175,
+STORE, 94173473714176, 94173473722367,
+SNULL, 140035515801599, 140035515805695,
+STORE, 140035515797504, 140035515801599,
+STORE, 140035515801600, 140035515805695,
+ERASE, 140035515768832, 140035515797503,
+STORE, 94173478645760, 94173478780927,
+STORE, 140035514068992, 140035515752447,
+STORE, 94173478645760, 94173478916095,
+STORE, 94173478645760, 94173479051263,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140724216176640, 140737488351231,
+SNULL, 140724216184831, 140737488351231,
+STORE, 140724216176640, 140724216184831,
+STORE, 140724216045568, 140724216184831,
+STORE, 94870930628608, 94870932963327,
+SNULL, 94870930841599, 94870932963327,
+STORE, 94870930628608, 94870930841599,
+STORE, 94870930841600, 94870932963327,
+ERASE, 94870930841600, 94870932963327,
+STORE, 94870932938752, 94870932951039,
+STORE, 94870932951040, 94870932963327,
+STORE, 140453683736576, 140453685989375,
+SNULL, 140453683879935, 140453685989375,
+STORE, 140453683736576, 140453683879935,
+STORE, 140453683879936, 140453685989375,
+ERASE, 140453683879936, 140453685989375,
+STORE, 140453685977088, 140453685985279,
+STORE, 140453685985280, 140453685989375,
+STORE, 140724216832000, 140724216836095,
+STORE, 140724216819712, 140724216831999,
+STORE, 140453685948416, 140453685977087,
+STORE, 140453685940224, 140453685948415,
+STORE, 140453681623040, 140453683736575,
+SNULL, 140453681623040, 140453681635327,
+STORE, 140453681635328, 140453683736575,
+STORE, 140453681623040, 140453681635327,
+SNULL, 140453683728383, 140453683736575,
+STORE, 140453681635328, 140453683728383,
+STORE, 140453683728384, 140453683736575,
+ERASE, 140453683728384, 140453683736575,
+STORE, 140453683728384, 140453683736575,
+STORE, 140453677826048, 140453681623039,
+SNULL, 140453677826048, 140453679484927,
+STORE, 140453679484928, 140453681623039,
+STORE, 140453677826048, 140453679484927,
+SNULL, 140453681582079, 140453681623039,
+STORE, 140453679484928, 140453681582079,
+STORE, 140453681582080, 140453681623039,
+SNULL, 140453681582080, 140453681606655,
+STORE, 140453681606656, 140453681623039,
+STORE, 140453681582080, 140453681606655,
+ERASE, 140453681582080, 140453681606655,
+STORE, 140453681582080, 140453681606655,
+ERASE, 140453681606656, 140453681623039,
+STORE, 140453681606656, 140453681623039,
+STORE, 140453685932032, 140453685948415,
+SNULL, 140453681598463, 140453681606655,
+STORE, 140453681582080, 140453681598463,
+STORE, 140453681598464, 140453681606655,
+SNULL, 140453683732479, 140453683736575,
+STORE, 140453683728384, 140453683732479,
+STORE, 140453683732480, 140453683736575,
+SNULL, 94870932942847, 94870932951039,
+STORE, 94870932938752, 94870932942847,
+STORE, 94870932942848, 94870932951039,
+SNULL, 140453685981183, 140453685985279,
+STORE, 140453685977088, 140453685981183,
+STORE, 140453685981184, 140453685985279,
+ERASE, 140453685948416, 140453685977087,
+STORE, 94870940565504, 94870940700671,
+STORE, 140453684248576, 140453685932031,
+STORE, 94870940565504, 94870940835839,
+STORE, 94870940565504, 94870940971007,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731275661312, 140737488351231,
+SNULL, 140731275669503, 140737488351231,
+STORE, 140731275661312, 140731275669503,
+STORE, 140731275530240, 140731275669503,
+STORE, 94642788548608, 94642790883327,
+SNULL, 94642788761599, 94642790883327,
+STORE, 94642788548608, 94642788761599,
+STORE, 94642788761600, 94642790883327,
+ERASE, 94642788761600, 94642790883327,
+STORE, 94642790858752, 94642790871039,
+STORE, 94642790871040, 94642790883327,
+STORE, 140228458749952, 140228461002751,
+SNULL, 140228458893311, 140228461002751,
+STORE, 140228458749952, 140228458893311,
+STORE, 140228458893312, 140228461002751,
+ERASE, 140228458893312, 140228461002751,
+STORE, 140228460990464, 140228460998655,
+STORE, 140228460998656, 140228461002751,
+STORE, 140731276349440, 140731276353535,
+STORE, 140731276337152, 140731276349439,
+STORE, 140228460961792, 140228460990463,
+STORE, 140228460953600, 140228460961791,
+STORE, 140228456636416, 140228458749951,
+SNULL, 140228456636416, 140228456648703,
+STORE, 140228456648704, 140228458749951,
+STORE, 140228456636416, 140228456648703,
+SNULL, 140228458741759, 140228458749951,
+STORE, 140228456648704, 140228458741759,
+STORE, 140228458741760, 140228458749951,
+ERASE, 140228458741760, 140228458749951,
+STORE, 140228458741760, 140228458749951,
+STORE, 140228452839424, 140228456636415,
+SNULL, 140228452839424, 140228454498303,
+STORE, 140228454498304, 140228456636415,
+STORE, 140228452839424, 140228454498303,
+SNULL, 140228456595455, 140228456636415,
+STORE, 140228454498304, 140228456595455,
+STORE, 140228456595456, 140228456636415,
+SNULL, 140228456595456, 140228456620031,
+STORE, 140228456620032, 140228456636415,
+STORE, 140228456595456, 140228456620031,
+ERASE, 140228456595456, 140228456620031,
+STORE, 140228456595456, 140228456620031,
+ERASE, 140228456620032, 140228456636415,
+STORE, 140228456620032, 140228456636415,
+STORE, 140228460945408, 140228460961791,
+SNULL, 140228456611839, 140228456620031,
+STORE, 140228456595456, 140228456611839,
+STORE, 140228456611840, 140228456620031,
+SNULL, 140228458745855, 140228458749951,
+STORE, 140228458741760, 140228458745855,
+STORE, 140228458745856, 140228458749951,
+SNULL, 94642790862847, 94642790871039,
+STORE, 94642790858752, 94642790862847,
+STORE, 94642790862848, 94642790871039,
+SNULL, 140228460994559, 140228460998655,
+STORE, 140228460990464, 140228460994559,
+STORE, 140228460994560, 140228460998655,
+ERASE, 140228460961792, 140228460990463,
+STORE, 94642801549312, 94642801684479,
+STORE, 140228459261952, 140228460945407,
+STORE, 94642801549312, 94642801819647,
+STORE, 94642801549312, 94642801954815,
+STORE, 94604087611392, 94604087824383,
+STORE, 94604089921536, 94604089925631,
+STORE, 94604089925632, 94604089933823,
+STORE, 94604089933824, 94604089946111,
+STORE, 94604105125888, 94604106424319,
+STORE, 140454937694208, 140454939353087,
+STORE, 140454939353088, 140454941450239,
+STORE, 140454941450240, 140454941466623,
+STORE, 140454941466624, 140454941474815,
+STORE, 140454941474816, 140454941491199,
+STORE, 140454941491200, 140454941503487,
+STORE, 140454941503488, 140454943596543,
+STORE, 140454943596544, 140454943600639,
+STORE, 140454943600640, 140454943604735,
+STORE, 140454943604736, 140454943748095,
+STORE, 140454944116736, 140454945800191,
+STORE, 140454945800192, 140454945816575,
+STORE, 140454945845248, 140454945849343,
+STORE, 140454945849344, 140454945853439,
+STORE, 140454945853440, 140454945857535,
+STORE, 140728438214656, 140728438353919,
+STORE, 140728439095296, 140728439107583,
+STORE, 140728439107584, 140728439111679,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140721843453952, 140737488351231,
+SNULL, 140721843462143, 140737488351231,
+STORE, 140721843453952, 140721843462143,
+STORE, 140721843322880, 140721843462143,
+STORE, 94465962455040, 94465964789759,
+SNULL, 94465962668031, 94465964789759,
+STORE, 94465962455040, 94465962668031,
+STORE, 94465962668032, 94465964789759,
+ERASE, 94465962668032, 94465964789759,
+STORE, 94465964765184, 94465964777471,
+STORE, 94465964777472, 94465964789759,
+STORE, 139913488314368, 139913490567167,
+SNULL, 139913488457727, 139913490567167,
+STORE, 139913488314368, 139913488457727,
+STORE, 139913488457728, 139913490567167,
+ERASE, 139913488457728, 139913490567167,
+STORE, 139913490554880, 139913490563071,
+STORE, 139913490563072, 139913490567167,
+STORE, 140721843503104, 140721843507199,
+STORE, 140721843490816, 140721843503103,
+STORE, 139913490526208, 139913490554879,
+STORE, 139913490518016, 139913490526207,
+STORE, 139913486200832, 139913488314367,
+SNULL, 139913486200832, 139913486213119,
+STORE, 139913486213120, 139913488314367,
+STORE, 139913486200832, 139913486213119,
+SNULL, 139913488306175, 139913488314367,
+STORE, 139913486213120, 139913488306175,
+STORE, 139913488306176, 139913488314367,
+ERASE, 139913488306176, 139913488314367,
+STORE, 139913488306176, 139913488314367,
+STORE, 139913482403840, 139913486200831,
+SNULL, 139913482403840, 139913484062719,
+STORE, 139913484062720, 139913486200831,
+STORE, 139913482403840, 139913484062719,
+SNULL, 139913486159871, 139913486200831,
+STORE, 139913484062720, 139913486159871,
+STORE, 139913486159872, 139913486200831,
+SNULL, 139913486159872, 139913486184447,
+STORE, 139913486184448, 139913486200831,
+STORE, 139913486159872, 139913486184447,
+ERASE, 139913486159872, 139913486184447,
+STORE, 139913486159872, 139913486184447,
+ERASE, 139913486184448, 139913486200831,
+STORE, 139913486184448, 139913486200831,
+STORE, 139913490509824, 139913490526207,
+SNULL, 139913486176255, 139913486184447,
+STORE, 139913486159872, 139913486176255,
+STORE, 139913486176256, 139913486184447,
+SNULL, 139913488310271, 139913488314367,
+STORE, 139913488306176, 139913488310271,
+STORE, 139913488310272, 139913488314367,
+SNULL, 94465964769279, 94465964777471,
+STORE, 94465964765184, 94465964769279,
+STORE, 94465964769280, 94465964777471,
+SNULL, 139913490558975, 139913490563071,
+STORE, 139913490554880, 139913490558975,
+STORE, 139913490558976, 139913490563071,
+ERASE, 139913490526208, 139913490554879,
+STORE, 94465970024448, 94465970159615,
+STORE, 139913488826368, 139913490509823,
+STORE, 94465970024448, 94465970294783,
+STORE, 94465970024448, 94465970429951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140720583307264, 140737488351231,
+SNULL, 140720583315455, 140737488351231,
+STORE, 140720583307264, 140720583315455,
+STORE, 140720583176192, 140720583315455,
+STORE, 94212322082816, 94212324417535,
+SNULL, 94212322295807, 94212324417535,
+STORE, 94212322082816, 94212322295807,
+STORE, 94212322295808, 94212324417535,
+ERASE, 94212322295808, 94212324417535,
+STORE, 94212324392960, 94212324405247,
+STORE, 94212324405248, 94212324417535,
+STORE, 139659688538112, 139659690790911,
+SNULL, 139659688681471, 139659690790911,
+STORE, 139659688538112, 139659688681471,
+STORE, 139659688681472, 139659690790911,
+ERASE, 139659688681472, 139659690790911,
+STORE, 139659690778624, 139659690786815,
+STORE, 139659690786816, 139659690790911,
+STORE, 140720584781824, 140720584785919,
+STORE, 140720584769536, 140720584781823,
+STORE, 139659690749952, 139659690778623,
+STORE, 139659690741760, 139659690749951,
+STORE, 139659686424576, 139659688538111,
+SNULL, 139659686424576, 139659686436863,
+STORE, 139659686436864, 139659688538111,
+STORE, 139659686424576, 139659686436863,
+SNULL, 139659688529919, 139659688538111,
+STORE, 139659686436864, 139659688529919,
+STORE, 139659688529920, 139659688538111,
+ERASE, 139659688529920, 139659688538111,
+STORE, 139659688529920, 139659688538111,
+STORE, 139659682627584, 139659686424575,
+SNULL, 139659682627584, 139659684286463,
+STORE, 139659684286464, 139659686424575,
+STORE, 139659682627584, 139659684286463,
+SNULL, 139659686383615, 139659686424575,
+STORE, 139659684286464, 139659686383615,
+STORE, 139659686383616, 139659686424575,
+SNULL, 139659686383616, 139659686408191,
+STORE, 139659686408192, 139659686424575,
+STORE, 139659686383616, 139659686408191,
+ERASE, 139659686383616, 139659686408191,
+STORE, 139659686383616, 139659686408191,
+ERASE, 139659686408192, 139659686424575,
+STORE, 139659686408192, 139659686424575,
+STORE, 139659690733568, 139659690749951,
+SNULL, 139659686399999, 139659686408191,
+STORE, 139659686383616, 139659686399999,
+STORE, 139659686400000, 139659686408191,
+SNULL, 139659688534015, 139659688538111,
+STORE, 139659688529920, 139659688534015,
+STORE, 139659688534016, 139659688538111,
+SNULL, 94212324397055, 94212324405247,
+STORE, 94212324392960, 94212324397055,
+STORE, 94212324397056, 94212324405247,
+SNULL, 139659690782719, 139659690786815,
+STORE, 139659690778624, 139659690782719,
+STORE, 139659690782720, 139659690786815,
+ERASE, 139659690749952, 139659690778623,
+STORE, 94212355014656, 94212355149823,
+STORE, 139659689050112, 139659690733567,
+STORE, 94212355014656, 94212355284991,
+STORE, 94212355014656, 94212355420159,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140727689830400, 140737488351231,
+SNULL, 140727689838591, 140737488351231,
+STORE, 140727689830400, 140727689838591,
+STORE, 140727689699328, 140727689838591,
+STORE, 94572390281216, 94572392615935,
+SNULL, 94572390494207, 94572392615935,
+STORE, 94572390281216, 94572390494207,
+STORE, 94572390494208, 94572392615935,
+ERASE, 94572390494208, 94572392615935,
+STORE, 94572392591360, 94572392603647,
+STORE, 94572392603648, 94572392615935,
+STORE, 140575923769344, 140575926022143,
+SNULL, 140575923912703, 140575926022143,
+STORE, 140575923769344, 140575923912703,
+STORE, 140575923912704, 140575926022143,
+ERASE, 140575923912704, 140575926022143,
+STORE, 140575926009856, 140575926018047,
+STORE, 140575926018048, 140575926022143,
+STORE, 140727689871360, 140727689875455,
+STORE, 140727689859072, 140727689871359,
+STORE, 140575925981184, 140575926009855,
+STORE, 140575925972992, 140575925981183,
+STORE, 140575921655808, 140575923769343,
+SNULL, 140575921655808, 140575921668095,
+STORE, 140575921668096, 140575923769343,
+STORE, 140575921655808, 140575921668095,
+SNULL, 140575923761151, 140575923769343,
+STORE, 140575921668096, 140575923761151,
+STORE, 140575923761152, 140575923769343,
+ERASE, 140575923761152, 140575923769343,
+STORE, 140575923761152, 140575923769343,
+STORE, 140575917858816, 140575921655807,
+SNULL, 140575917858816, 140575919517695,
+STORE, 140575919517696, 140575921655807,
+STORE, 140575917858816, 140575919517695,
+SNULL, 140575921614847, 140575921655807,
+STORE, 140575919517696, 140575921614847,
+STORE, 140575921614848, 140575921655807,
+SNULL, 140575921614848, 140575921639423,
+STORE, 140575921639424, 140575921655807,
+STORE, 140575921614848, 140575921639423,
+ERASE, 140575921614848, 140575921639423,
+STORE, 140575921614848, 140575921639423,
+ERASE, 140575921639424, 140575921655807,
+STORE, 140575921639424, 140575921655807,
+STORE, 140575925964800, 140575925981183,
+SNULL, 140575921631231, 140575921639423,
+STORE, 140575921614848, 140575921631231,
+STORE, 140575921631232, 140575921639423,
+SNULL, 140575923765247, 140575923769343,
+STORE, 140575923761152, 140575923765247,
+STORE, 140575923765248, 140575923769343,
+SNULL, 94572392595455, 94572392603647,
+STORE, 94572392591360, 94572392595455,
+STORE, 94572392595456, 94572392603647,
+SNULL, 140575926013951, 140575926018047,
+STORE, 140575926009856, 140575926013951,
+STORE, 140575926013952, 140575926018047,
+ERASE, 140575925981184, 140575926009855,
+STORE, 94572402278400, 94572402413567,
+STORE, 140575924281344, 140575925964799,
+STORE, 94572402278400, 94572402548735,
+STORE, 94572402278400, 94572402683903,
+STORE, 94572402278400, 94572402851839,
+SNULL, 94572402827263, 94572402851839,
+STORE, 94572402278400, 94572402827263,
+STORE, 94572402827264, 94572402851839,
+ERASE, 94572402827264, 94572402851839,
+STORE, 94572402278400, 94572402966527,
+STORE, 94572402278400, 94572403109887,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140725520506880, 140737488351231,
+SNULL, 140725520515071, 140737488351231,
+STORE, 140725520506880, 140725520515071,
+STORE, 140725520375808, 140725520515071,
+STORE, 93829948788736, 93829951012863,
+SNULL, 93829948899327, 93829951012863,
+STORE, 93829948788736, 93829948899327,
+STORE, 93829948899328, 93829951012863,
+ERASE, 93829948899328, 93829951012863,
+STORE, 93829950992384, 93829951004671,
+STORE, 93829951004672, 93829951012863,
+STORE, 140133696794624, 140133699047423,
+SNULL, 140133696937983, 140133699047423,
+STORE, 140133696794624, 140133696937983,
+STORE, 140133696937984, 140133699047423,
+ERASE, 140133696937984, 140133699047423,
+STORE, 140133699035136, 140133699043327,
+STORE, 140133699043328, 140133699047423,
+STORE, 140725520875520, 140725520879615,
+STORE, 140725520863232, 140725520875519,
+STORE, 140133699006464, 140133699035135,
+STORE, 140133698998272, 140133699006463,
+STORE, 140133692997632, 140133696794623,
+SNULL, 140133692997632, 140133694656511,
+STORE, 140133694656512, 140133696794623,
+STORE, 140133692997632, 140133694656511,
+SNULL, 140133696753663, 140133696794623,
+STORE, 140133694656512, 140133696753663,
+STORE, 140133696753664, 140133696794623,
+SNULL, 140133696753664, 140133696778239,
+STORE, 140133696778240, 140133696794623,
+STORE, 140133696753664, 140133696778239,
+ERASE, 140133696753664, 140133696778239,
+STORE, 140133696753664, 140133696778239,
+ERASE, 140133696778240, 140133696794623,
+STORE, 140133696778240, 140133696794623,
+SNULL, 140133696770047, 140133696778239,
+STORE, 140133696753664, 140133696770047,
+STORE, 140133696770048, 140133696778239,
+SNULL, 93829951000575, 93829951004671,
+STORE, 93829950992384, 93829951000575,
+STORE, 93829951000576, 93829951004671,
+SNULL, 140133699039231, 140133699043327,
+STORE, 140133699035136, 140133699039231,
+STORE, 140133699039232, 140133699043327,
+ERASE, 140133699006464, 140133699035135,
+STORE, 93829978693632, 93829978828799,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140736118022144, 140737488351231,
+SNULL, 140736118030335, 140737488351231,
+STORE, 140736118022144, 140736118030335,
+STORE, 140736117891072, 140736118030335,
+STORE, 94467663982592, 94467666206719,
+SNULL, 94467664093183, 94467666206719,
+STORE, 94467663982592, 94467664093183,
+STORE, 94467664093184, 94467666206719,
+ERASE, 94467664093184, 94467666206719,
+STORE, 94467666186240, 94467666198527,
+STORE, 94467666198528, 94467666206719,
+STORE, 140525377327104, 140525379579903,
+SNULL, 140525377470463, 140525379579903,
+STORE, 140525377327104, 140525377470463,
+STORE, 140525377470464, 140525379579903,
+ERASE, 140525377470464, 140525379579903,
+STORE, 140525379567616, 140525379575807,
+STORE, 140525379575808, 140525379579903,
+STORE, 140736118771712, 140736118775807,
+STORE, 140736118759424, 140736118771711,
+STORE, 140525379538944, 140525379567615,
+STORE, 140525379530752, 140525379538943,
+STORE, 140525373530112, 140525377327103,
+SNULL, 140525373530112, 140525375188991,
+STORE, 140525375188992, 140525377327103,
+STORE, 140525373530112, 140525375188991,
+SNULL, 140525377286143, 140525377327103,
+STORE, 140525375188992, 140525377286143,
+STORE, 140525377286144, 140525377327103,
+SNULL, 140525377286144, 140525377310719,
+STORE, 140525377310720, 140525377327103,
+STORE, 140525377286144, 140525377310719,
+ERASE, 140525377286144, 140525377310719,
+STORE, 140525377286144, 140525377310719,
+ERASE, 140525377310720, 140525377327103,
+STORE, 140525377310720, 140525377327103,
+SNULL, 140525377302527, 140525377310719,
+STORE, 140525377286144, 140525377302527,
+STORE, 140525377302528, 140525377310719,
+SNULL, 94467666194431, 94467666198527,
+STORE, 94467666186240, 94467666194431,
+STORE, 94467666194432, 94467666198527,
+SNULL, 140525379571711, 140525379575807,
+STORE, 140525379567616, 140525379571711,
+STORE, 140525379571712, 140525379575807,
+ERASE, 140525379538944, 140525379567615,
+STORE, 94467693379584, 94467693514751,
+STORE, 94200172744704, 94200172957695,
+STORE, 94200175054848, 94200175058943,
+STORE, 94200175058944, 94200175067135,
+STORE, 94200175067136, 94200175079423,
+STORE, 94200196673536, 94200198905855,
+STORE, 140053867720704, 140053869379583,
+STORE, 140053869379584, 140053871476735,
+STORE, 140053871476736, 140053871493119,
+STORE, 140053871493120, 140053871501311,
+STORE, 140053871501312, 140053871517695,
+STORE, 140053871517696, 140053871529983,
+STORE, 140053871529984, 140053873623039,
+STORE, 140053873623040, 140053873627135,
+STORE, 140053873627136, 140053873631231,
+STORE, 140053873631232, 140053873774591,
+STORE, 140053874143232, 140053875826687,
+STORE, 140053875826688, 140053875843071,
+STORE, 140053875871744, 140053875875839,
+STORE, 140053875875840, 140053875879935,
+STORE, 140053875879936, 140053875884031,
+STORE, 140728538484736, 140728538623999,
+STORE, 140728538652672, 140728538664959,
+STORE, 140728538664960, 140728538669055,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140732307775488, 140737488351231,
+SNULL, 140732307783679, 140737488351231,
+STORE, 140732307775488, 140732307783679,
+STORE, 140732307644416, 140732307783679,
+STORE, 93831417630720, 93831419965439,
+SNULL, 93831417843711, 93831419965439,
+STORE, 93831417630720, 93831417843711,
+STORE, 93831417843712, 93831419965439,
+ERASE, 93831417843712, 93831419965439,
+STORE, 93831419940864, 93831419953151,
+STORE, 93831419953152, 93831419965439,
+STORE, 140241062088704, 140241064341503,
+SNULL, 140241062232063, 140241064341503,
+STORE, 140241062088704, 140241062232063,
+STORE, 140241062232064, 140241064341503,
+ERASE, 140241062232064, 140241064341503,
+STORE, 140241064329216, 140241064337407,
+STORE, 140241064337408, 140241064341503,
+STORE, 140732308140032, 140732308144127,
+STORE, 140732308127744, 140732308140031,
+STORE, 140241064300544, 140241064329215,
+STORE, 140241064292352, 140241064300543,
+STORE, 140241059975168, 140241062088703,
+SNULL, 140241059975168, 140241059987455,
+STORE, 140241059987456, 140241062088703,
+STORE, 140241059975168, 140241059987455,
+SNULL, 140241062080511, 140241062088703,
+STORE, 140241059987456, 140241062080511,
+STORE, 140241062080512, 140241062088703,
+ERASE, 140241062080512, 140241062088703,
+STORE, 140241062080512, 140241062088703,
+STORE, 140241056178176, 140241059975167,
+SNULL, 140241056178176, 140241057837055,
+STORE, 140241057837056, 140241059975167,
+STORE, 140241056178176, 140241057837055,
+SNULL, 140241059934207, 140241059975167,
+STORE, 140241057837056, 140241059934207,
+STORE, 140241059934208, 140241059975167,
+SNULL, 140241059934208, 140241059958783,
+STORE, 140241059958784, 140241059975167,
+STORE, 140241059934208, 140241059958783,
+ERASE, 140241059934208, 140241059958783,
+STORE, 140241059934208, 140241059958783,
+ERASE, 140241059958784, 140241059975167,
+STORE, 140241059958784, 140241059975167,
+STORE, 140241064284160, 140241064300543,
+SNULL, 140241059950591, 140241059958783,
+STORE, 140241059934208, 140241059950591,
+STORE, 140241059950592, 140241059958783,
+SNULL, 140241062084607, 140241062088703,
+STORE, 140241062080512, 140241062084607,
+STORE, 140241062084608, 140241062088703,
+SNULL, 93831419944959, 93831419953151,
+STORE, 93831419940864, 93831419944959,
+STORE, 93831419944960, 93831419953151,
+SNULL, 140241064333311, 140241064337407,
+STORE, 140241064329216, 140241064333311,
+STORE, 140241064333312, 140241064337407,
+ERASE, 140241064300544, 140241064329215,
+STORE, 93831435284480, 93831435419647,
+STORE, 140241062600704, 140241064284159,
+STORE, 93831435284480, 93831435554815,
+STORE, 93831435284480, 93831435689983,
+STORE, 93831435284480, 93831435862015,
+SNULL, 93831435837439, 93831435862015,
+STORE, 93831435284480, 93831435837439,
+STORE, 93831435837440, 93831435862015,
+ERASE, 93831435837440, 93831435862015,
+STORE, 93831435284480, 93831435972607,
+STORE, 93831435284480, 93831436107775,
+SNULL, 93831436091391, 93831436107775,
+STORE, 93831435284480, 93831436091391,
+STORE, 93831436091392, 93831436107775,
+ERASE, 93831436091392, 93831436107775,
+STORE, 93831435284480, 93831436226559,
+STORE, 93831435284480, 93831436361727,
+STORE, 93831435284480, 93831436505087,
+STORE, 93831435284480, 93831436652543,
+STORE, 93831435284480, 93831436787711,
+STORE, 93831435284480, 93831436926975,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140728546775040, 140737488351231,
+SNULL, 140728546783231, 140737488351231,
+STORE, 140728546775040, 140728546783231,
+STORE, 140728546643968, 140728546783231,
+STORE, 94456178786304, 94456181010431,
+SNULL, 94456178896895, 94456181010431,
+STORE, 94456178786304, 94456178896895,
+STORE, 94456178896896, 94456181010431,
+ERASE, 94456178896896, 94456181010431,
+STORE, 94456180989952, 94456181002239,
+STORE, 94456181002240, 94456181010431,
+STORE, 140221893091328, 140221895344127,
+SNULL, 140221893234687, 140221895344127,
+STORE, 140221893091328, 140221893234687,
+STORE, 140221893234688, 140221895344127,
+ERASE, 140221893234688, 140221895344127,
+STORE, 140221895331840, 140221895340031,
+STORE, 140221895340032, 140221895344127,
+STORE, 140728547803136, 140728547807231,
+STORE, 140728547790848, 140728547803135,
+STORE, 140221895303168, 140221895331839,
+STORE, 140221895294976, 140221895303167,
+STORE, 140221889294336, 140221893091327,
+SNULL, 140221889294336, 140221890953215,
+STORE, 140221890953216, 140221893091327,
+STORE, 140221889294336, 140221890953215,
+SNULL, 140221893050367, 140221893091327,
+STORE, 140221890953216, 140221893050367,
+STORE, 140221893050368, 140221893091327,
+SNULL, 140221893050368, 140221893074943,
+STORE, 140221893074944, 140221893091327,
+STORE, 140221893050368, 140221893074943,
+ERASE, 140221893050368, 140221893074943,
+STORE, 140221893050368, 140221893074943,
+ERASE, 140221893074944, 140221893091327,
+STORE, 140221893074944, 140221893091327,
+SNULL, 140221893066751, 140221893074943,
+STORE, 140221893050368, 140221893066751,
+STORE, 140221893066752, 140221893074943,
+SNULL, 94456180998143, 94456181002239,
+STORE, 94456180989952, 94456180998143,
+STORE, 94456180998144, 94456181002239,
+SNULL, 140221895335935, 140221895340031,
+STORE, 140221895331840, 140221895335935,
+STORE, 140221895335936, 140221895340031,
+ERASE, 140221895303168, 140221895331839,
+STORE, 94456203730944, 94456203866111,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140734438637568, 140737488351231,
+SNULL, 140734438645759, 140737488351231,
+STORE, 140734438637568, 140734438645759,
+STORE, 140734438506496, 140734438645759,
+STORE, 94652233351168, 94652235575295,
+SNULL, 94652233461759, 94652235575295,
+STORE, 94652233351168, 94652233461759,
+STORE, 94652233461760, 94652235575295,
+ERASE, 94652233461760, 94652235575295,
+STORE, 94652235554816, 94652235567103,
+STORE, 94652235567104, 94652235575295,
+STORE, 140536493195264, 140536495448063,
+SNULL, 140536493338623, 140536495448063,
+STORE, 140536493195264, 140536493338623,
+STORE, 140536493338624, 140536495448063,
+ERASE, 140536493338624, 140536495448063,
+STORE, 140536495435776, 140536495443967,
+STORE, 140536495443968, 140536495448063,
+STORE, 140734439002112, 140734439006207,
+STORE, 140734438989824, 140734439002111,
+STORE, 140536495407104, 140536495435775,
+STORE, 140536495398912, 140536495407103,
+STORE, 140536489398272, 140536493195263,
+SNULL, 140536489398272, 140536491057151,
+STORE, 140536491057152, 140536493195263,
+STORE, 140536489398272, 140536491057151,
+SNULL, 140536493154303, 140536493195263,
+STORE, 140536491057152, 140536493154303,
+STORE, 140536493154304, 140536493195263,
+SNULL, 140536493154304, 140536493178879,
+STORE, 140536493178880, 140536493195263,
+STORE, 140536493154304, 140536493178879,
+ERASE, 140536493154304, 140536493178879,
+STORE, 140536493154304, 140536493178879,
+ERASE, 140536493178880, 140536493195263,
+STORE, 140536493178880, 140536493195263,
+SNULL, 140536493170687, 140536493178879,
+STORE, 140536493154304, 140536493170687,
+STORE, 140536493170688, 140536493178879,
+SNULL, 94652235563007, 94652235567103,
+STORE, 94652235554816, 94652235563007,
+STORE, 94652235563008, 94652235567103,
+SNULL, 140536495439871, 140536495443967,
+STORE, 140536495435776, 140536495439871,
+STORE, 140536495439872, 140536495443967,
+ERASE, 140536495407104, 140536495435775,
+STORE, 94652265619456, 94652265754623,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140721814200320, 140737488351231,
+SNULL, 140721814208511, 140737488351231,
+STORE, 140721814200320, 140721814208511,
+STORE, 140721814069248, 140721814208511,
+STORE, 94062800691200, 94062802915327,
+SNULL, 94062800801791, 94062802915327,
+STORE, 94062800691200, 94062800801791,
+STORE, 94062800801792, 94062802915327,
+ERASE, 94062800801792, 94062802915327,
+STORE, 94062802894848, 94062802907135,
+STORE, 94062802907136, 94062802915327,
+STORE, 139717739700224, 139717741953023,
+SNULL, 139717739843583, 139717741953023,
+STORE, 139717739700224, 139717739843583,
+STORE, 139717739843584, 139717741953023,
+ERASE, 139717739843584, 139717741953023,
+STORE, 139717741940736, 139717741948927,
+STORE, 139717741948928, 139717741953023,
+STORE, 140721814224896, 140721814228991,
+STORE, 140721814212608, 140721814224895,
+STORE, 139717741912064, 139717741940735,
+STORE, 139717741903872, 139717741912063,
+STORE, 139717735903232, 139717739700223,
+SNULL, 139717735903232, 139717737562111,
+STORE, 139717737562112, 139717739700223,
+STORE, 139717735903232, 139717737562111,
+SNULL, 139717739659263, 139717739700223,
+STORE, 139717737562112, 139717739659263,
+STORE, 139717739659264, 139717739700223,
+SNULL, 139717739659264, 139717739683839,
+STORE, 139717739683840, 139717739700223,
+STORE, 139717739659264, 139717739683839,
+ERASE, 139717739659264, 139717739683839,
+STORE, 139717739659264, 139717739683839,
+ERASE, 139717739683840, 139717739700223,
+STORE, 139717739683840, 139717739700223,
+SNULL, 139717739675647, 139717739683839,
+STORE, 139717739659264, 139717739675647,
+STORE, 139717739675648, 139717739683839,
+SNULL, 94062802903039, 94062802907135,
+STORE, 94062802894848, 94062802903039,
+STORE, 94062802903040, 94062802907135,
+SNULL, 139717741944831, 139717741948927,
+STORE, 139717741940736, 139717741944831,
+STORE, 139717741944832, 139717741948927,
+ERASE, 139717741912064, 139717741940735,
+STORE, 94062814060544, 94062814195711,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723945754624, 140737488351231,
+SNULL, 140723945762815, 140737488351231,
+STORE, 140723945754624, 140723945762815,
+STORE, 140723945623552, 140723945762815,
+STORE, 94886119305216, 94886121639935,
+SNULL, 94886119518207, 94886121639935,
+STORE, 94886119305216, 94886119518207,
+STORE, 94886119518208, 94886121639935,
+ERASE, 94886119518208, 94886121639935,
+STORE, 94886121615360, 94886121627647,
+STORE, 94886121627648, 94886121639935,
+STORE, 140152532131840, 140152534384639,
+SNULL, 140152532275199, 140152534384639,
+STORE, 140152532131840, 140152532275199,
+STORE, 140152532275200, 140152534384639,
+ERASE, 140152532275200, 140152534384639,
+STORE, 140152534372352, 140152534380543,
+STORE, 140152534380544, 140152534384639,
+STORE, 140723946213376, 140723946217471,
+STORE, 140723946201088, 140723946213375,
+STORE, 140152534343680, 140152534372351,
+STORE, 140152534335488, 140152534343679,
+STORE, 140152530018304, 140152532131839,
+SNULL, 140152530018304, 140152530030591,
+STORE, 140152530030592, 140152532131839,
+STORE, 140152530018304, 140152530030591,
+SNULL, 140152532123647, 140152532131839,
+STORE, 140152530030592, 140152532123647,
+STORE, 140152532123648, 140152532131839,
+ERASE, 140152532123648, 140152532131839,
+STORE, 140152532123648, 140152532131839,
+STORE, 140152526221312, 140152530018303,
+SNULL, 140152526221312, 140152527880191,
+STORE, 140152527880192, 140152530018303,
+STORE, 140152526221312, 140152527880191,
+SNULL, 140152529977343, 140152530018303,
+STORE, 140152527880192, 140152529977343,
+STORE, 140152529977344, 140152530018303,
+SNULL, 140152529977344, 140152530001919,
+STORE, 140152530001920, 140152530018303,
+STORE, 140152529977344, 140152530001919,
+ERASE, 140152529977344, 140152530001919,
+STORE, 140152529977344, 140152530001919,
+ERASE, 140152530001920, 140152530018303,
+STORE, 140152530001920, 140152530018303,
+STORE, 140152534327296, 140152534343679,
+SNULL, 140152529993727, 140152530001919,
+STORE, 140152529977344, 140152529993727,
+STORE, 140152529993728, 140152530001919,
+SNULL, 140152532127743, 140152532131839,
+STORE, 140152532123648, 140152532127743,
+STORE, 140152532127744, 140152532131839,
+SNULL, 94886121619455, 94886121627647,
+STORE, 94886121615360, 94886121619455,
+STORE, 94886121619456, 94886121627647,
+SNULL, 140152534376447, 140152534380543,
+STORE, 140152534372352, 140152534376447,
+STORE, 140152534376448, 140152534380543,
+ERASE, 140152534343680, 140152534372351,
+STORE, 94886129770496, 94886129905663,
+STORE, 140152532643840, 140152534327295,
+STORE, 94886129770496, 94886130040831,
+STORE, 94886129770496, 94886130175999,
+STORE, 94886129770496, 94886130348031,
+SNULL, 94886130323455, 94886130348031,
+STORE, 94886129770496, 94886130323455,
+STORE, 94886130323456, 94886130348031,
+ERASE, 94886130323456, 94886130348031,
+STORE, 94886129770496, 94886130458623,
+STORE, 94886129770496, 94886130606079,
+SNULL, 94886130573311, 94886130606079,
+STORE, 94886129770496, 94886130573311,
+STORE, 94886130573312, 94886130606079,
+ERASE, 94886130573312, 94886130606079,
+STORE, 94886129770496, 94886130724863,
+STORE, 94886129770496, 94886130876415,
+STORE, 94886129770496, 94886131023871,
+STORE, 94886129770496, 94886131175423,
+STORE, 94886129770496, 94886131318783,
+STORE, 94886129770496, 94886131453951,
+SNULL, 94886131449855, 94886131453951,
+STORE, 94886129770496, 94886131449855,
+STORE, 94886131449856, 94886131453951,
+ERASE, 94886131449856, 94886131453951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735450779648, 140737488351231,
+SNULL, 140735450787839, 140737488351231,
+STORE, 140735450779648, 140735450787839,
+STORE, 140735450648576, 140735450787839,
+STORE, 93947794079744, 93947796414463,
+SNULL, 93947794292735, 93947796414463,
+STORE, 93947794079744, 93947794292735,
+STORE, 93947794292736, 93947796414463,
+ERASE, 93947794292736, 93947796414463,
+STORE, 93947796389888, 93947796402175,
+STORE, 93947796402176, 93947796414463,
+STORE, 139841993433088, 139841995685887,
+SNULL, 139841993576447, 139841995685887,
+STORE, 139841993433088, 139841993576447,
+STORE, 139841993576448, 139841995685887,
+ERASE, 139841993576448, 139841995685887,
+STORE, 139841995673600, 139841995681791,
+STORE, 139841995681792, 139841995685887,
+STORE, 140735451308032, 140735451312127,
+STORE, 140735451295744, 140735451308031,
+STORE, 139841995644928, 139841995673599,
+STORE, 139841995636736, 139841995644927,
+STORE, 139841991319552, 139841993433087,
+SNULL, 139841991319552, 139841991331839,
+STORE, 139841991331840, 139841993433087,
+STORE, 139841991319552, 139841991331839,
+SNULL, 139841993424895, 139841993433087,
+STORE, 139841991331840, 139841993424895,
+STORE, 139841993424896, 139841993433087,
+ERASE, 139841993424896, 139841993433087,
+STORE, 139841993424896, 139841993433087,
+STORE, 139841987522560, 139841991319551,
+SNULL, 139841987522560, 139841989181439,
+STORE, 139841989181440, 139841991319551,
+STORE, 139841987522560, 139841989181439,
+SNULL, 139841991278591, 139841991319551,
+STORE, 139841989181440, 139841991278591,
+STORE, 139841991278592, 139841991319551,
+SNULL, 139841991278592, 139841991303167,
+STORE, 139841991303168, 139841991319551,
+STORE, 139841991278592, 139841991303167,
+ERASE, 139841991278592, 139841991303167,
+STORE, 139841991278592, 139841991303167,
+ERASE, 139841991303168, 139841991319551,
+STORE, 139841991303168, 139841991319551,
+STORE, 139841995628544, 139841995644927,
+SNULL, 139841991294975, 139841991303167,
+STORE, 139841991278592, 139841991294975,
+STORE, 139841991294976, 139841991303167,
+SNULL, 139841993428991, 139841993433087,
+STORE, 139841993424896, 139841993428991,
+STORE, 139841993428992, 139841993433087,
+SNULL, 93947796393983, 93947796402175,
+STORE, 93947796389888, 93947796393983,
+STORE, 93947796393984, 93947796402175,
+SNULL, 139841995677695, 139841995681791,
+STORE, 139841995673600, 139841995677695,
+STORE, 139841995677696, 139841995681791,
+ERASE, 139841995644928, 139841995673599,
+STORE, 93947829739520, 93947829874687,
+STORE, 139841993945088, 139841995628543,
+STORE, 93947829739520, 93947830009855,
+STORE, 93947829739520, 93947830145023,
+STORE, 94659351814144, 94659352027135,
+STORE, 94659354124288, 94659354128383,
+STORE, 94659354128384, 94659354136575,
+STORE, 94659354136576, 94659354148863,
+STORE, 94659383476224, 94659385057279,
+STORE, 139959054557184, 139959056216063,
+STORE, 139959056216064, 139959058313215,
+STORE, 139959058313216, 139959058329599,
+STORE, 139959058329600, 139959058337791,
+STORE, 139959058337792, 139959058354175,
+STORE, 139959058354176, 139959058366463,
+STORE, 139959058366464, 139959060459519,
+STORE, 139959060459520, 139959060463615,
+STORE, 139959060463616, 139959060467711,
+STORE, 139959060467712, 139959060611071,
+STORE, 139959060979712, 139959062663167,
+STORE, 139959062663168, 139959062679551,
+STORE, 139959062708224, 139959062712319,
+STORE, 139959062712320, 139959062716415,
+STORE, 139959062716416, 139959062720511,
+STORE, 140735532539904, 140735532679167,
+STORE, 140735532830720, 140735532843007,
+STORE, 140735532843008, 140735532847103,
+STORE, 93894361829376, 93894362042367,
+STORE, 93894364139520, 93894364143615,
+STORE, 93894364143616, 93894364151807,
+STORE, 93894364151808, 93894364164095,
+STORE, 93894396944384, 93894397624319,
+STORE, 140075612573696, 140075614232575,
+STORE, 140075614232576, 140075616329727,
+STORE, 140075616329728, 140075616346111,
+STORE, 140075616346112, 140075616354303,
+STORE, 140075616354304, 140075616370687,
+STORE, 140075616370688, 140075616382975,
+STORE, 140075616382976, 140075618476031,
+STORE, 140075618476032, 140075618480127,
+STORE, 140075618480128, 140075618484223,
+STORE, 140075618484224, 140075618627583,
+STORE, 140075618996224, 140075620679679,
+STORE, 140075620679680, 140075620696063,
+STORE, 140075620724736, 140075620728831,
+STORE, 140075620728832, 140075620732927,
+STORE, 140075620732928, 140075620737023,
+STORE, 140720830312448, 140720830451711,
+STORE, 140720830631936, 140720830644223,
+STORE, 140720830644224, 140720830648319,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735116226560, 140737488351231,
+SNULL, 140735116234751, 140737488351231,
+STORE, 140735116226560, 140735116234751,
+STORE, 140735116095488, 140735116234751,
+STORE, 94873398054912, 94873400279039,
+SNULL, 94873398165503, 94873400279039,
+STORE, 94873398054912, 94873398165503,
+STORE, 94873398165504, 94873400279039,
+ERASE, 94873398165504, 94873400279039,
+STORE, 94873400258560, 94873400270847,
+STORE, 94873400270848, 94873400279039,
+STORE, 140303828606976, 140303830859775,
+SNULL, 140303828750335, 140303830859775,
+STORE, 140303828606976, 140303828750335,
+STORE, 140303828750336, 140303830859775,
+ERASE, 140303828750336, 140303830859775,
+STORE, 140303830847488, 140303830855679,
+STORE, 140303830855680, 140303830859775,
+STORE, 140735116251136, 140735116255231,
+STORE, 140735116238848, 140735116251135,
+STORE, 140303830818816, 140303830847487,
+STORE, 140303830810624, 140303830818815,
+STORE, 140303824809984, 140303828606975,
+SNULL, 140303824809984, 140303826468863,
+STORE, 140303826468864, 140303828606975,
+STORE, 140303824809984, 140303826468863,
+SNULL, 140303828566015, 140303828606975,
+STORE, 140303826468864, 140303828566015,
+STORE, 140303828566016, 140303828606975,
+SNULL, 140303828566016, 140303828590591,
+STORE, 140303828590592, 140303828606975,
+STORE, 140303828566016, 140303828590591,
+ERASE, 140303828566016, 140303828590591,
+STORE, 140303828566016, 140303828590591,
+ERASE, 140303828590592, 140303828606975,
+STORE, 140303828590592, 140303828606975,
+SNULL, 140303828582399, 140303828590591,
+STORE, 140303828566016, 140303828582399,
+STORE, 140303828582400, 140303828590591,
+SNULL, 94873400266751, 94873400270847,
+STORE, 94873400258560, 94873400266751,
+STORE, 94873400266752, 94873400270847,
+SNULL, 140303830851583, 140303830855679,
+STORE, 140303830847488, 140303830851583,
+STORE, 140303830851584, 140303830855679,
+ERASE, 140303830818816, 140303830847487,
+STORE, 94873413713920, 94873413849087,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140732349956096, 140737488351231,
+SNULL, 140732349964287, 140737488351231,
+STORE, 140732349956096, 140732349964287,
+STORE, 140732349825024, 140732349964287,
+STORE, 94009652736000, 94009655070719,
+SNULL, 94009652948991, 94009655070719,
+STORE, 94009652736000, 94009652948991,
+STORE, 94009652948992, 94009655070719,
+ERASE, 94009652948992, 94009655070719,
+STORE, 94009655046144, 94009655058431,
+STORE, 94009655058432, 94009655070719,
+STORE, 140295688531968, 140295690784767,
+SNULL, 140295688675327, 140295690784767,
+STORE, 140295688531968, 140295688675327,
+STORE, 140295688675328, 140295690784767,
+ERASE, 140295688675328, 140295690784767,
+STORE, 140295690772480, 140295690780671,
+STORE, 140295690780672, 140295690784767,
+STORE, 140732350005248, 140732350009343,
+STORE, 140732349992960, 140732350005247,
+STORE, 140295690743808, 140295690772479,
+STORE, 140295690735616, 140295690743807,
+STORE, 140295686418432, 140295688531967,
+SNULL, 140295686418432, 140295686430719,
+STORE, 140295686430720, 140295688531967,
+STORE, 140295686418432, 140295686430719,
+SNULL, 140295688523775, 140295688531967,
+STORE, 140295686430720, 140295688523775,
+STORE, 140295688523776, 140295688531967,
+ERASE, 140295688523776, 140295688531967,
+STORE, 140295688523776, 140295688531967,
+STORE, 140295682621440, 140295686418431,
+SNULL, 140295682621440, 140295684280319,
+STORE, 140295684280320, 140295686418431,
+STORE, 140295682621440, 140295684280319,
+SNULL, 140295686377471, 140295686418431,
+STORE, 140295684280320, 140295686377471,
+STORE, 140295686377472, 140295686418431,
+SNULL, 140295686377472, 140295686402047,
+STORE, 140295686402048, 140295686418431,
+STORE, 140295686377472, 140295686402047,
+ERASE, 140295686377472, 140295686402047,
+STORE, 140295686377472, 140295686402047,
+ERASE, 140295686402048, 140295686418431,
+STORE, 140295686402048, 140295686418431,
+STORE, 140295690727424, 140295690743807,
+SNULL, 140295686393855, 140295686402047,
+STORE, 140295686377472, 140295686393855,
+STORE, 140295686393856, 140295686402047,
+SNULL, 140295688527871, 140295688531967,
+STORE, 140295688523776, 140295688527871,
+STORE, 140295688527872, 140295688531967,
+SNULL, 94009655050239, 94009655058431,
+STORE, 94009655046144, 94009655050239,
+STORE, 94009655050240, 94009655058431,
+SNULL, 140295690776575, 140295690780671,
+STORE, 140295690772480, 140295690776575,
+STORE, 140295690776576, 140295690780671,
+ERASE, 140295690743808, 140295690772479,
+STORE, 94009672114176, 94009672249343,
+STORE, 140295689043968, 140295690727423,
+STORE, 94009672114176, 94009672384511,
+STORE, 94009672114176, 94009672519679,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722376515584, 140737488351231,
+SNULL, 140722376523775, 140737488351231,
+STORE, 140722376515584, 140722376523775,
+STORE, 140722376384512, 140722376523775,
+STORE, 94089815773184, 94089818107903,
+SNULL, 94089815986175, 94089818107903,
+STORE, 94089815773184, 94089815986175,
+STORE, 94089815986176, 94089818107903,
+ERASE, 94089815986176, 94089818107903,
+STORE, 94089818083328, 94089818095615,
+STORE, 94089818095616, 94089818107903,
+STORE, 140265595711488, 140265597964287,
+SNULL, 140265595854847, 140265597964287,
+STORE, 140265595711488, 140265595854847,
+STORE, 140265595854848, 140265597964287,
+ERASE, 140265595854848, 140265597964287,
+STORE, 140265597952000, 140265597960191,
+STORE, 140265597960192, 140265597964287,
+STORE, 140722378297344, 140722378301439,
+STORE, 140722378285056, 140722378297343,
+STORE, 140265597923328, 140265597951999,
+STORE, 140265597915136, 140265597923327,
+STORE, 140265593597952, 140265595711487,
+SNULL, 140265593597952, 140265593610239,
+STORE, 140265593610240, 140265595711487,
+STORE, 140265593597952, 140265593610239,
+SNULL, 140265595703295, 140265595711487,
+STORE, 140265593610240, 140265595703295,
+STORE, 140265595703296, 140265595711487,
+ERASE, 140265595703296, 140265595711487,
+STORE, 140265595703296, 140265595711487,
+STORE, 140265589800960, 140265593597951,
+SNULL, 140265589800960, 140265591459839,
+STORE, 140265591459840, 140265593597951,
+STORE, 140265589800960, 140265591459839,
+SNULL, 140265593556991, 140265593597951,
+STORE, 140265591459840, 140265593556991,
+STORE, 140265593556992, 140265593597951,
+SNULL, 140265593556992, 140265593581567,
+STORE, 140265593581568, 140265593597951,
+STORE, 140265593556992, 140265593581567,
+ERASE, 140265593556992, 140265593581567,
+STORE, 140265593556992, 140265593581567,
+ERASE, 140265593581568, 140265593597951,
+STORE, 140265593581568, 140265593597951,
+STORE, 140265597906944, 140265597923327,
+SNULL, 140265593573375, 140265593581567,
+STORE, 140265593556992, 140265593573375,
+STORE, 140265593573376, 140265593581567,
+SNULL, 140265595707391, 140265595711487,
+STORE, 140265595703296, 140265595707391,
+STORE, 140265595707392, 140265595711487,
+SNULL, 94089818087423, 94089818095615,
+STORE, 94089818083328, 94089818087423,
+STORE, 94089818087424, 94089818095615,
+SNULL, 140265597956095, 140265597960191,
+STORE, 140265597952000, 140265597956095,
+STORE, 140265597956096, 140265597960191,
+ERASE, 140265597923328, 140265597951999,
+STORE, 94089837146112, 94089837281279,
+STORE, 140265596223488, 140265597906943,
+STORE, 94089837146112, 94089837416447,
+STORE, 94089837146112, 94089837551615,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735265218560, 140737488351231,
+SNULL, 140735265226751, 140737488351231,
+STORE, 140735265218560, 140735265226751,
+STORE, 140735265087488, 140735265226751,
+STORE, 94250422370304, 94250424705023,
+SNULL, 94250422583295, 94250424705023,
+STORE, 94250422370304, 94250422583295,
+STORE, 94250422583296, 94250424705023,
+ERASE, 94250422583296, 94250424705023,
+STORE, 94250424680448, 94250424692735,
+STORE, 94250424692736, 94250424705023,
+STORE, 140344442474496, 140344444727295,
+SNULL, 140344442617855, 140344444727295,
+STORE, 140344442474496, 140344442617855,
+STORE, 140344442617856, 140344444727295,
+ERASE, 140344442617856, 140344444727295,
+STORE, 140344444715008, 140344444723199,
+STORE, 140344444723200, 140344444727295,
+STORE, 140735265341440, 140735265345535,
+STORE, 140735265329152, 140735265341439,
+STORE, 140344444686336, 140344444715007,
+STORE, 140344444678144, 140344444686335,
+STORE, 140344440360960, 140344442474495,
+SNULL, 140344440360960, 140344440373247,
+STORE, 140344440373248, 140344442474495,
+STORE, 140344440360960, 140344440373247,
+SNULL, 140344442466303, 140344442474495,
+STORE, 140344440373248, 140344442466303,
+STORE, 140344442466304, 140344442474495,
+ERASE, 140344442466304, 140344442474495,
+STORE, 140344442466304, 140344442474495,
+STORE, 140344436563968, 140344440360959,
+SNULL, 140344436563968, 140344438222847,
+STORE, 140344438222848, 140344440360959,
+STORE, 140344436563968, 140344438222847,
+SNULL, 140344440319999, 140344440360959,
+STORE, 140344438222848, 140344440319999,
+STORE, 140344440320000, 140344440360959,
+SNULL, 140344440320000, 140344440344575,
+STORE, 140344440344576, 140344440360959,
+STORE, 140344440320000, 140344440344575,
+ERASE, 140344440320000, 140344440344575,
+STORE, 140344440320000, 140344440344575,
+ERASE, 140344440344576, 140344440360959,
+STORE, 140344440344576, 140344440360959,
+STORE, 140344444669952, 140344444686335,
+SNULL, 140344440336383, 140344440344575,
+STORE, 140344440320000, 140344440336383,
+STORE, 140344440336384, 140344440344575,
+SNULL, 140344442470399, 140344442474495,
+STORE, 140344442466304, 140344442470399,
+STORE, 140344442470400, 140344442474495,
+SNULL, 94250424684543, 94250424692735,
+STORE, 94250424680448, 94250424684543,
+STORE, 94250424684544, 94250424692735,
+SNULL, 140344444719103, 140344444723199,
+STORE, 140344444715008, 140344444719103,
+STORE, 140344444719104, 140344444723199,
+ERASE, 140344444686336, 140344444715007,
+STORE, 94250445512704, 94250445647871,
+STORE, 140344442986496, 140344444669951,
+STORE, 94250445512704, 94250445783039,
+STORE, 94250445512704, 94250445918207,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140725762719744, 140737488351231,
+SNULL, 140725762727935, 140737488351231,
+STORE, 140725762719744, 140725762727935,
+STORE, 140725762588672, 140725762727935,
+STORE, 94819009097728, 94819011432447,
+SNULL, 94819009310719, 94819011432447,
+STORE, 94819009097728, 94819009310719,
+STORE, 94819009310720, 94819011432447,
+ERASE, 94819009310720, 94819011432447,
+STORE, 94819011407872, 94819011420159,
+STORE, 94819011420160, 94819011432447,
+STORE, 139987985596416, 139987987849215,
+SNULL, 139987985739775, 139987987849215,
+STORE, 139987985596416, 139987985739775,
+STORE, 139987985739776, 139987987849215,
+ERASE, 139987985739776, 139987987849215,
+STORE, 139987987836928, 139987987845119,
+STORE, 139987987845120, 139987987849215,
+STORE, 140725763072000, 140725763076095,
+STORE, 140725763059712, 140725763071999,
+STORE, 139987987808256, 139987987836927,
+STORE, 139987987800064, 139987987808255,
+STORE, 139987983482880, 139987985596415,
+SNULL, 139987983482880, 139987983495167,
+STORE, 139987983495168, 139987985596415,
+STORE, 139987983482880, 139987983495167,
+SNULL, 139987985588223, 139987985596415,
+STORE, 139987983495168, 139987985588223,
+STORE, 139987985588224, 139987985596415,
+ERASE, 139987985588224, 139987985596415,
+STORE, 139987985588224, 139987985596415,
+STORE, 139987979685888, 139987983482879,
+SNULL, 139987979685888, 139987981344767,
+STORE, 139987981344768, 139987983482879,
+STORE, 139987979685888, 139987981344767,
+SNULL, 139987983441919, 139987983482879,
+STORE, 139987981344768, 139987983441919,
+STORE, 139987983441920, 139987983482879,
+SNULL, 139987983441920, 139987983466495,
+STORE, 139987983466496, 139987983482879,
+STORE, 139987983441920, 139987983466495,
+ERASE, 139987983441920, 139987983466495,
+STORE, 139987983441920, 139987983466495,
+ERASE, 139987983466496, 139987983482879,
+STORE, 139987983466496, 139987983482879,
+STORE, 139987987791872, 139987987808255,
+SNULL, 139987983458303, 139987983466495,
+STORE, 139987983441920, 139987983458303,
+STORE, 139987983458304, 139987983466495,
+SNULL, 139987985592319, 139987985596415,
+STORE, 139987985588224, 139987985592319,
+STORE, 139987985592320, 139987985596415,
+SNULL, 94819011411967, 94819011420159,
+STORE, 94819011407872, 94819011411967,
+STORE, 94819011411968, 94819011420159,
+SNULL, 139987987841023, 139987987845119,
+STORE, 139987987836928, 139987987841023,
+STORE, 139987987841024, 139987987845119,
+ERASE, 139987987808256, 139987987836927,
+STORE, 94819028176896, 94819028312063,
+STORE, 139987986108416, 139987987791871,
+STORE, 94819028176896, 94819028447231,
+STORE, 94819028176896, 94819028582399,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722475413504, 140737488351231,
+SNULL, 140722475421695, 140737488351231,
+STORE, 140722475413504, 140722475421695,
+STORE, 140722475282432, 140722475421695,
+STORE, 94620599119872, 94620601343999,
+SNULL, 94620599230463, 94620601343999,
+STORE, 94620599119872, 94620599230463,
+STORE, 94620599230464, 94620601343999,
+ERASE, 94620599230464, 94620601343999,
+STORE, 94620601323520, 94620601335807,
+STORE, 94620601335808, 94620601343999,
+STORE, 139891763060736, 139891765313535,
+SNULL, 139891763204095, 139891765313535,
+STORE, 139891763060736, 139891763204095,
+STORE, 139891763204096, 139891765313535,
+ERASE, 139891763204096, 139891765313535,
+STORE, 139891765301248, 139891765309439,
+STORE, 139891765309440, 139891765313535,
+STORE, 140722475700224, 140722475704319,
+STORE, 140722475687936, 140722475700223,
+STORE, 139891765272576, 139891765301247,
+STORE, 139891765264384, 139891765272575,
+STORE, 139891759263744, 139891763060735,
+SNULL, 139891759263744, 139891760922623,
+STORE, 139891760922624, 139891763060735,
+STORE, 139891759263744, 139891760922623,
+SNULL, 139891763019775, 139891763060735,
+STORE, 139891760922624, 139891763019775,
+STORE, 139891763019776, 139891763060735,
+SNULL, 139891763019776, 139891763044351,
+STORE, 139891763044352, 139891763060735,
+STORE, 139891763019776, 139891763044351,
+ERASE, 139891763019776, 139891763044351,
+STORE, 139891763019776, 139891763044351,
+ERASE, 139891763044352, 139891763060735,
+STORE, 139891763044352, 139891763060735,
+SNULL, 139891763036159, 139891763044351,
+STORE, 139891763019776, 139891763036159,
+STORE, 139891763036160, 139891763044351,
+SNULL, 94620601331711, 94620601335807,
+STORE, 94620601323520, 94620601331711,
+STORE, 94620601331712, 94620601335807,
+SNULL, 139891765305343, 139891765309439,
+STORE, 139891765301248, 139891765305343,
+STORE, 139891765305344, 139891765309439,
+ERASE, 139891765272576, 139891765301247,
+STORE, 94620610027520, 94620610162687,
+STORE, 94031976210432, 94031976423423,
+STORE, 94031978520576, 94031978524671,
+STORE, 94031978524672, 94031978532863,
+STORE, 94031978532864, 94031978545151,
+STORE, 94031990398976, 94031992565759,
+STORE, 140336240640000, 140336242298879,
+STORE, 140336242298880, 140336244396031,
+STORE, 140336244396032, 140336244412415,
+STORE, 140336244412416, 140336244420607,
+STORE, 140336244420608, 140336244436991,
+STORE, 140336244436992, 140336244449279,
+STORE, 140336244449280, 140336246542335,
+STORE, 140336246542336, 140336246546431,
+STORE, 140336246546432, 140336246550527,
+STORE, 140336246550528, 140336246693887,
+STORE, 140336247062528, 140336248745983,
+STORE, 140336248745984, 140336248762367,
+STORE, 140336248791040, 140336248795135,
+STORE, 140336248795136, 140336248799231,
+STORE, 140336248799232, 140336248803327,
+STORE, 140728500064256, 140728500203519,
+STORE, 140728501501952, 140728501514239,
+STORE, 140728501514240, 140728501518335,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140730503987200, 140737488351231,
+SNULL, 140730503995391, 140737488351231,
+STORE, 140730503987200, 140730503995391,
+STORE, 140730503856128, 140730503995391,
+STORE, 93866544205824, 93866546429951,
+SNULL, 93866544316415, 93866546429951,
+STORE, 93866544205824, 93866544316415,
+STORE, 93866544316416, 93866546429951,
+ERASE, 93866544316416, 93866546429951,
+STORE, 93866546409472, 93866546421759,
+STORE, 93866546421760, 93866546429951,
+STORE, 140216311959552, 140216314212351,
+SNULL, 140216312102911, 140216314212351,
+STORE, 140216311959552, 140216312102911,
+STORE, 140216312102912, 140216314212351,
+ERASE, 140216312102912, 140216314212351,
+STORE, 140216314200064, 140216314208255,
+STORE, 140216314208256, 140216314212351,
+STORE, 140730504626176, 140730504630271,
+STORE, 140730504613888, 140730504626175,
+STORE, 140216314171392, 140216314200063,
+STORE, 140216314163200, 140216314171391,
+STORE, 140216308162560, 140216311959551,
+SNULL, 140216308162560, 140216309821439,
+STORE, 140216309821440, 140216311959551,
+STORE, 140216308162560, 140216309821439,
+SNULL, 140216311918591, 140216311959551,
+STORE, 140216309821440, 140216311918591,
+STORE, 140216311918592, 140216311959551,
+SNULL, 140216311918592, 140216311943167,
+STORE, 140216311943168, 140216311959551,
+STORE, 140216311918592, 140216311943167,
+ERASE, 140216311918592, 140216311943167,
+STORE, 140216311918592, 140216311943167,
+ERASE, 140216311943168, 140216311959551,
+STORE, 140216311943168, 140216311959551,
+SNULL, 140216311934975, 140216311943167,
+STORE, 140216311918592, 140216311934975,
+STORE, 140216311934976, 140216311943167,
+SNULL, 93866546417663, 93866546421759,
+STORE, 93866546409472, 93866546417663,
+STORE, 93866546417664, 93866546421759,
+SNULL, 140216314204159, 140216314208255,
+STORE, 140216314200064, 140216314204159,
+STORE, 140216314204160, 140216314208255,
+ERASE, 140216314171392, 140216314200063,
+STORE, 93866550386688, 93866550521855,
+STORE, 94074292674560, 94074292887551,
+STORE, 94074294984704, 94074294988799,
+STORE, 94074294988800, 94074294996991,
+STORE, 94074294996992, 94074295009279,
+STORE, 94074300219392, 94074301378559,
+STORE, 139781563256832, 139781564915711,
+STORE, 139781564915712, 139781567012863,
+STORE, 139781567012864, 139781567029247,
+STORE, 139781567029248, 139781567037439,
+STORE, 139781567037440, 139781567053823,
+STORE, 139781567053824, 139781567066111,
+STORE, 139781567066112, 139781569159167,
+STORE, 139781569159168, 139781569163263,
+STORE, 139781569163264, 139781569167359,
+STORE, 139781569167360, 139781569310719,
+STORE, 139781569679360, 139781571362815,
+STORE, 139781571362816, 139781571379199,
+STORE, 139781571407872, 139781571411967,
+STORE, 139781571411968, 139781571416063,
+STORE, 139781571416064, 139781571420159,
+STORE, 140723688488960, 140723688628223,
+STORE, 140723689005056, 140723689017343,
+STORE, 140723689017344, 140723689021439,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735189745664, 140737488351231,
+SNULL, 140735189753855, 140737488351231,
+STORE, 140735189745664, 140735189753855,
+STORE, 140735189614592, 140735189753855,
+STORE, 94172072177664, 94172074512383,
+SNULL, 94172072390655, 94172074512383,
+STORE, 94172072177664, 94172072390655,
+STORE, 94172072390656, 94172074512383,
+ERASE, 94172072390656, 94172074512383,
+STORE, 94172074487808, 94172074500095,
+STORE, 94172074500096, 94172074512383,
+STORE, 140687827263488, 140687829516287,
+SNULL, 140687827406847, 140687829516287,
+STORE, 140687827263488, 140687827406847,
+STORE, 140687827406848, 140687829516287,
+ERASE, 140687827406848, 140687829516287,
+STORE, 140687829504000, 140687829512191,
+STORE, 140687829512192, 140687829516287,
+STORE, 140735189766144, 140735189770239,
+STORE, 140735189753856, 140735189766143,
+STORE, 140687829475328, 140687829503999,
+STORE, 140687829467136, 140687829475327,
+STORE, 140687825149952, 140687827263487,
+SNULL, 140687825149952, 140687825162239,
+STORE, 140687825162240, 140687827263487,
+STORE, 140687825149952, 140687825162239,
+SNULL, 140687827255295, 140687827263487,
+STORE, 140687825162240, 140687827255295,
+STORE, 140687827255296, 140687827263487,
+ERASE, 140687827255296, 140687827263487,
+STORE, 140687827255296, 140687827263487,
+STORE, 140687821352960, 140687825149951,
+SNULL, 140687821352960, 140687823011839,
+STORE, 140687823011840, 140687825149951,
+STORE, 140687821352960, 140687823011839,
+SNULL, 140687825108991, 140687825149951,
+STORE, 140687823011840, 140687825108991,
+STORE, 140687825108992, 140687825149951,
+SNULL, 140687825108992, 140687825133567,
+STORE, 140687825133568, 140687825149951,
+STORE, 140687825108992, 140687825133567,
+ERASE, 140687825108992, 140687825133567,
+STORE, 140687825108992, 140687825133567,
+ERASE, 140687825133568, 140687825149951,
+STORE, 140687825133568, 140687825149951,
+STORE, 140687829458944, 140687829475327,
+SNULL, 140687825125375, 140687825133567,
+STORE, 140687825108992, 140687825125375,
+STORE, 140687825125376, 140687825133567,
+SNULL, 140687827259391, 140687827263487,
+STORE, 140687827255296, 140687827259391,
+STORE, 140687827259392, 140687827263487,
+SNULL, 94172074491903, 94172074500095,
+STORE, 94172074487808, 94172074491903,
+STORE, 94172074491904, 94172074500095,
+SNULL, 140687829508095, 140687829512191,
+STORE, 140687829504000, 140687829508095,
+STORE, 140687829508096, 140687829512191,
+ERASE, 140687829475328, 140687829503999,
+STORE, 94172092432384, 94172092567551,
+STORE, 140687827775488, 140687829458943,
+STORE, 94172092432384, 94172092702719,
+STORE, 94172092432384, 94172092837887,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140737229504512, 140737488351231,
+SNULL, 140737229512703, 140737488351231,
+STORE, 140737229504512, 140737229512703,
+STORE, 140737229373440, 140737229512703,
+STORE, 94155246866432, 94155249090559,
+SNULL, 94155246977023, 94155249090559,
+STORE, 94155246866432, 94155246977023,
+STORE, 94155246977024, 94155249090559,
+ERASE, 94155246977024, 94155249090559,
+STORE, 94155249070080, 94155249082367,
+STORE, 94155249082368, 94155249090559,
+STORE, 140640993693696, 140640995946495,
+SNULL, 140640993837055, 140640995946495,
+STORE, 140640993693696, 140640993837055,
+STORE, 140640993837056, 140640995946495,
+ERASE, 140640993837056, 140640995946495,
+STORE, 140640995934208, 140640995942399,
+STORE, 140640995942400, 140640995946495,
+STORE, 140737230004224, 140737230008319,
+STORE, 140737229991936, 140737230004223,
+STORE, 140640995905536, 140640995934207,
+STORE, 140640995897344, 140640995905535,
+STORE, 140640989896704, 140640993693695,
+SNULL, 140640989896704, 140640991555583,
+STORE, 140640991555584, 140640993693695,
+STORE, 140640989896704, 140640991555583,
+SNULL, 140640993652735, 140640993693695,
+STORE, 140640991555584, 140640993652735,
+STORE, 140640993652736, 140640993693695,
+SNULL, 140640993652736, 140640993677311,
+STORE, 140640993677312, 140640993693695,
+STORE, 140640993652736, 140640993677311,
+ERASE, 140640993652736, 140640993677311,
+STORE, 140640993652736, 140640993677311,
+ERASE, 140640993677312, 140640993693695,
+STORE, 140640993677312, 140640993693695,
+SNULL, 140640993669119, 140640993677311,
+STORE, 140640993652736, 140640993669119,
+STORE, 140640993669120, 140640993677311,
+SNULL, 94155249078271, 94155249082367,
+STORE, 94155249070080, 94155249078271,
+STORE, 94155249078272, 94155249082367,
+SNULL, 140640995938303, 140640995942399,
+STORE, 140640995934208, 140640995938303,
+STORE, 140640995938304, 140640995942399,
+ERASE, 140640995905536, 140640995934207,
+STORE, 94155281035264, 94155281170431,
+STORE, 94088066453504, 94088066564095,
+STORE, 94088068657152, 94088068665343,
+STORE, 94088068665344, 94088068669439,
+STORE, 94088068669440, 94088068677631,
+STORE, 94088090214400, 94088090349567,
+STORE, 140503024627712, 140503026286591,
+STORE, 140503026286592, 140503028383743,
+STORE, 140503028383744, 140503028400127,
+STORE, 140503028400128, 140503028408319,
+STORE, 140503028408320, 140503028424703,
+STORE, 140503028424704, 140503028568063,
+STORE, 140503030628352, 140503030636543,
+STORE, 140503030665216, 140503030669311,
+STORE, 140503030669312, 140503030673407,
+STORE, 140503030673408, 140503030677503,
+STORE, 140730894725120, 140730894864383,
+STORE, 140730894880768, 140730894893055,
+STORE, 140730894893056, 140730894897151,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140730434342912, 140737488351231,
+SNULL, 140730434351103, 140737488351231,
+STORE, 140730434342912, 140730434351103,
+STORE, 140730434211840, 140730434351103,
+STORE, 4194304, 5128191,
+STORE, 7221248, 7241727,
+STORE, 7241728, 7249919,
+STORE, 140109041938432, 140109044191231,
+SNULL, 140109042081791, 140109044191231,
+STORE, 140109041938432, 140109042081791,
+STORE, 140109042081792, 140109044191231,
+ERASE, 140109042081792, 140109044191231,
+STORE, 140109044178944, 140109044187135,
+STORE, 140109044187136, 140109044191231,
+STORE, 140730434850816, 140730434854911,
+STORE, 140730434838528, 140730434850815,
+STORE, 140109044150272, 140109044178943,
+STORE, 140109044142080, 140109044150271,
+STORE, 140109038776320, 140109041938431,
+SNULL, 140109038776320, 140109039837183,
+STORE, 140109039837184, 140109041938431,
+STORE, 140109038776320, 140109039837183,
+SNULL, 140109041930239, 140109041938431,
+STORE, 140109039837184, 140109041930239,
+STORE, 140109041930240, 140109041938431,
+ERASE, 140109041930240, 140109041938431,
+STORE, 140109041930240, 140109041938431,
+STORE, 140109034979328, 140109038776319,
+SNULL, 140109034979328, 140109036638207,
+STORE, 140109036638208, 140109038776319,
+STORE, 140109034979328, 140109036638207,
+SNULL, 140109038735359, 140109038776319,
+STORE, 140109036638208, 140109038735359,
+STORE, 140109038735360, 140109038776319,
+SNULL, 140109038735360, 140109038759935,
+STORE, 140109038759936, 140109038776319,
+STORE, 140109038735360, 140109038759935,
+ERASE, 140109038735360, 140109038759935,
+STORE, 140109038735360, 140109038759935,
+ERASE, 140109038759936, 140109038776319,
+STORE, 140109038759936, 140109038776319,
+STORE, 140109044129792, 140109044150271,
+SNULL, 140109038751743, 140109038759935,
+STORE, 140109038735360, 140109038751743,
+STORE, 140109038751744, 140109038759935,
+SNULL, 140109041934335, 140109041938431,
+STORE, 140109041930240, 140109041934335,
+STORE, 140109041934336, 140109041938431,
+SNULL, 7233535, 7241727,
+STORE, 7221248, 7233535,
+STORE, 7233536, 7241727,
+SNULL, 140109044183039, 140109044187135,
+STORE, 140109044178944, 140109044183039,
+STORE, 140109044183040, 140109044187135,
+ERASE, 140109044150272, 140109044178943,
+STORE, 20000768, 20135935,
+STORE, 20000768, 20283391,
+STORE, 140109042446336, 140109044129791,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140730853408768, 140737488351231,
+SNULL, 140730853416959, 140737488351231,
+STORE, 140730853408768, 140730853416959,
+STORE, 140730853277696, 140730853416959,
+STORE, 94865902977024, 94865905311743,
+SNULL, 94865903190015, 94865905311743,
+STORE, 94865902977024, 94865903190015,
+STORE, 94865903190016, 94865905311743,
+ERASE, 94865903190016, 94865905311743,
+STORE, 94865905287168, 94865905299455,
+STORE, 94865905299456, 94865905311743,
+STORE, 139768865738752, 139768867991551,
+SNULL, 139768865882111, 139768867991551,
+STORE, 139768865738752, 139768865882111,
+STORE, 139768865882112, 139768867991551,
+ERASE, 139768865882112, 139768867991551,
+STORE, 139768867979264, 139768867987455,
+STORE, 139768867987456, 139768867991551,
+STORE, 140730853957632, 140730853961727,
+STORE, 140730853945344, 140730853957631,
+STORE, 139768867950592, 139768867979263,
+STORE, 139768867942400, 139768867950591,
+STORE, 139768863625216, 139768865738751,
+SNULL, 139768863625216, 139768863637503,
+STORE, 139768863637504, 139768865738751,
+STORE, 139768863625216, 139768863637503,
+SNULL, 139768865730559, 139768865738751,
+STORE, 139768863637504, 139768865730559,
+STORE, 139768865730560, 139768865738751,
+ERASE, 139768865730560, 139768865738751,
+STORE, 139768865730560, 139768865738751,
+STORE, 139768859828224, 139768863625215,
+SNULL, 139768859828224, 139768861487103,
+STORE, 139768861487104, 139768863625215,
+STORE, 139768859828224, 139768861487103,
+SNULL, 139768863584255, 139768863625215,
+STORE, 139768861487104, 139768863584255,
+STORE, 139768863584256, 139768863625215,
+SNULL, 139768863584256, 139768863608831,
+STORE, 139768863608832, 139768863625215,
+STORE, 139768863584256, 139768863608831,
+ERASE, 139768863584256, 139768863608831,
+STORE, 139768863584256, 139768863608831,
+ERASE, 139768863608832, 139768863625215,
+STORE, 139768863608832, 139768863625215,
+STORE, 139768867934208, 139768867950591,
+SNULL, 139768863600639, 139768863608831,
+STORE, 139768863584256, 139768863600639,
+STORE, 139768863600640, 139768863608831,
+SNULL, 139768865734655, 139768865738751,
+STORE, 139768865730560, 139768865734655,
+STORE, 139768865734656, 139768865738751,
+SNULL, 94865905291263, 94865905299455,
+STORE, 94865905287168, 94865905291263,
+STORE, 94865905291264, 94865905299455,
+SNULL, 139768867983359, 139768867987455,
+STORE, 139768867979264, 139768867983359,
+STORE, 139768867983360, 139768867987455,
+ERASE, 139768867950592, 139768867979263,
+STORE, 94865923670016, 94865923805183,
+STORE, 139768866250752, 139768867934207,
+STORE, 94865923670016, 94865923940351,
+STORE, 94865923670016, 94865924075519,
+STORE, 94865923670016, 94865924222975,
+SNULL, 94865924210687, 94865924222975,
+STORE, 94865923670016, 94865924210687,
+STORE, 94865924210688, 94865924222975,
+ERASE, 94865924210688, 94865924222975,
+STORE, 94865923670016, 94865924349951,
+STORE, 94865923670016, 94865924493311,
+STORE, 94865923670016, 94865924640767,
+SNULL, 94865924603903, 94865924640767,
+STORE, 94865923670016, 94865924603903,
+STORE, 94865924603904, 94865924640767,
+ERASE, 94865924603904, 94865924640767,
+STORE, 94865923670016, 94865924747263,
+STORE, 94865923670016, 94865924898815,
+SNULL, 94865924874239, 94865924898815,
+STORE, 94865923670016, 94865924874239,
+STORE, 94865924874240, 94865924898815,
+ERASE, 94865924874240, 94865924898815,
+STORE, 94865923670016, 94865925025791,
+SNULL, 94865925013503, 94865925025791,
+STORE, 94865923670016, 94865925013503,
+STORE, 94865925013504, 94865925025791,
+ERASE, 94865925013504, 94865925025791,
+SNULL, 94865924988927, 94865925013503,
+STORE, 94865923670016, 94865924988927,
+STORE, 94865924988928, 94865925013503,
+ERASE, 94865924988928, 94865925013503,
+STORE, 94865923670016, 94865925152767,
+SNULL, 94865925136383, 94865925152767,
+STORE, 94865923670016, 94865925136383,
+STORE, 94865925136384, 94865925152767,
+ERASE, 94865925136384, 94865925152767,
+STORE, 94865923670016, 94865925292031,
+SNULL, 94865925279743, 94865925292031,
+STORE, 94865923670016, 94865925279743,
+STORE, 94865925279744, 94865925292031,
+ERASE, 94865925279744, 94865925292031,
+SNULL, 94865925255167, 94865925279743,
+STORE, 94865923670016, 94865925255167,
+STORE, 94865925255168, 94865925279743,
+ERASE, 94865925255168, 94865925279743,
+STORE, 94865923670016, 94865925406719,
+SNULL, 94865925394431, 94865925406719,
+STORE, 94865923670016, 94865925394431,
+STORE, 94865925394432, 94865925406719,
+ERASE, 94865925394432, 94865925406719,
+STORE, 94865923670016, 94865925545983,
+SNULL, 94865925533695, 94865925545983,
+STORE, 94865923670016, 94865925533695,
+STORE, 94865925533696, 94865925545983,
+ERASE, 94865925533696, 94865925545983,
+SNULL, 94865925492735, 94865925533695,
+STORE, 94865923670016, 94865925492735,
+STORE, 94865925492736, 94865925533695,
+ERASE, 94865925492736, 94865925533695,
+STORE, 94865923670016, 94865925627903,
+SNULL, 94865925599231, 94865925627903,
+STORE, 94865923670016, 94865925599231,
+STORE, 94865925599232, 94865925627903,
+ERASE, 94865925599232, 94865925627903,
+STORE, 94865923670016, 94865925738495,
+SNULL, 94865925726207, 94865925738495,
+STORE, 94865923670016, 94865925726207,
+STORE, 94865925726208, 94865925738495,
+ERASE, 94865925726208, 94865925738495,
+STORE, 94865923670016, 94865925877759,
+SNULL, 94865925865471, 94865925877759,
+STORE, 94865923670016, 94865925865471,
+STORE, 94865925865472, 94865925877759,
+ERASE, 94865925865472, 94865925877759,
+STORE, 94865923670016, 94865926021119,
+SNULL, 94865926008831, 94865926021119,
+STORE, 94865923670016, 94865926008831,
+STORE, 94865926008832, 94865926021119,
+ERASE, 94865926008832, 94865926021119,
+SNULL, 94865925971967, 94865926008831,
+STORE, 94865923670016, 94865925971967,
+STORE, 94865925971968, 94865926008831,
+ERASE, 94865925971968, 94865926008831,
+STORE, 94865923670016, 94865926115327,
+STORE, 94865923670016, 94865926254591,
+SNULL, 94865926246399, 94865926254591,
+STORE, 94865923670016, 94865926246399,
+STORE, 94865926246400, 94865926254591,
+ERASE, 94865926246400, 94865926254591,
+STORE, 94865923670016, 94865926385663,
+STORE, 94865923670016, 94865926537215,
+STORE, 94865923670016, 94865926672383,
+STORE, 94865923670016, 94865926815743,
+STORE, 94865923670016, 94865926955007,
+STORE, 94865923670016, 94865927094271,
+STORE, 94865923670016, 94865927233535,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140731148435456, 140737488351231,
+SNULL, 140731148443647, 140737488351231,
+STORE, 140731148435456, 140731148443647,
+STORE, 140731148304384, 140731148443647,
+STORE, 94090775400448, 94090777735167,
+SNULL, 94090775613439, 94090777735167,
+STORE, 94090775400448, 94090775613439,
+STORE, 94090775613440, 94090777735167,
+ERASE, 94090775613440, 94090777735167,
+STORE, 94090777710592, 94090777722879,
+STORE, 94090777722880, 94090777735167,
+STORE, 140301090283520, 140301092536319,
+SNULL, 140301090426879, 140301092536319,
+STORE, 140301090283520, 140301090426879,
+STORE, 140301090426880, 140301092536319,
+ERASE, 140301090426880, 140301092536319,
+STORE, 140301092524032, 140301092532223,
+STORE, 140301092532224, 140301092536319,
+STORE, 140731148570624, 140731148574719,
+STORE, 140731148558336, 140731148570623,
+STORE, 140301092495360, 140301092524031,
+STORE, 140301092487168, 140301092495359,
+STORE, 140301088169984, 140301090283519,
+SNULL, 140301088169984, 140301088182271,
+STORE, 140301088182272, 140301090283519,
+STORE, 140301088169984, 140301088182271,
+SNULL, 140301090275327, 140301090283519,
+STORE, 140301088182272, 140301090275327,
+STORE, 140301090275328, 140301090283519,
+ERASE, 140301090275328, 140301090283519,
+STORE, 140301090275328, 140301090283519,
+STORE, 140301084372992, 140301088169983,
+SNULL, 140301084372992, 140301086031871,
+STORE, 140301086031872, 140301088169983,
+STORE, 140301084372992, 140301086031871,
+SNULL, 140301088129023, 140301088169983,
+STORE, 140301086031872, 140301088129023,
+STORE, 140301088129024, 140301088169983,
+SNULL, 140301088129024, 140301088153599,
+STORE, 140301088153600, 140301088169983,
+STORE, 140301088129024, 140301088153599,
+ERASE, 140301088129024, 140301088153599,
+STORE, 140301088129024, 140301088153599,
+ERASE, 140301088153600, 140301088169983,
+STORE, 140301088153600, 140301088169983,
+STORE, 140301092478976, 140301092495359,
+SNULL, 140301088145407, 140301088153599,
+STORE, 140301088129024, 140301088145407,
+STORE, 140301088145408, 140301088153599,
+SNULL, 140301090279423, 140301090283519,
+STORE, 140301090275328, 140301090279423,
+STORE, 140301090279424, 140301090283519,
+SNULL, 94090777714687, 94090777722879,
+STORE, 94090777710592, 94090777714687,
+STORE, 94090777714688, 94090777722879,
+SNULL, 140301092528127, 140301092532223,
+STORE, 140301092524032, 140301092528127,
+STORE, 140301092528128, 140301092532223,
+ERASE, 140301092495360, 140301092524031,
+STORE, 94090794590208, 94090794725375,
+STORE, 140301090795520, 140301092478975,
+STORE, 94090794590208, 94090794860543,
+STORE, 94090794590208, 94090794995711,
+STORE, 94090794590208, 94090795163647,
+SNULL, 94090795139071, 94090795163647,
+STORE, 94090794590208, 94090795139071,
+STORE, 94090795139072, 94090795163647,
+ERASE, 94090795139072, 94090795163647,
+STORE, 94090794590208, 94090795278335,
+STORE, 94090794590208, 94090795425791,
+SNULL, 94090795388927, 94090795425791,
+STORE, 94090794590208, 94090795388927,
+STORE, 94090795388928, 94090795425791,
+ERASE, 94090795388928, 94090795425791,
+STORE, 94090794590208, 94090795528191,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733084430336, 140737488351231,
+SNULL, 140733084438527, 140737488351231,
+STORE, 140733084430336, 140733084438527,
+STORE, 140733084299264, 140733084438527,
+STORE, 94116169183232, 94116171517951,
+SNULL, 94116169396223, 94116171517951,
+STORE, 94116169183232, 94116169396223,
+STORE, 94116169396224, 94116171517951,
+ERASE, 94116169396224, 94116171517951,
+STORE, 94116171493376, 94116171505663,
+STORE, 94116171505664, 94116171517951,
+STORE, 139772214128640, 139772216381439,
+SNULL, 139772214271999, 139772216381439,
+STORE, 139772214128640, 139772214271999,
+STORE, 139772214272000, 139772216381439,
+ERASE, 139772214272000, 139772216381439,
+STORE, 139772216369152, 139772216377343,
+STORE, 139772216377344, 139772216381439,
+STORE, 140733085270016, 140733085274111,
+STORE, 140733085257728, 140733085270015,
+STORE, 139772216340480, 139772216369151,
+STORE, 139772216332288, 139772216340479,
+STORE, 139772212015104, 139772214128639,
+SNULL, 139772212015104, 139772212027391,
+STORE, 139772212027392, 139772214128639,
+STORE, 139772212015104, 139772212027391,
+SNULL, 139772214120447, 139772214128639,
+STORE, 139772212027392, 139772214120447,
+STORE, 139772214120448, 139772214128639,
+ERASE, 139772214120448, 139772214128639,
+STORE, 139772214120448, 139772214128639,
+STORE, 139772208218112, 139772212015103,
+SNULL, 139772208218112, 139772209876991,
+STORE, 139772209876992, 139772212015103,
+STORE, 139772208218112, 139772209876991,
+SNULL, 139772211974143, 139772212015103,
+STORE, 139772209876992, 139772211974143,
+STORE, 139772211974144, 139772212015103,
+SNULL, 139772211974144, 139772211998719,
+STORE, 139772211998720, 139772212015103,
+STORE, 139772211974144, 139772211998719,
+ERASE, 139772211974144, 139772211998719,
+STORE, 139772211974144, 139772211998719,
+ERASE, 139772211998720, 139772212015103,
+STORE, 139772211998720, 139772212015103,
+STORE, 139772216324096, 139772216340479,
+SNULL, 139772211990527, 139772211998719,
+STORE, 139772211974144, 139772211990527,
+STORE, 139772211990528, 139772211998719,
+SNULL, 139772214124543, 139772214128639,
+STORE, 139772214120448, 139772214124543,
+STORE, 139772214124544, 139772214128639,
+SNULL, 94116171497471, 94116171505663,
+STORE, 94116171493376, 94116171497471,
+STORE, 94116171497472, 94116171505663,
+SNULL, 139772216373247, 139772216377343,
+STORE, 139772216369152, 139772216373247,
+STORE, 139772216373248, 139772216377343,
+ERASE, 139772216340480, 139772216369151,
+STORE, 94116199383040, 94116199518207,
+STORE, 139772214640640, 139772216324095,
+STORE, 94116199383040, 94116199653375,
+STORE, 94116199383040, 94116199788543,
+STORE, 140737488347136, 140737488351231,
+STORE, 140726067826688, 140737488351231,
+SNULL, 140726067830783, 140737488351231,
+STORE, 140726067826688, 140726067830783,
+STORE, 140726067695616, 140726067830783,
+STORE, 94535150673920, 94535152898047,
+SNULL, 94535150784511, 94535152898047,
+STORE, 94535150673920, 94535150784511,
+STORE, 94535150784512, 94535152898047,
+ERASE, 94535150784512, 94535152898047,
+STORE, 94535152877568, 94535152889855,
+STORE, 94535152889856, 94535152898047,
+STORE, 140381257314304, 140381259567103,
+SNULL, 140381257457663, 140381259567103,
+STORE, 140381257314304, 140381257457663,
+STORE, 140381257457664, 140381259567103,
+ERASE, 140381257457664, 140381259567103,
+STORE, 140381259554816, 140381259563007,
+STORE, 140381259563008, 140381259567103,
+STORE, 140726068060160, 140726068064255,
+STORE, 140726068047872, 140726068060159,
+STORE, 140381259526144, 140381259554815,
+STORE, 140381259517952, 140381259526143,
+STORE, 140381253517312, 140381257314303,
+SNULL, 140381253517312, 140381255176191,
+STORE, 140381255176192, 140381257314303,
+STORE, 140381253517312, 140381255176191,
+SNULL, 140381257273343, 140381257314303,
+STORE, 140381255176192, 140381257273343,
+STORE, 140381257273344, 140381257314303,
+SNULL, 140381257273344, 140381257297919,
+STORE, 140381257297920, 140381257314303,
+STORE, 140381257273344, 140381257297919,
+ERASE, 140381257273344, 140381257297919,
+STORE, 140381257273344, 140381257297919,
+ERASE, 140381257297920, 140381257314303,
+STORE, 140381257297920, 140381257314303,
+SNULL, 140381257289727, 140381257297919,
+STORE, 140381257273344, 140381257289727,
+STORE, 140381257289728, 140381257297919,
+SNULL, 94535152885759, 94535152889855,
+STORE, 94535152877568, 94535152885759,
+STORE, 94535152885760, 94535152889855,
+SNULL, 140381259558911, 140381259563007,
+STORE, 140381259554816, 140381259558911,
+STORE, 140381259558912, 140381259563007,
+ERASE, 140381259526144, 140381259554815,
+STORE, 94535186296832, 94535186431999,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140729189425152, 140737488351231,
+SNULL, 140729189433343, 140737488351231,
+STORE, 140729189425152, 140729189433343,
+STORE, 140729189294080, 140729189433343,
+STORE, 94428200128512, 94428202352639,
+SNULL, 94428200239103, 94428202352639,
+STORE, 94428200128512, 94428200239103,
+STORE, 94428200239104, 94428202352639,
+ERASE, 94428200239104, 94428202352639,
+STORE, 94428202332160, 94428202344447,
+STORE, 94428202344448, 94428202352639,
+STORE, 139707216986112, 139707219238911,
+SNULL, 139707217129471, 139707219238911,
+STORE, 139707216986112, 139707217129471,
+STORE, 139707217129472, 139707219238911,
+ERASE, 139707217129472, 139707219238911,
+STORE, 139707219226624, 139707219234815,
+STORE, 139707219234816, 139707219238911,
+STORE, 140729189785600, 140729189789695,
+STORE, 140729189773312, 140729189785599,
+STORE, 139707219197952, 139707219226623,
+STORE, 139707219189760, 139707219197951,
+STORE, 139707213189120, 139707216986111,
+SNULL, 139707213189120, 139707214847999,
+STORE, 139707214848000, 139707216986111,
+STORE, 139707213189120, 139707214847999,
+SNULL, 139707216945151, 139707216986111,
+STORE, 139707214848000, 139707216945151,
+STORE, 139707216945152, 139707216986111,
+SNULL, 139707216945152, 139707216969727,
+STORE, 139707216969728, 139707216986111,
+STORE, 139707216945152, 139707216969727,
+ERASE, 139707216945152, 139707216969727,
+STORE, 139707216945152, 139707216969727,
+ERASE, 139707216969728, 139707216986111,
+STORE, 139707216969728, 139707216986111,
+SNULL, 139707216961535, 139707216969727,
+STORE, 139707216945152, 139707216961535,
+STORE, 139707216961536, 139707216969727,
+SNULL, 94428202340351, 94428202344447,
+STORE, 94428202332160, 94428202340351,
+STORE, 94428202340352, 94428202344447,
+SNULL, 139707219230719, 139707219234815,
+STORE, 139707219226624, 139707219230719,
+STORE, 139707219230720, 139707219234815,
+ERASE, 139707219197952, 139707219226623,
+STORE, 94428208599040, 94428208734207,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722000953344, 140737488351231,
+SNULL, 140722000961535, 140737488351231,
+STORE, 140722000953344, 140722000961535,
+STORE, 140722000822272, 140722000961535,
+STORE, 94636494757888, 94636496982015,
+SNULL, 94636494868479, 94636496982015,
+STORE, 94636494757888, 94636494868479,
+STORE, 94636494868480, 94636496982015,
+ERASE, 94636494868480, 94636496982015,
+STORE, 94636496961536, 94636496973823,
+STORE, 94636496973824, 94636496982015,
+STORE, 140142275100672, 140142277353471,
+SNULL, 140142275244031, 140142277353471,
+STORE, 140142275100672, 140142275244031,
+STORE, 140142275244032, 140142277353471,
+ERASE, 140142275244032, 140142277353471,
+STORE, 140142277341184, 140142277349375,
+STORE, 140142277349376, 140142277353471,
+STORE, 140722002747392, 140722002751487,
+STORE, 140722002735104, 140722002747391,
+STORE, 140142277312512, 140142277341183,
+STORE, 140142277304320, 140142277312511,
+STORE, 140142271303680, 140142275100671,
+SNULL, 140142271303680, 140142272962559,
+STORE, 140142272962560, 140142275100671,
+STORE, 140142271303680, 140142272962559,
+SNULL, 140142275059711, 140142275100671,
+STORE, 140142272962560, 140142275059711,
+STORE, 140142275059712, 140142275100671,
+SNULL, 140142275059712, 140142275084287,
+STORE, 140142275084288, 140142275100671,
+STORE, 140142275059712, 140142275084287,
+ERASE, 140142275059712, 140142275084287,
+STORE, 140142275059712, 140142275084287,
+ERASE, 140142275084288, 140142275100671,
+STORE, 140142275084288, 140142275100671,
+SNULL, 140142275076095, 140142275084287,
+STORE, 140142275059712, 140142275076095,
+STORE, 140142275076096, 140142275084287,
+SNULL, 94636496969727, 94636496973823,
+STORE, 94636496961536, 94636496969727,
+STORE, 94636496969728, 94636496973823,
+SNULL, 140142277345279, 140142277349375,
+STORE, 140142277341184, 140142277345279,
+STORE, 140142277345280, 140142277349375,
+ERASE, 140142277312512, 140142277341183,
+STORE, 94636516286464, 94636516421631,
+STORE, 94071103692800, 94071103905791,
+STORE, 94071106002944, 94071106007039,
+STORE, 94071106007040, 94071106015231,
+STORE, 94071106015232, 94071106027519,
+STORE, 94071138521088, 94071140368383,
+STORE, 140145668190208, 140145669849087,
+STORE, 140145669849088, 140145671946239,
+STORE, 140145671946240, 140145671962623,
+STORE, 140145671962624, 140145671970815,
+STORE, 140145671970816, 140145671987199,
+STORE, 140145671987200, 140145671999487,
+STORE, 140145671999488, 140145674092543,
+STORE, 140145674092544, 140145674096639,
+STORE, 140145674096640, 140145674100735,
+STORE, 140145674100736, 140145674244095,
+STORE, 140145674612736, 140145676296191,
+STORE, 140145676296192, 140145676312575,
+STORE, 140145676341248, 140145676345343,
+STORE, 140145676345344, 140145676349439,
+STORE, 140145676349440, 140145676353535,
+STORE, 140734927740928, 140734927880191,
+STORE, 140734928842752, 140734928855039,
+STORE, 140734928855040, 140734928859135,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722342535168, 140737488351231,
+SNULL, 140722342543359, 140737488351231,
+STORE, 140722342535168, 140722342543359,
+STORE, 140722342404096, 140722342543359,
+STORE, 94399699714048, 94399702048767,
+SNULL, 94399699927039, 94399702048767,
+STORE, 94399699714048, 94399699927039,
+STORE, 94399699927040, 94399702048767,
+ERASE, 94399699927040, 94399702048767,
+STORE, 94399702024192, 94399702036479,
+STORE, 94399702036480, 94399702048767,
+STORE, 139811024748544, 139811027001343,
+SNULL, 139811024891903, 139811027001343,
+STORE, 139811024748544, 139811024891903,
+STORE, 139811024891904, 139811027001343,
+ERASE, 139811024891904, 139811027001343,
+STORE, 139811026989056, 139811026997247,
+STORE, 139811026997248, 139811027001343,
+STORE, 140722342707200, 140722342711295,
+STORE, 140722342694912, 140722342707199,
+STORE, 139811026960384, 139811026989055,
+STORE, 139811026952192, 139811026960383,
+STORE, 139811022635008, 139811024748543,
+SNULL, 139811022635008, 139811022647295,
+STORE, 139811022647296, 139811024748543,
+STORE, 139811022635008, 139811022647295,
+SNULL, 139811024740351, 139811024748543,
+STORE, 139811022647296, 139811024740351,
+STORE, 139811024740352, 139811024748543,
+ERASE, 139811024740352, 139811024748543,
+STORE, 139811024740352, 139811024748543,
+STORE, 139811018838016, 139811022635007,
+SNULL, 139811018838016, 139811020496895,
+STORE, 139811020496896, 139811022635007,
+STORE, 139811018838016, 139811020496895,
+SNULL, 139811022594047, 139811022635007,
+STORE, 139811020496896, 139811022594047,
+STORE, 139811022594048, 139811022635007,
+SNULL, 139811022594048, 139811022618623,
+STORE, 139811022618624, 139811022635007,
+STORE, 139811022594048, 139811022618623,
+ERASE, 139811022594048, 139811022618623,
+STORE, 139811022594048, 139811022618623,
+ERASE, 139811022618624, 139811022635007,
+STORE, 139811022618624, 139811022635007,
+STORE, 139811026944000, 139811026960383,
+SNULL, 139811022610431, 139811022618623,
+STORE, 139811022594048, 139811022610431,
+STORE, 139811022610432, 139811022618623,
+SNULL, 139811024744447, 139811024748543,
+STORE, 139811024740352, 139811024744447,
+STORE, 139811024744448, 139811024748543,
+SNULL, 94399702028287, 94399702036479,
+STORE, 94399702024192, 94399702028287,
+STORE, 94399702028288, 94399702036479,
+SNULL, 139811026993151, 139811026997247,
+STORE, 139811026989056, 139811026993151,
+STORE, 139811026993152, 139811026997247,
+ERASE, 139811026960384, 139811026989055,
+STORE, 94399723880448, 94399724015615,
+STORE, 139811025260544, 139811026943999,
+STORE, 94399723880448, 94399724150783,
+STORE, 94399723880448, 94399724285951,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140735364939776, 140737488351231,
+SNULL, 140735364947967, 140737488351231,
+STORE, 140735364939776, 140735364947967,
+STORE, 140735364808704, 140735364947967,
+STORE, 94421528674304, 94421531009023,
+SNULL, 94421528887295, 94421531009023,
+STORE, 94421528674304, 94421528887295,
+STORE, 94421528887296, 94421531009023,
+ERASE, 94421528887296, 94421531009023,
+STORE, 94421530984448, 94421530996735,
+STORE, 94421530996736, 94421531009023,
+STORE, 140162004742144, 140162006994943,
+SNULL, 140162004885503, 140162006994943,
+STORE, 140162004742144, 140162004885503,
+STORE, 140162004885504, 140162006994943,
+ERASE, 140162004885504, 140162006994943,
+STORE, 140162006982656, 140162006990847,
+STORE, 140162006990848, 140162006994943,
+STORE, 140735365402624, 140735365406719,
+STORE, 140735365390336, 140735365402623,
+STORE, 140162006953984, 140162006982655,
+STORE, 140162006945792, 140162006953983,
+STORE, 140162002628608, 140162004742143,
+SNULL, 140162002628608, 140162002640895,
+STORE, 140162002640896, 140162004742143,
+STORE, 140162002628608, 140162002640895,
+SNULL, 140162004733951, 140162004742143,
+STORE, 140162002640896, 140162004733951,
+STORE, 140162004733952, 140162004742143,
+ERASE, 140162004733952, 140162004742143,
+STORE, 140162004733952, 140162004742143,
+STORE, 140161998831616, 140162002628607,
+SNULL, 140161998831616, 140162000490495,
+STORE, 140162000490496, 140162002628607,
+STORE, 140161998831616, 140162000490495,
+SNULL, 140162002587647, 140162002628607,
+STORE, 140162000490496, 140162002587647,
+STORE, 140162002587648, 140162002628607,
+SNULL, 140162002587648, 140162002612223,
+STORE, 140162002612224, 140162002628607,
+STORE, 140162002587648, 140162002612223,
+ERASE, 140162002587648, 140162002612223,
+STORE, 140162002587648, 140162002612223,
+ERASE, 140162002612224, 140162002628607,
+STORE, 140162002612224, 140162002628607,
+STORE, 140162006937600, 140162006953983,
+SNULL, 140162002604031, 140162002612223,
+STORE, 140162002587648, 140162002604031,
+STORE, 140162002604032, 140162002612223,
+SNULL, 140162004738047, 140162004742143,
+STORE, 140162004733952, 140162004738047,
+STORE, 140162004738048, 140162004742143,
+SNULL, 94421530988543, 94421530996735,
+STORE, 94421530984448, 94421530988543,
+STORE, 94421530988544, 94421530996735,
+SNULL, 140162006986751, 140162006990847,
+STORE, 140162006982656, 140162006986751,
+STORE, 140162006986752, 140162006990847,
+ERASE, 140162006953984, 140162006982655,
+STORE, 94421551697920, 94421551833087,
+STORE, 140162005254144, 140162006937599,
+STORE, 94421551697920, 94421551968255,
+STORE, 94421551697920, 94421552103423,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140733498486784, 140737488351231,
+SNULL, 140733498494975, 140737488351231,
+STORE, 140733498486784, 140733498494975,
+STORE, 140733498355712, 140733498494975,
+STORE, 94567985836032, 94567988170751,
+SNULL, 94567986049023, 94567988170751,
+STORE, 94567985836032, 94567986049023,
+STORE, 94567986049024, 94567988170751,
+ERASE, 94567986049024, 94567988170751,
+STORE, 94567988146176, 94567988158463,
+STORE, 94567988158464, 94567988170751,
+STORE, 139634278572032, 139634280824831,
+SNULL, 139634278715391, 139634280824831,
+STORE, 139634278572032, 139634278715391,
+STORE, 139634278715392, 139634280824831,
+ERASE, 139634278715392, 139634280824831,
+STORE, 139634280812544, 139634280820735,
+STORE, 139634280820736, 139634280824831,
+STORE, 140733498544128, 140733498548223,
+STORE, 140733498531840, 140733498544127,
+STORE, 139634280783872, 139634280812543,
+STORE, 139634280775680, 139634280783871,
+STORE, 139634276458496, 139634278572031,
+SNULL, 139634276458496, 139634276470783,
+STORE, 139634276470784, 139634278572031,
+STORE, 139634276458496, 139634276470783,
+SNULL, 139634278563839, 139634278572031,
+STORE, 139634276470784, 139634278563839,
+STORE, 139634278563840, 139634278572031,
+ERASE, 139634278563840, 139634278572031,
+STORE, 139634278563840, 139634278572031,
+STORE, 139634272661504, 139634276458495,
+SNULL, 139634272661504, 139634274320383,
+STORE, 139634274320384, 139634276458495,
+STORE, 139634272661504, 139634274320383,
+SNULL, 139634276417535, 139634276458495,
+STORE, 139634274320384, 139634276417535,
+STORE, 139634276417536, 139634276458495,
+SNULL, 139634276417536, 139634276442111,
+STORE, 139634276442112, 139634276458495,
+STORE, 139634276417536, 139634276442111,
+ERASE, 139634276417536, 139634276442111,
+STORE, 139634276417536, 139634276442111,
+ERASE, 139634276442112, 139634276458495,
+STORE, 139634276442112, 139634276458495,
+STORE, 139634280767488, 139634280783871,
+SNULL, 139634276433919, 139634276442111,
+STORE, 139634276417536, 139634276433919,
+STORE, 139634276433920, 139634276442111,
+SNULL, 139634278567935, 139634278572031,
+STORE, 139634278563840, 139634278567935,
+STORE, 139634278567936, 139634278572031,
+SNULL, 94567988150271, 94567988158463,
+STORE, 94567988146176, 94567988150271,
+STORE, 94567988150272, 94567988158463,
+SNULL, 139634280816639, 139634280820735,
+STORE, 139634280812544, 139634280816639,
+STORE, 139634280816640, 139634280820735,
+ERASE, 139634280783872, 139634280812543,
+STORE, 94567996379136, 94567996514303,
+STORE, 139634279084032, 139634280767487,
+STORE, 94567996379136, 94567996649471,
+STORE, 94567996379136, 94567996784639,
+STORE, 94567996379136, 94567996960767,
+SNULL, 94567996932095, 94567996960767,
+STORE, 94567996379136, 94567996932095,
+STORE, 94567996932096, 94567996960767,
+ERASE, 94567996932096, 94567996960767,
+STORE, 94567996379136, 94567997071359,
+STORE, 94567996379136, 94567997206527,
+SNULL, 94567997186047, 94567997206527,
+STORE, 94567996379136, 94567997186047,
+STORE, 94567997186048, 94567997206527,
+ERASE, 94567997186048, 94567997206527,
+STORE, 94567996379136, 94567997358079,
+STORE, 94567996379136, 94567997493247,
+SNULL, 94567997476863, 94567997493247,
+STORE, 94567996379136, 94567997476863,
+STORE, 94567997476864, 94567997493247,
+ERASE, 94567997476864, 94567997493247,
+STORE, 94567996379136, 94567997612031,
+STORE, 94567996379136, 94567997767679,
+SNULL, 94567997739007, 94567997767679,
+STORE, 94567996379136, 94567997739007,
+STORE, 94567997739008, 94567997767679,
+ERASE, 94567997739008, 94567997767679,
+SNULL, 94567997698047, 94567997739007,
+STORE, 94567996379136, 94567997698047,
+STORE, 94567997698048, 94567997739007,
+ERASE, 94567997698048, 94567997739007,
+STORE, 94567996379136, 94567997853695,
+STORE, 94567996379136, 94567997988863,
+STORE, 94567996379136, 94567998132223,
+STORE, 94567996379136, 94567998275583,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140723667759104, 140737488351231,
+SNULL, 140723667767295, 140737488351231,
+STORE, 140723667759104, 140723667767295,
+STORE, 140723667628032, 140723667767295,
+STORE, 94231598800896, 94231601135615,
+SNULL, 94231599013887, 94231601135615,
+STORE, 94231598800896, 94231599013887,
+STORE, 94231599013888, 94231601135615,
+ERASE, 94231599013888, 94231601135615,
+STORE, 94231601111040, 94231601123327,
+STORE, 94231601123328, 94231601135615,
+STORE, 140269472649216, 140269474902015,
+SNULL, 140269472792575, 140269474902015,
+STORE, 140269472649216, 140269472792575,
+STORE, 140269472792576, 140269474902015,
+ERASE, 140269472792576, 140269474902015,
+STORE, 140269474889728, 140269474897919,
+STORE, 140269474897920, 140269474902015,
+STORE, 140723667836928, 140723667841023,
+STORE, 140723667824640, 140723667836927,
+STORE, 140269474861056, 140269474889727,
+STORE, 140269474852864, 140269474861055,
+STORE, 140269470535680, 140269472649215,
+SNULL, 140269470535680, 140269470547967,
+STORE, 140269470547968, 140269472649215,
+STORE, 140269470535680, 140269470547967,
+SNULL, 140269472641023, 140269472649215,
+STORE, 140269470547968, 140269472641023,
+STORE, 140269472641024, 140269472649215,
+ERASE, 140269472641024, 140269472649215,
+STORE, 140269472641024, 140269472649215,
+STORE, 140269466738688, 140269470535679,
+SNULL, 140269466738688, 140269468397567,
+STORE, 140269468397568, 140269470535679,
+STORE, 140269466738688, 140269468397567,
+SNULL, 140269470494719, 140269470535679,
+STORE, 140269468397568, 140269470494719,
+STORE, 140269470494720, 140269470535679,
+SNULL, 140269470494720, 140269470519295,
+STORE, 140269470519296, 140269470535679,
+STORE, 140269470494720, 140269470519295,
+ERASE, 140269470494720, 140269470519295,
+STORE, 140269470494720, 140269470519295,
+ERASE, 140269470519296, 140269470535679,
+STORE, 140269470519296, 140269470535679,
+STORE, 140269474844672, 140269474861055,
+SNULL, 140269470511103, 140269470519295,
+STORE, 140269470494720, 140269470511103,
+STORE, 140269470511104, 140269470519295,
+SNULL, 140269472645119, 140269472649215,
+STORE, 140269472641024, 140269472645119,
+STORE, 140269472645120, 140269472649215,
+SNULL, 94231601115135, 94231601123327,
+STORE, 94231601111040, 94231601115135,
+STORE, 94231601115136, 94231601123327,
+SNULL, 140269474893823, 140269474897919,
+STORE, 140269474889728, 140269474893823,
+STORE, 140269474893824, 140269474897919,
+ERASE, 140269474861056, 140269474889727,
+STORE, 94231626592256, 94231626727423,
+STORE, 140269473161216, 140269474844671,
+STORE, 94231626592256, 94231626862591,
+STORE, 94231626592256, 94231626997759,
+STORE, 94327178862592, 94327179075583,
+STORE, 94327181172736, 94327181176831,
+STORE, 94327181176832, 94327181185023,
+STORE, 94327181185024, 94327181197311,
+STORE, 94327185715200, 94327186685951,
+STORE, 140172071755776, 140172073414655,
+STORE, 140172073414656, 140172075511807,
+STORE, 140172075511808, 140172075528191,
+STORE, 140172075528192, 140172075536383,
+STORE, 140172075536384, 140172075552767,
+STORE, 140172075552768, 140172075565055,
+STORE, 140172075565056, 140172077658111,
+STORE, 140172077658112, 140172077662207,
+STORE, 140172077662208, 140172077666303,
+STORE, 140172077666304, 140172077809663,
+STORE, 140172078178304, 140172079861759,
+STORE, 140172079861760, 140172079878143,
+STORE, 140172079878144, 140172079906815,
+STORE, 140172079906816, 140172079910911,
+STORE, 140172079910912, 140172079915007,
+STORE, 140172079915008, 140172079919103,
+STORE, 140720358359040, 140720358494207,
+STORE, 140720358498304, 140720358510591,
+STORE, 140720358510592, 140720358514687,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140722548621312, 140737488351231,
+SNULL, 140722548629503, 140737488351231,
+STORE, 140722548621312, 140722548629503,
+STORE, 140722548490240, 140722548629503,
+STORE, 93949289504768, 93949291728895,
+SNULL, 93949289615359, 93949291728895,
+STORE, 93949289504768, 93949289615359,
+STORE, 93949289615360, 93949291728895,
+ERASE, 93949289615360, 93949291728895,
+STORE, 93949291708416, 93949291720703,
+STORE, 93949291720704, 93949291728895,
+STORE, 140305861902336, 140305864155135,
+SNULL, 140305862045695, 140305864155135,
+STORE, 140305861902336, 140305862045695,
+STORE, 140305862045696, 140305864155135,
+ERASE, 140305862045696, 140305864155135,
+STORE, 140305864142848, 140305864151039,
+STORE, 140305864151040, 140305864155135,
+STORE, 140722549821440, 140722549825535,
+STORE, 140722549809152, 140722549821439,
+STORE, 140305864114176, 140305864142847,
+STORE, 140305864105984, 140305864114175,
+STORE, 140305858105344, 140305861902335,
+SNULL, 140305858105344, 140305859764223,
+STORE, 140305859764224, 140305861902335,
+STORE, 140305858105344, 140305859764223,
+SNULL, 140305861861375, 140305861902335,
+STORE, 140305859764224, 140305861861375,
+STORE, 140305861861376, 140305861902335,
+SNULL, 140305861861376, 140305861885951,
+STORE, 140305861885952, 140305861902335,
+STORE, 140305861861376, 140305861885951,
+ERASE, 140305861861376, 140305861885951,
+STORE, 140305861861376, 140305861885951,
+ERASE, 140305861885952, 140305861902335,
+STORE, 140305861885952, 140305861902335,
+SNULL, 140305861877759, 140305861885951,
+STORE, 140305861861376, 140305861877759,
+STORE, 140305861877760, 140305861885951,
+SNULL, 93949291716607, 93949291720703,
+STORE, 93949291708416, 93949291716607,
+STORE, 93949291716608, 93949291720703,
+SNULL, 140305864146943, 140305864151039,
+STORE, 140305864142848, 140305864146943,
+STORE, 140305864146944, 140305864151039,
+ERASE, 140305864114176, 140305864142847,
+STORE, 93949324136448, 93949324271615,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140725754908672, 140737488351231,
+SNULL, 140725754916863, 140737488351231,
+STORE, 140725754908672, 140725754916863,
+STORE, 140725754777600, 140725754916863,
+STORE, 94831184375808, 94831186599935,
+SNULL, 94831184486399, 94831186599935,
+STORE, 94831184375808, 94831184486399,
+STORE, 94831184486400, 94831186599935,
+ERASE, 94831184486400, 94831186599935,
+STORE, 94831186579456, 94831186591743,
+STORE, 94831186591744, 94831186599935,
+STORE, 140605482479616, 140605484732415,
+SNULL, 140605482622975, 140605484732415,
+STORE, 140605482479616, 140605482622975,
+STORE, 140605482622976, 140605484732415,
+ERASE, 140605482622976, 140605484732415,
+STORE, 140605484720128, 140605484728319,
+STORE, 140605484728320, 140605484732415,
+STORE, 140725755670528, 140725755674623,
+STORE, 140725755658240, 140725755670527,
+STORE, 140605484691456, 140605484720127,
+STORE, 140605484683264, 140605484691455,
+STORE, 140605478682624, 140605482479615,
+SNULL, 140605478682624, 140605480341503,
+STORE, 140605480341504, 140605482479615,
+STORE, 140605478682624, 140605480341503,
+SNULL, 140605482438655, 140605482479615,
+STORE, 140605480341504, 140605482438655,
+STORE, 140605482438656, 140605482479615,
+SNULL, 140605482438656, 140605482463231,
+STORE, 140605482463232, 140605482479615,
+STORE, 140605482438656, 140605482463231,
+ERASE, 140605482438656, 140605482463231,
+STORE, 140605482438656, 140605482463231,
+ERASE, 140605482463232, 140605482479615,
+STORE, 140605482463232, 140605482479615,
+SNULL, 140605482455039, 140605482463231,
+STORE, 140605482438656, 140605482455039,
+STORE, 140605482455040, 140605482463231,
+SNULL, 94831186587647, 94831186591743,
+STORE, 94831186579456, 94831186587647,
+STORE, 94831186587648, 94831186591743,
+SNULL, 140605484724223, 140605484728319,
+STORE, 140605484720128, 140605484724223,
+STORE, 140605484724224, 140605484728319,
+ERASE, 140605484691456, 140605484720127,
+STORE, 94831217156096, 94831217291263,
+STORE, 94327178862592, 94327179075583,
+STORE, 94327181172736, 94327181176831,
+STORE, 94327181176832, 94327181185023,
+STORE, 94327181185024, 94327181197311,
+STORE, 94327185715200, 94327186685951,
+STORE, 140172071755776, 140172073414655,
+STORE, 140172073414656, 140172075511807,
+STORE, 140172075511808, 140172075528191,
+STORE, 140172075528192, 140172075536383,
+STORE, 140172075536384, 140172075552767,
+STORE, 140172075552768, 140172075565055,
+STORE, 140172075565056, 140172077658111,
+STORE, 140172077658112, 140172077662207,
+STORE, 140172077662208, 140172077666303,
+STORE, 140172077666304, 140172077809663,
+STORE, 140172078178304, 140172079861759,
+STORE, 140172079861760, 140172079878143,
+STORE, 140172079878144, 140172079906815,
+STORE, 140172079906816, 140172079910911,
+STORE, 140172079910912, 140172079915007,
+STORE, 140172079915008, 140172079919103,
+STORE, 140720358359040, 140720358494207,
+STORE, 140720358498304, 140720358510591,
+STORE, 140720358510592, 140720358514687,
+STORE, 140737488347136, 140737488351231,
+STORE, 140737488343040, 140737488351231,
+STORE, 140737488338944, 140737488351231,
+STORE, 140734529933312, 140737488351231,
+SNULL, 140734529945599, 140737488351231,
+STORE, 140734529933312, 140734529945599,
+STORE, 140734529802240, 140734529945599,
+STORE, 4194304, 26279935,
+STORE, 28372992, 28454911,
+STORE, 28454912, 29806591,
+STORE, 140249744060416, 140249746313215,
+SNULL, 140249744203775, 140249746313215,
+STORE, 140249744060416, 140249744203775,
+STORE, 140249744203776, 140249746313215,
+ERASE, 140249744203776, 140249746313215,
+STORE, 140249746300928, 140249746309119,
+STORE, 140249746309120, 140249746313215,
+STORE, 140734530174976, 140734530179071,
+STORE, 140734530162688, 140734530174975,
+STORE, 140249746272256, 140249746300927,
+STORE, 140249746264064, 140249746272255,
+STORE, 140249740226560, 140249744060415,
+SNULL, 140249740226560, 140249741934591,
+STORE, 140249741934592, 140249744060415,
+STORE, 140249740226560, 140249741934591,
+SNULL, 140249744027647, 140249744060415,
+STORE, 140249741934592, 140249744027647,
+STORE, 140249744027648, 140249744060415,
+ERASE, 140249744027648, 140249744060415,
+STORE, 140249744027648, 140249744060415,
+STORE, 140249738031104, 140249740226559,
+SNULL, 140249738031104, 140249738125311,
+STORE, 140249738125312, 140249740226559,
+STORE, 140249738031104, 140249738125311,
+SNULL, 140249740218367, 140249740226559,
+STORE, 140249738125312, 140249740218367,
+STORE, 140249740218368, 140249740226559,
+ERASE, 140249740218368, 140249740226559,
+STORE, 140249740218368, 140249740226559,
+STORE, 140249735512064, 140249738031103,
+SNULL, 140249735512064, 140249735925759,
+STORE, 140249735925760, 140249738031103,
+STORE, 140249735512064, 140249735925759,
+SNULL, 140249738018815, 140249738031103,
+STORE, 140249735925760, 140249738018815,
+STORE, 140249738018816, 140249738031103,
+ERASE, 140249738018816, 140249738031103,
+STORE, 140249738018816, 140249738031103,
+STORE, 140249732878336, 140249735512063,
+SNULL, 140249732878336, 140249733406719,
+STORE, 140249733406720, 140249735512063,
+STORE, 140249732878336, 140249733406719,
+SNULL, 140249735503871, 140249735512063,
+STORE, 140249733406720, 140249735503871,
+STORE, 140249735503872, 140249735512063,
+ERASE, 140249735503872, 140249735512063,
+STORE, 140249735503872, 140249735512063,
+STORE, 140249730764800, 140249732878335,
+SNULL, 140249730764800, 140249730777087,
+STORE, 140249730777088, 140249732878335,
+STORE, 140249730764800, 140249730777087,
+SNULL, 140249732870143, 140249732878335,
+STORE, 140249730777088, 140249732870143,
+STORE, 140249732870144, 140249732878335,
+ERASE, 140249732870144, 140249732878335,
+STORE, 140249732870144, 140249732878335,
+STORE, 140249728561152, 140249730764799,
+SNULL, 140249728561152, 140249728663551,
+STORE, 140249728663552, 140249730764799,
+STORE, 140249728561152, 140249728663551,
+SNULL, 140249730756607, 140249730764799,
+STORE, 140249728663552, 140249730756607,
+STORE, 140249730756608, 140249730764799,
+ERASE, 140249730756608, 140249730764799,
+STORE, 140249730756608, 140249730764799,
+STORE, 140249746255872, 140249746272255,
+STORE, 140249725399040, 140249728561151,
+SNULL, 140249725399040, 140249726459903,
+STORE, 140249726459904, 140249728561151,
+STORE, 140249725399040, 140249726459903,
+SNULL, 140249728552959, 140249728561151,
+STORE, 140249726459904, 140249728552959,
+STORE, 140249728552960, 140249728561151,
+ERASE, 140249728552960, 140249728561151,
+STORE, 140249728552960, 140249728561151,
+STORE, 140249721602048, 140249725399039,
+SNULL, 140249721602048, 140249723260927,
+STORE, 140249723260928, 140249725399039,
+STORE, 140249721602048, 140249723260927,
+SNULL, 140249725358079, 140249725399039,
+STORE, 140249723260928, 140249725358079,
+STORE, 140249725358080, 140249725399039,
+SNULL, 140249725358080, 140249725382655,
+STORE, 140249725382656, 140249725399039,
+STORE, 140249725358080, 140249725382655,
+ERASE, 140249725358080, 140249725382655,
+STORE, 140249725358080, 140249725382655,
+ERASE, 140249725382656, 140249725399039,
+STORE, 140249725382656, 140249725399039,
+STORE, 140249746243584, 140249746272255,
+SNULL, 140249725374463, 140249725382655,
+STORE, 140249725358080, 140249725374463,
+STORE, 140249725374464, 140249725382655,
+SNULL, 140249728557055, 140249728561151,
+STORE, 140249728552960, 140249728557055,
+STORE, 140249728557056, 140249728561151,
+SNULL, 140249730760703, 140249730764799,
+STORE, 140249730756608, 140249730760703,
+STORE, 140249730760704, 140249730764799,
+SNULL, 140249732874239, 140249732878335,
+STORE, 140249732870144, 140249732874239,
+STORE, 140249732874240, 140249732878335,
+SNULL, 140249735507967, 140249735512063,
+STORE, 140249735503872, 140249735507967,
+STORE, 140249735507968, 140249735512063,
+SNULL, 140249738027007, 140249738031103,
+STORE, 140249738018816, 140249738027007,
+STORE, 140249738027008, 140249738031103,
+SNULL, 140249740222463, 140249740226559,
+STORE, 140249740218368, 140249740222463,
+STORE, 140249740222464, 140249740226559,
+SNULL, 140249744031743, 140249744060415,
+STORE, 140249744027648, 140249744031743,
+STORE, 140249744031744, 140249744060415,
+SNULL, 28405759, 28454911,
+STORE, 28372992, 28405759,
+STORE, 28405760, 28454911,
+SNULL, 140249746305023, 140249746309119,
+STORE, 140249746300928, 140249746305023,
+STORE, 140249746305024, 140249746309119,
+ERASE, 140249746272256, 140249746300927,
+STORE, 33853440, 33988607,
+STORE, 140249744560128, 140249746243583,
+STORE, 140249746296832, 140249746300927,
+STORE, 140249744424960, 140249744560127,
+STORE, 33853440, 34131967,
+STORE, 140249719504896, 140249721602047,
+STORE, 140249746288640, 140249746300927,
+STORE, 140249746280448, 140249746300927,
+STORE, 140249746243584, 140249746280447,
+STORE, 140249744408576, 140249744560127,
+STORE, 33853440, 34267135,
+STORE, 33853440, 34422783,
+STORE, 140249744400384, 140249744560127,
+STORE, 140249744392192, 140249744560127,
+STORE, 33853440, 34557951,
+STORE, 33853440, 34693119,
+STORE, 140249744375808, 140249744560127,
+STORE, 140249744367616, 140249744560127,
+STORE, 33853440, 34832383,
+STORE, 140249719230464, 140249721602047,
+STORE, 140249744207872, 140249744560127,
+STORE, 33853440, 34971647,
+SNULL, 34963455, 34971647,
+STORE, 33853440, 34963455,
+STORE, 34963456, 34971647,
+ERASE, 34963456, 34971647,
+SNULL, 34955263, 34963455,
+STORE, 33853440, 34955263,
+STORE, 34955264, 34963455,
+ERASE, 34955264, 34963455,
+SNULL, 34947071, 34955263,
+STORE, 33853440, 34947071,
+STORE, 34947072, 34955263,
+ERASE, 34947072, 34955263,
+SNULL, 34938879, 34947071,
+STORE, 33853440, 34938879,
+STORE, 34938880, 34947071,
+ERASE, 34938880, 34947071,
+STORE, 140249719214080, 140249721602047,
+STORE, 140249719148544, 140249721602047,
+STORE, 140249719115776, 140249721602047,
+STORE, 140249717018624, 140249721602047,
+STORE, 140249716953088, 140249721602047,
+STORE, 33853440, 35086335,
+STORE, 140249716822016, 140249721602047,
+STORE, 140249716559872, 140249721602047,
+STORE, 140249716551680, 140249721602047,
+STORE, 140249716535296, 140249721602047,
+STORE, 140249716527104, 140249721602047,
+STORE, 140249716518912, 140249721602047,
+STORE, 33853440, 35221503,
+SNULL, 35213311, 35221503,
+STORE, 33853440, 35213311,
+STORE, 35213312, 35221503,
+ERASE, 35213312, 35221503,
+SNULL, 35205119, 35213311,
+STORE, 33853440, 35205119,
+STORE, 35205120, 35213311,
+ERASE, 35205120, 35213311,
+SNULL, 35192831, 35205119,
+STORE, 33853440, 35192831,
+STORE, 35192832, 35205119,
+ERASE, 35192832, 35205119,
+SNULL, 35176447, 35192831,
+STORE, 33853440, 35176447,
+STORE, 35176448, 35192831,
+ERASE, 35176448, 35192831,
+STORE, 140249716502528, 140249721602047,
+STORE, 33853440, 35311615,
+SNULL, 35307519, 35311615,
+STORE, 33853440, 35307519,
+STORE, 35307520, 35311615,
+ERASE, 35307520, 35311615,
+SNULL, 35303423, 35307519,
+STORE, 33853440, 35303423,
+STORE, 35303424, 35307519,
+ERASE, 35303424, 35307519,
+SNULL, 35299327, 35303423,
+STORE, 33853440, 35299327,
+STORE, 35299328, 35303423,
+ERASE, 35299328, 35303423,
+SNULL, 35295231, 35299327,
+STORE, 33853440, 35295231,
+STORE, 35295232, 35299327,
+ERASE, 35295232, 35299327,
+SNULL, 35291135, 35295231,
+STORE, 33853440, 35291135,
+STORE, 35291136, 35295231,
+ERASE, 35291136, 35295231,
+SNULL, 35287039, 35291135,
+STORE, 33853440, 35287039,
+STORE, 35287040, 35291135,
+ERASE, 35287040, 35291135,
+SNULL, 35282943, 35287039,
+STORE, 33853440, 35282943,
+STORE, 35282944, 35287039,
+ERASE, 35282944, 35287039,
+STORE, 140249716486144, 140249721602047,
+STORE, 140249716453376, 140249721602047,
+STORE, 33853440, 35418111,
+SNULL, 35401727, 35418111,
+STORE, 33853440, 35401727,
+STORE, 35401728, 35418111,
+ERASE, 35401728, 35418111,
+SNULL, 35389439, 35401727,
+STORE, 33853440, 35389439,
+STORE, 35389440, 35401727,
+ERASE, 35389440, 35401727,
+STORE, 140249714356224, 140249721602047,
+STORE, 33853440, 35540991,
+STORE, 140249714339840, 140249721602047,
+STORE, 140249714077696, 140249721602047,
+STORE, 140249714069504, 140249721602047,
+STORE, 140249714061312, 140249721602047,
+STORE, 33853440, 35680255,
+SNULL, 35672063, 35680255,
+STORE, 33853440, 35672063,
+STORE, 35672064, 35680255,
+ERASE, 35672064, 35680255,
+SNULL, 35627007, 35672063,
+STORE, 33853440, 35627007,
+STORE, 35627008, 35672063,
+ERASE, 35627008, 35672063,
+STORE, 140249711964160, 140249721602047,
+STORE, 33853440, 35762175,
+SNULL, 35753983, 35762175,
+STORE, 33853440, 35753983,
+STORE, 35753984, 35762175,
+ERASE, 35753984, 35762175,
+SNULL, 35745791, 35753983,
+STORE, 33853440, 35745791,
+STORE, 35745792, 35753983,
+ERASE, 35745792, 35753983,
+STORE, 140249711955968, 140249721602047,
+STORE, 140249711947776, 140249721602047,
+STORE, 140249710899200, 140249721602047,
+STORE, 140249710866432, 140249721602047,
+STORE, 140249710600192, 140249721602047,
+SNULL, 140249744424959, 140249744560127,
+STORE, 140249744207872, 140249744424959,
+STORE, 140249744424960, 140249744560127,
+ERASE, 140249744424960, 140249744560127,
+STORE, 140249708503040, 140249721602047,
+STORE, 33853440, 35885055,
+STORE, 140249707978752, 140249721602047,
+STORE, 140249705881600, 140249721602047,
+STORE, 33853440, 36036607,
+STORE, 33853440, 36175871,
+STORE, 140249744551936, 140249744560127,
+STORE, 140249744543744, 140249744560127,
+STORE, 140249744535552, 140249744560127,
+STORE, 140249744527360, 140249744560127,
+STORE, 140249744519168, 140249744560127,
+STORE, 140249705619456, 140249721602047,
+STORE, 140249744510976, 140249744560127,
+STORE, 140249744502784, 140249744560127,
+STORE, 140249744494592, 140249744560127,
+STORE, 140249744486400, 140249744560127,
+STORE, 140249744478208, 140249744560127,
+STORE, 140249744470016, 140249744560127,
+STORE, 140249744461824, 140249744560127,
+STORE, 140249744453632, 140249744560127,
+STORE, 140249744445440, 140249744560127,
+STORE, 140249744437248, 140249744560127,
+STORE, 140249744429056, 140249744560127,
+STORE, 140249703522304, 140249721602047,
+STORE, 33853440, 36311039,
+STORE, 140249703489536, 140249721602047,
+STORE, 33853440, 36474879,
+STORE, 140249703456768, 140249721602047,
+STORE, 33853440, 36622335,
+STORE, 140249703424000, 140249721602047,
+STORE, 140249703391232, 140249721602047,
+STORE, 33853440, 36810751,
+STORE, 140249703358464, 140249721602047,
+STORE, 140249703325696, 140249721602047,
+SNULL, 36655103, 36810751,
+STORE, 33853440, 36655103,
+STORE, 36655104, 36810751,
+ERASE, 36655104, 36810751,
+SNULL, 36438015, 36655103,
+STORE, 33853440, 36438015,
+STORE, 36438016, 36655103,
+ERASE, 36438016, 36655103,
+STORE, 140249703317504, 140249721602047,
+STORE, 140249701220352, 140249721602047,
+STORE, 33853440, 36585471,
+STORE, 33853440, 36782079,
+STORE, 140249701212160, 140249721602047,
+STORE, 140249701203968, 140249721602047,
+STORE, 140249701195776, 140249721602047,
+STORE, 140249701187584, 140249721602047,
+STORE, 140249701179392, 140249721602047,
+STORE, 140249701171200, 140249721602047,
+STORE, 140249701163008, 140249721602047,
+STORE, 140249701154816, 140249721602047,
+STORE, 140249701146624, 140249721602047,
+STORE, 140249701138432, 140249721602047,
+STORE, 140249701130240, 140249721602047,
+STORE, 140249700081664, 140249721602047,
+STORE, 140249700073472, 140249721602047,
+STORE, 33853440, 36978687,
+STORE, 140249697976320, 140249721602047,
+STORE, 33853440, 37240831,
+STORE, 140249695879168, 140249721602047,
+STORE, 140249695870976, 140249721602047,
+STORE, 140249695862784, 140249721602047,
+STORE, 140249695854592, 140249721602047,
+STORE, 140249695326208, 140249721602047,
+SNULL, 140249710600191, 140249721602047,
+STORE, 140249695326208, 140249710600191,
+STORE, 140249710600192, 140249721602047,
+SNULL, 140249710600192, 140249710866431,
+STORE, 140249710866432, 140249721602047,
+STORE, 140249710600192, 140249710866431,
+ERASE, 140249710600192, 140249710866431,
+STORE, 140249691131904, 140249710600191,
+STORE, 33853440, 37474303,
+STORE, 140249710858240, 140249721602047,
+STORE, 140249710850048, 140249721602047,
+STORE, 140249710841856, 140249721602047,
+STORE, 140249710833664, 140249721602047,
+STORE, 140249710825472, 140249721602047,
+STORE, 140249710817280, 140249721602047,
+STORE, 140249710809088, 140249721602047,
+STORE, 140249710800896, 140249721602047,
+STORE, 140249710792704, 140249721602047,
+STORE, 140249710784512, 140249721602047,
+STORE, 140249710776320, 140249721602047,
+STORE, 140249710768128, 140249721602047,
+STORE, 140249710759936, 140249721602047,
+STORE, 140249710751744, 140249721602047,
+STORE, 140249710743552, 140249721602047,
+STORE, 140249710735360, 140249721602047,
+STORE, 140249689034752, 140249710600191,
+STORE, 140249710727168, 140249721602047,
+STORE, 140249686937600, 140249710600191,
+STORE, 33853440, 37867519,
+STORE, 140249684840448, 140249710600191,
+STORE, 140249710718976, 140249721602047,
+STORE, 140249682743296, 140249710600191,
+STORE, 140249710710784, 140249721602047,
+STORE, 140249710702592, 140249721602047,
+STORE, 140249710694400, 140249721602047,
+STORE, 140249710686208, 140249721602047,
+STORE, 140249710678016, 140249721602047,
+STORE, 140249682612224, 140249710600191,
+STORE, 140249682087936, 140249710600191,
+SNULL, 140249705619455, 140249710600191,
+STORE, 140249682087936, 140249705619455,
+STORE, 140249705619456, 140249710600191,
+SNULL, 140249705619456, 140249705881599,
+STORE, 140249705881600, 140249710600191,
+STORE, 140249705619456, 140249705881599,
+ERASE, 140249705619456, 140249705881599,
+STORE, 140249679990784, 140249705619455,
+STORE, 140249710669824, 140249721602047,
+STORE, 140249677893632, 140249705619455,
+STORE, 140249710653440, 140249721602047,
+STORE, 140249710645248, 140249721602047,
+STORE, 140249710637056, 140249721602047,
+STORE, 140249710628864, 140249721602047,
+STORE, 140249710620672, 140249721602047,
+STORE, 140249710612480, 140249721602047,
+STORE, 140249710604288, 140249721602047,
+STORE, 140249705873408, 140249710600191,
+STORE, 140249705865216, 140249710600191,
+STORE, 140249705857024, 140249710600191,
+STORE, 140249705848832, 140249710600191,
+STORE, 140249705840640, 140249710600191,
+STORE, 140249705832448, 140249710600191,
+STORE, 140249705824256, 140249710600191,
+STORE, 140249705816064, 140249710600191,
+STORE, 140249705807872, 140249710600191,
+STORE, 140249705799680, 140249710600191,
+STORE, 33853440, 38129663,
+SNULL, 140249744207872, 140249744367615,
+STORE, 140249744367616, 140249744424959,
+STORE, 140249744207872, 140249744367615,
+ERASE, 140249744207872, 140249744367615,
+STORE, 140249677606912, 140249705619455,
+STORE, 140249675509760, 140249705619455,
+SNULL, 140249677606911, 140249705619455,
+STORE, 140249675509760, 140249677606911,
+STORE, 140249677606912, 140249705619455,
+SNULL, 140249677606912, 140249677893631,
+STORE, 140249677893632, 140249705619455,
+STORE, 140249677606912, 140249677893631,
+ERASE, 140249677606912, 140249677893631,
+STORE, 140249744359424, 140249744424959,
+STORE, 33853440, 38391807,
+STORE, 140249674981376, 140249677606911,
+STORE, 140249672884224, 140249677606911,
+SNULL, 140249719230463, 140249721602047,
+STORE, 140249710604288, 140249719230463,
+STORE, 140249719230464, 140249721602047,
+SNULL, 140249719230464, 140249719504895,
+STORE, 140249719504896, 140249721602047,
+STORE, 140249719230464, 140249719504895,
+ERASE, 140249719230464, 140249719504895,
+STORE, 140249744351232, 140249744424959,
+STORE, 140249744343040, 140249744424959,
+STORE, 140249744334848, 140249744424959,
+STORE, 140249744326656, 140249744424959,
+STORE, 140249744310272, 140249744424959,
+STORE, 140249744302080, 140249744424959,
+STORE, 140249744285696, 140249744424959,
+STORE, 140249744277504, 140249744424959,
+STORE, 140249744261120, 140249744424959,
+STORE, 140249744252928, 140249744424959,
+STORE, 140249744220160, 140249744424959,
+STORE, 140249744211968, 140249744424959,
+STORE, 140249719488512, 140249721602047,
+STORE, 140249744203776, 140249744424959,
+STORE, 140249719472128, 140249721602047,
+STORE, 140249719463936, 140249721602047,
+STORE, 140249719447552, 140249721602047,
+STORE, 140249719439360, 140249721602047,
+STORE, 140249719406592, 140249721602047,
+STORE, 140249719398400, 140249721602047,
+STORE, 140249719382016, 140249721602047,
+STORE, 140249719373824, 140249721602047,
+STORE, 140249719357440, 140249721602047,
+STORE, 140249719349248, 140249721602047,
+STORE, 140249719332864, 140249721602047,
+STORE, 140249719324672, 140249721602047,
+STORE, 140249719291904, 140249721602047,
+STORE, 140249719283712, 140249721602047,
+STORE, 140249719267328, 140249721602047,
+STORE, 140249719259136, 140249721602047,
+STORE, 140249719242752, 140249721602047,
+STORE, 140249719234560, 140249721602047,
+STORE, 140249705783296, 140249710600191,
+STORE, 140249705775104, 140249710600191,
+STORE, 140249705742336, 140249710600191,
+STORE, 140249705734144, 140249710600191,
+STORE, 140249705717760, 140249710600191,
+STORE, 140249670787072, 140249677606911,
+STORE, 140249705709568, 140249710600191,
+STORE, 140249705693184, 140249710600191,
+STORE, 140249705684992, 140249710600191,
+STORE, 140249705668608, 140249710600191,
+STORE, 140249705660416, 140249710600191,
+STORE, 140249705627648, 140249710600191,
+STORE, 140249677893632, 140249710600191,
+STORE, 140249677877248, 140249710600191,
+STORE, 140249677869056, 140249710600191,
+STORE, 140249677852672, 140249710600191,
+STORE, 140249677844480, 140249710600191,
+STORE, 140249677828096, 140249710600191,
+STORE, 140249668689920, 140249677606911,
+STORE, 140249677819904, 140249710600191,
+STORE, 140249677787136, 140249710600191,
+STORE, 140249677778944, 140249710600191,
+STORE, 140249677762560, 140249710600191,
+STORE, 140249677754368, 140249710600191,
+STORE, 140249677737984, 140249710600191,
+STORE, 140249677729792, 140249710600191,
+STORE, 140249677713408, 140249710600191,
+STORE, 140249677705216, 140249710600191,
+STORE, 140249677672448, 140249710600191,
+STORE, 140249677664256, 140249710600191,
+STORE, 140249677647872, 140249710600191,
+STORE, 140249677639680, 140249710600191,
+STORE, 140249677623296, 140249710600191,
+STORE, 140249677615104, 140249710600191,
+STORE, 140249668673536, 140249677606911,
+STORE, 140249668673536, 140249710600191,
+STORE, 140249668640768, 140249710600191,
+STORE, 140249668632576, 140249710600191,
+STORE, 140249668616192, 140249710600191,
+STORE, 140249668608000, 140249710600191,
+STORE, 140249668591616, 140249710600191,
+STORE, 140249668583424, 140249710600191,
+STORE, 140249668567040, 140249710600191,
+STORE, 140249668558848, 140249710600191,
+STORE, 140249668526080, 140249710600191,
+STORE, 140249668517888, 140249710600191,
+STORE, 140249668501504, 140249710600191,
+STORE, 140249668493312, 140249710600191,
+STORE, 140249668476928, 140249710600191,
+STORE, 140249668468736, 140249710600191,
+STORE, 140249668452352, 140249710600191,
+STORE, 140249668444160, 140249710600191,
+STORE, 140249668411392, 140249710600191,
+STORE, 140249668403200, 140249710600191,
+STORE, 140249668386816, 140249710600191,
+STORE, 140249668378624, 140249710600191,
+STORE, 140249668362240, 140249710600191,
+STORE, 140249668354048, 140249710600191,
+STORE, 140249668337664, 140249710600191,
+STORE, 140249668329472, 140249710600191,
+STORE, 140249668296704, 140249710600191,
+STORE, 140249668288512, 140249710600191,
+STORE, 140249668272128, 140249710600191,
+STORE, 140249668263936, 140249710600191,
+STORE, 140249668247552, 140249710600191,
+STORE, 140249668239360, 140249710600191,
+STORE, 140249668222976, 140249710600191,
+STORE, 140249668214784, 140249710600191,
+STORE, 140249668182016, 140249710600191,
+STORE, 140249668173824, 140249710600191,
+STORE, 140249668157440, 140249710600191,
+STORE, 140249668149248, 140249710600191,
+STORE, 140249668132864, 140249710600191,
+STORE, 140249668124672, 140249710600191,
+STORE, 140249668108288, 140249710600191,
+STORE, 140249668100096, 140249710600191,
+STORE, 140249668067328, 140249710600191,
+STORE, 140249668059136, 140249710600191,
+STORE, 140249668042752, 140249710600191,
+STORE, 140249668034560, 140249710600191,
+STORE, 140249668018176, 140249710600191,
+STORE, 140249668009984, 140249710600191,
+STORE, 140249667993600, 140249710600191,
+STORE, 140249667985408, 140249710600191,
+STORE, 140249667952640, 140249710600191,
+STORE, 140249667944448, 140249710600191,
+STORE, 140249667928064, 140249710600191,
+STORE, 140249667919872, 140249710600191,
+STORE, 140249667903488, 140249710600191,
+STORE, 140249667895296, 140249710600191,
+STORE, 140249667878912, 140249710600191,
+STORE, 140249667870720, 140249710600191,
+STORE, 140249667837952, 140249710600191,
+STORE, 140249667829760, 140249710600191,
+STORE, 140249667813376, 140249710600191,
+STORE, 140249667805184, 140249710600191,
+STORE, 140249667788800, 140249710600191,
+STORE, 140249667780608, 140249710600191,
+STORE, 140249667764224, 140249710600191,
+STORE, 140249667756032, 140249710600191,
+STORE, 140249667723264, 140249710600191,
+STORE, 140249667715072, 140249710600191,
+STORE, 140249667698688, 140249710600191,
+STORE, 140249667690496, 140249710600191,
+STORE, 140249667674112, 140249710600191,
+STORE, 140249667665920, 140249710600191,
+STORE, 140249667649536, 140249710600191,
+STORE, 140249667641344, 140249710600191,
+STORE, 140249667608576, 140249710600191,
+STORE, 140249667600384, 140249710600191,
+STORE, 140249667584000, 140249710600191,
+STORE, 140249667575808, 140249710600191,
+STORE, 140249667559424, 140249710600191,
+STORE, 140249667551232, 140249710600191,
+STORE, 140249667534848, 140249710600191,
+STORE, 140249667526656, 140249710600191,
+STORE, 140249667493888, 140249710600191,
+STORE, 140249667485696, 140249710600191,
+STORE, 140249667469312, 140249710600191,
+STORE, 140249667461120, 140249710600191,
+STORE, 140249667444736, 140249710600191,
+STORE, 140249667436544, 140249710600191,
+STORE, 140249667420160, 140249710600191,
+STORE, 140249665323008, 140249710600191,
+STORE, 140249665314816, 140249710600191,
+STORE, 140249665282048, 140249710600191,
+STORE, 140249665273856, 140249710600191,
+STORE, 140249665257472, 140249710600191,
+STORE, 140249665249280, 140249710600191,
+STORE, 140249665232896, 140249710600191,
+STORE, 140249665224704, 140249710600191,
+STORE, 140249665208320, 140249710600191,
+STORE, 140249665200128, 140249710600191,
+STORE, 140249665167360, 140249710600191,
+STORE, 140249665159168, 140249710600191,
+STORE, 140249665142784, 140249710600191,
+STORE, 140249665134592, 140249710600191,
+STORE, 140249665118208, 140249710600191,
+STORE, 140249665110016, 140249710600191,
+STORE, 140249665093632, 140249710600191,
+STORE, 140249665085440, 140249710600191,
+STORE, 140249665052672, 140249710600191,
+STORE, 140249665044480, 140249710600191,
+STORE, 140249665028096, 140249710600191,
+STORE, 140249665019904, 140249710600191,
+STORE, 140249665003520, 140249710600191,
+STORE, 140249664995328, 140249710600191,
+STORE, 140249664978944, 140249710600191,
+STORE, 140249664970752, 140249710600191,
+STORE, 140249664937984, 140249710600191,
+STORE, 140249664929792, 140249710600191,
+STORE, 140249664913408, 140249710600191,
+STORE, 140249664905216, 140249710600191,
+STORE, 140249664888832, 140249710600191,
+STORE, 140249664880640, 140249710600191,
+STORE, 140249664864256, 140249710600191,
+STORE, 140249664856064, 140249710600191,
+STORE, 140249664823296, 140249710600191,
+STORE, 140249664815104, 140249710600191,
+STORE, 140249664798720, 140249710600191,
+STORE, 140249664790528, 140249710600191,
+STORE, 140249664774144, 140249710600191,
+STORE, 140249664765952, 140249710600191,
+STORE, 140249664749568, 140249710600191,
+STORE, 140249664741376, 140249710600191,
+STORE, 140249664708608, 140249710600191,
+STORE, 140249664700416, 140249710600191,
+STORE, 140249664684032, 140249710600191,
+STORE, 140249664675840, 140249710600191,
+STORE, 140249664659456, 140249710600191,
+STORE, 140249664651264, 140249710600191,
+STORE, 140249664634880, 140249710600191,
+STORE, 140249664626688, 140249710600191,
+STORE, 140249664593920, 140249710600191,
+STORE, 140249664585728, 140249710600191,
+STORE, 140249664569344, 140249710600191,
+STORE, 140249664561152, 140249710600191,
+STORE, 140249664544768, 140249710600191,
+STORE, 140249664536576, 140249710600191,
+STORE, 140249664520192, 140249710600191,
+STORE, 140249664512000, 140249710600191,
+STORE, 140249664479232, 140249710600191,
+STORE, 140249664471040, 140249710600191,
+STORE, 140249664454656, 140249710600191,
+STORE, 140249664446464, 140249710600191,
+STORE, 140249664430080, 140249710600191,
+STORE, 140249664421888, 140249710600191,
+STORE, 140249664405504, 140249710600191,
+STORE, 140249664397312, 140249710600191,
+STORE, 140249664364544, 140249710600191,
+STORE, 140249664356352, 140249710600191,
+STORE, 140249664339968, 140249710600191,
+STORE, 140249664331776, 140249710600191,
+STORE, 140249664315392, 140249710600191,
+STORE, 140249664307200, 140249710600191,
+STORE, 140249664290816, 140249710600191,
+STORE, 140249664282624, 140249710600191,
+STORE, 140249664249856, 140249710600191,
+STORE, 140249664241664, 140249710600191,
+STORE, 140249664225280, 140249710600191,
+STORE, 140249664217088, 140249710600191,
+STORE, 140249664200704, 140249710600191,
+STORE, 140249664192512, 140249710600191,
+STORE, 140249664176128, 140249710600191,
+STORE, 140249664167936, 140249710600191,
+STORE, 140249664135168, 140249710600191,
+STORE, 140249664126976, 140249710600191,
+STORE, 140249664110592, 140249710600191,
+STORE, 140249664102400, 140249710600191,
+STORE, 140249664086016, 140249710600191,
+STORE, 140249664077824, 140249710600191,
+STORE, 140249664061440, 140249710600191,
+STORE, 140249664053248, 140249710600191,
+STORE, 140249664020480, 140249710600191,
+STORE, 140249664012288, 140249710600191,
+STORE, 140249663995904, 140249710600191,
+STORE, 140249663987712, 140249710600191,
+STORE, 140249663971328, 140249710600191,
+STORE, 140249663963136, 140249710600191,
+STORE, 140249663946752, 140249710600191,
+STORE, 140249663938560, 140249710600191,
+STORE, 140249663905792, 140249710600191,
+STORE, 140249663897600, 140249710600191,
+STORE, 140249663881216, 140249710600191,
+STORE, 140249663873024, 140249710600191,
+STORE, 140249663856640, 140249710600191,
+STORE, 140249663848448, 140249710600191,
+STORE, 140249663832064, 140249710600191,
+STORE, 140249663823872, 140249710600191,
+STORE, 140249663791104, 140249710600191,
+STORE, 140249663782912, 140249710600191,
+STORE, 140249663766528, 140249710600191,
+STORE, 140249663758336, 140249710600191,
+STORE, 140249663741952, 140249710600191,
+STORE, 140249663733760, 140249710600191,
+STORE, 140249663717376, 140249710600191,
+STORE, 140249663709184, 140249710600191,
+STORE, 140249663676416, 140249710600191,
+STORE, 140249663668224, 140249710600191,
+STORE, 140249663651840, 140249710600191,
+STORE, 140249663643648, 140249710600191,
+STORE, 140249663627264, 140249710600191,
+STORE, 33853440, 38526975,
+STORE, 140249663619072, 140249710600191,
+STORE, 140249663602688, 140249710600191,
+STORE, 140249661505536, 140249710600191,
+STORE, 140249661497344, 140249710600191,
+STORE, 140249661464576, 140249710600191,
+STORE, 140249661456384, 140249710600191,
+STORE, 140249661440000, 140249710600191,
+STORE, 140249661431808, 140249710600191,
+STORE, 140249661415424, 140249710600191,
+STORE, 140249661407232, 140249710600191,
+STORE, 140249661390848, 140249710600191,
+STORE, 140249661382656, 140249710600191,
+STORE, 140249661349888, 140249710600191,
+STORE, 140249661341696, 140249710600191,
+STORE, 140249661325312, 140249710600191,
+STORE, 140249661317120, 140249710600191,
+STORE, 140249661300736, 140249710600191,
+STORE, 140249661292544, 140249710600191,
+STORE, 140249661276160, 140249710600191,
+STORE, 140249661267968, 140249710600191,
+STORE, 140249661235200, 140249710600191,
+STORE, 140249661227008, 140249710600191,
+STORE, 140249661210624, 140249710600191,
+STORE, 140249661202432, 140249710600191,
+STORE, 140249661186048, 140249710600191,
+STORE, 140249661177856, 140249710600191,
+STORE, 140249661161472, 140249710600191,
+STORE, 140249661153280, 140249710600191,
+STORE, 140249661120512, 140249710600191,
+STORE, 140249661112320, 140249710600191,
+STORE, 140249661095936, 140249710600191,
+STORE, 140249661087744, 140249710600191,
+STORE, 140249661071360, 140249710600191,
+STORE, 140249661063168, 140249710600191,
+STORE, 140249661046784, 140249710600191,
+STORE, 140249661038592, 140249710600191,
+STORE, 140249661005824, 140249710600191,
+STORE, 140249660997632, 140249710600191,
+STORE, 140249660981248, 140249710600191,
+STORE, 140249660973056, 140249710600191,
+STORE, 140249660956672, 140249710600191,
+STORE, 140249660948480, 140249710600191,
+STORE, 140249660932096, 140249710600191,
+STORE, 140249660923904, 140249710600191,
+STORE, 140249660891136, 140249710600191,
+STORE, 140249660882944, 140249710600191,
+STORE, 140249660866560, 140249710600191,
+STORE, 140249660858368, 140249710600191,
+STORE, 140249660841984, 140249710600191,
+STORE, 140249660833792, 140249710600191,
+STORE, 140249660817408, 140249710600191,
+STORE, 140249660809216, 140249710600191,
+STORE, 140249660776448, 140249710600191,
+STORE, 140249660768256, 140249710600191,
+STORE, 140249660751872, 140249710600191,
+STORE, 140249660743680, 140249710600191,
+STORE, 140249660727296, 140249710600191,
+STORE, 140249660719104, 140249710600191,
+STORE, 140249660702720, 140249710600191,
+STORE, 140249660694528, 140249710600191,
+STORE, 140249660661760, 140249710600191,
+STORE, 140249660653568, 140249710600191,
+STORE, 140249660637184, 140249710600191,
+STORE, 140249660628992, 140249710600191,
+STORE, 140249660612608, 140249710600191,
+STORE, 140249660604416, 140249710600191,
+STORE, 140249660588032, 140249710600191,
+STORE, 140249660579840, 140249710600191,
+STORE, 140249660547072, 140249710600191,
+STORE, 140249660538880, 140249710600191,
+STORE, 140249660522496, 140249710600191,
+STORE, 140249660514304, 140249710600191,
+STORE, 140249660497920, 140249710600191,
+STORE, 140249660489728, 140249710600191,
+STORE, 140249660473344, 140249710600191,
+STORE, 140249660465152, 140249710600191,
+STORE, 140249660432384, 140249710600191,
+STORE, 140249660424192, 140249710600191,
+STORE, 140249660407808, 140249710600191,
+STORE, 140249660399616, 140249710600191,
+STORE, 140249660383232, 140249710600191,
+STORE, 140249660375040, 140249710600191,
+STORE, 140249660358656, 140249710600191,
+STORE, 140249660350464, 140249710600191,
+STORE, 140249660317696, 140249710600191,
+STORE, 140249660309504, 140249710600191,
+STORE, 140249660293120, 140249710600191,
+STORE, 140249660284928, 140249710600191,
+STORE, 140249660268544, 140249710600191,
+STORE, 140249660260352, 140249710600191,
+STORE, 140249660243968, 140249710600191,
+STORE, 140249660235776, 140249710600191,
+STORE, 140249660203008, 140249710600191,
+STORE, 140249660194816, 140249710600191,
+STORE, 140249660178432, 140249710600191,
+STORE, 140249660170240, 140249710600191,
+STORE, 140249660153856, 140249710600191,
+STORE, 140249660145664, 140249710600191,
+STORE, 140249660129280, 140249710600191,
+STORE, 140249660121088, 140249710600191,
+STORE, 140249660088320, 140249710600191,
+STORE, 140249660080128, 140249710600191,
+STORE, 140249660063744, 140249710600191,
+STORE, 140249660055552, 140249710600191,
+STORE, 140249660039168, 140249710600191,
+STORE, 140249660030976, 140249710600191,
+STORE, 140249660014592, 140249710600191,
+STORE, 140249660006400, 140249710600191,
+STORE, 140249659973632, 140249710600191,
+STORE, 140249659965440, 140249710600191,
+STORE, 140249659949056, 140249710600191,
+STORE, 140249659940864, 140249710600191,
+STORE, 140249659924480, 140249710600191,
+STORE, 140249659916288, 140249710600191,
+STORE, 140249659899904, 140249710600191,
+STORE, 140249659891712, 140249710600191,
+STORE, 140249659858944, 140249710600191,
+STORE, 140249659850752, 140249710600191,
+STORE, 140249659834368, 140249710600191,
+STORE, 140249659826176, 140249710600191,
+STORE, 140249659809792, 140249710600191,
+STORE, 140249659801600, 140249710600191,
+STORE, 140249659785216, 140249710600191,
+STORE, 140249657688064, 140249710600191,
+STORE, 140249657679872, 140249710600191,
+STORE, 140249657647104, 140249710600191,
+STORE, 140249657638912, 140249710600191,
+STORE, 140249657622528, 140249710600191,
+STORE, 140249657614336, 140249710600191,
+STORE, 140249657597952, 140249710600191,
+STORE, 140249657589760, 140249710600191,
+STORE, 140249657573376, 140249710600191,
+STORE, 140249657565184, 140249710600191,
+STORE, 140249657532416, 140249710600191,
+STORE, 140249657524224, 140249710600191,
+STORE, 140249657507840, 140249710600191,
+STORE, 140249657499648, 140249710600191,
+STORE, 140249657483264, 140249710600191,
+STORE, 140249657475072, 140249710600191,
+STORE, 140249657458688, 140249710600191,
+STORE, 140249657450496, 140249710600191,
+STORE, 140249657417728, 140249710600191,
+STORE, 140249657409536, 140249710600191,
+STORE, 140249657393152, 140249710600191,
+STORE, 140249657384960, 140249710600191,
+STORE, 140249657368576, 140249710600191,
+STORE, 140249657360384, 140249710600191,
+STORE, 140249657344000, 140249710600191,
+STORE, 140249657335808, 140249710600191,
+STORE, 140249657303040, 140249710600191,
+STORE, 140249657294848, 140249710600191,
+STORE, 140249657278464, 140249710600191,
+STORE, 140249657270272, 140249710600191,
+STORE, 140249657253888, 140249710600191,
+STORE, 140249657245696, 140249710600191,
+STORE, 140249657229312, 140249710600191,
+STORE, 140249657221120, 140249710600191,
+STORE, 140249657188352, 140249710600191,
+STORE, 140249657180160, 140249710600191,
+STORE, 140249657163776, 140249710600191,
+STORE, 140249657155584, 140249710600191,
+STORE, 140249657139200, 140249710600191,
+STORE, 140249657131008, 140249710600191,
+STORE, 140249657114624, 140249710600191,
+STORE, 140249657106432, 140249710600191,
+STORE, 140249657073664, 140249710600191,
+STORE, 140249657065472, 140249710600191,
+STORE, 140249657049088, 140249710600191,
+STORE, 140249657040896, 140249710600191,
+STORE, 140249657024512, 140249710600191,
+STORE, 140249657016320, 140249710600191,
+STORE, 140249656999936, 140249710600191,
+STORE, 140249656991744, 140249710600191,
+STORE, 140249656958976, 140249710600191,
+STORE, 140249656950784, 140249710600191,
+STORE, 140249656934400, 140249710600191,
+STORE, 140249656926208, 140249710600191,
+STORE, 140249656909824, 140249710600191,
+STORE, 140249656901632, 140249710600191,
+STORE, 140249656885248, 140249710600191,
+STORE, 140249656877056, 140249710600191,
+STORE, 140249656844288, 140249710600191,
+STORE, 140249656836096, 140249710600191,
+STORE, 140249656819712, 140249710600191,
+STORE, 140249656811520, 140249710600191,
+STORE, 140249656795136, 140249710600191,
+STORE, 33853440, 38662143,
+STORE, 140249656786944, 140249710600191,
+STORE, 140249656770560, 140249710600191,
+STORE, 140249656762368, 140249710600191,
+STORE, 140249656729600, 140249710600191,
+STORE, 140249656721408, 140249710600191,
+STORE, 140249656705024, 140249710600191,
+STORE, 140249656696832, 140249710600191,
+STORE, 140249656680448, 140249710600191,
+STORE, 140249656672256, 140249710600191,
+STORE, 140249656655872, 140249710600191,
+STORE, 140249656647680, 140249710600191,
+STORE, 140249656614912, 140249710600191,
+STORE, 140249656606720, 140249710600191,
+STORE, 140249656590336, 140249710600191,
+STORE, 140249656582144, 140249710600191,
+STORE, 140249656565760, 140249710600191,
+STORE, 140249656557568, 140249710600191,
+STORE, 140249656541184, 140249710600191,
+STORE, 140249656532992, 140249710600191,
+STORE, 140249656500224, 140249710600191,
+STORE, 140249656492032, 140249710600191,
+STORE, 140249656475648, 140249710600191,
+STORE, 140249656467456, 140249710600191,
+STORE, 140249656451072, 140249710600191,
+STORE, 140249656442880, 140249710600191,
+STORE, 140249656426496, 140249710600191,
+STORE, 140249656418304, 140249710600191,
+STORE, 140249656385536, 140249710600191,
+STORE, 140249656377344, 140249710600191,
+STORE, 140249656360960, 140249710600191,
+STORE, 140249656352768, 140249710600191,
+STORE, 140249656336384, 140249710600191,
+STORE, 140249656328192, 140249710600191,
+STORE, 140249656311808, 140249710600191,
+STORE, 140249656303616, 140249710600191,
+STORE, 140249656270848, 140249710600191,
+STORE, 140249656262656, 140249710600191,
+STORE, 140249656246272, 140249710600191,
+STORE, 140249656238080, 140249710600191,
+STORE, 140249656221696, 140249710600191,
+STORE, 140249656213504, 140249710600191,
+STORE, 140249656197120, 140249710600191,
+STORE, 140249656188928, 140249710600191,
+STORE, 140249656156160, 140249710600191,
+STORE, 140249656147968, 140249710600191,
+STORE, 140249656131584, 140249710600191,
+STORE, 140249656123392, 140249710600191,
+STORE, 140249656107008, 140249710600191,
+STORE, 140249656098816, 140249710600191,
+STORE, 140249656082432, 140249710600191,
+STORE, 140249656074240, 140249710600191,
+STORE, 140249656041472, 140249710600191,
+STORE, 140249656033280, 140249710600191,
+STORE, 140249656016896, 140249710600191,
+STORE, 140249656008704, 140249710600191,
+STORE, 140249655992320, 140249710600191,
+STORE, 140249655984128, 140249710600191,
+STORE, 140249655967744, 140249710600191,
+STORE, 140249653870592, 140249710600191,
+STORE, 140249653862400, 140249710600191,
+STORE, 140249653829632, 140249710600191,
+STORE, 140249653821440, 140249710600191,
+STORE, 140249653805056, 140249710600191,
+STORE, 140249653796864, 140249710600191,
+STORE, 140249653780480, 140249710600191,
+STORE, 140249653772288, 140249710600191,
+STORE, 140249653755904, 140249710600191,
+STORE, 140249652703232, 140249710600191,
+SNULL, 140249682087935, 140249710600191,
+STORE, 140249652703232, 140249682087935,
+STORE, 140249682087936, 140249710600191,
+ };
+
+ unsigned long set26[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140729464770560, 140737488351231,
+SNULL, 140729464774655, 140737488351231,
+STORE, 140729464770560, 140729464774655,
+STORE, 140729464639488, 140729464774655,
+STORE, 4194304, 5066751,
+STORE, 7159808, 7172095,
+STORE, 7172096, 7180287,
+STORE, 140729465114624, 140729465118719,
+STORE, 140729465102336, 140729465114623,
+STORE, 30867456, 30875647,
+STORE, 30867456, 31010815,
+STORE, 140109040988160, 140109042671615,
+STORE, 140109040959488, 140109040988159,
+STORE, 140109040943104, 140109040959487,
+ERASE, 140109040943104, 140109040959487,
+STORE, 140109040840704, 140109040959487,
+ERASE, 140109040840704, 140109040959487,
+STORE, 140109040951296, 140109040959487,
+ERASE, 140109040951296, 140109040959487,
+STORE, 140109040955392, 140109040959487,
+ERASE, 140109040955392, 140109040959487,
+ };
+ unsigned long set27[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140726128070656, 140737488351231,
+SNULL, 140726128074751, 140737488351231,
+STORE, 140726128070656, 140726128074751,
+STORE, 140726127939584, 140726128074751,
+STORE, 94478497189888, 94478499303423,
+SNULL, 94478497202175, 94478499303423,
+STORE, 94478497189888, 94478497202175,
+STORE, 94478497202176, 94478499303423,
+ERASE, 94478497202176, 94478499303423,
+STORE, 94478499295232, 94478499303423,
+STORE, 140415605723136, 140415607975935,
+SNULL, 140415605866495, 140415607975935,
+STORE, 140415605723136, 140415605866495,
+STORE, 140415605866496, 140415607975935,
+ERASE, 140415605866496, 140415607975935,
+STORE, 140415607963648, 140415607971839,
+STORE, 140415607971840, 140415607975935,
+STORE, 140726130024448, 140726130028543,
+STORE, 140726130012160, 140726130024447,
+STORE, 140415607934976, 140415607963647,
+STORE, 140415607926784, 140415607934975,
+STORE, 140415603245056, 140415605723135,
+SNULL, 140415603245056, 140415603613695,
+STORE, 140415603613696, 140415605723135,
+STORE, 140415603245056, 140415603613695,
+SNULL, 140415605710847, 140415605723135,
+STORE, 140415603613696, 140415605710847,
+STORE, 140415605710848, 140415605723135,
+ERASE, 140415605710848, 140415605723135,
+STORE, 140415605710848, 140415605723135,
+STORE, 140415599370240, 140415603245055,
+SNULL, 140415599370240, 140415601111039,
+STORE, 140415601111040, 140415603245055,
+STORE, 140415599370240, 140415601111039,
+SNULL, 140415603208191, 140415603245055,
+STORE, 140415601111040, 140415603208191,
+STORE, 140415603208192, 140415603245055,
+ERASE, 140415603208192, 140415603245055,
+STORE, 140415603208192, 140415603245055,
+STORE, 140415595692032, 140415599370239,
+SNULL, 140415595692032, 140415597207551,
+STORE, 140415597207552, 140415599370239,
+STORE, 140415595692032, 140415597207551,
+SNULL, 140415599304703, 140415599370239,
+STORE, 140415597207552, 140415599304703,
+STORE, 140415599304704, 140415599370239,
+SNULL, 140415599304704, 140415599353855,
+STORE, 140415599353856, 140415599370239,
+STORE, 140415599304704, 140415599353855,
+ERASE, 140415599304704, 140415599353855,
+STORE, 140415599304704, 140415599353855,
+ERASE, 140415599353856, 140415599370239,
+STORE, 140415599353856, 140415599370239,
+STORE, 140415593500672, 140415595692031,
+SNULL, 140415593500672, 140415593590783,
+STORE, 140415593590784, 140415595692031,
+STORE, 140415593500672, 140415593590783,
+SNULL, 140415595683839, 140415595692031,
+STORE, 140415593590784, 140415595683839,
+STORE, 140415595683840, 140415595692031,
+ERASE, 140415595683840, 140415595692031,
+STORE, 140415595683840, 140415595692031,
+STORE, 140415589703680, 140415593500671,
+SNULL, 140415589703680, 140415591362559,
+STORE, 140415591362560, 140415593500671,
+STORE, 140415589703680, 140415591362559,
+SNULL, 140415593459711, 140415593500671,
+STORE, 140415591362560, 140415593459711,
+STORE, 140415593459712, 140415593500671,
+SNULL, 140415593459712, 140415593484287,
+STORE, 140415593484288, 140415593500671,
+STORE, 140415593459712, 140415593484287,
+ERASE, 140415593459712, 140415593484287,
+STORE, 140415593459712, 140415593484287,
+ERASE, 140415593484288, 140415593500671,
+STORE, 140415593484288, 140415593500671,
+STORE, 140415587590144, 140415589703679,
+SNULL, 140415587590144, 140415587602431,
+STORE, 140415587602432, 140415589703679,
+STORE, 140415587590144, 140415587602431,
+SNULL, 140415589695487, 140415589703679,
+STORE, 140415587602432, 140415589695487,
+STORE, 140415589695488, 140415589703679,
+ERASE, 140415589695488, 140415589703679,
+STORE, 140415589695488, 140415589703679,
+STORE, 140415607918592, 140415607934975,
+STORE, 140415585398784, 140415587590143,
+SNULL, 140415585398784, 140415585480703,
+STORE, 140415585480704, 140415587590143,
+STORE, 140415585398784, 140415585480703,
+SNULL, 140415587573759, 140415587590143,
+STORE, 140415585480704, 140415587573759,
+STORE, 140415587573760, 140415587590143,
+SNULL, 140415587573760, 140415587581951,
+STORE, 140415587581952, 140415587590143,
+STORE, 140415587573760, 140415587581951,
+ERASE, 140415587573760, 140415587581951,
+STORE, 140415587573760, 140415587581951,
+ERASE, 140415587581952, 140415587590143,
+STORE, 140415587581952, 140415587590143,
+STORE, 140415583182848, 140415585398783,
+SNULL, 140415583182848, 140415583281151,
+STORE, 140415583281152, 140415585398783,
+STORE, 140415583182848, 140415583281151,
+SNULL, 140415585374207, 140415585398783,
+STORE, 140415583281152, 140415585374207,
+STORE, 140415585374208, 140415585398783,
+SNULL, 140415585374208, 140415585382399,
+STORE, 140415585382400, 140415585398783,
+STORE, 140415585374208, 140415585382399,
+ERASE, 140415585374208, 140415585382399,
+STORE, 140415585374208, 140415585382399,
+ERASE, 140415585382400, 140415585398783,
+STORE, 140415585382400, 140415585398783,
+STORE, 140415580979200, 140415583182847,
+SNULL, 140415580979200, 140415581081599,
+STORE, 140415581081600, 140415583182847,
+STORE, 140415580979200, 140415581081599,
+SNULL, 140415583174655, 140415583182847,
+STORE, 140415581081600, 140415583174655,
+STORE, 140415583174656, 140415583182847,
+ERASE, 140415583174656, 140415583182847,
+STORE, 140415583174656, 140415583182847,
+STORE, 140415578816512, 140415580979199,
+SNULL, 140415578816512, 140415578877951,
+STORE, 140415578877952, 140415580979199,
+STORE, 140415578816512, 140415578877951,
+SNULL, 140415580971007, 140415580979199,
+STORE, 140415578877952, 140415580971007,
+STORE, 140415580971008, 140415580979199,
+ERASE, 140415580971008, 140415580979199,
+STORE, 140415580971008, 140415580979199,
+STORE, 140415576563712, 140415578816511,
+SNULL, 140415576563712, 140415576715263,
+STORE, 140415576715264, 140415578816511,
+STORE, 140415576563712, 140415576715263,
+SNULL, 140415578808319, 140415578816511,
+STORE, 140415576715264, 140415578808319,
+STORE, 140415578808320, 140415578816511,
+ERASE, 140415578808320, 140415578816511,
+STORE, 140415578808320, 140415578816511,
+STORE, 140415574392832, 140415576563711,
+SNULL, 140415574392832, 140415574462463,
+STORE, 140415574462464, 140415576563711,
+STORE, 140415574392832, 140415574462463,
+SNULL, 140415576555519, 140415576563711,
+STORE, 140415574462464, 140415576555519,
+STORE, 140415576555520, 140415576563711,
+ERASE, 140415576555520, 140415576563711,
+STORE, 140415576555520, 140415576563711,
+STORE, 140415607910400, 140415607934975,
+STORE, 140415571230720, 140415574392831,
+SNULL, 140415571230720, 140415572291583,
+STORE, 140415572291584, 140415574392831,
+STORE, 140415571230720, 140415572291583,
+SNULL, 140415574384639, 140415574392831,
+STORE, 140415572291584, 140415574384639,
+STORE, 140415574384640, 140415574392831,
+ERASE, 140415574384640, 140415574392831,
+STORE, 140415574384640, 140415574392831,
+STORE, 140415607902208, 140415607934975,
+SNULL, 140415593476095, 140415593484287,
+STORE, 140415593459712, 140415593476095,
+STORE, 140415593476096, 140415593484287,
+SNULL, 140415574388735, 140415574392831,
+STORE, 140415574384640, 140415574388735,
+STORE, 140415574388736, 140415574392831,
+SNULL, 140415576559615, 140415576563711,
+STORE, 140415576555520, 140415576559615,
+STORE, 140415576559616, 140415576563711,
+SNULL, 140415589699583, 140415589703679,
+STORE, 140415589695488, 140415589699583,
+STORE, 140415589699584, 140415589703679,
+SNULL, 140415585378303, 140415585382399,
+STORE, 140415585374208, 140415585378303,
+STORE, 140415585378304, 140415585382399,
+SNULL, 140415578812415, 140415578816511,
+STORE, 140415578808320, 140415578812415,
+STORE, 140415578812416, 140415578816511,
+SNULL, 140415580975103, 140415580979199,
+STORE, 140415580971008, 140415580975103,
+STORE, 140415580975104, 140415580979199,
+SNULL, 140415583178751, 140415583182847,
+STORE, 140415583174656, 140415583178751,
+STORE, 140415583178752, 140415583182847,
+SNULL, 140415587577855, 140415587581951,
+STORE, 140415587573760, 140415587577855,
+STORE, 140415587577856, 140415587581951,
+SNULL, 140415595687935, 140415595692031,
+STORE, 140415595683840, 140415595687935,
+STORE, 140415595687936, 140415595692031,
+STORE, 140415607894016, 140415607934975,
+SNULL, 140415599345663, 140415599353855,
+STORE, 140415599304704, 140415599345663,
+STORE, 140415599345664, 140415599353855,
+SNULL, 140415603240959, 140415603245055,
+STORE, 140415603208192, 140415603240959,
+STORE, 140415603240960, 140415603245055,
+SNULL, 140415605719039, 140415605723135,
+STORE, 140415605710848, 140415605719039,
+STORE, 140415605719040, 140415605723135,
+SNULL, 94478499299327, 94478499303423,
+STORE, 94478499295232, 94478499299327,
+STORE, 94478499299328, 94478499303423,
+SNULL, 140415607967743, 140415607971839,
+STORE, 140415607963648, 140415607967743,
+STORE, 140415607967744, 140415607971839,
+ERASE, 140415607934976, 140415607963647,
+STORE, 94478511173632, 94478511378431,
+STORE, 140415606210560, 140415607894015,
+STORE, 140415607934976, 140415607963647,
+STORE, 94478511173632, 94478511513599,
+STORE, 94478511173632, 94478511648767,
+SNULL, 94478511615999, 94478511648767,
+STORE, 94478511173632, 94478511615999,
+STORE, 94478511616000, 94478511648767,
+ERASE, 94478511616000, 94478511648767,
+STORE, 94478511173632, 94478511751167,
+SNULL, 94478511747071, 94478511751167,
+STORE, 94478511173632, 94478511747071,
+STORE, 94478511747072, 94478511751167,
+ERASE, 94478511747072, 94478511751167,
+STORE, 94478511173632, 94478511882239,
+SNULL, 94478511878143, 94478511882239,
+STORE, 94478511173632, 94478511878143,
+STORE, 94478511878144, 94478511882239,
+ERASE, 94478511878144, 94478511882239,
+STORE, 94478511173632, 94478512013311,
+SNULL, 94478512009215, 94478512013311,
+STORE, 94478511173632, 94478512009215,
+STORE, 94478512009216, 94478512013311,
+ERASE, 94478512009216, 94478512013311,
+STORE, 94478511173632, 94478512144383,
+STORE, 94478511173632, 94478512279551,
+STORE, 140415606181888, 140415606210559,
+STORE, 140415569100800, 140415571230719,
+SNULL, 140415569100800, 140415569129471,
+STORE, 140415569129472, 140415571230719,
+STORE, 140415569100800, 140415569129471,
+SNULL, 140415571222527, 140415571230719,
+STORE, 140415569129472, 140415571222527,
+STORE, 140415571222528, 140415571230719,
+ERASE, 140415571222528, 140415571230719,
+STORE, 140415571222528, 140415571230719,
+STORE, 140415566905344, 140415569100799,
+SNULL, 140415566905344, 140415566987263,
+STORE, 140415566987264, 140415569100799,
+STORE, 140415566905344, 140415566987263,
+SNULL, 140415569084415, 140415569100799,
+STORE, 140415566987264, 140415569084415,
+STORE, 140415569084416, 140415569100799,
+SNULL, 140415569084416, 140415569092607,
+STORE, 140415569092608, 140415569100799,
+STORE, 140415569084416, 140415569092607,
+ERASE, 140415569084416, 140415569092607,
+STORE, 140415569084416, 140415569092607,
+ERASE, 140415569092608, 140415569100799,
+STORE, 140415569092608, 140415569100799,
+SNULL, 140415569088511, 140415569092607,
+STORE, 140415569084416, 140415569088511,
+STORE, 140415569088512, 140415569092607,
+SNULL, 140415571226623, 140415571230719,
+STORE, 140415571222528, 140415571226623,
+STORE, 140415571226624, 140415571230719,
+ERASE, 140415606181888, 140415606210559,
+STORE, 140415606181888, 140415606210559,
+STORE, 140415564759040, 140415566905343,
+SNULL, 140415564759040, 140415564804095,
+STORE, 140415564804096, 140415566905343,
+STORE, 140415564759040, 140415564804095,
+SNULL, 140415566897151, 140415566905343,
+STORE, 140415564804096, 140415566897151,
+STORE, 140415566897152, 140415566905343,
+ERASE, 140415566897152, 140415566905343,
+STORE, 140415566897152, 140415566905343,
+STORE, 140415562588160, 140415564759039,
+SNULL, 140415562588160, 140415562629119,
+STORE, 140415562629120, 140415564759039,
+STORE, 140415562588160, 140415562629119,
+SNULL, 140415564726271, 140415564759039,
+STORE, 140415562629120, 140415564726271,
+STORE, 140415564726272, 140415564759039,
+SNULL, 140415564726272, 140415564734463,
+STORE, 140415564734464, 140415564759039,
+STORE, 140415564726272, 140415564734463,
+ERASE, 140415564726272, 140415564734463,
+STORE, 140415564726272, 140415564734463,
+ERASE, 140415564734464, 140415564759039,
+STORE, 140415564734464, 140415564759039,
+SNULL, 140415564730367, 140415564734463,
+STORE, 140415564726272, 140415564730367,
+STORE, 140415564730368, 140415564734463,
+SNULL, 140415566901247, 140415566905343,
+STORE, 140415566897152, 140415566901247,
+STORE, 140415566901248, 140415566905343,
+ERASE, 140415606181888, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415605944320, 140415606210559,
+ERASE, 140415605944320, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 94478511173632, 94478512414719,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 140415606206464, 140415606210559,
+ERASE, 140415606206464, 140415606210559,
+STORE, 94478511173632, 94478512652287,
+STORE, 94478511173632, 94478512787455,
+STORE, 94478511173632, 94478512922623,
+STORE, 94478511173632, 94478513057791,
+STORE, 140415537422336, 140415562588159,
+STORE, 94478511173632, 94478513192959,
+STORE, 94478511173632, 94478513356799,
+STORE, 94478511173632, 94478513491967,
+STORE, 94478511173632, 94478513627135,
+STORE, 94478511173632, 94478513790975,
+STORE, 94478511173632, 94478513926143,
+STORE, 94478511173632, 94478514061311,
+STORE, 94478511173632, 94478514196479,
+STORE, 94478511173632, 94478514331647,
+STORE, 94478511173632, 94478514606079,
+STORE, 94478511173632, 94478514741247,
+STORE, 94478511173632, 94478514876415,
+STORE, 94478511173632, 94478515011583,
+STORE, 94478511173632, 94478515146751,
+STORE, 94478511173632, 94478515281919,
+STORE, 94478511173632, 94478515474431,
+STORE, 94478511173632, 94478515609599,
+STORE, 94478511173632, 94478515744767,
+STORE, 140415536922624, 140415562588159,
+STORE, 94478511173632, 94478515879935,
+STORE, 94478511173632, 94478516015103,
+STORE, 94478511173632, 94478516150271,
+STORE, 94478511173632, 94478516285439,
+STORE, 94478511173632, 94478516420607,
+STORE, 94478511173632, 94478516555775,
+STORE, 94478511173632, 94478516690943,
+STORE, 94478511173632, 94478516826111,
+STORE, 94478511173632, 94478516961279,
+STORE, 94478511173632, 94478517231615,
+STORE, 94478511173632, 94478517366783,
+STORE, 94478511173632, 94478517501951,
+STORE, 94478511173632, 94478517637119,
+STORE, 94478511173632, 94478517772287,
+STORE, 94478511173632, 94478517907455,
+STORE, 94478511173632, 94478518042623,
+STORE, 94478511173632, 94478518177791,
+STORE, 94478511173632, 94478518312959,
+STORE, 94478511173632, 94478518448127,
+STORE, 140415535910912, 140415562588159,
+SNULL, 140415536922623, 140415562588159,
+STORE, 140415535910912, 140415536922623,
+STORE, 140415536922624, 140415562588159,
+SNULL, 140415536922624, 140415537422335,
+STORE, 140415537422336, 140415562588159,
+STORE, 140415536922624, 140415537422335,
+ERASE, 140415536922624, 140415537422335,
+STORE, 94478511173632, 94478518583295,
+STORE, 94478511173632, 94478518718463,
+STORE, 94478511173632, 94478518853631,
+STORE, 94478511173632, 94478518988799,
+STORE, 94478511173632, 94478519123967,
+STORE, 94478511173632, 94478519259135,
+STORE, 140415509696512, 140415535910911,
+ERASE, 140415537422336, 140415562588159,
+STORE, 140415482433536, 140415509696511,
+ };
+ unsigned long set28[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140722475622400, 140737488351231,
+SNULL, 140722475626495, 140737488351231,
+STORE, 140722475622400, 140722475626495,
+STORE, 140722475491328, 140722475626495,
+STORE, 93865834291200, 93865836548095,
+SNULL, 93865834422271, 93865836548095,
+STORE, 93865834291200, 93865834422271,
+STORE, 93865834422272, 93865836548095,
+ERASE, 93865834422272, 93865836548095,
+STORE, 93865836519424, 93865836527615,
+STORE, 93865836527616, 93865836548095,
+STORE, 139918411104256, 139918413357055,
+SNULL, 139918411247615, 139918413357055,
+STORE, 139918411104256, 139918411247615,
+STORE, 139918411247616, 139918413357055,
+ERASE, 139918411247616, 139918413357055,
+STORE, 139918413344768, 139918413352959,
+STORE, 139918413352960, 139918413357055,
+STORE, 140722476642304, 140722476646399,
+STORE, 140722476630016, 140722476642303,
+STORE, 139918413316096, 139918413344767,
+STORE, 139918413307904, 139918413316095,
+STORE, 139918408888320, 139918411104255,
+SNULL, 139918408888320, 139918408986623,
+STORE, 139918408986624, 139918411104255,
+STORE, 139918408888320, 139918408986623,
+SNULL, 139918411079679, 139918411104255,
+STORE, 139918408986624, 139918411079679,
+STORE, 139918411079680, 139918411104255,
+SNULL, 139918411079680, 139918411087871,
+STORE, 139918411087872, 139918411104255,
+STORE, 139918411079680, 139918411087871,
+ERASE, 139918411079680, 139918411087871,
+STORE, 139918411079680, 139918411087871,
+ERASE, 139918411087872, 139918411104255,
+STORE, 139918411087872, 139918411104255,
+STORE, 139918405091328, 139918408888319,
+SNULL, 139918405091328, 139918406750207,
+STORE, 139918406750208, 139918408888319,
+STORE, 139918405091328, 139918406750207,
+SNULL, 139918408847359, 139918408888319,
+STORE, 139918406750208, 139918408847359,
+STORE, 139918408847360, 139918408888319,
+SNULL, 139918408847360, 139918408871935,
+STORE, 139918408871936, 139918408888319,
+STORE, 139918408847360, 139918408871935,
+ERASE, 139918408847360, 139918408871935,
+STORE, 139918408847360, 139918408871935,
+ERASE, 139918408871936, 139918408888319,
+STORE, 139918408871936, 139918408888319,
+STORE, 139918413299712, 139918413316095,
+SNULL, 139918408863743, 139918408871935,
+STORE, 139918408847360, 139918408863743,
+STORE, 139918408863744, 139918408871935,
+SNULL, 139918411083775, 139918411087871,
+STORE, 139918411079680, 139918411083775,
+STORE, 139918411083776, 139918411087871,
+SNULL, 93865836523519, 93865836527615,
+STORE, 93865836519424, 93865836523519,
+STORE, 93865836523520, 93865836527615,
+SNULL, 139918413348863, 139918413352959,
+STORE, 139918413344768, 139918413348863,
+STORE, 139918413348864, 139918413352959,
+ERASE, 139918413316096, 139918413344767,
+STORE, 93865848528896, 93865848664063,
+ };
+ unsigned long set29[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140734467944448, 140737488351231,
+SNULL, 140734467948543, 140737488351231,
+STORE, 140734467944448, 140734467948543,
+STORE, 140734467813376, 140734467948543,
+STORE, 94880407924736, 94880410177535,
+SNULL, 94880408055807, 94880410177535,
+STORE, 94880407924736, 94880408055807,
+STORE, 94880408055808, 94880410177535,
+ERASE, 94880408055808, 94880410177535,
+STORE, 94880410148864, 94880410157055,
+STORE, 94880410157056, 94880410177535,
+STORE, 140143367815168, 140143370067967,
+SNULL, 140143367958527, 140143370067967,
+STORE, 140143367815168, 140143367958527,
+STORE, 140143367958528, 140143370067967,
+ERASE, 140143367958528, 140143370067967,
+STORE, 140143370055680, 140143370063871,
+STORE, 140143370063872, 140143370067967,
+STORE, 140734468329472, 140734468333567,
+STORE, 140734468317184, 140734468329471,
+STORE, 140143370027008, 140143370055679,
+STORE, 140143370018816, 140143370027007,
+STORE, 140143365599232, 140143367815167,
+SNULL, 140143365599232, 140143365697535,
+STORE, 140143365697536, 140143367815167,
+STORE, 140143365599232, 140143365697535,
+SNULL, 140143367790591, 140143367815167,
+STORE, 140143365697536, 140143367790591,
+STORE, 140143367790592, 140143367815167,
+SNULL, 140143367790592, 140143367798783,
+STORE, 140143367798784, 140143367815167,
+STORE, 140143367790592, 140143367798783,
+ERASE, 140143367790592, 140143367798783,
+STORE, 140143367790592, 140143367798783,
+ERASE, 140143367798784, 140143367815167,
+STORE, 140143367798784, 140143367815167,
+STORE, 140143361802240, 140143365599231,
+SNULL, 140143361802240, 140143363461119,
+STORE, 140143363461120, 140143365599231,
+STORE, 140143361802240, 140143363461119,
+SNULL, 140143365558271, 140143365599231,
+STORE, 140143363461120, 140143365558271,
+STORE, 140143365558272, 140143365599231,
+SNULL, 140143365558272, 140143365582847,
+STORE, 140143365582848, 140143365599231,
+STORE, 140143365558272, 140143365582847,
+ERASE, 140143365558272, 140143365582847,
+STORE, 140143365558272, 140143365582847,
+ERASE, 140143365582848, 140143365599231,
+STORE, 140143365582848, 140143365599231,
+STORE, 140143370010624, 140143370027007,
+SNULL, 140143365574655, 140143365582847,
+STORE, 140143365558272, 140143365574655,
+STORE, 140143365574656, 140143365582847,
+SNULL, 140143367794687, 140143367798783,
+STORE, 140143367790592, 140143367794687,
+STORE, 140143367794688, 140143367798783,
+SNULL, 94880410152959, 94880410157055,
+STORE, 94880410148864, 94880410152959,
+STORE, 94880410152960, 94880410157055,
+SNULL, 140143370059775, 140143370063871,
+STORE, 140143370055680, 140143370059775,
+STORE, 140143370059776, 140143370063871,
+ERASE, 140143370027008, 140143370055679,
+STORE, 94880442400768, 94880442535935,
+STORE, 140143353409536, 140143361802239,
+SNULL, 140143353413631, 140143361802239,
+STORE, 140143353409536, 140143353413631,
+STORE, 140143353413632, 140143361802239,
+STORE, 140143345016832, 140143353409535,
+STORE, 140143210799104, 140143345016831,
+SNULL, 140143210799104, 140143239364607,
+STORE, 140143239364608, 140143345016831,
+STORE, 140143210799104, 140143239364607,
+ERASE, 140143210799104, 140143239364607,
+SNULL, 140143306473471, 140143345016831,
+STORE, 140143239364608, 140143306473471,
+STORE, 140143306473472, 140143345016831,
+ERASE, 140143306473472, 140143345016831,
+SNULL, 140143239499775, 140143306473471,
+STORE, 140143239364608, 140143239499775,
+STORE, 140143239499776, 140143306473471,
+SNULL, 140143345020927, 140143353409535,
+STORE, 140143345016832, 140143345020927,
+STORE, 140143345020928, 140143353409535,
+STORE, 140143336624128, 140143345016831,
+SNULL, 140143336628223, 140143345016831,
+STORE, 140143336624128, 140143336628223,
+STORE, 140143336628224, 140143345016831,
+STORE, 140143328231424, 140143336624127,
+SNULL, 140143328235519, 140143336624127,
+STORE, 140143328231424, 140143328235519,
+STORE, 140143328235520, 140143336624127,
+STORE, 140143319838720, 140143328231423,
+SNULL, 140143319842815, 140143328231423,
+STORE, 140143319838720, 140143319842815,
+STORE, 140143319842816, 140143328231423,
+STORE, 140143311446016, 140143319838719,
+STORE, 140143105146880, 140143239364607,
+STORE, 140143096754176, 140143105146879,
+STORE, 140143029645312, 140143096754175,
+ERASE, 140143029645312, 140143096754175,
+STORE, 140142962536448, 140143096754175,
+SNULL, 140142962536448, 140142970929151,
+STORE, 140142970929152, 140143096754175,
+STORE, 140142962536448, 140142970929151,
+ERASE, 140142962536448, 140142970929151,
+STORE, 140142962536448, 140142970929151,
+STORE, 140142828318720, 140142962536447,
+STORE, 140142819926016, 140142828318719,
+SNULL, 140142828318720, 140142836711423,
+STORE, 140142836711424, 140142962536447,
+STORE, 140142828318720, 140142836711423,
+ERASE, 140142828318720, 140142836711423,
+SNULL, 140143172255743, 140143239364607,
+STORE, 140143105146880, 140143172255743,
+STORE, 140143172255744, 140143239364607,
+ERASE, 140143172255744, 140143239364607,
+SNULL, 140143105282047, 140143172255743,
+STORE, 140143105146880, 140143105282047,
+STORE, 140143105282048, 140143172255743,
+SNULL, 140143038038015, 140143096754175,
+STORE, 140142970929152, 140143038038015,
+STORE, 140143038038016, 140143096754175,
+ERASE, 140143038038016, 140143096754175,
+SNULL, 140142971064319, 140143038038015,
+STORE, 140142970929152, 140142971064319,
+STORE, 140142971064320, 140143038038015,
+SNULL, 140142903820287, 140142962536447,
+STORE, 140142836711424, 140142903820287,
+STORE, 140142903820288, 140142962536447,
+ERASE, 140142903820288, 140142962536447,
+SNULL, 140142836846591, 140142903820287,
+STORE, 140142836711424, 140142836846591,
+STORE, 140142836846592, 140142903820287,
+STORE, 140142685708288, 140142819926015,
+SNULL, 140143311450111, 140143319838719,
+STORE, 140143311446016, 140143311450111,
+STORE, 140143311450112, 140143319838719,
+SNULL, 140142962540543, 140142970929151,
+STORE, 140142962536448, 140142962540543,
+STORE, 140142962540544, 140142970929151,
+SNULL, 140142685708288, 140142702493695,
+STORE, 140142702493696, 140142819926015,
+STORE, 140142685708288, 140142702493695,
+ERASE, 140142685708288, 140142702493695,
+SNULL, 140142769602559, 140142819926015,
+STORE, 140142702493696, 140142769602559,
+STORE, 140142769602560, 140142819926015,
+ERASE, 140142769602560, 140142819926015,
+SNULL, 140142702628863, 140142769602559,
+STORE, 140142702493696, 140142702628863,
+STORE, 140142702628864, 140142769602559,
+STORE, 140143230971904, 140143239364607,
+SNULL, 140143230975999, 140143239364607,
+STORE, 140143230971904, 140143230975999,
+STORE, 140143230976000, 140143239364607,
+SNULL, 140143096758271, 140143105146879,
+STORE, 140143096754176, 140143096758271,
+STORE, 140143096758272, 140143105146879,
+STORE, 140143222579200, 140143230971903,
+SNULL, 140143222583295, 140143230971903,
+STORE, 140143222579200, 140143222583295,
+STORE, 140143222583296, 140143230971903,
+STORE, 140143214186496, 140143222579199,
+SNULL, 140142819930111, 140142828318719,
+STORE, 140142819926016, 140142819930111,
+STORE, 140142819930112, 140142828318719,
+STORE, 140143205793792, 140143222579199,
+SNULL, 140143205793792, 140143214186495,
+STORE, 140143214186496, 140143222579199,
+STORE, 140143205793792, 140143214186495,
+SNULL, 140143214190591, 140143222579199,
+STORE, 140143214186496, 140143214190591,
+STORE, 140143214190592, 140143222579199,
+SNULL, 140143205797887, 140143214186495,
+STORE, 140143205793792, 140143205797887,
+STORE, 140143205797888, 140143214186495,
+STORE, 140143197401088, 140143205793791,
+SNULL, 140143197405183, 140143205793791,
+STORE, 140143197401088, 140143197405183,
+STORE, 140143197405184, 140143205793791,
+STORE, 140143189008384, 140143197401087,
+STORE, 140143180615680, 140143197401087,
+STORE, 140143088361472, 140143096754175,
+SNULL, 140143180619775, 140143197401087,
+STORE, 140143180615680, 140143180619775,
+STORE, 140143180619776, 140143197401087,
+SNULL, 140143180619776, 140143189008383,
+STORE, 140143189008384, 140143197401087,
+STORE, 140143180619776, 140143189008383,
+SNULL, 140143189012479, 140143197401087,
+STORE, 140143189008384, 140143189012479,
+STORE, 140143189012480, 140143197401087,
+SNULL, 140143088365567, 140143096754175,
+STORE, 140143088361472, 140143088365567,
+STORE, 140143088365568, 140143096754175,
+STORE, 140143079968768, 140143088361471,
+SNULL, 140143079972863, 140143088361471,
+STORE, 140143079968768, 140143079972863,
+STORE, 140143079972864, 140143088361471,
+STORE, 140143071576064, 140143079968767,
+SNULL, 140143071580159, 140143079968767,
+STORE, 140143071576064, 140143071580159,
+STORE, 140143071580160, 140143079968767,
+STORE, 140143063183360, 140143071576063,
+STORE, 140143054790656, 140143071576063,
+SNULL, 140143054794751, 140143071576063,
+STORE, 140143054790656, 140143054794751,
+STORE, 140143054794752, 140143071576063,
+SNULL, 140143054794752, 140143063183359,
+STORE, 140143063183360, 140143071576063,
+STORE, 140143054794752, 140143063183359,
+SNULL, 140143063187455, 140143071576063,
+STORE, 140143063183360, 140143063187455,
+STORE, 140143063187456, 140143071576063,
+STORE, 140143046397952, 140143054790655,
+STORE, 140142954143744, 140142962536447,
+STORE, 140142945751040, 140142962536447,
+STORE, 140142937358336, 140142962536447,
+STORE, 140142928965632, 140142962536447,
+STORE, 140142568275968, 140142702493695,
+SNULL, 140142635384831, 140142702493695,
+STORE, 140142568275968, 140142635384831,
+STORE, 140142635384832, 140142702493695,
+ERASE, 140142635384832, 140142702493695,
+STORE, 140142920572928, 140142962536447,
+STORE, 140142912180224, 140142962536447,
+STORE, 140142568275968, 140142702493695,
+SNULL, 140142568275968, 140142635384831,
+STORE, 140142635384832, 140142702493695,
+STORE, 140142568275968, 140142635384831,
+SNULL, 140142635519999, 140142702493695,
+STORE, 140142635384832, 140142635519999,
+STORE, 140142635520000, 140142702493695,
+STORE, 140142819930112, 140142836711423,
+STORE, 140142811533312, 140142819926015,
+STORE, 140142434058240, 140142635384831,
+SNULL, 140142501167103, 140142635384831,
+STORE, 140142434058240, 140142501167103,
+STORE, 140142501167104, 140142635384831,
+SNULL, 140142501167104, 140142568275967,
+STORE, 140142568275968, 140142635384831,
+STORE, 140142501167104, 140142568275967,
+ERASE, 140142501167104, 140142568275967,
+STORE, 140142299840512, 140142501167103,
+STORE, 140142803140608, 140142819926015,
+SNULL, 140142366949375, 140142501167103,
+STORE, 140142299840512, 140142366949375,
+STORE, 140142366949376, 140142501167103,
+SNULL, 140142366949376, 140142434058239,
+STORE, 140142434058240, 140142501167103,
+STORE, 140142366949376, 140142434058239,
+ERASE, 140142366949376, 140142434058239,
+STORE, 140142794747904, 140142819926015,
+STORE, 140142786355200, 140142819926015,
+STORE, 140142299840512, 140142501167103,
+STORE, 140142777962496, 140142819926015,
+STORE, 140142559883264, 140142568275967,
+STORE, 140142232731648, 140142501167103,
+STORE, 140142551490560, 140142568275967,
+SNULL, 140142777962496, 140142803140607,
+STORE, 140142803140608, 140142819926015,
+STORE, 140142777962496, 140142803140607,
+SNULL, 140142803144703, 140142819926015,
+STORE, 140142803140608, 140142803144703,
+STORE, 140142803144704, 140142819926015,
+STORE, 140142543097856, 140142568275967,
+STORE, 140142098513920, 140142501167103,
+SNULL, 140142165622783, 140142501167103,
+STORE, 140142098513920, 140142165622783,
+STORE, 140142165622784, 140142501167103,
+SNULL, 140142165622784, 140142232731647,
+STORE, 140142232731648, 140142501167103,
+STORE, 140142165622784, 140142232731647,
+ERASE, 140142165622784, 140142232731647,
+SNULL, 140142568411135, 140142635384831,
+STORE, 140142568275968, 140142568411135,
+STORE, 140142568411136, 140142635384831,
+STORE, 140141964296192, 140142165622783,
+SNULL, 140142912180224, 140142928965631,
+STORE, 140142928965632, 140142962536447,
+STORE, 140142912180224, 140142928965631,
+SNULL, 140142928969727, 140142962536447,
+STORE, 140142928965632, 140142928969727,
+STORE, 140142928969728, 140142962536447,
+STORE, 140141830078464, 140142165622783,
+SNULL, 140142912184319, 140142928965631,
+STORE, 140142912180224, 140142912184319,
+STORE, 140142912184320, 140142928965631,
+SNULL, 140142232731648, 140142434058239,
+STORE, 140142434058240, 140142501167103,
+STORE, 140142232731648, 140142434058239,
+SNULL, 140142434193407, 140142501167103,
+STORE, 140142434058240, 140142434193407,
+STORE, 140142434193408, 140142501167103,
+SNULL, 140142232731648, 140142299840511,
+STORE, 140142299840512, 140142434058239,
+STORE, 140142232731648, 140142299840511,
+SNULL, 140142299975679, 140142434058239,
+STORE, 140142299840512, 140142299975679,
+STORE, 140142299975680, 140142434058239,
+SNULL, 140142928969728, 140142954143743,
+STORE, 140142954143744, 140142962536447,
+STORE, 140142928969728, 140142954143743,
+SNULL, 140142954147839, 140142962536447,
+STORE, 140142954143744, 140142954147839,
+STORE, 140142954147840, 140142962536447,
+STORE, 140141830078464, 140142299840511,
+SNULL, 140142543097856, 140142559883263,
+STORE, 140142559883264, 140142568275967,
+STORE, 140142543097856, 140142559883263,
+SNULL, 140142559887359, 140142568275967,
+STORE, 140142559883264, 140142559887359,
+STORE, 140142559887360, 140142568275967,
+STORE, 140142534705152, 140142559883263,
+SNULL, 140142928969728, 140142945751039,
+STORE, 140142945751040, 140142954143743,
+STORE, 140142928969728, 140142945751039,
+SNULL, 140142945755135, 140142954143743,
+STORE, 140142945751040, 140142945755135,
+STORE, 140142945755136, 140142954143743,
+SNULL, 140142299975680, 140142366949375,
+STORE, 140142366949376, 140142434058239,
+STORE, 140142299975680, 140142366949375,
+SNULL, 140142367084543, 140142434058239,
+STORE, 140142366949376, 140142367084543,
+STORE, 140142367084544, 140142434058239,
+SNULL, 140142928969728, 140142937358335,
+STORE, 140142937358336, 140142945751039,
+STORE, 140142928969728, 140142937358335,
+SNULL, 140142937362431, 140142945751039,
+STORE, 140142937358336, 140142937362431,
+STORE, 140142937362432, 140142945751039,
+SNULL, 140141830078464, 140142232731647,
+STORE, 140142232731648, 140142299840511,
+STORE, 140141830078464, 140142232731647,
+SNULL, 140142232866815, 140142299840511,
+STORE, 140142232731648, 140142232866815,
+STORE, 140142232866816, 140142299840511,
+SNULL, 140142534705152, 140142543097855,
+STORE, 140142543097856, 140142559883263,
+STORE, 140142534705152, 140142543097855,
+SNULL, 140142543101951, 140142559883263,
+STORE, 140142543097856, 140142543101951,
+STORE, 140142543101952, 140142559883263,
+STORE, 140142526312448, 140142543097855,
+STORE, 140142517919744, 140142543097855,
+SNULL, 140141830078464, 140142098513919,
+STORE, 140142098513920, 140142232731647,
+STORE, 140141830078464, 140142098513919,
+SNULL, 140142098649087, 140142232731647,
+STORE, 140142098513920, 140142098649087,
+STORE, 140142098649088, 140142232731647,
+SNULL, 140142031405055, 140142098513919,
+STORE, 140141830078464, 140142031405055,
+STORE, 140142031405056, 140142098513919,
+ERASE, 140142031405056, 140142098513919,
+SNULL, 140141830078464, 140141964296191,
+STORE, 140141964296192, 140142031405055,
+STORE, 140141830078464, 140141964296191,
+SNULL, 140141964431359, 140142031405055,
+STORE, 140141964296192, 140141964431359,
+STORE, 140141964431360, 140142031405055,
+STORE, 140142509527040, 140142543097855,
+SNULL, 140141897187327, 140141964296191,
+STORE, 140141830078464, 140141897187327,
+STORE, 140141897187328, 140141964296191,
+ERASE, 140141897187328, 140141964296191,
+SNULL, 140141830213631, 140141897187327,
+STORE, 140141830078464, 140141830213631,
+STORE, 140141830213632, 140141897187327,
+SNULL, 140142803144704, 140142811533311,
+STORE, 140142811533312, 140142819926015,
+STORE, 140142803144704, 140142811533311,
+SNULL, 140142811537407, 140142819926015,
+STORE, 140142811533312, 140142811537407,
+STORE, 140142811537408, 140142819926015,
+SNULL, 140142098649088, 140142165622783,
+STORE, 140142165622784, 140142232731647,
+STORE, 140142098649088, 140142165622783,
+SNULL, 140142165757951, 140142232731647,
+STORE, 140142165622784, 140142165757951,
+STORE, 140142165757952, 140142232731647,
+STORE, 140142090121216, 140142098513919,
+SNULL, 140142777962496, 140142786355199,
+STORE, 140142786355200, 140142803140607,
+STORE, 140142777962496, 140142786355199,
+SNULL, 140142786359295, 140142803140607,
+STORE, 140142786355200, 140142786359295,
+STORE, 140142786359296, 140142803140607,
+SNULL, 140142509527040, 140142534705151,
+STORE, 140142534705152, 140142543097855,
+STORE, 140142509527040, 140142534705151,
+SNULL, 140142534709247, 140142543097855,
+STORE, 140142534705152, 140142534709247,
+STORE, 140142534709248, 140142543097855,
+STORE, 140142081728512, 140142098513919,
+SNULL, 140142786359296, 140142794747903,
+STORE, 140142794747904, 140142803140607,
+STORE, 140142786359296, 140142794747903,
+SNULL, 140142794751999, 140142803140607,
+STORE, 140142794747904, 140142794751999,
+STORE, 140142794752000, 140142803140607,
+STORE, 140142073335808, 140142098513919,
+SNULL, 140142073339903, 140142098513919,
+STORE, 140142073335808, 140142073339903,
+STORE, 140142073339904, 140142098513919,
+SNULL, 140142543101952, 140142551490559,
+STORE, 140142551490560, 140142559883263,
+STORE, 140142543101952, 140142551490559,
+SNULL, 140142551494655, 140142559883263,
+STORE, 140142551490560, 140142551494655,
+STORE, 140142551494656, 140142559883263,
+SNULL, 140142509527040, 140142517919743,
+STORE, 140142517919744, 140142534705151,
+STORE, 140142509527040, 140142517919743,
+SNULL, 140142517923839, 140142534705151,
+STORE, 140142517919744, 140142517923839,
+STORE, 140142517923840, 140142534705151,
+STORE, 140142064943104, 140142073335807,
+SNULL, 140142073339904, 140142090121215,
+STORE, 140142090121216, 140142098513919,
+STORE, 140142073339904, 140142090121215,
+SNULL, 140142090125311, 140142098513919,
+STORE, 140142090121216, 140142090125311,
+STORE, 140142090125312, 140142098513919,
+STORE, 140142056550400, 140142073335807,
+SNULL, 140142056554495, 140142073335807,
+STORE, 140142056550400, 140142056554495,
+STORE, 140142056554496, 140142073335807,
+STORE, 140142048157696, 140142056550399,
+SNULL, 140142509531135, 140142517919743,
+STORE, 140142509527040, 140142509531135,
+STORE, 140142509531136, 140142517919743,
+SNULL, 140142777966591, 140142786355199,
+STORE, 140142777962496, 140142777966591,
+STORE, 140142777966592, 140142786355199,
+SNULL, 140143046402047, 140143054790655,
+STORE, 140143046397952, 140143046402047,
+STORE, 140143046402048, 140143054790655,
+SNULL, 140142912184320, 140142920572927,
+STORE, 140142920572928, 140142928965631,
+STORE, 140142912184320, 140142920572927,
+SNULL, 140142920577023, 140142928965631,
+STORE, 140142920572928, 140142920577023,
+STORE, 140142920577024, 140142928965631,
+STORE, 140142039764992, 140142056550399,
+STORE, 140141955903488, 140141964296191,
+SNULL, 140142819930112, 140142828318719,
+STORE, 140142828318720, 140142836711423,
+STORE, 140142819930112, 140142828318719,
+SNULL, 140142828322815, 140142836711423,
+STORE, 140142828318720, 140142828322815,
+STORE, 140142828322816, 140142836711423,
+SNULL, 140142517923840, 140142526312447,
+STORE, 140142526312448, 140142534705151,
+STORE, 140142517923840, 140142526312447,
+SNULL, 140142526316543, 140142534705151,
+STORE, 140142526312448, 140142526316543,
+STORE, 140142526316544, 140142534705151,
+STORE, 140141947510784, 140141964296191,
+SNULL, 140142056554496, 140142064943103,
+STORE, 140142064943104, 140142073335807,
+STORE, 140142056554496, 140142064943103,
+SNULL, 140142064947199, 140142073335807,
+STORE, 140142064943104, 140142064947199,
+STORE, 140142064947200, 140142073335807,
+SNULL, 140142073339904, 140142081728511,
+STORE, 140142081728512, 140142090121215,
+STORE, 140142073339904, 140142081728511,
+SNULL, 140142081732607, 140142090121215,
+STORE, 140142081728512, 140142081732607,
+STORE, 140142081732608, 140142090121215,
+STORE, 140141939118080, 140141964296191,
+STORE, 140141930725376, 140141964296191,
+STORE, 140141922332672, 140141964296191,
+STORE, 140141913939968, 140141964296191,
+SNULL, 140141913939968, 140141922332671,
+STORE, 140141922332672, 140141964296191,
+STORE, 140141913939968, 140141922332671,
+SNULL, 140141922336767, 140141964296191,
+STORE, 140141922332672, 140141922336767,
+STORE, 140141922336768, 140141964296191,
+STORE, 140141905547264, 140141922332671,
+SNULL, 140141905551359, 140141922332671,
+STORE, 140141905547264, 140141905551359,
+STORE, 140141905551360, 140141922332671,
+STORE, 140141821685760, 140141830078463,
+STORE, 140141813293056, 140141830078463,
+STORE, 140141804900352, 140141830078463,
+STORE, 140141796507648, 140141830078463,
+SNULL, 140141796511743, 140141830078463,
+STORE, 140141796507648, 140141796511743,
+STORE, 140141796511744, 140141830078463,
+SNULL, 140141922336768, 140141955903487,
+STORE, 140141955903488, 140141964296191,
+STORE, 140141922336768, 140141955903487,
+SNULL, 140141955907583, 140141964296191,
+STORE, 140141955903488, 140141955907583,
+STORE, 140141955907584, 140141964296191,
+STORE, 140141788114944, 140141796507647,
+STORE, 140141779722240, 140141796507647,
+SNULL, 140141779722240, 140141788114943,
+STORE, 140141788114944, 140141796507647,
+STORE, 140141779722240, 140141788114943,
+SNULL, 140141788119039, 140141796507647,
+STORE, 140141788114944, 140141788119039,
+STORE, 140141788119040, 140141796507647,
+SNULL, 140141922336768, 140141947510783,
+STORE, 140141947510784, 140141955903487,
+STORE, 140141922336768, 140141947510783,
+SNULL, 140141947514879, 140141955903487,
+STORE, 140141947510784, 140141947514879,
+STORE, 140141947514880, 140141955903487,
+SNULL, 140142039764992, 140142048157695,
+STORE, 140142048157696, 140142056550399,
+STORE, 140142039764992, 140142048157695,
+SNULL, 140142048161791, 140142056550399,
+STORE, 140142048157696, 140142048161791,
+STORE, 140142048161792, 140142056550399,
+SNULL, 140142039769087, 140142048157695,
+STORE, 140142039764992, 140142039769087,
+STORE, 140142039769088, 140142048157695,
+SNULL, 140141796511744, 140141804900351,
+STORE, 140141804900352, 140141830078463,
+STORE, 140141796511744, 140141804900351,
+SNULL, 140141804904447, 140141830078463,
+STORE, 140141804900352, 140141804904447,
+STORE, 140141804904448, 140141830078463,
+STORE, 140141771329536, 140141788114943,
+STORE, 140141762936832, 140141788114943,
+STORE, 140141754544128, 140141788114943,
+SNULL, 140141804904448, 140141821685759,
+STORE, 140141821685760, 140141830078463,
+STORE, 140141804904448, 140141821685759,
+SNULL, 140141821689855, 140141830078463,
+STORE, 140141821685760, 140141821689855,
+STORE, 140141821689856, 140141830078463,
+SNULL, 140141922336768, 140141939118079,
+STORE, 140141939118080, 140141947510783,
+STORE, 140141922336768, 140141939118079,
+SNULL, 140141939122175, 140141947510783,
+STORE, 140141939118080, 140141939122175,
+STORE, 140141939122176, 140141947510783,
+SNULL, 140141905551360, 140141913939967,
+STORE, 140141913939968, 140141922332671,
+STORE, 140141905551360, 140141913939967,
+SNULL, 140141913944063, 140141922332671,
+STORE, 140141913939968, 140141913944063,
+STORE, 140141913944064, 140141922332671,
+STORE, 140141746151424, 140141788114943,
+STORE, 140141737758720, 140141788114943,
+SNULL, 140141804904448, 140141813293055,
+STORE, 140141813293056, 140141821685759,
+STORE, 140141804904448, 140141813293055,
+SNULL, 140141813297151, 140141821685759,
+STORE, 140141813293056, 140141813297151,
+STORE, 140141813297152, 140141821685759,
+STORE, 140141729366016, 140141788114943,
+STORE, 140141720973312, 140141788114943,
+STORE, 140141712580608, 140141788114943,
+SNULL, 140141712584703, 140141788114943,
+STORE, 140141712580608, 140141712584703,
+STORE, 140141712584704, 140141788114943,
+SNULL, 140141922336768, 140141930725375,
+STORE, 140141930725376, 140141939118079,
+STORE, 140141922336768, 140141930725375,
+SNULL, 140141930729471, 140141939118079,
+STORE, 140141930725376, 140141930729471,
+STORE, 140141930729472, 140141939118079,
+STORE, 140141704187904, 140141712580607,
+SNULL, 140141704191999, 140141712580607,
+STORE, 140141704187904, 140141704191999,
+STORE, 140141704192000, 140141712580607,
+STORE, 140141695795200, 140141704187903,
+STORE, 140141687402496, 140141704187903,
+SNULL, 140141712584704, 140141771329535,
+STORE, 140141771329536, 140141788114943,
+STORE, 140141712584704, 140141771329535,
+SNULL, 140141771333631, 140141788114943,
+STORE, 140141771329536, 140141771333631,
+STORE, 140141771333632, 140141788114943,
+SNULL, 140141771333632, 140141779722239,
+STORE, 140141779722240, 140141788114943,
+STORE, 140141771333632, 140141779722239,
+SNULL, 140141779726335, 140141788114943,
+STORE, 140141779722240, 140141779726335,
+STORE, 140141779726336, 140141788114943,
+STORE, 140141679009792, 140141704187903,
+SNULL, 140141679013887, 140141704187903,
+STORE, 140141679009792, 140141679013887,
+STORE, 140141679013888, 140141704187903,
+STORE, 140141670617088, 140141679009791,
+SNULL, 140141670621183, 140141679009791,
+STORE, 140141670617088, 140141670621183,
+STORE, 140141670621184, 140141679009791,
+STORE, 140141662224384, 140141670617087,
+SNULL, 140141712584704, 140141737758719,
+STORE, 140141737758720, 140141771329535,
+STORE, 140141712584704, 140141737758719,
+SNULL, 140141737762815, 140141771329535,
+STORE, 140141737758720, 140141737762815,
+STORE, 140141737762816, 140141771329535,
+SNULL, 140141712584704, 140141729366015,
+STORE, 140141729366016, 140141737758719,
+STORE, 140141712584704, 140141729366015,
+SNULL, 140141729370111, 140141737758719,
+STORE, 140141729366016, 140141729370111,
+STORE, 140141729370112, 140141737758719,
+SNULL, 140141737762816, 140141746151423,
+STORE, 140141746151424, 140141771329535,
+STORE, 140141737762816, 140141746151423,
+SNULL, 140141746155519, 140141771329535,
+STORE, 140141746151424, 140141746155519,
+STORE, 140141746155520, 140141771329535,
+STORE, 140141653831680, 140141670617087,
+SNULL, 140141746155520, 140141762936831,
+STORE, 140141762936832, 140141771329535,
+STORE, 140141746155520, 140141762936831,
+SNULL, 140141762940927, 140141771329535,
+STORE, 140141762936832, 140141762940927,
+STORE, 140141762940928, 140141771329535,
+STORE, 140141645438976, 140141670617087,
+SNULL, 140141645443071, 140141670617087,
+STORE, 140141645438976, 140141645443071,
+STORE, 140141645443072, 140141670617087,
+SNULL, 140141712584704, 140141720973311,
+STORE, 140141720973312, 140141729366015,
+STORE, 140141712584704, 140141720973311,
+SNULL, 140141720977407, 140141729366015,
+STORE, 140141720973312, 140141720977407,
+STORE, 140141720977408, 140141729366015,
+STORE, 140141637046272, 140141645438975,
+SNULL, 140141637050367, 140141645438975,
+STORE, 140141637046272, 140141637050367,
+STORE, 140141637050368, 140141645438975,
+STORE, 140141628653568, 140141637046271,
+SNULL, 140141628657663, 140141637046271,
+STORE, 140141628653568, 140141628657663,
+STORE, 140141628657664, 140141637046271,
+STORE, 140141620260864, 140141628653567,
+SNULL, 140141679013888, 140141687402495,
+STORE, 140141687402496, 140141704187903,
+STORE, 140141679013888, 140141687402495,
+SNULL, 140141687406591, 140141704187903,
+STORE, 140141687402496, 140141687406591,
+STORE, 140141687406592, 140141704187903,
+SNULL, 140141746155520, 140141754544127,
+STORE, 140141754544128, 140141762936831,
+STORE, 140141746155520, 140141754544127,
+SNULL, 140141754548223, 140141762936831,
+STORE, 140141754544128, 140141754548223,
+STORE, 140141754548224, 140141762936831,
+SNULL, 140141687406592, 140141695795199,
+STORE, 140141695795200, 140141704187903,
+STORE, 140141687406592, 140141695795199,
+SNULL, 140141695799295, 140141704187903,
+STORE, 140141695795200, 140141695799295,
+STORE, 140141695799296, 140141704187903,
+STORE, 140141611868160, 140141628653567,
+SNULL, 140141611872255, 140141628653567,
+STORE, 140141611868160, 140141611872255,
+STORE, 140141611872256, 140141628653567,
+SNULL, 140141645443072, 140141662224383,
+STORE, 140141662224384, 140141670617087,
+STORE, 140141645443072, 140141662224383,
+SNULL, 140141662228479, 140141670617087,
+STORE, 140141662224384, 140141662228479,
+STORE, 140141662228480, 140141670617087,
+STORE, 140141603475456, 140141611868159,
+SNULL, 140141603479551, 140141611868159,
+STORE, 140141603475456, 140141603479551,
+STORE, 140141603479552, 140141611868159,
+STORE, 140141595082752, 140141603475455,
+SNULL, 140141645443072, 140141653831679,
+STORE, 140141653831680, 140141662224383,
+STORE, 140141645443072, 140141653831679,
+SNULL, 140141653835775, 140141662224383,
+STORE, 140141653831680, 140141653835775,
+STORE, 140141653835776, 140141662224383,
+STORE, 140141586690048, 140141603475455,
+SNULL, 140141611872256, 140141620260863,
+STORE, 140141620260864, 140141628653567,
+STORE, 140141611872256, 140141620260863,
+SNULL, 140141620264959, 140141628653567,
+STORE, 140141620260864, 140141620264959,
+STORE, 140141620264960, 140141628653567,
+SNULL, 140141586690048, 140141595082751,
+STORE, 140141595082752, 140141603475455,
+STORE, 140141586690048, 140141595082751,
+SNULL, 140141595086847, 140141603475455,
+STORE, 140141595082752, 140141595086847,
+STORE, 140141595086848, 140141603475455,
+STORE, 140141578297344, 140141595082751,
+SNULL, 140141578301439, 140141595082751,
+STORE, 140141578297344, 140141578301439,
+STORE, 140141578301440, 140141595082751,
+SNULL, 140141578301440, 140141586690047,
+STORE, 140141586690048, 140141595082751,
+STORE, 140141578301440, 140141586690047,
+SNULL, 140141586694143, 140141595082751,
+STORE, 140141586690048, 140141586694143,
+STORE, 140141586694144, 140141595082751,
+STORE, 140143370027008, 140143370055679,
+STORE, 140143309254656, 140143311446015,
+SNULL, 140143309254656, 140143309344767,
+STORE, 140143309344768, 140143311446015,
+STORE, 140143309254656, 140143309344767,
+SNULL, 140143311437823, 140143311446015,
+STORE, 140143309344768, 140143311437823,
+STORE, 140143311437824, 140143311446015,
+ERASE, 140143311437824, 140143311446015,
+STORE, 140143311437824, 140143311446015,
+SNULL, 140143311441919, 140143311446015,
+STORE, 140143311437824, 140143311441919,
+STORE, 140143311441920, 140143311446015,
+ERASE, 140143370027008, 140143370055679,
+ERASE, 140142912180224, 140142912184319,
+ERASE, 140142912184320, 140142920572927,
+ERASE, 140142945751040, 140142945755135,
+ERASE, 140142945755136, 140142954143743,
+ERASE, 140142090121216, 140142090125311,
+ERASE, 140142090125312, 140142098513919,
+ERASE, 140142794747904, 140142794751999,
+ERASE, 140142794752000, 140142803140607,
+ERASE, 140141913939968, 140141913944063,
+ERASE, 140141913944064, 140141922332671,
+ERASE, 140141746151424, 140141746155519,
+ERASE, 140141746155520, 140141754544127,
+ERASE, 140142954143744, 140142954147839,
+ERASE, 140142954147840, 140142962536447,
+ERASE, 140142081728512, 140142081732607,
+ERASE, 140142081732608, 140142090121215,
+ERASE, 140141905547264, 140141905551359,
+ERASE, 140141905551360, 140141913939967,
+ERASE, 140141729366016, 140141729370111,
+ERASE, 140141729370112, 140141737758719,
+ERASE, 140142920572928, 140142920577023,
+ERASE, 140142920577024, 140142928965631,
+ERASE, 140142039764992, 140142039769087,
+ERASE, 140142039769088, 140142048157695,
+ERASE, 140141679009792, 140141679013887,
+ERASE, 140141679013888, 140141687402495,
+ERASE, 140142551490560, 140142551494655,
+ERASE, 140142551494656, 140142559883263,
+ERASE, 140141947510784, 140141947514879,
+ERASE, 140141947514880, 140141955903487,
+ERASE, 140141771329536, 140141771333631,
+ERASE, 140141771333632, 140141779722239,
+ERASE, 140142928965632, 140142928969727,
+ERASE, 140142928969728, 140142937358335,
+ERASE, 140142073335808, 140142073339903,
+ERASE, 140142073339904, 140142081728511,
+ERASE, 140142543097856, 140142543101951,
+ERASE, 140142543101952, 140142551490559,
+ERASE, 140141955903488, 140141955907583,
+ERASE, 140141955907584, 140141964296191,
+ERASE, 140141704187904, 140141704191999,
+ERASE, 140141704192000, 140141712580607,
+ERASE, 140142786355200, 140142786359295,
+ERASE, 140142786359296, 140142794747903,
+ERASE, 140142056550400, 140142056554495,
+ERASE, 140142056554496, 140142064943103,
+ERASE, 140142828318720, 140142828322815,
+ERASE, 140142828322816, 140142836711423,
+ERASE, 140141788114944, 140141788119039,
+ERASE, 140141788119040, 140141796507647,
+ERASE, 140141695795200, 140141695799295,
+ERASE, 140141695799296, 140141704187903,
+ERASE, 140141578297344, 140141578301439,
+ERASE, 140141578301440, 140141586690047,
+ERASE, 140141611868160, 140141611872255,
+ERASE, 140141611872256, 140141620260863,
+ERASE, 140142811533312, 140142811537407,
+ERASE, 140142811537408, 140142819926015,
+ERASE, 140142064943104, 140142064947199,
+ERASE, 140142064947200, 140142073335807,
+ERASE, 140141628653568, 140141628657663,
+ERASE, 140141628657664, 140141637046271,
+ERASE, 140143046397952, 140143046402047,
+ERASE, 140143046402048, 140143054790655,
+ERASE, 140141796507648, 140141796511743,
+ERASE, 140141796511744, 140141804900351,
+ERASE, 140142803140608, 140142803144703,
+ERASE, 140142803144704, 140142811533311,
+ERASE, 140142509527040, 140142509531135,
+ERASE, 140142509531136, 140142517919743,
+ERASE, 140141821685760, 140141821689855,
+ERASE, 140141821689856, 140141830078463,
+ERASE, 140142777962496, 140142777966591,
+ERASE, 140142777966592, 140142786355199,
+ERASE, 140141804900352, 140141804904447,
+ERASE, 140141804904448, 140141813293055,
+ERASE, 140141930725376, 140141930729471,
+ERASE, 140141930729472, 140141939118079,
+ERASE, 140142937358336, 140142937362431,
+ERASE, 140142937362432, 140142945751039,
+ERASE, 140142559883264, 140142559887359,
+ERASE, 140142559887360, 140142568275967,
+ERASE, 140142534705152, 140142534709247,
+ERASE, 140142534709248, 140142543097855,
+ERASE, 140142048157696, 140142048161791,
+ERASE, 140142048161792, 140142056550399,
+ERASE, 140141754544128, 140141754548223,
+ERASE, 140141754548224, 140141762936831,
+ERASE, 140141939118080, 140141939122175,
+ERASE, 140141939122176, 140141947510783,
+ERASE, 140141653831680, 140141653835775,
+ERASE, 140141653835776, 140141662224383,
+ERASE, 140141712580608, 140141712584703,
+ERASE, 140141712584704, 140141720973311,
+ERASE, 140141645438976, 140141645443071,
+ERASE, 140141645443072, 140141653831679,
+ERASE, 140141687402496, 140141687406591,
+ERASE, 140141687406592, 140141695795199,
+ERASE, 140141662224384, 140141662228479,
+ERASE, 140141662228480, 140141670617087,
+ERASE, 140141922332672, 140141922336767,
+ERASE, 140141922336768, 140141930725375,
+ERASE, 140141737758720, 140141737762815,
+ERASE, 140141737762816, 140141746151423,
+ERASE, 140141637046272, 140141637050367,
+ERASE, 140141637050368, 140141645438975,
+ERASE, 140142517919744, 140142517923839,
+ERASE, 140142517923840, 140142526312447,
+ERASE, 140143096754176, 140143096758271,
+ERASE, 140143096758272, 140143105146879,
+ERASE, 140141595082752, 140141595086847,
+ERASE, 140141595086848, 140141603475455,
+ERASE, 140141762936832, 140141762940927,
+ERASE, 140141762940928, 140141771329535,
+ERASE, 140143311446016, 140143311450111,
+ERASE, 140143311450112, 140143319838719,
+ERASE, 140142526312448, 140142526316543,
+ERASE, 140142526316544, 140142534705151,
+ERASE, 140142819926016, 140142819930111,
+ERASE, 140142819930112, 140142828318719,
+ERASE, 140143180615680, 140143180619775,
+ERASE, 140143180619776, 140143189008383,
+ERASE, 140142962536448, 140142962540543,
+ERASE, 140142962540544, 140142970929151,
+ERASE, 140143214186496, 140143214190591,
+ERASE, 140143214190592, 140143222579199,
+ERASE, 140143088361472, 140143088365567,
+ERASE, 140143088365568, 140143096754175,
+ERASE, 140141586690048, 140141586694143,
+ERASE, 140141586694144, 140141595082751,
+ERASE, 140143230971904, 140143230975999,
+ERASE, 140143230976000, 140143239364607,
+ERASE, 140141779722240, 140141779726335,
+ERASE, 140141779726336, 140141788114943,
+ERASE, 140141670617088, 140141670621183,
+ERASE, 140141670621184, 140141679009791,
+ERASE, 140141813293056, 140141813297151,
+ERASE, 140141813297152, 140141821685759,
+ERASE, 140143222579200, 140143222583295,
+ERASE, 140143222583296, 140143230971903,
+ERASE, 140143189008384, 140143189012479,
+ERASE, 140143189012480, 140143197401087,
+ERASE, 140143071576064, 140143071580159,
+ERASE, 140143071580160, 140143079968767,
+ERASE, 140141620260864, 140141620264959,
+ERASE, 140141620264960, 140141628653567,
+ERASE, 140141603475456, 140141603479551,
+ERASE, 140141603479552, 140141611868159,
+ERASE, 140141720973312, 140141720977407,
+ERASE, 140141720977408, 140141729366015,
+ERASE, 140143079968768, 140143079972863,
+ERASE, 140143079972864, 140143088361471,
+ERASE, 140143205793792, 140143205797887,
+ERASE, 140143205797888, 140143214186495,
+ };
+ unsigned long set30[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140733436743680, 140737488351231,
+SNULL, 140733436747775, 140737488351231,
+STORE, 140733436743680, 140733436747775,
+STORE, 140733436612608, 140733436747775,
+STORE, 94630728904704, 94630731157503,
+SNULL, 94630729035775, 94630731157503,
+STORE, 94630728904704, 94630729035775,
+STORE, 94630729035776, 94630731157503,
+ERASE, 94630729035776, 94630731157503,
+STORE, 94630731128832, 94630731137023,
+STORE, 94630731137024, 94630731157503,
+STORE, 140165750841344, 140165753094143,
+SNULL, 140165750984703, 140165753094143,
+STORE, 140165750841344, 140165750984703,
+STORE, 140165750984704, 140165753094143,
+ERASE, 140165750984704, 140165753094143,
+STORE, 140165753081856, 140165753090047,
+STORE, 140165753090048, 140165753094143,
+STORE, 140733436887040, 140733436891135,
+STORE, 140733436874752, 140733436887039,
+STORE, 140165753053184, 140165753081855,
+STORE, 140165753044992, 140165753053183,
+STORE, 140165748625408, 140165750841343,
+SNULL, 140165748625408, 140165748723711,
+STORE, 140165748723712, 140165750841343,
+STORE, 140165748625408, 140165748723711,
+SNULL, 140165750816767, 140165750841343,
+STORE, 140165748723712, 140165750816767,
+STORE, 140165750816768, 140165750841343,
+SNULL, 140165750816768, 140165750824959,
+STORE, 140165750824960, 140165750841343,
+STORE, 140165750816768, 140165750824959,
+ERASE, 140165750816768, 140165750824959,
+STORE, 140165750816768, 140165750824959,
+ERASE, 140165750824960, 140165750841343,
+STORE, 140165750824960, 140165750841343,
+STORE, 140165744828416, 140165748625407,
+SNULL, 140165744828416, 140165746487295,
+STORE, 140165746487296, 140165748625407,
+STORE, 140165744828416, 140165746487295,
+SNULL, 140165748584447, 140165748625407,
+STORE, 140165746487296, 140165748584447,
+STORE, 140165748584448, 140165748625407,
+SNULL, 140165748584448, 140165748609023,
+STORE, 140165748609024, 140165748625407,
+STORE, 140165748584448, 140165748609023,
+ERASE, 140165748584448, 140165748609023,
+STORE, 140165748584448, 140165748609023,
+ERASE, 140165748609024, 140165748625407,
+STORE, 140165748609024, 140165748625407,
+STORE, 140165753036800, 140165753053183,
+SNULL, 140165748600831, 140165748609023,
+STORE, 140165748584448, 140165748600831,
+STORE, 140165748600832, 140165748609023,
+SNULL, 140165750820863, 140165750824959,
+STORE, 140165750816768, 140165750820863,
+STORE, 140165750820864, 140165750824959,
+SNULL, 94630731132927, 94630731137023,
+STORE, 94630731128832, 94630731132927,
+STORE, 94630731132928, 94630731137023,
+SNULL, 140165753085951, 140165753090047,
+STORE, 140165753081856, 140165753085951,
+STORE, 140165753085952, 140165753090047,
+ERASE, 140165753053184, 140165753081855,
+STORE, 94630743547904, 94630743683071,
+STORE, 140165736435712, 140165744828415,
+SNULL, 140165736439807, 140165744828415,
+STORE, 140165736435712, 140165736439807,
+STORE, 140165736439808, 140165744828415,
+STORE, 140165728043008, 140165736435711,
+STORE, 140165593825280, 140165728043007,
+SNULL, 140165593825280, 140165653725183,
+STORE, 140165653725184, 140165728043007,
+STORE, 140165593825280, 140165653725183,
+ERASE, 140165593825280, 140165653725183,
+SNULL, 140165720834047, 140165728043007,
+STORE, 140165653725184, 140165720834047,
+STORE, 140165720834048, 140165728043007,
+ERASE, 140165720834048, 140165728043007,
+SNULL, 140165653860351, 140165720834047,
+STORE, 140165653725184, 140165653860351,
+STORE, 140165653860352, 140165720834047,
+SNULL, 140165728047103, 140165736435711,
+STORE, 140165728043008, 140165728047103,
+STORE, 140165728047104, 140165736435711,
+STORE, 140165645332480, 140165653725183,
+SNULL, 140165645336575, 140165653725183,
+STORE, 140165645332480, 140165645336575,
+STORE, 140165645336576, 140165653725183,
+STORE, 140165636939776, 140165645332479,
+SNULL, 140165636943871, 140165645332479,
+STORE, 140165636939776, 140165636943871,
+STORE, 140165636943872, 140165645332479,
+STORE, 140165628547072, 140165636939775,
+SNULL, 140165628551167, 140165636939775,
+STORE, 140165628547072, 140165628551167,
+STORE, 140165628551168, 140165636939775,
+STORE, 140165620154368, 140165628547071,
+STORE, 140165611761664, 140165628547071,
+STORE, 140165603368960, 140165628547071,
+STORE, 140165469151232, 140165603368959,
+SNULL, 140165469151232, 140165519507455,
+STORE, 140165519507456, 140165603368959,
+STORE, 140165469151232, 140165519507455,
+ERASE, 140165469151232, 140165519507455,
+SNULL, 140165586616319, 140165603368959,
+STORE, 140165519507456, 140165586616319,
+STORE, 140165586616320, 140165603368959,
+ERASE, 140165586616320, 140165603368959,
+STORE, 140165594976256, 140165628547071,
+STORE, 140165385289728, 140165586616319,
+SNULL, 140165452398591, 140165586616319,
+STORE, 140165385289728, 140165452398591,
+STORE, 140165452398592, 140165586616319,
+SNULL, 140165452398592, 140165519507455,
+STORE, 140165519507456, 140165586616319,
+STORE, 140165452398592, 140165519507455,
+ERASE, 140165452398592, 140165519507455,
+STORE, 140165251072000, 140165452398591,
+SNULL, 140165318180863, 140165452398591,
+STORE, 140165251072000, 140165318180863,
+STORE, 140165318180864, 140165452398591,
+SNULL, 140165318180864, 140165385289727,
+STORE, 140165385289728, 140165452398591,
+STORE, 140165318180864, 140165385289727,
+ERASE, 140165318180864, 140165385289727,
+SNULL, 140165519642623, 140165586616319,
+STORE, 140165519507456, 140165519642623,
+STORE, 140165519642624, 140165586616319,
+SNULL, 140165594976256, 140165611761663,
+STORE, 140165611761664, 140165628547071,
+STORE, 140165594976256, 140165611761663,
+SNULL, 140165611765759, 140165628547071,
+STORE, 140165611761664, 140165611765759,
+STORE, 140165611765760, 140165628547071,
+STORE, 140165385289728, 140165519507455,
+SNULL, 140165385424895, 140165519507455,
+STORE, 140165385289728, 140165385424895,
+STORE, 140165385424896, 140165519507455,
+SNULL, 140165594976256, 140165603368959,
+STORE, 140165603368960, 140165611761663,
+STORE, 140165594976256, 140165603368959,
+SNULL, 140165603373055, 140165611761663,
+STORE, 140165603368960, 140165603373055,
+STORE, 140165603373056, 140165611761663,
+SNULL, 140165251207167, 140165318180863,
+STORE, 140165251072000, 140165251207167,
+STORE, 140165251207168, 140165318180863,
+STORE, 140165376897024, 140165385289727,
+SNULL, 140165376901119, 140165385289727,
+STORE, 140165376897024, 140165376901119,
+STORE, 140165376901120, 140165385289727,
+SNULL, 140165385424896, 140165452398591,
+STORE, 140165452398592, 140165519507455,
+STORE, 140165385424896, 140165452398591,
+SNULL, 140165452533759, 140165519507455,
+STORE, 140165452398592, 140165452533759,
+STORE, 140165452533760, 140165519507455,
+STORE, 140165368504320, 140165376897023,
+SNULL, 140165594980351, 140165603368959,
+STORE, 140165594976256, 140165594980351,
+STORE, 140165594980352, 140165603368959,
+SNULL, 140165368508415, 140165376897023,
+STORE, 140165368504320, 140165368508415,
+STORE, 140165368508416, 140165376897023,
+SNULL, 140165611765760, 140165620154367,
+STORE, 140165620154368, 140165628547071,
+STORE, 140165611765760, 140165620154367,
+SNULL, 140165620158463, 140165628547071,
+STORE, 140165620154368, 140165620158463,
+STORE, 140165620158464, 140165628547071,
+STORE, 140165360111616, 140165368504319,
+STORE, 140165351718912, 140165368504319,
+STORE, 140165343326208, 140165368504319,
+SNULL, 140165343326208, 140165351718911,
+STORE, 140165351718912, 140165368504319,
+STORE, 140165343326208, 140165351718911,
+SNULL, 140165351723007, 140165368504319,
+STORE, 140165351718912, 140165351723007,
+STORE, 140165351723008, 140165368504319,
+SNULL, 140165343330303, 140165351718911,
+STORE, 140165343326208, 140165343330303,
+STORE, 140165343330304, 140165351718911,
+SNULL, 140165351723008, 140165360111615,
+STORE, 140165360111616, 140165368504319,
+STORE, 140165351723008, 140165360111615,
+SNULL, 140165360115711, 140165368504319,
+STORE, 140165360111616, 140165360115711,
+STORE, 140165360115712, 140165368504319,
+STORE, 140165334933504, 140165343326207,
+SNULL, 140165334937599, 140165343326207,
+STORE, 140165334933504, 140165334937599,
+STORE, 140165334937600, 140165343326207,
+STORE, 140165326540800, 140165334933503,
+STORE, 140165242679296, 140165251071999,
+SNULL, 140165242683391, 140165251071999,
+STORE, 140165242679296, 140165242683391,
+STORE, 140165242683392, 140165251071999,
+STORE, 140165234286592, 140165242679295,
+STORE, 140165225893888, 140165242679295,
+SNULL, 140165225897983, 140165242679295,
+STORE, 140165225893888, 140165225897983,
+STORE, 140165225897984, 140165242679295,
+SNULL, 140165225897984, 140165234286591,
+STORE, 140165234286592, 140165242679295,
+STORE, 140165225897984, 140165234286591,
+SNULL, 140165234290687, 140165242679295,
+STORE, 140165234286592, 140165234290687,
+STORE, 140165234290688, 140165242679295,
+SNULL, 140165326544895, 140165334933503,
+STORE, 140165326540800, 140165326544895,
+STORE, 140165326544896, 140165334933503,
+STORE, 140165217501184, 140165225893887,
+STORE, 140165209108480, 140165225893887,
+SNULL, 140165209108480, 140165217501183,
+STORE, 140165217501184, 140165225893887,
+STORE, 140165209108480, 140165217501183,
+SNULL, 140165217505279, 140165225893887,
+STORE, 140165217501184, 140165217505279,
+STORE, 140165217505280, 140165225893887,
+SNULL, 140165209112575, 140165217501183,
+STORE, 140165209108480, 140165209112575,
+STORE, 140165209112576, 140165217501183,
+STORE, 140165200715776, 140165209108479,
+STORE, 140165066498048, 140165200715775,
+SNULL, 140165066498048, 140165116854271,
+STORE, 140165116854272, 140165200715775,
+STORE, 140165066498048, 140165116854271,
+ERASE, 140165066498048, 140165116854271,
+SNULL, 140165183963135, 140165200715775,
+STORE, 140165116854272, 140165183963135,
+STORE, 140165183963136, 140165200715775,
+ERASE, 140165183963136, 140165200715775,
+SNULL, 140165116989439, 140165183963135,
+STORE, 140165116854272, 140165116989439,
+STORE, 140165116989440, 140165183963135,
+STORE, 140165192323072, 140165209108479,
+STORE, 140165108461568, 140165116854271,
+STORE, 140164974243840, 140165108461567,
+STORE, 140164965851136, 140164974243839,
+SNULL, 140164974243840, 140164982636543,
+STORE, 140164982636544, 140165108461567,
+STORE, 140164974243840, 140164982636543,
+ERASE, 140164974243840, 140164982636543,
+STORE, 140164965851136, 140164982636543,
+STORE, 140164957458432, 140164982636543,
+STORE, 140164949065728, 140164982636543,
+STORE, 140164940673024, 140164982636543,
+STORE, 140164806455296, 140164940673023,
+STORE, 140164798062592, 140164806455295,
+STORE, 140164789669888, 140164806455295,
+STORE, 140164655452160, 140164789669887,
+STORE, 140164647059456, 140164655452159,
+STORE, 140164638666752, 140164655452159,
+SNULL, 140164655452160, 140164714201087,
+STORE, 140164714201088, 140164789669887,
+STORE, 140164655452160, 140164714201087,
+ERASE, 140164655452160, 140164714201087,
+STORE, 140164705808384, 140164714201087,
+STORE, 140164697415680, 140164714201087,
+STORE, 140164504449024, 140164638666751,
+SNULL, 140164504449024, 140164512874495,
+STORE, 140164512874496, 140164638666751,
+STORE, 140164504449024, 140164512874495,
+ERASE, 140164504449024, 140164512874495,
+STORE, 140164689022976, 140164714201087,
+STORE, 140164680630272, 140164714201087,
+SNULL, 140164680634367, 140164714201087,
+STORE, 140164680630272, 140164680634367,
+STORE, 140164680634368, 140164714201087,
+STORE, 140164378656768, 140164638666751,
+SNULL, 140165192323072, 140165200715775,
+STORE, 140165200715776, 140165209108479,
+STORE, 140165192323072, 140165200715775,
+SNULL, 140165200719871, 140165209108479,
+STORE, 140165200715776, 140165200719871,
+STORE, 140165200719872, 140165209108479,
+SNULL, 140165049745407, 140165108461567,
+STORE, 140164982636544, 140165049745407,
+STORE, 140165049745408, 140165108461567,
+ERASE, 140165049745408, 140165108461567,
+SNULL, 140164982771711, 140165049745407,
+STORE, 140164982636544, 140164982771711,
+STORE, 140164982771712, 140165049745407,
+STORE, 140164244439040, 140164638666751,
+SNULL, 140164311547903, 140164638666751,
+STORE, 140164244439040, 140164311547903,
+STORE, 140164311547904, 140164638666751,
+SNULL, 140164311547904, 140164378656767,
+STORE, 140164378656768, 140164638666751,
+STORE, 140164311547904, 140164378656767,
+ERASE, 140164311547904, 140164378656767,
+SNULL, 140164806455296, 140164848418815,
+STORE, 140164848418816, 140164940673023,
+STORE, 140164806455296, 140164848418815,
+ERASE, 140164806455296, 140164848418815,
+SNULL, 140164915527679, 140164940673023,
+STORE, 140164848418816, 140164915527679,
+STORE, 140164915527680, 140164940673023,
+ERASE, 140164915527680, 140164940673023,
+STORE, 140164110221312, 140164311547903,
+SNULL, 140164177330175, 140164311547903,
+STORE, 140164110221312, 140164177330175,
+STORE, 140164177330176, 140164311547903,
+SNULL, 140164177330176, 140164244439039,
+STORE, 140164244439040, 140164311547903,
+STORE, 140164177330176, 140164244439039,
+ERASE, 140164177330176, 140164244439039,
+SNULL, 140164781309951, 140164789669887,
+STORE, 140164714201088, 140164781309951,
+STORE, 140164781309952, 140164789669887,
+ERASE, 140164781309952, 140164789669887,
+STORE, 140163976003584, 140164177330175,
+SNULL, 140164043112447, 140164177330175,
+STORE, 140163976003584, 140164043112447,
+STORE, 140164043112448, 140164177330175,
+SNULL, 140164043112448, 140164110221311,
+STORE, 140164110221312, 140164177330175,
+STORE, 140164043112448, 140164110221311,
+ERASE, 140164043112448, 140164110221311,
+SNULL, 140164579983359, 140164638666751,
+STORE, 140164378656768, 140164579983359,
+STORE, 140164579983360, 140164638666751,
+ERASE, 140164579983360, 140164638666751,
+STORE, 140163841785856, 140164043112447,
+SNULL, 140163908894719, 140164043112447,
+STORE, 140163841785856, 140163908894719,
+STORE, 140163908894720, 140164043112447,
+SNULL, 140163908894720, 140163976003583,
+STORE, 140163976003584, 140164043112447,
+STORE, 140163908894720, 140163976003583,
+ERASE, 140163908894720, 140163976003583,
+SNULL, 140164940673024, 140164965851135,
+STORE, 140164965851136, 140164982636543,
+STORE, 140164940673024, 140164965851135,
+SNULL, 140164965855231, 140164982636543,
+STORE, 140164965851136, 140164965855231,
+STORE, 140164965855232, 140164982636543,
+SNULL, 140164965855232, 140164974243839,
+STORE, 140164974243840, 140164982636543,
+STORE, 140164965855232, 140164974243839,
+SNULL, 140164974247935, 140164982636543,
+STORE, 140164974243840, 140164974247935,
+STORE, 140164974247936, 140164982636543,
+SNULL, 140164445765631, 140164579983359,
+STORE, 140164378656768, 140164445765631,
+STORE, 140164445765632, 140164579983359,
+SNULL, 140164445765632, 140164512874495,
+STORE, 140164512874496, 140164579983359,
+STORE, 140164445765632, 140164512874495,
+ERASE, 140164445765632, 140164512874495,
+SNULL, 140164378791935, 140164445765631,
+STORE, 140164378656768, 140164378791935,
+STORE, 140164378791936, 140164445765631,
+SNULL, 140164789673983, 140164806455295,
+STORE, 140164789669888, 140164789673983,
+STORE, 140164789673984, 140164806455295,
+SNULL, 140164789673984, 140164798062591,
+STORE, 140164798062592, 140164806455295,
+STORE, 140164789673984, 140164798062591,
+SNULL, 140164798066687, 140164806455295,
+STORE, 140164798062592, 140164798066687,
+STORE, 140164798066688, 140164806455295,
+SNULL, 140164638670847, 140164655452159,
+STORE, 140164638666752, 140164638670847,
+STORE, 140164638670848, 140164655452159,
+STORE, 140165100068864, 140165116854271,
+STORE, 140165091676160, 140165116854271,
+STORE, 140165083283456, 140165116854271,
+SNULL, 140164244574207, 140164311547903,
+STORE, 140164244439040, 140164244574207,
+STORE, 140164244574208, 140164311547903,
+SNULL, 140164848553983, 140164915527679,
+STORE, 140164848418816, 140164848553983,
+STORE, 140164848553984, 140164915527679,
+SNULL, 140164110356479, 140164177330175,
+STORE, 140164110221312, 140164110356479,
+STORE, 140164110356480, 140164177330175,
+SNULL, 140164714336255, 140164781309951,
+STORE, 140164714201088, 140164714336255,
+STORE, 140164714336256, 140164781309951,
+SNULL, 140163976138751, 140164043112447,
+STORE, 140163976003584, 140163976138751,
+STORE, 140163976138752, 140164043112447,
+SNULL, 140164513009663, 140164579983359,
+STORE, 140164512874496, 140164513009663,
+STORE, 140164513009664, 140164579983359,
+SNULL, 140163841921023, 140163908894719,
+STORE, 140163841785856, 140163841921023,
+STORE, 140163841921024, 140163908894719,
+SNULL, 140165083283456, 140165100068863,
+STORE, 140165100068864, 140165116854271,
+STORE, 140165083283456, 140165100068863,
+SNULL, 140165100072959, 140165116854271,
+STORE, 140165100068864, 140165100072959,
+STORE, 140165100072960, 140165116854271,
+SNULL, 140165100072960, 140165108461567,
+STORE, 140165108461568, 140165116854271,
+STORE, 140165100072960, 140165108461567,
+SNULL, 140165108465663, 140165116854271,
+STORE, 140165108461568, 140165108465663,
+STORE, 140165108465664, 140165116854271,
+STORE, 140165074890752, 140165100068863,
+SNULL, 140165074894847, 140165100068863,
+STORE, 140165074890752, 140165074894847,
+STORE, 140165074894848, 140165100068863,
+STORE, 140165066498048, 140165074890751,
+STORE, 140165058105344, 140165074890751,
+STORE, 140164932280320, 140164965851135,
+SNULL, 140165192327167, 140165200715775,
+STORE, 140165192323072, 140165192327167,
+STORE, 140165192327168, 140165200715775,
+STORE, 140164923887616, 140164965851135,
+SNULL, 140164923891711, 140164965851135,
+STORE, 140164923887616, 140164923891711,
+STORE, 140164923891712, 140164965851135,
+SNULL, 140164680634368, 140164705808383,
+STORE, 140164705808384, 140164714201087,
+STORE, 140164680634368, 140164705808383,
+SNULL, 140164705812479, 140164714201087,
+STORE, 140164705808384, 140164705812479,
+STORE, 140164705812480, 140164714201087,
+SNULL, 140164680634368, 140164697415679,
+STORE, 140164697415680, 140164705808383,
+STORE, 140164680634368, 140164697415679,
+SNULL, 140164697419775, 140164705808383,
+STORE, 140164697415680, 140164697419775,
+STORE, 140164697419776, 140164705808383,
+STORE, 140164840026112, 140164848418815,
+STORE, 140164831633408, 140164848418815,
+STORE, 140164823240704, 140164848418815,
+SNULL, 140165074894848, 140165083283455,
+STORE, 140165083283456, 140165100068863,
+STORE, 140165074894848, 140165083283455,
+SNULL, 140165083287551, 140165100068863,
+STORE, 140165083283456, 140165083287551,
+STORE, 140165083287552, 140165100068863,
+SNULL, 140165083287552, 140165091676159,
+STORE, 140165091676160, 140165100068863,
+STORE, 140165083287552, 140165091676159,
+SNULL, 140165091680255, 140165100068863,
+STORE, 140165091676160, 140165091680255,
+STORE, 140165091680256, 140165100068863,
+SNULL, 140164638670848, 140164647059455,
+STORE, 140164647059456, 140164655452159,
+STORE, 140164638670848, 140164647059455,
+SNULL, 140164647063551, 140164655452159,
+STORE, 140164647059456, 140164647063551,
+STORE, 140164647063552, 140164655452159,
+SNULL, 140164923891712, 140164940673023,
+STORE, 140164940673024, 140164965851135,
+STORE, 140164923891712, 140164940673023,
+SNULL, 140164940677119, 140164965851135,
+STORE, 140164940673024, 140164940677119,
+STORE, 140164940677120, 140164965851135,
+SNULL, 140164940677120, 140164949065727,
+STORE, 140164949065728, 140164965851135,
+STORE, 140164940677120, 140164949065727,
+SNULL, 140164949069823, 140164965851135,
+STORE, 140164949065728, 140164949069823,
+STORE, 140164949069824, 140164965851135,
+SNULL, 140164949069824, 140164957458431,
+STORE, 140164957458432, 140164965851135,
+STORE, 140164949069824, 140164957458431,
+SNULL, 140164957462527, 140164965851135,
+STORE, 140164957458432, 140164957462527,
+STORE, 140164957462528, 140164965851135,
+SNULL, 140164680634368, 140164689022975,
+STORE, 140164689022976, 140164697415679,
+STORE, 140164680634368, 140164689022975,
+SNULL, 140164689027071, 140164697415679,
+STORE, 140164689022976, 140164689027071,
+STORE, 140164689027072, 140164697415679,
+STORE, 140164814848000, 140164848418815,
+SNULL, 140165058105344, 140165066498047,
+STORE, 140165066498048, 140165074890751,
+STORE, 140165058105344, 140165066498047,
+SNULL, 140165066502143, 140165074890751,
+STORE, 140165066498048, 140165066502143,
+STORE, 140165066502144, 140165074890751,
+SNULL, 140165058109439, 140165066498047,
+STORE, 140165058105344, 140165058109439,
+STORE, 140165058109440, 140165066498047,
+STORE, 140164798066688, 140164814847999,
+SNULL, 140164798066688, 140164806455295,
+STORE, 140164806455296, 140164814847999,
+STORE, 140164798066688, 140164806455295,
+SNULL, 140164806459391, 140164814847999,
+STORE, 140164806455296, 140164806459391,
+STORE, 140164806459392, 140164814847999,
+SNULL, 140164923891712, 140164932280319,
+STORE, 140164932280320, 140164940673023,
+STORE, 140164923891712, 140164932280319,
+SNULL, 140164932284415, 140164940673023,
+STORE, 140164932280320, 140164932284415,
+STORE, 140164932284416, 140164940673023,
+STORE, 140164672237568, 140164680630271,
+STORE, 140164663844864, 140164680630271,
+STORE, 140164647063552, 140164680630271,
+SNULL, 140164647063552, 140164655452159,
+STORE, 140164655452160, 140164680630271,
+STORE, 140164647063552, 140164655452159,
+SNULL, 140164655456255, 140164680630271,
+STORE, 140164655452160, 140164655456255,
+STORE, 140164655456256, 140164680630271,
+STORE, 140164630274048, 140164638666751,
+SNULL, 140164814852095, 140164848418815,
+STORE, 140164814848000, 140164814852095,
+STORE, 140164814852096, 140164848418815,
+SNULL, 140164814852096, 140164831633407,
+STORE, 140164831633408, 140164848418815,
+STORE, 140164814852096, 140164831633407,
+SNULL, 140164831637503, 140164848418815,
+STORE, 140164831633408, 140164831637503,
+STORE, 140164831637504, 140164848418815,
+STORE, 140164621881344, 140164638666751,
+SNULL, 140164831637504, 140164840026111,
+STORE, 140164840026112, 140164848418815,
+STORE, 140164831637504, 140164840026111,
+SNULL, 140164840030207, 140164848418815,
+STORE, 140164840026112, 140164840030207,
+STORE, 140164840030208, 140164848418815,
+STORE, 140164613488640, 140164638666751,
+SNULL, 140164613492735, 140164638666751,
+STORE, 140164613488640, 140164613492735,
+STORE, 140164613492736, 140164638666751,
+STORE, 140164605095936, 140164613488639,
+SNULL, 140164605100031, 140164613488639,
+STORE, 140164605095936, 140164605100031,
+STORE, 140164605100032, 140164613488639,
+STORE, 140164596703232, 140164605095935,
+STORE, 140164588310528, 140164605095935,
+SNULL, 140164588314623, 140164605095935,
+STORE, 140164588310528, 140164588314623,
+STORE, 140164588314624, 140164605095935,
+STORE, 140164504481792, 140164512874495,
+STORE, 140164496089088, 140164512874495,
+SNULL, 140164496089088, 140164504481791,
+STORE, 140164504481792, 140164512874495,
+STORE, 140164496089088, 140164504481791,
+SNULL, 140164504485887, 140164512874495,
+STORE, 140164504481792, 140164504485887,
+STORE, 140164504485888, 140164512874495,
+SNULL, 140164613492736, 140164630274047,
+STORE, 140164630274048, 140164638666751,
+STORE, 140164613492736, 140164630274047,
+SNULL, 140164630278143, 140164638666751,
+STORE, 140164630274048, 140164630278143,
+STORE, 140164630278144, 140164638666751,
+STORE, 140164487696384, 140164504481791,
+STORE, 140164479303680, 140164504481791,
+SNULL, 140164814852096, 140164823240703,
+STORE, 140164823240704, 140164831633407,
+STORE, 140164814852096, 140164823240703,
+SNULL, 140164823244799, 140164831633407,
+STORE, 140164823240704, 140164823244799,
+STORE, 140164823244800, 140164831633407,
+STORE, 140164470910976, 140164504481791,
+SNULL, 140164470910976, 140164496089087,
+STORE, 140164496089088, 140164504481791,
+STORE, 140164470910976, 140164496089087,
+SNULL, 140164496093183, 140164504481791,
+STORE, 140164496089088, 140164496093183,
+STORE, 140164496093184, 140164504481791,
+SNULL, 140164655456256, 140164672237567,
+STORE, 140164672237568, 140164680630271,
+STORE, 140164655456256, 140164672237567,
+SNULL, 140164672241663, 140164680630271,
+STORE, 140164672237568, 140164672241663,
+STORE, 140164672241664, 140164680630271,
+STORE, 140164462518272, 140164496089087,
+STORE, 140164454125568, 140164496089087,
+SNULL, 140164655456256, 140164663844863,
+STORE, 140164663844864, 140164672237567,
+STORE, 140164655456256, 140164663844863,
+SNULL, 140164663848959, 140164672237567,
+STORE, 140164663844864, 140164663848959,
+STORE, 140164663848960, 140164672237567,
+STORE, 140164370264064, 140164378656767,
+STORE, 140164361871360, 140164378656767,
+STORE, 140164353478656, 140164378656767,
+STORE, 140164345085952, 140164378656767,
+SNULL, 140164345085952, 140164353478655,
+STORE, 140164353478656, 140164378656767,
+STORE, 140164345085952, 140164353478655,
+SNULL, 140164353482751, 140164378656767,
+STORE, 140164353478656, 140164353482751,
+STORE, 140164353482752, 140164378656767,
+SNULL, 140164454125568, 140164487696383,
+STORE, 140164487696384, 140164496089087,
+STORE, 140164454125568, 140164487696383,
+SNULL, 140164487700479, 140164496089087,
+STORE, 140164487696384, 140164487700479,
+STORE, 140164487700480, 140164496089087,
+STORE, 140164336693248, 140164353478655,
+SNULL, 140164336697343, 140164353478655,
+STORE, 140164336693248, 140164336697343,
+STORE, 140164336697344, 140164353478655,
+STORE, 140164328300544, 140164336693247,
+SNULL, 140164454125568, 140164479303679,
+STORE, 140164479303680, 140164487696383,
+STORE, 140164454125568, 140164479303679,
+SNULL, 140164479307775, 140164487696383,
+STORE, 140164479303680, 140164479307775,
+STORE, 140164479307776, 140164487696383,
+STORE, 140164319907840, 140164336693247,
+STORE, 140164236046336, 140164244439039,
+SNULL, 140164588314624, 140164596703231,
+STORE, 140164596703232, 140164605095935,
+STORE, 140164588314624, 140164596703231,
+SNULL, 140164596707327, 140164605095935,
+STORE, 140164596703232, 140164596707327,
+STORE, 140164596707328, 140164605095935,
+SNULL, 140164454125568, 140164462518271,
+STORE, 140164462518272, 140164479303679,
+STORE, 140164454125568, 140164462518271,
+SNULL, 140164462522367, 140164479303679,
+STORE, 140164462518272, 140164462522367,
+STORE, 140164462522368, 140164479303679,
+STORE, 140164227653632, 140164244439039,
+SNULL, 140164227657727, 140164244439039,
+STORE, 140164227653632, 140164227657727,
+STORE, 140164227657728, 140164244439039,
+SNULL, 140164462522368, 140164470910975,
+STORE, 140164470910976, 140164479303679,
+STORE, 140164462522368, 140164470910975,
+SNULL, 140164470915071, 140164479303679,
+STORE, 140164470910976, 140164470915071,
+STORE, 140164470915072, 140164479303679,
+SNULL, 140164613492736, 140164621881343,
+STORE, 140164621881344, 140164630274047,
+STORE, 140164613492736, 140164621881343,
+SNULL, 140164621885439, 140164630274047,
+STORE, 140164621881344, 140164621885439,
+STORE, 140164621885440, 140164630274047,
+SNULL, 140164353482752, 140164370264063,
+STORE, 140164370264064, 140164378656767,
+STORE, 140164353482752, 140164370264063,
+SNULL, 140164370268159, 140164378656767,
+STORE, 140164370264064, 140164370268159,
+STORE, 140164370268160, 140164378656767,
+STORE, 140164219260928, 140164227653631,
+SNULL, 140164319911935, 140164336693247,
+STORE, 140164319907840, 140164319911935,
+STORE, 140164319911936, 140164336693247,
+SNULL, 140164336697344, 140164345085951,
+STORE, 140164345085952, 140164353478655,
+STORE, 140164336697344, 140164345085951,
+SNULL, 140164345090047, 140164353478655,
+STORE, 140164345085952, 140164345090047,
+STORE, 140164345090048, 140164353478655,
+SNULL, 140164319911936, 140164328300543,
+STORE, 140164328300544, 140164336693247,
+STORE, 140164319911936, 140164328300543,
+SNULL, 140164328304639, 140164336693247,
+STORE, 140164328300544, 140164328304639,
+STORE, 140164328304640, 140164336693247,
+SNULL, 140164454129663, 140164462518271,
+STORE, 140164454125568, 140164454129663,
+STORE, 140164454129664, 140164462518271,
+STORE, 140164210868224, 140164227653631,
+STORE, 140164202475520, 140164227653631,
+STORE, 140164194082816, 140164227653631,
+SNULL, 140164194086911, 140164227653631,
+STORE, 140164194082816, 140164194086911,
+STORE, 140164194086912, 140164227653631,
+SNULL, 140164353482752, 140164361871359,
+STORE, 140164361871360, 140164370264063,
+STORE, 140164353482752, 140164361871359,
+SNULL, 140164361875455, 140164370264063,
+STORE, 140164361871360, 140164361875455,
+STORE, 140164361875456, 140164370264063,
+SNULL, 140164227657728, 140164236046335,
+STORE, 140164236046336, 140164244439039,
+STORE, 140164227657728, 140164236046335,
+SNULL, 140164236050431, 140164244439039,
+STORE, 140164236046336, 140164236050431,
+STORE, 140164236050432, 140164244439039,
+STORE, 140164185690112, 140164194082815,
+SNULL, 140164194086912, 140164219260927,
+STORE, 140164219260928, 140164227653631,
+STORE, 140164194086912, 140164219260927,
+SNULL, 140164219265023, 140164227653631,
+STORE, 140164219260928, 140164219265023,
+STORE, 140164219265024, 140164227653631,
+STORE, 140164101828608, 140164110221311,
+STORE, 140164093435904, 140164110221311,
+STORE, 140164085043200, 140164110221311,
+SNULL, 140164085047295, 140164110221311,
+STORE, 140164085043200, 140164085047295,
+STORE, 140164085047296, 140164110221311,
+STORE, 140164076650496, 140164085043199,
+SNULL, 140164185694207, 140164194082815,
+STORE, 140164185690112, 140164185694207,
+STORE, 140164185694208, 140164194082815,
+SNULL, 140164085047296, 140164101828607,
+STORE, 140164101828608, 140164110221311,
+STORE, 140164085047296, 140164101828607,
+SNULL, 140164101832703, 140164110221311,
+STORE, 140164101828608, 140164101832703,
+STORE, 140164101832704, 140164110221311,
+SNULL, 140164085047296, 140164093435903,
+STORE, 140164093435904, 140164101828607,
+STORE, 140164085047296, 140164093435903,
+SNULL, 140164093439999, 140164101828607,
+STORE, 140164093435904, 140164093439999,
+STORE, 140164093440000, 140164101828607,
+SNULL, 140164194086912, 140164202475519,
+STORE, 140164202475520, 140164219260927,
+STORE, 140164194086912, 140164202475519,
+SNULL, 140164202479615, 140164219260927,
+STORE, 140164202475520, 140164202479615,
+STORE, 140164202479616, 140164219260927,
+SNULL, 140164202479616, 140164210868223,
+STORE, 140164210868224, 140164219260927,
+STORE, 140164202479616, 140164210868223,
+SNULL, 140164210872319, 140164219260927,
+STORE, 140164210868224, 140164210872319,
+STORE, 140164210872320, 140164219260927,
+SNULL, 140164076654591, 140164085043199,
+STORE, 140164076650496, 140164076654591,
+STORE, 140164076654592, 140164085043199,
+STORE, 140164068257792, 140164076650495,
+SNULL, 140164068261887, 140164076650495,
+STORE, 140164068257792, 140164068261887,
+STORE, 140164068261888, 140164076650495,
+STORE, 140165753053184, 140165753081855,
+STORE, 140165725851648, 140165728043007,
+SNULL, 140165725851648, 140165725941759,
+STORE, 140165725941760, 140165728043007,
+STORE, 140165725851648, 140165725941759,
+SNULL, 140165728034815, 140165728043007,
+STORE, 140165725941760, 140165728034815,
+STORE, 140165728034816, 140165728043007,
+ERASE, 140165728034816, 140165728043007,
+STORE, 140165728034816, 140165728043007,
+SNULL, 140165728038911, 140165728043007,
+STORE, 140165728034816, 140165728038911,
+STORE, 140165728038912, 140165728043007,
+ERASE, 140165753053184, 140165753081855,
+ERASE, 140164638666752, 140164638670847,
+ERASE, 140164638670848, 140164647059455,
+ERASE, 140165091676160, 140165091680255,
+ERASE, 140165091680256, 140165100068863,
+ERASE, 140164613488640, 140164613492735,
+ERASE, 140164613492736, 140164621881343,
+ERASE, 140164319907840, 140164319911935,
+ERASE, 140164319911936, 140164328300543,
+ERASE, 140165620154368, 140165620158463,
+ERASE, 140165620158464, 140165628547071,
+ERASE, 140164798062592, 140164798066687,
+ERASE, 140164798066688, 140164806455295,
+ERASE, 140164789669888, 140164789673983,
+ERASE, 140164789673984, 140164798062591,
+ERASE, 140164965851136, 140164965855231,
+ERASE, 140164965855232, 140164974243839,
+ERASE, 140165074890752, 140165074894847,
+ERASE, 140165074894848, 140165083283455,
+ERASE, 140164672237568, 140164672241663,
+ERASE, 140164672241664, 140164680630271,
+ERASE, 140164454125568, 140164454129663,
+ERASE, 140164454129664, 140164462518271,
+ERASE, 140165200715776, 140165200719871,
+ERASE, 140165200719872, 140165209108479,
+ERASE, 140164932280320, 140164932284415,
+ERASE, 140164932284416, 140164940673023,
+ERASE, 140164663844864, 140164663848959,
+ERASE, 140164663848960, 140164672237567,
+ERASE, 140164697415680, 140164697419775,
+ERASE, 140164697419776, 140164705808383,
+ERASE, 140164831633408, 140164831637503,
+ERASE, 140164831637504, 140164840026111,
+ERASE, 140165192323072, 140165192327167,
+ERASE, 140165192327168, 140165200715775,
+ERASE, 140165108461568, 140165108465663,
+ERASE, 140165108465664, 140165116854271,
+ERASE, 140164840026112, 140164840030207,
+ERASE, 140164840030208, 140164848418815,
+ERASE, 140164647059456, 140164647063551,
+ERASE, 140164647063552, 140164655452159,
+ERASE, 140165083283456, 140165083287551,
+ERASE, 140165083287552, 140165091676159,
+ERASE, 140164923887616, 140164923891711,
+ERASE, 140164923891712, 140164932280319,
+ERASE, 140164823240704, 140164823244799,
+ERASE, 140164823244800, 140164831633407,
+ERASE, 140164227653632, 140164227657727,
+ERASE, 140164227657728, 140164236046335,
+ERASE, 140164957458432, 140164957462527,
+ERASE, 140164957462528, 140164965851135,
+ERASE, 140164680630272, 140164680634367,
+ERASE, 140164680634368, 140164689022975,
+ERASE, 140164974243840, 140164974247935,
+ERASE, 140164974247936, 140164982636543,
+ERASE, 140165066498048, 140165066502143,
+ERASE, 140165066502144, 140165074890751,
+ERASE, 140164621881344, 140164621885439,
+ERASE, 140164621885440, 140164630274047,
+ERASE, 140164949065728, 140164949069823,
+ERASE, 140164949069824, 140164957458431,
+ERASE, 140164588310528, 140164588314623,
+ERASE, 140164588314624, 140164596703231,
+ERASE, 140164806455296, 140164806459391,
+ERASE, 140164806459392, 140164814847999,
+ERASE, 140164940673024, 140164940677119,
+ERASE, 140164940677120, 140164949065727,
+ERASE, 140164596703232, 140164596707327,
+ERASE, 140164596707328, 140164605095935,
+ERASE, 140164605095936, 140164605100031,
+ERASE, 140164605100032, 140164613488639,
+ERASE, 140164655452160, 140164655456255,
+ERASE, 140164655456256, 140164663844863,
+ERASE, 140164705808384, 140164705812479,
+ERASE, 140164705812480, 140164714201087,
+ERASE, 140164689022976, 140164689027071,
+ERASE, 140164689027072, 140164697415679,
+ERASE, 140164630274048, 140164630278143,
+ERASE, 140164630278144, 140164638666751,
+ERASE, 140164479303680, 140164479307775,
+ERASE, 140164479307776, 140164487696383,
+ERASE, 140164236046336, 140164236050431,
+ERASE, 140164236050432, 140164244439039,
+ERASE, 140164085043200, 140164085047295,
+ERASE, 140164085047296, 140164093435903,
+ERASE, 140164345085952, 140164345090047,
+ERASE, 140164345090048, 140164353478655,
+ERASE, 140164101828608, 140164101832703,
+ERASE, 140164101832704, 140164110221311,
+ERASE, 140164370264064, 140164370268159,
+ERASE, 140164370268160, 140164378656767,
+ERASE, 140164336693248, 140164336697343,
+ERASE, 140164336697344, 140164345085951,
+ERASE, 140164194082816, 140164194086911,
+ERASE, 140164194086912, 140164202475519,
+ERASE, 140164353478656, 140164353482751,
+ERASE, 140164353482752, 140164361871359,
+ERASE, 140164210868224, 140164210872319,
+ERASE, 140164210872320, 140164219260927,
+ERASE, 140164814848000, 140164814852095,
+ERASE, 140164814852096, 140164823240703,
+ERASE, 140164504481792, 140164504485887,
+ERASE, 140164504485888, 140164512874495,
+ERASE, 140165100068864, 140165100072959,
+ERASE, 140165100072960, 140165108461567,
+ERASE, 140164361871360, 140164361875455,
+ERASE, 140164361875456, 140164370264063,
+ERASE, 140164470910976, 140164470915071,
+ERASE, 140164470915072, 140164479303679,
+ERASE, 140164076650496, 140164076654591,
+ERASE, 140164076654592, 140164085043199,
+ERASE, 140164202475520, 140164202479615,
+ERASE, 140164202479616, 140164210868223,
+ERASE, 140164462518272, 140164462522367,
+ERASE, 140164462522368, 140164470910975,
+ERASE, 140165351718912, 140165351723007,
+ERASE, 140165351723008, 140165360111615,
+ERASE, 140164328300544, 140164328304639,
+ERASE, 140164328304640, 140164336693247,
+ERASE, 140164093435904, 140164093439999,
+ERASE, 140164093440000, 140164101828607,
+ERASE, 140165603368960, 140165603373055,
+ERASE, 140165603373056, 140165611761663,
+ERASE, 140165368504320, 140165368508415,
+ERASE, 140165368508416, 140165376897023,
+ERASE, 140165334933504, 140165334937599,
+ERASE, 140165334937600, 140165343326207,
+ERASE, 140165594976256, 140165594980351,
+ERASE, 140165594980352, 140165603368959,
+ERASE, 140164487696384, 140164487700479,
+ERASE, 140164487700480, 140164496089087,
+ERASE, 140164219260928, 140164219265023,
+ERASE, 140164219265024, 140164227653631,
+ERASE, 140164185690112, 140164185694207,
+ERASE, 140164185694208, 140164194082815,
+ERASE, 140164068257792, 140164068261887,
+ERASE, 140164068261888, 140164076650495,
+ERASE, 140165225893888, 140165225897983,
+ERASE, 140165225897984, 140165234286591,
+ERASE, 140165058105344, 140165058109439,
+ };
+ unsigned long set31[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140730890784768, 140737488351231,
+SNULL, 140730890788863, 140737488351231,
+STORE, 140730890784768, 140730890788863,
+STORE, 140730890653696, 140730890788863,
+STORE, 94577123659776, 94577125912575,
+SNULL, 94577123790847, 94577125912575,
+STORE, 94577123659776, 94577123790847,
+STORE, 94577123790848, 94577125912575,
+ERASE, 94577123790848, 94577125912575,
+STORE, 94577125883904, 94577125892095,
+STORE, 94577125892096, 94577125912575,
+STORE, 140624060407808, 140624062660607,
+SNULL, 140624060551167, 140624062660607,
+STORE, 140624060407808, 140624060551167,
+STORE, 140624060551168, 140624062660607,
+ERASE, 140624060551168, 140624062660607,
+STORE, 140624062648320, 140624062656511,
+STORE, 140624062656512, 140624062660607,
+STORE, 140730892140544, 140730892144639,
+STORE, 140730892128256, 140730892140543,
+STORE, 140624062619648, 140624062648319,
+STORE, 140624062611456, 140624062619647,
+STORE, 140624058191872, 140624060407807,
+SNULL, 140624058191872, 140624058290175,
+STORE, 140624058290176, 140624060407807,
+STORE, 140624058191872, 140624058290175,
+SNULL, 140624060383231, 140624060407807,
+STORE, 140624058290176, 140624060383231,
+STORE, 140624060383232, 140624060407807,
+SNULL, 140624060383232, 140624060391423,
+STORE, 140624060391424, 140624060407807,
+STORE, 140624060383232, 140624060391423,
+ERASE, 140624060383232, 140624060391423,
+STORE, 140624060383232, 140624060391423,
+ERASE, 140624060391424, 140624060407807,
+STORE, 140624060391424, 140624060407807,
+STORE, 140624054394880, 140624058191871,
+SNULL, 140624054394880, 140624056053759,
+STORE, 140624056053760, 140624058191871,
+STORE, 140624054394880, 140624056053759,
+SNULL, 140624058150911, 140624058191871,
+STORE, 140624056053760, 140624058150911,
+STORE, 140624058150912, 140624058191871,
+SNULL, 140624058150912, 140624058175487,
+STORE, 140624058175488, 140624058191871,
+STORE, 140624058150912, 140624058175487,
+ERASE, 140624058150912, 140624058175487,
+STORE, 140624058150912, 140624058175487,
+ERASE, 140624058175488, 140624058191871,
+STORE, 140624058175488, 140624058191871,
+STORE, 140624062603264, 140624062619647,
+SNULL, 140624058167295, 140624058175487,
+STORE, 140624058150912, 140624058167295,
+STORE, 140624058167296, 140624058175487,
+SNULL, 140624060387327, 140624060391423,
+STORE, 140624060383232, 140624060387327,
+STORE, 140624060387328, 140624060391423,
+SNULL, 94577125887999, 94577125892095,
+STORE, 94577125883904, 94577125887999,
+STORE, 94577125888000, 94577125892095,
+SNULL, 140624062652415, 140624062656511,
+STORE, 140624062648320, 140624062652415,
+STORE, 140624062652416, 140624062656511,
+ERASE, 140624062619648, 140624062648319,
+STORE, 94577157709824, 94577157844991,
+STORE, 140624046002176, 140624054394879,
+SNULL, 140624046006271, 140624054394879,
+STORE, 140624046002176, 140624046006271,
+STORE, 140624046006272, 140624054394879,
+STORE, 140624037609472, 140624046002175,
+STORE, 140623903391744, 140624037609471,
+SNULL, 140623903391744, 140623940157439,
+STORE, 140623940157440, 140624037609471,
+STORE, 140623903391744, 140623940157439,
+ERASE, 140623903391744, 140623940157439,
+SNULL, 140624007266303, 140624037609471,
+STORE, 140623940157440, 140624007266303,
+STORE, 140624007266304, 140624037609471,
+ERASE, 140624007266304, 140624037609471,
+SNULL, 140623940292607, 140624007266303,
+STORE, 140623940157440, 140623940292607,
+STORE, 140623940292608, 140624007266303,
+SNULL, 140624037613567, 140624046002175,
+STORE, 140624037609472, 140624037613567,
+STORE, 140624037613568, 140624046002175,
+STORE, 140624029216768, 140624037609471,
+SNULL, 140624029220863, 140624037609471,
+STORE, 140624029216768, 140624029220863,
+STORE, 140624029220864, 140624037609471,
+STORE, 140624020824064, 140624029216767,
+SNULL, 140624020828159, 140624029216767,
+STORE, 140624020824064, 140624020828159,
+STORE, 140624020828160, 140624029216767,
+STORE, 140624012431360, 140624020824063,
+SNULL, 140624012435455, 140624020824063,
+STORE, 140624012431360, 140624012435455,
+STORE, 140624012435456, 140624020824063,
+STORE, 140623931764736, 140623940157439,
+STORE, 140623797547008, 140623931764735,
+SNULL, 140623797547008, 140623805939711,
+STORE, 140623805939712, 140623931764735,
+STORE, 140623797547008, 140623805939711,
+ERASE, 140623797547008, 140623805939711,
+SNULL, 140623873048575, 140623931764735,
+STORE, 140623805939712, 140623873048575,
+STORE, 140623873048576, 140623931764735,
+ERASE, 140623873048576, 140623931764735,
+STORE, 140623923372032, 140623940157439,
+STORE, 140623914979328, 140623940157439,
+STORE, 140623906586624, 140623940157439,
+STORE, 140623671721984, 140623873048575,
+SNULL, 140623738830847, 140623873048575,
+STORE, 140623671721984, 140623738830847,
+STORE, 140623738830848, 140623873048575,
+SNULL, 140623738830848, 140623805939711,
+STORE, 140623805939712, 140623873048575,
+STORE, 140623738830848, 140623805939711,
+ERASE, 140623738830848, 140623805939711,
+SNULL, 140623806074879, 140623873048575,
+STORE, 140623805939712, 140623806074879,
+STORE, 140623806074880, 140623873048575,
+SNULL, 140623906586624, 140623931764735,
+STORE, 140623931764736, 140623940157439,
+STORE, 140623906586624, 140623931764735,
+SNULL, 140623931768831, 140623940157439,
+STORE, 140623931764736, 140623931768831,
+STORE, 140623931768832, 140623940157439,
+STORE, 140623537504256, 140623738830847,
+SNULL, 140623537504256, 140623671721983,
+STORE, 140623671721984, 140623738830847,
+STORE, 140623537504256, 140623671721983,
+SNULL, 140623671857151, 140623738830847,
+STORE, 140623671721984, 140623671857151,
+STORE, 140623671857152, 140623738830847,
+SNULL, 140623604613119, 140623671721983,
+STORE, 140623537504256, 140623604613119,
+STORE, 140623604613120, 140623671721983,
+ERASE, 140623604613120, 140623671721983,
+SNULL, 140623537639423, 140623604613119,
+STORE, 140623537504256, 140623537639423,
+STORE, 140623537639424, 140623604613119,
+STORE, 140623537639424, 140623671721983,
+SNULL, 140623537639424, 140623604613119,
+STORE, 140623604613120, 140623671721983,
+STORE, 140623537639424, 140623604613119,
+SNULL, 140623604748287, 140623671721983,
+STORE, 140623604613120, 140623604748287,
+STORE, 140623604748288, 140623671721983,
+STORE, 140623898193920, 140623931764735,
+SNULL, 140623898193920, 140623923372031,
+STORE, 140623923372032, 140623931764735,
+STORE, 140623898193920, 140623923372031,
+SNULL, 140623923376127, 140623931764735,
+STORE, 140623923372032, 140623923376127,
+STORE, 140623923376128, 140623931764735,
+STORE, 140623889801216, 140623923372031,
+SNULL, 140623889801216, 140623898193919,
+STORE, 140623898193920, 140623923372031,
+STORE, 140623889801216, 140623898193919,
+SNULL, 140623898198015, 140623923372031,
+STORE, 140623898193920, 140623898198015,
+STORE, 140623898198016, 140623923372031,
+SNULL, 140623889805311, 140623898193919,
+STORE, 140623889801216, 140623889805311,
+STORE, 140623889805312, 140623898193919,
+SNULL, 140623898198016, 140623906586623,
+STORE, 140623906586624, 140623923372031,
+STORE, 140623898198016, 140623906586623,
+SNULL, 140623906590719, 140623923372031,
+STORE, 140623906586624, 140623906590719,
+STORE, 140623906590720, 140623923372031,
+STORE, 140623881408512, 140623889801215,
+SNULL, 140623906590720, 140623914979327,
+STORE, 140623914979328, 140623923372031,
+STORE, 140623906590720, 140623914979327,
+SNULL, 140623914983423, 140623923372031,
+STORE, 140623914979328, 140623914983423,
+STORE, 140623914983424, 140623923372031,
+SNULL, 140623881412607, 140623889801215,
+STORE, 140623881408512, 140623881412607,
+STORE, 140623881412608, 140623889801215,
+STORE, 140623797547008, 140623805939711,
+STORE, 140623789154304, 140623805939711,
+STORE, 140623780761600, 140623805939711,
+SNULL, 140623780761600, 140623789154303,
+STORE, 140623789154304, 140623805939711,
+STORE, 140623780761600, 140623789154303,
+SNULL, 140623789158399, 140623805939711,
+STORE, 140623789154304, 140623789158399,
+STORE, 140623789158400, 140623805939711,
+STORE, 140623772368896, 140623789154303,
+STORE, 140623763976192, 140623789154303,
+SNULL, 140623763976192, 140623780761599,
+STORE, 140623780761600, 140623789154303,
+STORE, 140623763976192, 140623780761599,
+SNULL, 140623780765695, 140623789154303,
+STORE, 140623780761600, 140623780765695,
+STORE, 140623780765696, 140623789154303,
+SNULL, 140623789158400, 140623797547007,
+STORE, 140623797547008, 140623805939711,
+STORE, 140623789158400, 140623797547007,
+SNULL, 140623797551103, 140623805939711,
+STORE, 140623797547008, 140623797551103,
+STORE, 140623797551104, 140623805939711,
+SNULL, 140623763976192, 140623772368895,
+STORE, 140623772368896, 140623780761599,
+STORE, 140623763976192, 140623772368895,
+SNULL, 140623772372991, 140623780761599,
+STORE, 140623772368896, 140623772372991,
+STORE, 140623772372992, 140623780761599,
+SNULL, 140623763980287, 140623772368895,
+STORE, 140623763976192, 140623763980287,
+STORE, 140623763980288, 140623772368895,
+STORE, 140623755583488, 140623763976191,
+STORE, 140623747190784, 140623763976191,
+SNULL, 140623747190784, 140623755583487,
+STORE, 140623755583488, 140623763976191,
+STORE, 140623747190784, 140623755583487,
+SNULL, 140623755587583, 140623763976191,
+STORE, 140623755583488, 140623755587583,
+STORE, 140623755587584, 140623763976191,
+STORE, 140623529111552, 140623537504255,
+SNULL, 140623747194879, 140623755583487,
+STORE, 140623747190784, 140623747194879,
+STORE, 140623747194880, 140623755583487,
+SNULL, 140623529115647, 140623537504255,
+STORE, 140623529111552, 140623529115647,
+STORE, 140623529115648, 140623537504255,
+STORE, 140623520718848, 140623529111551,
+SNULL, 140623520722943, 140623529111551,
+STORE, 140623520718848, 140623520722943,
+STORE, 140623520722944, 140623529111551,
+STORE, 140623512326144, 140623520718847,
+STORE, 140623503933440, 140623520718847,
+STORE, 140623495540736, 140623520718847,
+STORE, 140623361323008, 140623495540735,
+STORE, 140623227105280, 140623495540735,
+STORE, 140623218712576, 140623227105279,
+STORE, 140623084494848, 140623218712575,
+STORE, 140623076102144, 140623084494847,
+STORE, 140622941884416, 140623076102143,
+SNULL, 140622941884416, 140623000633343,
+STORE, 140623000633344, 140623076102143,
+STORE, 140622941884416, 140623000633343,
+ERASE, 140622941884416, 140623000633343,
+STORE, 140622992240640, 140623000633343,
+STORE, 140622983847936, 140623000633343,
+STORE, 140622849630208, 140622983847935,
+STORE, 140622841237504, 140622849630207,
+SNULL, 140622849630208, 140622866415615,
+STORE, 140622866415616, 140622983847935,
+STORE, 140622849630208, 140622866415615,
+ERASE, 140622849630208, 140622866415615,
+STORE, 140622858022912, 140622866415615,
+SNULL, 140622933524479, 140622983847935,
+STORE, 140622866415616, 140622933524479,
+STORE, 140622933524480, 140622983847935,
+ERASE, 140622933524480, 140622983847935,
+STORE, 140622975455232, 140623000633343,
+STORE, 140622707019776, 140622841237503,
+STORE, 140622967062528, 140623000633343,
+STORE, 140622572802048, 140622841237503,
+STORE, 140622958669824, 140623000633343,
+STORE, 140622438584320, 140622841237503,
+STORE, 140622950277120, 140623000633343,
+SNULL, 140622858027007, 140622866415615,
+STORE, 140622858022912, 140622858027007,
+STORE, 140622858027008, 140622866415615,
+STORE, 140622941884416, 140623000633343,
+STORE, 140622841237504, 140622858022911,
+SNULL, 140622841237504, 140622849630207,
+STORE, 140622849630208, 140622858022911,
+STORE, 140622841237504, 140622849630207,
+SNULL, 140622849634303, 140622858022911,
+STORE, 140622849630208, 140622849634303,
+STORE, 140622849634304, 140622858022911,
+STORE, 140622430191616, 140622438584319,
+SNULL, 140622430195711, 140622438584319,
+STORE, 140622430191616, 140622430195711,
+STORE, 140622430195712, 140622438584319,
+SNULL, 140623361323007, 140623495540735,
+STORE, 140623227105280, 140623361323007,
+STORE, 140623361323008, 140623495540735,
+SNULL, 140623361323008, 140623403286527,
+STORE, 140623403286528, 140623495540735,
+STORE, 140623361323008, 140623403286527,
+ERASE, 140623361323008, 140623403286527,
+SNULL, 140623470395391, 140623495540735,
+STORE, 140623403286528, 140623470395391,
+STORE, 140623470395392, 140623495540735,
+ERASE, 140623470395392, 140623495540735,
+SNULL, 140623227105280, 140623269068799,
+STORE, 140623269068800, 140623361323007,
+STORE, 140623227105280, 140623269068799,
+ERASE, 140623227105280, 140623269068799,
+SNULL, 140623084494848, 140623134851071,
+STORE, 140623134851072, 140623218712575,
+STORE, 140623084494848, 140623134851071,
+ERASE, 140623084494848, 140623134851071,
+SNULL, 140623201959935, 140623218712575,
+STORE, 140623134851072, 140623201959935,
+STORE, 140623201959936, 140623218712575,
+ERASE, 140623201959936, 140623218712575,
+SNULL, 140623067742207, 140623076102143,
+STORE, 140623000633344, 140623067742207,
+STORE, 140623067742208, 140623076102143,
+ERASE, 140623067742208, 140623076102143,
+STORE, 140622295973888, 140622430191615,
+SNULL, 140622295973888, 140622329544703,
+STORE, 140622329544704, 140622430191615,
+STORE, 140622295973888, 140622329544703,
+ERASE, 140622295973888, 140622329544703,
+SNULL, 140622866550783, 140622933524479,
+STORE, 140622866415616, 140622866550783,
+STORE, 140622866550784, 140622933524479,
+SNULL, 140622707019775, 140622841237503,
+STORE, 140622438584320, 140622707019775,
+STORE, 140622707019776, 140622841237503,
+SNULL, 140622707019776, 140622732197887,
+STORE, 140622732197888, 140622841237503,
+STORE, 140622707019776, 140622732197887,
+ERASE, 140622707019776, 140622732197887,
+SNULL, 140622799306751, 140622841237503,
+STORE, 140622732197888, 140622799306751,
+STORE, 140622799306752, 140622841237503,
+ERASE, 140622799306752, 140622841237503,
+SNULL, 140622572802047, 140622707019775,
+STORE, 140622438584320, 140622572802047,
+STORE, 140622572802048, 140622707019775,
+SNULL, 140622572802048, 140622597980159,
+STORE, 140622597980160, 140622707019775,
+STORE, 140622572802048, 140622597980159,
+ERASE, 140622572802048, 140622597980159,
+SNULL, 140622438584320, 140622463762431,
+STORE, 140622463762432, 140622572802047,
+STORE, 140622438584320, 140622463762431,
+ERASE, 140622438584320, 140622463762431,
+SNULL, 140622530871295, 140622572802047,
+STORE, 140622463762432, 140622530871295,
+STORE, 140622530871296, 140622572802047,
+ERASE, 140622530871296, 140622572802047,
+STORE, 140622195326976, 140622430191615,
+SNULL, 140622262435839, 140622430191615,
+STORE, 140622195326976, 140622262435839,
+STORE, 140622262435840, 140622430191615,
+SNULL, 140622262435840, 140622329544703,
+STORE, 140622329544704, 140622430191615,
+STORE, 140622262435840, 140622329544703,
+ERASE, 140622262435840, 140622329544703,
+SNULL, 140622841241599, 140622849630207,
+STORE, 140622841237504, 140622841241599,
+STORE, 140622841241600, 140622849630207,
+STORE, 140623487148032, 140623520718847,
+STORE, 140623478755328, 140623520718847,
+SNULL, 140622941884416, 140622983847935,
+STORE, 140622983847936, 140623000633343,
+STORE, 140622941884416, 140622983847935,
+SNULL, 140622983852031, 140623000633343,
+STORE, 140622983847936, 140622983852031,
+STORE, 140622983852032, 140623000633343,
+STORE, 140623394893824, 140623403286527,
+SNULL, 140623394897919, 140623403286527,
+STORE, 140623394893824, 140623394897919,
+STORE, 140623394897920, 140623403286527,
+SNULL, 140623403421695, 140623470395391,
+STORE, 140623403286528, 140623403421695,
+STORE, 140623403421696, 140623470395391,
+SNULL, 140623478755328, 140623503933439,
+STORE, 140623503933440, 140623520718847,
+STORE, 140623478755328, 140623503933439,
+SNULL, 140623503937535, 140623520718847,
+STORE, 140623503933440, 140623503937535,
+STORE, 140623503937536, 140623520718847,
+SNULL, 140623336177663, 140623361323007,
+STORE, 140623269068800, 140623336177663,
+STORE, 140623336177664, 140623361323007,
+ERASE, 140623336177664, 140623361323007,
+SNULL, 140623269203967, 140623336177663,
+STORE, 140623269068800, 140623269203967,
+STORE, 140623269203968, 140623336177663,
+SNULL, 140623134986239, 140623201959935,
+STORE, 140623134851072, 140623134986239,
+STORE, 140623134986240, 140623201959935,
+SNULL, 140623000768511, 140623067742207,
+STORE, 140623000633344, 140623000768511,
+STORE, 140623000768512, 140623067742207,
+SNULL, 140622396653567, 140622430191615,
+STORE, 140622329544704, 140622396653567,
+STORE, 140622396653568, 140622430191615,
+ERASE, 140622396653568, 140622430191615,
+SNULL, 140622732333055, 140622799306751,
+STORE, 140622732197888, 140622732333055,
+STORE, 140622732333056, 140622799306751,
+SNULL, 140622941884416, 140622975455231,
+STORE, 140622975455232, 140622983847935,
+STORE, 140622941884416, 140622975455231,
+SNULL, 140622975459327, 140622983847935,
+STORE, 140622975455232, 140622975459327,
+STORE, 140622975459328, 140622983847935,
+SNULL, 140622665089023, 140622707019775,
+STORE, 140622597980160, 140622665089023,
+STORE, 140622665089024, 140622707019775,
+ERASE, 140622665089024, 140622707019775,
+SNULL, 140622598115327, 140622665089023,
+STORE, 140622597980160, 140622598115327,
+STORE, 140622598115328, 140622665089023,
+SNULL, 140622463897599, 140622530871295,
+STORE, 140622463762432, 140622463897599,
+STORE, 140622463897600, 140622530871295,
+SNULL, 140622195462143, 140622262435839,
+STORE, 140622195326976, 140622195462143,
+STORE, 140622195462144, 140622262435839,
+STORE, 140623386501120, 140623394893823,
+SNULL, 140622941884416, 140622950277119,
+STORE, 140622950277120, 140622975455231,
+STORE, 140622941884416, 140622950277119,
+SNULL, 140622950281215, 140622975455231,
+STORE, 140622950277120, 140622950281215,
+STORE, 140622950281216, 140622975455231,
+SNULL, 140622941888511, 140622950277119,
+STORE, 140622941884416, 140622941888511,
+STORE, 140622941888512, 140622950277119,
+STORE, 140623378108416, 140623394893823,
+SNULL, 140623478755328, 140623495540735,
+STORE, 140623495540736, 140623503933439,
+STORE, 140623478755328, 140623495540735,
+SNULL, 140623495544831, 140623503933439,
+STORE, 140623495540736, 140623495544831,
+STORE, 140623495544832, 140623503933439,
+SNULL, 140623478755328, 140623487148031,
+STORE, 140623487148032, 140623495540735,
+STORE, 140623478755328, 140623487148031,
+SNULL, 140623487152127, 140623495540735,
+STORE, 140623487148032, 140623487152127,
+STORE, 140623487152128, 140623495540735,
+SNULL, 140623218716671, 140623227105279,
+STORE, 140623218712576, 140623218716671,
+STORE, 140623218716672, 140623227105279,
+SNULL, 140623076106239, 140623084494847,
+STORE, 140623076102144, 140623076106239,
+STORE, 140623076106240, 140623084494847,
+SNULL, 140622329679871, 140622396653567,
+STORE, 140622329544704, 140622329679871,
+STORE, 140622329679872, 140622396653567,
+SNULL, 140622950281216, 140622958669823,
+STORE, 140622958669824, 140622975455231,
+STORE, 140622950281216, 140622958669823,
+SNULL, 140622958673919, 140622975455231,
+STORE, 140622958669824, 140622958673919,
+STORE, 140622958673920, 140622975455231,
+SNULL, 140623503937536, 140623512326143,
+STORE, 140623512326144, 140623520718847,
+STORE, 140623503937536, 140623512326143,
+SNULL, 140623512330239, 140623520718847,
+STORE, 140623512326144, 140623512330239,
+STORE, 140623512330240, 140623520718847,
+SNULL, 140623378108416, 140623386501119,
+STORE, 140623386501120, 140623394893823,
+STORE, 140623378108416, 140623386501119,
+SNULL, 140623386505215, 140623394893823,
+STORE, 140623386501120, 140623386505215,
+STORE, 140623386505216, 140623394893823,
+STORE, 140623369715712, 140623386501119,
+STORE, 140623361323008, 140623386501119,
+STORE, 140623352930304, 140623386501119,
+SNULL, 140623352930304, 140623361323007,
+STORE, 140623361323008, 140623386501119,
+STORE, 140623352930304, 140623361323007,
+SNULL, 140623361327103, 140623386501119,
+STORE, 140623361323008, 140623361327103,
+STORE, 140623361327104, 140623386501119,
+SNULL, 140623478759423, 140623487148031,
+STORE, 140623478755328, 140623478759423,
+STORE, 140623478759424, 140623487148031,
+STORE, 140623344537600, 140623361323007,
+STORE, 140623260676096, 140623269068799,
+SNULL, 140622958673920, 140622967062527,
+STORE, 140622967062528, 140622975455231,
+STORE, 140622958673920, 140622967062527,
+SNULL, 140622967066623, 140622975455231,
+STORE, 140622967062528, 140622967066623,
+STORE, 140622967066624, 140622975455231,
+STORE, 140623252283392, 140623269068799,
+STORE, 140623243890688, 140623269068799,
+SNULL, 140622983852032, 140622992240639,
+STORE, 140622992240640, 140623000633343,
+STORE, 140622983852032, 140622992240639,
+SNULL, 140622992244735, 140623000633343,
+STORE, 140622992240640, 140622992244735,
+STORE, 140622992244736, 140623000633343,
+STORE, 140623235497984, 140623269068799,
+STORE, 140623218716672, 140623235497983,
+STORE, 140623210319872, 140623218712575,
+STORE, 140623126458368, 140623134851071,
+SNULL, 140623210323967, 140623218712575,
+STORE, 140623210319872, 140623210323967,
+STORE, 140623210323968, 140623218712575,
+SNULL, 140623218716672, 140623227105279,
+STORE, 140623227105280, 140623235497983,
+STORE, 140623218716672, 140623227105279,
+SNULL, 140623227109375, 140623235497983,
+STORE, 140623227105280, 140623227109375,
+STORE, 140623227109376, 140623235497983,
+STORE, 140623118065664, 140623134851071,
+STORE, 140623109672960, 140623134851071,
+SNULL, 140623109677055, 140623134851071,
+STORE, 140623109672960, 140623109677055,
+STORE, 140623109677056, 140623134851071,
+STORE, 140623101280256, 140623109672959,
+STORE, 140623092887552, 140623109672959,
+SNULL, 140623092887552, 140623101280255,
+STORE, 140623101280256, 140623109672959,
+STORE, 140623092887552, 140623101280255,
+SNULL, 140623101284351, 140623109672959,
+STORE, 140623101280256, 140623101284351,
+STORE, 140623101284352, 140623109672959,
+SNULL, 140623361327104, 140623378108415,
+STORE, 140623378108416, 140623386501119,
+STORE, 140623361327104, 140623378108415,
+SNULL, 140623378112511, 140623386501119,
+STORE, 140623378108416, 140623378112511,
+STORE, 140623378112512, 140623386501119,
+SNULL, 140623235497984, 140623243890687,
+STORE, 140623243890688, 140623269068799,
+STORE, 140623235497984, 140623243890687,
+SNULL, 140623243894783, 140623269068799,
+STORE, 140623243890688, 140623243894783,
+STORE, 140623243894784, 140623269068799,
+SNULL, 140623361327104, 140623369715711,
+STORE, 140623369715712, 140623378108415,
+STORE, 140623361327104, 140623369715711,
+SNULL, 140623369719807, 140623378108415,
+STORE, 140623369715712, 140623369719807,
+STORE, 140623369719808, 140623378108415,
+SNULL, 140623243894784, 140623252283391,
+STORE, 140623252283392, 140623269068799,
+STORE, 140623243894784, 140623252283391,
+SNULL, 140623252287487, 140623269068799,
+STORE, 140623252283392, 140623252287487,
+STORE, 140623252287488, 140623269068799,
+SNULL, 140623235502079, 140623243890687,
+STORE, 140623235497984, 140623235502079,
+STORE, 140623235502080, 140623243890687,
+SNULL, 140623344541695, 140623361323007,
+STORE, 140623344537600, 140623344541695,
+STORE, 140623344541696, 140623361323007,
+STORE, 140623076106240, 140623092887551,
+SNULL, 140623076106240, 140623084494847,
+STORE, 140623084494848, 140623092887551,
+STORE, 140623076106240, 140623084494847,
+SNULL, 140623084498943, 140623092887551,
+STORE, 140623084494848, 140623084498943,
+STORE, 140623084498944, 140623092887551,
+SNULL, 140623344541696, 140623352930303,
+STORE, 140623352930304, 140623361323007,
+STORE, 140623344541696, 140623352930303,
+SNULL, 140623352934399, 140623361323007,
+STORE, 140623352930304, 140623352934399,
+STORE, 140623352934400, 140623361323007,
+SNULL, 140623109677056, 140623118065663,
+STORE, 140623118065664, 140623134851071,
+STORE, 140623109677056, 140623118065663,
+SNULL, 140623118069759, 140623134851071,
+STORE, 140623118065664, 140623118069759,
+STORE, 140623118069760, 140623134851071,
+STORE, 140622832844800, 140622841237503,
+STORE, 140622824452096, 140622841237503,
+SNULL, 140622824452096, 140622832844799,
+STORE, 140622832844800, 140622841237503,
+STORE, 140622824452096, 140622832844799,
+SNULL, 140622832848895, 140622841237503,
+STORE, 140622832844800, 140622832848895,
+STORE, 140622832848896, 140622841237503,
+STORE, 140622816059392, 140622832844799,
+SNULL, 140623092891647, 140623101280255,
+STORE, 140623092887552, 140623092891647,
+STORE, 140623092891648, 140623101280255,
+SNULL, 140623118069760, 140623126458367,
+STORE, 140623126458368, 140623134851071,
+STORE, 140623118069760, 140623126458367,
+SNULL, 140623126462463, 140623134851071,
+STORE, 140623126458368, 140623126462463,
+STORE, 140623126462464, 140623134851071,
+SNULL, 140623252287488, 140623260676095,
+STORE, 140623260676096, 140623269068799,
+STORE, 140623252287488, 140623260676095,
+SNULL, 140623260680191, 140623269068799,
+STORE, 140623260676096, 140623260680191,
+STORE, 140623260680192, 140623269068799,
+STORE, 140622807666688, 140622832844799,
+STORE, 140622723805184, 140622732197887,
+STORE, 140622715412480, 140622732197887,
+STORE, 140622707019776, 140622732197887,
+SNULL, 140622707023871, 140622732197887,
+STORE, 140622707019776, 140622707023871,
+STORE, 140622707023872, 140622732197887,
+STORE, 140622698627072, 140622707019775,
+STORE, 140622690234368, 140622707019775,
+SNULL, 140622690238463, 140622707019775,
+STORE, 140622690234368, 140622690238463,
+STORE, 140622690238464, 140622707019775,
+SNULL, 140622807666688, 140622816059391,
+STORE, 140622816059392, 140622832844799,
+STORE, 140622807666688, 140622816059391,
+SNULL, 140622816063487, 140622832844799,
+STORE, 140622816059392, 140622816063487,
+STORE, 140622816063488, 140622832844799,
+STORE, 140622681841664, 140622690234367,
+STORE, 140622673448960, 140622690234367,
+SNULL, 140622673453055, 140622690234367,
+STORE, 140622673448960, 140622673453055,
+STORE, 140622673453056, 140622690234367,
+STORE, 140622589587456, 140622597980159,
+SNULL, 140622807670783, 140622816059391,
+STORE, 140622807666688, 140622807670783,
+STORE, 140622807670784, 140622816059391,
+STORE, 140622581194752, 140622597980159,
+SNULL, 140622581198847, 140622597980159,
+STORE, 140622581194752, 140622581198847,
+STORE, 140622581198848, 140622597980159,
+SNULL, 140622816063488, 140622824452095,
+STORE, 140622824452096, 140622832844799,
+STORE, 140622816063488, 140622824452095,
+SNULL, 140622824456191, 140622832844799,
+STORE, 140622824452096, 140622824456191,
+STORE, 140622824456192, 140622832844799,
+STORE, 140622572802048, 140622581194751,
+SNULL, 140622572806143, 140622581194751,
+STORE, 140622572802048, 140622572806143,
+STORE, 140622572806144, 140622581194751,
+STORE, 140622564409344, 140622572802047,
+STORE, 140622556016640, 140622572802047,
+SNULL, 140622556016640, 140622564409343,
+STORE, 140622564409344, 140622572802047,
+STORE, 140622556016640, 140622564409343,
+SNULL, 140622564413439, 140622572802047,
+STORE, 140622564409344, 140622564413439,
+STORE, 140622564413440, 140622572802047,
+SNULL, 140622690238464, 140622698627071,
+STORE, 140622698627072, 140622707019775,
+STORE, 140622690238464, 140622698627071,
+SNULL, 140622698631167, 140622707019775,
+STORE, 140622698627072, 140622698631167,
+STORE, 140622698631168, 140622707019775,
+SNULL, 140622707023872, 140622723805183,
+STORE, 140622723805184, 140622732197887,
+STORE, 140622707023872, 140622723805183,
+SNULL, 140622723809279, 140622732197887,
+STORE, 140622723805184, 140622723809279,
+STORE, 140622723809280, 140622732197887,
+SNULL, 140622707023872, 140622715412479,
+STORE, 140622715412480, 140622723805183,
+STORE, 140622707023872, 140622715412479,
+SNULL, 140622715416575, 140622723805183,
+STORE, 140622715412480, 140622715416575,
+STORE, 140622715416576, 140622723805183,
+STORE, 140622547623936, 140622564409343,
+SNULL, 140622547628031, 140622564409343,
+STORE, 140622547623936, 140622547628031,
+STORE, 140622547628032, 140622564409343,
+STORE, 140622539231232, 140622547623935,
+SNULL, 140622539235327, 140622547623935,
+STORE, 140622539231232, 140622539235327,
+STORE, 140622539235328, 140622547623935,
+SNULL, 140622581198848, 140622589587455,
+STORE, 140622589587456, 140622597980159,
+STORE, 140622581198848, 140622589587455,
+SNULL, 140622589591551, 140622597980159,
+STORE, 140622589587456, 140622589591551,
+STORE, 140622589591552, 140622597980159,
+STORE, 140622455369728, 140622463762431,
+SNULL, 140622455373823, 140622463762431,
+STORE, 140622455369728, 140622455373823,
+STORE, 140622455373824, 140622463762431,
+STORE, 140622446977024, 140622455369727,
+SNULL, 140622446981119, 140622455369727,
+STORE, 140622446977024, 140622446981119,
+STORE, 140622446981120, 140622455369727,
+SNULL, 140622547628032, 140622556016639,
+STORE, 140622556016640, 140622564409343,
+STORE, 140622547628032, 140622556016639,
+SNULL, 140622556020735, 140622564409343,
+STORE, 140622556016640, 140622556020735,
+STORE, 140622556020736, 140622564409343,
+STORE, 140622430195712, 140622446977023,
+STORE, 140622421798912, 140622430191615,
+SNULL, 140622430195712, 140622438584319,
+STORE, 140622438584320, 140622446977023,
+STORE, 140622430195712, 140622438584319,
+SNULL, 140622438588415, 140622446977023,
+STORE, 140622438584320, 140622438588415,
+STORE, 140622438588416, 140622446977023,
+STORE, 140622413406208, 140622430191615,
+STORE, 140622405013504, 140622430191615,
+SNULL, 140622405013504, 140622413406207,
+STORE, 140622413406208, 140622430191615,
+STORE, 140622405013504, 140622413406207,
+SNULL, 140622413410303, 140622430191615,
+STORE, 140622413406208, 140622413410303,
+STORE, 140622413410304, 140622430191615,
+SNULL, 140622673453056, 140622681841663,
+STORE, 140622681841664, 140622690234367,
+STORE, 140622673453056, 140622681841663,
+SNULL, 140622681845759, 140622690234367,
+STORE, 140622681841664, 140622681845759,
+STORE, 140622681845760, 140622690234367,
+STORE, 140622321152000, 140622329544703,
+SNULL, 140622413410304, 140622421798911,
+STORE, 140622421798912, 140622430191615,
+STORE, 140622413410304, 140622421798911,
+SNULL, 140622421803007, 140622430191615,
+STORE, 140622421798912, 140622421803007,
+STORE, 140622421803008, 140622430191615,
+STORE, 140622312759296, 140622329544703,
+SNULL, 140622312763391, 140622329544703,
+STORE, 140622312759296, 140622312763391,
+STORE, 140622312763392, 140622329544703,
+SNULL, 140622405017599, 140622413406207,
+STORE, 140622405013504, 140622405017599,
+STORE, 140622405017600, 140622413406207,
+STORE, 140622304366592, 140622312759295,
+SNULL, 140622304370687, 140622312759295,
+STORE, 140622304366592, 140622304370687,
+STORE, 140622304370688, 140622312759295,
+SNULL, 140622312763392, 140622321151999,
+STORE, 140622321152000, 140622329544703,
+STORE, 140622312763392, 140622321151999,
+SNULL, 140622321156095, 140622329544703,
+STORE, 140622321152000, 140622321156095,
+STORE, 140622321156096, 140622329544703,
+STORE, 140624062619648, 140624062648319,
+STORE, 140624010240000, 140624012431359,
+SNULL, 140624010240000, 140624010330111,
+STORE, 140624010330112, 140624012431359,
+STORE, 140624010240000, 140624010330111,
+SNULL, 140624012423167, 140624012431359,
+STORE, 140624010330112, 140624012423167,
+STORE, 140624012423168, 140624012431359,
+ERASE, 140624012423168, 140624012431359,
+STORE, 140624012423168, 140624012431359,
+SNULL, 140624012427263, 140624012431359,
+STORE, 140624012423168, 140624012427263,
+STORE, 140624012427264, 140624012431359,
+ERASE, 140624062619648, 140624062648319,
+ERASE, 140622849630208, 140622849634303,
+ERASE, 140622849634304, 140622858022911,
+ERASE, 140623394893824, 140623394897919,
+ERASE, 140623394897920, 140623403286527,
+ERASE, 140623361323008, 140623361327103,
+ERASE, 140623361327104, 140623369715711,
+ERASE, 140623084494848, 140623084498943,
+ERASE, 140623084498944, 140623092887551,
+ERASE, 140623931764736, 140623931768831,
+ERASE, 140623931768832, 140623940157439,
+ERASE, 140622841237504, 140622841241599,
+ERASE, 140622841241600, 140622849630207,
+ERASE, 140623487148032, 140623487152127,
+ERASE, 140623487152128, 140623495540735,
+ERASE, 140623109672960, 140623109677055,
+ERASE, 140623109677056, 140623118065663,
+ERASE, 140622983847936, 140622983852031,
+ERASE, 140622983852032, 140622992240639,
+ERASE, 140623352930304, 140623352934399,
+ERASE, 140623352934400, 140623361323007,
+ERASE, 140622564409344, 140622564413439,
+ERASE, 140622564413440, 140622572802047,
+ERASE, 140622430191616, 140622430195711,
+ERASE, 140622430195712, 140622438584319,
+ERASE, 140622958669824, 140622958673919,
+ERASE, 140622958673920, 140622967062527,
+ERASE, 140622992240640, 140622992244735,
+ERASE, 140622992244736, 140623000633343,
+ERASE, 140623227105280, 140623227109375,
+ERASE, 140623227109376, 140623235497983,
+ERASE, 140622321152000, 140622321156095,
+ERASE, 140622321156096, 140622329544703,
+ERASE, 140622858022912, 140622858027007,
+ERASE, 140622858027008, 140622866415615,
+ERASE, 140622975455232, 140622975459327,
+ERASE, 140622975459328, 140622983847935,
+ERASE, 140623378108416, 140623378112511,
+ERASE, 140623378112512, 140623386501119,
+ERASE, 140623495540736, 140623495544831,
+ERASE, 140623495544832, 140623503933439,
+ERASE, 140623118065664, 140623118069759,
+ERASE, 140623118069760, 140623126458367,
+ERASE, 140622572802048, 140622572806143,
+ERASE, 140622572806144, 140622581194751,
+ERASE, 140622421798912, 140622421803007,
+ERASE, 140622421803008, 140622430191615,
+ERASE, 140622967062528, 140622967066623,
+ERASE, 140622967066624, 140622975455231,
+ERASE, 140623252283392, 140623252287487,
+ERASE, 140623252287488, 140623260676095,
+ERASE, 140622673448960, 140622673453055,
+ERASE, 140622673453056, 140622681841663,
+ERASE, 140623076102144, 140623076106239,
+ERASE, 140623076106240, 140623084494847,
+ERASE, 140623101280256, 140623101284351,
+ERASE, 140623101284352, 140623109672959,
+ERASE, 140622715412480, 140622715416575,
+ERASE, 140622715416576, 140622723805183,
+ERASE, 140622405013504, 140622405017599,
+ERASE, 140622405017600, 140622413406207,
+ERASE, 140623478755328, 140623478759423,
+ERASE, 140623478759424, 140623487148031,
+ERASE, 140623906586624, 140623906590719,
+ERASE, 140623906590720, 140623914979327,
+ERASE, 140622950277120, 140622950281215,
+ERASE, 140622950281216, 140622958669823,
+ };
+ unsigned long set32[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140731244212224, 140737488351231,
+SNULL, 140731244216319, 140737488351231,
+STORE, 140731244212224, 140731244216319,
+STORE, 140731244081152, 140731244216319,
+STORE, 94427773984768, 94427776237567,
+SNULL, 94427774115839, 94427776237567,
+STORE, 94427773984768, 94427774115839,
+STORE, 94427774115840, 94427776237567,
+ERASE, 94427774115840, 94427776237567,
+STORE, 94427776208896, 94427776217087,
+STORE, 94427776217088, 94427776237567,
+STORE, 140401464893440, 140401467146239,
+SNULL, 140401465036799, 140401467146239,
+STORE, 140401464893440, 140401465036799,
+STORE, 140401465036800, 140401467146239,
+ERASE, 140401465036800, 140401467146239,
+STORE, 140401467133952, 140401467142143,
+STORE, 140401467142144, 140401467146239,
+STORE, 140731244507136, 140731244511231,
+STORE, 140731244494848, 140731244507135,
+STORE, 140401467105280, 140401467133951,
+STORE, 140401467097088, 140401467105279,
+STORE, 140401462677504, 140401464893439,
+SNULL, 140401462677504, 140401462775807,
+STORE, 140401462775808, 140401464893439,
+STORE, 140401462677504, 140401462775807,
+SNULL, 140401464868863, 140401464893439,
+STORE, 140401462775808, 140401464868863,
+STORE, 140401464868864, 140401464893439,
+SNULL, 140401464868864, 140401464877055,
+STORE, 140401464877056, 140401464893439,
+STORE, 140401464868864, 140401464877055,
+ERASE, 140401464868864, 140401464877055,
+STORE, 140401464868864, 140401464877055,
+ERASE, 140401464877056, 140401464893439,
+STORE, 140401464877056, 140401464893439,
+STORE, 140401458880512, 140401462677503,
+SNULL, 140401458880512, 140401460539391,
+STORE, 140401460539392, 140401462677503,
+STORE, 140401458880512, 140401460539391,
+SNULL, 140401462636543, 140401462677503,
+STORE, 140401460539392, 140401462636543,
+STORE, 140401462636544, 140401462677503,
+SNULL, 140401462636544, 140401462661119,
+STORE, 140401462661120, 140401462677503,
+STORE, 140401462636544, 140401462661119,
+ERASE, 140401462636544, 140401462661119,
+STORE, 140401462636544, 140401462661119,
+ERASE, 140401462661120, 140401462677503,
+STORE, 140401462661120, 140401462677503,
+STORE, 140401467088896, 140401467105279,
+SNULL, 140401462652927, 140401462661119,
+STORE, 140401462636544, 140401462652927,
+STORE, 140401462652928, 140401462661119,
+SNULL, 140401464872959, 140401464877055,
+STORE, 140401464868864, 140401464872959,
+STORE, 140401464872960, 140401464877055,
+SNULL, 94427776212991, 94427776217087,
+STORE, 94427776208896, 94427776212991,
+STORE, 94427776212992, 94427776217087,
+SNULL, 140401467138047, 140401467142143,
+STORE, 140401467133952, 140401467138047,
+STORE, 140401467138048, 140401467142143,
+ERASE, 140401467105280, 140401467133951,
+STORE, 94427784683520, 94427784818687,
+STORE, 140401450487808, 140401458880511,
+SNULL, 140401450491903, 140401458880511,
+STORE, 140401450487808, 140401450491903,
+STORE, 140401450491904, 140401458880511,
+STORE, 140401442095104, 140401450487807,
+STORE, 140401307877376, 140401442095103,
+SNULL, 140401307877376, 140401340055551,
+STORE, 140401340055552, 140401442095103,
+STORE, 140401307877376, 140401340055551,
+ERASE, 140401307877376, 140401340055551,
+SNULL, 140401407164415, 140401442095103,
+STORE, 140401340055552, 140401407164415,
+STORE, 140401407164416, 140401442095103,
+ERASE, 140401407164416, 140401442095103,
+SNULL, 140401340190719, 140401407164415,
+STORE, 140401340055552, 140401340190719,
+STORE, 140401340190720, 140401407164415,
+SNULL, 140401442099199, 140401450487807,
+STORE, 140401442095104, 140401442099199,
+STORE, 140401442099200, 140401450487807,
+STORE, 140401433702400, 140401442095103,
+SNULL, 140401433706495, 140401442095103,
+STORE, 140401433702400, 140401433706495,
+STORE, 140401433706496, 140401442095103,
+STORE, 140401425309696, 140401433702399,
+SNULL, 140401425313791, 140401433702399,
+STORE, 140401425309696, 140401425313791,
+STORE, 140401425313792, 140401433702399,
+STORE, 140401416916992, 140401425309695,
+SNULL, 140401416921087, 140401425309695,
+STORE, 140401416916992, 140401416921087,
+STORE, 140401416921088, 140401425309695,
+STORE, 140401408524288, 140401416916991,
+STORE, 140401205837824, 140401340055551,
+SNULL, 140401272946687, 140401340055551,
+STORE, 140401205837824, 140401272946687,
+STORE, 140401272946688, 140401340055551,
+ERASE, 140401272946688, 140401340055551,
+SNULL, 140401205972991, 140401272946687,
+STORE, 140401205837824, 140401205972991,
+STORE, 140401205972992, 140401272946687,
+STORE, 140401331662848, 140401340055551,
+STORE, 140401323270144, 140401340055551,
+STORE, 140401138728960, 140401205837823,
+STORE, 140401314877440, 140401340055551,
+SNULL, 140401408528383, 140401416916991,
+STORE, 140401408524288, 140401408528383,
+STORE, 140401408528384, 140401416916991,
+SNULL, 140401138864127, 140401205837823,
+STORE, 140401138728960, 140401138864127,
+STORE, 140401138864128, 140401205837823,
+STORE, 140401004511232, 140401138728959,
+SNULL, 140401071620095, 140401138728959,
+STORE, 140401004511232, 140401071620095,
+STORE, 140401071620096, 140401138728959,
+ERASE, 140401071620096, 140401138728959,
+STORE, 140400870293504, 140401071620095,
+SNULL, 140400937402367, 140401071620095,
+STORE, 140400870293504, 140400937402367,
+STORE, 140400937402368, 140401071620095,
+SNULL, 140400937402368, 140401004511231,
+STORE, 140401004511232, 140401071620095,
+STORE, 140400937402368, 140401004511231,
+ERASE, 140400937402368, 140401004511231,
+STORE, 140401306484736, 140401340055551,
+SNULL, 140401306484736, 140401323270143,
+STORE, 140401323270144, 140401340055551,
+STORE, 140401306484736, 140401323270143,
+SNULL, 140401323274239, 140401340055551,
+STORE, 140401323270144, 140401323274239,
+STORE, 140401323274240, 140401340055551,
+SNULL, 140401004646399, 140401071620095,
+STORE, 140401004511232, 140401004646399,
+STORE, 140401004646400, 140401071620095,
+SNULL, 140400870428671, 140400937402367,
+STORE, 140400870293504, 140400870428671,
+STORE, 140400870428672, 140400937402367,
+SNULL, 140401306488831, 140401323270143,
+STORE, 140401306484736, 140401306488831,
+STORE, 140401306488832, 140401323270143,
+STORE, 140401298092032, 140401306484735,
+SNULL, 140401306488832, 140401314877439,
+STORE, 140401314877440, 140401323270143,
+STORE, 140401306488832, 140401314877439,
+SNULL, 140401314881535, 140401323270143,
+STORE, 140401314877440, 140401314881535,
+STORE, 140401314881536, 140401323270143,
+SNULL, 140401323274240, 140401331662847,
+STORE, 140401331662848, 140401340055551,
+STORE, 140401323274240, 140401331662847,
+SNULL, 140401331666943, 140401340055551,
+STORE, 140401331662848, 140401331666943,
+STORE, 140401331666944, 140401340055551,
+SNULL, 140401298096127, 140401306484735,
+STORE, 140401298092032, 140401298096127,
+STORE, 140401298096128, 140401306484735,
+STORE, 140401289699328, 140401298092031,
+STORE, 140401281306624, 140401298092031,
+STORE, 140401130336256, 140401138728959,
+SNULL, 140401281306624, 140401289699327,
+STORE, 140401289699328, 140401298092031,
+STORE, 140401281306624, 140401289699327,
+SNULL, 140401289703423, 140401298092031,
+STORE, 140401289699328, 140401289703423,
+STORE, 140401289703424, 140401298092031,
+STORE, 140401121943552, 140401138728959,
+STORE, 140401113550848, 140401138728959,
+SNULL, 140401281310719, 140401289699327,
+STORE, 140401281306624, 140401281310719,
+STORE, 140401281310720, 140401289699327,
+SNULL, 140401113550848, 140401121943551,
+STORE, 140401121943552, 140401138728959,
+STORE, 140401113550848, 140401121943551,
+SNULL, 140401121947647, 140401138728959,
+STORE, 140401121943552, 140401121947647,
+STORE, 140401121947648, 140401138728959,
+STORE, 140401105158144, 140401121943551,
+SNULL, 140401121947648, 140401130336255,
+STORE, 140401130336256, 140401138728959,
+STORE, 140401121947648, 140401130336255,
+SNULL, 140401130340351, 140401138728959,
+STORE, 140401130336256, 140401130340351,
+STORE, 140401130340352, 140401138728959,
+STORE, 140401096765440, 140401121943551,
+SNULL, 140401096765440, 140401113550847,
+STORE, 140401113550848, 140401121943551,
+STORE, 140401096765440, 140401113550847,
+SNULL, 140401113554943, 140401121943551,
+STORE, 140401113550848, 140401113554943,
+STORE, 140401113554944, 140401121943551,
+STORE, 140401088372736, 140401113550847,
+SNULL, 140401088372736, 140401096765439,
+STORE, 140401096765440, 140401113550847,
+STORE, 140401088372736, 140401096765439,
+SNULL, 140401096769535, 140401113550847,
+STORE, 140401096765440, 140401096769535,
+STORE, 140401096769536, 140401113550847,
+SNULL, 140401096769536, 140401105158143,
+STORE, 140401105158144, 140401113550847,
+STORE, 140401096769536, 140401105158143,
+SNULL, 140401105162239, 140401113550847,
+STORE, 140401105158144, 140401105162239,
+STORE, 140401105162240, 140401113550847,
+SNULL, 140401088376831, 140401096765439,
+STORE, 140401088372736, 140401088376831,
+STORE, 140401088376832, 140401096765439,
+STORE, 140401079980032, 140401088372735,
+STORE, 140400996118528, 140401004511231,
+SNULL, 140401079984127, 140401088372735,
+STORE, 140401079980032, 140401079984127,
+STORE, 140401079984128, 140401088372735,
+SNULL, 140400996122623, 140401004511231,
+STORE, 140400996118528, 140400996122623,
+STORE, 140400996122624, 140401004511231,
+STORE, 140400987725824, 140400996118527,
+STORE, 140400979333120, 140400996118527,
+STORE, 140400803184640, 140400870293503,
+SNULL, 140400803319807, 140400870293503,
+STORE, 140400803184640, 140400803319807,
+STORE, 140400803319808, 140400870293503,
+SNULL, 140400979333120, 140400987725823,
+STORE, 140400987725824, 140400996118527,
+STORE, 140400979333120, 140400987725823,
+SNULL, 140400987729919, 140400996118527,
+STORE, 140400987725824, 140400987729919,
+STORE, 140400987729920, 140400996118527,
+STORE, 140400970940416, 140400987725823,
+STORE, 140400962547712, 140400987725823,
+STORE, 140400668966912, 140400803184639,
+STORE, 140400954155008, 140400987725823,
+STORE, 140400945762304, 140400987725823,
+STORE, 140400660574208, 140400668966911,
+STORE, 140400593465344, 140400660574207,
+STORE, 140400585072640, 140400593465343,
+STORE, 140400450854912, 140400585072639,
+STORE, 140400442462208, 140400450854911,
+STORE, 140400434069504, 140400450854911,
+STORE, 140400299851776, 140400434069503,
+STORE, 140400291459072, 140400299851775,
+SNULL, 140400299851776, 140400333422591,
+STORE, 140400333422592, 140400434069503,
+STORE, 140400299851776, 140400333422591,
+ERASE, 140400299851776, 140400333422591,
+STORE, 140400325029888, 140400333422591,
+STORE, 140400157241344, 140400291459071,
+STORE, 140400316637184, 140400333422591,
+STORE, 140400308244480, 140400333422591,
+STORE, 140400023023616, 140400291459071,
+STORE, 140400291459072, 140400333422591,
+SNULL, 140400023023616, 140400064987135,
+STORE, 140400064987136, 140400291459071,
+STORE, 140400023023616, 140400064987135,
+ERASE, 140400023023616, 140400064987135,
+STORE, 140400056594432, 140400064987135,
+SNULL, 140400056598527, 140400064987135,
+STORE, 140400056594432, 140400056598527,
+STORE, 140400056598528, 140400064987135,
+STORE, 140399989485568, 140400056594431,
+SNULL, 140400291459072, 140400316637183,
+STORE, 140400316637184, 140400333422591,
+STORE, 140400291459072, 140400316637183,
+SNULL, 140400316641279, 140400333422591,
+STORE, 140400316637184, 140400316641279,
+STORE, 140400316641280, 140400333422591,
+STORE, 140399855267840, 140400056594431,
+SNULL, 140399855267840, 140399863660543,
+STORE, 140399863660544, 140400056594431,
+STORE, 140399855267840, 140399863660543,
+ERASE, 140399855267840, 140399863660543,
+SNULL, 140400736075775, 140400803184639,
+STORE, 140400668966912, 140400736075775,
+STORE, 140400736075776, 140400803184639,
+ERASE, 140400736075776, 140400803184639,
+SNULL, 140400669102079, 140400736075775,
+STORE, 140400668966912, 140400669102079,
+STORE, 140400669102080, 140400736075775,
+STORE, 140400669102080, 140400803184639,
+SNULL, 140400669102080, 140400736075775,
+STORE, 140400736075776, 140400803184639,
+STORE, 140400669102080, 140400736075775,
+SNULL, 140400736210943, 140400803184639,
+STORE, 140400736075776, 140400736210943,
+STORE, 140400736210944, 140400803184639,
+ERASE, 140400593465344, 140400660574207,
+SNULL, 140400450854912, 140400467640319,
+STORE, 140400467640320, 140400585072639,
+STORE, 140400450854912, 140400467640319,
+ERASE, 140400450854912, 140400467640319,
+STORE, 140399729442816, 140400056594431,
+SNULL, 140400400531455, 140400434069503,
+STORE, 140400333422592, 140400400531455,
+STORE, 140400400531456, 140400434069503,
+ERASE, 140400400531456, 140400434069503,
+SNULL, 140400333557759, 140400400531455,
+STORE, 140400333422592, 140400333557759,
+STORE, 140400333557760, 140400400531455,
+SNULL, 140400157241343, 140400291459071,
+STORE, 140400064987136, 140400157241343,
+STORE, 140400157241344, 140400291459071,
+SNULL, 140400157241344, 140400199204863,
+STORE, 140400199204864, 140400291459071,
+STORE, 140400157241344, 140400199204863,
+ERASE, 140400157241344, 140400199204863,
+SNULL, 140400266313727, 140400291459071,
+STORE, 140400199204864, 140400266313727,
+STORE, 140400266313728, 140400291459071,
+ERASE, 140400266313728, 140400291459071,
+SNULL, 140400132095999, 140400157241343,
+STORE, 140400064987136, 140400132095999,
+STORE, 140400132096000, 140400157241343,
+ERASE, 140400132096000, 140400157241343,
+SNULL, 140400065122303, 140400132095999,
+STORE, 140400064987136, 140400065122303,
+STORE, 140400065122304, 140400132095999,
+SNULL, 140400945762304, 140400954155007,
+STORE, 140400954155008, 140400987725823,
+STORE, 140400945762304, 140400954155007,
+SNULL, 140400954159103, 140400987725823,
+STORE, 140400954155008, 140400954159103,
+STORE, 140400954159104, 140400987725823,
+SNULL, 140400434069504, 140400442462207,
+STORE, 140400442462208, 140400450854911,
+STORE, 140400434069504, 140400442462207,
+SNULL, 140400442466303, 140400450854911,
+STORE, 140400442462208, 140400442466303,
+STORE, 140400442466304, 140400450854911,
+SNULL, 140400291463167, 140400316637183,
+STORE, 140400291459072, 140400291463167,
+STORE, 140400291463168, 140400316637183,
+STORE, 140400652181504, 140400668966911,
+STORE, 140400643788800, 140400668966911,
+SNULL, 140400291463168, 140400299851775,
+STORE, 140400299851776, 140400316637183,
+STORE, 140400291463168, 140400299851775,
+SNULL, 140400299855871, 140400316637183,
+STORE, 140400299851776, 140400299855871,
+STORE, 140400299855872, 140400316637183,
+STORE, 140400635396096, 140400668966911,
+SNULL, 140400635396096, 140400643788799,
+STORE, 140400643788800, 140400668966911,
+STORE, 140400635396096, 140400643788799,
+SNULL, 140400643792895, 140400668966911,
+STORE, 140400643788800, 140400643792895,
+STORE, 140400643792896, 140400668966911,
+SNULL, 140399989485567, 140400056594431,
+STORE, 140399729442816, 140399989485567,
+STORE, 140399989485568, 140400056594431,
+ERASE, 140399989485568, 140400056594431,
+SNULL, 140399930769407, 140399989485567,
+STORE, 140399729442816, 140399930769407,
+STORE, 140399930769408, 140399989485567,
+ERASE, 140399930769408, 140399989485567,
+SNULL, 140400945766399, 140400954155007,
+STORE, 140400945762304, 140400945766399,
+STORE, 140400945766400, 140400954155007,
+SNULL, 140400534749183, 140400585072639,
+STORE, 140400467640320, 140400534749183,
+STORE, 140400534749184, 140400585072639,
+ERASE, 140400534749184, 140400585072639,
+SNULL, 140399796551679, 140399930769407,
+STORE, 140399729442816, 140399796551679,
+STORE, 140399796551680, 140399930769407,
+SNULL, 140399796551680, 140399863660543,
+STORE, 140399863660544, 140399930769407,
+STORE, 140399796551680, 140399863660543,
+ERASE, 140399796551680, 140399863660543,
+SNULL, 140400199340031, 140400266313727,
+STORE, 140400199204864, 140400199340031,
+STORE, 140400199340032, 140400266313727,
+STORE, 140400627003392, 140400643788799,
+SNULL, 140400316641280, 140400325029887,
+STORE, 140400325029888, 140400333422591,
+STORE, 140400316641280, 140400325029887,
+SNULL, 140400325033983, 140400333422591,
+STORE, 140400325029888, 140400325033983,
+STORE, 140400325033984, 140400333422591,
+SNULL, 140400627003392, 140400635396095,
+STORE, 140400635396096, 140400643788799,
+STORE, 140400627003392, 140400635396095,
+SNULL, 140400635400191, 140400643788799,
+STORE, 140400635396096, 140400635400191,
+STORE, 140400635400192, 140400643788799,
+SNULL, 140400434073599, 140400442462207,
+STORE, 140400434069504, 140400434073599,
+STORE, 140400434073600, 140400442462207,
+STORE, 140400618610688, 140400635396095,
+STORE, 140400610217984, 140400635396095,
+SNULL, 140400954159104, 140400962547711,
+STORE, 140400962547712, 140400987725823,
+STORE, 140400954159104, 140400962547711,
+SNULL, 140400962551807, 140400987725823,
+STORE, 140400962547712, 140400962551807,
+STORE, 140400962551808, 140400987725823,
+SNULL, 140400299855872, 140400308244479,
+STORE, 140400308244480, 140400316637183,
+STORE, 140400299855872, 140400308244479,
+SNULL, 140400308248575, 140400316637183,
+STORE, 140400308244480, 140400308248575,
+STORE, 140400308248576, 140400316637183,
+STORE, 140400601825280, 140400635396095,
+SNULL, 140400601829375, 140400635396095,
+STORE, 140400601825280, 140400601829375,
+STORE, 140400601829376, 140400635396095,
+STORE, 140400576679936, 140400593465343,
+SNULL, 140400576684031, 140400593465343,
+STORE, 140400576679936, 140400576684031,
+STORE, 140400576684032, 140400593465343,
+SNULL, 140400643792896, 140400652181503,
+STORE, 140400652181504, 140400668966911,
+STORE, 140400643792896, 140400652181503,
+SNULL, 140400652185599, 140400668966911,
+STORE, 140400652181504, 140400652185599,
+STORE, 140400652185600, 140400668966911,
+STORE, 140399595225088, 140399796551679,
+SNULL, 140399662333951, 140399796551679,
+STORE, 140399595225088, 140399662333951,
+STORE, 140399662333952, 140399796551679,
+SNULL, 140399662333952, 140399729442815,
+STORE, 140399729442816, 140399796551679,
+STORE, 140399662333952, 140399729442815,
+ERASE, 140399662333952, 140399729442815,
+SNULL, 140399863795711, 140399930769407,
+STORE, 140399863660544, 140399863795711,
+STORE, 140399863795712, 140399930769407,
+STORE, 140400568287232, 140400576679935,
+SNULL, 140400568291327, 140400576679935,
+STORE, 140400568287232, 140400568291327,
+STORE, 140400568291328, 140400576679935,
+SNULL, 140400467775487, 140400534749183,
+STORE, 140400467640320, 140400467775487,
+STORE, 140400467775488, 140400534749183,
+SNULL, 140399729577983, 140399796551679,
+STORE, 140399729442816, 140399729577983,
+STORE, 140399729577984, 140399796551679,
+SNULL, 140400601829376, 140400627003391,
+STORE, 140400627003392, 140400635396095,
+STORE, 140400601829376, 140400627003391,
+SNULL, 140400627007487, 140400635396095,
+STORE, 140400627003392, 140400627007487,
+STORE, 140400627007488, 140400635396095,
+STORE, 140400559894528, 140400568287231,
+STORE, 140400551501824, 140400568287231,
+STORE, 140400543109120, 140400568287231,
+STORE, 140400459247616, 140400467640319,
+STORE, 140400442466304, 140400467640319,
+SNULL, 140399595360255, 140399662333951,
+STORE, 140399595225088, 140399595360255,
+STORE, 140399595360256, 140399662333951,
+SNULL, 140400962551808, 140400970940415,
+STORE, 140400970940416, 140400987725823,
+STORE, 140400962551808, 140400970940415,
+SNULL, 140400970944511, 140400987725823,
+STORE, 140400970940416, 140400970944511,
+STORE, 140400970944512, 140400987725823,
+SNULL, 140400652185600, 140400660574207,
+STORE, 140400660574208, 140400668966911,
+STORE, 140400652185600, 140400660574207,
+SNULL, 140400660578303, 140400668966911,
+STORE, 140400660574208, 140400660578303,
+STORE, 140400660578304, 140400668966911,
+SNULL, 140400576684032, 140400585072639,
+STORE, 140400585072640, 140400593465343,
+STORE, 140400576684032, 140400585072639,
+SNULL, 140400585076735, 140400593465343,
+STORE, 140400585072640, 140400585076735,
+STORE, 140400585076736, 140400593465343,
+STORE, 140400425676800, 140400434069503,
+STORE, 140400417284096, 140400434069503,
+STORE, 140400408891392, 140400434069503,
+SNULL, 140400408891392, 140400417284095,
+STORE, 140400417284096, 140400434069503,
+STORE, 140400408891392, 140400417284095,
+SNULL, 140400417288191, 140400434069503,
+STORE, 140400417284096, 140400417288191,
+STORE, 140400417288192, 140400434069503,
+STORE, 140400283066368, 140400291459071,
+SNULL, 140400601829376, 140400618610687,
+STORE, 140400618610688, 140400627003391,
+STORE, 140400601829376, 140400618610687,
+SNULL, 140400618614783, 140400627003391,
+STORE, 140400618610688, 140400618614783,
+STORE, 140400618614784, 140400627003391,
+SNULL, 140400601829376, 140400610217983,
+STORE, 140400610217984, 140400618610687,
+STORE, 140400601829376, 140400610217983,
+SNULL, 140400610222079, 140400618610687,
+STORE, 140400610217984, 140400610222079,
+STORE, 140400610222080, 140400618610687,
+STORE, 140400274673664, 140400291459071,
+STORE, 140400190812160, 140400199204863,
+STORE, 140400182419456, 140400199204863,
+SNULL, 140400442466304, 140400450854911,
+STORE, 140400450854912, 140400467640319,
+STORE, 140400442466304, 140400450854911,
+SNULL, 140400450859007, 140400467640319,
+STORE, 140400450854912, 140400450859007,
+STORE, 140400450859008, 140400467640319,
+SNULL, 140400543109120, 140400559894527,
+STORE, 140400559894528, 140400568287231,
+STORE, 140400543109120, 140400559894527,
+SNULL, 140400559898623, 140400568287231,
+STORE, 140400559894528, 140400559898623,
+STORE, 140400559898624, 140400568287231,
+SNULL, 140400450859008, 140400459247615,
+STORE, 140400459247616, 140400467640319,
+STORE, 140400450859008, 140400459247615,
+SNULL, 140400459251711, 140400467640319,
+STORE, 140400459247616, 140400459251711,
+STORE, 140400459251712, 140400467640319,
+SNULL, 140400543113215, 140400559894527,
+STORE, 140400543109120, 140400543113215,
+STORE, 140400543113216, 140400559894527,
+SNULL, 140400970944512, 140400979333119,
+STORE, 140400979333120, 140400987725823,
+STORE, 140400970944512, 140400979333119,
+SNULL, 140400979337215, 140400987725823,
+STORE, 140400979333120, 140400979337215,
+STORE, 140400979337216, 140400987725823,
+STORE, 140400174026752, 140400199204863,
+SNULL, 140400174030847, 140400199204863,
+STORE, 140400174026752, 140400174030847,
+STORE, 140400174030848, 140400199204863,
+SNULL, 140400274673664, 140400283066367,
+STORE, 140400283066368, 140400291459071,
+STORE, 140400274673664, 140400283066367,
+SNULL, 140400283070463, 140400291459071,
+STORE, 140400283066368, 140400283070463,
+STORE, 140400283070464, 140400291459071,
+STORE, 140400165634048, 140400174026751,
+SNULL, 140400165638143, 140400174026751,
+STORE, 140400165634048, 140400165638143,
+STORE, 140400165638144, 140400174026751,
+SNULL, 140400174030848, 140400182419455,
+STORE, 140400182419456, 140400199204863,
+STORE, 140400174030848, 140400182419455,
+SNULL, 140400182423551, 140400199204863,
+STORE, 140400182419456, 140400182423551,
+STORE, 140400182423552, 140400199204863,
+SNULL, 140400182423552, 140400190812159,
+STORE, 140400190812160, 140400199204863,
+STORE, 140400182423552, 140400190812159,
+SNULL, 140400190816255, 140400199204863,
+STORE, 140400190812160, 140400190816255,
+STORE, 140400190816256, 140400199204863,
+STORE, 140400157241344, 140400165634047,
+SNULL, 140400157245439, 140400165634047,
+STORE, 140400157241344, 140400157245439,
+STORE, 140400157245440, 140400165634047,
+SNULL, 140400408895487, 140400417284095,
+STORE, 140400408891392, 140400408895487,
+STORE, 140400408895488, 140400417284095,
+SNULL, 140400417288192, 140400425676799,
+STORE, 140400425676800, 140400434069503,
+STORE, 140400417288192, 140400425676799,
+SNULL, 140400425680895, 140400434069503,
+STORE, 140400425676800, 140400425680895,
+STORE, 140400425680896, 140400434069503,
+STORE, 140400148848640, 140400157241343,
+SNULL, 140400148852735, 140400157241343,
+STORE, 140400148848640, 140400148852735,
+STORE, 140400148852736, 140400157241343,
+SNULL, 140400543113216, 140400551501823,
+STORE, 140400551501824, 140400559894527,
+STORE, 140400543113216, 140400551501823,
+SNULL, 140400551505919, 140400559894527,
+STORE, 140400551501824, 140400551505919,
+STORE, 140400551505920, 140400559894527,
+STORE, 140400140455936, 140400148848639,
+STORE, 140400048201728, 140400056594431,
+SNULL, 140400140460031, 140400148848639,
+STORE, 140400140455936, 140400140460031,
+STORE, 140400140460032, 140400148848639,
+STORE, 140400039809024, 140400056594431,
+SNULL, 140400039813119, 140400056594431,
+STORE, 140400039809024, 140400039813119,
+STORE, 140400039813120, 140400056594431,
+STORE, 140400031416320, 140400039809023,
+STORE, 140400023023616, 140400039809023,
+SNULL, 140400274677759, 140400283066367,
+STORE, 140400274673664, 140400274677759,
+STORE, 140400274677760, 140400283066367,
+STORE, 140400014630912, 140400039809023,
+STORE, 140400006238208, 140400039809023,
+STORE, 140399997845504, 140400039809023,
+SNULL, 140399997849599, 140400039809023,
+STORE, 140399997845504, 140399997849599,
+STORE, 140399997849600, 140400039809023,
+STORE, 140399989452800, 140399997845503,
+SNULL, 140399989456895, 140399997845503,
+STORE, 140399989452800, 140399989456895,
+STORE, 140399989456896, 140399997845503,
+STORE, 140399981060096, 140399989452799,
+SNULL, 140399981064191, 140399989452799,
+STORE, 140399981060096, 140399981064191,
+STORE, 140399981064192, 140399989452799,
+STORE, 140399972667392, 140399981060095,
+STORE, 140399964274688, 140399981060095,
+SNULL, 140399964278783, 140399981060095,
+STORE, 140399964274688, 140399964278783,
+STORE, 140399964278784, 140399981060095,
+SNULL, 140400039813120, 140400048201727,
+STORE, 140400048201728, 140400056594431,
+STORE, 140400039813120, 140400048201727,
+SNULL, 140400048205823, 140400056594431,
+STORE, 140400048201728, 140400048205823,
+STORE, 140400048205824, 140400056594431,
+SNULL, 140399997849600, 140400031416319,
+STORE, 140400031416320, 140400039809023,
+STORE, 140399997849600, 140400031416319,
+SNULL, 140400031420415, 140400039809023,
+STORE, 140400031416320, 140400031420415,
+STORE, 140400031420416, 140400039809023,
+STORE, 140399955881984, 140399964274687,
+SNULL, 140399955886079, 140399964274687,
+STORE, 140399955881984, 140399955886079,
+STORE, 140399955886080, 140399964274687,
+STORE, 140399947489280, 140399955881983,
+STORE, 140399939096576, 140399955881983,
+STORE, 140399855267840, 140399863660543,
+SNULL, 140399939100671, 140399955881983,
+STORE, 140399939096576, 140399939100671,
+STORE, 140399939100672, 140399955881983,
+SNULL, 140399997849600, 140400014630911,
+STORE, 140400014630912, 140400031416319,
+STORE, 140399997849600, 140400014630911,
+SNULL, 140400014635007, 140400031416319,
+STORE, 140400014630912, 140400014635007,
+STORE, 140400014635008, 140400031416319,
+SNULL, 140400014635008, 140400023023615,
+STORE, 140400023023616, 140400031416319,
+STORE, 140400014635008, 140400023023615,
+SNULL, 140400023027711, 140400031416319,
+STORE, 140400023023616, 140400023027711,
+STORE, 140400023027712, 140400031416319,
+SNULL, 140399997849600, 140400006238207,
+STORE, 140400006238208, 140400014630911,
+STORE, 140399997849600, 140400006238207,
+SNULL, 140400006242303, 140400014630911,
+STORE, 140400006238208, 140400006242303,
+STORE, 140400006242304, 140400014630911,
+STORE, 140399846875136, 140399863660543,
+STORE, 140399838482432, 140399863660543,
+SNULL, 140399838486527, 140399863660543,
+STORE, 140399838482432, 140399838486527,
+STORE, 140399838486528, 140399863660543,
+SNULL, 140399939100672, 140399947489279,
+STORE, 140399947489280, 140399955881983,
+STORE, 140399939100672, 140399947489279,
+SNULL, 140399947493375, 140399955881983,
+STORE, 140399947489280, 140399947493375,
+STORE, 140399947493376, 140399955881983,
+SNULL, 140399964278784, 140399972667391,
+STORE, 140399972667392, 140399981060095,
+STORE, 140399964278784, 140399972667391,
+SNULL, 140399972671487, 140399981060095,
+STORE, 140399972667392, 140399972671487,
+STORE, 140399972671488, 140399981060095,
+SNULL, 140399838486528, 140399855267839,
+STORE, 140399855267840, 140399863660543,
+STORE, 140399838486528, 140399855267839,
+SNULL, 140399855271935, 140399863660543,
+STORE, 140399855267840, 140399855271935,
+STORE, 140399855271936, 140399863660543,
+STORE, 140399830089728, 140399838482431,
+SNULL, 140399830093823, 140399838482431,
+STORE, 140399830089728, 140399830093823,
+STORE, 140399830093824, 140399838482431,
+STORE, 140399821697024, 140399830089727,
+SNULL, 140399821701119, 140399830089727,
+STORE, 140399821697024, 140399821701119,
+STORE, 140399821701120, 140399830089727,
+SNULL, 140399838486528, 140399846875135,
+STORE, 140399846875136, 140399855267839,
+STORE, 140399838486528, 140399846875135,
+SNULL, 140399846879231, 140399855267839,
+STORE, 140399846875136, 140399846879231,
+STORE, 140399846879232, 140399855267839,
+STORE, 140399813304320, 140399821697023,
+STORE, 140399804911616, 140399821697023,
+SNULL, 140399804915711, 140399821697023,
+STORE, 140399804911616, 140399804915711,
+STORE, 140399804915712, 140399821697023,
+STORE, 140399721050112, 140399729442815,
+SNULL, 140399804915712, 140399813304319,
+STORE, 140399813304320, 140399821697023,
+STORE, 140399804915712, 140399813304319,
+SNULL, 140399813308415, 140399821697023,
+STORE, 140399813304320, 140399813308415,
+STORE, 140399813308416, 140399821697023,
+SNULL, 140399721054207, 140399729442815,
+STORE, 140399721050112, 140399721054207,
+STORE, 140399721054208, 140399729442815,
+STORE, 140401467105280, 140401467133951,
+STORE, 140401279115264, 140401281306623,
+SNULL, 140401279115264, 140401279205375,
+STORE, 140401279205376, 140401281306623,
+STORE, 140401279115264, 140401279205375,
+SNULL, 140401281298431, 140401281306623,
+STORE, 140401279205376, 140401281298431,
+STORE, 140401281298432, 140401281306623,
+ERASE, 140401281298432, 140401281306623,
+STORE, 140401281298432, 140401281306623,
+SNULL, 140401281302527, 140401281306623,
+STORE, 140401281298432, 140401281302527,
+STORE, 140401281302528, 140401281306623,
+ERASE, 140401467105280, 140401467133951,
+ERASE, 140400056594432, 140400056598527,
+ERASE, 140400056598528, 140400064987135,
+ERASE, 140400635396096, 140400635400191,
+ERASE, 140400635400192, 140400643788799,
+ERASE, 140400408891392, 140400408895487,
+ERASE, 140400408895488, 140400417284095,
+ERASE, 140400299851776, 140400299855871,
+ERASE, 140400299855872, 140400308244479,
+ERASE, 140400627003392, 140400627007487,
+ERASE, 140400627007488, 140400635396095,
+ERASE, 140400954155008, 140400954159103,
+ERASE, 140400954159104, 140400962547711,
+ERASE, 140400291459072, 140400291463167,
+ERASE, 140400291463168, 140400299851775,
+ERASE, 140400643788800, 140400643792895,
+ERASE, 140400643792896, 140400652181503,
+ERASE, 140400325029888, 140400325033983,
+ERASE, 140400325033984, 140400333422591,
+ERASE, 140400610217984, 140400610222079,
+ERASE, 140400610222080, 140400618610687,
+ERASE, 140400190812160, 140400190816255,
+ERASE, 140400190816256, 140400199204863,
+ERASE, 140399964274688, 140399964278783,
+ERASE, 140399964278784, 140399972667391,
+ERASE, 140400945762304, 140400945766399,
+ERASE, 140400945766400, 140400954155007,
+ERASE, 140400568287232, 140400568291327,
+ERASE, 140400568291328, 140400576679935,
+ERASE, 140399972667392, 140399972671487,
+ERASE, 140399972671488, 140399981060095,
+ERASE, 140400962547712, 140400962551807,
+ERASE, 140400962551808, 140400970940415,
+ERASE, 140400987725824, 140400987729919,
+ERASE, 140400987729920, 140400996118527,
+ERASE, 140400652181504, 140400652185599,
+ERASE, 140400652185600, 140400660574207,
+ERASE, 140400450854912, 140400450859007,
+ERASE, 140400450859008, 140400459247615,
+ERASE, 140400031416320, 140400031420415,
+ERASE, 140400031420416, 140400039809023,
+ERASE, 140400308244480, 140400308248575,
+ERASE, 140400308248576, 140400316637183,
+ERASE, 140400434069504, 140400434073599,
+ERASE, 140400434073600, 140400442462207,
+ERASE, 140400543109120, 140400543113215,
+ERASE, 140400543113216, 140400551501823,
+ERASE, 140400023023616, 140400023027711,
+ERASE, 140400023027712, 140400031416319,
+ERASE, 140399813304320, 140399813308415,
+ERASE, 140399813308416, 140399821697023,
+ERASE, 140400316637184, 140400316641279,
+ERASE, 140400316641280, 140400325029887,
+ERASE, 140400585072640, 140400585076735,
+ERASE, 140400585076736, 140400593465343,
+ERASE, 140400148848640, 140400148852735,
+ERASE, 140400148852736, 140400157241343,
+ERASE, 140399955881984, 140399955886079,
+ERASE, 140399955886080, 140399964274687,
+ERASE, 140399821697024, 140399821701119,
+ERASE, 140399821701120, 140399830089727,
+ERASE, 140400601825280, 140400601829375,
+ERASE, 140400601829376, 140400610217983,
+ERASE, 140400979333120, 140400979337215,
+ERASE, 140400979337216, 140400987725823,
+ERASE, 140399997845504, 140399997849599,
+ERASE, 140399997849600, 140400006238207,
+ERASE, 140400459247616, 140400459251711,
+ERASE, 140400459251712, 140400467640319,
+ERASE, 140400551501824, 140400551505919,
+ERASE, 140400551505920, 140400559894527,
+ERASE, 140399939096576, 140399939100671,
+ERASE, 140399939100672, 140399947489279,
+ERASE, 140400442462208, 140400442466303,
+ERASE, 140400442466304, 140400450854911,
+ERASE, 140400576679936, 140400576684031,
+ERASE, 140400576684032, 140400585072639,
+ERASE, 140400559894528, 140400559898623,
+ERASE, 140400559898624, 140400568287231,
+ERASE, 140400417284096, 140400417288191,
+ERASE, 140400417288192, 140400425676799,
+ERASE, 140400283066368, 140400283070463,
+ERASE, 140400283070464, 140400291459071,
+ };
+ unsigned long set33[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140734562918400, 140737488351231,
+SNULL, 140734562922495, 140737488351231,
+STORE, 140734562918400, 140734562922495,
+STORE, 140734562787328, 140734562922495,
+STORE, 94133878984704, 94133881237503,
+SNULL, 94133879115775, 94133881237503,
+STORE, 94133878984704, 94133879115775,
+STORE, 94133879115776, 94133881237503,
+ERASE, 94133879115776, 94133881237503,
+STORE, 94133881208832, 94133881217023,
+STORE, 94133881217024, 94133881237503,
+STORE, 140583654043648, 140583656296447,
+SNULL, 140583654187007, 140583656296447,
+STORE, 140583654043648, 140583654187007,
+STORE, 140583654187008, 140583656296447,
+ERASE, 140583654187008, 140583656296447,
+STORE, 140583656284160, 140583656292351,
+STORE, 140583656292352, 140583656296447,
+STORE, 140734564319232, 140734564323327,
+STORE, 140734564306944, 140734564319231,
+STORE, 140583656255488, 140583656284159,
+STORE, 140583656247296, 140583656255487,
+STORE, 140583651827712, 140583654043647,
+SNULL, 140583651827712, 140583651926015,
+STORE, 140583651926016, 140583654043647,
+STORE, 140583651827712, 140583651926015,
+SNULL, 140583654019071, 140583654043647,
+STORE, 140583651926016, 140583654019071,
+STORE, 140583654019072, 140583654043647,
+SNULL, 140583654019072, 140583654027263,
+STORE, 140583654027264, 140583654043647,
+STORE, 140583654019072, 140583654027263,
+ERASE, 140583654019072, 140583654027263,
+STORE, 140583654019072, 140583654027263,
+ERASE, 140583654027264, 140583654043647,
+STORE, 140583654027264, 140583654043647,
+STORE, 140583648030720, 140583651827711,
+SNULL, 140583648030720, 140583649689599,
+STORE, 140583649689600, 140583651827711,
+STORE, 140583648030720, 140583649689599,
+SNULL, 140583651786751, 140583651827711,
+STORE, 140583649689600, 140583651786751,
+STORE, 140583651786752, 140583651827711,
+SNULL, 140583651786752, 140583651811327,
+STORE, 140583651811328, 140583651827711,
+STORE, 140583651786752, 140583651811327,
+ERASE, 140583651786752, 140583651811327,
+STORE, 140583651786752, 140583651811327,
+ERASE, 140583651811328, 140583651827711,
+STORE, 140583651811328, 140583651827711,
+STORE, 140583656239104, 140583656255487,
+SNULL, 140583651803135, 140583651811327,
+STORE, 140583651786752, 140583651803135,
+STORE, 140583651803136, 140583651811327,
+SNULL, 140583654023167, 140583654027263,
+STORE, 140583654019072, 140583654023167,
+STORE, 140583654023168, 140583654027263,
+SNULL, 94133881212927, 94133881217023,
+STORE, 94133881208832, 94133881212927,
+STORE, 94133881212928, 94133881217023,
+SNULL, 140583656288255, 140583656292351,
+STORE, 140583656284160, 140583656288255,
+STORE, 140583656288256, 140583656292351,
+ERASE, 140583656255488, 140583656284159,
+STORE, 94133881733120, 94133881868287,
+STORE, 140583639638016, 140583648030719,
+SNULL, 140583639642111, 140583648030719,
+STORE, 140583639638016, 140583639642111,
+STORE, 140583639642112, 140583648030719,
+STORE, 140583631245312, 140583639638015,
+STORE, 140583497027584, 140583631245311,
+SNULL, 140583497027584, 140583540621311,
+STORE, 140583540621312, 140583631245311,
+STORE, 140583497027584, 140583540621311,
+ERASE, 140583497027584, 140583540621311,
+SNULL, 140583607730175, 140583631245311,
+STORE, 140583540621312, 140583607730175,
+STORE, 140583607730176, 140583631245311,
+ERASE, 140583607730176, 140583631245311,
+SNULL, 140583540756479, 140583607730175,
+STORE, 140583540621312, 140583540756479,
+STORE, 140583540756480, 140583607730175,
+SNULL, 140583631249407, 140583639638015,
+STORE, 140583631245312, 140583631249407,
+STORE, 140583631249408, 140583639638015,
+STORE, 140583622852608, 140583631245311,
+SNULL, 140583622856703, 140583631245311,
+STORE, 140583622852608, 140583622856703,
+STORE, 140583622856704, 140583631245311,
+STORE, 140583614459904, 140583622852607,
+SNULL, 140583614463999, 140583622852607,
+STORE, 140583614459904, 140583614463999,
+STORE, 140583614464000, 140583622852607,
+STORE, 140583532228608, 140583540621311,
+SNULL, 140583532232703, 140583540621311,
+STORE, 140583532228608, 140583532232703,
+STORE, 140583532232704, 140583540621311,
+STORE, 140583523835904, 140583532228607,
+STORE, 140583515443200, 140583532228607,
+STORE, 140583507050496, 140583532228607,
+STORE, 140583372832768, 140583507050495,
+STORE, 140583364440064, 140583372832767,
+STORE, 140583230222336, 140583364440063,
+STORE, 140583096004608, 140583364440063,
+SNULL, 140583230222335, 140583364440063,
+STORE, 140583096004608, 140583230222335,
+STORE, 140583230222336, 140583364440063,
+SNULL, 140583230222336, 140583272185855,
+STORE, 140583272185856, 140583364440063,
+STORE, 140583230222336, 140583272185855,
+ERASE, 140583230222336, 140583272185855,
+STORE, 140582961786880, 140583230222335,
+SNULL, 140583372832768, 140583406403583,
+STORE, 140583406403584, 140583507050495,
+STORE, 140583372832768, 140583406403583,
+ERASE, 140583372832768, 140583406403583,
+SNULL, 140583473512447, 140583507050495,
+STORE, 140583406403584, 140583473512447,
+STORE, 140583473512448, 140583507050495,
+ERASE, 140583473512448, 140583507050495,
+SNULL, 140583096004607, 140583230222335,
+STORE, 140582961786880, 140583096004607,
+STORE, 140583096004608, 140583230222335,
+SNULL, 140583096004608, 140583137968127,
+STORE, 140583137968128, 140583230222335,
+STORE, 140583096004608, 140583137968127,
+ERASE, 140583096004608, 140583137968127,
+SNULL, 140583339294719, 140583364440063,
+STORE, 140583272185856, 140583339294719,
+STORE, 140583339294720, 140583364440063,
+ERASE, 140583339294720, 140583364440063,
+SNULL, 140583272321023, 140583339294719,
+STORE, 140583272185856, 140583272321023,
+STORE, 140583272321024, 140583339294719,
+SNULL, 140582961786880, 140583003750399,
+STORE, 140583003750400, 140583096004607,
+STORE, 140582961786880, 140583003750399,
+ERASE, 140582961786880, 140583003750399,
+ };
+
+ unsigned long set34[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140731327180800, 140737488351231,
+SNULL, 140731327184895, 140737488351231,
+STORE, 140731327180800, 140731327184895,
+STORE, 140731327049728, 140731327184895,
+STORE, 94632924487680, 94632926740479,
+SNULL, 94632924618751, 94632926740479,
+STORE, 94632924487680, 94632924618751,
+STORE, 94632924618752, 94632926740479,
+ERASE, 94632924618752, 94632926740479,
+STORE, 94632926711808, 94632926719999,
+STORE, 94632926720000, 94632926740479,
+STORE, 140012544888832, 140012547141631,
+SNULL, 140012545032191, 140012547141631,
+STORE, 140012544888832, 140012545032191,
+STORE, 140012545032192, 140012547141631,
+ERASE, 140012545032192, 140012547141631,
+STORE, 140012547129344, 140012547137535,
+STORE, 140012547137536, 140012547141631,
+STORE, 140731327725568, 140731327729663,
+STORE, 140731327713280, 140731327725567,
+STORE, 140012547100672, 140012547129343,
+STORE, 140012547092480, 140012547100671,
+STORE, 140012542672896, 140012544888831,
+SNULL, 140012542672896, 140012542771199,
+STORE, 140012542771200, 140012544888831,
+STORE, 140012542672896, 140012542771199,
+SNULL, 140012544864255, 140012544888831,
+STORE, 140012542771200, 140012544864255,
+STORE, 140012544864256, 140012544888831,
+SNULL, 140012544864256, 140012544872447,
+STORE, 140012544872448, 140012544888831,
+STORE, 140012544864256, 140012544872447,
+ERASE, 140012544864256, 140012544872447,
+STORE, 140012544864256, 140012544872447,
+ERASE, 140012544872448, 140012544888831,
+STORE, 140012544872448, 140012544888831,
+STORE, 140012538875904, 140012542672895,
+SNULL, 140012538875904, 140012540534783,
+STORE, 140012540534784, 140012542672895,
+STORE, 140012538875904, 140012540534783,
+SNULL, 140012542631935, 140012542672895,
+STORE, 140012540534784, 140012542631935,
+STORE, 140012542631936, 140012542672895,
+SNULL, 140012542631936, 140012542656511,
+STORE, 140012542656512, 140012542672895,
+STORE, 140012542631936, 140012542656511,
+ERASE, 140012542631936, 140012542656511,
+STORE, 140012542631936, 140012542656511,
+ERASE, 140012542656512, 140012542672895,
+STORE, 140012542656512, 140012542672895,
+STORE, 140012547084288, 140012547100671,
+SNULL, 140012542648319, 140012542656511,
+STORE, 140012542631936, 140012542648319,
+STORE, 140012542648320, 140012542656511,
+SNULL, 140012544868351, 140012544872447,
+STORE, 140012544864256, 140012544868351,
+STORE, 140012544868352, 140012544872447,
+SNULL, 94632926715903, 94632926719999,
+STORE, 94632926711808, 94632926715903,
+STORE, 94632926715904, 94632926719999,
+SNULL, 140012547133439, 140012547137535,
+STORE, 140012547129344, 140012547133439,
+STORE, 140012547133440, 140012547137535,
+ERASE, 140012547100672, 140012547129343,
+STORE, 94632939606016, 94632939741183,
+STORE, 140012530483200, 140012538875903,
+SNULL, 140012530487295, 140012538875903,
+STORE, 140012530483200, 140012530487295,
+STORE, 140012530487296, 140012538875903,
+STORE, 140012522090496, 140012530483199,
+STORE, 140012387872768, 140012522090495,
+SNULL, 140012387872768, 140012444188671,
+STORE, 140012444188672, 140012522090495,
+STORE, 140012387872768, 140012444188671,
+ERASE, 140012387872768, 140012444188671,
+SNULL, 140012511297535, 140012522090495,
+STORE, 140012444188672, 140012511297535,
+STORE, 140012511297536, 140012522090495,
+ERASE, 140012511297536, 140012522090495,
+SNULL, 140012444323839, 140012511297535,
+STORE, 140012444188672, 140012444323839,
+STORE, 140012444323840, 140012511297535,
+SNULL, 140012522094591, 140012530483199,
+STORE, 140012522090496, 140012522094591,
+STORE, 140012522094592, 140012530483199,
+STORE, 140012513697792, 140012522090495,
+SNULL, 140012513701887, 140012522090495,
+STORE, 140012513697792, 140012513701887,
+STORE, 140012513701888, 140012522090495,
+STORE, 140012435795968, 140012444188671,
+SNULL, 140012435800063, 140012444188671,
+STORE, 140012435795968, 140012435800063,
+STORE, 140012435800064, 140012444188671,
+STORE, 140012427403264, 140012435795967,
+SNULL, 140012427407359, 140012435795967,
+STORE, 140012427403264, 140012427407359,
+STORE, 140012427407360, 140012435795967,
+STORE, 140012419010560, 140012427403263,
+STORE, 140012410617856, 140012427403263,
+STORE, 140012276400128, 140012410617855,
+STORE, 140012268007424, 140012276400127,
+STORE, 140012133789696, 140012268007423,
+SNULL, 140012133789696, 140012175753215,
+STORE, 140012175753216, 140012268007423,
+STORE, 140012133789696, 140012175753215,
+ERASE, 140012133789696, 140012175753215,
+STORE, 140012041535488, 140012268007423,
+SNULL, 140012108644351, 140012268007423,
+STORE, 140012041535488, 140012108644351,
+STORE, 140012108644352, 140012268007423,
+SNULL, 140012108644352, 140012175753215,
+STORE, 140012175753216, 140012268007423,
+STORE, 140012108644352, 140012175753215,
+ERASE, 140012108644352, 140012175753215,
+SNULL, 140012276400128, 140012309970943,
+STORE, 140012309970944, 140012410617855,
+STORE, 140012276400128, 140012309970943,
+ERASE, 140012276400128, 140012309970943,
+STORE, 140012301578240, 140012309970943,
+STORE, 140012041535488, 140012268007423,
+SNULL, 140012242862079, 140012268007423,
+STORE, 140012041535488, 140012242862079,
+STORE, 140012242862080, 140012268007423,
+ERASE, 140012242862080, 140012268007423,
+SNULL, 140012041670655, 140012242862079,
+STORE, 140012041535488, 140012041670655,
+STORE, 140012041670656, 140012242862079,
+SNULL, 140012041670656, 140012108644351,
+STORE, 140012108644352, 140012242862079,
+STORE, 140012041670656, 140012108644351,
+SNULL, 140012108779519, 140012242862079,
+STORE, 140012108644352, 140012108779519,
+STORE, 140012108779520, 140012242862079,
+SNULL, 140012377079807, 140012410617855,
+STORE, 140012309970944, 140012377079807,
+STORE, 140012377079808, 140012410617855,
+ERASE, 140012377079808, 140012410617855,
+SNULL, 140012310106111, 140012377079807,
+STORE, 140012309970944, 140012310106111,
+STORE, 140012310106112, 140012377079807,
+SNULL, 140012410621951, 140012427403263,
+STORE, 140012410617856, 140012410621951,
+STORE, 140012410621952, 140012427403263,
+SNULL, 140012108779520, 140012175753215,
+STORE, 140012175753216, 140012242862079,
+STORE, 140012108779520, 140012175753215,
+SNULL, 140012175888383, 140012242862079,
+STORE, 140012175753216, 140012175888383,
+STORE, 140012175888384, 140012242862079,
+SNULL, 140012301582335, 140012309970943,
+STORE, 140012301578240, 140012301582335,
+STORE, 140012301582336, 140012309970943,
+SNULL, 140012410621952, 140012419010559,
+STORE, 140012419010560, 140012427403263,
+STORE, 140012410621952, 140012419010559,
+SNULL, 140012419014655, 140012427403263,
+STORE, 140012419010560, 140012419014655,
+STORE, 140012419014656, 140012427403263,
+SNULL, 140012268011519, 140012276400127,
+STORE, 140012268007424, 140012268011519,
+STORE, 140012268011520, 140012276400127,
+STORE, 140012402225152, 140012410617855,
+STORE, 140012393832448, 140012410617855,
+SNULL, 140012393832448, 140012402225151,
+STORE, 140012402225152, 140012410617855,
+STORE, 140012393832448, 140012402225151,
+SNULL, 140012402229247, 140012410617855,
+STORE, 140012402225152, 140012402229247,
+STORE, 140012402229248, 140012410617855,
+STORE, 140012385439744, 140012402225151,
+SNULL, 140012385439744, 140012393832447,
+STORE, 140012393832448, 140012402225151,
+STORE, 140012385439744, 140012393832447,
+SNULL, 140012393836543, 140012402225151,
+STORE, 140012393832448, 140012393836543,
+STORE, 140012393836544, 140012402225151,
+STORE, 140012293185536, 140012301578239,
+STORE, 140012284792832, 140012301578239,
+SNULL, 140012284792832, 140012293185535,
+STORE, 140012293185536, 140012301578239,
+STORE, 140012284792832, 140012293185535,
+SNULL, 140012293189631, 140012301578239,
+STORE, 140012293185536, 140012293189631,
+STORE, 140012293189632, 140012301578239,
+STORE, 140012268011520, 140012284792831,
+SNULL, 140012385443839, 140012393832447,
+STORE, 140012385439744, 140012385443839,
+STORE, 140012385443840, 140012393832447,
+STORE, 140012259614720, 140012268007423,
+SNULL, 140012259618815, 140012268007423,
+STORE, 140012259614720, 140012259618815,
+STORE, 140012259618816, 140012268007423,
+STORE, 140012251222016, 140012259614719,
+SNULL, 140012251226111, 140012259614719,
+STORE, 140012251222016, 140012251226111,
+STORE, 140012251226112, 140012259614719,
+SNULL, 140012284796927, 140012293185535,
+STORE, 140012284792832, 140012284796927,
+STORE, 140012284796928, 140012293185535,
+SNULL, 140012268011520, 140012276400127,
+STORE, 140012276400128, 140012284792831,
+STORE, 140012268011520, 140012276400127,
+SNULL, 140012276404223, 140012284792831,
+STORE, 140012276400128, 140012276404223,
+STORE, 140012276404224, 140012284792831,
+STORE, 140012033142784, 140012041535487,
+SNULL, 140012033146879, 140012041535487,
+STORE, 140012033142784, 140012033146879,
+STORE, 140012033146880, 140012041535487,
+STORE, 140012024750080, 140012033142783,
+STORE, 140012016357376, 140012033142783,
+SNULL, 140012016357376, 140012024750079,
+STORE, 140012024750080, 140012033142783,
+STORE, 140012016357376, 140012024750079,
+SNULL, 140012024754175, 140012033142783,
+STORE, 140012024750080, 140012024754175,
+STORE, 140012024754176, 140012033142783,
+SNULL, 140012016361471, 140012024750079,
+STORE, 140012016357376, 140012016361471,
+STORE, 140012016361472, 140012024750079,
+STORE, 140012007964672, 140012016357375,
+SNULL, 140012007968767, 140012016357375,
+STORE, 140012007964672, 140012007968767,
+STORE, 140012007968768, 140012016357375,
+STORE, 140011999571968, 140012007964671,
+STORE, 140011991179264, 140012007964671,
+STORE, 140011856961536, 140011991179263,
+STORE, 140011848568832, 140011856961535,
+STORE, 140011714351104, 140011848568831,
+SNULL, 140011714351104, 140011773100031,
+STORE, 140011773100032, 140011848568831,
+STORE, 140011714351104, 140011773100031,
+ERASE, 140011714351104, 140011773100031,
+STORE, 140011764707328, 140011773100031,
+STORE, 140011756314624, 140011773100031,
+STORE, 140011622096896, 140011756314623,
+STORE, 140011613704192, 140011622096895,
+STORE, 140011479486464, 140011613704191,
+STORE, 140011471093760, 140011479486463,
+SNULL, 140011479486464, 140011504664575,
+STORE, 140011504664576, 140011613704191,
+STORE, 140011479486464, 140011504664575,
+ERASE, 140011479486464, 140011504664575,
+STORE, 140011496271872, 140011504664575,
+STORE, 140011487879168, 140011504664575,
+STORE, 140011336876032, 140011471093759,
+SNULL, 140011336876032, 140011370446847,
+STORE, 140011370446848, 140011471093759,
+STORE, 140011336876032, 140011370446847,
+ERASE, 140011336876032, 140011370446847,
+STORE, 140011471093760, 140011487879167,
+STORE, 140011362054144, 140011370446847,
+SNULL, 140011362058239, 140011370446847,
+STORE, 140011362054144, 140011362058239,
+STORE, 140011362058240, 140011370446847,
+STORE, 140011353661440, 140011362054143,
+STORE, 140011345268736, 140011362054143,
+SNULL, 140011345272831, 140011362054143,
+STORE, 140011345268736, 140011345272831,
+STORE, 140011345272832, 140011362054143,
+STORE, 140011336876032, 140011345268735,
+STORE, 140011328483328, 140011345268735,
+SNULL, 140011328487423, 140011345268735,
+STORE, 140011328483328, 140011328487423,
+STORE, 140011328487424, 140011345268735,
+STORE, 140011320090624, 140011328483327,
+STORE, 140011185872896, 140011320090623,
+SNULL, 140011185872896, 140011236229119,
+STORE, 140011236229120, 140011320090623,
+STORE, 140011185872896, 140011236229119,
+ERASE, 140011185872896, 140011236229119,
+SNULL, 140011856961536, 140011907317759,
+STORE, 140011907317760, 140011991179263,
+STORE, 140011856961536, 140011907317759,
+ERASE, 140011856961536, 140011907317759,
+SNULL, 140011974426623, 140011991179263,
+STORE, 140011907317760, 140011974426623,
+STORE, 140011974426624, 140011991179263,
+ERASE, 140011974426624, 140011991179263,
+SNULL, 140011840208895, 140011848568831,
+STORE, 140011773100032, 140011840208895,
+STORE, 140011840208896, 140011848568831,
+ERASE, 140011840208896, 140011848568831,
+SNULL, 140011773235199, 140011840208895,
+STORE, 140011773100032, 140011773235199,
+STORE, 140011773235200, 140011840208895,
+STORE, 140011102011392, 140011320090623,
+SNULL, 140011169120255, 140011320090623,
+STORE, 140011102011392, 140011169120255,
+STORE, 140011169120256, 140011320090623,
+SNULL, 140011169120256, 140011236229119,
+STORE, 140011236229120, 140011320090623,
+STORE, 140011169120256, 140011236229119,
+ERASE, 140011169120256, 140011236229119,
+SNULL, 140011622096896, 140011638882303,
+STORE, 140011638882304, 140011756314623,
+STORE, 140011622096896, 140011638882303,
+ERASE, 140011622096896, 140011638882303,
+SNULL, 140011705991167, 140011756314623,
+STORE, 140011638882304, 140011705991167,
+STORE, 140011705991168, 140011756314623,
+ERASE, 140011705991168, 140011756314623,
+SNULL, 140011571773439, 140011613704191,
+STORE, 140011504664576, 140011571773439,
+STORE, 140011571773440, 140011613704191,
+ERASE, 140011571773440, 140011613704191,
+STORE, 140010967793664, 140011169120255,
+SNULL, 140011034902527, 140011169120255,
+STORE, 140010967793664, 140011034902527,
+STORE, 140011034902528, 140011169120255,
+SNULL, 140011034902528, 140011102011391,
+STORE, 140011102011392, 140011169120255,
+STORE, 140011034902528, 140011102011391,
+ERASE, 140011034902528, 140011102011391,
+STORE, 140010833575936, 140011034902527,
+SNULL, 140011437555711, 140011471093759,
+STORE, 140011370446848, 140011437555711,
+STORE, 140011437555712, 140011471093759,
+ERASE, 140011437555712, 140011471093759,
+SNULL, 140011370582015, 140011437555711,
+STORE, 140011370446848, 140011370582015,
+STORE, 140011370582016, 140011437555711,
+STORE, 140010699358208, 140011034902527,
+SNULL, 140011487883263, 140011504664575,
+STORE, 140011487879168, 140011487883263,
+STORE, 140011487883264, 140011504664575,
+SNULL, 140011345272832, 140011353661439,
+STORE, 140011353661440, 140011362054143,
+STORE, 140011345272832, 140011353661439,
+SNULL, 140011353665535, 140011362054143,
+STORE, 140011353661440, 140011353665535,
+STORE, 140011353665536, 140011362054143,
+SNULL, 140011328487424, 140011336876031,
+STORE, 140011336876032, 140011345268735,
+STORE, 140011328487424, 140011336876031,
+SNULL, 140011336880127, 140011345268735,
+STORE, 140011336876032, 140011336880127,
+STORE, 140011336880128, 140011345268735,
+SNULL, 140011303337983, 140011320090623,
+STORE, 140011236229120, 140011303337983,
+STORE, 140011303337984, 140011320090623,
+ERASE, 140011303337984, 140011320090623,
+SNULL, 140011907452927, 140011974426623,
+STORE, 140011907317760, 140011907452927,
+STORE, 140011907452928, 140011974426623,
+SNULL, 140011102146559, 140011169120255,
+STORE, 140011102011392, 140011102146559,
+STORE, 140011102146560, 140011169120255,
+SNULL, 140011639017471, 140011705991167,
+STORE, 140011638882304, 140011639017471,
+STORE, 140011639017472, 140011705991167,
+SNULL, 140011504799743, 140011571773439,
+STORE, 140011504664576, 140011504799743,
+STORE, 140011504799744, 140011571773439,
+SNULL, 140011613708287, 140011622096895,
+STORE, 140011613704192, 140011613708287,
+STORE, 140011613708288, 140011622096895,
+SNULL, 140010699358208, 140010967793663,
+STORE, 140010967793664, 140011034902527,
+STORE, 140010699358208, 140010967793663,
+SNULL, 140010967928831, 140011034902527,
+STORE, 140010967793664, 140010967928831,
+STORE, 140010967928832, 140011034902527,
+SNULL, 140010900684799, 140010967793663,
+STORE, 140010699358208, 140010900684799,
+STORE, 140010900684800, 140010967793663,
+ERASE, 140010900684800, 140010967793663,
+SNULL, 140010766467071, 140010900684799,
+STORE, 140010699358208, 140010766467071,
+STORE, 140010766467072, 140010900684799,
+SNULL, 140010766467072, 140010833575935,
+STORE, 140010833575936, 140010900684799,
+STORE, 140010766467072, 140010833575935,
+ERASE, 140010766467072, 140010833575935,
+SNULL, 140010699493375, 140010766467071,
+STORE, 140010699358208, 140010699493375,
+STORE, 140010699493376, 140010766467071,
+SNULL, 140011848572927, 140011856961535,
+STORE, 140011848568832, 140011848572927,
+STORE, 140011848572928, 140011856961535,
+STORE, 140011982786560, 140012007964671,
+STORE, 140011898925056, 140011907317759,
+SNULL, 140011898929151, 140011907317759,
+STORE, 140011898925056, 140011898929151,
+STORE, 140011898929152, 140011907317759,
+SNULL, 140011320094719, 140011328483327,
+STORE, 140011320090624, 140011320094719,
+STORE, 140011320094720, 140011328483327,
+STORE, 140011890532352, 140011898925055,
+STORE, 140011882139648, 140011898925055,
+SNULL, 140011882143743, 140011898925055,
+STORE, 140011882139648, 140011882143743,
+STORE, 140011882143744, 140011898925055,
+STORE, 140011873746944, 140011882139647,
+SNULL, 140011873751039, 140011882139647,
+STORE, 140011873746944, 140011873751039,
+STORE, 140011873751040, 140011882139647,
+SNULL, 140011236364287, 140011303337983,
+STORE, 140011236229120, 140011236364287,
+STORE, 140011236364288, 140011303337983,
+SNULL, 140011756318719, 140011773100031,
+STORE, 140011756314624, 140011756318719,
+STORE, 140011756318720, 140011773100031,
+SNULL, 140011756318720, 140011764707327,
+STORE, 140011764707328, 140011773100031,
+STORE, 140011756318720, 140011764707327,
+SNULL, 140011764711423, 140011773100031,
+STORE, 140011764707328, 140011764711423,
+STORE, 140011764711424, 140011773100031,
+SNULL, 140011471097855, 140011487879167,
+STORE, 140011471093760, 140011471097855,
+STORE, 140011471097856, 140011487879167,
+SNULL, 140010833711103, 140010900684799,
+STORE, 140010833575936, 140010833711103,
+STORE, 140010833711104, 140010900684799,
+SNULL, 140011982790655, 140012007964671,
+STORE, 140011982786560, 140011982790655,
+STORE, 140011982790656, 140012007964671,
+STORE, 140011865354240, 140011873746943,
+STORE, 140011848572928, 140011865354239,
+SNULL, 140011848572928, 140011856961535,
+STORE, 140011856961536, 140011865354239,
+STORE, 140011848572928, 140011856961535,
+SNULL, 140011856965631, 140011865354239,
+STORE, 140011856961536, 140011856965631,
+STORE, 140011856965632, 140011865354239,
+STORE, 140011747921920, 140011756314623,
+STORE, 140011739529216, 140011756314623,
+SNULL, 140011471097856, 140011479486463,
+STORE, 140011479486464, 140011487879167,
+STORE, 140011471097856, 140011479486463,
+SNULL, 140011479490559, 140011487879167,
+STORE, 140011479486464, 140011479490559,
+STORE, 140011479490560, 140011487879167,
+STORE, 140011731136512, 140011756314623,
+STORE, 140011722743808, 140011756314623,
+SNULL, 140011982790656, 140011999571967,
+STORE, 140011999571968, 140012007964671,
+STORE, 140011982790656, 140011999571967,
+SNULL, 140011999576063, 140012007964671,
+STORE, 140011999571968, 140011999576063,
+STORE, 140011999576064, 140012007964671,
+STORE, 140011714351104, 140011756314623,
+SNULL, 140011882143744, 140011890532351,
+STORE, 140011890532352, 140011898925055,
+STORE, 140011882143744, 140011890532351,
+SNULL, 140011890536447, 140011898925055,
+STORE, 140011890532352, 140011890536447,
+STORE, 140011890536448, 140011898925055,
+STORE, 140011630489600, 140011638882303,
+STORE, 140011613708288, 140011638882303,
+STORE, 140011605311488, 140011613704191,
+STORE, 140011596918784, 140011613704191,
+STORE, 140011588526080, 140011613704191,
+SNULL, 140011487883264, 140011496271871,
+STORE, 140011496271872, 140011504664575,
+STORE, 140011487883264, 140011496271871,
+SNULL, 140011496275967, 140011504664575,
+STORE, 140011496271872, 140011496275967,
+STORE, 140011496275968, 140011504664575,
+STORE, 140011580133376, 140011613704191,
+SNULL, 140011580137471, 140011613704191,
+STORE, 140011580133376, 140011580137471,
+STORE, 140011580137472, 140011613704191,
+SNULL, 140011982790656, 140011991179263,
+STORE, 140011991179264, 140011999571967,
+STORE, 140011982790656, 140011991179263,
+SNULL, 140011991183359, 140011999571967,
+STORE, 140011991179264, 140011991183359,
+STORE, 140011991183360, 140011999571967,
+SNULL, 140011865358335, 140011873746943,
+STORE, 140011865354240, 140011865358335,
+STORE, 140011865358336, 140011873746943,
+STORE, 140011462701056, 140011471093759,
+SNULL, 140011714351104, 140011739529215,
+STORE, 140011739529216, 140011756314623,
+STORE, 140011714351104, 140011739529215,
+SNULL, 140011739533311, 140011756314623,
+STORE, 140011739529216, 140011739533311,
+STORE, 140011739533312, 140011756314623,
+SNULL, 140011739533312, 140011747921919,
+STORE, 140011747921920, 140011756314623,
+STORE, 140011739533312, 140011747921919,
+SNULL, 140011747926015, 140011756314623,
+STORE, 140011747921920, 140011747926015,
+STORE, 140011747926016, 140011756314623,
+SNULL, 140011613708288, 140011630489599,
+STORE, 140011630489600, 140011638882303,
+STORE, 140011613708288, 140011630489599,
+SNULL, 140011630493695, 140011638882303,
+STORE, 140011630489600, 140011630493695,
+STORE, 140011630493696, 140011638882303,
+SNULL, 140011714351104, 140011722743807,
+STORE, 140011722743808, 140011739529215,
+STORE, 140011714351104, 140011722743807,
+SNULL, 140011722747903, 140011739529215,
+STORE, 140011722743808, 140011722747903,
+STORE, 140011722747904, 140011739529215,
+SNULL, 140011714355199, 140011722743807,
+STORE, 140011714351104, 140011714355199,
+STORE, 140011714355200, 140011722743807,
+SNULL, 140011722747904, 140011731136511,
+STORE, 140011731136512, 140011739529215,
+STORE, 140011722747904, 140011731136511,
+SNULL, 140011731140607, 140011739529215,
+STORE, 140011731136512, 140011731140607,
+STORE, 140011731140608, 140011739529215,
+STORE, 140011454308352, 140011471093759,
+STORE, 140011445915648, 140011471093759,
+SNULL, 140011580137472, 140011588526079,
+STORE, 140011588526080, 140011613704191,
+STORE, 140011580137472, 140011588526079,
+SNULL, 140011588530175, 140011613704191,
+STORE, 140011588526080, 140011588530175,
+STORE, 140011588530176, 140011613704191,
+SNULL, 140011445915648, 140011462701055,
+STORE, 140011462701056, 140011471093759,
+STORE, 140011445915648, 140011462701055,
+SNULL, 140011462705151, 140011471093759,
+STORE, 140011462701056, 140011462705151,
+STORE, 140011462705152, 140011471093759,
+SNULL, 140011588530176, 140011596918783,
+STORE, 140011596918784, 140011613704191,
+STORE, 140011588530176, 140011596918783,
+SNULL, 140011596922879, 140011613704191,
+STORE, 140011596918784, 140011596922879,
+STORE, 140011596922880, 140011613704191,
+SNULL, 140011596922880, 140011605311487,
+STORE, 140011605311488, 140011613704191,
+STORE, 140011596922880, 140011605311487,
+SNULL, 140011605315583, 140011613704191,
+STORE, 140011605311488, 140011605315583,
+STORE, 140011605315584, 140011613704191,
+SNULL, 140011613708288, 140011622096895,
+STORE, 140011622096896, 140011630489599,
+STORE, 140011613708288, 140011622096895,
+SNULL, 140011622100991, 140011630489599,
+STORE, 140011622096896, 140011622100991,
+STORE, 140011622100992, 140011630489599,
+STORE, 140011311697920, 140011320090623,
+STORE, 140011227836416, 140011236229119,
+STORE, 140011219443712, 140011236229119,
+SNULL, 140011219447807, 140011236229119,
+STORE, 140011219443712, 140011219447807,
+STORE, 140011219447808, 140011236229119,
+STORE, 140011211051008, 140011219443711,
+STORE, 140011202658304, 140011219443711,
+SNULL, 140011202662399, 140011219443711,
+STORE, 140011202658304, 140011202662399,
+STORE, 140011202662400, 140011219443711,
+STORE, 140011194265600, 140011202658303,
+STORE, 140011185872896, 140011202658303,
+STORE, 140011177480192, 140011202658303,
+STORE, 140011093618688, 140011102011391,
+SNULL, 140011445915648, 140011454308351,
+STORE, 140011454308352, 140011462701055,
+STORE, 140011445915648, 140011454308351,
+SNULL, 140011454312447, 140011462701055,
+STORE, 140011454308352, 140011454312447,
+STORE, 140011454312448, 140011462701055,
+STORE, 140011085225984, 140011102011391,
+SNULL, 140011085230079, 140011102011391,
+STORE, 140011085225984, 140011085230079,
+STORE, 140011085230080, 140011102011391,
+SNULL, 140011177484287, 140011202658303,
+STORE, 140011177480192, 140011177484287,
+STORE, 140011177484288, 140011202658303,
+SNULL, 140011445919743, 140011454308351,
+STORE, 140011445915648, 140011445919743,
+STORE, 140011445919744, 140011454308351,
+SNULL, 140011177484288, 140011185872895,
+STORE, 140011185872896, 140011202658303,
+STORE, 140011177484288, 140011185872895,
+SNULL, 140011185876991, 140011202658303,
+STORE, 140011185872896, 140011185876991,
+STORE, 140011185876992, 140011202658303,
+STORE, 140011076833280, 140011085225983,
+SNULL, 140011202662400, 140011211051007,
+STORE, 140011211051008, 140011219443711,
+STORE, 140011202662400, 140011211051007,
+SNULL, 140011211055103, 140011219443711,
+STORE, 140011211051008, 140011211055103,
+STORE, 140011211055104, 140011219443711,
+SNULL, 140011185876992, 140011194265599,
+STORE, 140011194265600, 140011202658303,
+STORE, 140011185876992, 140011194265599,
+SNULL, 140011194269695, 140011202658303,
+STORE, 140011194265600, 140011194269695,
+STORE, 140011194269696, 140011202658303,
+STORE, 140011068440576, 140011085225983,
+SNULL, 140011311702015, 140011320090623,
+STORE, 140011311697920, 140011311702015,
+STORE, 140011311702016, 140011320090623,
+STORE, 140011060047872, 140011085225983,
+SNULL, 140011060051967, 140011085225983,
+STORE, 140011060047872, 140011060051967,
+STORE, 140011060051968, 140011085225983,
+STORE, 140011051655168, 140011060047871,
+STORE, 140011043262464, 140011060047871,
+SNULL, 140011043266559, 140011060047871,
+STORE, 140011043262464, 140011043266559,
+STORE, 140011043266560, 140011060047871,
+SNULL, 140011219447808, 140011227836415,
+STORE, 140011227836416, 140011236229119,
+STORE, 140011219447808, 140011227836415,
+SNULL, 140011227840511, 140011236229119,
+STORE, 140011227836416, 140011227840511,
+STORE, 140011227840512, 140011236229119,
+SNULL, 140011085230080, 140011093618687,
+STORE, 140011093618688, 140011102011391,
+STORE, 140011085230080, 140011093618687,
+SNULL, 140011093622783, 140011102011391,
+STORE, 140011093618688, 140011093622783,
+STORE, 140011093622784, 140011102011391,
+STORE, 140010959400960, 140010967793663,
+STORE, 140010951008256, 140010967793663,
+SNULL, 140010951008256, 140010959400959,
+STORE, 140010959400960, 140010967793663,
+STORE, 140010951008256, 140010959400959,
+SNULL, 140010959405055, 140010967793663,
+STORE, 140010959400960, 140010959405055,
+STORE, 140010959405056, 140010967793663,
+STORE, 140010942615552, 140010959400959,
+STORE, 140010934222848, 140010959400959,
+SNULL, 140011060051968, 140011076833279,
+STORE, 140011076833280, 140011085225983,
+STORE, 140011060051968, 140011076833279,
+SNULL, 140011076837375, 140011085225983,
+STORE, 140011076833280, 140011076837375,
+STORE, 140011076837376, 140011085225983,
+SNULL, 140011043266560, 140011051655167,
+STORE, 140011051655168, 140011060047871,
+STORE, 140011043266560, 140011051655167,
+SNULL, 140011051659263, 140011060047871,
+STORE, 140011051655168, 140011051659263,
+STORE, 140011051659264, 140011060047871,
+STORE, 140010925830144, 140010959400959,
+SNULL, 140011060051968, 140011068440575,
+STORE, 140011068440576, 140011076833279,
+STORE, 140011060051968, 140011068440575,
+SNULL, 140011068444671, 140011076833279,
+STORE, 140011068440576, 140011068444671,
+STORE, 140011068444672, 140011076833279,
+STORE, 140010917437440, 140010959400959,
+STORE, 140010909044736, 140010959400959,
+STORE, 140010825183232, 140010833575935,
+SNULL, 140010909044736, 140010942615551,
+STORE, 140010942615552, 140010959400959,
+STORE, 140010909044736, 140010942615551,
+SNULL, 140010942619647, 140010959400959,
+STORE, 140010942615552, 140010942619647,
+STORE, 140010942619648, 140010959400959,
+SNULL, 140010909044736, 140010934222847,
+STORE, 140010934222848, 140010942615551,
+STORE, 140010909044736, 140010934222847,
+SNULL, 140010934226943, 140010942615551,
+STORE, 140010934222848, 140010934226943,
+STORE, 140010934226944, 140010942615551,
+SNULL, 140010909048831, 140010934222847,
+STORE, 140010909044736, 140010909048831,
+STORE, 140010909048832, 140010934222847,
+STORE, 140010816790528, 140010833575935,
+SNULL, 140010816794623, 140010833575935,
+STORE, 140010816790528, 140010816794623,
+STORE, 140010816794624, 140010833575935,
+STORE, 140010808397824, 140010816790527,
+SNULL, 140010942619648, 140010951008255,
+STORE, 140010951008256, 140010959400959,
+STORE, 140010942619648, 140010951008255,
+SNULL, 140010951012351, 140010959400959,
+STORE, 140010951008256, 140010951012351,
+STORE, 140010951012352, 140010959400959,
+STORE, 140010800005120, 140010816790527,
+SNULL, 140010800009215, 140010816790527,
+STORE, 140010800005120, 140010800009215,
+STORE, 140010800009216, 140010816790527,
+SNULL, 140010909048832, 140010925830143,
+STORE, 140010925830144, 140010934222847,
+STORE, 140010909048832, 140010925830143,
+SNULL, 140010925834239, 140010934222847,
+STORE, 140010925830144, 140010925834239,
+STORE, 140010925834240, 140010934222847,
+SNULL, 140010816794624, 140010825183231,
+STORE, 140010825183232, 140010833575935,
+STORE, 140010816794624, 140010825183231,
+SNULL, 140010825187327, 140010833575935,
+STORE, 140010825183232, 140010825187327,
+STORE, 140010825187328, 140010833575935,
+SNULL, 140010909048832, 140010917437439,
+STORE, 140010917437440, 140010925830143,
+STORE, 140010909048832, 140010917437439,
+SNULL, 140010917441535, 140010925830143,
+STORE, 140010917437440, 140010917441535,
+STORE, 140010917441536, 140010925830143,
+SNULL, 140010800009216, 140010808397823,
+STORE, 140010808397824, 140010816790527,
+STORE, 140010800009216, 140010808397823,
+SNULL, 140010808401919, 140010816790527,
+STORE, 140010808397824, 140010808401919,
+STORE, 140010808401920, 140010816790527,
+STORE, 140010791612416, 140010800005119,
+SNULL, 140010791616511, 140010800005119,
+STORE, 140010791612416, 140010791616511,
+STORE, 140010791616512, 140010800005119,
+STORE, 140012547100672, 140012547129343,
+STORE, 140012511506432, 140012513697791,
+SNULL, 140012511506432, 140012511596543,
+STORE, 140012511596544, 140012513697791,
+STORE, 140012511506432, 140012511596543,
+SNULL, 140012513689599, 140012513697791,
+STORE, 140012511596544, 140012513689599,
+STORE, 140012513689600, 140012513697791,
+ERASE, 140012513689600, 140012513697791,
+STORE, 140012513689600, 140012513697791,
+SNULL, 140012513693695, 140012513697791,
+STORE, 140012513689600, 140012513693695,
+STORE, 140012513693696, 140012513697791,
+ERASE, 140012547100672, 140012547129343,
+ERASE, 140011362054144, 140011362058239,
+ERASE, 140011362058240, 140011370446847,
+ERASE, 140011882139648, 140011882143743,
+ERASE, 140011882143744, 140011890532351,
+ERASE, 140011873746944, 140011873751039,
+ERASE, 140011873751040, 140011882139647,
+ERASE, 140011588526080, 140011588530175,
+ERASE, 140011588530176, 140011596918783,
+ERASE, 140011328483328, 140011328487423,
+ERASE, 140011328487424, 140011336876031,
+ERASE, 140011898925056, 140011898929151,
+ERASE, 140011898929152, 140011907317759,
+ERASE, 140011353661440, 140011353665535,
+ERASE, 140011353665536, 140011362054143,
+ERASE, 140011336876032, 140011336880127,
+ERASE, 140011336880128, 140011345268735,
+ERASE, 140011731136512, 140011731140607,
+ERASE, 140011731140608, 140011739529215,
+ERASE, 140011479486464, 140011479490559,
+ERASE, 140011479490560, 140011487879167,
+ERASE, 140011756314624, 140011756318719,
+ERASE, 140011756318720, 140011764707327,
+ERASE, 140011580133376, 140011580137471,
+ERASE, 140011580137472, 140011588526079,
+ERASE, 140011219443712, 140011219447807,
+ERASE, 140011219447808, 140011227836415,
+ERASE, 140011051655168, 140011051659263,
+ERASE, 140011051659264, 140011060047871,
+ERASE, 140011999571968, 140011999576063,
+ERASE, 140011999576064, 140012007964671,
+ERASE, 140011714351104, 140011714355199,
+ERASE, 140011714355200, 140011722743807,
+ERASE, 140011739529216, 140011739533311,
+ERASE, 140011739533312, 140011747921919,
+ERASE, 140011320090624, 140011320094719,
+ERASE, 140011320094720, 140011328483327,
+ERASE, 140011630489600, 140011630493695,
+ERASE, 140011630493696, 140011638882303,
+ERASE, 140011345268736, 140011345272831,
+ERASE, 140011345272832, 140011353661439,
+ERASE, 140011496271872, 140011496275967,
+ERASE, 140011496275968, 140011504664575,
+ERASE, 140011194265600, 140011194269695,
+ERASE, 140011194269696, 140011202658303,
+ERASE, 140011068440576, 140011068444671,
+ERASE, 140011068444672, 140011076833279,
+ERASE, 140010909044736, 140010909048831,
+ERASE, 140010909048832, 140010917437439,
+ERASE, 140011764707328, 140011764711423,
+ERASE, 140011764711424, 140011773100031,
+ERASE, 140011462701056, 140011462705151,
+ERASE, 140011462705152, 140011471093759,
+ERASE, 140011076833280, 140011076837375,
+ERASE, 140011076837376, 140011085225983,
+ERASE, 140011991179264, 140011991183359,
+ERASE, 140011991183360, 140011999571967,
+ERASE, 140011211051008, 140011211055103,
+ERASE, 140011211055104, 140011219443711,
+ERASE, 140010917437440, 140010917441535,
+ERASE, 140010917441536, 140010925830143,
+ERASE, 140011085225984, 140011085230079,
+ERASE, 140011085230080, 140011093618687,
+ERASE, 140011487879168, 140011487883263,
+ERASE, 140011487883264, 140011496271871,
+ERASE, 140011856961536, 140011856965631,
+ERASE, 140011856965632, 140011865354239,
+ERASE, 140011982786560, 140011982790655,
+ERASE, 140011982790656, 140011991179263,
+ERASE, 140011722743808, 140011722747903,
+ERASE, 140011722747904, 140011731136511,
+ERASE, 140011177480192, 140011177484287,
+ERASE, 140011177484288, 140011185872895,
+ERASE, 140011848568832, 140011848572927,
+ERASE, 140011848572928, 140011856961535,
+ERASE, 140011890532352, 140011890536447,
+ERASE, 140011890536448, 140011898925055,
+ERASE, 140011622096896, 140011622100991,
+ERASE, 140011622100992, 140011630489599,
+ERASE, 140011311697920, 140011311702015,
+ERASE, 140011311702016, 140011320090623,
+ERASE, 140011471093760, 140011471097855,
+ERASE, 140011471097856, 140011479486463,
+ERASE, 140011605311488, 140011605315583,
+ERASE, 140011605315584, 140011613704191,
+ERASE, 140010791612416, 140010791616511,
+ERASE, 140010791616512, 140010800005119,
+ERASE, 140010959400960, 140010959405055,
+ERASE, 140010959405056, 140010967793663,
+ERASE, 140011185872896, 140011185876991,
+ERASE, 140011185876992, 140011194265599,
+ERASE, 140011454308352, 140011454312447,
+ERASE, 140011454312448, 140011462701055,
+ERASE, 140011596918784, 140011596922879,
+ERASE, 140011596922880, 140011605311487,
+ERASE, 140011060047872, 140011060051967,
+ERASE, 140011060051968, 140011068440575,
+ERASE, 140010925830144, 140010925834239,
+ERASE, 140010925834240, 140010934222847,
+ERASE, 140011747921920, 140011747926015,
+ERASE, 140011747926016, 140011756314623,
+ERASE, 140011202658304, 140011202662399,
+ERASE, 140011202662400, 140011211051007,
+ERASE, 140010800005120, 140010800009215,
+ERASE, 140010800009216, 140010808397823,
+ERASE, 140011093618688, 140011093622783,
+ERASE, 140011093622784, 140011102011391,
+ERASE, 140010808397824, 140010808401919,
+ERASE, 140010808401920, 140010816790527,
+ERASE, 140012419010560, 140012419014655,
+ERASE, 140012419014656, 140012427403263,
+ERASE, 140010934222848, 140010934226943,
+ERASE, 140010934226944, 140010942615551,
+ERASE, 140010942615552, 140010942619647,
+ERASE, 140010942619648, 140010951008255,
+ERASE, 140011613704192, 140011613708287,
+ERASE, 140011613708288, 140011622096895,
+ERASE, 140011865354240, 140011865358335,
+ERASE, 140011865358336, 140011873746943,
+ERASE, 140012301578240, 140012301582335,
+ERASE, 140012301582336, 140012309970943,
+ERASE, 140012393832448, 140012393836543,
+ERASE, 140012393836544, 140012402225151,
+ERASE, 140012410617856, 140012410621951,
+ERASE, 140012410621952, 140012419010559,
+ERASE, 140012402225152, 140012402229247,
+ERASE, 140012402229248, 140012410617855,
+ERASE, 140012259614720, 140012259618815,
+ERASE, 140012259618816, 140012268007423,
+ERASE, 140012251222016, 140012251226111,
+ERASE, 140012251226112, 140012259614719,
+ERASE, 140012284792832, 140012284796927,
+ERASE, 140012284796928, 140012293185535,
+ERASE, 140011445915648, 140011445919743,
+ERASE, 140011445919744, 140011454308351,
+ERASE, 140010951008256, 140010951012351,
+ERASE, 140010951012352, 140010959400959,
+ERASE, 140011043262464, 140011043266559,
+ERASE, 140011043266560, 140011051655167,
+ERASE, 140010825183232, 140010825187327,
+ERASE, 140010825187328, 140010833575935,
+ERASE, 140012293185536, 140012293189631,
+ERASE, 140012293189632, 140012301578239,
+ERASE, 140012276400128, 140012276404223,
+ERASE, 140012276404224, 140012284792831,
+ERASE, 140012016357376, 140012016361471,
+ERASE, 140012016361472, 140012024750079,
+ERASE, 140012024750080, 140012024754175,
+ERASE, 140012024754176, 140012033142783,
+ERASE, 140011227836416, 140011227840511,
+ERASE, 140011227840512, 140011236229119,
+ERASE, 140010816790528, 140010816794623,
+ERASE, 140010816794624, 140010825183231,
+ERASE, 140012268007424, 140012268011519,
+ERASE, 140012268011520, 140012276400127,
+ERASE, 140012385439744, 140012385443839,
+ERASE, 140012385443840, 140012393832447,
+ERASE, 140012522090496, 140012522094591,
+ERASE, 140012522094592, 140012530483199,
+ERASE, 140012033142784, 140012033146879,
+ERASE, 140012033146880, 140012041535487,
+ };
+ unsigned long set35[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140730536939520, 140737488351231,
+SNULL, 140730536943615, 140737488351231,
+STORE, 140730536939520, 140730536943615,
+STORE, 140730536808448, 140730536943615,
+STORE, 94245239877632, 94245242130431,
+SNULL, 94245240008703, 94245242130431,
+STORE, 94245239877632, 94245240008703,
+STORE, 94245240008704, 94245242130431,
+ERASE, 94245240008704, 94245242130431,
+STORE, 94245242101760, 94245242109951,
+STORE, 94245242109952, 94245242130431,
+STORE, 140475575263232, 140475577516031,
+SNULL, 140475575406591, 140475577516031,
+STORE, 140475575263232, 140475575406591,
+STORE, 140475575406592, 140475577516031,
+ERASE, 140475575406592, 140475577516031,
+STORE, 140475577503744, 140475577511935,
+STORE, 140475577511936, 140475577516031,
+STORE, 140730538164224, 140730538168319,
+STORE, 140730538151936, 140730538164223,
+STORE, 140475577475072, 140475577503743,
+STORE, 140475577466880, 140475577475071,
+STORE, 140475573047296, 140475575263231,
+SNULL, 140475573047296, 140475573145599,
+STORE, 140475573145600, 140475575263231,
+STORE, 140475573047296, 140475573145599,
+SNULL, 140475575238655, 140475575263231,
+STORE, 140475573145600, 140475575238655,
+STORE, 140475575238656, 140475575263231,
+SNULL, 140475575238656, 140475575246847,
+STORE, 140475575246848, 140475575263231,
+STORE, 140475575238656, 140475575246847,
+ERASE, 140475575238656, 140475575246847,
+STORE, 140475575238656, 140475575246847,
+ERASE, 140475575246848, 140475575263231,
+STORE, 140475575246848, 140475575263231,
+STORE, 140475569250304, 140475573047295,
+SNULL, 140475569250304, 140475570909183,
+STORE, 140475570909184, 140475573047295,
+STORE, 140475569250304, 140475570909183,
+SNULL, 140475573006335, 140475573047295,
+STORE, 140475570909184, 140475573006335,
+STORE, 140475573006336, 140475573047295,
+SNULL, 140475573006336, 140475573030911,
+STORE, 140475573030912, 140475573047295,
+STORE, 140475573006336, 140475573030911,
+ERASE, 140475573006336, 140475573030911,
+STORE, 140475573006336, 140475573030911,
+ERASE, 140475573030912, 140475573047295,
+STORE, 140475573030912, 140475573047295,
+STORE, 140475577458688, 140475577475071,
+SNULL, 140475573022719, 140475573030911,
+STORE, 140475573006336, 140475573022719,
+STORE, 140475573022720, 140475573030911,
+SNULL, 140475575242751, 140475575246847,
+STORE, 140475575238656, 140475575242751,
+STORE, 140475575242752, 140475575246847,
+SNULL, 94245242105855, 94245242109951,
+STORE, 94245242101760, 94245242105855,
+STORE, 94245242105856, 94245242109951,
+SNULL, 140475577507839, 140475577511935,
+STORE, 140475577503744, 140475577507839,
+STORE, 140475577507840, 140475577511935,
+ERASE, 140475577475072, 140475577503743,
+STORE, 94245271216128, 94245271351295,
+STORE, 140475560857600, 140475569250303,
+SNULL, 140475560861695, 140475569250303,
+STORE, 140475560857600, 140475560861695,
+STORE, 140475560861696, 140475569250303,
+STORE, 140475552464896, 140475560857599,
+STORE, 140475418247168, 140475552464895,
+SNULL, 140475418247168, 140475428241407,
+STORE, 140475428241408, 140475552464895,
+STORE, 140475418247168, 140475428241407,
+ERASE, 140475418247168, 140475428241407,
+SNULL, 140475495350271, 140475552464895,
+STORE, 140475428241408, 140475495350271,
+STORE, 140475495350272, 140475552464895,
+ERASE, 140475495350272, 140475552464895,
+SNULL, 140475428376575, 140475495350271,
+STORE, 140475428241408, 140475428376575,
+STORE, 140475428376576, 140475495350271,
+SNULL, 140475552468991, 140475560857599,
+STORE, 140475552464896, 140475552468991,
+STORE, 140475552468992, 140475560857599,
+STORE, 140475544072192, 140475552464895,
+SNULL, 140475544076287, 140475552464895,
+STORE, 140475544072192, 140475544076287,
+STORE, 140475544076288, 140475552464895,
+STORE, 140475535679488, 140475544072191,
+SNULL, 140475535683583, 140475544072191,
+STORE, 140475535679488, 140475535683583,
+STORE, 140475535683584, 140475544072191,
+STORE, 140475527286784, 140475535679487,
+SNULL, 140475527290879, 140475535679487,
+STORE, 140475527286784, 140475527290879,
+STORE, 140475527290880, 140475535679487,
+STORE, 140475518894080, 140475527286783,
+STORE, 140475510501376, 140475527286783,
+STORE, 140475502108672, 140475527286783,
+STORE, 140475419848704, 140475428241407,
+STORE, 140475285630976, 140475419848703,
+SNULL, 140475285630976, 140475294023679,
+STORE, 140475294023680, 140475419848703,
+STORE, 140475285630976, 140475294023679,
+ERASE, 140475285630976, 140475294023679,
+STORE, 140475159805952, 140475419848703,
+STORE, 140475025588224, 140475419848703,
+SNULL, 140475092697087, 140475419848703,
+STORE, 140475025588224, 140475092697087,
+STORE, 140475092697088, 140475419848703,
+SNULL, 140475092697088, 140475159805951,
+STORE, 140475159805952, 140475419848703,
+STORE, 140475092697088, 140475159805951,
+ERASE, 140475092697088, 140475159805951,
+STORE, 140474891370496, 140475092697087,
+SNULL, 140474958479359, 140475092697087,
+STORE, 140474891370496, 140474958479359,
+STORE, 140474958479360, 140475092697087,
+SNULL, 140474958479360, 140475025588223,
+STORE, 140475025588224, 140475092697087,
+STORE, 140474958479360, 140475025588223,
+ERASE, 140474958479360, 140475025588223,
+SNULL, 140475361132543, 140475419848703,
+STORE, 140475159805952, 140475361132543,
+STORE, 140475361132544, 140475419848703,
+ERASE, 140475361132544, 140475419848703,
+SNULL, 140475159805952, 140475294023679,
+STORE, 140475294023680, 140475361132543,
+STORE, 140475159805952, 140475294023679,
+SNULL, 140475294158847, 140475361132543,
+STORE, 140475294023680, 140475294158847,
+STORE, 140475294158848, 140475361132543,
+SNULL, 140475226914815, 140475294023679,
+STORE, 140475159805952, 140475226914815,
+STORE, 140475226914816, 140475294023679,
+ERASE, 140475226914816, 140475294023679,
+SNULL, 140475025723391, 140475092697087,
+STORE, 140475025588224, 140475025723391,
+STORE, 140475025723392, 140475092697087,
+SNULL, 140475159941119, 140475226914815,
+STORE, 140475159805952, 140475159941119,
+STORE, 140475159941120, 140475226914815,
+SNULL, 140474891505663, 140474958479359,
+STORE, 140474891370496, 140474891505663,
+STORE, 140474891505664, 140474958479359,
+SNULL, 140475502108672, 140475518894079,
+STORE, 140475518894080, 140475527286783,
+STORE, 140475502108672, 140475518894079,
+SNULL, 140475518898175, 140475527286783,
+STORE, 140475518894080, 140475518898175,
+STORE, 140475518898176, 140475527286783,
+STORE, 140475411456000, 140475428241407,
+SNULL, 140475502112767, 140475518894079,
+STORE, 140475502108672, 140475502112767,
+STORE, 140475502112768, 140475518894079,
+SNULL, 140475411460095, 140475428241407,
+STORE, 140475411456000, 140475411460095,
+STORE, 140475411460096, 140475428241407,
+SNULL, 140475411460096, 140475419848703,
+STORE, 140475419848704, 140475428241407,
+STORE, 140475411460096, 140475419848703,
+SNULL, 140475419852799, 140475428241407,
+STORE, 140475419848704, 140475419852799,
+STORE, 140475419852800, 140475428241407,
+STORE, 140475403063296, 140475411455999,
+SNULL, 140475502112768, 140475510501375,
+STORE, 140475510501376, 140475518894079,
+STORE, 140475502112768, 140475510501375,
+SNULL, 140475510505471, 140475518894079,
+STORE, 140475510501376, 140475510505471,
+STORE, 140475510505472, 140475518894079,
+SNULL, 140475403067391, 140475411455999,
+STORE, 140475403063296, 140475403067391,
+STORE, 140475403067392, 140475411455999,
+STORE, 140475394670592, 140475403063295,
+SNULL, 140475394674687, 140475403063295,
+STORE, 140475394670592, 140475394674687,
+STORE, 140475394674688, 140475403063295,
+STORE, 140475386277888, 140475394670591,
+STORE, 140475377885184, 140475394670591,
+STORE, 140475369492480, 140475394670591,
+SNULL, 140475369496575, 140475394670591,
+STORE, 140475369492480, 140475369496575,
+STORE, 140475369496576, 140475394670591,
+SNULL, 140475369496576, 140475377885183,
+STORE, 140475377885184, 140475394670591,
+STORE, 140475369496576, 140475377885183,
+SNULL, 140475377889279, 140475394670591,
+STORE, 140475377885184, 140475377889279,
+STORE, 140475377889280, 140475394670591,
+STORE, 140475285630976, 140475294023679,
+SNULL, 140475377889280, 140475386277887,
+STORE, 140475386277888, 140475394670591,
+STORE, 140475377889280, 140475386277887,
+SNULL, 140475386281983, 140475394670591,
+STORE, 140475386277888, 140475386281983,
+STORE, 140475386281984, 140475394670591,
+SNULL, 140475285635071, 140475294023679,
+STORE, 140475285630976, 140475285635071,
+STORE, 140475285635072, 140475294023679,
+STORE, 140475277238272, 140475285630975,
+STORE, 140475268845568, 140475285630975,
+SNULL, 140475268845568, 140475277238271,
+STORE, 140475277238272, 140475285630975,
+STORE, 140475268845568, 140475277238271,
+SNULL, 140475277242367, 140475285630975,
+STORE, 140475277238272, 140475277242367,
+STORE, 140475277242368, 140475285630975,
+STORE, 140475260452864, 140475277238271,
+SNULL, 140475260452864, 140475268845567,
+STORE, 140475268845568, 140475277238271,
+STORE, 140475260452864, 140475268845567,
+SNULL, 140475268849663, 140475277238271,
+STORE, 140475268845568, 140475268849663,
+STORE, 140475268849664, 140475277238271,
+SNULL, 140475260456959, 140475268845567,
+STORE, 140475260452864, 140475260456959,
+STORE, 140475260456960, 140475268845567,
+STORE, 140475252060160, 140475260452863,
+SNULL, 140475252064255, 140475260452863,
+STORE, 140475252060160, 140475252064255,
+STORE, 140475252064256, 140475260452863,
+STORE, 140475243667456, 140475252060159,
+SNULL, 140475243671551, 140475252060159,
+STORE, 140475243667456, 140475243671551,
+STORE, 140475243671552, 140475252060159,
+STORE, 140475235274752, 140475243667455,
+STORE, 140475151413248, 140475159805951,
+STORE, 140474891505664, 140475025588223,
+STORE, 140475143020544, 140475159805951,
+SNULL, 140474891505664, 140474958479359,
+STORE, 140474958479360, 140475025588223,
+STORE, 140474891505664, 140474958479359,
+SNULL, 140474958614527, 140475025588223,
+STORE, 140474958479360, 140474958614527,
+STORE, 140474958614528, 140475025588223,
+STORE, 140474824261632, 140474891370495,
+SNULL, 140474824396799, 140474891370495,
+STORE, 140474824261632, 140474824396799,
+STORE, 140474824396800, 140474891370495,
+STORE, 140475134627840, 140475159805951,
+STORE, 140474690043904, 140474824261631,
+STORE, 140475126235136, 140475159805951,
+STORE, 140475117842432, 140475159805951,
+STORE, 140474622935040, 140474824261631,
+STORE, 140475109449728, 140475159805951,
+STORE, 140474488717312, 140474824261631,
+STORE, 140475101057024, 140475159805951,
+STORE, 140474480324608, 140474488717311,
+STORE, 140474413215744, 140474480324607,
+STORE, 140474404823040, 140474413215743,
+ERASE, 140474413215744, 140474480324607,
+STORE, 140474471931904, 140474488717311,
+STORE, 140474270605312, 140474404823039,
+SNULL, 140475101057024, 140475126235135,
+STORE, 140475126235136, 140475159805951,
+STORE, 140475101057024, 140475126235135,
+SNULL, 140475126239231, 140475159805951,
+STORE, 140475126235136, 140475126239231,
+STORE, 140475126239232, 140475159805951,
+STORE, 140474463539200, 140474488717311,
+STORE, 140474455146496, 140474488717311,
+SNULL, 140474455150591, 140474488717311,
+STORE, 140474455146496, 140474455150591,
+STORE, 140474455150592, 140474488717311,
+STORE, 140474446753792, 140474455146495,
+SNULL, 140474446757887, 140474455146495,
+STORE, 140474446753792, 140474446757887,
+STORE, 140474446757888, 140474455146495,
+STORE, 140474438361088, 140474446753791,
+STORE, 140474429968384, 140474446753791,
+SNULL, 140474429972479, 140474446753791,
+STORE, 140474429968384, 140474429972479,
+STORE, 140474429972480, 140474446753791,
+SNULL, 140475235278847, 140475243667455,
+STORE, 140475235274752, 140475235278847,
+STORE, 140475235278848, 140475243667455,
+SNULL, 140474757152767, 140474824261631,
+STORE, 140474488717312, 140474757152767,
+STORE, 140474757152768, 140474824261631,
+ERASE, 140474757152768, 140474824261631,
+SNULL, 140474488717312, 140474690043903,
+STORE, 140474690043904, 140474757152767,
+STORE, 140474488717312, 140474690043903,
+SNULL, 140474690179071, 140474757152767,
+STORE, 140474690043904, 140474690179071,
+STORE, 140474690179072, 140474757152767,
+SNULL, 140474488717312, 140474622935039,
+STORE, 140474622935040, 140474690043903,
+STORE, 140474488717312, 140474622935039,
+SNULL, 140474623070207, 140474690043903,
+STORE, 140474622935040, 140474623070207,
+STORE, 140474623070208, 140474690043903,
+SNULL, 140475101057024, 140475117842431,
+STORE, 140475117842432, 140475126235135,
+STORE, 140475101057024, 140475117842431,
+SNULL, 140475117846527, 140475126235135,
+STORE, 140475117842432, 140475117846527,
+STORE, 140475117846528, 140475126235135,
+SNULL, 140474555826175, 140474622935039,
+STORE, 140474488717312, 140474555826175,
+STORE, 140474555826176, 140474622935039,
+ERASE, 140474555826176, 140474622935039,
+STORE, 140474136387584, 140474404823039,
+SNULL, 140474136387584, 140474153172991,
+STORE, 140474153172992, 140474404823039,
+STORE, 140474136387584, 140474153172991,
+ERASE, 140474136387584, 140474153172991,
+STORE, 140474018955264, 140474404823039,
+STORE, 140473884737536, 140474404823039,
+SNULL, 140474086064127, 140474404823039,
+STORE, 140473884737536, 140474086064127,
+STORE, 140474086064128, 140474404823039,
+SNULL, 140474086064128, 140474153172991,
+STORE, 140474153172992, 140474404823039,
+STORE, 140474086064128, 140474153172991,
+ERASE, 140474086064128, 140474153172991,
+STORE, 140473750519808, 140474086064127,
+SNULL, 140473817628671, 140474086064127,
+STORE, 140473750519808, 140473817628671,
+STORE, 140473817628672, 140474086064127,
+SNULL, 140473817628672, 140473884737535,
+STORE, 140473884737536, 140474086064127,
+STORE, 140473817628672, 140473884737535,
+ERASE, 140473817628672, 140473884737535,
+SNULL, 140475126239232, 140475151413247,
+STORE, 140475151413248, 140475159805951,
+STORE, 140475126239232, 140475151413247,
+SNULL, 140475151417343, 140475159805951,
+STORE, 140475151413248, 140475151417343,
+STORE, 140475151417344, 140475159805951,
+SNULL, 140474270605311, 140474404823039,
+STORE, 140474153172992, 140474270605311,
+STORE, 140474270605312, 140474404823039,
+SNULL, 140474270605312, 140474287390719,
+STORE, 140474287390720, 140474404823039,
+STORE, 140474270605312, 140474287390719,
+ERASE, 140474270605312, 140474287390719,
+SNULL, 140474429972480, 140474438361087,
+STORE, 140474438361088, 140474446753791,
+STORE, 140474429972480, 140474438361087,
+SNULL, 140474438365183, 140474446753791,
+STORE, 140474438361088, 140474438365183,
+STORE, 140474438365184, 140474446753791,
+STORE, 140474815868928, 140474824261631,
+SNULL, 140474815873023, 140474824261631,
+STORE, 140474815868928, 140474815873023,
+STORE, 140474815873024, 140474824261631,
+SNULL, 140474220281855, 140474270605311,
+STORE, 140474153172992, 140474220281855,
+STORE, 140474220281856, 140474270605311,
+ERASE, 140474220281856, 140474270605311,
+SNULL, 140474488852479, 140474555826175,
+STORE, 140474488717312, 140474488852479,
+STORE, 140474488852480, 140474555826175,
+SNULL, 140475101057024, 140475109449727,
+STORE, 140475109449728, 140475117842431,
+STORE, 140475101057024, 140475109449727,
+SNULL, 140475109453823, 140475117842431,
+STORE, 140475109449728, 140475109453823,
+STORE, 140475109453824, 140475117842431,
+SNULL, 140473951846399, 140474086064127,
+STORE, 140473884737536, 140473951846399,
+STORE, 140473951846400, 140474086064127,
+SNULL, 140473951846400, 140474018955263,
+STORE, 140474018955264, 140474086064127,
+STORE, 140473951846400, 140474018955263,
+ERASE, 140473951846400, 140474018955263,
+SNULL, 140473884872703, 140473951846399,
+STORE, 140473884737536, 140473884872703,
+STORE, 140473884872704, 140473951846399,
+SNULL, 140474019090431, 140474086064127,
+STORE, 140474018955264, 140474019090431,
+STORE, 140474019090432, 140474086064127,
+SNULL, 140473750654975, 140473817628671,
+STORE, 140473750519808, 140473750654975,
+STORE, 140473750654976, 140473817628671,
+SNULL, 140474455150592, 140474463539199,
+STORE, 140474463539200, 140474488717311,
+STORE, 140474455150592, 140474463539199,
+SNULL, 140474463543295, 140474488717311,
+STORE, 140474463539200, 140474463543295,
+STORE, 140474463543296, 140474488717311,
+STORE, 140474807476224, 140474815868927,
+SNULL, 140474463543296, 140474471931903,
+STORE, 140474471931904, 140474488717311,
+STORE, 140474463543296, 140474471931903,
+SNULL, 140474471935999, 140474488717311,
+STORE, 140474471931904, 140474471935999,
+STORE, 140474471936000, 140474488717311,
+STORE, 140474799083520, 140474815868927,
+STORE, 140474790690816, 140474815868927,
+SNULL, 140474790690816, 140474799083519,
+STORE, 140474799083520, 140474815868927,
+STORE, 140474790690816, 140474799083519,
+SNULL, 140474799087615, 140474815868927,
+STORE, 140474799083520, 140474799087615,
+STORE, 140474799087616, 140474815868927,
+SNULL, 140474354499583, 140474404823039,
+STORE, 140474287390720, 140474354499583,
+STORE, 140474354499584, 140474404823039,
+ERASE, 140474354499584, 140474404823039,
+SNULL, 140474287525887, 140474354499583,
+STORE, 140474287390720, 140474287525887,
+STORE, 140474287525888, 140474354499583,
+STORE, 140474782298112, 140474799083519,
+STORE, 140474773905408, 140474799083519,
+SNULL, 140474773909503, 140474799083519,
+STORE, 140474773905408, 140474773909503,
+STORE, 140474773909504, 140474799083519,
+SNULL, 140475126239232, 140475134627839,
+STORE, 140475134627840, 140475151413247,
+STORE, 140475126239232, 140475134627839,
+SNULL, 140475134631935, 140475151413247,
+STORE, 140475134627840, 140475134631935,
+STORE, 140475134631936, 140475151413247,
+STORE, 140474765512704, 140474773905407,
+STORE, 140474614542336, 140474622935039,
+SNULL, 140474153308159, 140474220281855,
+STORE, 140474153172992, 140474153308159,
+STORE, 140474153308160, 140474220281855,
+SNULL, 140474404827135, 140474413215743,
+STORE, 140474404823040, 140474404827135,
+STORE, 140474404827136, 140474413215743,
+STORE, 140474606149632, 140474622935039,
+SNULL, 140474606153727, 140474622935039,
+STORE, 140474606149632, 140474606153727,
+STORE, 140474606153728, 140474622935039,
+STORE, 140474597756928, 140474606149631,
+SNULL, 140474597761023, 140474606149631,
+STORE, 140474597756928, 140474597761023,
+STORE, 140474597761024, 140474606149631,
+SNULL, 140475134631936, 140475143020543,
+STORE, 140475143020544, 140475151413247,
+STORE, 140475134631936, 140475143020543,
+SNULL, 140475143024639, 140475151413247,
+STORE, 140475143020544, 140475143024639,
+STORE, 140475143024640, 140475151413247,
+STORE, 140474589364224, 140474597756927,
+SNULL, 140474606153728, 140474614542335,
+STORE, 140474614542336, 140474622935039,
+STORE, 140474606153728, 140474614542335,
+SNULL, 140474614546431, 140474622935039,
+STORE, 140474614542336, 140474614546431,
+STORE, 140474614546432, 140474622935039,
+SNULL, 140474765516799, 140474773905407,
+STORE, 140474765512704, 140474765516799,
+STORE, 140474765516800, 140474773905407,
+STORE, 140474580971520, 140474597756927,
+SNULL, 140474773909504, 140474782298111,
+STORE, 140474782298112, 140474799083519,
+STORE, 140474773909504, 140474782298111,
+SNULL, 140474782302207, 140474799083519,
+STORE, 140474782298112, 140474782302207,
+STORE, 140474782302208, 140474799083519,
+SNULL, 140474471936000, 140474480324607,
+STORE, 140474480324608, 140474488717311,
+STORE, 140474471936000, 140474480324607,
+SNULL, 140474480328703, 140474488717311,
+STORE, 140474480324608, 140474480328703,
+STORE, 140474480328704, 140474488717311,
+STORE, 140474572578816, 140474597756927,
+SNULL, 140474572582911, 140474597756927,
+STORE, 140474572578816, 140474572582911,
+STORE, 140474572582912, 140474597756927,
+SNULL, 140474782302208, 140474790690815,
+STORE, 140474790690816, 140474799083519,
+STORE, 140474782302208, 140474790690815,
+SNULL, 140474790694911, 140474799083519,
+STORE, 140474790690816, 140474790694911,
+STORE, 140474790694912, 140474799083519,
+STORE, 140474564186112, 140474572578815,
+STORE, 140474421575680, 140474429968383,
+STORE, 140474396430336, 140474404823039,
+SNULL, 140474396434431, 140474404823039,
+STORE, 140474396430336, 140474396434431,
+STORE, 140474396434432, 140474404823039,
+STORE, 140474388037632, 140474396430335,
+SNULL, 140474799087616, 140474807476223,
+STORE, 140474807476224, 140474815868927,
+STORE, 140474799087616, 140474807476223,
+SNULL, 140474807480319, 140474815868927,
+STORE, 140474807476224, 140474807480319,
+STORE, 140474807480320, 140474815868927,
+SNULL, 140475101061119, 140475109449727,
+STORE, 140475101057024, 140475101061119,
+STORE, 140475101061120, 140475109449727,
+STORE, 140474379644928, 140474396430335,
+SNULL, 140474572582912, 140474589364223,
+STORE, 140474589364224, 140474597756927,
+STORE, 140474572582912, 140474589364223,
+SNULL, 140474589368319, 140474597756927,
+STORE, 140474589364224, 140474589368319,
+STORE, 140474589368320, 140474597756927,
+STORE, 140474371252224, 140474396430335,
+STORE, 140474362859520, 140474396430335,
+STORE, 140474278998016, 140474287390719,
+STORE, 140474270605312, 140474287390719,
+STORE, 140474262212608, 140474287390719,
+SNULL, 140474262216703, 140474287390719,
+STORE, 140474262212608, 140474262216703,
+STORE, 140474262216704, 140474287390719,
+STORE, 140474253819904, 140474262212607,
+SNULL, 140474253823999, 140474262212607,
+STORE, 140474253819904, 140474253823999,
+STORE, 140474253824000, 140474262212607,
+SNULL, 140474362859520, 140474388037631,
+STORE, 140474388037632, 140474396430335,
+STORE, 140474362859520, 140474388037631,
+SNULL, 140474388041727, 140474396430335,
+STORE, 140474388037632, 140474388041727,
+STORE, 140474388041728, 140474396430335,
+SNULL, 140474362859520, 140474379644927,
+STORE, 140474379644928, 140474388037631,
+STORE, 140474362859520, 140474379644927,
+SNULL, 140474379649023, 140474388037631,
+STORE, 140474379644928, 140474379649023,
+STORE, 140474379649024, 140474388037631,
+STORE, 140474245427200, 140474253819903,
+STORE, 140474237034496, 140474253819903,
+STORE, 140474228641792, 140474253819903,
+STORE, 140474144780288, 140474153172991,
+SNULL, 140474228645887, 140474253819903,
+STORE, 140474228641792, 140474228645887,
+STORE, 140474228645888, 140474253819903,
+SNULL, 140474564190207, 140474572578815,
+STORE, 140474564186112, 140474564190207,
+STORE, 140474564190208, 140474572578815,
+STORE, 140474136387584, 140474153172991,
+SNULL, 140474362859520, 140474371252223,
+STORE, 140474371252224, 140474379644927,
+STORE, 140474362859520, 140474371252223,
+SNULL, 140474371256319, 140474379644927,
+STORE, 140474371252224, 140474371256319,
+STORE, 140474371256320, 140474379644927,
+STORE, 140474127994880, 140474153172991,
+STORE, 140474119602176, 140474153172991,
+SNULL, 140474421579775, 140474429968383,
+STORE, 140474421575680, 140474421579775,
+STORE, 140474421579776, 140474429968383,
+STORE, 140474111209472, 140474153172991,
+SNULL, 140474111213567, 140474153172991,
+STORE, 140474111209472, 140474111213567,
+STORE, 140474111213568, 140474153172991,
+SNULL, 140474262216704, 140474270605311,
+STORE, 140474270605312, 140474287390719,
+STORE, 140474262216704, 140474270605311,
+SNULL, 140474270609407, 140474287390719,
+STORE, 140474270605312, 140474270609407,
+STORE, 140474270609408, 140474287390719,
+STORE, 140474102816768, 140474111209471,
+SNULL, 140474102820863, 140474111209471,
+STORE, 140474102816768, 140474102820863,
+STORE, 140474102820864, 140474111209471,
+SNULL, 140474270609408, 140474278998015,
+STORE, 140474278998016, 140474287390719,
+STORE, 140474270609408, 140474278998015,
+SNULL, 140474279002111, 140474287390719,
+STORE, 140474278998016, 140474279002111,
+STORE, 140474279002112, 140474287390719,
+STORE, 140474094424064, 140474102816767,
+SNULL, 140474572582912, 140474580971519,
+STORE, 140474580971520, 140474589364223,
+STORE, 140474572582912, 140474580971519,
+SNULL, 140474580975615, 140474589364223,
+STORE, 140474580971520, 140474580975615,
+STORE, 140474580975616, 140474589364223,
+SNULL, 140474362863615, 140474371252223,
+STORE, 140474362859520, 140474362863615,
+STORE, 140474362863616, 140474371252223,
+STORE, 140474010562560, 140474018955263,
+SNULL, 140474228645888, 140474245427199,
+STORE, 140474245427200, 140474253819903,
+STORE, 140474228645888, 140474245427199,
+SNULL, 140474245431295, 140474253819903,
+STORE, 140474245427200, 140474245431295,
+STORE, 140474245431296, 140474253819903,
+SNULL, 140474111213568, 140474136387583,
+STORE, 140474136387584, 140474153172991,
+STORE, 140474111213568, 140474136387583,
+SNULL, 140474136391679, 140474153172991,
+STORE, 140474136387584, 140474136391679,
+STORE, 140474136391680, 140474153172991,
+STORE, 140474002169856, 140474018955263,
+STORE, 140473993777152, 140474018955263,
+SNULL, 140474111213568, 140474127994879,
+STORE, 140474127994880, 140474136387583,
+STORE, 140474111213568, 140474127994879,
+SNULL, 140474127998975, 140474136387583,
+STORE, 140474127994880, 140474127998975,
+STORE, 140474127998976, 140474136387583,
+SNULL, 140474228645888, 140474237034495,
+STORE, 140474237034496, 140474245427199,
+STORE, 140474228645888, 140474237034495,
+SNULL, 140474237038591, 140474245427199,
+STORE, 140474237034496, 140474237038591,
+STORE, 140474237038592, 140474245427199,
+SNULL, 140474136391680, 140474144780287,
+STORE, 140474144780288, 140474153172991,
+STORE, 140474136391680, 140474144780287,
+SNULL, 140474144784383, 140474153172991,
+STORE, 140474144780288, 140474144784383,
+STORE, 140474144784384, 140474153172991,
+STORE, 140473985384448, 140474018955263,
+STORE, 140473976991744, 140474018955263,
+STORE, 140473968599040, 140474018955263,
+SNULL, 140473968603135, 140474018955263,
+STORE, 140473968599040, 140473968603135,
+STORE, 140473968603136, 140474018955263,
+SNULL, 140474111213568, 140474119602175,
+STORE, 140474119602176, 140474127994879,
+STORE, 140474111213568, 140474119602175,
+SNULL, 140474119606271, 140474127994879,
+STORE, 140474119602176, 140474119606271,
+STORE, 140474119606272, 140474127994879,
+STORE, 140473960206336, 140473968599039,
+SNULL, 140474094428159, 140474102816767,
+STORE, 140474094424064, 140474094428159,
+STORE, 140474094428160, 140474102816767,
+STORE, 140473876344832, 140473884737535,
+STORE, 140473867952128, 140473884737535,
+STORE, 140473859559424, 140473884737535,
+SNULL, 140473859563519, 140473884737535,
+STORE, 140473859559424, 140473859563519,
+STORE, 140473859563520, 140473884737535,
+SNULL, 140473968603136, 140473993777151,
+STORE, 140473993777152, 140474018955263,
+STORE, 140473968603136, 140473993777151,
+SNULL, 140473993781247, 140474018955263,
+STORE, 140473993777152, 140473993781247,
+STORE, 140473993781248, 140474018955263,
+SNULL, 140473960210431, 140473968599039,
+STORE, 140473960206336, 140473960210431,
+STORE, 140473960210432, 140473968599039,
+SNULL, 140473993781248, 140474010562559,
+STORE, 140474010562560, 140474018955263,
+STORE, 140473993781248, 140474010562559,
+SNULL, 140474010566655, 140474018955263,
+STORE, 140474010562560, 140474010566655,
+STORE, 140474010566656, 140474018955263,
+SNULL, 140473968603136, 140473985384447,
+STORE, 140473985384448, 140473993777151,
+STORE, 140473968603136, 140473985384447,
+SNULL, 140473985388543, 140473993777151,
+STORE, 140473985384448, 140473985388543,
+STORE, 140473985388544, 140473993777151,
+SNULL, 140473993781248, 140474002169855,
+STORE, 140474002169856, 140474010562559,
+STORE, 140473993781248, 140474002169855,
+SNULL, 140474002173951, 140474010562559,
+STORE, 140474002169856, 140474002173951,
+STORE, 140474002173952, 140474010562559,
+STORE, 140473851166720, 140473859559423,
+SNULL, 140473851170815, 140473859559423,
+STORE, 140473851166720, 140473851170815,
+STORE, 140473851170816, 140473859559423,
+SNULL, 140473968603136, 140473976991743,
+STORE, 140473976991744, 140473985384447,
+STORE, 140473968603136, 140473976991743,
+SNULL, 140473976995839, 140473985384447,
+STORE, 140473976991744, 140473976995839,
+STORE, 140473976995840, 140473985384447,
+STORE, 140473842774016, 140473851166719,
+SNULL, 140473859563520, 140473867952127,
+STORE, 140473867952128, 140473884737535,
+STORE, 140473859563520, 140473867952127,
+SNULL, 140473867956223, 140473884737535,
+STORE, 140473867952128, 140473867956223,
+STORE, 140473867956224, 140473884737535,
+SNULL, 140473867956224, 140473876344831,
+STORE, 140473876344832, 140473884737535,
+STORE, 140473867956224, 140473876344831,
+SNULL, 140473876348927, 140473884737535,
+STORE, 140473876344832, 140473876348927,
+STORE, 140473876348928, 140473884737535,
+STORE, 140473834381312, 140473851166719,
+SNULL, 140473834385407, 140473851166719,
+STORE, 140473834381312, 140473834385407,
+STORE, 140473834385408, 140473851166719,
+SNULL, 140473834385408, 140473842774015,
+STORE, 140473842774016, 140473851166719,
+STORE, 140473834385408, 140473842774015,
+SNULL, 140473842778111, 140473851166719,
+STORE, 140473842774016, 140473842778111,
+STORE, 140473842778112, 140473851166719,
+STORE, 140473825988608, 140473834381311,
+SNULL, 140473825992703, 140473834381311,
+STORE, 140473825988608, 140473825992703,
+STORE, 140473825992704, 140473834381311,
+STORE, 140475577475072, 140475577503743,
+STORE, 140475499917312, 140475502108671,
+SNULL, 140475499917312, 140475500007423,
+STORE, 140475500007424, 140475502108671,
+STORE, 140475499917312, 140475500007423,
+SNULL, 140475502100479, 140475502108671,
+STORE, 140475500007424, 140475502100479,
+STORE, 140475502100480, 140475502108671,
+ERASE, 140475502100480, 140475502108671,
+STORE, 140475502100480, 140475502108671,
+SNULL, 140475502104575, 140475502108671,
+STORE, 140475502100480, 140475502104575,
+STORE, 140475502104576, 140475502108671,
+ERASE, 140475577475072, 140475577503743,
+ERASE, 140475235274752, 140475235278847,
+ERASE, 140475235278848, 140475243667455,
+ERASE, 140474815868928, 140474815873023,
+ERASE, 140474815873024, 140474824261631,
+ERASE, 140474606149632, 140474606153727,
+ERASE, 140474606153728, 140474614542335,
+ERASE, 140474270605312, 140474270609407,
+ERASE, 140474270609408, 140474278998015,
+ERASE, 140474438361088, 140474438365183,
+ERASE, 140474438365184, 140474446753791,
+ERASE, 140474597756928, 140474597761023,
+ERASE, 140474597761024, 140474606149631,
+ERASE, 140475126235136, 140475126239231,
+ERASE, 140475126239232, 140475134627839,
+ERASE, 140474463539200, 140474463543295,
+ERASE, 140474463543296, 140474471931903,
+ERASE, 140474388037632, 140474388041727,
+ERASE, 140474388041728, 140474396430335,
+ERASE, 140474404823040, 140474404827135,
+ERASE, 140474404827136, 140474413215743,
+ERASE, 140474278998016, 140474279002111,
+ERASE, 140474279002112, 140474287390719,
+ERASE, 140474094424064, 140474094428159,
+ERASE, 140474094428160, 140474102816767,
+ERASE, 140473867952128, 140473867956223,
+ERASE, 140473867956224, 140473876344831,
+ERASE, 140475151413248, 140475151417343,
+ERASE, 140475151417344, 140475159805951,
+ERASE, 140474455146496, 140474455150591,
+ERASE, 140474455150592, 140474463539199,
+ERASE, 140474807476224, 140474807480319,
+ERASE, 140474807480320, 140474815868927,
+ERASE, 140475117842432, 140475117846527,
+ERASE, 140475117846528, 140475126235135,
+ERASE, 140474446753792, 140474446757887,
+ERASE, 140474446757888, 140474455146495,
+ERASE, 140474429968384, 140474429972479,
+ERASE, 140474429972480, 140474438361087,
+ERASE, 140474782298112, 140474782302207,
+ERASE, 140474782302208, 140474790690815,
+ERASE, 140474136387584, 140474136391679,
+ERASE, 140474136391680, 140474144780287,
+ERASE, 140474002169856, 140474002173951,
+ERASE, 140474002173952, 140474010562559,
+ERASE, 140475134627840, 140475134631935,
+ERASE, 140475134631936, 140475143020543,
+ERASE, 140474471931904, 140474471935999,
+ERASE, 140474471936000, 140474480324607,
+ERASE, 140474396430336, 140474396434431,
+ERASE, 140474396434432, 140474404823039,
+ };
+ unsigned long set36[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140723893125120, 140737488351231,
+SNULL, 140723893129215, 140737488351231,
+STORE, 140723893125120, 140723893129215,
+STORE, 140723892994048, 140723893129215,
+STORE, 94076829786112, 94076832038911,
+SNULL, 94076829917183, 94076832038911,
+STORE, 94076829786112, 94076829917183,
+STORE, 94076829917184, 94076832038911,
+ERASE, 94076829917184, 94076832038911,
+STORE, 94076832010240, 94076832018431,
+STORE, 94076832018432, 94076832038911,
+STORE, 140122444345344, 140122446598143,
+SNULL, 140122444488703, 140122446598143,
+STORE, 140122444345344, 140122444488703,
+STORE, 140122444488704, 140122446598143,
+ERASE, 140122444488704, 140122446598143,
+STORE, 140122446585856, 140122446594047,
+STORE, 140122446594048, 140122446598143,
+STORE, 140723893538816, 140723893542911,
+STORE, 140723893526528, 140723893538815,
+STORE, 140122446557184, 140122446585855,
+STORE, 140122446548992, 140122446557183,
+STORE, 140122442129408, 140122444345343,
+SNULL, 140122442129408, 140122442227711,
+STORE, 140122442227712, 140122444345343,
+STORE, 140122442129408, 140122442227711,
+SNULL, 140122444320767, 140122444345343,
+STORE, 140122442227712, 140122444320767,
+STORE, 140122444320768, 140122444345343,
+SNULL, 140122444320768, 140122444328959,
+STORE, 140122444328960, 140122444345343,
+STORE, 140122444320768, 140122444328959,
+ERASE, 140122444320768, 140122444328959,
+STORE, 140122444320768, 140122444328959,
+ERASE, 140122444328960, 140122444345343,
+STORE, 140122444328960, 140122444345343,
+STORE, 140122438332416, 140122442129407,
+SNULL, 140122438332416, 140122439991295,
+STORE, 140122439991296, 140122442129407,
+STORE, 140122438332416, 140122439991295,
+SNULL, 140122442088447, 140122442129407,
+STORE, 140122439991296, 140122442088447,
+STORE, 140122442088448, 140122442129407,
+SNULL, 140122442088448, 140122442113023,
+STORE, 140122442113024, 140122442129407,
+STORE, 140122442088448, 140122442113023,
+ERASE, 140122442088448, 140122442113023,
+STORE, 140122442088448, 140122442113023,
+ERASE, 140122442113024, 140122442129407,
+STORE, 140122442113024, 140122442129407,
+STORE, 140122446540800, 140122446557183,
+SNULL, 140122442104831, 140122442113023,
+STORE, 140122442088448, 140122442104831,
+STORE, 140122442104832, 140122442113023,
+SNULL, 140122444324863, 140122444328959,
+STORE, 140122444320768, 140122444324863,
+STORE, 140122444324864, 140122444328959,
+SNULL, 94076832014335, 94076832018431,
+STORE, 94076832010240, 94076832014335,
+STORE, 94076832014336, 94076832018431,
+SNULL, 140122446589951, 140122446594047,
+STORE, 140122446585856, 140122446589951,
+STORE, 140122446589952, 140122446594047,
+ERASE, 140122446557184, 140122446585855,
+STORE, 94076845723648, 94076845858815,
+STORE, 140122429939712, 140122438332415,
+SNULL, 140122429943807, 140122438332415,
+STORE, 140122429939712, 140122429943807,
+STORE, 140122429943808, 140122438332415,
+STORE, 140122421547008, 140122429939711,
+STORE, 140122287329280, 140122421547007,
+SNULL, 140122287329280, 140122301399039,
+STORE, 140122301399040, 140122421547007,
+STORE, 140122287329280, 140122301399039,
+ERASE, 140122287329280, 140122301399039,
+SNULL, 140122368507903, 140122421547007,
+STORE, 140122301399040, 140122368507903,
+STORE, 140122368507904, 140122421547007,
+ERASE, 140122368507904, 140122421547007,
+SNULL, 140122301534207, 140122368507903,
+STORE, 140122301399040, 140122301534207,
+STORE, 140122301534208, 140122368507903,
+SNULL, 140122421551103, 140122429939711,
+STORE, 140122421547008, 140122421551103,
+STORE, 140122421551104, 140122429939711,
+STORE, 140122413154304, 140122421547007,
+SNULL, 140122413158399, 140122421547007,
+STORE, 140122413154304, 140122413158399,
+STORE, 140122413158400, 140122421547007,
+STORE, 140122404761600, 140122413154303,
+SNULL, 140122404765695, 140122413154303,
+STORE, 140122404761600, 140122404765695,
+STORE, 140122404765696, 140122413154303,
+STORE, 140122396368896, 140122404761599,
+SNULL, 140122396372991, 140122404761599,
+STORE, 140122396368896, 140122396372991,
+STORE, 140122396372992, 140122404761599,
+STORE, 140122387976192, 140122396368895,
+STORE, 140122167181312, 140122301399039,
+SNULL, 140122234290175, 140122301399039,
+STORE, 140122167181312, 140122234290175,
+STORE, 140122234290176, 140122301399039,
+ERASE, 140122234290176, 140122301399039,
+SNULL, 140122167316479, 140122234290175,
+STORE, 140122167181312, 140122167316479,
+STORE, 140122167316480, 140122234290175,
+STORE, 140122379583488, 140122396368895,
+STORE, 140122371190784, 140122396368895,
+STORE, 140122167316480, 140122301399039,
+STORE, 140122158788608, 140122167181311,
+SNULL, 140122371190784, 140122387976191,
+STORE, 140122387976192, 140122396368895,
+STORE, 140122371190784, 140122387976191,
+SNULL, 140122387980287, 140122396368895,
+STORE, 140122387976192, 140122387980287,
+STORE, 140122387980288, 140122396368895,
+SNULL, 140122167316480, 140122234290175,
+STORE, 140122234290176, 140122301399039,
+STORE, 140122167316480, 140122234290175,
+SNULL, 140122234425343, 140122301399039,
+STORE, 140122234290176, 140122234425343,
+STORE, 140122234425344, 140122301399039,
+STORE, 140122024570880, 140122158788607,
+SNULL, 140122024570880, 140122032963583,
+STORE, 140122032963584, 140122158788607,
+STORE, 140122024570880, 140122032963583,
+ERASE, 140122024570880, 140122032963583,
+STORE, 140121898745856, 140122158788607,
+STORE, 140121890353152, 140121898745855,
+SNULL, 140122100072447, 140122158788607,
+STORE, 140121898745856, 140122100072447,
+STORE, 140122100072448, 140122158788607,
+ERASE, 140122100072448, 140122158788607,
+SNULL, 140121965854719, 140122100072447,
+STORE, 140121898745856, 140121965854719,
+STORE, 140121965854720, 140122100072447,
+SNULL, 140121965854720, 140122032963583,
+STORE, 140122032963584, 140122100072447,
+STORE, 140121965854720, 140122032963583,
+ERASE, 140121965854720, 140122032963583,
+SNULL, 140121898881023, 140121965854719,
+STORE, 140121898745856, 140121898881023,
+STORE, 140121898881024, 140121965854719,
+SNULL, 140121890357247, 140121898745855,
+STORE, 140121890353152, 140121890357247,
+STORE, 140121890357248, 140121898745855,
+SNULL, 140122371190784, 140122379583487,
+STORE, 140122379583488, 140122387976191,
+STORE, 140122371190784, 140122379583487,
+SNULL, 140122379587583, 140122387976191,
+STORE, 140122379583488, 140122379587583,
+STORE, 140122379587584, 140122387976191,
+SNULL, 140122033098751, 140122100072447,
+STORE, 140122032963584, 140122033098751,
+STORE, 140122033098752, 140122100072447,
+SNULL, 140122158792703, 140122167181311,
+STORE, 140122158788608, 140122158792703,
+STORE, 140122158792704, 140122167181311,
+STORE, 140122150395904, 140122158788607,
+STORE, 140122142003200, 140122158788607,
+SNULL, 140122142007295, 140122158788607,
+STORE, 140122142003200, 140122142007295,
+STORE, 140122142007296, 140122158788607,
+SNULL, 140122371194879, 140122379583487,
+STORE, 140122371190784, 140122371194879,
+STORE, 140122371194880, 140122379583487,
+SNULL, 140122142007296, 140122150395903,
+STORE, 140122150395904, 140122158788607,
+STORE, 140122142007296, 140122150395903,
+SNULL, 140122150399999, 140122158788607,
+STORE, 140122150395904, 140122150399999,
+STORE, 140122150400000, 140122158788607,
+STORE, 140122133610496, 140122142003199,
+STORE, 140122125217792, 140122142003199,
+STORE, 140122116825088, 140122142003199,
+SNULL, 140122116829183, 140122142003199,
+STORE, 140122116825088, 140122116829183,
+STORE, 140122116829184, 140122142003199,
+SNULL, 140122116829184, 140122133610495,
+STORE, 140122133610496, 140122142003199,
+STORE, 140122116829184, 140122133610495,
+SNULL, 140122133614591, 140122142003199,
+STORE, 140122133610496, 140122133614591,
+STORE, 140122133614592, 140122142003199,
+SNULL, 140122116829184, 140122125217791,
+STORE, 140122125217792, 140122133610495,
+STORE, 140122116829184, 140122125217791,
+SNULL, 140122125221887, 140122133610495,
+STORE, 140122125217792, 140122125221887,
+STORE, 140122125221888, 140122133610495,
+STORE, 140122108432384, 140122116825087,
+SNULL, 140122108436479, 140122116825087,
+STORE, 140122108432384, 140122108436479,
+STORE, 140122108436480, 140122116825087,
+STORE, 140122024570880, 140122032963583,
+STORE, 140122016178176, 140122032963583,
+SNULL, 140122016182271, 140122032963583,
+STORE, 140122016178176, 140122016182271,
+STORE, 140122016182272, 140122032963583,
+SNULL, 140122016182272, 140122024570879,
+STORE, 140122024570880, 140122032963583,
+STORE, 140122016182272, 140122024570879,
+SNULL, 140122024574975, 140122032963583,
+STORE, 140122024570880, 140122024574975,
+STORE, 140122024574976, 140122032963583,
+STORE, 140122007785472, 140122016178175,
+SNULL, 140122007789567, 140122016178175,
+STORE, 140122007785472, 140122007789567,
+STORE, 140122007789568, 140122016178175,
+STORE, 140121999392768, 140122007785471,
+STORE, 140121991000064, 140122007785471,
+SNULL, 140121991004159, 140122007785471,
+STORE, 140121991000064, 140121991004159,
+STORE, 140121991004160, 140122007785471,
+SNULL, 140121991004160, 140121999392767,
+STORE, 140121999392768, 140122007785471,
+STORE, 140121991004160, 140121999392767,
+SNULL, 140121999396863, 140122007785471,
+STORE, 140121999392768, 140121999396863,
+STORE, 140121999396864, 140122007785471,
+STORE, 140121982607360, 140121991000063,
+STORE, 140121823244288, 140121890353151,
+ERASE, 140121823244288, 140121890353151,
+STORE, 140121756135424, 140121890353151,
+SNULL, 140121756135424, 140121764528127,
+STORE, 140121764528128, 140121890353151,
+STORE, 140121756135424, 140121764528127,
+ERASE, 140121756135424, 140121764528127,
+SNULL, 140121831636991, 140121890353151,
+STORE, 140121764528128, 140121831636991,
+STORE, 140121831636992, 140121890353151,
+ERASE, 140121831636992, 140121890353151,
+STORE, 140121974214656, 140121991000063,
+STORE, 140121630310400, 140121831636991,
+SNULL, 140121697419263, 140121831636991,
+STORE, 140121630310400, 140121697419263,
+STORE, 140121697419264, 140121831636991,
+SNULL, 140121697419264, 140121764528127,
+STORE, 140121764528128, 140121831636991,
+STORE, 140121697419264, 140121764528127,
+ERASE, 140121697419264, 140121764528127,
+STORE, 140121881960448, 140121890353151,
+STORE, 140121630310400, 140121831636991,
+STORE, 140121873567744, 140121890353151,
+SNULL, 140121630310400, 140121697419263,
+STORE, 140121697419264, 140121831636991,
+STORE, 140121630310400, 140121697419263,
+SNULL, 140121697554431, 140121831636991,
+STORE, 140121697419264, 140121697554431,
+STORE, 140121697554432, 140121831636991,
+STORE, 140121865175040, 140121890353151,
+STORE, 140121856782336, 140121890353151,
+STORE, 140121848389632, 140121890353151,
+STORE, 140121839996928, 140121890353151,
+STORE, 140121496092672, 140121697419263,
+STORE, 140121487699968, 140121496092671,
+STORE, 140121420591104, 140121487699967,
+STORE, 140121412198400, 140121420591103,
+ERASE, 140121420591104, 140121487699967,
+STORE, 140121479307264, 140121496092671,
+STORE, 140121277980672, 140121412198399,
+SNULL, 140121277980672, 140121294766079,
+STORE, 140121294766080, 140121412198399,
+STORE, 140121277980672, 140121294766079,
+ERASE, 140121277980672, 140121294766079,
+STORE, 140121470914560, 140121496092671,
+STORE, 140121462521856, 140121496092671,
+STORE, 140121160548352, 140121412198399,
+STORE, 140121454129152, 140121496092671,
+SNULL, 140121227657215, 140121412198399,
+STORE, 140121160548352, 140121227657215,
+STORE, 140121227657216, 140121412198399,
+SNULL, 140121227657216, 140121294766079,
+STORE, 140121294766080, 140121412198399,
+STORE, 140121227657216, 140121294766079,
+ERASE, 140121227657216, 140121294766079,
+STORE, 140121445736448, 140121496092671,
+STORE, 140121437343744, 140121496092671,
+SNULL, 140121437343744, 140121445736447,
+STORE, 140121445736448, 140121496092671,
+STORE, 140121437343744, 140121445736447,
+SNULL, 140121445740543, 140121496092671,
+STORE, 140121445736448, 140121445740543,
+STORE, 140121445740544, 140121496092671,
+SNULL, 140121697554432, 140121764528127,
+STORE, 140121764528128, 140121831636991,
+STORE, 140121697554432, 140121764528127,
+SNULL, 140121764663295, 140121831636991,
+STORE, 140121764528128, 140121764663295,
+STORE, 140121764663296, 140121831636991,
+SNULL, 140121496092672, 140121630310399,
+STORE, 140121630310400, 140121697419263,
+STORE, 140121496092672, 140121630310399,
+SNULL, 140121630445567, 140121697419263,
+STORE, 140121630310400, 140121630445567,
+STORE, 140121630445568, 140121697419263,
+SNULL, 140121445740544, 140121454129151,
+STORE, 140121454129152, 140121496092671,
+STORE, 140121445740544, 140121454129151,
+SNULL, 140121454133247, 140121496092671,
+STORE, 140121454129152, 140121454133247,
+STORE, 140121454133248, 140121496092671,
+STORE, 140121026330624, 140121227657215,
+SNULL, 140121093439487, 140121227657215,
+STORE, 140121026330624, 140121093439487,
+STORE, 140121093439488, 140121227657215,
+SNULL, 140121093439488, 140121160548351,
+STORE, 140121160548352, 140121227657215,
+STORE, 140121093439488, 140121160548351,
+ERASE, 140121093439488, 140121160548351,
+SNULL, 140121563201535, 140121630310399,
+STORE, 140121496092672, 140121563201535,
+STORE, 140121563201536, 140121630310399,
+ERASE, 140121563201536, 140121630310399,
+STORE, 140120892112896, 140121093439487,
+SNULL, 140120959221759, 140121093439487,
+STORE, 140120892112896, 140120959221759,
+STORE, 140120959221760, 140121093439487,
+SNULL, 140120959221760, 140121026330623,
+STORE, 140121026330624, 140121093439487,
+STORE, 140120959221760, 140121026330623,
+ERASE, 140120959221760, 140121026330623,
+STORE, 140120757895168, 140120959221759,
+SNULL, 140121361874943, 140121412198399,
+STORE, 140121294766080, 140121361874943,
+STORE, 140121361874944, 140121412198399,
+ERASE, 140121361874944, 140121412198399,
+SNULL, 140121294901247, 140121361874943,
+STORE, 140121294766080, 140121294901247,
+STORE, 140121294901248, 140121361874943,
+STORE, 140120623677440, 140120959221759,
+SNULL, 140120690786303, 140120959221759,
+STORE, 140120623677440, 140120690786303,
+STORE, 140120690786304, 140120959221759,
+SNULL, 140120690786304, 140120757895167,
+STORE, 140120757895168, 140120959221759,
+STORE, 140120690786304, 140120757895167,
+ERASE, 140120690786304, 140120757895167,
+SNULL, 140121160683519, 140121227657215,
+STORE, 140121160548352, 140121160683519,
+STORE, 140121160683520, 140121227657215,
+SNULL, 140121974214656, 140121982607359,
+STORE, 140121982607360, 140121991000063,
+STORE, 140121974214656, 140121982607359,
+SNULL, 140121982611455, 140121991000063,
+STORE, 140121982607360, 140121982611455,
+STORE, 140121982611456, 140121991000063,
+SNULL, 140121839996928, 140121873567743,
+STORE, 140121873567744, 140121890353151,
+STORE, 140121839996928, 140121873567743,
+SNULL, 140121873571839, 140121890353151,
+STORE, 140121873567744, 140121873571839,
+STORE, 140121873571840, 140121890353151,
+SNULL, 140121873571840, 140121881960447,
+STORE, 140121881960448, 140121890353151,
+STORE, 140121873571840, 140121881960447,
+SNULL, 140121881964543, 140121890353151,
+STORE, 140121881960448, 140121881964543,
+STORE, 140121881964544, 140121890353151,
+SNULL, 140121840001023, 140121873567743,
+STORE, 140121839996928, 140121840001023,
+STORE, 140121840001024, 140121873567743,
+SNULL, 140121840001024, 140121865175039,
+STORE, 140121865175040, 140121873567743,
+STORE, 140121840001024, 140121865175039,
+SNULL, 140121865179135, 140121873567743,
+STORE, 140121865175040, 140121865179135,
+STORE, 140121865179136, 140121873567743,
+SNULL, 140121437347839, 140121445736447,
+STORE, 140121437343744, 140121437347839,
+STORE, 140121437347840, 140121445736447,
+STORE, 140121621917696, 140121630310399,
+STORE, 140121613524992, 140121630310399,
+SNULL, 140121026465791, 140121093439487,
+STORE, 140121026330624, 140121026465791,
+STORE, 140121026465792, 140121093439487,
+SNULL, 140121496227839, 140121563201535,
+STORE, 140121496092672, 140121496227839,
+STORE, 140121496227840, 140121563201535,
+SNULL, 140120757895168, 140120892112895,
+STORE, 140120892112896, 140120959221759,
+STORE, 140120757895168, 140120892112895,
+SNULL, 140120892248063, 140120959221759,
+STORE, 140120892112896, 140120892248063,
+STORE, 140120892248064, 140120959221759,
+SNULL, 140120825004031, 140120892112895,
+STORE, 140120757895168, 140120825004031,
+STORE, 140120825004032, 140120892112895,
+ERASE, 140120825004032, 140120892112895,
+SNULL, 140120623812607, 140120690786303,
+STORE, 140120623677440, 140120623812607,
+STORE, 140120623812608, 140120690786303,
+SNULL, 140120758030335, 140120825004031,
+STORE, 140120757895168, 140120758030335,
+STORE, 140120758030336, 140120825004031,
+SNULL, 140121454133248, 140121462521855,
+STORE, 140121462521856, 140121496092671,
+STORE, 140121454133248, 140121462521855,
+SNULL, 140121462525951, 140121496092671,
+STORE, 140121462521856, 140121462525951,
+STORE, 140121462525952, 140121496092671,
+STORE, 140121605132288, 140121630310399,
+SNULL, 140121605136383, 140121630310399,
+STORE, 140121605132288, 140121605136383,
+STORE, 140121605136384, 140121630310399,
+STORE, 140121596739584, 140121605132287,
+SNULL, 140121605136384, 140121621917695,
+STORE, 140121621917696, 140121630310399,
+STORE, 140121605136384, 140121621917695,
+SNULL, 140121621921791, 140121630310399,
+STORE, 140121621917696, 140121621921791,
+STORE, 140121621921792, 140121630310399,
+STORE, 140121588346880, 140121605132287,
+STORE, 140121579954176, 140121605132287,
+SNULL, 140121412202495, 140121420591103,
+STORE, 140121412198400, 140121412202495,
+STORE, 140121412202496, 140121420591103,
+SNULL, 140121974218751, 140121982607359,
+STORE, 140121974214656, 140121974218751,
+STORE, 140121974218752, 140121982607359,
+SNULL, 140121462525952, 140121479307263,
+STORE, 140121479307264, 140121496092671,
+STORE, 140121462525952, 140121479307263,
+SNULL, 140121479311359, 140121496092671,
+STORE, 140121479307264, 140121479311359,
+STORE, 140121479311360, 140121496092671,
+STORE, 140121571561472, 140121605132287,
+SNULL, 140121571565567, 140121605132287,
+STORE, 140121571561472, 140121571565567,
+STORE, 140121571565568, 140121605132287,
+STORE, 140121428951040, 140121437343743,
+SNULL, 140121428955135, 140121437343743,
+STORE, 140121428951040, 140121428955135,
+STORE, 140121428955136, 140121437343743,
+SNULL, 140121840001024, 140121856782335,
+STORE, 140121856782336, 140121865175039,
+STORE, 140121840001024, 140121856782335,
+SNULL, 140121856786431, 140121865175039,
+STORE, 140121856782336, 140121856786431,
+STORE, 140121856786432, 140121865175039,
+STORE, 140121403805696, 140121412198399,
+SNULL, 140121840001024, 140121848389631,
+STORE, 140121848389632, 140121856782335,
+STORE, 140121840001024, 140121848389631,
+SNULL, 140121848393727, 140121856782335,
+STORE, 140121848389632, 140121848393727,
+STORE, 140121848393728, 140121856782335,
+SNULL, 140121479311360, 140121487699967,
+STORE, 140121487699968, 140121496092671,
+STORE, 140121479311360, 140121487699967,
+SNULL, 140121487704063, 140121496092671,
+STORE, 140121487699968, 140121487704063,
+STORE, 140121487704064, 140121496092671,
+STORE, 140121395412992, 140121412198399,
+STORE, 140121387020288, 140121412198399,
+SNULL, 140121387024383, 140121412198399,
+STORE, 140121387020288, 140121387024383,
+STORE, 140121387024384, 140121412198399,
+SNULL, 140121605136384, 140121613524991,
+STORE, 140121613524992, 140121621917695,
+STORE, 140121605136384, 140121613524991,
+SNULL, 140121613529087, 140121621917695,
+STORE, 140121613524992, 140121613529087,
+STORE, 140121613529088, 140121621917695,
+SNULL, 140121462525952, 140121470914559,
+STORE, 140121470914560, 140121479307263,
+STORE, 140121462525952, 140121470914559,
+SNULL, 140121470918655, 140121479307263,
+STORE, 140121470914560, 140121470918655,
+STORE, 140121470918656, 140121479307263,
+STORE, 140121378627584, 140121387020287,
+SNULL, 140121378631679, 140121387020287,
+STORE, 140121378627584, 140121378631679,
+STORE, 140121378631680, 140121387020287,
+SNULL, 140121571565568, 140121596739583,
+STORE, 140121596739584, 140121605132287,
+STORE, 140121571565568, 140121596739583,
+SNULL, 140121596743679, 140121605132287,
+STORE, 140121596739584, 140121596743679,
+STORE, 140121596743680, 140121605132287,
+SNULL, 140121387024384, 140121403805695,
+STORE, 140121403805696, 140121412198399,
+STORE, 140121387024384, 140121403805695,
+SNULL, 140121403809791, 140121412198399,
+STORE, 140121403805696, 140121403809791,
+STORE, 140121403809792, 140121412198399,
+STORE, 140121370234880, 140121378627583,
+SNULL, 140121387024384, 140121395412991,
+STORE, 140121395412992, 140121403805695,
+STORE, 140121387024384, 140121395412991,
+SNULL, 140121395417087, 140121403805695,
+STORE, 140121395412992, 140121395417087,
+STORE, 140121395417088, 140121403805695,
+SNULL, 140121571565568, 140121588346879,
+STORE, 140121588346880, 140121596739583,
+STORE, 140121571565568, 140121588346879,
+SNULL, 140121588350975, 140121596739583,
+STORE, 140121588346880, 140121588350975,
+STORE, 140121588350976, 140121596739583,
+SNULL, 140121571565568, 140121579954175,
+STORE, 140121579954176, 140121588346879,
+STORE, 140121571565568, 140121579954175,
+SNULL, 140121579958271, 140121588346879,
+STORE, 140121579954176, 140121579958271,
+STORE, 140121579958272, 140121588346879,
+STORE, 140121286373376, 140121294766079,
+STORE, 140121277980672, 140121294766079,
+SNULL, 140121277980672, 140121286373375,
+STORE, 140121286373376, 140121294766079,
+STORE, 140121277980672, 140121286373375,
+SNULL, 140121286377471, 140121294766079,
+STORE, 140121286373376, 140121286377471,
+STORE, 140121286377472, 140121294766079,
+STORE, 140121269587968, 140121286373375,
+STORE, 140121261195264, 140121286373375,
+SNULL, 140121261195264, 140121269587967,
+STORE, 140121269587968, 140121286373375,
+STORE, 140121261195264, 140121269587967,
+SNULL, 140121269592063, 140121286373375,
+STORE, 140121269587968, 140121269592063,
+STORE, 140121269592064, 140121286373375,
+STORE, 140121252802560, 140121269587967,
+SNULL, 140121252806655, 140121269587967,
+STORE, 140121252802560, 140121252806655,
+STORE, 140121252806656, 140121269587967,
+STORE, 140121244409856, 140121252802559,
+STORE, 140121236017152, 140121252802559,
+SNULL, 140121236017152, 140121244409855,
+STORE, 140121244409856, 140121252802559,
+STORE, 140121236017152, 140121244409855,
+SNULL, 140121244413951, 140121252802559,
+STORE, 140121244409856, 140121244413951,
+STORE, 140121244413952, 140121252802559,
+SNULL, 140121370238975, 140121378627583,
+STORE, 140121370234880, 140121370238975,
+STORE, 140121370238976, 140121378627583,
+STORE, 140121152155648, 140121160548351,
+STORE, 140121143762944, 140121160548351,
+STORE, 140121135370240, 140121160548351,
+SNULL, 140121135374335, 140121160548351,
+STORE, 140121135370240, 140121135374335,
+STORE, 140121135374336, 140121160548351,
+STORE, 140121126977536, 140121135370239,
+STORE, 140121118584832, 140121135370239,
+STORE, 140121110192128, 140121135370239,
+SNULL, 140121110192128, 140121118584831,
+STORE, 140121118584832, 140121135370239,
+STORE, 140121110192128, 140121118584831,
+SNULL, 140121118588927, 140121135370239,
+STORE, 140121118584832, 140121118588927,
+STORE, 140121118588928, 140121135370239,
+STORE, 140121101799424, 140121118584831,
+STORE, 140121017937920, 140121026330623,
+STORE, 140121009545216, 140121026330623,
+SNULL, 140121009545216, 140121017937919,
+STORE, 140121017937920, 140121026330623,
+STORE, 140121009545216, 140121017937919,
+SNULL, 140121017942015, 140121026330623,
+STORE, 140121017937920, 140121017942015,
+STORE, 140121017942016, 140121026330623,
+SNULL, 140121269592064, 140121277980671,
+STORE, 140121277980672, 140121286373375,
+STORE, 140121269592064, 140121277980671,
+SNULL, 140121277984767, 140121286373375,
+STORE, 140121277980672, 140121277984767,
+STORE, 140121277984768, 140121286373375,
+STORE, 140121001152512, 140121017937919,
+SNULL, 140121252806656, 140121261195263,
+STORE, 140121261195264, 140121269587967,
+STORE, 140121252806656, 140121261195263,
+SNULL, 140121261199359, 140121269587967,
+STORE, 140121261195264, 140121261199359,
+STORE, 140121261199360, 140121269587967,
+SNULL, 140121135374336, 140121152155647,
+STORE, 140121152155648, 140121160548351,
+STORE, 140121135374336, 140121152155647,
+SNULL, 140121152159743, 140121160548351,
+STORE, 140121152155648, 140121152159743,
+STORE, 140121152159744, 140121160548351,
+STORE, 140120992759808, 140121017937919,
+STORE, 140120984367104, 140121017937919,
+STORE, 140120975974400, 140121017937919,
+SNULL, 140121101799424, 140121110192127,
+STORE, 140121110192128, 140121118584831,
+STORE, 140121101799424, 140121110192127,
+SNULL, 140121110196223, 140121118584831,
+STORE, 140121110192128, 140121110196223,
+STORE, 140121110196224, 140121118584831,
+SNULL, 140121118588928, 140121126977535,
+STORE, 140121126977536, 140121135370239,
+STORE, 140121118588928, 140121126977535,
+SNULL, 140121126981631, 140121135370239,
+STORE, 140121126977536, 140121126981631,
+STORE, 140121126981632, 140121135370239,
+STORE, 140120967581696, 140121017937919,
+STORE, 140120883720192, 140120892112895,
+SNULL, 140120883724287, 140120892112895,
+STORE, 140120883720192, 140120883724287,
+STORE, 140120883724288, 140120892112895,
+STORE, 140120875327488, 140120883720191,
+SNULL, 140121101803519, 140121110192127,
+STORE, 140121101799424, 140121101803519,
+STORE, 140121101803520, 140121110192127,
+SNULL, 140121135374336, 140121143762943,
+STORE, 140121143762944, 140121152155647,
+STORE, 140121135374336, 140121143762943,
+SNULL, 140121143767039, 140121152155647,
+STORE, 140121143762944, 140121143767039,
+STORE, 140121143767040, 140121152155647,
+STORE, 140120866934784, 140120883720191,
+SNULL, 140120967581696, 140120984367103,
+STORE, 140120984367104, 140121017937919,
+STORE, 140120967581696, 140120984367103,
+SNULL, 140120984371199, 140121017937919,
+STORE, 140120984367104, 140120984371199,
+STORE, 140120984371200, 140121017937919,
+STORE, 140120858542080, 140120883720191,
+SNULL, 140121236021247, 140121244409855,
+STORE, 140121236017152, 140121236021247,
+STORE, 140121236021248, 140121244409855,
+SNULL, 140120984371200, 140121009545215,
+STORE, 140121009545216, 140121017937919,
+STORE, 140120984371200, 140121009545215,
+SNULL, 140121009549311, 140121017937919,
+STORE, 140121009545216, 140121009549311,
+STORE, 140121009549312, 140121017937919,
+SNULL, 140120984371200, 140120992759807,
+STORE, 140120992759808, 140121009545215,
+STORE, 140120984371200, 140120992759807,
+SNULL, 140120992763903, 140121009545215,
+STORE, 140120992759808, 140120992763903,
+STORE, 140120992763904, 140121009545215,
+SNULL, 140120992763904, 140121001152511,
+STORE, 140121001152512, 140121009545215,
+STORE, 140120992763904, 140121001152511,
+SNULL, 140121001156607, 140121009545215,
+STORE, 140121001152512, 140121001156607,
+STORE, 140121001156608, 140121009545215,
+STORE, 140120850149376, 140120883720191,
+SNULL, 140120850153471, 140120883720191,
+STORE, 140120850149376, 140120850153471,
+STORE, 140120850153472, 140120883720191,
+SNULL, 140120967585791, 140120984367103,
+STORE, 140120967581696, 140120967585791,
+STORE, 140120967585792, 140120984367103,
+SNULL, 140120850153472, 140120866934783,
+STORE, 140120866934784, 140120883720191,
+STORE, 140120850153472, 140120866934783,
+SNULL, 140120866938879, 140120883720191,
+STORE, 140120866934784, 140120866938879,
+STORE, 140120866938880, 140120883720191,
+STORE, 140120841756672, 140120850149375,
+SNULL, 140120967585792, 140120975974399,
+STORE, 140120975974400, 140120984367103,
+STORE, 140120967585792, 140120975974399,
+SNULL, 140120975978495, 140120984367103,
+STORE, 140120975974400, 140120975978495,
+STORE, 140120975978496, 140120984367103,
+SNULL, 140120866938880, 140120875327487,
+STORE, 140120875327488, 140120883720191,
+STORE, 140120866938880, 140120875327487,
+SNULL, 140120875331583, 140120883720191,
+STORE, 140120875327488, 140120875331583,
+STORE, 140120875331584, 140120883720191,
+STORE, 140120833363968, 140120850149375,
+STORE, 140120749502464, 140120757895167,
+STORE, 140120741109760, 140120757895167,
+STORE, 140120732717056, 140120757895167,
+STORE, 140120724324352, 140120757895167,
+SNULL, 140120724324352, 140120732717055,
+STORE, 140120732717056, 140120757895167,
+STORE, 140120724324352, 140120732717055,
+SNULL, 140120732721151, 140120757895167,
+STORE, 140120732717056, 140120732721151,
+STORE, 140120732721152, 140120757895167,
+STORE, 140120715931648, 140120732717055,
+SNULL, 140120715935743, 140120732717055,
+STORE, 140120715931648, 140120715935743,
+STORE, 140120715935744, 140120732717055,
+SNULL, 140120850153472, 140120858542079,
+STORE, 140120858542080, 140120866934783,
+STORE, 140120850153472, 140120858542079,
+SNULL, 140120858546175, 140120866934783,
+STORE, 140120858542080, 140120858546175,
+STORE, 140120858546176, 140120866934783,
+STORE, 140120707538944, 140120715931647,
+SNULL, 140120707543039, 140120715931647,
+STORE, 140120707538944, 140120707543039,
+STORE, 140120707543040, 140120715931647,
+SNULL, 140120833368063, 140120850149375,
+STORE, 140120833363968, 140120833368063,
+STORE, 140120833368064, 140120850149375,
+SNULL, 140120833368064, 140120841756671,
+STORE, 140120841756672, 140120850149375,
+STORE, 140120833368064, 140120841756671,
+SNULL, 140120841760767, 140120850149375,
+STORE, 140120841756672, 140120841760767,
+STORE, 140120841760768, 140120850149375,
+STORE, 140120699146240, 140120707538943,
+SNULL, 140120715935744, 140120724324351,
+STORE, 140120724324352, 140120732717055,
+STORE, 140120715935744, 140120724324351,
+SNULL, 140120724328447, 140120732717055,
+STORE, 140120724324352, 140120724328447,
+STORE, 140120724328448, 140120732717055,
+SNULL, 140120732721152, 140120741109759,
+STORE, 140120741109760, 140120757895167,
+STORE, 140120732721152, 140120741109759,
+SNULL, 140120741113855, 140120757895167,
+STORE, 140120741109760, 140120741113855,
+STORE, 140120741113856, 140120757895167,
+SNULL, 140120741113856, 140120749502463,
+STORE, 140120749502464, 140120757895167,
+STORE, 140120741113856, 140120749502463,
+SNULL, 140120749506559, 140120757895167,
+STORE, 140120749502464, 140120749506559,
+STORE, 140120749506560, 140120757895167,
+SNULL, 140120699150335, 140120707538943,
+STORE, 140120699146240, 140120699150335,
+STORE, 140120699150336, 140120707538943,
+STORE, 140122446557184, 140122446585855,
+STORE, 140122368999424, 140122371190783,
+SNULL, 140122368999424, 140122369089535,
+STORE, 140122369089536, 140122371190783,
+STORE, 140122368999424, 140122369089535,
+SNULL, 140122371182591, 140122371190783,
+STORE, 140122369089536, 140122371182591,
+STORE, 140122371182592, 140122371190783,
+ERASE, 140122371182592, 140122371190783,
+STORE, 140122371182592, 140122371190783,
+SNULL, 140122371186687, 140122371190783,
+STORE, 140122371182592, 140122371186687,
+STORE, 140122371186688, 140122371190783,
+ERASE, 140122446557184, 140122446585855,
+ERASE, 140121445736448, 140121445740543,
+ERASE, 140121445740544, 140121454129151,
+ERASE, 140121621917696, 140121621921791,
+ERASE, 140121621921792, 140121630310399,
+ERASE, 140121579954176, 140121579958271,
+ERASE, 140121579958272, 140121588346879,
+ERASE, 140121261195264, 140121261199359,
+ERASE, 140121261199360, 140121269587967,
+ERASE, 140121454129152, 140121454133247,
+ERASE, 140121454133248, 140121462521855,
+ERASE, 140121588346880, 140121588350975,
+ERASE, 140121588350976, 140121596739583,
+ERASE, 140121135370240, 140121135374335,
+ERASE, 140121135374336, 140121143762943,
+ERASE, 140121881960448, 140121881964543,
+ERASE, 140121881964544, 140121890353151,
+ERASE, 140121428951040, 140121428955135,
+ERASE, 140121428955136, 140121437343743,
+ERASE, 140121387020288, 140121387024383,
+ERASE, 140121387024384, 140121395412991,
+ERASE, 140121487699968, 140121487704063,
+ERASE, 140121487704064, 140121496092671,
+ERASE, 140121437343744, 140121437347839,
+ERASE, 140121437347840, 140121445736447,
+ERASE, 140121613524992, 140121613529087,
+ERASE, 140121613529088, 140121621917695,
+ERASE, 140121856782336, 140121856786431,
+ERASE, 140121856786432, 140121865175039,
+ERASE, 140121252802560, 140121252806655,
+ERASE, 140121252806656, 140121261195263,
+ERASE, 140121839996928, 140121840001023,
+ERASE, 140121840001024, 140121848389631,
+ERASE, 140121596739584, 140121596743679,
+ERASE, 140121596743680, 140121605132287,
+ERASE, 140121009545216, 140121009549311,
+ERASE, 140121009549312, 140121017937919,
+ERASE, 140120724324352, 140120724328447,
+ERASE, 140120724328448, 140120732717055,
+ERASE, 140120883720192, 140120883724287,
+ERASE, 140120883724288, 140120892112895,
+ERASE, 140121982607360, 140121982611455,
+ERASE, 140121982611456, 140121991000063,
+ERASE, 140121571561472, 140121571565567,
+ERASE, 140121571565568, 140121579954175,
+ERASE, 140121286373376, 140121286377471,
+ERASE, 140121286377472, 140121294766079,
+ERASE, 140120875327488, 140120875331583,
+ERASE, 140120875331584, 140120883720191,
+ERASE, 140121848389632, 140121848393727,
+ERASE, 140121848393728, 140121856782335,
+ERASE, 140121370234880, 140121370238975,
+ERASE, 140121370238976, 140121378627583,
+ERASE, 140121143762944, 140121143767039,
+ERASE, 140121143767040, 140121152155647,
+ERASE, 140121118584832, 140121118588927,
+ERASE, 140121118588928, 140121126977535,
+ERASE, 140120866934784, 140120866938879,
+ERASE, 140120866938880, 140120875327487,
+ERASE, 140120741109760, 140120741113855,
+ERASE, 140120741113856, 140120749502463,
+ERASE, 140121865175040, 140121865179135,
+ERASE, 140121865179136, 140121873567743,
+ERASE, 140121403805696, 140121403809791,
+ERASE, 140121403809792, 140121412198399,
+ERASE, 140121236017152, 140121236021247,
+ERASE, 140121236021248, 140121244409855,
+ERASE, 140120732717056, 140120732721151,
+ERASE, 140120732721152, 140120741109759,
+ERASE, 140121017937920, 140121017942015,
+ERASE, 140121017942016, 140121026330623,
+ERASE, 140121873567744, 140121873571839,
+ERASE, 140121873571840, 140121881960447,
+ERASE, 140121470914560, 140121470918655,
+ERASE, 140121470918656, 140121479307263,
+ERASE, 140121126977536, 140121126981631,
+ERASE, 140121126981632, 140121135370239,
+ERASE, 140120850149376, 140120850153471,
+ERASE, 140120850153472, 140120858542079,
+ERASE, 140120707538944, 140120707543039,
+ERASE, 140120707543040, 140120715931647,
+ERASE, 140121479307264, 140121479311359,
+ERASE, 140121479311360, 140121487699967,
+ERASE, 140120967581696, 140120967585791,
+ERASE, 140120967585792, 140120975974399,
+ERASE, 140120841756672, 140120841760767,
+ERASE, 140120841760768, 140120850149375,
+ERASE, 140121412198400, 140121412202495,
+ERASE, 140121412202496, 140121420591103,
+ERASE, 140122158788608, 140122158792703,
+ERASE, 140122158792704, 140122167181311,
+ERASE, 140122142003200, 140122142007295,
+ERASE, 140122142007296, 140122150395903,
+ERASE, 140121101799424, 140121101803519,
+ERASE, 140121101803520, 140121110192127,
+ERASE, 140120858542080, 140120858546175,
+ERASE, 140120858546176, 140120866934783,
+ERASE, 140120833363968, 140120833368063,
+ERASE, 140120833368064, 140120841756671,
+ERASE, 140121277980672, 140121277984767,
+ERASE, 140121277984768, 140121286373375,
+ERASE, 140121001152512, 140121001156607,
+ERASE, 140121001156608, 140121009545215,
+ERASE, 140120749502464, 140120749506559,
+ERASE, 140120749506560, 140120757895167,
+ERASE, 140121605132288, 140121605136383,
+ERASE, 140121605136384, 140121613524991,
+ERASE, 140121378627584, 140121378631679,
+ERASE, 140121378631680, 140121387020287,
+ERASE, 140121110192128, 140121110196223,
+ERASE, 140121110196224, 140121118584831,
+ERASE, 140121462521856, 140121462525951,
+ERASE, 140121462525952, 140121470914559,
+ERASE, 140121395412992, 140121395417087,
+ERASE, 140121395417088, 140121403805695,
+ERASE, 140121152155648, 140121152159743,
+ERASE, 140121152159744, 140121160548351,
+ERASE, 140120992759808, 140120992763903,
+ERASE, 140120992763904, 140121001152511,
+ERASE, 140122387976192, 140122387980287,
+ERASE, 140122387980288, 140122396368895,
+ERASE, 140121890353152, 140121890357247,
+ERASE, 140121890357248, 140121898745855,
+ERASE, 140121269587968, 140121269592063,
+ERASE, 140121269592064, 140121277980671,
+ };
+ unsigned long set37[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140722404016128, 140737488351231,
+SNULL, 140722404020223, 140737488351231,
+STORE, 140722404016128, 140722404020223,
+STORE, 140722403885056, 140722404020223,
+STORE, 94637010001920, 94637012254719,
+SNULL, 94637010132991, 94637012254719,
+STORE, 94637010001920, 94637010132991,
+STORE, 94637010132992, 94637012254719,
+ERASE, 94637010132992, 94637012254719,
+STORE, 94637012226048, 94637012234239,
+STORE, 94637012234240, 94637012254719,
+STORE, 139760240594944, 139760242847743,
+SNULL, 139760240738303, 139760242847743,
+STORE, 139760240594944, 139760240738303,
+STORE, 139760240738304, 139760242847743,
+ERASE, 139760240738304, 139760242847743,
+STORE, 139760242835456, 139760242843647,
+STORE, 139760242843648, 139760242847743,
+STORE, 140722405232640, 140722405236735,
+STORE, 140722405220352, 140722405232639,
+STORE, 139760242806784, 139760242835455,
+STORE, 139760242798592, 139760242806783,
+STORE, 139760238379008, 139760240594943,
+SNULL, 139760238379008, 139760238477311,
+STORE, 139760238477312, 139760240594943,
+STORE, 139760238379008, 139760238477311,
+SNULL, 139760240570367, 139760240594943,
+STORE, 139760238477312, 139760240570367,
+STORE, 139760240570368, 139760240594943,
+SNULL, 139760240570368, 139760240578559,
+STORE, 139760240578560, 139760240594943,
+STORE, 139760240570368, 139760240578559,
+ERASE, 139760240570368, 139760240578559,
+STORE, 139760240570368, 139760240578559,
+ERASE, 139760240578560, 139760240594943,
+STORE, 139760240578560, 139760240594943,
+STORE, 139760234582016, 139760238379007,
+SNULL, 139760234582016, 139760236240895,
+STORE, 139760236240896, 139760238379007,
+STORE, 139760234582016, 139760236240895,
+SNULL, 139760238338047, 139760238379007,
+STORE, 139760236240896, 139760238338047,
+STORE, 139760238338048, 139760238379007,
+SNULL, 139760238338048, 139760238362623,
+STORE, 139760238362624, 139760238379007,
+STORE, 139760238338048, 139760238362623,
+ERASE, 139760238338048, 139760238362623,
+STORE, 139760238338048, 139760238362623,
+ERASE, 139760238362624, 139760238379007,
+STORE, 139760238362624, 139760238379007,
+STORE, 139760242790400, 139760242806783,
+SNULL, 139760238354431, 139760238362623,
+STORE, 139760238338048, 139760238354431,
+STORE, 139760238354432, 139760238362623,
+SNULL, 139760240574463, 139760240578559,
+STORE, 139760240570368, 139760240574463,
+STORE, 139760240574464, 139760240578559,
+SNULL, 94637012230143, 94637012234239,
+STORE, 94637012226048, 94637012230143,
+STORE, 94637012230144, 94637012234239,
+SNULL, 139760242839551, 139760242843647,
+STORE, 139760242835456, 139760242839551,
+STORE, 139760242839552, 139760242843647,
+ERASE, 139760242806784, 139760242835455,
+STORE, 94637033324544, 94637033459711,
+STORE, 139760226189312, 139760234582015,
+SNULL, 139760226193407, 139760234582015,
+STORE, 139760226189312, 139760226193407,
+STORE, 139760226193408, 139760234582015,
+STORE, 139760217796608, 139760226189311,
+STORE, 139760083578880, 139760217796607,
+SNULL, 139760083578880, 139760114860031,
+STORE, 139760114860032, 139760217796607,
+STORE, 139760083578880, 139760114860031,
+ERASE, 139760083578880, 139760114860031,
+SNULL, 139760181968895, 139760217796607,
+STORE, 139760114860032, 139760181968895,
+STORE, 139760181968896, 139760217796607,
+ERASE, 139760181968896, 139760217796607,
+SNULL, 139760114995199, 139760181968895,
+STORE, 139760114860032, 139760114995199,
+STORE, 139760114995200, 139760181968895,
+SNULL, 139760217800703, 139760226189311,
+STORE, 139760217796608, 139760217800703,
+STORE, 139760217800704, 139760226189311,
+STORE, 139760209403904, 139760217796607,
+SNULL, 139760209407999, 139760217796607,
+STORE, 139760209403904, 139760209407999,
+STORE, 139760209408000, 139760217796607,
+STORE, 139760201011200, 139760209403903,
+SNULL, 139760201015295, 139760209403903,
+STORE, 139760201011200, 139760201015295,
+STORE, 139760201015296, 139760209403903,
+STORE, 139760192618496, 139760201011199,
+SNULL, 139760192622591, 139760201011199,
+STORE, 139760192618496, 139760192622591,
+STORE, 139760192622592, 139760201011199,
+STORE, 139760184225792, 139760192618495,
+STORE, 139759980642304, 139760114860031,
+STORE, 139759972249600, 139759980642303,
+STORE, 139759963856896, 139759980642303,
+STORE, 139759955464192, 139759980642303,
+STORE, 139759888355328, 139759955464191,
+SNULL, 139760047751167, 139760114860031,
+STORE, 139759980642304, 139760047751167,
+STORE, 139760047751168, 139760114860031,
+ERASE, 139760047751168, 139760114860031,
+SNULL, 139759980777471, 139760047751167,
+STORE, 139759980642304, 139759980777471,
+STORE, 139759980777472, 139760047751167,
+STORE, 139759980777472, 139760114860031,
+SNULL, 139759980777472, 139760047751167,
+STORE, 139760047751168, 139760114860031,
+STORE, 139759980777472, 139760047751167,
+SNULL, 139760047886335, 139760114860031,
+STORE, 139760047751168, 139760047886335,
+STORE, 139760047886336, 139760114860031,
+STORE, 139759821246464, 139759955464191,
+SNULL, 139759821246464, 139759888355327,
+STORE, 139759888355328, 139759955464191,
+STORE, 139759821246464, 139759888355327,
+ERASE, 139759821246464, 139759888355327,
+ERASE, 139759888355328, 139759955464191,
+ };
+ unsigned long set38[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140730666221568, 140737488351231,
+SNULL, 140730666225663, 140737488351231,
+STORE, 140730666221568, 140730666225663,
+STORE, 140730666090496, 140730666225663,
+STORE, 94177584803840, 94177587056639,
+SNULL, 94177584934911, 94177587056639,
+STORE, 94177584803840, 94177584934911,
+STORE, 94177584934912, 94177587056639,
+ERASE, 94177584934912, 94177587056639,
+STORE, 94177587027968, 94177587036159,
+STORE, 94177587036160, 94177587056639,
+STORE, 140614382714880, 140614384967679,
+SNULL, 140614382858239, 140614384967679,
+STORE, 140614382714880, 140614382858239,
+STORE, 140614382858240, 140614384967679,
+ERASE, 140614382858240, 140614384967679,
+STORE, 140614384955392, 140614384963583,
+STORE, 140614384963584, 140614384967679,
+STORE, 140730666315776, 140730666319871,
+STORE, 140730666303488, 140730666315775,
+STORE, 140614384926720, 140614384955391,
+STORE, 140614384918528, 140614384926719,
+STORE, 140614380498944, 140614382714879,
+SNULL, 140614380498944, 140614380597247,
+STORE, 140614380597248, 140614382714879,
+STORE, 140614380498944, 140614380597247,
+SNULL, 140614382690303, 140614382714879,
+STORE, 140614380597248, 140614382690303,
+STORE, 140614382690304, 140614382714879,
+SNULL, 140614382690304, 140614382698495,
+STORE, 140614382698496, 140614382714879,
+STORE, 140614382690304, 140614382698495,
+ERASE, 140614382690304, 140614382698495,
+STORE, 140614382690304, 140614382698495,
+ERASE, 140614382698496, 140614382714879,
+STORE, 140614382698496, 140614382714879,
+STORE, 140614376701952, 140614380498943,
+SNULL, 140614376701952, 140614378360831,
+STORE, 140614378360832, 140614380498943,
+STORE, 140614376701952, 140614378360831,
+SNULL, 140614380457983, 140614380498943,
+STORE, 140614378360832, 140614380457983,
+STORE, 140614380457984, 140614380498943,
+SNULL, 140614380457984, 140614380482559,
+STORE, 140614380482560, 140614380498943,
+STORE, 140614380457984, 140614380482559,
+ERASE, 140614380457984, 140614380482559,
+STORE, 140614380457984, 140614380482559,
+ERASE, 140614380482560, 140614380498943,
+STORE, 140614380482560, 140614380498943,
+STORE, 140614384910336, 140614384926719,
+SNULL, 140614380474367, 140614380482559,
+STORE, 140614380457984, 140614380474367,
+STORE, 140614380474368, 140614380482559,
+SNULL, 140614382694399, 140614382698495,
+STORE, 140614382690304, 140614382694399,
+STORE, 140614382694400, 140614382698495,
+SNULL, 94177587032063, 94177587036159,
+STORE, 94177587027968, 94177587032063,
+STORE, 94177587032064, 94177587036159,
+SNULL, 140614384959487, 140614384963583,
+STORE, 140614384955392, 140614384959487,
+STORE, 140614384959488, 140614384963583,
+ERASE, 140614384926720, 140614384955391,
+STORE, 94177619791872, 94177619927039,
+STORE, 140614368309248, 140614376701951,
+SNULL, 140614368313343, 140614376701951,
+STORE, 140614368309248, 140614368313343,
+STORE, 140614368313344, 140614376701951,
+STORE, 140614359916544, 140614368309247,
+STORE, 140614225698816, 140614359916543,
+SNULL, 140614225698816, 140614276481023,
+STORE, 140614276481024, 140614359916543,
+STORE, 140614225698816, 140614276481023,
+ERASE, 140614225698816, 140614276481023,
+SNULL, 140614343589887, 140614359916543,
+STORE, 140614276481024, 140614343589887,
+STORE, 140614343589888, 140614359916543,
+ERASE, 140614343589888, 140614359916543,
+SNULL, 140614276616191, 140614343589887,
+STORE, 140614276481024, 140614276616191,
+STORE, 140614276616192, 140614343589887,
+SNULL, 140614359920639, 140614368309247,
+STORE, 140614359916544, 140614359920639,
+STORE, 140614359920640, 140614368309247,
+STORE, 140614351523840, 140614359916543,
+SNULL, 140614351527935, 140614359916543,
+STORE, 140614351523840, 140614351527935,
+STORE, 140614351527936, 140614359916543,
+STORE, 140614268088320, 140614276481023,
+SNULL, 140614268092415, 140614276481023,
+STORE, 140614268088320, 140614268092415,
+STORE, 140614268092416, 140614276481023,
+STORE, 140614259695616, 140614268088319,
+SNULL, 140614259699711, 140614268088319,
+STORE, 140614259695616, 140614259699711,
+STORE, 140614259699712, 140614268088319,
+STORE, 140614251302912, 140614259695615,
+STORE, 140614242910208, 140614259695615,
+STORE, 140614108692480, 140614242910207,
+SNULL, 140614108692480, 140614142263295,
+STORE, 140614142263296, 140614242910207,
+STORE, 140614108692480, 140614142263295,
+ERASE, 140614108692480, 140614142263295,
+STORE, 140614133870592, 140614142263295,
+STORE, 140613999652864, 140614133870591,
+SNULL, 140613999652864, 140614008045567,
+STORE, 140614008045568, 140614133870591,
+STORE, 140613999652864, 140614008045567,
+ERASE, 140613999652864, 140614008045567,
+STORE, 140613999652864, 140614008045567,
+STORE, 140613865435136, 140613999652863,
+SNULL, 140613865435136, 140613873827839,
+STORE, 140613873827840, 140613999652863,
+STORE, 140613865435136, 140613873827839,
+ERASE, 140613865435136, 140613873827839,
+SNULL, 140614209372159, 140614242910207,
+STORE, 140614142263296, 140614209372159,
+STORE, 140614209372160, 140614242910207,
+ERASE, 140614209372160, 140614242910207,
+SNULL, 140614142398463, 140614209372159,
+STORE, 140614142263296, 140614142398463,
+STORE, 140614142398464, 140614209372159,
+SNULL, 140614075154431, 140614133870591,
+STORE, 140614008045568, 140614075154431,
+STORE, 140614075154432, 140614133870591,
+ERASE, 140614075154432, 140614133870591,
+SNULL, 140614008180735, 140614075154431,
+STORE, 140614008045568, 140614008180735,
+STORE, 140614008180736, 140614075154431,
+SNULL, 140613940936703, 140613999652863,
+STORE, 140613873827840, 140613940936703,
+STORE, 140613940936704, 140613999652863,
+ERASE, 140613940936704, 140613999652863,
+SNULL, 140614242914303, 140614259695615,
+STORE, 140614242910208, 140614242914303,
+STORE, 140614242914304, 140614259695615,
+STORE, 140613739610112, 140613940936703,
+STORE, 140614234517504, 140614242910207,
+SNULL, 140614242914304, 140614251302911,
+STORE, 140614251302912, 140614259695615,
+STORE, 140614242914304, 140614251302911,
+SNULL, 140614251307007, 140614259695615,
+STORE, 140614251302912, 140614251307007,
+STORE, 140614251307008, 140614259695615,
+SNULL, 140613739610112, 140613873827839,
+STORE, 140613873827840, 140613940936703,
+STORE, 140613739610112, 140613873827839,
+SNULL, 140613873963007, 140613940936703,
+STORE, 140613873827840, 140613873963007,
+STORE, 140613873963008, 140613940936703,
+SNULL, 140614133874687, 140614142263295,
+STORE, 140614133870592, 140614133874687,
+STORE, 140614133874688, 140614142263295,
+SNULL, 140613806718975, 140613873827839,
+STORE, 140613739610112, 140613806718975,
+STORE, 140613806718976, 140613873827839,
+ERASE, 140613806718976, 140613873827839,
+STORE, 140614226124800, 140614242910207,
+SNULL, 140613739745279, 140613806718975,
+STORE, 140613739610112, 140613739745279,
+STORE, 140613739745280, 140613806718975,
+SNULL, 140613999656959, 140614008045567,
+STORE, 140613999652864, 140613999656959,
+STORE, 140613999656960, 140614008045567,
+SNULL, 140614226124800, 140614234517503,
+STORE, 140614234517504, 140614242910207,
+STORE, 140614226124800, 140614234517503,
+SNULL, 140614234521599, 140614242910207,
+STORE, 140614234517504, 140614234521599,
+STORE, 140614234521600, 140614242910207,
+STORE, 140614217732096, 140614234517503,
+STORE, 140614125477888, 140614133870591,
+SNULL, 140614125481983, 140614133870591,
+STORE, 140614125477888, 140614125481983,
+STORE, 140614125481984, 140614133870591,
+STORE, 140614117085184, 140614125477887,
+SNULL, 140614217736191, 140614234517503,
+STORE, 140614217732096, 140614217736191,
+STORE, 140614217736192, 140614234517503,
+SNULL, 140614117089279, 140614125477887,
+STORE, 140614117085184, 140614117089279,
+STORE, 140614117089280, 140614125477887,
+SNULL, 140614217736192, 140614226124799,
+STORE, 140614226124800, 140614234517503,
+STORE, 140614217736192, 140614226124799,
+SNULL, 140614226128895, 140614234517503,
+STORE, 140614226124800, 140614226128895,
+STORE, 140614226128896, 140614234517503,
+STORE, 140614108692480, 140614117085183,
+STORE, 140614100299776, 140614117085183,
+STORE, 140614091907072, 140614117085183,
+SNULL, 140614091907072, 140614108692479,
+STORE, 140614108692480, 140614117085183,
+STORE, 140614091907072, 140614108692479,
+SNULL, 140614108696575, 140614117085183,
+STORE, 140614108692480, 140614108696575,
+STORE, 140614108696576, 140614117085183,
+SNULL, 140614091907072, 140614100299775,
+STORE, 140614100299776, 140614108692479,
+STORE, 140614091907072, 140614100299775,
+SNULL, 140614100303871, 140614108692479,
+STORE, 140614100299776, 140614100303871,
+STORE, 140614100303872, 140614108692479,
+STORE, 140614083514368, 140614100299775,
+SNULL, 140614083518463, 140614100299775,
+STORE, 140614083514368, 140614083518463,
+STORE, 140614083518464, 140614100299775,
+STORE, 140613991260160, 140613999652863,
+SNULL, 140614083518464, 140614091907071,
+STORE, 140614091907072, 140614100299775,
+STORE, 140614083518464, 140614091907071,
+SNULL, 140614091911167, 140614100299775,
+STORE, 140614091907072, 140614091911167,
+STORE, 140614091911168, 140614100299775,
+SNULL, 140613991264255, 140613999652863,
+STORE, 140613991260160, 140613991264255,
+STORE, 140613991264256, 140613999652863,
+STORE, 140613982867456, 140613991260159,
+SNULL, 140613982871551, 140613991260159,
+STORE, 140613982867456, 140613982871551,
+STORE, 140613982871552, 140613991260159,
+STORE, 140613974474752, 140613982867455,
+SNULL, 140613974478847, 140613982867455,
+STORE, 140613974474752, 140613974478847,
+STORE, 140613974478848, 140613982867455,
+STORE, 140613966082048, 140613974474751,
+STORE, 140613739745280, 140613873827839,
+SNULL, 140613739745280, 140613806718975,
+STORE, 140613806718976, 140613873827839,
+STORE, 140613739745280, 140613806718975,
+SNULL, 140613806854143, 140613873827839,
+STORE, 140613806718976, 140613806854143,
+STORE, 140613806854144, 140613873827839,
+SNULL, 140613966086143, 140613974474751,
+STORE, 140613966082048, 140613966086143,
+STORE, 140613966086144, 140613974474751,
+STORE, 140613957689344, 140613966082047,
+STORE, 140613605392384, 140613739610111,
+STORE, 140613949296640, 140613966082047,
+STORE, 140613596999680, 140613605392383,
+STORE, 140613529890816, 140613596999679,
+STORE, 140613521498112, 140613529890815,
+STORE, 140613513105408, 140613529890815,
+STORE, 140613378887680, 140613513105407,
+SNULL, 140613378887680, 140613404065791,
+STORE, 140613404065792, 140613513105407,
+STORE, 140613378887680, 140613404065791,
+ERASE, 140613378887680, 140613404065791,
+STORE, 140613395673088, 140613404065791,
+STORE, 140613261455360, 140613395673087,
+SNULL, 140613261455360, 140613269848063,
+STORE, 140613269848064, 140613395673087,
+STORE, 140613261455360, 140613269848063,
+ERASE, 140613261455360, 140613269848063,
+STORE, 140613261455360, 140613269848063,
+STORE, 140613253062656, 140613269848063,
+STORE, 140613118844928, 140613253062655,
+STORE, 140613110452224, 140613118844927,
+SNULL, 140613118844928, 140613135630335,
+STORE, 140613135630336, 140613253062655,
+STORE, 140613118844928, 140613135630335,
+ERASE, 140613118844928, 140613135630335,
+STORE, 140613127237632, 140613135630335,
+STORE, 140613110452224, 140613135630335,
+STORE, 140612976234496, 140613110452223,
+STORE, 140612967841792, 140612976234495,
+STORE, 140612833624064, 140612967841791,
+STORE, 140612825231360, 140612833624063,
+STORE, 140612816838656, 140612833624063,
+STORE, 140612682620928, 140612816838655,
+STORE, 140612674228224, 140612682620927,
+SNULL, 140612682620928, 140612732977151,
+STORE, 140612732977152, 140612816838655,
+STORE, 140612682620928, 140612732977151,
+ERASE, 140612682620928, 140612732977151,
+SNULL, 140613672501247, 140613739610111,
+STORE, 140613605392384, 140613672501247,
+STORE, 140613672501248, 140613739610111,
+ERASE, 140613672501248, 140613739610111,
+SNULL, 140613605527551, 140613672501247,
+STORE, 140613605392384, 140613605527551,
+STORE, 140613605527552, 140613672501247,
+ERASE, 140613529890816, 140613596999679,
+STORE, 140612540010496, 140612674228223,
+SNULL, 140612540010496, 140612598759423,
+STORE, 140612598759424, 140612674228223,
+STORE, 140612540010496, 140612598759423,
+ERASE, 140612540010496, 140612598759423,
+SNULL, 140613471174655, 140613513105407,
+STORE, 140613404065792, 140613471174655,
+STORE, 140613471174656, 140613513105407,
+ERASE, 140613471174656, 140613513105407,
+SNULL, 140613404200959, 140613471174655,
+STORE, 140613404065792, 140613404200959,
+STORE, 140613404200960, 140613471174655,
+SNULL, 140613336956927, 140613395673087,
+STORE, 140613269848064, 140613336956927,
+STORE, 140613336956928, 140613395673087,
+ERASE, 140613336956928, 140613395673087,
+SNULL, 140612833624064, 140612867194879,
+STORE, 140612867194880, 140612967841791,
+STORE, 140612833624064, 140612867194879,
+ERASE, 140612833624064, 140612867194879,
+SNULL, 140612976234496, 140613001412607,
+STORE, 140613001412608, 140613110452223,
+STORE, 140612976234496, 140613001412607,
+ERASE, 140612976234496, 140613001412607,
+SNULL, 140613202739199, 140613253062655,
+STORE, 140613135630336, 140613202739199,
+STORE, 140613202739200, 140613253062655,
+ERASE, 140613202739200, 140613253062655,
+SNULL, 140613135765503, 140613202739199,
+STORE, 140613135630336, 140613135765503,
+STORE, 140613135765504, 140613202739199,
+SNULL, 140612816842751, 140612833624063,
+STORE, 140612816838656, 140612816842751,
+STORE, 140612816842752, 140612833624063,
+SNULL, 140613110456319, 140613135630335,
+STORE, 140613110452224, 140613110456319,
+STORE, 140613110456320, 140613135630335,
+SNULL, 140613949300735, 140613966082047,
+STORE, 140613949296640, 140613949300735,
+STORE, 140613949300736, 140613966082047,
+SNULL, 140613110456320, 140613118844927,
+STORE, 140613118844928, 140613135630335,
+STORE, 140613110456320, 140613118844927,
+SNULL, 140613118849023, 140613135630335,
+STORE, 140613118844928, 140613118849023,
+STORE, 140613118849024, 140613135630335,
+SNULL, 140612800086015, 140612816838655,
+STORE, 140612732977152, 140612800086015,
+STORE, 140612800086016, 140612816838655,
+ERASE, 140612800086016, 140612816838655,
+SNULL, 140613253062656, 140613261455359,
+STORE, 140613261455360, 140613269848063,
+STORE, 140613253062656, 140613261455359,
+SNULL, 140613261459455, 140613269848063,
+STORE, 140613261455360, 140613261459455,
+STORE, 140613261459456, 140613269848063,
+SNULL, 140612674232319, 140612682620927,
+STORE, 140612674228224, 140612674232319,
+STORE, 140612674232320, 140612682620927,
+STORE, 140613731217408, 140613739610111,
+STORE, 140613722824704, 140613739610111,
+SNULL, 140613949300736, 140613957689343,
+STORE, 140613957689344, 140613966082047,
+STORE, 140613949300736, 140613957689343,
+SNULL, 140613957693439, 140613966082047,
+STORE, 140613957689344, 140613957693439,
+STORE, 140613957693440, 140613966082047,
+STORE, 140612464541696, 140612674228223,
+SNULL, 140612531650559, 140612674228223,
+STORE, 140612464541696, 140612531650559,
+STORE, 140612531650560, 140612674228223,
+SNULL, 140612531650560, 140612598759423,
+STORE, 140612598759424, 140612674228223,
+STORE, 140612531650560, 140612598759423,
+ERASE, 140612531650560, 140612598759423,
+SNULL, 140612665868287, 140612674228223,
+STORE, 140612598759424, 140612665868287,
+STORE, 140612665868288, 140612674228223,
+ERASE, 140612665868288, 140612674228223,
+SNULL, 140613269983231, 140613336956927,
+STORE, 140613269848064, 140613269983231,
+STORE, 140613269983232, 140613336956927,
+SNULL, 140612934303743, 140612967841791,
+STORE, 140612867194880, 140612934303743,
+STORE, 140612934303744, 140612967841791,
+ERASE, 140612934303744, 140612967841791,
+SNULL, 140613068521471, 140613110452223,
+STORE, 140613001412608, 140613068521471,
+STORE, 140613068521472, 140613110452223,
+ERASE, 140613068521472, 140613110452223,
+STORE, 140613714432000, 140613739610111,
+SNULL, 140613001547775, 140613068521471,
+STORE, 140613001412608, 140613001547775,
+STORE, 140613001547776, 140613068521471,
+SNULL, 140612733112319, 140612800086015,
+STORE, 140612732977152, 140612733112319,
+STORE, 140612733112320, 140612800086015,
+SNULL, 140613513109503, 140613529890815,
+STORE, 140613513105408, 140613513109503,
+STORE, 140613513109504, 140613529890815,
+STORE, 140613706039296, 140613739610111,
+STORE, 140613697646592, 140613739610111,
+STORE, 140613689253888, 140613739610111,
+SNULL, 140613689257983, 140613739610111,
+STORE, 140613689253888, 140613689257983,
+STORE, 140613689257984, 140613739610111,
+SNULL, 140613253066751, 140613261455359,
+STORE, 140613253062656, 140613253066751,
+STORE, 140613253066752, 140613261455359,
+STORE, 140613680861184, 140613689253887,
+STORE, 140613588606976, 140613605392383,
+SNULL, 140613689257984, 140613731217407,
+STORE, 140613731217408, 140613739610111,
+STORE, 140613689257984, 140613731217407,
+SNULL, 140613731221503, 140613739610111,
+STORE, 140613731217408, 140613731221503,
+STORE, 140613731221504, 140613739610111,
+STORE, 140613580214272, 140613605392383,
+SNULL, 140612464676863, 140612531650559,
+STORE, 140612464541696, 140612464676863,
+STORE, 140612464676864, 140612531650559,
+SNULL, 140612598894591, 140612665868287,
+STORE, 140612598759424, 140612598894591,
+STORE, 140612598894592, 140612665868287,
+SNULL, 140612867330047, 140612934303743,
+STORE, 140612867194880, 140612867330047,
+STORE, 140612867330048, 140612934303743,
+STORE, 140613571821568, 140613605392383,
+SNULL, 140613571825663, 140613605392383,
+STORE, 140613571821568, 140613571825663,
+STORE, 140613571825664, 140613605392383,
+SNULL, 140613689257984, 140613722824703,
+STORE, 140613722824704, 140613731217407,
+STORE, 140613689257984, 140613722824703,
+SNULL, 140613722828799, 140613731217407,
+STORE, 140613722824704, 140613722828799,
+STORE, 140613722828800, 140613731217407,
+SNULL, 140613689257984, 140613714431999,
+STORE, 140613714432000, 140613722824703,
+STORE, 140613689257984, 140613714431999,
+SNULL, 140613714436095, 140613722824703,
+STORE, 140613714432000, 140613714436095,
+STORE, 140613714436096, 140613722824703,
+SNULL, 140612816842752, 140612825231359,
+STORE, 140612825231360, 140612833624063,
+STORE, 140612816842752, 140612825231359,
+SNULL, 140612825235455, 140612833624063,
+STORE, 140612825231360, 140612825235455,
+STORE, 140612825235456, 140612833624063,
+SNULL, 140613395677183, 140613404065791,
+STORE, 140613395673088, 140613395677183,
+STORE, 140613395677184, 140613404065791,
+SNULL, 140613689257984, 140613706039295,
+STORE, 140613706039296, 140613714431999,
+STORE, 140613689257984, 140613706039295,
+SNULL, 140613706043391, 140613714431999,
+STORE, 140613706039296, 140613706043391,
+STORE, 140613706043392, 140613714431999,
+SNULL, 140613118849024, 140613127237631,
+STORE, 140613127237632, 140613135630335,
+STORE, 140613118849024, 140613127237631,
+SNULL, 140613127241727, 140613135630335,
+STORE, 140613127237632, 140613127241727,
+STORE, 140613127241728, 140613135630335,
+SNULL, 140613571825664, 140613580214271,
+STORE, 140613580214272, 140613605392383,
+STORE, 140613571825664, 140613580214271,
+SNULL, 140613580218367, 140613605392383,
+STORE, 140613580214272, 140613580218367,
+STORE, 140613580218368, 140613605392383,
+SNULL, 140613689257984, 140613697646591,
+STORE, 140613697646592, 140613706039295,
+STORE, 140613689257984, 140613697646591,
+SNULL, 140613697650687, 140613706039295,
+STORE, 140613697646592, 140613697650687,
+STORE, 140613697650688, 140613706039295,
+SNULL, 140613680865279, 140613689253887,
+STORE, 140613680861184, 140613680865279,
+STORE, 140613680865280, 140613689253887,
+STORE, 140613563428864, 140613571821567,
+SNULL, 140613563432959, 140613571821567,
+STORE, 140613563428864, 140613563432959,
+STORE, 140613563432960, 140613571821567,
+SNULL, 140613580218368, 140613588606975,
+STORE, 140613588606976, 140613605392383,
+STORE, 140613580218368, 140613588606975,
+SNULL, 140613588611071, 140613605392383,
+STORE, 140613588606976, 140613588611071,
+STORE, 140613588611072, 140613605392383,
+SNULL, 140613513109504, 140613521498111,
+STORE, 140613521498112, 140613529890815,
+STORE, 140613513109504, 140613521498111,
+SNULL, 140613521502207, 140613529890815,
+STORE, 140613521498112, 140613521502207,
+STORE, 140613521502208, 140613529890815,
+SNULL, 140613588611072, 140613596999679,
+STORE, 140613596999680, 140613605392383,
+STORE, 140613588611072, 140613596999679,
+SNULL, 140613597003775, 140613605392383,
+STORE, 140613596999680, 140613597003775,
+STORE, 140613597003776, 140613605392383,
+STORE, 140613555036160, 140613563428863,
+SNULL, 140613555040255, 140613563428863,
+STORE, 140613555036160, 140613555040255,
+STORE, 140613555040256, 140613563428863,
+STORE, 140613546643456, 140613555036159,
+STORE, 140613538250752, 140613555036159,
+SNULL, 140613538250752, 140613546643455,
+STORE, 140613546643456, 140613555036159,
+STORE, 140613538250752, 140613546643455,
+SNULL, 140613546647551, 140613555036159,
+STORE, 140613546643456, 140613546647551,
+STORE, 140613546647552, 140613555036159,
+STORE, 140613504712704, 140613513105407,
+STORE, 140613496320000, 140613513105407,
+SNULL, 140613496324095, 140613513105407,
+STORE, 140613496320000, 140613496324095,
+STORE, 140613496324096, 140613513105407,
+STORE, 140613487927296, 140613496319999,
+SNULL, 140613487931391, 140613496319999,
+STORE, 140613487927296, 140613487931391,
+STORE, 140613487931392, 140613496319999,
+STORE, 140613479534592, 140613487927295,
+SNULL, 140612967845887, 140612976234495,
+STORE, 140612967841792, 140612967845887,
+STORE, 140612967845888, 140612976234495,
+STORE, 140613387280384, 140613395673087,
+STORE, 140613378887680, 140613395673087,
+SNULL, 140613378887680, 140613387280383,
+STORE, 140613387280384, 140613395673087,
+STORE, 140613378887680, 140613387280383,
+SNULL, 140613387284479, 140613395673087,
+STORE, 140613387280384, 140613387284479,
+STORE, 140613387284480, 140613395673087,
+STORE, 140613370494976, 140613387280383,
+STORE, 140613362102272, 140613387280383,
+SNULL, 140613479538687, 140613487927295,
+STORE, 140613479534592, 140613479538687,
+STORE, 140613479538688, 140613487927295,
+STORE, 140613353709568, 140613387280383,
+STORE, 140613345316864, 140613387280383,
+STORE, 140613244669952, 140613253062655,
+SNULL, 140613345320959, 140613387280383,
+STORE, 140613345316864, 140613345320959,
+STORE, 140613345320960, 140613387280383,
+SNULL, 140613538254847, 140613546643455,
+STORE, 140613538250752, 140613538254847,
+STORE, 140613538254848, 140613546643455,
+STORE, 140613236277248, 140613253062655,
+STORE, 140613227884544, 140613253062655,
+STORE, 140613219491840, 140613253062655,
+STORE, 140613211099136, 140613253062655,
+SNULL, 140613211103231, 140613253062655,
+STORE, 140613211099136, 140613211103231,
+STORE, 140613211103232, 140613253062655,
+STORE, 140613102059520, 140613110452223,
+STORE, 140613093666816, 140613110452223,
+SNULL, 140613093670911, 140613110452223,
+STORE, 140613093666816, 140613093670911,
+STORE, 140613093670912, 140613110452223,
+STORE, 140613085274112, 140613093666815,
+SNULL, 140613496324096, 140613504712703,
+STORE, 140613504712704, 140613513105407,
+STORE, 140613496324096, 140613504712703,
+SNULL, 140613504716799, 140613513105407,
+STORE, 140613504712704, 140613504716799,
+STORE, 140613504716800, 140613513105407,
+SNULL, 140613345320960, 140613378887679,
+STORE, 140613378887680, 140613387280383,
+STORE, 140613345320960, 140613378887679,
+SNULL, 140613378891775, 140613387280383,
+STORE, 140613378887680, 140613378891775,
+STORE, 140613378891776, 140613387280383,
+SNULL, 140613345320960, 140613362102271,
+STORE, 140613362102272, 140613378887679,
+STORE, 140613345320960, 140613362102271,
+SNULL, 140613362106367, 140613378887679,
+STORE, 140613362102272, 140613362106367,
+STORE, 140613362106368, 140613378887679,
+SNULL, 140613362106368, 140613370494975,
+STORE, 140613370494976, 140613378887679,
+STORE, 140613362106368, 140613370494975,
+SNULL, 140613370499071, 140613378887679,
+STORE, 140613370494976, 140613370499071,
+STORE, 140613370499072, 140613378887679,
+STORE, 140613076881408, 140613093666815,
+STORE, 140612993019904, 140613001412607,
+SNULL, 140613076885503, 140613093666815,
+STORE, 140613076881408, 140613076885503,
+STORE, 140613076885504, 140613093666815,
+SNULL, 140613093670912, 140613102059519,
+STORE, 140613102059520, 140613110452223,
+STORE, 140613093670912, 140613102059519,
+SNULL, 140613102063615, 140613110452223,
+STORE, 140613102059520, 140613102063615,
+STORE, 140613102063616, 140613110452223,
+SNULL, 140613076885504, 140613085274111,
+STORE, 140613085274112, 140613093666815,
+STORE, 140613076885504, 140613085274111,
+SNULL, 140613085278207, 140613093666815,
+STORE, 140613085274112, 140613085278207,
+STORE, 140613085278208, 140613093666815,
+STORE, 140612984627200, 140613001412607,
+STORE, 140612967845888, 140612984627199,
+SNULL, 140613211103232, 140613219491839,
+STORE, 140613219491840, 140613253062655,
+STORE, 140613211103232, 140613219491839,
+SNULL, 140613219495935, 140613253062655,
+STORE, 140613219491840, 140613219495935,
+STORE, 140613219495936, 140613253062655,
+STORE, 140612959449088, 140612967841791,
+STORE, 140612951056384, 140612967841791,
+SNULL, 140612951060479, 140612967841791,
+STORE, 140612951056384, 140612951060479,
+STORE, 140612951060480, 140612967841791,
+SNULL, 140613345320960, 140613353709567,
+STORE, 140613353709568, 140613362102271,
+STORE, 140613345320960, 140613353709567,
+SNULL, 140613353713663, 140613362102271,
+STORE, 140613353709568, 140613353713663,
+STORE, 140613353713664, 140613362102271,
+SNULL, 140613219495936, 140613244669951,
+STORE, 140613244669952, 140613253062655,
+STORE, 140613219495936, 140613244669951,
+SNULL, 140613244674047, 140613253062655,
+STORE, 140613244669952, 140613244674047,
+STORE, 140613244674048, 140613253062655,
+STORE, 140612942663680, 140612951056383,
+SNULL, 140613219495936, 140613236277247,
+STORE, 140613236277248, 140613244669951,
+STORE, 140613219495936, 140613236277247,
+SNULL, 140613236281343, 140613244669951,
+STORE, 140613236277248, 140613236281343,
+STORE, 140613236281344, 140613244669951,
+SNULL, 140613219495936, 140613227884543,
+STORE, 140613227884544, 140613236277247,
+STORE, 140613219495936, 140613227884543,
+SNULL, 140613227888639, 140613236277247,
+STORE, 140613227884544, 140613227888639,
+STORE, 140613227888640, 140613236277247,
+SNULL, 140612984627200, 140612993019903,
+STORE, 140612993019904, 140613001412607,
+STORE, 140612984627200, 140612993019903,
+SNULL, 140612993023999, 140613001412607,
+STORE, 140612993019904, 140612993023999,
+STORE, 140612993024000, 140613001412607,
+STORE, 140612858802176, 140612867194879,
+STORE, 140612850409472, 140612867194879,
+SNULL, 140612951060480, 140612959449087,
+STORE, 140612959449088, 140612967841791,
+STORE, 140612951060480, 140612959449087,
+SNULL, 140612959453183, 140612967841791,
+STORE, 140612959449088, 140612959453183,
+STORE, 140612959453184, 140612967841791,
+SNULL, 140612967845888, 140612976234495,
+STORE, 140612976234496, 140612984627199,
+STORE, 140612967845888, 140612976234495,
+SNULL, 140612976238591, 140612984627199,
+STORE, 140612976234496, 140612976238591,
+STORE, 140612976238592, 140612984627199,
+STORE, 140612842016768, 140612867194879,
+SNULL, 140612842020863, 140612867194879,
+STORE, 140612842016768, 140612842020863,
+STORE, 140612842020864, 140612867194879,
+SNULL, 140612984631295, 140612993019903,
+STORE, 140612984627200, 140612984631295,
+STORE, 140612984631296, 140612993019903,
+STORE, 140612825235456, 140612842016767,
+STORE, 140612808445952, 140612816838655,
+SNULL, 140612942667775, 140612951056383,
+STORE, 140612942663680, 140612942667775,
+STORE, 140612942667776, 140612951056383,
+STORE, 140612724584448, 140612732977151,
+SNULL, 140612724588543, 140612732977151,
+STORE, 140612724584448, 140612724588543,
+STORE, 140612724588544, 140612732977151,
+STORE, 140612716191744, 140612724584447,
+SNULL, 140612842020864, 140612850409471,
+STORE, 140612850409472, 140612867194879,
+STORE, 140612842020864, 140612850409471,
+SNULL, 140612850413567, 140612867194879,
+STORE, 140612850409472, 140612850413567,
+STORE, 140612850413568, 140612867194879,
+SNULL, 140612850413568, 140612858802175,
+STORE, 140612858802176, 140612867194879,
+STORE, 140612850413568, 140612858802175,
+SNULL, 140612858806271, 140612867194879,
+STORE, 140612858802176, 140612858806271,
+STORE, 140612858806272, 140612867194879,
+STORE, 140612707799040, 140612724584447,
+SNULL, 140612707803135, 140612724584447,
+STORE, 140612707799040, 140612707803135,
+STORE, 140612707803136, 140612724584447,
+SNULL, 140612707803136, 140612716191743,
+STORE, 140612716191744, 140612724584447,
+STORE, 140612707803136, 140612716191743,
+SNULL, 140612716195839, 140612724584447,
+STORE, 140612716191744, 140612716195839,
+STORE, 140612716195840, 140612724584447,
+SNULL, 140612808450047, 140612816838655,
+STORE, 140612808445952, 140612808450047,
+STORE, 140612808450048, 140612816838655,
+SNULL, 140612825235456, 140612833624063,
+STORE, 140612833624064, 140612842016767,
+STORE, 140612825235456, 140612833624063,
+SNULL, 140612833628159, 140612842016767,
+STORE, 140612833624064, 140612833628159,
+STORE, 140612833628160, 140612842016767,
+STORE, 140612699406336, 140612707799039,
+SNULL, 140612699410431, 140612707799039,
+STORE, 140612699406336, 140612699410431,
+STORE, 140612699410432, 140612707799039,
+STORE, 140614384926720, 140614384955391,
+STORE, 140614349332480, 140614351523839,
+SNULL, 140614349332480, 140614349422591,
+STORE, 140614349422592, 140614351523839,
+STORE, 140614349332480, 140614349422591,
+SNULL, 140614351515647, 140614351523839,
+STORE, 140614349422592, 140614351515647,
+STORE, 140614351515648, 140614351523839,
+ERASE, 140614351515648, 140614351523839,
+STORE, 140614351515648, 140614351523839,
+SNULL, 140614351519743, 140614351523839,
+STORE, 140614351515648, 140614351519743,
+STORE, 140614351519744, 140614351523839,
+ERASE, 140614384926720, 140614384955391,
+ERASE, 140613949296640, 140613949300735,
+ERASE, 140613949300736, 140613957689343,
+ERASE, 140613689253888, 140613689257983,
+ERASE, 140613689257984, 140613697646591,
+ERASE, 140613563428864, 140613563432959,
+ERASE, 140613563432960, 140613571821567,
+ERASE, 140613211099136, 140613211103231,
+ERASE, 140613211103232, 140613219491839,
+ERASE, 140614133870592, 140614133874687,
+ERASE, 140614133874688, 140614142263295,
+ERASE, 140612967841792, 140612967845887,
+ERASE, 140612967845888, 140612976234495,
+ERASE, 140613076881408, 140613076885503,
+ERASE, 140613076885504, 140613085274111,
+ERASE, 140612850409472, 140612850413567,
+ERASE, 140612850413568, 140612858802175,
+ERASE, 140613110452224, 140613110456319,
+ERASE, 140613110456320, 140613118844927,
+ERASE, 140613706039296, 140613706043391,
+ERASE, 140613706043392, 140613714431999,
+ERASE, 140613521498112, 140613521502207,
+ERASE, 140613521502208, 140613529890815,
+ERASE, 140613362102272, 140613362106367,
+ERASE, 140613362106368, 140613370494975,
+ERASE, 140613253062656, 140613253066751,
+ERASE, 140613253066752, 140613261455359,
+ERASE, 140612816838656, 140612816842751,
+ERASE, 140612816842752, 140612825231359,
+ERASE, 140613261455360, 140613261459455,
+ERASE, 140613261459456, 140613269848063,
+ERASE, 140613118844928, 140613118849023,
+ERASE, 140613118849024, 140613127237631,
+ERASE, 140613714432000, 140613714436095,
+ERASE, 140613714436096, 140613722824703,
+ERASE, 140613496320000, 140613496324095,
+ERASE, 140613496324096, 140613504712703,
+ERASE, 140613513105408, 140613513109503,
+ERASE, 140613513109504, 140613521498111,
+ERASE, 140613697646592, 140613697650687,
+ERASE, 140613697650688, 140613706039295,
+ERASE, 140613093666816, 140613093670911,
+ERASE, 140613093670912, 140613102059519,
+ERASE, 140612993019904, 140612993023999,
+ERASE, 140612993024000, 140613001412607,
+ERASE, 140613127237632, 140613127241727,
+ERASE, 140613127241728, 140613135630335,
+ERASE, 140613957689344, 140613957693439,
+ERASE, 140613957693440, 140613966082047,
+ERASE, 140613571821568, 140613571825663,
+ERASE, 140613571825664, 140613580214271,
+ERASE, 140613479534592, 140613479538687,
+ERASE, 140613479538688, 140613487927295,
+ERASE, 140612984627200, 140612984631295,
+ERASE, 140612984631296, 140612993019903,
+ERASE, 140613588606976, 140613588611071,
+ERASE, 140613588611072, 140613596999679,
+ERASE, 140613680861184, 140613680865279,
+ERASE, 140613680865280, 140613689253887,
+ERASE, 140613345316864, 140613345320959,
+ERASE, 140613345320960, 140613353709567,
+ERASE, 140613596999680, 140613597003775,
+ERASE, 140613597003776, 140613605392383,
+ERASE, 140613966082048, 140613966086143,
+ERASE, 140613966086144, 140613974474751,
+ERASE, 140613731217408, 140613731221503,
+ERASE, 140613731221504, 140613739610111,
+ERASE, 140613395673088, 140613395677183,
+ERASE, 140613395677184, 140613404065791,
+ERASE, 140612825231360, 140612825235455,
+ERASE, 140612825235456, 140612833624063,
+ERASE, 140612674228224, 140612674232319,
+ERASE, 140612674232320, 140612682620927,
+ERASE, 140613722824704, 140613722828799,
+ERASE, 140613722828800, 140613731217407,
+ERASE, 140613487927296, 140613487931391,
+ERASE, 140613487931392, 140613496319999,
+ERASE, 140613102059520, 140613102063615,
+ERASE, 140613102063616, 140613110452223,
+ERASE, 140614242910208, 140614242914303,
+ERASE, 140614242914304, 140614251302911,
+ERASE, 140612808445952, 140612808450047,
+ERASE, 140612808450048, 140612816838655,
+ERASE, 140613236277248, 140613236281343,
+ERASE, 140613236281344, 140613244669951,
+ERASE, 140613580214272, 140613580218367,
+ERASE, 140613580218368, 140613588606975,
+ERASE, 140613370494976, 140613370499071,
+ERASE, 140613370499072, 140613378887679,
+ERASE, 140613244669952, 140613244674047,
+ERASE, 140613244674048, 140613253062655,
+ERASE, 140612724584448, 140612724588543,
+ERASE, 140612724588544, 140612732977151,
+ERASE, 140612707799040, 140612707803135,
+ERASE, 140612707803136, 140612716191743,
+ERASE, 140613504712704, 140613504716799,
+ERASE, 140613504716800, 140613513105407,
+ };
+
+ unsigned long set39[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140736271417344, 140737488351231,
+SNULL, 140736271421439, 140737488351231,
+STORE, 140736271417344, 140736271421439,
+STORE, 140736271286272, 140736271421439,
+STORE, 94412930822144, 94412933074943,
+SNULL, 94412930953215, 94412933074943,
+STORE, 94412930822144, 94412930953215,
+STORE, 94412930953216, 94412933074943,
+ERASE, 94412930953216, 94412933074943,
+STORE, 94412933046272, 94412933054463,
+STORE, 94412933054464, 94412933074943,
+STORE, 140326136901632, 140326139154431,
+SNULL, 140326137044991, 140326139154431,
+STORE, 140326136901632, 140326137044991,
+STORE, 140326137044992, 140326139154431,
+ERASE, 140326137044992, 140326139154431,
+STORE, 140326139142144, 140326139150335,
+STORE, 140326139150336, 140326139154431,
+STORE, 140736271585280, 140736271589375,
+STORE, 140736271572992, 140736271585279,
+STORE, 140326139113472, 140326139142143,
+STORE, 140326139105280, 140326139113471,
+STORE, 140326134685696, 140326136901631,
+SNULL, 140326134685696, 140326134783999,
+STORE, 140326134784000, 140326136901631,
+STORE, 140326134685696, 140326134783999,
+SNULL, 140326136877055, 140326136901631,
+STORE, 140326134784000, 140326136877055,
+STORE, 140326136877056, 140326136901631,
+SNULL, 140326136877056, 140326136885247,
+STORE, 140326136885248, 140326136901631,
+STORE, 140326136877056, 140326136885247,
+ERASE, 140326136877056, 140326136885247,
+STORE, 140326136877056, 140326136885247,
+ERASE, 140326136885248, 140326136901631,
+STORE, 140326136885248, 140326136901631,
+STORE, 140326130888704, 140326134685695,
+SNULL, 140326130888704, 140326132547583,
+STORE, 140326132547584, 140326134685695,
+STORE, 140326130888704, 140326132547583,
+SNULL, 140326134644735, 140326134685695,
+STORE, 140326132547584, 140326134644735,
+STORE, 140326134644736, 140326134685695,
+SNULL, 140326134644736, 140326134669311,
+STORE, 140326134669312, 140326134685695,
+STORE, 140326134644736, 140326134669311,
+ERASE, 140326134644736, 140326134669311,
+STORE, 140326134644736, 140326134669311,
+ERASE, 140326134669312, 140326134685695,
+STORE, 140326134669312, 140326134685695,
+STORE, 140326139097088, 140326139113471,
+SNULL, 140326134661119, 140326134669311,
+STORE, 140326134644736, 140326134661119,
+STORE, 140326134661120, 140326134669311,
+SNULL, 140326136881151, 140326136885247,
+STORE, 140326136877056, 140326136881151,
+STORE, 140326136881152, 140326136885247,
+SNULL, 94412933050367, 94412933054463,
+STORE, 94412933046272, 94412933050367,
+STORE, 94412933050368, 94412933054463,
+SNULL, 140326139146239, 140326139150335,
+STORE, 140326139142144, 140326139146239,
+STORE, 140326139146240, 140326139150335,
+ERASE, 140326139113472, 140326139142143,
+STORE, 94412939493376, 94412939628543,
+STORE, 140326122496000, 140326130888703,
+SNULL, 140326122500095, 140326130888703,
+STORE, 140326122496000, 140326122500095,
+STORE, 140326122500096, 140326130888703,
+STORE, 140326114103296, 140326122495999,
+STORE, 140325979885568, 140326114103295,
+SNULL, 140325979885568, 140326043910143,
+STORE, 140326043910144, 140326114103295,
+STORE, 140325979885568, 140326043910143,
+ERASE, 140325979885568, 140326043910143,
+SNULL, 140326111019007, 140326114103295,
+STORE, 140326043910144, 140326111019007,
+STORE, 140326111019008, 140326114103295,
+ERASE, 140326111019008, 140326114103295,
+SNULL, 140326044045311, 140326111019007,
+STORE, 140326043910144, 140326044045311,
+STORE, 140326044045312, 140326111019007,
+SNULL, 140326114107391, 140326122495999,
+STORE, 140326114103296, 140326114107391,
+STORE, 140326114107392, 140326122495999,
+STORE, 140326035517440, 140326043910143,
+SNULL, 140326035521535, 140326043910143,
+STORE, 140326035517440, 140326035521535,
+STORE, 140326035521536, 140326043910143,
+STORE, 140326027124736, 140326035517439,
+SNULL, 140326027128831, 140326035517439,
+STORE, 140326027124736, 140326027128831,
+STORE, 140326027128832, 140326035517439,
+STORE, 140326018732032, 140326027124735,
+SNULL, 140326018736127, 140326027124735,
+STORE, 140326018732032, 140326018736127,
+STORE, 140326018736128, 140326027124735,
+STORE, 140326010339328, 140326018732031,
+STORE, 140326001946624, 140326018732031,
+STORE, 140325993553920, 140326018732031,
+STORE, 140325859336192, 140325993553919,
+SNULL, 140325859336192, 140325909692415,
+STORE, 140325909692416, 140325993553919,
+STORE, 140325859336192, 140325909692415,
+ERASE, 140325859336192, 140325909692415,
+SNULL, 140325976801279, 140325993553919,
+STORE, 140325909692416, 140325976801279,
+STORE, 140325976801280, 140325993553919,
+ERASE, 140325976801280, 140325993553919,
+STORE, 140325985161216, 140326018732031,
+STORE, 140325775474688, 140325976801279,
+STORE, 140325708365824, 140325976801279,
+SNULL, 140325708500991, 140325976801279,
+STORE, 140325708365824, 140325708500991,
+STORE, 140325708500992, 140325976801279,
+SNULL, 140325708500992, 140325909692415,
+STORE, 140325909692416, 140325976801279,
+STORE, 140325708500992, 140325909692415,
+SNULL, 140325909827583, 140325976801279,
+STORE, 140325909692416, 140325909827583,
+STORE, 140325909827584, 140325976801279,
+SNULL, 140325842583551, 140325909692415,
+STORE, 140325708500992, 140325842583551,
+STORE, 140325842583552, 140325909692415,
+ERASE, 140325842583552, 140325909692415,
+SNULL, 140325708500992, 140325775474687,
+STORE, 140325775474688, 140325842583551,
+STORE, 140325708500992, 140325775474687,
+SNULL, 140325775609855, 140325842583551,
+STORE, 140325775474688, 140325775609855,
+STORE, 140325775609856, 140325842583551,
+STORE, 140325775609856, 140325909692415,
+SNULL, 140325775609856, 140325842583551,
+STORE, 140325842583552, 140325909692415,
+STORE, 140325775609856, 140325842583551,
+SNULL, 140325842718719, 140325909692415,
+STORE, 140325842583552, 140325842718719,
+STORE, 140325842718720, 140325909692415,
+SNULL, 140325985161216, 140325993553919,
+STORE, 140325993553920, 140326018732031,
+STORE, 140325985161216, 140325993553919,
+SNULL, 140325993558015, 140326018732031,
+STORE, 140325993553920, 140325993558015,
+STORE, 140325993558016, 140326018732031,
+SNULL, 140325985165311, 140325993553919,
+STORE, 140325985161216, 140325985165311,
+STORE, 140325985165312, 140325993553919,
+SNULL, 140325993558016, 140326001946623,
+STORE, 140326001946624, 140326018732031,
+STORE, 140325993558016, 140326001946623,
+SNULL, 140326001950719, 140326018732031,
+STORE, 140326001946624, 140326001950719,
+STORE, 140326001950720, 140326018732031,
+SNULL, 140326001950720, 140326010339327,
+STORE, 140326010339328, 140326018732031,
+STORE, 140326001950720, 140326010339327,
+SNULL, 140326010343423, 140326018732031,
+STORE, 140326010339328, 140326010343423,
+STORE, 140326010343424, 140326018732031,
+STORE, 140325699973120, 140325708365823,
+STORE, 140325691580416, 140325708365823,
+STORE, 140325683187712, 140325708365823,
+SNULL, 140325683191807, 140325708365823,
+STORE, 140325683187712, 140325683191807,
+STORE, 140325683191808, 140325708365823,
+SNULL, 140325683191808, 140325699973119,
+STORE, 140325699973120, 140325708365823,
+STORE, 140325683191808, 140325699973119,
+SNULL, 140325699977215, 140325708365823,
+STORE, 140325699973120, 140325699977215,
+STORE, 140325699977216, 140325708365823,
+STORE, 140325674795008, 140325683187711,
+STORE, 140325666402304, 140325683187711,
+STORE, 140325658009600, 140325683187711,
+SNULL, 140325658009600, 140325666402303,
+STORE, 140325666402304, 140325683187711,
+STORE, 140325658009600, 140325666402303,
+SNULL, 140325666406399, 140325683187711,
+STORE, 140325666402304, 140325666406399,
+STORE, 140325666406400, 140325683187711,
+SNULL, 140325683191808, 140325691580415,
+STORE, 140325691580416, 140325699973119,
+STORE, 140325683191808, 140325691580415,
+SNULL, 140325691584511, 140325699973119,
+STORE, 140325691580416, 140325691584511,
+STORE, 140325691584512, 140325699973119,
+SNULL, 140325666406400, 140325674795007,
+STORE, 140325674795008, 140325683187711,
+STORE, 140325666406400, 140325674795007,
+SNULL, 140325674799103, 140325683187711,
+STORE, 140325674795008, 140325674799103,
+STORE, 140325674799104, 140325683187711,
+STORE, 140325649616896, 140325666402303,
+SNULL, 140325649616896, 140325658009599,
+STORE, 140325658009600, 140325666402303,
+STORE, 140325649616896, 140325658009599,
+SNULL, 140325658013695, 140325666402303,
+STORE, 140325658009600, 140325658013695,
+STORE, 140325658013696, 140325666402303,
+SNULL, 140325649620991, 140325658009599,
+STORE, 140325649616896, 140325649620991,
+STORE, 140325649620992, 140325658009599,
+STORE, 140325641224192, 140325649616895,
+STORE, 140325632831488, 140325649616895,
+SNULL, 140325632835583, 140325649616895,
+STORE, 140325632831488, 140325632835583,
+STORE, 140325632835584, 140325649616895,
+STORE, 140325624438784, 140325632831487,
+SNULL, 140325624442879, 140325632831487,
+STORE, 140325624438784, 140325624442879,
+STORE, 140325624442880, 140325632831487,
+SNULL, 140325632835584, 140325641224191,
+STORE, 140325641224192, 140325649616895,
+STORE, 140325632835584, 140325641224191,
+SNULL, 140325641228287, 140325649616895,
+STORE, 140325641224192, 140325641228287,
+STORE, 140325641228288, 140325649616895,
+STORE, 140325616046080, 140325624438783,
+SNULL, 140325616050175, 140325624438783,
+STORE, 140325616046080, 140325616050175,
+STORE, 140325616050176, 140325624438783,
+STORE, 140325607653376, 140325616046079,
+SNULL, 140325607657471, 140325616046079,
+STORE, 140325607653376, 140325607657471,
+STORE, 140325607657472, 140325616046079,
+STORE, 140325599260672, 140325607653375,
+STORE, 140325590867968, 140325607653375,
+STORE, 140325456650240, 140325590867967,
+SNULL, 140325456650240, 140325507039231,
+STORE, 140325507039232, 140325590867967,
+STORE, 140325456650240, 140325507039231,
+ERASE, 140325456650240, 140325507039231,
+STORE, 140325498646528, 140325507039231,
+STORE, 140325364428800, 140325498646527,
+SNULL, 140325364428800, 140325372821503,
+STORE, 140325372821504, 140325498646527,
+STORE, 140325364428800, 140325372821503,
+ERASE, 140325364428800, 140325372821503,
+STORE, 140325364428800, 140325372821503,
+STORE, 140325356036096, 140325372821503,
+STORE, 140325221818368, 140325356036095,
+SNULL, 140325221818368, 140325238603775,
+STORE, 140325238603776, 140325356036095,
+STORE, 140325221818368, 140325238603775,
+ERASE, 140325221818368, 140325238603775,
+STORE, 140325230211072, 140325238603775,
+STORE, 140325221818368, 140325238603775,
+STORE, 140325087600640, 140325221818367,
+STORE, 140325079207936, 140325087600639,
+SNULL, 140325087600640, 140325104386047,
+STORE, 140325104386048, 140325221818367,
+STORE, 140325087600640, 140325104386047,
+ERASE, 140325087600640, 140325104386047,
+STORE, 140325095993344, 140325104386047,
+STORE, 140325079207936, 140325104386047,
+STORE, 140324944990208, 140325079207935,
+SNULL, 140324944990208, 140324970168319,
+STORE, 140324970168320, 140325079207935,
+STORE, 140324944990208, 140324970168319,
+ERASE, 140324944990208, 140324970168319,
+STORE, 140324961775616, 140324970168319,
+STORE, 140324953382912, 140324970168319,
+STORE, 140324819165184, 140324953382911,
+STORE, 140324684947456, 140324953382911,
+STORE, 140324676554752, 140324684947455,
+STORE, 140324668162048, 140324684947455,
+STORE, 140324533944320, 140324668162047,
+STORE, 140324525551616, 140324533944319,
+SNULL, 140324533944320, 140324567515135,
+STORE, 140324567515136, 140324668162047,
+STORE, 140324533944320, 140324567515135,
+ERASE, 140324533944320, 140324567515135,
+STORE, 140324559122432, 140324567515135,
+STORE, 140324391333888, 140324525551615,
+SNULL, 140325574148095, 140325590867967,
+STORE, 140325507039232, 140325574148095,
+STORE, 140325574148096, 140325590867967,
+ERASE, 140325574148096, 140325590867967,
+SNULL, 140325439930367, 140325498646527,
+STORE, 140325372821504, 140325439930367,
+STORE, 140325439930368, 140325498646527,
+ERASE, 140325439930368, 140325498646527,
+SNULL, 140325305712639, 140325356036095,
+STORE, 140325238603776, 140325305712639,
+STORE, 140325305712640, 140325356036095,
+ERASE, 140325305712640, 140325356036095,
+SNULL, 140325171494911, 140325221818367,
+STORE, 140325104386048, 140325171494911,
+STORE, 140325171494912, 140325221818367,
+ERASE, 140325171494912, 140325221818367,
+SNULL, 140325104521215, 140325171494911,
+STORE, 140325104386048, 140325104521215,
+STORE, 140325104521216, 140325171494911,
+STORE, 140324257116160, 140324525551615,
+SNULL, 140324257116160, 140324299079679,
+STORE, 140324299079680, 140324525551615,
+STORE, 140324257116160, 140324299079679,
+ERASE, 140324257116160, 140324299079679,
+SNULL, 140325037277183, 140325079207935,
+STORE, 140324970168320, 140325037277183,
+STORE, 140325037277184, 140325079207935,
+ERASE, 140325037277184, 140325079207935,
+SNULL, 140324819165183, 140324953382911,
+STORE, 140324684947456, 140324819165183,
+STORE, 140324819165184, 140324953382911,
+SNULL, 140324819165184, 140324835950591,
+STORE, 140324835950592, 140324953382911,
+STORE, 140324819165184, 140324835950591,
+ERASE, 140324819165184, 140324835950591,
+SNULL, 140324903059455, 140324953382911,
+STORE, 140324835950592, 140324903059455,
+STORE, 140324903059456, 140324953382911,
+ERASE, 140324903059456, 140324953382911,
+SNULL, 140324684947456, 140324701732863,
+STORE, 140324701732864, 140324819165183,
+STORE, 140324684947456, 140324701732863,
+ERASE, 140324684947456, 140324701732863,
+SNULL, 140324768841727, 140324819165183,
+STORE, 140324701732864, 140324768841727,
+STORE, 140324768841728, 140324819165183,
+ERASE, 140324768841728, 140324819165183,
+SNULL, 140324634623999, 140324668162047,
+STORE, 140324567515136, 140324634623999,
+STORE, 140324634624000, 140324668162047,
+ERASE, 140324634624000, 140324668162047,
+SNULL, 140324391333887, 140324525551615,
+STORE, 140324299079680, 140324391333887,
+STORE, 140324391333888, 140324525551615,
+SNULL, 140324391333888, 140324433297407,
+STORE, 140324433297408, 140324525551615,
+STORE, 140324391333888, 140324433297407,
+ERASE, 140324391333888, 140324433297407,
+SNULL, 140325507174399, 140325574148095,
+STORE, 140325507039232, 140325507174399,
+STORE, 140325507174400, 140325574148095,
+SNULL, 140325590867968, 140325599260671,
+STORE, 140325599260672, 140325607653375,
+STORE, 140325590867968, 140325599260671,
+SNULL, 140325599264767, 140325607653375,
+STORE, 140325599260672, 140325599264767,
+STORE, 140325599264768, 140325607653375,
+SNULL, 140325372956671, 140325439930367,
+STORE, 140325372821504, 140325372956671,
+STORE, 140325372956672, 140325439930367,
+SNULL, 140324668166143, 140324684947455,
+STORE, 140324668162048, 140324668166143,
+STORE, 140324668166144, 140324684947455,
+SNULL, 140324525555711, 140324533944319,
+STORE, 140324525551616, 140324525555711,
+STORE, 140324525555712, 140324533944319,
+SNULL, 140324953382912, 140324961775615,
+STORE, 140324961775616, 140324970168319,
+STORE, 140324953382912, 140324961775615,
+SNULL, 140324961779711, 140324970168319,
+STORE, 140324961775616, 140324961779711,
+STORE, 140324961779712, 140324970168319,
+SNULL, 140325079212031, 140325104386047,
+STORE, 140325079207936, 140325079212031,
+STORE, 140325079212032, 140325104386047,
+SNULL, 140325221818368, 140325230211071,
+STORE, 140325230211072, 140325238603775,
+STORE, 140325221818368, 140325230211071,
+SNULL, 140325230215167, 140325238603775,
+STORE, 140325230211072, 140325230215167,
+STORE, 140325230215168, 140325238603775,
+SNULL, 140325356036096, 140325364428799,
+STORE, 140325364428800, 140325372821503,
+STORE, 140325356036096, 140325364428799,
+SNULL, 140325364432895, 140325372821503,
+ };
+ unsigned long set40[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140734309167104, 140737488351231,
+SNULL, 140734309171199, 140737488351231,
+STORE, 140734309167104, 140734309171199,
+STORE, 140734309036032, 140734309171199,
+STORE, 94270500081664, 94270502334463,
+SNULL, 94270500212735, 94270502334463,
+STORE, 94270500081664, 94270500212735,
+STORE, 94270500212736, 94270502334463,
+ERASE, 94270500212736, 94270502334463,
+STORE, 94270502305792, 94270502313983,
+STORE, 94270502313984, 94270502334463,
+STORE, 140321935110144, 140321937362943,
+SNULL, 140321935253503, 140321937362943,
+STORE, 140321935110144, 140321935253503,
+STORE, 140321935253504, 140321937362943,
+ERASE, 140321935253504, 140321937362943,
+STORE, 140321937350656, 140321937358847,
+STORE, 140321937358848, 140321937362943,
+STORE, 140734309625856, 140734309629951,
+STORE, 140734309613568, 140734309625855,
+STORE, 140321937321984, 140321937350655,
+STORE, 140321937313792, 140321937321983,
+STORE, 140321932894208, 140321935110143,
+SNULL, 140321932894208, 140321932992511,
+STORE, 140321932992512, 140321935110143,
+STORE, 140321932894208, 140321932992511,
+SNULL, 140321935085567, 140321935110143,
+STORE, 140321932992512, 140321935085567,
+STORE, 140321935085568, 140321935110143,
+SNULL, 140321935085568, 140321935093759,
+STORE, 140321935093760, 140321935110143,
+STORE, 140321935085568, 140321935093759,
+ERASE, 140321935085568, 140321935093759,
+STORE, 140321935085568, 140321935093759,
+ERASE, 140321935093760, 140321935110143,
+STORE, 140321935093760, 140321935110143,
+STORE, 140321929097216, 140321932894207,
+SNULL, 140321929097216, 140321930756095,
+STORE, 140321930756096, 140321932894207,
+STORE, 140321929097216, 140321930756095,
+SNULL, 140321932853247, 140321932894207,
+STORE, 140321930756096, 140321932853247,
+STORE, 140321932853248, 140321932894207,
+SNULL, 140321932853248, 140321932877823,
+STORE, 140321932877824, 140321932894207,
+STORE, 140321932853248, 140321932877823,
+ERASE, 140321932853248, 140321932877823,
+STORE, 140321932853248, 140321932877823,
+ERASE, 140321932877824, 140321932894207,
+STORE, 140321932877824, 140321932894207,
+STORE, 140321937305600, 140321937321983,
+SNULL, 140321932869631, 140321932877823,
+STORE, 140321932853248, 140321932869631,
+STORE, 140321932869632, 140321932877823,
+SNULL, 140321935089663, 140321935093759,
+STORE, 140321935085568, 140321935089663,
+STORE, 140321935089664, 140321935093759,
+SNULL, 94270502309887, 94270502313983,
+STORE, 94270502305792, 94270502309887,
+STORE, 94270502309888, 94270502313983,
+SNULL, 140321937354751, 140321937358847,
+STORE, 140321937350656, 140321937354751,
+STORE, 140321937354752, 140321937358847,
+ERASE, 140321937321984, 140321937350655,
+STORE, 94270507364352, 94270507499519,
+STORE, 140321920704512, 140321929097215,
+SNULL, 140321920708607, 140321929097215,
+STORE, 140321920704512, 140321920708607,
+STORE, 140321920708608, 140321929097215,
+STORE, 140321912311808, 140321920704511,
+STORE, 140321778094080, 140321912311807,
+SNULL, 140321778094080, 140321816051711,
+STORE, 140321816051712, 140321912311807,
+STORE, 140321778094080, 140321816051711,
+ERASE, 140321778094080, 140321816051711,
+SNULL, 140321883160575, 140321912311807,
+STORE, 140321816051712, 140321883160575,
+STORE, 140321883160576, 140321912311807,
+ERASE, 140321883160576, 140321912311807,
+SNULL, 140321816186879, 140321883160575,
+STORE, 140321816051712, 140321816186879,
+STORE, 140321816186880, 140321883160575,
+SNULL, 140321912315903, 140321920704511,
+STORE, 140321912311808, 140321912315903,
+STORE, 140321912315904, 140321920704511,
+STORE, 140321903919104, 140321912311807,
+SNULL, 140321903923199, 140321912311807,
+STORE, 140321903919104, 140321903923199,
+STORE, 140321903923200, 140321912311807,
+STORE, 140321895526400, 140321903919103,
+SNULL, 140321895530495, 140321903919103,
+STORE, 140321895526400, 140321895530495,
+STORE, 140321895530496, 140321903919103,
+STORE, 140321887133696, 140321895526399,
+SNULL, 140321887137791, 140321895526399,
+STORE, 140321887133696, 140321887137791,
+STORE, 140321887137792, 140321895526399,
+STORE, 140321807659008, 140321816051711,
+STORE, 140321673441280, 140321807659007,
+SNULL, 140321673441280, 140321681833983,
+STORE, 140321681833984, 140321807659007,
+STORE, 140321673441280, 140321681833983,
+ERASE, 140321673441280, 140321681833983,
+SNULL, 140321748942847, 140321807659007,
+STORE, 140321681833984, 140321748942847,
+STORE, 140321748942848, 140321807659007,
+ERASE, 140321748942848, 140321807659007,
+STORE, 140321799266304, 140321816051711,
+STORE, 140321790873600, 140321816051711,
+STORE, 140321782480896, 140321816051711,
+STORE, 140321547616256, 140321748942847,
+SNULL, 140321614725119, 140321748942847,
+STORE, 140321547616256, 140321614725119,
+STORE, 140321614725120, 140321748942847,
+SNULL, 140321614725120, 140321681833983,
+STORE, 140321681833984, 140321748942847,
+STORE, 140321614725120, 140321681833983,
+ERASE, 140321614725120, 140321681833983,
+SNULL, 140321681969151, 140321748942847,
+STORE, 140321681833984, 140321681969151,
+STORE, 140321681969152, 140321748942847,
+STORE, 140321547616256, 140321681833983,
+SNULL, 140321547616256, 140321614725119,
+STORE, 140321614725120, 140321681833983,
+STORE, 140321547616256, 140321614725119,
+SNULL, 140321614860287, 140321681833983,
+STORE, 140321614725120, 140321614860287,
+STORE, 140321614860288, 140321681833983,
+SNULL, 140321547751423, 140321614725119,
+STORE, 140321547616256, 140321547751423,
+STORE, 140321547751424, 140321614725119,
+STORE, 140321480507392, 140321547616255,
+SNULL, 140321782480896, 140321799266303,
+STORE, 140321799266304, 140321816051711,
+STORE, 140321782480896, 140321799266303,
+SNULL, 140321799270399, 140321816051711,
+STORE, 140321799266304, 140321799270399,
+STORE, 140321799270400, 140321816051711,
+STORE, 140321774088192, 140321799266303,
+SNULL, 140321774088192, 140321790873599,
+STORE, 140321790873600, 140321799266303,
+STORE, 140321774088192, 140321790873599,
+SNULL, 140321790877695, 140321799266303,
+STORE, 140321790873600, 140321790877695,
+STORE, 140321790877696, 140321799266303,
+SNULL, 140321480642559, 140321547616255,
+STORE, 140321480507392, 140321480642559,
+STORE, 140321480642560, 140321547616255,
+SNULL, 140321774088192, 140321782480895,
+STORE, 140321782480896, 140321790873599,
+STORE, 140321774088192, 140321782480895,
+SNULL, 140321782484991, 140321790873599,
+STORE, 140321782480896, 140321782484991,
+STORE, 140321782484992, 140321790873599,
+SNULL, 140321799270400, 140321807659007,
+STORE, 140321807659008, 140321816051711,
+STORE, 140321799270400, 140321807659007,
+SNULL, 140321807663103, 140321816051711,
+STORE, 140321807659008, 140321807663103,
+STORE, 140321807663104, 140321816051711,
+STORE, 140321765695488, 140321782480895,
+STORE, 140321757302784, 140321782480895,
+SNULL, 140321757306879, 140321782480895,
+STORE, 140321757302784, 140321757306879,
+STORE, 140321757306880, 140321782480895,
+STORE, 140321472114688, 140321480507391,
+STORE, 140321463721984, 140321480507391,
+SNULL, 140321463726079, 140321480507391,
+STORE, 140321463721984, 140321463726079,
+STORE, 140321463726080, 140321480507391,
+SNULL, 140321757306880, 140321774088191,
+STORE, 140321774088192, 140321782480895,
+STORE, 140321757306880, 140321774088191,
+SNULL, 140321774092287, 140321782480895,
+STORE, 140321774088192, 140321774092287,
+STORE, 140321774092288, 140321782480895,
+SNULL, 140321463726080, 140321472114687,
+STORE, 140321472114688, 140321480507391,
+STORE, 140321463726080, 140321472114687,
+SNULL, 140321472118783, 140321480507391,
+STORE, 140321472114688, 140321472118783,
+STORE, 140321472118784, 140321480507391,
+SNULL, 140321757306880, 140321765695487,
+STORE, 140321765695488, 140321774088191,
+STORE, 140321757306880, 140321765695487,
+SNULL, 140321765699583, 140321774088191,
+STORE, 140321765695488, 140321765699583,
+STORE, 140321765699584, 140321774088191,
+STORE, 140321455329280, 140321463721983,
+SNULL, 140321455333375, 140321463721983,
+STORE, 140321455329280, 140321455333375,
+STORE, 140321455333376, 140321463721983,
+STORE, 140321446936576, 140321455329279,
+STORE, 140321438543872, 140321455329279,
+STORE, 140321430151168, 140321455329279,
+SNULL, 140321430155263, 140321455329279,
+STORE, 140321430151168, 140321430155263,
+STORE, 140321430155264, 140321455329279,
+SNULL, 140321430155264, 140321446936575,
+STORE, 140321446936576, 140321455329279,
+STORE, 140321430155264, 140321446936575,
+SNULL, 140321446940671, 140321455329279,
+STORE, 140321446936576, 140321446940671,
+STORE, 140321446940672, 140321455329279,
+SNULL, 140321430155264, 140321438543871,
+STORE, 140321438543872, 140321446936575,
+STORE, 140321430155264, 140321438543871,
+SNULL, 140321438547967, 140321446936575,
+STORE, 140321438543872, 140321438547967,
+STORE, 140321438547968, 140321446936575,
+STORE, 140321421758464, 140321430151167,
+SNULL, 140321421762559, 140321430151167,
+STORE, 140321421758464, 140321421762559,
+STORE, 140321421762560, 140321430151167,
+STORE, 140321413365760, 140321421758463,
+SNULL, 140321413369855, 140321421758463,
+STORE, 140321413365760, 140321413369855,
+STORE, 140321413369856, 140321421758463,
+STORE, 140321404973056, 140321413365759,
+SNULL, 140321404977151, 140321413365759,
+STORE, 140321404973056, 140321404977151,
+STORE, 140321404977152, 140321413365759,
+STORE, 140321396580352, 140321404973055,
+STORE, 140321388187648, 140321404973055,
+STORE, 140321253969920, 140321388187647,
+SNULL, 140321253969920, 140321279180799,
+STORE, 140321279180800, 140321388187647,
+STORE, 140321253969920, 140321279180799,
+ERASE, 140321253969920, 140321279180799,
+SNULL, 140321346289663, 140321388187647,
+STORE, 140321279180800, 140321346289663,
+STORE, 140321346289664, 140321388187647,
+ERASE, 140321346289664, 140321388187647,
+STORE, 140321144963072, 140321346289663,
+STORE, 140321379794944, 140321404973055,
+STORE, 140321371402240, 140321404973055,
+STORE, 140321010745344, 140321346289663,
+STORE, 140321363009536, 140321404973055,
+SNULL, 140321077854207, 140321346289663,
+STORE, 140321010745344, 140321077854207,
+STORE, 140321077854208, 140321346289663,
+SNULL, 140321077854208, 140321144963071,
+STORE, 140321144963072, 140321346289663,
+STORE, 140321077854208, 140321144963071,
+ERASE, 140321077854208, 140321144963071,
+STORE, 140321354616832, 140321404973055,
+STORE, 140321136570368, 140321144963071,
+STORE, 140320943636480, 140321077854207,
+STORE, 140320876527616, 140321077854207,
+STORE, 140321128177664, 140321144963071,
+SNULL, 140320876662783, 140321077854207,
+STORE, 140320876527616, 140320876662783,
+STORE, 140320876662784, 140321077854207,
+STORE, 140321119784960, 140321144963071,
+STORE, 140321111392256, 140321144963071,
+STORE, 140320742309888, 140320876527615,
+STORE, 140321102999552, 140321144963071,
+STORE, 140320608092160, 140320876527615,
+SNULL, 140320675201023, 140320876527615,
+STORE, 140320608092160, 140320675201023,
+STORE, 140320675201024, 140320876527615,
+SNULL, 140320675201024, 140320742309887,
+STORE, 140320742309888, 140320876527615,
+STORE, 140320675201024, 140320742309887,
+ERASE, 140320675201024, 140320742309887,
+STORE, 140321094606848, 140321144963071,
+STORE, 140321086214144, 140321144963071,
+STORE, 140320608092160, 140320876527615,
+SNULL, 140320608092160, 140320675201023,
+STORE, 140320675201024, 140320876527615,
+STORE, 140320608092160, 140320675201023,
+SNULL, 140320675336191, 140320876527615,
+STORE, 140320675201024, 140320675336191,
+STORE, 140320675336192, 140320876527615,
+STORE, 140320599699456, 140320608092159,
+STORE, 140320591306752, 140320608092159,
+STORE, 140320457089024, 140320591306751,
+STORE, 140320448696320, 140320457089023,
+STORE, 140320314478592, 140320448696319,
+SNULL, 140321144963072, 140321279180799,
+STORE, 140321279180800, 140321346289663,
+STORE, 140321144963072, 140321279180799,
+SNULL, 140321279315967, 140321346289663,
+STORE, 140321279180800, 140321279315967,
+STORE, 140321279315968, 140321346289663,
+SNULL, 140321086214144, 140321136570367,
+STORE, 140321136570368, 140321144963071,
+STORE, 140321086214144, 140321136570367,
+SNULL, 140321136574463, 140321144963071,
+STORE, 140321136570368, 140321136574463,
+STORE, 140321136574464, 140321144963071,
+SNULL, 140321212071935, 140321279180799,
+STORE, 140321144963072, 140321212071935,
+STORE, 140321212071936, 140321279180799,
+ERASE, 140321212071936, 140321279180799,
+SNULL, 140321145098239, 140321212071935,
+STORE, 140321144963072, 140321145098239,
+STORE, 140321145098240, 140321212071935,
+SNULL, 140320876662784, 140321010745343,
+STORE, 140321010745344, 140321077854207,
+STORE, 140320876662784, 140321010745343,
+SNULL, 140321010880511, 140321077854207,
+STORE, 140321010745344, 140321010880511,
+STORE, 140321010880512, 140321077854207,
+SNULL, 140321354616832, 140321379794943,
+STORE, 140321379794944, 140321404973055,
+STORE, 140321354616832, 140321379794943,
+SNULL, 140321379799039, 140321404973055,
+STORE, 140321379794944, 140321379799039,
+STORE, 140321379799040, 140321404973055,
+SNULL, 140320876662784, 140320943636479,
+STORE, 140320943636480, 140321010745343,
+STORE, 140320876662784, 140320943636479,
+SNULL, 140320943771647, 140321010745343,
+STORE, 140320943636480, 140320943771647,
+STORE, 140320943771648, 140321010745343,
+SNULL, 140320809418751, 140320876527615,
+STORE, 140320675336192, 140320809418751,
+STORE, 140320809418752, 140320876527615,
+ERASE, 140320809418752, 140320876527615,
+SNULL, 140320675336192, 140320742309887,
+STORE, 140320742309888, 140320809418751,
+STORE, 140320675336192, 140320742309887,
+SNULL, 140320742445055, 140320809418751,
+STORE, 140320742309888, 140320742445055,
+STORE, 140320742445056, 140320809418751,
+SNULL, 140320608227327, 140320675201023,
+STORE, 140320608092160, 140320608227327,
+STORE, 140320608227328, 140320675201023,
+SNULL, 140320457089024, 140320473874431,
+STORE, 140320473874432, 140320591306751,
+STORE, 140320457089024, 140320473874431,
+ERASE, 140320457089024, 140320473874431,
+SNULL, 140320540983295, 140320591306751,
+STORE, 140320473874432, 140320540983295,
+STORE, 140320540983296, 140320591306751,
+ERASE, 140320540983296, 140320591306751,
+SNULL, 140320314478592, 140320339656703,
+STORE, 140320339656704, 140320448696319,
+STORE, 140320314478592, 140320339656703,
+ERASE, 140320314478592, 140320339656703,
+SNULL, 140321086214144, 140321128177663,
+STORE, 140321128177664, 140321136570367,
+STORE, 140321086214144, 140321128177663,
+SNULL, 140321128181759, 140321136570367,
+STORE, 140321128177664, 140321128181759,
+STORE, 140321128181760, 140321136570367,
+SNULL, 140321354616832, 140321371402239,
+STORE, 140321371402240, 140321379794943,
+STORE, 140321354616832, 140321371402239,
+SNULL, 140321371406335, 140321379794943,
+STORE, 140321371402240, 140321371406335,
+STORE, 140321371406336, 140321379794943,
+SNULL, 140320591310847, 140320608092159,
+STORE, 140320591306752, 140320591310847,
+STORE, 140320591310848, 140320608092159,
+SNULL, 140321354616832, 140321363009535,
+STORE, 140321363009536, 140321371402239,
+STORE, 140321354616832, 140321363009535,
+SNULL, 140321363013631, 140321371402239,
+STORE, 140321363009536, 140321363013631,
+STORE, 140321363013632, 140321371402239,
+SNULL, 140321086214144, 140321119784959,
+STORE, 140321119784960, 140321128177663,
+STORE, 140321086214144, 140321119784959,
+SNULL, 140321119789055, 140321128177663,
+STORE, 140321119784960, 140321119789055,
+STORE, 140321119789056, 140321128177663,
+SNULL, 140321086218239, 140321119784959,
+STORE, 140321086214144, 140321086218239,
+STORE, 140321086218240, 140321119784959,
+SNULL, 140321086218240, 140321094606847,
+STORE, 140321094606848, 140321119784959,
+STORE, 140321086218240, 140321094606847,
+SNULL, 140321094610943, 140321119784959,
+STORE, 140321094606848, 140321094610943,
+STORE, 140321094610944, 140321119784959,
+SNULL, 140320474009599, 140320540983295,
+STORE, 140320473874432, 140320474009599,
+STORE, 140320474009600, 140320540983295,
+SNULL, 140320406765567, 140320448696319,
+STORE, 140320339656704, 140320406765567,
+STORE, 140320406765568, 140320448696319,
+ERASE, 140320406765568, 140320448696319,
+SNULL, 140320339791871, 140320406765567,
+STORE, 140320339656704, 140320339791871,
+STORE, 140320339791872, 140320406765567,
+STORE, 140321270788096, 140321279180799,
+STORE, 140321262395392, 140321279180799,
+STORE, 140321254002688, 140321279180799,
+SNULL, 140321254002688, 140321262395391,
+STORE, 140321262395392, 140321279180799,
+STORE, 140321254002688, 140321262395391,
+SNULL, 140321262399487, 140321279180799,
+STORE, 140321262395392, 140321262399487,
+STORE, 140321262399488, 140321279180799,
+STORE, 140321245609984, 140321262395391,
+STORE, 140321237217280, 140321262395391,
+SNULL, 140321237217280, 140321245609983,
+STORE, 140321245609984, 140321262395391,
+STORE, 140321237217280, 140321245609983,
+SNULL, 140321245614079, 140321262395391,
+STORE, 140321245609984, 140321245614079,
+STORE, 140321245614080, 140321262395391,
+SNULL, 140321379799040, 140321388187647,
+STORE, 140321388187648, 140321404973055,
+STORE, 140321379799040, 140321388187647,
+SNULL, 140321388191743, 140321404973055,
+STORE, 140321388187648, 140321388191743,
+STORE, 140321388191744, 140321404973055,
+SNULL, 140321354620927, 140321363009535,
+STORE, 140321354616832, 140321354620927,
+STORE, 140321354620928, 140321363009535,
+SNULL, 140321388191744, 140321396580351,
+STORE, 140321396580352, 140321404973055,
+STORE, 140321388191744, 140321396580351,
+SNULL, 140321396584447, 140321404973055,
+STORE, 140321396580352, 140321396584447,
+STORE, 140321396584448, 140321404973055,
+SNULL, 140321094610944, 140321111392255,
+STORE, 140321111392256, 140321119784959,
+STORE, 140321094610944, 140321111392255,
+SNULL, 140321111396351, 140321119784959,
+STORE, 140321111392256, 140321111396351,
+STORE, 140321111396352, 140321119784959,
+STORE, 140321228824576, 140321245609983,
+SNULL, 140321094610944, 140321102999551,
+STORE, 140321102999552, 140321111392255,
+STORE, 140321094610944, 140321102999551,
+SNULL, 140321103003647, 140321111392255,
+STORE, 140321102999552, 140321103003647,
+STORE, 140321103003648, 140321111392255,
+STORE, 140321220431872, 140321245609983,
+SNULL, 140321220435967, 140321245609983,
+STORE, 140321220431872, 140321220435967,
+STORE, 140321220435968, 140321245609983,
+STORE, 140320868134912, 140320876527615,
+SNULL, 140320868139007, 140320876527615,
+STORE, 140320868134912, 140320868139007,
+STORE, 140320868139008, 140320876527615,
+SNULL, 140320591310848, 140320599699455,
+STORE, 140320599699456, 140320608092159,
+STORE, 140320591310848, 140320599699455,
+SNULL, 140320599703551, 140320608092159,
+STORE, 140320599699456, 140320599703551,
+STORE, 140320599703552, 140320608092159,
+STORE, 140320859742208, 140320868134911,
+SNULL, 140321262399488, 140321270788095,
+STORE, 140321270788096, 140321279180799,
+STORE, 140321262399488, 140321270788095,
+SNULL, 140321270792191, 140321279180799,
+STORE, 140321270788096, 140321270792191,
+STORE, 140321270792192, 140321279180799,
+STORE, 140320851349504, 140320868134911,
+STORE, 140320842956800, 140320868134911,
+STORE, 140320834564096, 140320868134911,
+STORE, 140320826171392, 140320868134911,
+SNULL, 140320826171392, 140320834564095,
+STORE, 140320834564096, 140320868134911,
+STORE, 140320826171392, 140320834564095,
+SNULL, 140320834568191, 140320868134911,
+STORE, 140320834564096, 140320834568191,
+STORE, 140320834568192, 140320868134911,
+SNULL, 140321220435968, 140321228824575,
+STORE, 140321228824576, 140321245609983,
+STORE, 140321220435968, 140321228824575,
+SNULL, 140321228828671, 140321245609983,
+STORE, 140321228824576, 140321228828671,
+STORE, 140321228828672, 140321245609983,
+STORE, 140320817778688, 140320834564095,
+SNULL, 140320817782783, 140320834564095,
+STORE, 140320817778688, 140320817782783,
+STORE, 140320817782784, 140320834564095,
+STORE, 140320582914048, 140320591306751,
+SNULL, 140321228828672, 140321237217279,
+STORE, 140321237217280, 140321245609983,
+STORE, 140321228828672, 140321237217279,
+SNULL, 140321237221375, 140321245609983,
+STORE, 140321237217280, 140321237221375,
+STORE, 140321237221376, 140321245609983,
+SNULL, 140320448700415, 140320457089023,
+STORE, 140320448696320, 140320448700415,
+STORE, 140320448700416, 140320457089023,
+SNULL, 140321245614080, 140321254002687,
+STORE, 140321254002688, 140321262395391,
+STORE, 140321245614080, 140321254002687,
+SNULL, 140321254006783, 140321262395391,
+STORE, 140321254002688, 140321254006783,
+STORE, 140321254006784, 140321262395391,
+STORE, 140320574521344, 140320591306751,
+SNULL, 140320574525439, 140320591306751,
+STORE, 140320574521344, 140320574525439,
+STORE, 140320574525440, 140320591306751,
+STORE, 140320566128640, 140320574521343,
+SNULL, 140320566132735, 140320574521343,
+STORE, 140320566128640, 140320566132735,
+STORE, 140320566132736, 140320574521343,
+SNULL, 140320574525440, 140320582914047,
+STORE, 140320582914048, 140320591306751,
+STORE, 140320574525440, 140320582914047,
+SNULL, 140320582918143, 140320591306751,
+STORE, 140320582914048, 140320582918143,
+STORE, 140320582918144, 140320591306751,
+STORE, 140320557735936, 140320566128639,
+SNULL, 140320557740031, 140320566128639,
+STORE, 140320557735936, 140320557740031,
+STORE, 140320557740032, 140320566128639,
+STORE, 140320549343232, 140320557735935,
+STORE, 140320465481728, 140320473874431,
+STORE, 140320448700416, 140320473874431,
+SNULL, 140320834568192, 140320859742207,
+STORE, 140320859742208, 140320868134911,
+STORE, 140320834568192, 140320859742207,
+SNULL, 140320859746303, 140320868134911,
+STORE, 140320859742208, 140320859746303,
+STORE, 140320859746304, 140320868134911,
+STORE, 140320440303616, 140320448696319,
+STORE, 140320431910912, 140320448696319,
+SNULL, 140320834568192, 140320851349503,
+STORE, 140320851349504, 140320859742207,
+STORE, 140320834568192, 140320851349503,
+SNULL, 140320851353599, 140320859742207,
+STORE, 140320851349504, 140320851353599,
+STORE, 140320851353600, 140320859742207,
+SNULL, 140320817782784, 140320826171391,
+STORE, 140320826171392, 140320834564095,
+STORE, 140320817782784, 140320826171391,
+SNULL, 140320826175487, 140320834564095,
+STORE, 140320826171392, 140320826175487,
+STORE, 140320826175488, 140320834564095,
+SNULL, 140320834568192, 140320842956799,
+STORE, 140320842956800, 140320851349503,
+STORE, 140320834568192, 140320842956799,
+SNULL, 140320842960895, 140320851349503,
+STORE, 140320842956800, 140320842960895,
+STORE, 140320842960896, 140320851349503,
+STORE, 140320423518208, 140320448696319,
+SNULL, 140320423522303, 140320448696319,
+STORE, 140320423518208, 140320423522303,
+STORE, 140320423522304, 140320448696319,
+STORE, 140320415125504, 140320423518207,
+STORE, 140320331264000, 140320339656703,
+STORE, 140320322871296, 140320339656703,
+STORE, 140320314478592, 140320339656703,
+SNULL, 140320314482687, 140320339656703,
+STORE, 140320314478592, 140320314482687,
+STORE, 140320314482688, 140320339656703,
+STORE, 140320306085888, 140320314478591,
+SNULL, 140320306089983, 140320314478591,
+STORE, 140320306085888, 140320306089983,
+STORE, 140320306089984, 140320314478591,
+STORE, 140320297693184, 140320306085887,
+SNULL, 140320297697279, 140320306085887,
+STORE, 140320297693184, 140320297697279,
+STORE, 140320297697280, 140320306085887,
+STORE, 140320289300480, 140320297693183,
+STORE, 140320280907776, 140320297693183,
+SNULL, 140320280911871, 140320297693183,
+STORE, 140320280907776, 140320280911871,
+STORE, 140320280911872, 140320297693183,
+SNULL, 140320423522304, 140320431910911,
+STORE, 140320431910912, 140320448696319,
+STORE, 140320423522304, 140320431910911,
+SNULL, 140320431915007, 140320448696319,
+STORE, 140320431910912, 140320431915007,
+STORE, 140320431915008, 140320448696319,
+SNULL, 140320549347327, 140320557735935,
+STORE, 140320549343232, 140320549347327,
+STORE, 140320549347328, 140320557735935,
+STORE, 140320272515072, 140320280907775,
+SNULL, 140320448700416, 140320457089023,
+STORE, 140320457089024, 140320473874431,
+STORE, 140320448700416, 140320457089023,
+SNULL, 140320457093119, 140320473874431,
+STORE, 140320457089024, 140320457093119,
+STORE, 140320457093120, 140320473874431,
+STORE, 140320264122368, 140320280907775,
+SNULL, 140320457093120, 140320465481727,
+STORE, 140320465481728, 140320473874431,
+STORE, 140320457093120, 140320465481727,
+SNULL, 140320465485823, 140320473874431,
+STORE, 140320465481728, 140320465485823,
+STORE, 140320465485824, 140320473874431,
+SNULL, 140320431915008, 140320440303615,
+STORE, 140320440303616, 140320448696319,
+STORE, 140320431915008, 140320440303615,
+SNULL, 140320440307711, 140320448696319,
+STORE, 140320440303616, 140320440307711,
+STORE, 140320440307712, 140320448696319,
+STORE, 140320255729664, 140320280907775,
+STORE, 140320247336960, 140320280907775,
+SNULL, 140320247341055, 140320280907775,
+STORE, 140320247336960, 140320247341055,
+STORE, 140320247341056, 140320280907775,
+STORE, 140320238944256, 140320247336959,
+STORE, 140320230551552, 140320247336959,
+SNULL, 140320230551552, 140320238944255,
+STORE, 140320238944256, 140320247336959,
+STORE, 140320230551552, 140320238944255,
+SNULL, 140320238948351, 140320247336959,
+STORE, 140320238944256, 140320238948351,
+STORE, 140320238948352, 140320247336959,
+SNULL, 140320314482688, 140320331263999,
+STORE, 140320331264000, 140320339656703,
+STORE, 140320314482688, 140320331263999,
+SNULL, 140320331268095, 140320339656703,
+STORE, 140320331264000, 140320331268095,
+STORE, 140320331268096, 140320339656703,
+SNULL, 140320280911872, 140320289300479,
+STORE, 140320289300480, 140320297693183,
+STORE, 140320280911872, 140320289300479,
+SNULL, 140320289304575, 140320297693183,
+STORE, 140320289300480, 140320289304575,
+STORE, 140320289304576, 140320297693183,
+SNULL, 140320415129599, 140320423518207,
+STORE, 140320415125504, 140320415129599,
+STORE, 140320415129600, 140320423518207,
+STORE, 140320222158848, 140320238944255,
+STORE, 140320213766144, 140320238944255,
+STORE, 140320205373440, 140320238944255,
+SNULL, 140320205377535, 140320238944255,
+STORE, 140320205373440, 140320205377535,
+STORE, 140320205377536, 140320238944255,
+SNULL, 140320314482688, 140320322871295,
+STORE, 140320322871296, 140320331263999,
+STORE, 140320314482688, 140320322871295,
+SNULL, 140320322875391, 140320331263999,
+STORE, 140320322871296, 140320322875391,
+STORE, 140320322875392, 140320331263999,
+SNULL, 140320247341056, 140320272515071,
+STORE, 140320272515072, 140320280907775,
+STORE, 140320247341056, 140320272515071,
+SNULL, 140320272519167, 140320280907775,
+STORE, 140320272515072, 140320272519167,
+STORE, 140320272519168, 140320280907775,
+SNULL, 140320247341056, 140320264122367,
+STORE, 140320264122368, 140320272515071,
+STORE, 140320247341056, 140320264122367,
+SNULL, 140320264126463, 140320272515071,
+STORE, 140320264122368, 140320264126463,
+STORE, 140320264126464, 140320272515071,
+SNULL, 140320205377536, 140320230551551,
+STORE, 140320230551552, 140320238944255,
+STORE, 140320205377536, 140320230551551,
+SNULL, 140320230555647, 140320238944255,
+STORE, 140320230551552, 140320230555647,
+STORE, 140320230555648, 140320238944255,
+STORE, 140320196980736, 140320205373439,
+SNULL, 140320196984831, 140320205373439,
+STORE, 140320196980736, 140320196984831,
+STORE, 140320196984832, 140320205373439,
+STORE, 140320188588032, 140320196980735,
+SNULL, 140320247341056, 140320255729663,
+STORE, 140320255729664, 140320264122367,
+STORE, 140320247341056, 140320255729663,
+SNULL, 140320255733759, 140320264122367,
+STORE, 140320255729664, 140320255733759,
+STORE, 140320255733760, 140320264122367,
+STORE, 140320180195328, 140320196980735,
+SNULL, 140320180199423, 140320196980735,
+STORE, 140320180195328, 140320180199423,
+STORE, 140320180199424, 140320196980735,
+STORE, 140320171802624, 140320180195327,
+STORE, 140320163409920, 140320180195327,
+SNULL, 140320163414015, 140320180195327,
+STORE, 140320163409920, 140320163414015,
+STORE, 140320163414016, 140320180195327,
+SNULL, 140320205377536, 140320222158847,
+STORE, 140320222158848, 140320230551551,
+STORE, 140320205377536, 140320222158847,
+SNULL, 140320222162943, 140320230551551,
+STORE, 140320222158848, 140320222162943,
+STORE, 140320222162944, 140320230551551,
+SNULL, 140320205377536, 140320213766143,
+STORE, 140320213766144, 140320222158847,
+STORE, 140320205377536, 140320213766143,
+SNULL, 140320213770239, 140320222158847,
+STORE, 140320213766144, 140320213770239,
+STORE, 140320213770240, 140320222158847,
+STORE, 140320155017216, 140320163409919,
+SNULL, 140320180199424, 140320188588031,
+STORE, 140320188588032, 140320196980735,
+STORE, 140320180199424, 140320188588031,
+SNULL, 140320188592127, 140320196980735,
+STORE, 140320188588032, 140320188592127,
+STORE, 140320188592128, 140320196980735,
+SNULL, 140320155021311, 140320163409919,
+STORE, 140320155017216, 140320155021311,
+STORE, 140320155021312, 140320163409919,
+SNULL, 140320163414016, 140320171802623,
+STORE, 140320171802624, 140320180195327,
+STORE, 140320163414016, 140320171802623,
+SNULL, 140320171806719, 140320180195327,
+STORE, 140320171802624, 140320171806719,
+STORE, 140320171806720, 140320180195327,
+STORE, 140320146624512, 140320155017215,
+SNULL, 140320146628607, 140320155017215,
+STORE, 140320146624512, 140320146628607,
+STORE, 140320146628608, 140320155017215,
+STORE, 140321937321984, 140321937350655,
+STORE, 140321884942336, 140321887133695,
+SNULL, 140321884942336, 140321885032447,
+STORE, 140321885032448, 140321887133695,
+STORE, 140321884942336, 140321885032447,
+SNULL, 140321887125503, 140321887133695,
+STORE, 140321885032448, 140321887125503,
+STORE, 140321887125504, 140321887133695,
+ERASE, 140321887125504, 140321887133695,
+STORE, 140321887125504, 140321887133695,
+SNULL, 140321887129599, 140321887133695,
+STORE, 140321887125504, 140321887129599,
+STORE, 140321887129600, 140321887133695,
+ERASE, 140321937321984, 140321937350655,
+ERASE, 140321086214144, 140321086218239,
+ERASE, 140321086218240, 140321094606847,
+ERASE, 140321119784960, 140321119789055,
+ERASE, 140321119789056, 140321128177663,
+ERASE, 140321245609984, 140321245614079,
+ERASE, 140321245614080, 140321254002687,
+ERASE, 140320574521344, 140320574525439,
+ERASE, 140320574525440, 140320582914047,
+ERASE, 140320297693184, 140320297697279,
+ERASE, 140320297697280, 140320306085887,
+ERASE, 140321354616832, 140321354620927,
+ERASE, 140321354620928, 140321363009535,
+ERASE, 140320834564096, 140320834568191,
+ERASE, 140320834568192, 140320842956799,
+ERASE, 140320591306752, 140320591310847,
+ERASE, 140320591310848, 140320599699455,
+ERASE, 140321136570368, 140321136574463,
+ERASE, 140321136574464, 140321144963071,
+ERASE, 140321237217280, 140321237221375,
+ERASE, 140321237221376, 140321245609983,
+ERASE, 140321363009536, 140321363013631,
+ERASE, 140321363013632, 140321371402239,
+ERASE, 140320599699456, 140320599703551,
+ERASE, 140320599703552, 140320608092159,
+ERASE, 140321396580352, 140321396584447,
+ERASE, 140321396584448, 140321404973055,
+ERASE, 140320566128640, 140320566132735,
+ERASE, 140320566132736, 140320574521343,
+ERASE, 140321094606848, 140321094610943,
+ERASE, 140321094610944, 140321102999551,
+ERASE, 140320582914048, 140320582918143,
+ERASE, 140320582918144, 140320591306751,
+ERASE, 140320289300480, 140320289304575,
+ERASE, 140320289304576, 140320297693183,
+ERASE, 140320163409920, 140320163414015,
+ };
+ unsigned long set41[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140728157171712, 140737488351231,
+SNULL, 140728157175807, 140737488351231,
+STORE, 140728157171712, 140728157175807,
+STORE, 140728157040640, 140728157175807,
+STORE, 94376106364928, 94376108613631,
+SNULL, 94376106487807, 94376108613631,
+STORE, 94376106364928, 94376106487807,
+STORE, 94376106487808, 94376108613631,
+SNULL, 94376106487808, 94376108613631,
+STORE, 94376108584960, 94376108593151,
+STORE, 94376108593152, 94376108613631,
+STORE, 140113496432640, 140113498685439,
+SNULL, 140113496575999, 140113498685439,
+STORE, 140113496432640, 140113496575999,
+STORE, 140113496576000, 140113498685439,
+SNULL, 140113496576000, 140113498685439,
+STORE, 140113498673152, 140113498681343,
+STORE, 140113498681344, 140113498685439,
+STORE, 140728157609984, 140728157618175,
+STORE, 140728157593600, 140728157609983,
+STORE, 140113498636288, 140113498673151,
+STORE, 140113498628096, 140113498636287,
+STORE, 140113492635648, 140113496432639,
+SNULL, 140113492635648, 140113494294527,
+STORE, 140113494294528, 140113496432639,
+STORE, 140113492635648, 140113494294527,
+SNULL, 140113496391679, 140113496432639,
+STORE, 140113494294528, 140113496391679,
+STORE, 140113496391680, 140113496432639,
+SNULL, 140113496391680, 140113496416255,
+STORE, 140113496416256, 140113496432639,
+STORE, 140113496391680, 140113496416255,
+SNULL, 140113496391680, 140113496416255,
+STORE, 140113496391680, 140113496416255,
+SNULL, 140113496416256, 140113496432639,
+STORE, 140113496416256, 140113496432639,
+SNULL, 140113496408063, 140113496416255,
+STORE, 140113496391680, 140113496408063,
+STORE, 140113496408064, 140113496416255,
+SNULL, 94376108589055, 94376108593151,
+STORE, 94376108584960, 94376108589055,
+STORE, 94376108589056, 94376108593151,
+SNULL, 140113498677247, 140113498681343,
+STORE, 140113498673152, 140113498677247,
+STORE, 140113498677248, 140113498681343,
+SNULL, 140113498636288, 140113498673151,
+STORE, 94376135090176, 94376135094271,
+STORE, 94376135090176, 94376135098367,
+STORE, 94376139288576, 94376139292671,
+STORE, 94376143482880, 94376143486975,
+STORE, 94376147677184, 94376147681279,
+STORE, 94376151871488, 94376151875583,
+STORE, 94376156065792, 94376156069887,
+STORE, 94376160260096, 94376160264191,
+STORE, 94376164454400, 94376164458495,
+STORE, 94376168648704, 94376168652799,
+STORE, 94376172843008, 94376172847103,
+STORE, 94376177037312, 94376177041407,
+STORE, 94376181231616, 94376181235711,
+STORE, 94376185425920, 94376185430015,
+STORE, 94376189620224, 94376189624319,
+STORE, 94376193814528, 94376193818623,
+STORE, 94376198008832, 94376198012927,
+STORE, 94376202203136, 94376202207231,
+STORE, 94376206397440, 94376206401535,
+STORE, 94376210591744, 94376210595839,
+STORE, 94376214786048, 94376214790143,
+STORE, 94376218980352, 94376218984447,
+STORE, 94376223174656, 94376223178751,
+STORE, 94376227368960, 94376227373055,
+STORE, 94376231563264, 94376231567359,
+STORE, 94376235757568, 94376235761663,
+STORE, 94376239951872, 94376239955967,
+STORE, 94376244146176, 94376244150271,
+STORE, 94376248340480, 94376248344575,
+STORE, 94376252534784, 94376252538879,
+STORE, 94376256729088, 94376256733183,
+STORE, 94376260923392, 94376260927487,
+STORE, 94376265117696, 94376265121791,
+STORE, 94376269312000, 94376269316095,
+STORE, 94376273506304, 94376273510399,
+STORE, 94376277700608, 94376277704703,
+STORE, 94376281894912, 94376281899007,
+STORE, 94376286089216, 94376286093311,
+STORE, 94376290283520, 94376290287615,
+STORE, 94376294477824, 94376294481919,
+STORE, 94376298672128, 94376298676223,
+STORE, 94376302866432, 94376302870527,
+STORE, 94376307060736, 94376307064831,
+STORE, 94376311255040, 94376311259135,
+STORE, 94376315449344, 94376315453439,
+STORE, 94376319643648, 94376319647743,
+STORE, 94376323837952, 94376323842047,
+STORE, 94376328032256, 94376328036351,
+STORE, 94376332226560, 94376332230655,
+STORE, 94376336420864, 94376336424959,
+STORE, 94376340615168, 94376340619263,
+STORE, 94376344809472, 94376344813567,
+STORE, 94376349003776, 94376349007871,
+STORE, 94376353198080, 94376353202175,
+STORE, 94376357392384, 94376357396479,
+STORE, 94376361586688, 94376361590783,
+STORE, 94376365780992, 94376365785087,
+STORE, 94376369975296, 94376369979391,
+STORE, 94376374169600, 94376374173695,
+STORE, 94376378363904, 94376378367999,
+STORE, 94376382558208, 94376382562303,
+STORE, 94376386752512, 94376386756607,
+STORE, 94376390946816, 94376390950911,
+STORE, 94376395141120, 94376395145215,
+STORE, 94376399335424, 94376399339519,
+STORE, 94376403529728, 94376403533823,
+STORE, 94376407724032, 94376407728127,
+STORE, 94376411918336, 94376411922431,
+STORE, 94376416112640, 94376416116735,
+STORE, 94376420306944, 94376420311039,
+STORE, 94376424501248, 94376424505343,
+STORE, 94376428695552, 94376428699647,
+STORE, 94376432889856, 94376432893951,
+STORE, 94376437084160, 94376437088255,
+STORE, 94376441278464, 94376441282559,
+STORE, 94376445472768, 94376445476863,
+STORE, 94376449667072, 94376449671167,
+STORE, 94376453861376, 94376453865471,
+STORE, 94376458055680, 94376458059775,
+STORE, 94376462249984, 94376462254079,
+STORE, 94376466444288, 94376466448383,
+STORE, 94376470638592, 94376470642687,
+STORE, 94376474832896, 94376474836991,
+STORE, 94376479027200, 94376479031295,
+STORE, 94376483221504, 94376483225599,
+STORE, 94376487415808, 94376487419903,
+STORE, 94376491610112, 94376491614207,
+STORE, 94376495804416, 94376495808511,
+STORE, 94376499998720, 94376500002815,
+STORE, 94376504193024, 94376504197119,
+STORE, 94376508387328, 94376508391423,
+STORE, 94376512581632, 94376512585727,
+STORE, 94376516775936, 94376516780031,
+STORE, 94376520970240, 94376520974335,
+STORE, 94376525164544, 94376525168639,
+STORE, 94376529358848, 94376529362943,
+STORE, 94376533553152, 94376533557247,
+STORE, 94376537747456, 94376537751551,
+STORE, 94376541941760, 94376541945855,
+STORE, 94376546136064, 94376546140159,
+STORE, 94376550330368, 94376550334463,
+STORE, 94376554524672, 94376554528767,
+STORE, 94376558718976, 94376558723071,
+STORE, 94376562913280, 94376562917375,
+STORE, 94376567107584, 94376567111679,
+STORE, 94376571301888, 94376571305983,
+STORE, 94376575496192, 94376575500287,
+STORE, 94376579690496, 94376579694591,
+STORE, 94376583884800, 94376583888895,
+STORE, 94376588079104, 94376588083199,
+STORE, 94376592273408, 94376592277503,
+STORE, 94376596467712, 94376596471807,
+STORE, 94376600662016, 94376600666111,
+STORE, 94376604856320, 94376604860415,
+STORE, 94376609050624, 94376609054719,
+STORE, 94376613244928, 94376613249023,
+STORE, 94376617439232, 94376617443327,
+STORE, 94376621633536, 94376621637631,
+STORE, 94376625827840, 94376625831935,
+STORE, 94376630022144, 94376630026239,
+STORE, 94376634216448, 94376634220543,
+STORE, 94376638410752, 94376638414847,
+STORE, 94376642605056, 94376642609151,
+STORE, 94376646799360, 94376646803455,
+STORE, 94376650993664, 94376650997759,
+STORE, 94376655187968, 94376655192063,
+STORE, 94376659382272, 94376659386367,
+STORE, 94376663576576, 94376663580671,
+STORE, 94376667770880, 94376667774975,
+STORE, 94376671965184, 94376671969279,
+STORE, 94376676159488, 94376676163583,
+STORE, 94376680353792, 94376680357887,
+STORE, 94376684548096, 94376684552191,
+STORE, 94376688742400, 94376688746495,
+STORE, 94376692936704, 94376692940799,
+STORE, 94376697131008, 94376697135103,
+STORE, 94376701325312, 94376701329407,
+STORE, 94376705519616, 94376705523711,
+STORE, 94376709713920, 94376709718015,
+STORE, 94376713908224, 94376713912319,
+STORE, 94376718102528, 94376718106623,
+STORE, 94376722296832, 94376722300927,
+STORE, 94376726491136, 94376726495231,
+STORE, 94376730685440, 94376730689535,
+STORE, 94376734879744, 94376734883839,
+STORE, 94376739074048, 94376739078143,
+STORE, 94376743268352, 94376743272447,
+STORE, 94376747462656, 94376747466751,
+STORE, 94376751656960, 94376751661055,
+STORE, 94376755851264, 94376755855359,
+STORE, 94376760045568, 94376760049663,
+STORE, 94376764239872, 94376764243967,
+STORE, 94376768434176, 94376768438271,
+STORE, 94376772628480, 94376772632575,
+STORE, 94376776822784, 94376776826879,
+STORE, 94376781017088, 94376781021183,
+STORE, 94376785211392, 94376785215487,
+STORE, 94376789405696, 94376789409791,
+STORE, 94376793600000, 94376793604095,
+STORE, 94376797794304, 94376797798399,
+STORE, 94376801988608, 94376801992703,
+STORE, 94376806182912, 94376806187007,
+STORE, 94376810377216, 94376810381311,
+STORE, 94376814571520, 94376814575615,
+STORE, 94376818765824, 94376818769919,
+STORE, 94376822960128, 94376822964223,
+STORE, 94376827154432, 94376827158527,
+STORE, 94376831348736, 94376831352831,
+STORE, 94376835543040, 94376835547135,
+STORE, 94376839737344, 94376839741439,
+STORE, 94376843931648, 94376843935743,
+STORE, 94376848125952, 94376848130047,
+STORE, 94376852320256, 94376852324351,
+STORE, 94376856514560, 94376856518655,
+STORE, 94376860708864, 94376860712959,
+STORE, 94376864903168, 94376864907263,
+STORE, 94376869097472, 94376869101567,
+STORE, 94376873291776, 94376873295871,
+STORE, 94376877486080, 94376877490175,
+STORE, 94376881680384, 94376881684479,
+STORE, 94376885874688, 94376885878783,
+STORE, 94376890068992, 94376890073087,
+STORE, 94376894263296, 94376894267391,
+STORE, 94376898457600, 94376898461695,
+STORE, 94376902651904, 94376902655999,
+STORE, 94376906846208, 94376906850303,
+STORE, 94376911040512, 94376911044607,
+STORE, 94376915234816, 94376915238911,
+STORE, 94376919429120, 94376919433215,
+STORE, 94376923623424, 94376923627519,
+STORE, 94376927817728, 94376927821823,
+STORE, 94376932012032, 94376932016127,
+STORE, 94376936206336, 94376936210431,
+STORE, 94376940400640, 94376940404735,
+STORE, 94376944594944, 94376944599039,
+STORE, 94376948789248, 94376948793343,
+STORE, 94376952983552, 94376952987647,
+STORE, 94376957177856, 94376957181951,
+STORE, 94376961372160, 94376961376255,
+STORE, 94376965566464, 94376965570559,
+STORE, 94376969760768, 94376969764863,
+STORE, 94376973955072, 94376973959167,
+STORE, 94376978149376, 94376978153471,
+STORE, 94376982343680, 94376982347775,
+STORE, 94376986537984, 94376986542079,
+STORE, 94376990732288, 94376990736383,
+STORE, 94376994926592, 94376994930687,
+STORE, 94376999120896, 94376999124991,
+STORE, 94377003315200, 94377003319295,
+STORE, 94377007509504, 94377007513599,
+STORE, 94377011703808, 94377011707903,
+STORE, 94377015898112, 94377015902207,
+STORE, 94377020092416, 94377020096511,
+STORE, 94377024286720, 94377024290815,
+STORE, 94377028481024, 94377028485119,
+STORE, 94377032675328, 94377032679423,
+STORE, 94377036869632, 94377036873727,
+STORE, 94377041063936, 94377041068031,
+STORE, 94377045258240, 94377045262335,
+STORE, 94377049452544, 94377049456639,
+STORE, 94377053646848, 94377053650943,
+STORE, 94377057841152, 94377057845247,
+STORE, 94377062035456, 94377062039551,
+STORE, 94377066229760, 94377066233855,
+STORE, 94377070424064, 94377070428159,
+STORE, 94377074618368, 94377074622463,
+STORE, 94377078812672, 94377078816767,
+STORE, 94377083006976, 94377083011071,
+STORE, 94377087201280, 94377087205375,
+STORE, 94377091395584, 94377091399679,
+STORE, 94377095589888, 94377095593983,
+STORE, 94377099784192, 94377099788287,
+STORE, 94377103978496, 94377103982591,
+STORE, 94377108172800, 94377108176895,
+STORE, 94377112367104, 94377112371199,
+STORE, 94377116561408, 94377116565503,
+STORE, 94377120755712, 94377120759807,
+STORE, 94377124950016, 94377124954111,
+STORE, 94377129144320, 94377129148415,
+STORE, 94377133338624, 94377133342719,
+STORE, 94377137532928, 94377137537023,
+STORE, 94377141727232, 94377141731327,
+STORE, 94377145921536, 94377145925631,
+STORE, 94377150115840, 94377150119935,
+STORE, 94377154310144, 94377154314239,
+STORE, 94377158504448, 94377158508543,
+STORE, 94377162698752, 94377162702847,
+STORE, 94377166893056, 94377166897151,
+STORE, 94377171087360, 94377171091455,
+STORE, 94377175281664, 94377175285759,
+STORE, 94377179475968, 94377179480063,
+STORE, 94377183670272, 94377183674367,
+STORE, 94377187864576, 94377187868671,
+STORE, 94377192058880, 94377192062975,
+STORE, 94377196253184, 94377196257279,
+STORE, 94377200447488, 94377200451583,
+STORE, 94377204641792, 94377204645887,
+SNULL, 94376135094271, 94376135098367,
+STORE, 94376135090176, 94376135094271,
+STORE, 94376135094272, 94376135098367,
+SNULL, 94376135094272, 94377208836095,
+ };
+ unsigned long set42[] = {
+STORE, 314572800, 1388314623,
+STORE, 1462157312, 1462169599,
+STORE, 1462169600, 1462185983,
+STORE, 1462185984, 1462190079,
+STORE, 1462190080, 1462194175,
+STORE, 1462194176, 1462198271,
+STORE, 1879986176, 1881800703,
+STORE, 1881800704, 1882034175,
+STORE, 1882034176, 1882193919,
+STORE, 1882193920, 1882406911,
+STORE, 1882406912, 1882451967,
+STORE, 1882451968, 1882996735,
+STORE, 1882996736, 1885892607,
+STORE, 1885892608, 1885896703,
+STORE, 1885896704, 1885904895,
+STORE, 1885904896, 1885908991,
+STORE, 1885908992, 1885913087,
+STORE, 1885913088, 1885966335,
+STORE, 1885966336, 1886232575,
+STORE, 1886232576, 1886236671,
+STORE, 1886236672, 1886240767,
+STORE, 1886240768, 1886244863,
+STORE, 1886244864, 1886248959,
+STORE, 1886248960, 1886294015,
+STORE, 1886294016, 1886494719,
+STORE, 1886494720, 1886498815,
+STORE, 1886498816, 1886502911,
+STORE, 1886502912, 1886507007,
+STORE, 1886507008, 1886511103,
+STORE, 1886511104, 1886556159,
+STORE, 1886556160, 1886629887,
+STORE, 1886629888, 1886633983,
+STORE, 1886633984, 1886638079,
+STORE, 1886638080, 1886642175,
+STORE, 1886642176, 1886646271,
+STORE, 1886646272, 1886666751,
+STORE, 1886666752, 1886670847,
+STORE, 1886670848, 1886674943,
+STORE, 1886674944, 1886679039,
+STORE, 1886679040, 1895419903,
+STORE, 1895419904, 1895550975,
+STORE, 1895550976, 1896148991,
+STORE, 1896148992, 1897189375,
+STORE, 1897189376, 1897701375,
+STORE, 1897701376, 1897803775,
+STORE, 1897803776, 1897816063,
+STORE, 1897816064, 1899913215,
+STORE, 1899913216, 1909379071,
+STORE, 1909379072, 1909387263,
+STORE, 1909387264, 1909391359,
+STORE, 1909391360, 1909432319,
+STORE, 1909432320, 1909436415,
+STORE, 1909436416, 1909440511,
+STORE, 1909440512, 1909460991,
+STORE, 1909460992, 1909547007,
+STORE, 1909547008, 1909551103,
+STORE, 1909551104, 1909555199,
+STORE, 1909555200, 1909559295,
+STORE, 1909559296, 1909563391,
+STORE, 1909563392, 1909739519,
+STORE, 1909739520, 1910566911,
+STORE, 1910566912, 1910571007,
+STORE, 1910571008, 1910575103,
+STORE, 1910575104, 1910579199,
+STORE, 1910579200, 1910583295,
+STORE, 1910583296, 1910587391,
+STORE, 1910587392, 1910620159,
+STORE, 1910620160, 1910624255,
+STORE, 1910624256, 1910628351,
+STORE, 1910628352, 1910632447,
+STORE, 1910632448, 1910652927,
+STORE, 1910652928, 1910657023,
+STORE, 1910657024, 1910661119,
+STORE, 1910661120, 1910665215,
+STORE, 1910665216, 1910669311,
+STORE, 1910669312, 1910677503,
+STORE, 1910677504, 1910681599,
+STORE, 1910681600, 1910685695,
+STORE, 1910685696, 1910689791,
+STORE, 1910689792, 1910697983,
+STORE, 1910697984, 1910702079,
+STORE, 1910702080, 1910706175,
+STORE, 1910706176, 1910710271,
+STORE, 1910710272, 1914093567,
+STORE, 1914093568, 1914097663,
+STORE, 1914097664, 1969434623,
+STORE, 1969434624, 1977819135,
+STORE, 3290435584, 3426750463,
+STORE, 3426750464, 3426754559,
+STORE, 3426754560, 3426762751,
+STORE, 3426762752, 3426766847,
+STORE, 3426766848, 3426770943,
+STORE, 3427037184, 3427061759,
+STORE, 3427061760, 3427135487,
+STORE, 3427135488, 3427143679,
+STORE, 3427143680, 3427147775,
+STORE, 3427147776, 3427209215,
+STORE, 3427319808, 3432116223,
+STORE, 3432116224, 3450130431,
+STORE, 3450130432, 3451027455,
+STORE, 3451027456, 3451031551,
+STORE, 3451031552, 3451461631,
+STORE, 3451736064, 3456688127,
+STORE, 3456688128, 3475222527,
+STORE, 3475222528, 3476119551,
+STORE, 3476119552, 3476127743,
+STORE, 3476127744, 3476553727,
+STORE, 3476631552, 3477315583,
+STORE, 3477315584, 3479949311,
+STORE, 3479949312, 3480002559,
+STORE, 3480002560, 3480006655,
+STORE, 3480006656, 3480432639,
+STORE, 3480539136, 3480543231,
+STORE, 3480543232, 3480547327,
+STORE, 3480547328, 3480555519,
+STORE, 3480854528, 3480903679,
+STORE, 3480903680, 3480969215,
+STORE, 3480969216, 3480977407,
+STORE, 3480977408, 3480981503,
+STORE, 3481030656, 3481092095,
+STORE, 3481092096, 3481235455,
+STORE, 3481235456, 3481243647,
+STORE, 3481243648, 3481247743,
+STORE, 3481436160, 3481444351,
+STORE, 3481444352, 3481456639,
+STORE, 3481456640, 3481460735,
+STORE, 3481460736, 3481464831,
+STORE, 3481587712, 3481645055,
+STORE, 3481645056, 3481772031,
+STORE, 3481772032, 3481776127,
+STORE, 3481776128, 3481780223,
+STORE, 3481874432, 3481935871,
+STORE, 3481935872, 3482030079,
+STORE, 3482030080, 3482038271,
+STORE, 3482038272, 3482042367,
+STORE, 3482198016, 3482230783,
+STORE, 3482230784, 3482271743,
+STORE, 3482271744, 3482279935,
+STORE, 3482279936, 3482284031,
+STORE, 3482562560, 3482566655,
+STORE, 3482566656, 3482570751,
+STORE, 3482570752, 3482574847,
+STORE, 3482636288, 3482689535,
+STORE, 3482689536, 3482746879,
+STORE, 3482746880, 3482755071,
+STORE, 3482755072, 3482759167,
+STORE, 3482972160, 3483062271,
+STORE, 3483062272, 3483242495,
+STORE, 3483242496, 3483246591,
+STORE, 3483246592, 3483250687,
+STORE, 3483398144, 3483688959,
+STORE, 3483688960, 3484114943,
+STORE, 3484114944, 3484131327,
+STORE, 3484131328, 3484135423,
+STORE, 3484135424, 3484143615,
+STORE, 3484184576, 3484475391,
+STORE, 3484475392, 3485028351,
+STORE, 3485028352, 3485057023,
+STORE, 3485057024, 3485061119,
+STORE, 3485360128, 3485364223,
+STORE, 3485364224, 3485368319,
+STORE, 3485368320, 3485372415,
+STORE, 3485589504, 3485593599,
+STORE, 3485593600, 3485597695,
+STORE, 3485597696, 3485601791,
+STORE, 3485913088, 3485937663,
+STORE, 3485937664, 3485974527,
+STORE, 3485974528, 3485982719,
+STORE, 3485982720, 3485986815,
+STORE, 3486052352, 3486056447,
+STORE, 3486056448, 3486064639,
+STORE, 3486064640, 3486068735,
+STORE, 3486068736, 3486072831,
+STORE, 3486294016, 3486302207,
+STORE, 3486302208, 3486306303,
+STORE, 3486306304, 3486310399,
+STORE, 3486310400, 3486314495,
+STORE, 3486670848, 3486679039,
+STORE, 3486679040, 3486683135,
+STORE, 3486683136, 3486687231,
+STORE, 3486687232, 3486691327,
+STORE, 3486863360, 3486871551,
+STORE, 3486871552, 3486875647,
+STORE, 3486875648, 3486879743,
+STORE, 3486879744, 3486883839,
+STORE, 3487584256, 3522543615,
+STORE, 3522543616, 3523321855,
+STORE, 3523321856, 3523342335,
+STORE, 3523342336, 3523387391,
+STORE, 3523387392, 3523391487,
+STORE, 3523391488, 3523395583,
+STORE, 3523477504, 3523686399,
+STORE, 3523686400, 3523981311,
+STORE, 3523981312, 3523997695,
+STORE, 3523997696, 3524001791,
+STORE, 3524177920, 3525013503,
+STORE, 3525013504, 3526582271,
+STORE, 3526582272, 3526606847,
+STORE, 3526606848, 3526610943,
+STORE, 3526610944, 3526615039,
+STORE, 3526672384, 3526746111,
+STORE, 3526746112, 3526860799,
+STORE, 3526860800, 3526868991,
+STORE, 3526868992, 3526873087,
+STORE, 3527000064, 3527475199,
+STORE, 3527475200, 3527479295,
+STORE, 3527479296, 3527573503,
+STORE, 3527573504, 3527581695,
+STORE, 3527581696, 3527585791,
+STORE, 3527585792, 3527606271,
+STORE, 3527909376, 3527913471,
+STORE, 3527913472, 3527917567,
+STORE, 3527917568, 3527921663,
+STORE, 3527950336, 3528011775,
+STORE, 3528011776, 3528093695,
+STORE, 3528093696, 3528101887,
+STORE, 3528101888, 3528105983,
+STORE, 3528228864, 3528241151,
+STORE, 3528241152, 3528261631,
+STORE, 3528261632, 3528265727,
+STORE, 3528273920, 3528593407,
+STORE, 3528593408, 3528609791,
+STORE, 3528609792, 3528638463,
+STORE, 3528638464, 3528642559,
+STORE, 3528642560, 3528646655,
+STORE, 3528880128, 3528912895,
+STORE, 3528912896, 3528962047,
+STORE, 3528962048, 3528966143,
+STORE, 3528966144, 3528970239,
+STORE, 3528982528, 3530293247,
+STORE, 3530366976, 3530825727,
+STORE, 3530825728, 3531317247,
+STORE, 3531317248, 3541041151,
+STORE, 3541041152, 3541303295,
+STORE, 3541430272, 3566206975,
+STORE, 3566206976, 3566993407,
+STORE, 3567239168, 3587571711,
+STORE, 3587571712, 3588284415,
+STORE, 3588284416, 3588661247,
+STORE, 3588661248, 3589066751,
+STORE, 3589066752, 3589574655,
+STORE, 3589574656, 3590078463,
+STORE, 3590078464, 3590373375,
+STORE, 3590373376, 3590668287,
+STORE, 3590668288, 3590963199,
+STORE, 3590963200, 3591294975,
+STORE, 3591294976, 3591602175,
+STORE, 3591602176, 3591933951,
+STORE, 3591933952, 3592241151,
+STORE, 3592241152, 3592572927,
+STORE, 3592572928, 3592876031,
+STORE, 3592876032, 3593211903,
+STORE, 3593211904, 3593547775,
+STORE, 3593547776, 3593650175,
+STORE, 3593650176, 3593928703,
+STORE, 3593928704, 3593936895,
+STORE, 3593936896, 3593940991,
+STORE, 3594006528, 3594301439,
+STORE, 3594301440, 3594739711,
+STORE, 3594739712, 3594756095,
+STORE, 3594756096, 3594760191,
+STORE, 3594760192, 3594768383,
+STORE, 3594952704, 3595051007,
+STORE, 3595051008, 3595223039,
+STORE, 3595223040, 3595227135,
+STORE, 3595227136, 3595235327,
+STORE, 3595431936, 3595775999,
+STORE, 3595776000, 3596701695,
+STORE, 3596701696, 3596742655,
+STORE, 3596742656, 3596746751,
+STORE, 3596746752, 3596750847,
+STORE, 3596767232, 3597070335,
+STORE, 3597070336, 3597402111,
+STORE, 3597402112, 3598188543,
+STORE, 3598262272, 3623428095,
+STORE, 3623428096, 3623432191,
+STORE, 3623432192, 3623436287,
+STORE, 3623436288, 3623440383,
+STORE, 3623616512, 3623878655,
+STORE, 3624169472, 3624300543,
+STORE, 3627524096, 3628523519,
+STORE, 3628523520, 3629522943,
+STORE, 3696631808, 3730186239,
+STORE, 3730186240, 3763740671,
+STORE, 3763740672, 3764027391,
+STORE, 3764027392, 3765133311,
+STORE, 3765133312, 3765145599,
+STORE, 3765145600, 3765149695,
+STORE, 3765178368, 3766022143,
+STORE, 3766022144, 3768791039,
+STORE, 3768791040, 3768840191,
+STORE, 3768840192, 3768844287,
+STORE, 3768897536, 3768913919,
+STORE, 3768913920, 3768934399,
+STORE, 3768934400, 3768938495,
+STORE, 3769016320, 3769147391,
+STORE, 3769147392, 3769233407,
+STORE, 3769233408, 3769356287,
+STORE, 3769356288, 3769360383,
+STORE, 3769360384, 3769368575,
+STORE, 3769376768, 3794542591,
+STORE, 3794542592, 3794599935,
+STORE, 3794599936, 3794731007,
+STORE, 3794731008, 3794735103,
+STORE, 3794735104, 3794743295,
+STORE, 3794849792, 3794980863,
+STORE, 3794980864, 3794984959,
+STORE, 3794984960, 3794989055,
+STORE, 3794989056, 3794993151,
+STORE, 3794993152, 3794997247,
+STORE, 3795103744, 3795128319,
+STORE, 3795128320, 3795165183,
+STORE, 3795165184, 3795169279,
+STORE, 3795169280, 3795173375,
+STORE, 3795210240, 3795357695,
+STORE, 3795357696, 3795365887,
+STORE, 3795365888, 3795374079,
+STORE, 3795374080, 3795378175,
+STORE, 3795378176, 3795382271,
+STORE, 3795406848, 3795738623,
+STORE, 3795738624, 3795742719,
+STORE, 3795742720, 3795755007,
+STORE, 3795755008, 3795759103,
+STORE, 3795763200, 3795894271,
+STORE, 3795894272, 3796041727,
+STORE, 3796041728, 3796054015,
+STORE, 3796054016, 3796066303,
+STORE, 3796066304, 3796070399,
+STORE, 3796176896, 3796205567,
+STORE, 3796205568, 3796250623,
+STORE, 3796250624, 3796254719,
+STORE, 3796254720, 3796258815,
+STORE, 3796262912, 3796393983,
+STORE, 3796393984, 3796516863,
+STORE, 3796516864, 3796873215,
+STORE, 3796873216, 3796885503,
+STORE, 3796885504, 3796889599,
+STORE, 3796963328, 3796967423,
+STORE, 3796967424, 3796975615,
+STORE, 3796975616, 3796979711,
+STORE, 3797000192, 3797307391,
+STORE, 3797307392, 3797311487,
+STORE, 3797311488, 3797315583,
+STORE, 3797315584, 3797323775,
+STORE, 3797327872, 3797450751,
+STORE, 3797450752, 3797458943,
+STORE, 3797458944, 3797471231,
+STORE, 3797471232, 3797475327,
+STORE, 3797577728, 3797700607,
+STORE, 3797700608, 3797721087,
+STORE, 3797721088, 3797733375,
+STORE, 3797733376, 3797741567,
+STORE, 3797741568, 3797864447,
+STORE, 3797864448, 3797995519,
+STORE, 3797995520, 3798048767,
+STORE, 3798048768, 3798179839,
+STORE, 3798179840, 3798188031,
+STORE, 3798188032, 3798192127,
+STORE, 3798290432, 3798302719,
+STORE, 3798302720, 3798323199,
+STORE, 3798323200, 3798327295,
+STORE, 3798327296, 3798331391,
+STORE, 3798429696, 3798433791,
+STORE, 3798433792, 3798552575,
+STORE, 3798552576, 3798556671,
+STORE, 3798556672, 3798568959,
+STORE, 3798568960, 3798573055,
+STORE, 3798573056, 3798581247,
+STORE, 3798618112, 3798749183,
+STORE, 3798749184, 3798855679,
+STORE, 3798855680, 3798966271,
+STORE, 3798966272, 3798982655,
+STORE, 3798982656, 3798986751,
+STORE, 3799101440, 3799171071,
+STORE, 3799171072, 3799240703,
+STORE, 3799240704, 3799248895,
+STORE, 3799248896, 3799252991,
+STORE, 3799326720, 3799650303,
+STORE, 3799650304, 3800629247,
+STORE, 3800629248, 3800641535,
+STORE, 3800641536, 3800645631,
+STORE, 3800645632, 3800649727,
+STORE, 3800649728, 3800903679,
+STORE, 3800903680, 3800936447,
+STORE, 3800936448, 3800969215,
+STORE, 3800969216, 3800981503,
+STORE, 3800981504, 3800985599,
+STORE, 3801001984, 3801133055,
+STORE, 3801133056, 3801202687,
+STORE, 3801202688, 3801591807,
+STORE, 3801591808, 3801599999,
+STORE, 3801600000, 3801604095,
+STORE, 3801604096, 3801608191,
+STORE, 3801608192, 3801739263,
+STORE, 3801739264, 3801755647,
+STORE, 3801755648, 3801796607,
+STORE, 3801796608, 3801804799,
+STORE, 3801804800, 3801808895,
+STORE, 3801878528, 3801944063,
+STORE, 3801944064, 3802116095,
+STORE, 3802116096, 3802124287,
+STORE, 3802124288, 3802128383,
+STORE, 3802136576, 3803447295,
+STORE, 3803492352, 3803553791,
+STORE, 3803553792, 3804233727,
+STORE, 3804233728, 3806068735,
+STORE, 3806121984, 3806253055,
+STORE, 3806253056, 3806674943,
+STORE, 3806674944, 3807117311,
+STORE, 3807117312, 3807379455,
+STORE, 3807379456, 3807432703,
+STORE, 3807432704, 3807563775,
+STORE, 3807563776, 3809202175,
+STORE, 3809202176, 3810250751,
+STORE, 3810250752, 3827027967,
+STORE, 3827027968, 3829125119,
+STORE, 3829125120, 3837513727,
+STORE, 3837513728, 3839610879,
+STORE, 3839610880, 3847999487,
+STORE, 3847999488, 3856392191,
+STORE, 3856392192, 3864784895,
+STORE, 3864784896, 3868983295,
+STORE, 3868983296, 3885760511,
+STORE, 3885760512, 3886809087,
+STORE, 3886809088, 3887857663,
+STORE, 3887857664, 3888119807,
+STORE, 3888144384, 3888148479,
+STORE, 3888148480, 3888218111,
+STORE, 3888218112, 3888222207,
+STORE, 3888222208, 3888353279,
+STORE, 3888353280, 3889172479,
+STORE, 3889172480, 3892314111,
+STORE, 3892314112, 3892576255,
+STORE, 3892588544, 3892637695,
+STORE, 3892637696, 3892686847,
+STORE, 3892686848, 3892744191,
+STORE, 3892748288, 3892785151,
+STORE, 3892785152, 3895459839,
+STORE, 3895459840, 3895721983,
+STORE, 3895738368, 3895885823,
+STORE, 3895885824, 3897081855,
+STORE, 3897081856, 3906482175,
+STORE, 3906482176, 3916144639,
+STORE, 3916144640, 3925766143,
+STORE, 3925766144, 3926974463,
+STORE, 3926974464, 3928367103,
+STORE, 3928367104, 3928911871,
+STORE, 3928911872, 3933995007,
+STORE, 3933995008, 3935830015,
+STORE, 3935830016, 3935846399,
+STORE, 3935879168, 3936010239,
+STORE, 3936010240, 3936026623,
+STORE, 3936026624, 3936034815,
+STORE, 3936034816, 3936051199,
+STORE, 3936051200, 3936055295,
+STORE, 3936071680, 3936137215,
+STORE, 3936137216, 3936202751,
+STORE, 3936202752, 3936219135,
+STORE, 3936235520, 3936251903,
+STORE, 3936268288, 3936276479,
+STORE, 3936276480, 3936284671,
+STORE, 3936284672, 3936288767,
+STORE, 3936288768, 3936292863,
+STORE, 3936296960, 3936354303,
+STORE, 3936354304, 3936616447,
+STORE, 3936628736, 3936669695,
+STORE, 3936669696, 3936747519,
+STORE, 3936747520, 3936870399,
+STORE, 3936870400, 3936874495,
+STORE, 3936874496, 3936878591,
+STORE, 3936882688, 3936903167,
+STORE, 3936911360, 3936948223,
+STORE, 3936948224, 3936964607,
+STORE, 3936964608, 3937103871,
+STORE, 3937103872, 3937107967,
+STORE, 3937132544, 3937161215,
+STORE, 3937189888, 3937255423,
+STORE, 3937255424, 3938512895,
+STORE, 3938512896, 3945435135,
+STORE, 3945435136, 3945476095,
+STORE, 3945476096, 3945484287,
+STORE, 3945484288, 3945496575,
+STORE, 3945500672, 3945541631,
+STORE, 3945558016, 3945566207,
+STORE, 3945566208, 3945594879,
+STORE, 3945594880, 3945598975,
+STORE, 3945598976, 3945603071,
+STORE, 3945611264, 3945742335,
+STORE, 3945742336, 3945844735,
+STORE, 3945844736, 3945848831,
+STORE, 3945848832, 3945861119,
+STORE, 3945861120, 3945865215,
+STORE, 3945869312, 3945897983,
+STORE, 3945897984, 3946303487,
+STORE, 3946303488, 3946397695,
+STORE, 3946397696, 3946569727,
+STORE, 3946569728, 3946573823,
+STORE, 3946573824, 3946594303,
+STORE, 3946594304, 3946663935,
+STORE, 3946663936, 3946708991,
+STORE, 3946708992, 3946823679,
+STORE, 3946823680, 3946827775,
+STORE, 3946827776, 3946831871,
+STORE, 3946831872, 3946860543,
+STORE, 3946893312, 3946897407,
+STORE, 3946897408, 3946905599,
+STORE, 3946905600, 3946909695,
+STORE, 3946909696, 3946913791,
+STORE, 3946913792, 3946930175,
+STORE, 3946930176, 3946967039,
+STORE, 3946967040, 3947102207,
+STORE, 3947102208, 3948412927,
+STORE, 3948441600, 3948556287,
+STORE, 3948556288, 3948576767,
+STORE, 3948576768, 3948597247,
+STORE, 3948597248, 3948605439,
+STORE, 3948605440, 3948609535,
+STORE, 3948609536, 3948654591,
+STORE, 3948654592, 3948781567,
+STORE, 3948781568, 3948822527,
+STORE, 3948822528, 3948904447,
+STORE, 3948904448, 3948908543,
+STORE, 3948908544, 3948912639,
+STORE, 3948945408, 3949043711,
+STORE, 3949043712, 3949174783,
+STORE, 3949174784, 3949191167,
+STORE, 3949191168, 3949195263,
+STORE, 3949207552, 3949252607,
+STORE, 3949252608, 3949256703,
+STORE, 3949256704, 3949363199,
+STORE, 3949363200, 3949367295,
+STORE, 3949367296, 3949379583,
+STORE, 3949379584, 3949383679,
+STORE, 3949383680, 3949400063,
+STORE, 3949400064, 3949404159,
+STORE, 3949416448, 3949481983,
+STORE, 3949481984, 3949486079,
+STORE, 3949486080, 3949592575,
+STORE, 3949592576, 3949596671,
+STORE, 3949596672, 3949621247,
+STORE, 3949621248, 3949662207,
+STORE, 3949662208, 3949666303,
+STORE, 3949694976, 3949727743,
+STORE, 3949727744, 3949731839,
+STORE, 3949731840, 3949838335,
+STORE, 3949838336, 3949842431,
+STORE, 3949842432, 3949846527,
+STORE, 3949846528, 3949854719,
+STORE, 3949854720, 3949858815,
+STORE, 3949858816, 3949862911,
+STORE, 3949867008, 3949891583,
+STORE, 3949891584, 3949928447,
+STORE, 3949928448, 3949993983,
+STORE, 3949993984, 3950043135,
+STORE, 3950043136, 3950059519,
+STORE, 3950059520, 3950096383,
+STORE, 3950096384, 3950100479,
+STORE, 3950100480, 3950104575,
+STORE, 3950104576, 3950157823,
+STORE, 3950157824, 3950292991,
+STORE, 3950292992, 3950346239,
+STORE, 3950346240, 3950477311,
+STORE, 3950477312, 3950485503,
+STORE, 3950485504, 3950489599,
+STORE, 3950493696, 3950510079,
+STORE, 3950510080, 3950661631,
+STORE, 3950661632, 3951005695,
+STORE, 3951005696, 3951026175,
+STORE, 3951026176, 3951030271,
+STORE, 3951030272, 3951054847,
+STORE, 3951054848, 3951116287,
+STORE, 3951116288, 3951144959,
+STORE, 3951144960, 3951149055,
+STORE, 3951149056, 3951194111,
+STORE, 3951194112, 3951202303,
+STORE, 3951202304, 3951206399,
+STORE, 3951210496, 3951226879,
+STORE, 3951226880, 3951329279,
+STORE, 3951329280, 3951366143,
+STORE, 3951366144, 3951411199,
+STORE, 3951411200, 3951415295,
+STORE, 3951415296, 3951419391,
+STORE, 3951419392, 3951452159,
+STORE, 3951452160, 3951566847,
+STORE, 3951566848, 3951812607,
+STORE, 3951812608, 3952173055,
+STORE, 3952173056, 3952214015,
+STORE, 3952214016, 3952218111,
+STORE, 3952222208, 3952250879,
+STORE, 3952250880, 3952369663,
+STORE, 3952369664, 3952488447,
+STORE, 3952488448, 3952627711,
+STORE, 3952627712, 3952635903,
+STORE, 3952635904, 3952639999,
+STORE, 3952652288, 3952668671,
+STORE, 3952668672, 3953000447,
+STORE, 3953000448, 3953004543,
+STORE, 3953004544, 3953008639,
+STORE, 3953008640, 3953012735,
+STORE, 3953012736, 3953037311,
+STORE, 3953037312, 3953151999,
+STORE, 3953152000, 3953291263,
+STORE, 3953291264, 3953324031,
+STORE, 3953324032, 3953364991,
+STORE, 3953364992, 3953373183,
+STORE, 3953373184, 3953377279,
+STORE, 3953381376, 3953410047,
+STORE, 3953410048, 3953491967,
+STORE, 3953491968, 3953643519,
+STORE, 3953643520, 3953651711,
+STORE, 3953651712, 3953655807,
+STORE, 3953659904, 3953766399,
+STORE, 3953766400, 3953774591,
+STORE, 3953774592, 3953786879,
+STORE, 3953786880, 3953790975,
+STORE, 3953790976, 3953823743,
+STORE, 3953823744, 3953963007,
+STORE, 3953963008, 3954024447,
+STORE, 3954024448, 3954118655,
+STORE, 3954118656, 3954122751,
+STORE, 3954122752, 3954126847,
+STORE, 3954130944, 3954184191,
+STORE, 3954184192, 3954294783,
+STORE, 3954294784, 3954323455,
+STORE, 3954323456, 3954393087,
+STORE, 3954393088, 3954397183,
+STORE, 3954397184, 3954401279,
+STORE, 3954401280, 3954405375,
+STORE, 3954409472, 3954528255,
+STORE, 3954528256, 3954737151,
+STORE, 3954737152, 3955052543,
+STORE, 3955052544, 3955060735,
+STORE, 3955060736, 3955064831,
+STORE, 3955068928, 3955105791,
+STORE, 3955105792, 3955167231,
+STORE, 3955167232, 3955277823,
+STORE, 3955277824, 3955310591,
+STORE, 3955310592, 3955351551,
+STORE, 3955351552, 3955359743,
+STORE, 3955359744, 3955363839,
+STORE, 3955363840, 3955392511,
+STORE, 3955392512, 3955453951,
+STORE, 3955453952, 3955601407,
+STORE, 3955601408, 3955777535,
+STORE, 3955777536, 3955982335,
+STORE, 3955982336, 3956011007,
+STORE, 3956011008, 3956015103,
+STORE, 3956023296, 3956039679,
+STORE, 3956039680, 3956125695,
+STORE, 3956125696, 3956129791,
+STORE, 3956129792, 3956133887,
+STORE, 3956133888, 3956137983,
+STORE, 3956142080, 3956449279,
+STORE, 3956449280, 3956543487,
+STORE, 3956543488, 3956719615,
+STORE, 3956719616, 3956731903,
+STORE, 3956731904, 3956735999,
+STORE, 3956744192, 3956793343,
+STORE, 3956793344, 3956887551,
+STORE, 3956887552, 3956953087,
+STORE, 3956953088, 3957035007,
+STORE, 3957035008, 3957039103,
+STORE, 3957039104, 3957047295,
+STORE, 3957047296, 3957071871,
+STORE, 3957071872, 3957231615,
+STORE, 3957231616, 3957563391,
+STORE, 3957563392, 3957579775,
+STORE, 3957579776, 3957583871,
+STORE, 3957592064, 3957608447,
+STORE, 3957608448, 3957878783,
+STORE, 3957878784, 3958591487,
+STORE, 3958591488, 3958599679,
+STORE, 3958599680, 3958607871,
+STORE, 3958607872, 3958620159,
+STORE, 3958620160, 3958624255,
+STORE, 3958624256, 3963199487,
+STORE, 3963199488, 3963285503,
+STORE, 3963285504, 3963371519,
+STORE, 3963371520, 3963428863,
+STORE, 3963428864, 3963555839,
+STORE, 3963555840, 3963559935,
+STORE, 3963559936, 3963564031,
+STORE, 3963568128, 3963596799,
+STORE, 3963596800, 3963682815,
+STORE, 3963682816, 3963695103,
+STORE, 3963695104, 3963711487,
+STORE, 3963711488, 3963715583,
+STORE, 3963719680, 3963752447,
+STORE, 3963752448, 3963846655,
+STORE, 3963846656, 3963932671,
+STORE, 3963932672, 3964444671,
+STORE, 3964444672, 3964448767,
+STORE, 3964448768, 3965808639,
+STORE, 3965808640, 3965845503,
+STORE, 3965845504, 3965849599,
+STORE, 3965853696, 3965935615,
+STORE, 3965935616, 3966017535,
+STORE, 3966017536, 3966103551,
+STORE, 3966103552, 3966685183,
+STORE, 3966685184, 3967705087,
+STORE, 3967705088, 3967758335,
+STORE, 3967758336, 3967762431,
+STORE, 3967762432, 3967770623,
+STORE, 3967770624, 3967799295,
+STORE, 3967799296, 3967848447,
+STORE, 3967848448, 3967868927,
+STORE, 3967868928, 3967901695,
+STORE, 3967901696, 3967905791,
+STORE, 3967905792, 3967909887,
+STORE, 3967909888, 3967995903,
+STORE, 3967995904, 3968077823,
+STORE, 3968077824, 3968159743,
+STORE, 3968159744, 3968167935,
+STORE, 3968167936, 3968172031,
+STORE, 3968172032, 3968192511,
+STORE, 3968192512, 3968196607,
+STORE, 3968196608, 3968200703,
+STORE, 3968208896, 3968516095,
+STORE, 3968516096, 3968528383,
+STORE, 3968528384, 3968552959,
+STORE, 3968552960, 3968557055,
+STORE, 3968561152, 3968593919,
+STORE, 3968593920, 3968626687,
+STORE, 3968626688, 3971153919,
+STORE, 3971153920, 3973754879,
+STORE, 3973754880, 3973804031,
+STORE, 3973804032, 3973820415,
+STORE, 3973820416, 3973832703,
+STORE, 3973840896, 3973873663,
+STORE, 3973873664, 3973967871,
+STORE, 3973967872, 3973976063,
+STORE, 3973976064, 3973984255,
+STORE, 3973984256, 3973988351,
+STORE, 3973988352, 3973992447,
+STORE, 3973996544, 3974008831,
+STORE, 3974008832, 3974045695,
+STORE, 3974045696, 3974139903,
+STORE, 3974139904, 3974254591,
+STORE, 3974254592, 3974275071,
+STORE, 3974275072, 3974291455,
+STORE, 3974291456, 3974295551,
+STORE, 3974295552, 3974373375,
+STORE, 3974373376, 3974524927,
+STORE, 3974524928, 3974529023,
+STORE, 3974529024, 3974537215,
+STORE, 3974537216, 3974541311,
+STORE, 3974541312, 3974545407,
+STORE, 3974545408, 3974627327,
+STORE, 3974627328, 3974680575,
+STORE, 3974680576, 3974811647,
+STORE, 3974811648, 3974819839,
+STORE, 3974819840, 3974823935,
+STORE, 3974832128, 3974918143,
+STORE, 3974918144, 3974963199,
+STORE, 3974963200, 3975077887,
+STORE, 3975077888, 3975090175,
+STORE, 3975090176, 3975094271,
+STORE, 3975094272, 3975102463,
+STORE, 3975102464, 3975114751,
+STORE, 3975114752, 3975266303,
+STORE, 3975266304, 3975274495,
+STORE, 3975274496, 3975286783,
+STORE, 3975286784, 3975290879,
+STORE, 3975290880, 3975299071,
+STORE, 3975299072, 3975315455,
+STORE, 3975315456, 3975430143,
+STORE, 3975430144, 3975536639,
+STORE, 3975536640, 3975651327,
+STORE, 3975651328, 3975655423,
+STORE, 3975655424, 3975659519,
+STORE, 3975659520, 3975770111,
+STORE, 3975770112, 3975778303,
+STORE, 3975778304, 3975790591,
+STORE, 3975790592, 3975794687,
+STORE, 3975794688, 3975798783,
+STORE, 3975798784, 3975831551,
+STORE, 3975831552, 3975872511,
+STORE, 3975872512, 3975987199,
+STORE, 3975987200, 3976134655,
+STORE, 3976134656, 3977175039,
+STORE, 3977175040, 3977183231,
+STORE, 3977183232, 3977191423,
+STORE, 3977191424, 3977195519,
+STORE, 3977199616, 3977248767,
+STORE, 3977248768, 3977539583,
+STORE, 3977539584, 3977965567,
+STORE, 3977965568, 3977981951,
+STORE, 3977981952, 3977986047,
+STORE, 3977986048, 3977994239,
+STORE, 3977994240, 3978002431,
+STORE, 3978002432, 3978084351,
+STORE, 3978084352, 3978125311,
+STORE, 3978125312, 3978174463,
+STORE, 3978174464, 3978178559,
+STORE, 3978178560, 3978182655,
+STORE, 3978182656, 3978207231,
+STORE, 3978207232, 3978297343,
+STORE, 3978297344, 3978301439,
+STORE, 3978301440, 3978305535,
+STORE, 3978305536, 3978309631,
+STORE, 3978309632, 3978317823,
+STORE, 3978317824, 3978625023,
+STORE, 3978625024, 3978657791,
+STORE, 3978657792, 3978727423,
+STORE, 3978727424, 3978735615,
+STORE, 3978735616, 3978739711,
+STORE, 3978739712, 3978760191,
+STORE, 3978760192, 3978842111,
+STORE, 3978842112, 3978850303,
+STORE, 3978850304, 3978858495,
+STORE, 3978858496, 3978862591,
+STORE, 3978862592, 3978895359,
+STORE, 3978895360, 3979014143,
+STORE, 3979014144, 3979132927,
+STORE, 3979132928, 3979288575,
+STORE, 3979288576, 3979481087,
+STORE, 3979481088, 3979489279,
+STORE, 3979489280, 3979493375,
+STORE, 3979497472, 3979583487,
+STORE, 3979583488, 3979673599,
+STORE, 3979673600, 3979718655,
+STORE, 3979718656, 3979829247,
+STORE, 3979829248, 3979841535,
+STORE, 3979841536, 3979882495,
+STORE, 3979882496, 3979964415,
+STORE, 3979964416, 3980013567,
+STORE, 3980013568, 3980148735,
+STORE, 3980148736, 3980152831,
+STORE, 3980152832, 3980320767,
+STORE, 3980320768, 3980337151,
+STORE, 3980337152, 3980341247,
+STORE, 3980345344, 3980365823,
+STORE, 3980365824, 3980423167,
+STORE, 3980423168, 3980460031,
+STORE, 3980460032, 3980500991,
+STORE, 3980500992, 3980509183,
+STORE, 3980509184, 3980513279,
+STORE, 3980513280, 3980546047,
+STORE, 3980546048, 3980660735,
+STORE, 3980660736, 3980951551,
+STORE, 3980951552, 3981500415,
+STORE, 3981500416, 3981529087,
+STORE, 3981529088, 3981533183,
+STORE, 3981537280, 3981549567,
+STORE, 3981549568, 3981598719,
+STORE, 3981598720, 3981717503,
+STORE, 3981717504, 3982127103,
+STORE, 3982127104, 3982675967,
+STORE, 3982675968, 3982733311,
+STORE, 3982733312, 3982737407,
+STORE, 3982741504, 3982860287,
+STORE, 3982860288, 3982905343,
+STORE, 3982905344, 3982966783,
+STORE, 3982966784, 3982974975,
+STORE, 3982974976, 3982979071,
+STORE, 3982979072, 3983032319,
+STORE, 3983032320, 3983085567,
+STORE, 3983085568, 3983208447,
+STORE, 3983208448, 3983212543,
+STORE, 3983212544, 3983220735,
+STORE, 3983220736, 3983224831,
+STORE, 3983224832, 3983237119,
+STORE, 3983237120, 3983351807,
+STORE, 3983351808, 3983376383,
+STORE, 3983376384, 3983392767,
+STORE, 3983392768, 3983396863,
+STORE, 3983396864, 3983400959,
+STORE, 3983400960, 3983417343,
+STORE, 3983417344, 3983753215,
+STORE, 3983753216, 3983757311,
+STORE, 3983757312, 3983761407,
+STORE, 3983761408, 3983765503,
+STORE, 3983765504, 3983769599,
+STORE, 3983769600, 3983880191,
+STORE, 3983880192, 3983892479,
+STORE, 3983892480, 3983900671,
+STORE, 3983900672, 3983904767,
+STORE, 3983904768, 3983908863,
+STORE, 3983908864, 3983941631,
+STORE, 3983941632, 3983990783,
+STORE, 3983990784, 3984097279,
+STORE, 3984097280, 3984105471,
+STORE, 3984105472, 3984117759,
+STORE, 3984117760, 3984121855,
+STORE, 3984121856, 3984125951,
+STORE, 3984125952, 3984134143,
+STORE, 3984134144, 3984150527,
+STORE, 3984150528, 3984416767,
+STORE, 3984416768, 3984470015,
+STORE, 3984470016, 3984564223,
+STORE, 3984564224, 3984568319,
+STORE, 3984572416, 3984629759,
+STORE, 3984629760, 3984805887,
+STORE, 3984805888, 3985096703,
+STORE, 3985096704, 3985104895,
+STORE, 3985104896, 3985108991,
+STORE, 3985113088, 3986862079,
+STORE, 3986862080, 3993640959,
+STORE, 3993640960, 3993739263,
+STORE, 3993739264, 3993743359,
+STORE, 3993743360, 3993759743,
+STORE, 3993759744, 3993780223,
+STORE, 3993780224, 3993784319,
+STORE, 3993784320, 3993792511,
+STORE, 3993792512, 3993796607,
+STORE, 3993796608, 3993800703,
+STORE, 3993804800, 3994214399,
+STORE, 3994214400, 3994218495,
+STORE, 3994218496, 3994222591,
+STORE, 3994222592, 3994226687,
+STORE, 3994230784, 3994243071,
+STORE, 3994243072, 3994255359,
+STORE, 3994255360, 3994304511,
+STORE, 3994304512, 3994386431,
+STORE, 3994386432, 3994509311,
+STORE, 3994509312, 3994521599,
+STORE, 3994521600, 3994525695,
+STORE, 3994529792, 3994542079,
+STORE, 3994542080, 3994660863,
+STORE, 3994660864, 3994705919,
+STORE, 3994705920, 3994796031,
+STORE, 3994796032, 3994800127,
+STORE, 3994800128, 3994804223,
+STORE, 3994804224, 3994812415,
+STORE, 3994812416, 3994845183,
+STORE, 3994845184, 3994898431,
+STORE, 3994898432, 3994902527,
+STORE, 3994902528, 3994906623,
+STORE, 3994910720, 3994931199,
+STORE, 3994931200, 3995181055,
+STORE, 3995181056, 3995222015,
+STORE, 3995222016, 3995275263,
+STORE, 3995275264, 3995279359,
+STORE, 3995279360, 3995283455,
+STORE, 3995283456, 3995291647,
+STORE, 3995291648, 3995324415,
+STORE, 3995324416, 3995451391,
+STORE, 3995451392, 3995697151,
+STORE, 3995697152, 3996078079,
+STORE, 3996078080, 3996086271,
+STORE, 3996086272, 3996090367,
+STORE, 3996094464, 3996119039,
+STORE, 3996119040, 3996200959,
+STORE, 3996200960, 3996229631,
+STORE, 3996229632, 3996233727,
+STORE, 3996233728, 3996282879,
+STORE, 3996282880, 3996291071,
+STORE, 3996291072, 3996295167,
+STORE, 3996299264, 3996311551,
+STORE, 3996311552, 3996430335,
+STORE, 3996430336, 3996467199,
+STORE, 3996467200, 3996504063,
+STORE, 3996504064, 3996512255,
+STORE, 3996512256, 3996516351,
+STORE, 3996516352, 3996540927,
+STORE, 3996540928, 3996671999,
+STORE, 3996672000, 3996676095,
+STORE, 3996676096, 3996684287,
+STORE, 3996684288, 3996688383,
+STORE, 3996688384, 3996692479,
+STORE, 3996692480, 3996717055,
+STORE, 3996717056, 3997048831,
+STORE, 3997048832, 3997057023,
+STORE, 3997057024, 3997073407,
+STORE, 3997073408, 3997077503,
+STORE, 3997077504, 3997081599,
+STORE, 3997081600, 3997097983,
+STORE, 3997097984, 3997179903,
+STORE, 3997179904, 3997356031,
+STORE, 3997356032, 3997650943,
+STORE, 3997650944, 3997675519,
+STORE, 3997675520, 3997679615,
+STORE, 3997683712, 3997700095,
+STORE, 3997700096, 3997745151,
+STORE, 3997745152, 3997802495,
+STORE, 3997802496, 3997810687,
+STORE, 3997810688, 3997814783,
+STORE, 3997814784, 3998064639,
+STORE, 3998064640, 3998081023,
+STORE, 3998081024, 3998085119,
+STORE, 3998085120, 3998130175,
+STORE, 3998130176, 3998134271,
+STORE, 3998134272, 3998142463,
+STORE, 3998142464, 3998179327,
+STORE, 3998179328, 3998212095,
+STORE, 3998212096, 3998326783,
+STORE, 3998326784, 3998351359,
+STORE, 3998351360, 3998392319,
+STORE, 3998392320, 3998396415,
+STORE, 3998396416, 3998400511,
+STORE, 3998400512, 3998433279,
+STORE, 3998433280, 3998466047,
+STORE, 3998466048, 3998613503,
+STORE, 3998613504, 3998666751,
+STORE, 3998666752, 3998724095,
+STORE, 3998724096, 3998732287,
+STORE, 3998732288, 3998736383,
+STORE, 3998736384, 3998760959,
+STORE, 3998760960, 3998777343,
+STORE, 3998777344, 3998822399,
+STORE, 3998822400, 3998826495,
+STORE, 3998826496, 3998830591,
+STORE, 3998830592, 3998863359,
+STORE, 3998863360, 3998900223,
+STORE, 3998900224, 3999043583,
+STORE, 3999043584, 3999121407,
+STORE, 3999121408, 3999215615,
+STORE, 3999215616, 3999223807,
+STORE, 3999223808, 3999227903,
+STORE, 3999227904, 3999236095,
+STORE, 3999236096, 3999268863,
+STORE, 3999268864, 3999301631,
+STORE, 3999301632, 3999354879,
+STORE, 3999354880, 3999428607,
+STORE, 3999428608, 3999436799,
+STORE, 3999436800, 3999440895,
+STORE, 3999444992, 3999461375,
+STORE, 3999461376, 3999584255,
+STORE, 3999584256, 3999760383,
+STORE, 3999760384, 4000219135,
+STORE, 4000219136, 4000235519,
+STORE, 4000235520, 4000251903,
+STORE, 4000251904, 4000501759,
+STORE, 4000501760, 4000505855,
+STORE, 4000505856, 4000509951,
+STORE, 4000509952, 4000518143,
+STORE, 4000518144, 4000522239,
+STORE, 4000522240, 4000587775,
+STORE, 4000587776, 4000645119,
+STORE, 4000645120, 4000813055,
+STORE, 4000813056, 4000817151,
+STORE, 4000821248, 4000837631,
+STORE, 4000837632, 4000870399,
+STORE, 4000870400, 4000874495,
+STORE, 4000874496, 4000878591,
+STORE, 4000878592, 4000882687,
+STORE, 4000882688, 4000886783,
+STORE, 4000886784, 4000890879,
+STORE, 4000890880, 4000907263,
+STORE, 4000907264, 4001214463,
+STORE, 4001214464, 4001558527,
+STORE, 4001558528, 4002484223,
+STORE, 4002484224, 4002525183,
+STORE, 4002525184, 4002529279,
+STORE, 4002529280, 4002533375,
+STORE, 4002533376, 4002537471,
+STORE, 4002537472, 4002660351,
+STORE, 4002660352, 4002779135,
+STORE, 4002779136, 4002791423,
+STORE, 4002791424, 4002799615,
+STORE, 4002799616, 4002807807,
+STORE, 4002807808, 4002811903,
+STORE, 4002811904, 4002828287,
+STORE, 4002828288, 4002910207,
+STORE, 4002910208, 4003028991,
+STORE, 4003028992, 4003037183,
+STORE, 4003037184, 4003045375,
+STORE, 4003045376, 4003049471,
+STORE, 4003049472, 4003053567,
+STORE, 4003053568, 4003057663,
+STORE, 4003057664, 4003065855,
+STORE, 4003065856, 4003135487,
+STORE, 4003135488, 4003446783,
+STORE, 4003446784, 4003450879,
+STORE, 4003450880, 4003454975,
+STORE, 4003454976, 4003459071,
+STORE, 4003459072, 4003463167,
+STORE, 4003463168, 4003495935,
+STORE, 4003495936, 4003569663,
+STORE, 4003569664, 4003573759,
+STORE, 4003573760, 4003704831,
+STORE, 4003704832, 4003708927,
+STORE, 4003708928, 4003713023,
+STORE, 4003713024, 4003737599,
+STORE, 4003737600, 4003770367,
+STORE, 4003770368, 4003876863,
+STORE, 4003876864, 4003880959,
+STORE, 4003880960, 4003885055,
+STORE, 4003885056, 4003889151,
+STORE, 4003889152, 4003893247,
+STORE, 4003893248, 4003897343,
+STORE, 4003897344, 4003962879,
+STORE, 4003962880, 4004069375,
+STORE, 4004069376, 4004093951,
+STORE, 4004093952, 4004118527,
+STORE, 4004118528, 4004122623,
+STORE, 4004122624, 4004126719,
+STORE, 4004126720, 4004155391,
+STORE, 4004155392, 4004286463,
+STORE, 4004286464, 4004384767,
+STORE, 4004384768, 4004388863,
+STORE, 4004388864, 4004646911,
+STORE, 4004646912, 4004655103,
+STORE, 4004655104, 4004659199,
+STORE, 4004659200, 4004667391,
+STORE, 4004667392, 4004683775,
+STORE, 4004683776, 4004814847,
+STORE, 4004814848, 4004818943,
+STORE, 4004818944, 4004823039,
+STORE, 4004823040, 4004827135,
+STORE, 4004827136, 4004835327,
+STORE, 4004835328, 4004954111,
+STORE, 4004954112, 4005085183,
+STORE, 4005085184, 4005306367,
+STORE, 4005306368, 4005765119,
+STORE, 4005765120, 4005789695,
+STORE, 4005789696, 4005793791,
+STORE, 4005793792, 4005801983,
+STORE, 4005801984, 4005920767,
+STORE, 4005920768, 4005945343,
+STORE, 4005945344, 4005949439,
+STORE, 4005949440, 4005986303,
+STORE, 4005986304, 4005990399,
+STORE, 4005990400, 4005994495,
+STORE, 4005994496, 4006002687,
+STORE, 4006002688, 4006109183,
+STORE, 4006109184, 4006117375,
+STORE, 4006117376, 4006121471,
+STORE, 4006121472, 4006133759,
+STORE, 4006133760, 4006137855,
+STORE, 4006137856, 4006141951,
+STORE, 4006141952, 4006150143,
+STORE, 4006150144, 4006391807,
+STORE, 4006391808, 4006445055,
+STORE, 4006445056, 4006563839,
+STORE, 4006563840, 4006572031,
+STORE, 4006572032, 4006576127,
+STORE, 4006576128, 4006584319,
+STORE, 4006584320, 4006694911,
+STORE, 4006694912, 4006739967,
+STORE, 4006739968, 4006776831,
+STORE, 4006776832, 4006785023,
+STORE, 4006785024, 4006789119,
+STORE, 4006789120, 4006797311,
+STORE, 4006797312, 4006813695,
+STORE, 4006813696, 4006846463,
+STORE, 4006846464, 4006977535,
+STORE, 4006977536, 4007006207,
+STORE, 4007006208, 4007010303,
+STORE, 4007010304, 4007067647,
+STORE, 4007067648, 4007075839,
+STORE, 4007075840, 4007084031,
+STORE, 4007084032, 4007100415,
+STORE, 4007100416, 4007116799,
+STORE, 4007116800, 4007133183,
+STORE, 4007133184, 4007153663,
+STORE, 4007153664, 4007178239,
+STORE, 4007178240, 4007202815,
+STORE, 4007202816, 4007206911,
+STORE, 4007206912, 4007272447,
+STORE, 4007272448, 4007276543,
+STORE, 4007276544, 4007280639,
+STORE, 4007280640, 4007284735,
+STORE, 4007284736, 4007292927,
+STORE, 4007292928, 4007423999,
+STORE, 4007424000, 4007448575,
+STORE, 4007448576, 4007452671,
+STORE, 4007452672, 4007505919,
+STORE, 4007505920, 4007510015,
+STORE, 4007510016, 4007514111,
+STORE, 4007514112, 4007645183,
+STORE, 4007645184, 4007776255,
+STORE, 4007776256, 4007780351,
+STORE, 4007780352, 4007784447,
+STORE, 4007784448, 4007788543,
+STORE, 4007788544, 4007809023,
+STORE, 4007809024, 4007829503,
+STORE, 4007829504, 4007960575,
+STORE, 4007960576, 4008091647,
+STORE, 4008091648, 4008296447,
+STORE, 4008296448, 4008890367,
+STORE, 4008890368, 4008898559,
+STORE, 4008898560, 4008902655,
+STORE, 4008902656, 4008996863,
+STORE, 4008996864, 4009041919,
+STORE, 4009041920, 4009082879,
+STORE, 4009082880, 4009091071,
+STORE, 4009091072, 4009107455,
+STORE, 4009107456, 4009349119,
+STORE, 4009349120, 4009373695,
+STORE, 4009373696, 4009414655,
+STORE, 4009414656, 4009422847,
+STORE, 4009422848, 4009426943,
+STORE, 4009426944, 4009447423,
+STORE, 4009447424, 4009471999,
+STORE, 4009472000, 4009512959,
+STORE, 4009512960, 4009594879,
+STORE, 4009594880, 4009598975,
+STORE, 4009598976, 4009697279,
+STORE, 4009697280, 4009713663,
+STORE, 4009713664, 4009717759,
+STORE, 4009717760, 4009721855,
+STORE, 4009721856, 4009730047,
+STORE, 4009730048, 4009861119,
+STORE, 4009861120, 4009951231,
+STORE, 4009951232, 4010131455,
+STORE, 4010131456, 4010135551,
+STORE, 4010135552, 4010139647,
+STORE, 4010139648, 4010143743,
+STORE, 4010143744, 4010164223,
+STORE, 4010164224, 4010295295,
+STORE, 4010295296, 4010299391,
+STORE, 4010299392, 4010491903,
+STORE, 4010491904, 4010495999,
+STORE, 4010496000, 4010668031,
+STORE, 4010668032, 4011028479,
+STORE, 4011028480, 4011053055,
+STORE, 4011053056, 4011057151,
+STORE, 4011057152, 4011118591,
+STORE, 4011118592, 4011126783,
+STORE, 4011126784, 4011130879,
+STORE, 4011130880, 4011143167,
+STORE, 4011143168, 4011147263,
+STORE, 4011147264, 4011167743,
+STORE, 4011167744, 4011171839,
+STORE, 4011171840, 4011360255,
+STORE, 4011360256, 4011364351,
+STORE, 4011364352, 4011626495,
+STORE, 4011626496, 4012216319,
+STORE, 4012216320, 4012228607,
+STORE, 4012228608, 4012232703,
+STORE, 4012232704, 4012236799,
+STORE, 4012236800, 4012240895,
+STORE, 4012240896, 4012261375,
+STORE, 4012261376, 4012392447,
+STORE, 4012392448, 4012466175,
+STORE, 4012466176, 4012597247,
+STORE, 4012597248, 4012601343,
+STORE, 4012601344, 4012605439,
+STORE, 4012605440, 4012609535,
+STORE, 4012609536, 4012679167,
+STORE, 4012679168, 4013563903,
+STORE, 4013563904, 4015366143,
+STORE, 4015366144, 4015411199,
+STORE, 4015411200, 4015415295,
+STORE, 4015415296, 4015419391,
+STORE, 4015419392, 4015542271,
+STORE, 4015542272, 4015550463,
+STORE, 4015550464, 4015558655,
+STORE, 4015558656, 4015562751,
+STORE, 4015562752, 4015583231,
+STORE, 4015583232, 4015587327,
+STORE, 4015587328, 4015603711,
+STORE, 4015665152, 4015669247,
+STORE, 4015669248, 4015812607,
+STORE, 4015812608, 4015816703,
+STORE, 4015816704, 4016111615,
+STORE, 4016111616, 4016467967,
+STORE, 4016467968, 4016508927,
+STORE, 4016508928, 4016517119,
+STORE, 4016517120, 4016525311,
+STORE, 4016525312, 4016586751,
+STORE, 4016586752, 4016664575,
+STORE, 4016664576, 4016697343,
+STORE, 4016697344, 4016742399,
+STORE, 4016742400, 4016746495,
+STORE, 4016746496, 4016750591,
+STORE, 4016750592, 4016758783,
+STORE, 4016799744, 4016844799,
+STORE, 4016844800, 4016902143,
+STORE, 4016902144, 4016992255,
+STORE, 4016992256, 4017000447,
+STORE, 4017000448, 4017004543,
+STORE, 4017004544, 4017008639,
+STORE, 4017008640, 4017016831,
+STORE, 4017016832, 4017020927,
+STORE, 4017020928, 4017127423,
+STORE, 4017127424, 4017131519,
+STORE, 4017131520, 4017229823,
+STORE, 4017229824, 4017422335,
+STORE, 4017422336, 4017438719,
+STORE, 4017438720, 4017442815,
+STORE, 4017442816, 4017446911,
+STORE, 4017446912, 4017455103,
+STORE, 4017455104, 4017766399,
+STORE, 4017766400, 4017909759,
+STORE, 4017909760, 4018081791,
+STORE, 4018081792, 4018089983,
+STORE, 4018089984, 4018094079,
+STORE, 4018094080, 4018098175,
+STORE, 4018098176, 4018327551,
+STORE, 4018327552, 4018331647,
+STORE, 4018331648, 4018339839,
+STORE, 4018339840, 4018348031,
+STORE, 4018348032, 4018610175,
+STORE, 4018610176, 4018626559,
+STORE, 4018626560, 4018647039,
+STORE, 4018647040, 4018651135,
+STORE, 4018651136, 4018749439,
+STORE, 4018749440, 4018761727,
+STORE, 4018761728, 4018802687,
+STORE, 4018802688, 4018806783,
+STORE, 4018806784, 4018810879,
+STORE, 4018810880, 4018814975,
+STORE, 4018814976, 4018823167,
+STORE, 4018823168, 4018954239,
+STORE, 4018954240, 4019007487,
+STORE, 4019007488, 4019068927,
+STORE, 4019068928, 4019077119,
+STORE, 4019077120, 4019081215,
+STORE, 4019081216, 4019093503,
+STORE, 4019093504, 4019208191,
+STORE, 4019208192, 4019232767,
+STORE, 4019232768, 4019265535,
+STORE, 4019265536, 4019269631,
+STORE, 4019269632, 4019277823,
+STORE, 4019277824, 4019458047,
+STORE, 4019458048, 4019519487,
+STORE, 4019519488, 4019613695,
+STORE, 4019613696, 4019621887,
+STORE, 4019621888, 4019625983,
+STORE, 4019625984, 4019630079,
+STORE, 4019630080, 4019744767,
+STORE, 4019744768, 4019822591,
+STORE, 4019822592, 4019929087,
+STORE, 4019929088, 4019941375,
+STORE, 4019941376, 4019945471,
+STORE, 4019945472, 4019961855,
+STORE, 4019961856, 4019994623,
+STORE, 4019994624, 4019998719,
+STORE, 4019998720, 4020002815,
+STORE, 4020002816, 4020006911,
+STORE, 4020006912, 4020011007,
+STORE, 4020011008, 4020256767,
+STORE, 4020256768, 4020326399,
+STORE, 4020326400, 4020457471,
+STORE, 4020457472, 4020469759,
+STORE, 4020469760, 4020473855,
+STORE, 4020473856, 4020482047,
+STORE, 4020482048, 4020711423,
+STORE, 4020711424, 4020715519,
+STORE, 4020715520, 4020719615,
+STORE, 4020719616, 4020723711,
+STORE, 4020723712, 4020805631,
+STORE, 4020805632, 4021051391,
+STORE, 4021051392, 4021460991,
+STORE, 4021460992, 4021469183,
+STORE, 4021469184, 4021473279,
+STORE, 4021473280, 4021571583,
+STORE, 4021571584, 4021633023,
+STORE, 4021633024, 4021727231,
+STORE, 4021727232, 4021735423,
+STORE, 4021735424, 4021739519,
+STORE, 4021739520, 4021747711,
+STORE, 4021747712, 4021829631,
+STORE, 4021829632, 4021866495,
+STORE, 4021866496, 4021919743,
+STORE, 4021919744, 4021927935,
+STORE, 4021927936, 4021932031,
+STORE, 4021932032, 4021944319,
+STORE, 4021944320, 4022157311,
+STORE, 4022157312, 4022161407,
+STORE, 4022161408, 4022173695,
+STORE, 4022173696, 4022177791,
+STORE, 4022177792, 4022472703,
+STORE, 4022472704, 4022509567,
+STORE, 4022509568, 4022583295,
+STORE, 4022583296, 4022587391,
+STORE, 4022587392, 4022591487,
+STORE, 4022591488, 4022607871,
+STORE, 4022607872, 4022657023,
+STORE, 4022657024, 4022722559,
+STORE, 4022722560, 4022730751,
+STORE, 4022730752, 4022734847,
+STORE, 4022734848, 4022865919,
+STORE, 4022865920, 4022943743,
+STORE, 4022943744, 4023062527,
+STORE, 4023062528, 4023074815,
+STORE, 4023074816, 4023078911,
+STORE, 4023078912, 4023128063,
+STORE, 4023128064, 4023218175,
+STORE, 4023218176, 4023361535,
+STORE, 4023361536, 4023373823,
+STORE, 4023373824, 4023377919,
+STORE, 4023377920, 4023558143,
+STORE, 4023558144, 4023631871,
+STORE, 4023631872, 4023816191,
+STORE, 4023816192, 4023820287,
+STORE, 4023820288, 4023824383,
+STORE, 4023824384, 4023832575,
+STORE, 4023832576, 4024078335,
+STORE, 4024078336, 4024197119,
+STORE, 4024197120, 4024389631,
+STORE, 4024389632, 4024406015,
+STORE, 4024406016, 4024410111,
+STORE, 4024410112, 4024422399,
+STORE, 4024422400, 4024619007,
+STORE, 4024619008, 4024639487,
+STORE, 4024639488, 4024655871,
+STORE, 4024655872, 4024664063,
+STORE, 4024664064, 4024668159,
+STORE, 4024668160, 4024676351,
+STORE, 4024676352, 4024905727,
+STORE, 4024905728, 4024909823,
+STORE, 4024909824, 4024918015,
+STORE, 4024918016, 4024922111,
+STORE, 4024922112, 4024930303,
+STORE, 4024930304, 4025110527,
+STORE, 4025110528, 4025176063,
+STORE, 4025176064, 4025208831,
+STORE, 4025208832, 4025212927,
+STORE, 4025212928, 4025217023,
+STORE, 4025217024, 4025348095,
+STORE, 4025348096, 4025372671,
+STORE, 4025372672, 4025458687,
+STORE, 4025458688, 4025466879,
+STORE, 4025466880, 4025565183,
+STORE, 4025565184, 4025757695,
+STORE, 4025757696, 4026249215,
+STORE, 4026249216, 4026261503,
+STORE, 4026261504, 4026265599,
+STORE, 4026265600, 4026269695,
+STORE, 4026269696, 4026302463,
+STORE, 4026302464, 4026306559,
+STORE, 4026306560, 4026314751,
+STORE, 4026314752, 4026318847,
+STORE, 4026318848, 4026322943,
+STORE, 4026322944, 4026327039,
+STORE, 4026327040, 4026654719,
+STORE, 4026654720, 4026671103,
+STORE, 4026671104, 4026720255,
+STORE, 4026720256, 4026724351,
+STORE, 4026724352, 4026728447,
+STORE, 4026728448, 4026732543,
+STORE, 4026732544, 4026863615,
+STORE, 4026863616, 4027027455,
+STORE, 4027027456, 4027031551,
+STORE, 4027031552, 4027514879,
+STORE, 4027514880, 4027531263,
+STORE, 4027531264, 4027535359,
+STORE, 4027535360, 4027539455,
+STORE, 4027539456, 4027785215,
+STORE, 4027785216, 4027789311,
+STORE, 4027789312, 4027793407,
+STORE, 4027793408, 4027797503,
+STORE, 4027797504, 4027863039,
+STORE, 4027863040, 4027899903,
+STORE, 4027899904, 4027949055,
+STORE, 4027949056, 4027957247,
+STORE, 4027957248, 4027961343,
+STORE, 4027961344, 4027965439,
+STORE, 4027965440, 4028194815,
+STORE, 4028194816, 4028252159,
+STORE, 4028252160, 4028338175,
+STORE, 4028338176, 4028350463,
+STORE, 4028350464, 4028354559,
+STORE, 4028354560, 4028452863,
+STORE, 4028452864, 4028489727,
+STORE, 4028489728, 4028530687,
+STORE, 4028530688, 4028538879,
+STORE, 4028538880, 4028542975,
+STORE, 4028542976, 4028551167,
+STORE, 4028551168, 4028665855,
+STORE, 4028665856, 4029349887,
+STORE, 4029349888, 4030468095,
+STORE, 4030468096, 4030513151,
+STORE, 4030513152, 4030517247,
+STORE, 4030517248, 4030525439,
+STORE, 4030525440, 4030529535,
+STORE, 4030529536, 4030758911,
+STORE, 4030758912, 4030828543,
+STORE, 4030828544, 4030943231,
+STORE, 4030943232, 4030951423,
+STORE, 4030951424, 4030955519,
+STORE, 4030955520, 4030967807,
+STORE, 4030967808, 4031131647,
+STORE, 4031131648, 4031135743,
+STORE, 4031135744, 4031139839,
+STORE, 4031139840, 4031148031,
+STORE, 4031148032, 4031152127,
+STORE, 4031152128, 4031160319,
+STORE, 4031160320, 4031504383,
+STORE, 4031504384, 4031598591,
+STORE, 4031598592, 4031754239,
+STORE, 4031754240, 4031766527,
+STORE, 4031766528, 4031770623,
+STORE, 4031770624, 4031774719,
+STORE, 4031774720, 4031782911,
+STORE, 4031782912, 4031799295,
+STORE, 4031799296, 4031856639,
+STORE, 4031856640, 4031983615,
+STORE, 4031983616, 4031987711,
+STORE, 4031987712, 4031991807,
+STORE, 4031991808, 4032270335,
+STORE, 4032270336, 4032274431,
+STORE, 4032274432, 4032282623,
+STORE, 4032282624, 4032286719,
+STORE, 4032286720, 4032290815,
+STORE, 4032290816, 4032389119,
+STORE, 4032389120, 4032397311,
+STORE, 4032397312, 4032405503,
+STORE, 4032405504, 4032413695,
+STORE, 4032413696, 4032417791,
+STORE, 4032417792, 4032565247,
+STORE, 4032565248, 4032593919,
+STORE, 4032593920, 4032737279,
+STORE, 4032737280, 4032741375,
+STORE, 4032741376, 4032745471,
+STORE, 4032745472, 4032770047,
+STORE, 4032770048, 4032933887,
+STORE, 4032933888, 4032999423,
+STORE, 4032999424, 4033032191,
+STORE, 4033032192, 4033036287,
+STORE, 4033036288, 4033040383,
+STORE, 4033040384, 4033105919,
+STORE, 4033105920, 4033396735,
+STORE, 4033396736, 4033822719,
+STORE, 4033822720, 4033839103,
+STORE, 4033839104, 4033843199,
+STORE, 4033843200, 4033851391,
+STORE, 4033851392, 4033863679,
+STORE, 4033863680, 4033880063,
+STORE, 4033880064, 4033933311,
+STORE, 4033933312, 4034023423,
+STORE, 4034023424, 4034031615,
+STORE, 4034031616, 4034035711,
+STORE, 4034035712, 4034043903,
+STORE, 4034043904, 4034142207,
+STORE, 4034142208, 4034191359,
+STORE, 4034191360, 4034260991,
+STORE, 4034260992, 4034269183,
+STORE, 4034269184, 4034273279,
+STORE, 4034273280, 4034281471,
+STORE, 4034281472, 4034412543,
+STORE, 4034412544, 4034445311,
+STORE, 4034445312, 4034490367,
+STORE, 4034490368, 4034494463,
+STORE, 4034494464, 4034498559,
+STORE, 4034498560, 4034662399,
+STORE, 4034662400, 4034666495,
+STORE, 4034666496, 4034670591,
+STORE, 4034670592, 4034674687,
+STORE, 4034674688, 4034678783,
+STORE, 4034678784, 4034682879,
+STORE, 4034682880, 4034781183,
+STORE, 4034781184, 4035043327,
+STORE, 4035043328, 4035047423,
+STORE, 4035047424, 4035055615,
+STORE, 4035055616, 4035059711,
+STORE, 4035059712, 4035063807,
+STORE, 4035063808, 4035067903,
+STORE, 4035067904, 4035100671,
+STORE, 4035100672, 4035375103,
+STORE, 4035375104, 4035383295,
+STORE, 4035383296, 4035395583,
+STORE, 4035395584, 4035399679,
+STORE, 4035399680, 4035403775,
+STORE, 4035403776, 4035407871,
+STORE, 4035407872, 4035411967,
+STORE, 4035411968, 4035477503,
+STORE, 4035477504, 4035608575,
+STORE, 4035608576, 4035641343,
+STORE, 4035641344, 4035682303,
+STORE, 4035682304, 4035686399,
+STORE, 4035686400, 4035690495,
+STORE, 4035690496, 4035694591,
+STORE, 4035694592, 4035743743,
+STORE, 4035743744, 4035784703,
+STORE, 4035784704, 4035829759,
+STORE, 4035829760, 4035837951,
+STORE, 4035837952, 4035842047,
+STORE, 4035842048, 4035846143,
+STORE, 4035846144, 4035850239,
+STORE, 4035850240, 4036001791,
+STORE, 4036001792, 4036005887,
+STORE, 4036005888, 4036214783,
+STORE, 4036214784, 4036218879,
+STORE, 4036218880, 4036603903,
+STORE, 4036603904, 4036648959,
+STORE, 4036648960, 4036653055,
+STORE, 4036653056, 4036657151,
+STORE, 4036657152, 4036665343,
+STORE, 4036665344, 4036780031,
+STORE, 4036780032, 4036829183,
+STORE, 4036829184, 4036984831,
+STORE, 4036984832, 4036993023,
+STORE, 4036993024, 4036997119,
+STORE, 4036997120, 4037001215,
+STORE, 4037001216, 4037009407,
+STORE, 4037009408, 4037025791,
+STORE, 4037025792, 4037095423,
+STORE, 4037095424, 4037181439,
+STORE, 4037181440, 4037193727,
+STORE, 4037193728, 4037197823,
+STORE, 4037197824, 4037206015,
+STORE, 4037206016, 4037320703,
+STORE, 4037320704, 4037337087,
+STORE, 4037337088, 4037349375,
+STORE, 4037349376, 4037357567,
+STORE, 4037357568, 4037361663,
+STORE, 4037369856, 4037386239,
+STORE, 4037386240, 4037672959,
+STORE, 4037672960, 4037689343,
+STORE, 4037689344, 4037730303,
+STORE, 4037730304, 4037734399,
+STORE, 4037734400, 4037738495,
+STORE, 4037738496, 4037742591,
+STORE, 4037742592, 4037758975,
+STORE, 4037758976, 4037890047,
+STORE, 4037890048, 4037931007,
+STORE, 4037931008, 4037976063,
+STORE, 4037976064, 4037984255,
+STORE, 4037984256, 4037988351,
+STORE, 4037988352, 4038053887,
+STORE, 4038053888, 4038184959,
+STORE, 4038184960, 4038189055,
+STORE, 4038189056, 4038197247,
+STORE, 4038197248, 4038201343,
+STORE, 4038201344, 4038205439,
+STORE, 4038205440, 4038209535,
+STORE, 4038217728, 4038250495,
+STORE, 4038250496, 4038512639,
+STORE, 4038512640, 4038516735,
+STORE, 4038516736, 4038520831,
+STORE, 4038520832, 4038524927,
+STORE, 4038524928, 4038529023,
+STORE, 4038529024, 4038533119,
+STORE, 4038541312, 4038623231,
+STORE, 4038623232, 4038754303,
+STORE, 4038754304, 4038885375,
+STORE, 4038885376, 4038889471,
+STORE, 4038897664, 4038963199,
+STORE, 4038963200, 4038967295,
+STORE, 4038967296, 4038983679,
+STORE, 4038983680, 4039114751,
+STORE, 4039114752, 4039245823,
+STORE, 4039245824, 4039376895,
+STORE, 4039376896, 4040687615,
+STORE, 4040687616, 4040691711,
+STORE, 4040691712, 4040806399,
+STORE, 4040806400, 4040937471,
+STORE, 4040937472, 4040941567,
+STORE, 4040945664, 4040949759,
+STORE, 4040949760, 4041080831,
+STORE, 4041080832, 4041211903,
+STORE, 4041211904, 4043046911,
+STORE, 4043046912, 4043051007,
+STORE, 4043051008, 4043055103,
+STORE, 4043055104, 4043137023,
+STORE, 4043137024, 4043141119,
+STORE, 4043141120, 4043145215,
+STORE, 4043145216, 4043153407,
+STORE, 4043153408, 4043186175,
+STORE, 4043186176, 4043317247,
+STORE, 4043317248, 4043448319,
+STORE, 4043448320, 4043579391,
+STORE, 4043579392, 4043583487,
+STORE, 4043583488, 4043599871,
+STORE, 4043599872, 4043661311,
+STORE, 4043661312, 4043792383,
+STORE, 4043792384, 4043796479,
+STORE, 4043796480, 4043800575,
+STORE, 4043800576, 4043816959,
+STORE, 4043816960, 4043821055,
+STORE, 4043821056, 4043825151,
+STORE, 4043825152, 4043829247,
+STORE, 4043829248, 4043833343,
+STORE, 4043833344, 4047241215,
+STORE, 4047241216, 4047249407,
+STORE, 4047249408, 4047253503,
+STORE, 4047253504, 4047323135,
+STORE, 4047323136, 4047327231,
+STORE, 4047327232, 4047458303,
+STORE, 4047458304, 4047589375,
+STORE, 4047589376, 4047720447,
+STORE, 4047720448, 4047773695,
+STORE, 4047773696, 4047790079,
+STORE, 4047790080, 4047921151,
+STORE, 4047921152, 4048052223,
+STORE, 4048052224, 4048183295,
+STORE, 4048183296, 4049002495,
+STORE, 4049002496, 4049133567,
+STORE, 4049133568, 4049154047,
+STORE, 4049154048, 4049158143,
+STORE, 4049158144, 4049162239,
+STORE, 4049162240, 4049166335,
+STORE, 4049166336, 4049174527,
+STORE, 4049174528, 4049182719,
+STORE, 4049182720, 4049186815,
+STORE, 4049186816, 4049190911,
+STORE, 4049190912, 4049195007,
+STORE, 4049195008, 4049203199,
+STORE, 4049203200, 4049207295,
+STORE, 4049207296, 4049211391,
+STORE, 4049211392, 4049215487,
+STORE, 4049215488, 4049219583,
+STORE, 4049219584, 4049227775,
+STORE, 4049227776, 4049231871,
+STORE, 4049231872, 4049235967,
+STORE, 4049235968, 4049244159,
+STORE, 4049244160, 4049248255,
+STORE, 4049248256, 4049252351,
+STORE, 4049252352, 4049256447,
+STORE, 4049256448, 4049268735,
+STORE, 4049268736, 4049272831,
+STORE, 4049272832, 4049313791,
+STORE, 4049313792, 4049723391,
+STORE, 4049723392, 4049727487,
+STORE, 4049727488, 4049858559,
+STORE, 4049858560, 4049989631,
+STORE, 4049989632, 4049993727,
+STORE, 4049993728, 4050026495,
+STORE, 4050026496, 4050030591,
+STORE, 4050030592, 4050161663,
+STORE, 4050161664, 4050169855,
+STORE, 4050169856, 4050223103,
+STORE, 4050223104, 4050632703,
+STORE, 4050632704, 4050636799,
+STORE, 4050636800, 4050640895,
+STORE, 4050640896, 4050644991,
+STORE, 4050644992, 4050661375,
+STORE, 4050661376, 4050665471,
+STORE, 4050665472, 4050673663,
+STORE, 4050673664, 4050677759,
+STORE, 4050677760, 4050694143,
+STORE, 4050694144, 4050702335,
+STORE, 4050702336, 4050956287,
+STORE, 4050956288, 4051963903,
+STORE, 4051963904, 4051980287,
+STORE, 4051980288, 4051988479,
+STORE, 4051988480, 4052000767,
+STORE, 4052000768, 4052004863,
+STORE, 4052004864, 4052029439,
+STORE, 4284014592, 4284018687,
+STORE, 4284018688, 4292403199,
+SNULL, 4041080832, 4041211903,
+SNULL, 3795763200, 3795894271,
+STORE, 3629522944, 3696631807,
+SNULL, 3663077375, 3696631807,
+STORE, 3629522944, 3663077375,
+STORE, 3663077376, 3696631807,
+SNULL, 3663077376, 3696631807,
+STORE, 3663077376, 3696631807,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3626471424, 3627524095,
+SNULL, 3626471424, 3626475519,
+STORE, 3626475520, 3627524095,
+STORE, 3626471424, 3626475519,
+SNULL, 3627519999, 3627524095,
+STORE, 3626475520, 3627519999,
+STORE, 3627520000, 3627524095,
+STORE, 3625418752, 3626475519,
+SNULL, 3625418752, 3625422847,
+STORE, 3625422848, 3626475519,
+STORE, 3625418752, 3625422847,
+SNULL, 3626467327, 3626475519,
+STORE, 3625422848, 3626467327,
+STORE, 3626467328, 3626475519,
+STORE, 3624366080, 3625422847,
+SNULL, 3624366080, 3624370175,
+STORE, 3624370176, 3625422847,
+STORE, 3624366080, 3624370175,
+SNULL, 3625414655, 3625422847,
+STORE, 3624370176, 3625414655,
+STORE, 3625414656, 3625422847,
+STORE, 4041191424, 4041211903,
+SNULL, 4041195519, 4041211903,
+STORE, 4041191424, 4041195519,
+STORE, 4041195520, 4041211903,
+STORE, 4041170944, 4041191423,
+SNULL, 4041175039, 4041191423,
+STORE, 4041170944, 4041175039,
+STORE, 4041175040, 4041191423,
+SNULL, 3625426943, 3626467327,
+STORE, 3625422848, 3625426943,
+STORE, 3625426944, 3626467327,
+STORE, 4041162752, 4041170943,
+SNULL, 3626479615, 3627519999,
+STORE, 3626475520, 3626479615,
+STORE, 3626479616, 3627519999,
+STORE, 4041154560, 4041162751,
+STORE, 4041154560, 4041170943,
+STORE, 4041134080, 4041154559,
+SNULL, 4041138175, 4041154559,
+STORE, 4041134080, 4041138175,
+STORE, 4041138176, 4041154559,
+SNULL, 3624374271, 3625414655,
+STORE, 3624370176, 3624374271,
+STORE, 3624374272, 3625414655,
+STORE, 4041125888, 4041134079,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+STORE, 3487174656, 3487584255,
+STORE, 4041121792, 4041125887,
+SNULL, 4041121792, 4041125887,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 3487174656, 3487584255,
+STORE, 3222274048, 3223326719,
+SNULL, 3222274048, 3222278143,
+STORE, 3222278144, 3223326719,
+STORE, 3222274048, 3222278143,
+SNULL, 3223322623, 3223326719,
+STORE, 3222278144, 3223322623,
+STORE, 3223322624, 3223326719,
+STORE, 3221221376, 3222278143,
+SNULL, 3221221376, 3221225471,
+STORE, 3221225472, 3222278143,
+STORE, 3221221376, 3221225471,
+SNULL, 3222269951, 3222278143,
+STORE, 3221225472, 3222269951,
+STORE, 3222269952, 3222278143,
+STORE, 3220168704, 3221225471,
+SNULL, 3220168704, 3220172799,
+STORE, 3220172800, 3221225471,
+STORE, 3220168704, 3220172799,
+SNULL, 3221217279, 3221225471,
+STORE, 3220172800, 3221217279,
+STORE, 3221217280, 3221225471,
+STORE, 4041117696, 4041125887,
+STORE, 4041117696, 4041134079,
+STORE, 3219083264, 3220172799,
+SNULL, 3219083264, 3219087359,
+STORE, 3219087360, 3220172799,
+STORE, 3219083264, 3219087359,
+SNULL, 3220164607, 3220172799,
+STORE, 3219087360, 3220164607,
+STORE, 3220164608, 3220172799,
+STORE, 4041109504, 4041117695,
+STORE, 4041109504, 4041134079,
+STORE, 3217997824, 3219087359,
+SNULL, 3217997824, 3218001919,
+STORE, 3218001920, 3219087359,
+STORE, 3217997824, 3218001919,
+SNULL, 3219079167, 3219087359,
+STORE, 3218001920, 3219079167,
+STORE, 3219079168, 3219087359,
+STORE, 4041101312, 4041109503,
+STORE, 4041101312, 4041134079,
+STORE, 3216912384, 3218001919,
+SNULL, 3216912384, 3216916479,
+STORE, 3216916480, 3218001919,
+STORE, 3216912384, 3216916479,
+SNULL, 3217993727, 3218001919,
+STORE, 3216916480, 3217993727,
+STORE, 3217993728, 3218001919,
+STORE, 4041093120, 4041101311,
+STORE, 4041093120, 4041134079,
+STORE, 3215826944, 3216916479,
+SNULL, 3215826944, 3215831039,
+STORE, 3215831040, 3216916479,
+STORE, 3215826944, 3215831039,
+SNULL, 3216908287, 3216916479,
+STORE, 3215831040, 3216908287,
+STORE, 3216908288, 3216916479,
+STORE, 4016779264, 4016799743,
+SNULL, 4016783359, 4016799743,
+STORE, 4016779264, 4016783359,
+STORE, 4016783360, 4016799743,
+STORE, 4016758784, 4016779263,
+SNULL, 4016762879, 4016779263,
+STORE, 4016758784, 4016762879,
+STORE, 4016762880, 4016779263,
+SNULL, 3222282239, 3223322623,
+STORE, 3222278144, 3222282239,
+STORE, 3222282240, 3223322623,
+STORE, 4041084928, 4041093119,
+STORE, 4041084928, 4041134079,
+SNULL, 3221229567, 3222269951,
+STORE, 3221225472, 3221229567,
+STORE, 3221229568, 3222269951,
+STORE, 4015644672, 4015665151,
+STORE, 4038889472, 4038897663,
+SNULL, 4015648767, 4015665151,
+STORE, 4015644672, 4015648767,
+STORE, 4015648768, 4015665151,
+STORE, 4015624192, 4015644671,
+SNULL, 4015628287, 4015644671,
+STORE, 4015624192, 4015628287,
+STORE, 4015628288, 4015644671,
+SNULL, 3219091455, 3220164607,
+STORE, 3219087360, 3219091455,
+STORE, 3219091456, 3220164607,
+STORE, 4015603712, 4015624191,
+SNULL, 4015607807, 4015624191,
+STORE, 4015603712, 4015607807,
+STORE, 4015607808, 4015624191,
+SNULL, 3218006015, 3219079167,
+STORE, 3218001920, 3218006015,
+STORE, 3218006016, 3219079167,
+STORE, 3949674496, 3949694975,
+SNULL, 3949678591, 3949694975,
+STORE, 3949674496, 3949678591,
+STORE, 3949678592, 3949694975,
+SNULL, 3216920575, 3217993727,
+STORE, 3216916480, 3216920575,
+STORE, 3216920576, 3217993727,
+STORE, 3948924928, 3948945407,
+SNULL, 3948929023, 3948945407,
+STORE, 3948924928, 3948929023,
+STORE, 3948929024, 3948945407,
+SNULL, 3215835135, 3216908287,
+STORE, 3215831040, 3215835135,
+STORE, 3215835136, 3216908287,
+SNULL, 3220176895, 3221217279,
+STORE, 3220172800, 3220176895,
+STORE, 3220176896, 3221217279,
+STORE, 3214786560, 3215826943,
+STORE, 3213733888, 3214786559,
+SNULL, 3213733888, 3213737983,
+STORE, 3213737984, 3214786559,
+STORE, 3213733888, 3213737983,
+SNULL, 3214782463, 3214786559,
+STORE, 3213737984, 3214782463,
+STORE, 3214782464, 3214786559,
+STORE, 4038533120, 4038541311,
+STORE, 3948421120, 3948441599,
+SNULL, 3948425215, 3948441599,
+STORE, 3948421120, 3948425215,
+STORE, 3948425216, 3948441599,
+SNULL, 3213742079, 3214782463,
+STORE, 3213737984, 3213742079,
+STORE, 3213742080, 3214782463,
+STORE, 4038209536, 4038217727,
+STORE, 3212681216, 3213737983,
+SNULL, 3212681216, 3212685311,
+STORE, 3212685312, 3213737983,
+STORE, 3212681216, 3212685311,
+SNULL, 3213729791, 3213737983,
+STORE, 3212685312, 3213729791,
+STORE, 3213729792, 3213737983,
+STORE, 3795763200, 3795894271,
+STORE, 3946872832, 3946893311,
+SNULL, 3946876927, 3946893311,
+STORE, 3946872832, 3946876927,
+STORE, 3946876928, 3946893311,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+STORE, 3487174656, 3487584255,
+SNULL, 3212689407, 3213729791,
+STORE, 3212685312, 3212689407,
+STORE, 3212689408, 3213729791,
+STORE, 4041080832, 4041084927,
+STORE, 4040941568, 4040945663,
+STORE, 4037361664, 4037369855,
+STORE, 4000817152, 4000821247,
+STORE, 3999440896, 3999444991,
+STORE, 3212161024, 3212681215,
+SNULL, 3212161024, 3212439551,
+STORE, 3212439552, 3212681215,
+STORE, 3212161024, 3212439551,
+SNULL, 3212161024, 3212439551,
+SNULL, 3212464127, 3212681215,
+STORE, 3212439552, 3212464127,
+STORE, 3212464128, 3212681215,
+SNULL, 3212464128, 3212681215,
+SNULL, 3212439552, 3212451839,
+STORE, 3212451840, 3212464127,
+STORE, 3212439552, 3212451839,
+SNULL, 3212439552, 3212451839,
+STORE, 3212439552, 3212451839,
+SNULL, 3212451840, 3212455935,
+STORE, 3212455936, 3212464127,
+STORE, 3212451840, 3212455935,
+SNULL, 3212451840, 3212455935,
+STORE, 3212451840, 3212455935,
+SNULL, 3212455936, 3212460031,
+STORE, 3212460032, 3212464127,
+STORE, 3212455936, 3212460031,
+SNULL, 3212455936, 3212460031,
+STORE, 3212455936, 3212460031,
+SNULL, 3212460032, 3212464127,
+STORE, 3212460032, 3212464127,
+STORE, 3997679616, 3997683711,
+SNULL, 4049235968, 4049240063,
+STORE, 4049240064, 4049244159,
+STORE, 4049235968, 4049240063,
+SNULL, 4049240064, 4049244159,
+STORE, 4049240064, 4049244159,
+SNULL, 3997679616, 3997683711,
+SNULL, 3999440896, 3999444991,
+SNULL, 4000817152, 4000821247,
+SNULL, 4040941568, 4040945663,
+SNULL, 4041080832, 4041084927,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 3487174656, 3487584255,
+SNULL, 3212451840, 3212455935,
+STORE, 3212451840, 3212455935,
+STORE, 4041080832, 4041084927,
+STORE, 3623890944, 3624169471,
+SNULL, 4041080832, 4041084927,
+STORE, 4041080832, 4041084927,
+SNULL, 4041080832, 4041084927,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+STORE, 4041080832, 4041084927,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+STORE, 3211386880, 3212439551,
+SNULL, 3211386880, 3211390975,
+STORE, 3211390976, 3212439551,
+STORE, 3211386880, 3211390975,
+SNULL, 3212435455, 3212439551,
+STORE, 3211390976, 3212435455,
+STORE, 3212435456, 3212439551,
+STORE, 4040941568, 4040945663,
+STORE, 3937169408, 3937189887,
+STORE, 3623485440, 3623616511,
+SNULL, 717225983, 1388314623,
+STORE, 314572800, 717225983,
+STORE, 717225984, 1388314623,
+SNULL, 717225984, 1388314623,
+STORE, 3937112064, 3937132543,
+SNULL, 3937116159, 3937132543,
+STORE, 3937112064, 3937116159,
+STORE, 3937116160, 3937132543,
+SNULL, 3211395071, 3212435455,
+STORE, 3211390976, 3211395071,
+STORE, 3211395072, 3212435455,
+STORE, 4000817152, 4000821247,
+STORE, 3974823936, 3974832127,
+STORE, 3595284480, 3595431935,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+STORE, 3487174656, 3487584255,
+STORE, 3999440896, 3999444991,
+STORE, 3997679616, 3997683711,
+STORE, 3996295168, 3996299263,
+STORE, 3996090368, 3996094463,
+STORE, 3210866688, 3211386879,
+SNULL, 3210866688, 3211001855,
+STORE, 3211001856, 3211386879,
+STORE, 3210866688, 3211001855,
+SNULL, 3210866688, 3211001855,
+SNULL, 3211038719, 3211386879,
+STORE, 3211001856, 3211038719,
+STORE, 3211038720, 3211386879,
+SNULL, 3211038720, 3211386879,
+SNULL, 3211001856, 3211022335,
+STORE, 3211022336, 3211038719,
+STORE, 3211001856, 3211022335,
+SNULL, 3211001856, 3211022335,
+STORE, 3211001856, 3211022335,
+SNULL, 3211022336, 3211030527,
+STORE, 3211030528, 3211038719,
+STORE, 3211022336, 3211030527,
+SNULL, 3211022336, 3211030527,
+STORE, 3211022336, 3211030527,
+SNULL, 3211030528, 3211034623,
+STORE, 3211034624, 3211038719,
+STORE, 3211030528, 3211034623,
+SNULL, 3211030528, 3211034623,
+STORE, 3211030528, 3211034623,
+SNULL, 3211034624, 3211038719,
+STORE, 3211034624, 3211038719,
+STORE, 3994906624, 3994910719,
+SNULL, 4049240064, 4049244159,
+STORE, 4049240064, 4049244159,
+SNULL, 3994906624, 3994910719,
+SNULL, 3996090368, 3996094463,
+SNULL, 3996295168, 3996299263,
+SNULL, 3997679616, 3997683711,
+SNULL, 3999440896, 3999444991,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 3487174656, 3487584255,
+SNULL, 3211022336, 3211030527,
+STORE, 3211022336, 3211030527,
+STORE, 3999440896, 3999444991,
+STORE, 3210199040, 3211001855,
+SNULL, 3999440896, 3999444991,
+STORE, 3999440896, 3999444991,
+SNULL, 3999440896, 3999444991,
+STORE, 3594821632, 3594952703,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 4048183296, 4048592895,
+STORE, 4048592896, 4049002495,
+STORE, 4048183296, 4048592895,
+STORE, 4048183296, 4049002495,
+SNULL, 1914101759, 1969434623,
+STORE, 1914097664, 1914101759,
+STORE, 1914101760, 1969434623,
+STORE, 3567108096, 3567239167,
+STORE, 3973832704, 3973840895,
+STORE, 3209113600, 3210199039,
+SNULL, 3209113600, 3209117695,
+STORE, 3209117696, 3210199039,
+STORE, 3209113600, 3209117695,
+SNULL, 3210194943, 3210199039,
+STORE, 3209117696, 3210194943,
+STORE, 3210194944, 3210199039,
+STORE, 3935858688, 3935879167,
+SNULL, 3935862783, 3935879167,
+STORE, 3935858688, 3935862783,
+STORE, 3935862784, 3935879167,
+SNULL, 3209121791, 3210194943,
+STORE, 3209117696, 3209121791,
+STORE, 3209121792, 3210194943,
+STORE, 3528749056, 3528880127,
+STORE, 3968200704, 3968208895,
+STORE, 3208028160, 3209117695,
+SNULL, 3208028160, 3208032255,
+STORE, 3208032256, 3209117695,
+STORE, 3208028160, 3208032255,
+SNULL, 3209109503, 3209117695,
+STORE, 3208032256, 3209109503,
+STORE, 3209109504, 3209117695,
+STORE, 3888123904, 3888144383,
+SNULL, 3888127999, 3888144383,
+STORE, 3888123904, 3888127999,
+STORE, 3888128000, 3888144383,
+SNULL, 3208036351, 3209109503,
+STORE, 3208032256, 3208036351,
+STORE, 3208036352, 3209109503,
+SNULL, 3968200704, 3968208895,
+SNULL, 3888123904, 3888144383,
+SNULL, 3209109504, 3209113599,
+STORE, 3209113600, 3209117695,
+STORE, 3209109504, 3209113599,
+SNULL, 3208028160, 3209113599,
+STORE, 3208060928, 3209117695,
+SNULL, 3208060928, 3208065023,
+STORE, 3208065024, 3209117695,
+STORE, 3208060928, 3208065023,
+SNULL, 3209109503, 3209117695,
+STORE, 3208065024, 3209109503,
+STORE, 3209109504, 3209117695,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3888123904, 3888144383,
+SNULL, 3888127999, 3888144383,
+STORE, 3888123904, 3888127999,
+STORE, 3888128000, 3888144383,
+SNULL, 3208069119, 3209109503,
+STORE, 3208065024, 3208069119,
+STORE, 3208069120, 3209109503,
+STORE, 3968200704, 3968208895,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3527778304, 3527909375,
+STORE, 3999440896, 3999444991,
+STORE, 3997679616, 3997683711,
+STORE, 1914097664, 1914105855,
+STORE, 1914105856, 1969434623,
+STORE, 3957583872, 3957592063,
+STORE, 3206975488, 3208065023,
+SNULL, 3206975488, 3206979583,
+STORE, 3206979584, 3208065023,
+STORE, 3206975488, 3206979583,
+SNULL, 3208056831, 3208065023,
+STORE, 3206979584, 3208056831,
+STORE, 3208056832, 3208065023,
+STORE, 3956736000, 3956744191,
+STORE, 3205890048, 3206979583,
+SNULL, 3205890048, 3205894143,
+STORE, 3205894144, 3206979583,
+STORE, 3205890048, 3205894143,
+SNULL, 3206971391, 3206979583,
+STORE, 3205894144, 3206971391,
+STORE, 3206971392, 3206979583,
+STORE, 3806101504, 3806121983,
+SNULL, 3806105599, 3806121983,
+STORE, 3806101504, 3806105599,
+STORE, 3806105600, 3806121983,
+SNULL, 3206983679, 3208056831,
+STORE, 3206979584, 3206983679,
+STORE, 3206983680, 3208056831,
+STORE, 3806081024, 3806101503,
+SNULL, 3806085119, 3806101503,
+STORE, 3806081024, 3806085119,
+STORE, 3806085120, 3806101503,
+SNULL, 3205898239, 3206971391,
+STORE, 3205894144, 3205898239,
+STORE, 3205898240, 3206971391,
+STORE, 3956015104, 3956023295,
+STORE, 3204804608, 3205894143,
+SNULL, 3204804608, 3204808703,
+STORE, 3204808704, 3205894143,
+STORE, 3204804608, 3204808703,
+SNULL, 3205885951, 3205894143,
+STORE, 3204808704, 3205885951,
+STORE, 3205885952, 3205894143,
+STORE, 3803471872, 3803492351,
+STORE, 3803451392, 3803471871,
+STORE, 3803451392, 3803492351,
+SNULL, 3957583872, 3957592063,
+SNULL, 3806101504, 3806121983,
+SNULL, 3206975487, 3206979583,
+STORE, 3206971392, 3206975487,
+STORE, 3206975488, 3206979583,
+SNULL, 3208056832, 3208060927,
+STORE, 3208060928, 3208065023,
+STORE, 3208056832, 3208060927,
+SNULL, 3206975488, 3208060927,
+STORE, 3801845760, 3801878527,
+STORE, 3806101504, 3806121983,
+SNULL, 3806105599, 3806121983,
+STORE, 3806101504, 3806105599,
+STORE, 3806105600, 3806121983,
+SNULL, 3204812799, 3205885951,
+STORE, 3204808704, 3204812799,
+STORE, 3204812800, 3205885951,
+STORE, 1914097664, 1914109951,
+STORE, 1914109952, 1969434623,
+STORE, 3957583872, 3957592063,
+STORE, 3206971392, 3208065023,
+SNULL, 3206971392, 3206979583,
+STORE, 3206979584, 3208065023,
+STORE, 3206971392, 3206979583,
+SNULL, 3208056831, 3208065023,
+STORE, 3206979584, 3208056831,
+STORE, 3208056832, 3208065023,
+STORE, 3801825280, 3801845759,
+SNULL, 3801829375, 3801845759,
+STORE, 3801825280, 3801829375,
+STORE, 3801829376, 3801845759,
+SNULL, 3206983679, 3208056831,
+STORE, 3206979584, 3206983679,
+STORE, 3206983680, 3208056831,
+STORE, 3202707456, 3204804607,
+SNULL, 3202707456, 3204804607,
+STORE, 3202707456, 3204804607,
+STORE, 3200610304, 3202707455,
+SNULL, 3202707456, 3204804607,
+SNULL, 3200610304, 3202707455,
+STORE, 3202707456, 3204804607,
+SNULL, 3202707456, 3204804607,
+STORE, 3202707456, 3204804607,
+SNULL, 3202707456, 3204804607,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3527647232, 3527778303,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+STORE, 3487059968, 3487584255,
+SNULL, 3487059968, 3487301631,
+STORE, 3487301632, 3487584255,
+STORE, 3487059968, 3487301631,
+SNULL, 3487059968, 3487301631,
+SNULL, 3487563775, 3487584255,
+STORE, 3487301632, 3487563775,
+STORE, 3487563776, 3487584255,
+SNULL, 3487563776, 3487584255,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3524046848, 3524177919,
+STORE, 3487170560, 3487301631,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3487039488, 3487170559,
+STORE, 3487039488, 3487301631,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3204280320, 3204804607,
+SNULL, 3204280320, 3204448255,
+STORE, 3204448256, 3204804607,
+STORE, 3204280320, 3204448255,
+SNULL, 3204280320, 3204448255,
+SNULL, 3204710399, 3204804607,
+STORE, 3204448256, 3204710399,
+STORE, 3204710400, 3204804607,
+SNULL, 3204710400, 3204804607,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3996295168, 3996299263,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+SNULL, 3996295168, 3996299263,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3486908416, 3487039487,
+STORE, 3486908416, 3487301631,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3223326720, 3290435583,
+SNULL, 3223326720, 3256881151,
+STORE, 3256881152, 3290435583,
+STORE, 3223326720, 3256881151,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+STORE, 3201826816, 3202351103,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+STORE, 3202351104, 3204448255,
+SNULL, 3202351104, 3204448255,
+SNULL, 3803471871, 3803492351,
+STORE, 3803451392, 3803471871,
+STORE, 3803471872, 3803492351,
+SNULL, 3803471872, 3803492351,
+SNULL, 3803451392, 3803471871,
+STORE, 3798999040, 3799101439,
+SNULL, 3798999040, 3799101439,
+STORE, 3952644096, 3952652287,
+STORE, 3203362816, 3204448255,
+SNULL, 3203362816, 3203366911,
+STORE, 3203366912, 3204448255,
+STORE, 3203362816, 3203366911,
+SNULL, 3204444159, 3204448255,
+STORE, 3203366912, 3204444159,
+STORE, 3204444160, 3204448255,
+STORE, 3803471872, 3803492351,
+SNULL, 3803475967, 3803492351,
+STORE, 3803471872, 3803475967,
+STORE, 3803475968, 3803492351,
+SNULL, 3203371007, 3204444159,
+STORE, 3203366912, 3203371007,
+STORE, 3203371008, 3204444159,
+STORE, 3199729664, 3201826815,
+SNULL, 3199729664, 3201826815,
+STORE, 3199729664, 3201826815,
+SNULL, 3199729664, 3201826815,
+STORE, 3199729664, 3201826815,
+SNULL, 3199729664, 3201826815,
+STORE, 3199729664, 3201826815,
+SNULL, 3199729664, 3201826815,
+STORE, 3199729664, 3201826815,
+SNULL, 3199729664, 3201826815,
+STORE, 3200774144, 3201826815,
+SNULL, 3200774144, 3200778239,
+STORE, 3200778240, 3201826815,
+STORE, 3200774144, 3200778239,
+SNULL, 3201822719, 3201826815,
+STORE, 3200778240, 3201822719,
+STORE, 3201822720, 3201826815,
+STORE, 3803451392, 3803471871,
+SNULL, 3803455487, 3803471871,
+STORE, 3803451392, 3803455487,
+STORE, 3803455488, 3803471871,
+SNULL, 3200782335, 3201822719,
+STORE, 3200778240, 3200782335,
+STORE, 3200782336, 3201822719,
+STORE, 3949666304, 3949674495,
+STORE, 3949408256, 3949416447,
+STORE, 3199688704, 3200778239,
+SNULL, 3199688704, 3199692799,
+STORE, 3199692800, 3200778239,
+STORE, 3199688704, 3199692799,
+SNULL, 3200770047, 3200778239,
+STORE, 3199692800, 3200770047,
+STORE, 3200770048, 3200778239,
+STORE, 3799306240, 3799326719,
+SNULL, 3799310335, 3799326719,
+STORE, 3799306240, 3799310335,
+STORE, 3799310336, 3799326719,
+SNULL, 3199696895, 3200770047,
+STORE, 3199692800, 3199696895,
+STORE, 3199696896, 3200770047,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+STORE, 3799277568, 3799306239,
+SNULL, 3799277568, 3799306239,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+SNULL, 4041162751, 4041170943,
+STORE, 4041154560, 4041162751,
+STORE, 4041162752, 4041170943,
+SNULL, 4041162752, 4041170943,
+SNULL, 4041154560, 4041162751,
+SNULL, 4041191424, 4041211903,
+SNULL, 4041170944, 4041191423,
+SNULL, 3626471423, 3626475519,
+STORE, 3626467328, 3626471423,
+STORE, 3626471424, 3626475519,
+SNULL, 3626471424, 3627524095,
+SNULL, 3625418751, 3625422847,
+STORE, 3625414656, 3625418751,
+STORE, 3625418752, 3625422847,
+SNULL, 3625418752, 3626471423,
+STORE, 3627393024, 3627524095,
+STORE, 3627261952, 3627393023,
+STORE, 3627261952, 3627524095,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+STORE, 3195494400, 3197591551,
+SNULL, 3197591552, 3199688703,
+SNULL, 3195494400, 3197591551,
+STORE, 3197591552, 3199688703,
+SNULL, 3197591552, 3199688703,
+STORE, 3197591552, 3199688703,
+STORE, 3195494400, 3197591551,
+SNULL, 3197591552, 3199688703,
+SNULL, 3195494400, 3197591551,
+STORE, 3798999040, 3799101439,
+SNULL, 3798999040, 3799101439,
+/*
+ * mmap: unmapped_area_topdown: ffff9a9f14ddaa80
+ * Gap was found: mt 4041162752 gap_end 4041183232
+ * mmap: window was 4052029440 - 4096 size 28672
+ * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423
+ * mmap: mas.index 4041162752 align mask 0 offset 0
+ * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678)
+ */
+ };
+
+ unsigned long set43[] = {
+STORE, 140737488347136, 140737488351231,
+STORE, 140734187720704, 140737488351231,
+SNULL, 140734187724800, 140737488351231,
+STORE, 140734187589632, 140734187724799,
+STORE, 4194304, 6443007,
+STORE, 4337664, 6443007,
+STORE, 4194304, 4337663,
+SNULL, 4337664, 6443007,
+STORE, 6430720, 6443007,
+STORE, 206158430208, 206160674815,
+STORE, 206158569472, 206160674815,
+STORE, 206158430208, 206158569471,
+SNULL, 206158569472, 206160674815,
+STORE, 206160662528, 206160670719,
+STORE, 206160670720, 206160674815,
+STORE, 140734188756992, 140734188765183,
+STORE, 140734188740608, 140734188756991,
+STORE, 140501948112896, 140501948116991,
+ };
+
+ int count = 0;
+ void *ptr = NULL;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ mt_set_non_kernel(3);
+ check_erase2_testset(mt, set, ARRAY_SIZE(set));
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set2, ARRAY_SIZE(set2));
+ start = 140735933894656;
+ MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL));
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(2);
+ mt_init_flags(mt, 0);
+ check_erase2_testset(mt, set3, ARRAY_SIZE(set3));
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, 0);
+ check_erase2_testset(mt, set4, ARRAY_SIZE(set4));
+ rcu_read_lock();
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ if (xa_is_zero(entry))
+ continue;
+ }
+ rcu_read_unlock();
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ mt_set_non_kernel(100);
+ check_erase2_testset(mt, set5, ARRAY_SIZE(set5));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set6, ARRAY_SIZE(set6));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set7, ARRAY_SIZE(set7));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set8, ARRAY_SIZE(set8));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set9, ARRAY_SIZE(set9));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set10, ARRAY_SIZE(set10));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set11, ARRAY_SIZE(set11));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000);
+ MT_BUG_ON(mt, mas.last != 140014592573439);
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mas.tree = mt;
+ count = 0;
+ mas.index = 0;
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set12, ARRAY_SIZE(set12));
+ rcu_barrier();
+ mas_for_each(&mas, entry, ULONG_MAX) {
+ if (xa_is_zero(entry))
+ continue;
+ BUG_ON(count > 12);
+ count++;
+ }
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set13, ARRAY_SIZE(set13));
+ mtree_erase(mt, 140373516443648);
+ rcu_read_lock();
+ mas_empty_area_rev(&mas, 0, 140373518663680, 4096);
+ rcu_read_unlock();
+ mtree_destroy(mt);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set14, ARRAY_SIZE(set14));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set15, ARRAY_SIZE(set15));
+ rcu_barrier();
+ mtree_destroy(mt);
+
+ /* set16 was to find a bug on limit updating at slot 0. */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set16, ARRAY_SIZE(set16));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000);
+ MT_BUG_ON(mt, mas.last != 139921865547775);
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ /*
+ * set17 found a bug in walking backwards and not counting nulls at
+ * the end. This could cause a gap to be missed if the null had any
+ * size.
+ */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set17, ARRAY_SIZE(set17));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000);
+ MT_BUG_ON(mt, mas.last != 139953197322239);
+/* MT_BUG_ON(mt, mas.index != 139953197318144); */
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ /*
+ * set18 found a bug in walking backwards and not setting the max from
+ * the node, but using the parent node. This was only an issue if the
+ * next slot in the parent had what we needed.
+ */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set18, ARRAY_SIZE(set18));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936);
+ MT_BUG_ON(mt, mas.last != 140222968475647);
+ /*MT_BUG_ON(mt, mas.index != 140222966259712); */
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ /*
+ * set19 found 2 bugs in prev.
+ * 1. If we hit root without finding anything, then there was an
+ * infinite loop.
+ * 2. The first ascending wasn't using the correct slot which may have
+ * caused missed entries.
+ */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set19, ARRAY_SIZE(set19));
+ rcu_barrier();
+ mas.index = 140656779083776;
+ entry = mas_find(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, entry != xa_mk_value(140656779083776));
+ entry = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, entry != xa_mk_value(140656766251008));
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ /*
+ * set20 found a bug in mas_may_move_gap due to the slot being
+ * overwritten during the __mas_add operation and setting it to zero.
+ */
+ mt_set_non_kernel(99);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set20, ARRAY_SIZE(set20));
+ rcu_barrier();
+ check_load(mt, 94849009414144, NULL);
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(99);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set21, ARRAY_SIZE(set21));
+ rcu_barrier();
+ mt_validate(mt);
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(999);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set22, ARRAY_SIZE(set22));
+ rcu_barrier();
+ mt_validate(mt);
+ ptr = mtree_load(mt, 140551363362816);
+ MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159));
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(99);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set23, ARRAY_SIZE(set23));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+
+ mt_set_non_kernel(99);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set24, ARRAY_SIZE(set24));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(99);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set25, ARRAY_SIZE(set25));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* Split on NULL followed by delete - causes gap issues. */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set26, ARRAY_SIZE(set26));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 140109042671616, 409600);
+ MT_BUG_ON(mt, mas.last != 140109040959487);
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* Split on NULL followed by delete - causes gap issues. */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set27, ARRAY_SIZE(set27));
+ rcu_barrier();
+ MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336));
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set28, ARRAY_SIZE(set28));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152);
+ /* Search for the size of gap then align it (offset 0) */
+ mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056);
+ MT_BUG_ON(mt, mas.index != 139918401601536);
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* This test found issues with retry moving rebalanced nodes so the
+ * incorrect parent pivot was updated.
+ */
+ mt_set_non_kernel(999);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set29, ARRAY_SIZE(set29));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* This test found issues with deleting all entries in a node when
+ * surrounded by entries in the next nodes, then deleting the entries
+ * surrounding the node filled with deleted entries.
+ */
+ mt_set_non_kernel(999);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set30, ARRAY_SIZE(set30));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* This test found an issue with deleting all entries in a node that was
+ * the end node and mas_gap incorrectly set next = curr, and curr = prev
+ * then moved next to the left, losing data.
+ */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set31, ARRAY_SIZE(set31));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set32, ARRAY_SIZE(set32));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+/*
+ * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found:
+ * mt 140582827569152 gap_end 140582869532672
+ * mmap: window was 140583656296448 - 4096 size 134217728
+ * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879
+ * mmap: mas.index 140582827569152 align mask 0 offset 0
+ * mmap: rb_find_vma find on
+ * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00)
+ */
+
+ /* move gap failed due to an entirely empty node */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set33, ARRAY_SIZE(set33));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728);
+ MT_BUG_ON(mt, mas.last != 140583003750399);
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /*
+ * Incorrect gap in tree caused by mas_prev not setting the limits
+ * correctly while walking down.
+ */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set34, ARRAY_SIZE(set34));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* Empty leaf at the end of a parent caused incorrect gap. */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set35, ARRAY_SIZE(set35));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(99);
+ /* Empty leaf at the end of a parent caused incorrect gap. */
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set36, ARRAY_SIZE(set36));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set37, ARRAY_SIZE(set37));
+ rcu_barrier();
+ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712));
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set38, ARRAY_SIZE(set38));
+ rcu_barrier();
+ MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712));
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set39, ARRAY_SIZE(set39));
+ rcu_barrier();
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set40, ARRAY_SIZE(set40));
+ rcu_barrier();
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set41, ARRAY_SIZE(set41));
+ rcu_barrier();
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* move gap failed due to an entirely empty node. */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set42, ARRAY_SIZE(set42));
+ rcu_barrier();
+ mas_empty_area_rev(&mas, 4096, 4052029440, 28672);
+ MT_BUG_ON(mt, mas.last != 4041211903);
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* gap calc off by one */
+ mt_set_non_kernel(99);
+ mas_reset(&mas);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_erase2_testset(mt, set43, ARRAY_SIZE(set43));
+ rcu_barrier();
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ mtree_destroy(mt);
+}
+
+static noinline void check_alloc_rev_range(struct maple_tree *mt)
+{
+ /*
+ * Generated by:
+ * cat /proc/self/maps | awk '{print $1}'|
+ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}'
+ */
+
+ unsigned long range[] = {
+ /* Inclusive , Exclusive. */
+ 0x565234af2000, 0x565234af4000,
+ 0x565234af4000, 0x565234af9000,
+ 0x565234af9000, 0x565234afb000,
+ 0x565234afc000, 0x565234afd000,
+ 0x565234afd000, 0x565234afe000,
+ 0x565235def000, 0x565235e10000,
+ 0x7f36d4bfd000, 0x7f36d4ee2000,
+ 0x7f36d4ee2000, 0x7f36d4f04000,
+ 0x7f36d4f04000, 0x7f36d504c000,
+ 0x7f36d504c000, 0x7f36d5098000,
+ 0x7f36d5098000, 0x7f36d5099000,
+ 0x7f36d5099000, 0x7f36d509d000,
+ 0x7f36d509d000, 0x7f36d509f000,
+ 0x7f36d509f000, 0x7f36d50a5000,
+ 0x7f36d50b9000, 0x7f36d50db000,
+ 0x7f36d50db000, 0x7f36d50dc000,
+ 0x7f36d50dc000, 0x7f36d50fa000,
+ 0x7f36d50fa000, 0x7f36d5102000,
+ 0x7f36d5102000, 0x7f36d5103000,
+ 0x7f36d5103000, 0x7f36d5104000,
+ 0x7f36d5104000, 0x7f36d5105000,
+ 0x7fff5876b000, 0x7fff5878d000,
+ 0x7fff5878e000, 0x7fff58791000,
+ 0x7fff58791000, 0x7fff58793000,
+ };
+
+ unsigned long holes[] = {
+ /*
+ * Note: start of hole is INCLUSIVE
+ * end of hole is EXCLUSIVE
+ * (opposite of the above table.)
+ * Start of hole, end of hole, size of hole (+1)
+ */
+ 0x565234afb000, 0x565234afc000, 0x1000,
+ 0x565234afe000, 0x565235def000, 0x12F1000,
+ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000,
+ };
+
+ /*
+ * req_range consists of 4 values.
+ * 1. min index
+ * 2. max index
+ * 3. size
+ * 4. number that should be returned.
+ * 5. return value
+ */
+ unsigned long req_range[] = {
+ 0x565234af9000, /* Min */
+ 0x7fff58791000, /* Max */
+ 0x1000, /* Size */
+ 0x7fff5878d << 12, /* First rev hole of size 0x1000 */
+ 0, /* Return value success. */
+
+ 0x0, /* Min */
+ 0x565234AF1 << 12, /* Max */
+ 0x3000, /* Size */
+ 0x565234AEE << 12, /* max - 3. */
+ 0, /* Return value success. */
+
+ 0x0, /* Min */
+ -1, /* Max */
+ 0x1000, /* Size */
+ 562949953421311 << 12,/* First rev hole of size 0x1000 */
+ 0, /* Return value success. */
+
+ 0x0, /* Min */
+ 0x7F36D510A << 12, /* Max */
+ 0x4000, /* Size */
+ 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */
+ 0, /* Return value success. */
+
+ /* Ascend test. */
+ 0x0,
+ 34148798629 << 12,
+ 19 << 12,
+ 34148797418 << 12,
+ 0x0,
+
+ /* Too big test. */
+ 0x0,
+ 18446744073709551615UL,
+ 562915594369134UL << 12,
+ 0x0,
+ -EBUSY,
+
+ };
+
+ int i, range_count = ARRAY_SIZE(range);
+ int req_range_count = ARRAY_SIZE(req_range);
+ unsigned long min = 0;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY,
+ GFP_KERNEL);
+#define DEBUG_REV_RANGE 0
+ for (i = 0; i < range_count; i += 2) {
+ /* Inclusive, Inclusive (with the -1) */
+
+#if DEBUG_REV_RANGE
+ pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12,
+ (range[i + 1] >> 12) - 1);
+#endif
+ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1,
+ xa_mk_value(range[i] >> 12), 0);
+ mt_validate(mt);
+ }
+
+
+ for (i = 0; i < ARRAY_SIZE(holes); i += 3) {
+#if DEBUG_REV_RANGE
+ pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n",
+ min, holes[i+1]>>12, holes[i+2]>>12,
+ holes[i] >> 12);
+#endif
+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, min,
+ holes[i+1] >> 12,
+ holes[i+2] >> 12));
+#if DEBUG_REV_RANGE
+ pr_debug("Found %lu %lu\n", mas.index, mas.last);
+ pr_debug("gap %lu %lu\n", (holes[i] >> 12),
+ (holes[i+1] >> 12));
+#endif
+ MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12));
+ MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12));
+ min = holes[i+1] >> 12;
+ mas_reset(&mas);
+ }
+
+ for (i = 0; i < req_range_count; i += 5) {
+#if DEBUG_REV_RANGE
+ pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n",
+ req_range[i] >> 12,
+ (req_range[i + 1] >> 12) - 1,
+ req_range[i+2] >> 12,
+ req_range[i+3] >> 12);
+#endif
+ check_mtree_alloc_rrange(mt,
+ req_range[i] >> 12, /* start */
+ req_range[i+1] >> 12, /* end */
+ req_range[i+2] >> 12, /* size */
+ req_range[i+3] >> 12, /* expected address */
+ req_range[i+4], /* expected return */
+ xa_mk_value(req_range[i] >> 12)); /* pointer */
+ mt_validate(mt);
+ }
+
+ mt_set_non_kernel(1);
+ mtree_erase(mt, 34148798727); /* create a deleted range. */
+ check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414,
+ 34148798725, 0, mt);
+
+ mtree_destroy(mt);
+}
+
+static noinline void check_alloc_range(struct maple_tree *mt)
+{
+ /*
+ * Generated by:
+ * cat /proc/self/maps|awk '{print $1}'|
+ * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}'
+ */
+
+ unsigned long range[] = {
+ /* Inclusive , Exclusive. */
+ 0x565234af2000, 0x565234af4000,
+ 0x565234af4000, 0x565234af9000,
+ 0x565234af9000, 0x565234afb000,
+ 0x565234afc000, 0x565234afd000,
+ 0x565234afd000, 0x565234afe000,
+ 0x565235def000, 0x565235e10000,
+ 0x7f36d4bfd000, 0x7f36d4ee2000,
+ 0x7f36d4ee2000, 0x7f36d4f04000,
+ 0x7f36d4f04000, 0x7f36d504c000,
+ 0x7f36d504c000, 0x7f36d5098000,
+ 0x7f36d5098000, 0x7f36d5099000,
+ 0x7f36d5099000, 0x7f36d509d000,
+ 0x7f36d509d000, 0x7f36d509f000,
+ 0x7f36d509f000, 0x7f36d50a5000,
+ 0x7f36d50b9000, 0x7f36d50db000,
+ 0x7f36d50db000, 0x7f36d50dc000,
+ 0x7f36d50dc000, 0x7f36d50fa000,
+ 0x7f36d50fa000, 0x7f36d5102000,
+ 0x7f36d5102000, 0x7f36d5103000,
+ 0x7f36d5103000, 0x7f36d5104000,
+ 0x7f36d5104000, 0x7f36d5105000,
+ 0x7fff5876b000, 0x7fff5878d000,
+ 0x7fff5878e000, 0x7fff58791000,
+ 0x7fff58791000, 0x7fff58793000,
+ };
+ unsigned long holes[] = {
+ /* Start of hole, end of hole, size of hole (+1) */
+ 0x565234afb000, 0x565234afc000, 0x1000,
+ 0x565234afe000, 0x565235def000, 0x12F1000,
+ 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000,
+ };
+
+ /*
+ * req_range consists of 4 values.
+ * 1. min index
+ * 2. max index
+ * 3. size
+ * 4. number that should be returned.
+ * 5. return value
+ */
+ unsigned long req_range[] = {
+ 0x565234af9000, /* Min */
+ 0x7fff58791000, /* Max */
+ 0x1000, /* Size */
+ 0x565234afb000, /* First hole in our data of size 1000. */
+ 0, /* Return value success. */
+
+ 0x0, /* Min */
+ 0x7fff58791000, /* Max */
+ 0x1F00, /* Size */
+ 0x0, /* First hole in our data of size 2000. */
+ 0, /* Return value success. */
+
+ /* Test ascend. */
+ 34148797436 << 12, /* Min */
+ 0x7fff587AF000, /* Max */
+ 0x3000, /* Size */
+ 34148798629 << 12, /* Expected location */
+ 0, /* Return value success. */
+
+ /* Test failing. */
+ 34148798623 << 12, /* Min */
+ 34148798683 << 12, /* Max */
+ 0x15000, /* Size */
+ 0, /* Expected location */
+ -EBUSY, /* Return value failed. */
+
+ /* Test filling entire gap. */
+ 34148798623 << 12, /* Min */
+ 0x7fff587AF000, /* Max */
+ 0x10000, /* Size */
+ 34148798632 << 12, /* Expected location */
+ 0, /* Return value success. */
+
+ /* Test walking off the end of root. */
+ 0, /* Min */
+ -1, /* Max */
+ -1, /* Size */
+ 0, /* Expected location */
+ -EBUSY, /* Return value failure. */
+
+ /* Test looking for too large a hole across entire range. */
+ 0, /* Min */
+ -1, /* Max */
+ 4503599618982063UL << 12, /* Size */
+ 34359052178 << 12, /* Expected location */
+ -EBUSY, /* Return failure. */
+ };
+ int i, range_count = ARRAY_SIZE(range);
+ int req_range_count = ARRAY_SIZE(req_range);
+ unsigned long min = 0x565234af2000;
+
+ mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY,
+ GFP_KERNEL);
+ for (i = 0; i < range_count; i += 2) {
+#define DEBUG_ALLOC_RANGE 0
+#if DEBUG_ALLOC_RANGE
+ pr_debug("\tInsert %lu-%lu\n", range[i] >> 12,
+ (range[i + 1] >> 12) - 1);
+ mt_dump(mt);
+#endif
+ check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1,
+ xa_mk_value(range[i] >> 12), 0);
+ mt_validate(mt);
+ }
+
+
+ MA_STATE(mas, mt, 0, 0);
+
+ for (i = 0; i < ARRAY_SIZE(holes); i += 3) {
+
+#if DEBUG_ALLOC_RANGE
+ pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12,
+ holes[i+1] >> 12, holes[i+2] >> 12,
+ min, holes[i+1]);
+#endif
+ MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12,
+ holes[i+1] >> 12,
+ holes[i+2] >> 12));
+ MT_BUG_ON(mt, mas.index != holes[i] >> 12);
+ min = holes[i+1];
+ mas_reset(&mas);
+ }
+ for (i = 0; i < req_range_count; i += 5) {
+#if DEBUG_ALLOC_RANGE
+ pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n",
+ i/5, req_range[i] >> 12, req_range[i + 1] >> 12,
+ req_range[i + 2] >> 12, req_range[i + 3] >> 12,
+ req_range[i], req_range[i+1]);
+#endif
+ check_mtree_alloc_range(mt,
+ req_range[i] >> 12, /* start */
+ req_range[i+1] >> 12, /* end */
+ req_range[i+2] >> 12, /* size */
+ req_range[i+3] >> 12, /* expected address */
+ req_range[i+4], /* expected return */
+ xa_mk_value(req_range[i] >> 12)); /* pointer */
+ mt_validate(mt);
+#if DEBUG_ALLOC_RANGE
+ mt_dump(mt);
+#endif
+ }
+
+ mtree_destroy(mt);
+}
+
+static noinline void check_ranges(struct maple_tree *mt)
+{
+ int i, val, val2;
+ unsigned long r[] = {
+ 10, 15,
+ 20, 25,
+ 17, 22, /* Overlaps previous range. */
+ 9, 1000, /* Huge. */
+ 100, 200,
+ 45, 168,
+ 118, 128,
+ };
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+ check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0);
+ check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0);
+ check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST);
+ MT_BUG_ON(mt, !mt_height(mt));
+ /* Store */
+ check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0);
+ check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0);
+ check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+ MT_BUG_ON(mt, mt_height(mt));
+
+ check_seq(mt, 50, false);
+ mt_set_non_kernel(4);
+ check_store_range(mt, 5, 47, xa_mk_value(47), 0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ /* Create tree of 1-100 */
+ check_seq(mt, 100, false);
+ /* Store 45-168 */
+ mt_set_non_kernel(10);
+ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ /* Create tree of 1-200 */
+ check_seq(mt, 200, false);
+ /* Store 45-168 */
+ check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ check_seq(mt, 30, false);
+ check_store_range(mt, 6, 18, xa_mk_value(6), 0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ /* Overwrite across multiple levels. */
+ /* Create tree of 1-400 */
+ check_seq(mt, 400, false);
+ mt_set_non_kernel(50);
+ /* Store 118-128 */
+ check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0);
+ mt_set_non_kernel(50);
+ mtree_test_erase(mt, 140);
+ mtree_test_erase(mt, 141);
+ mtree_test_erase(mt, 142);
+ mtree_test_erase(mt, 143);
+ mtree_test_erase(mt, 130);
+ mtree_test_erase(mt, 131);
+ mtree_test_erase(mt, 132);
+ mtree_test_erase(mt, 133);
+ mtree_test_erase(mt, 134);
+ mtree_test_erase(mt, 135);
+ check_load(mt, r[12], xa_mk_value(r[12]));
+ check_load(mt, r[13], xa_mk_value(r[12]));
+ check_load(mt, r[13] - 1, xa_mk_value(r[12]));
+ check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1));
+ check_load(mt, 135, NULL);
+ check_load(mt, 140, NULL);
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+
+
+ /* Overwrite multiple levels at the end of the tree (slot 7) */
+ mt_set_non_kernel(50);
+ check_seq(mt, 400, false);
+ check_store_range(mt, 353, 361, xa_mk_value(353), 0);
+ check_store_range(mt, 347, 352, xa_mk_value(347), 0);
+
+ check_load(mt, 346, xa_mk_value(346));
+ for (i = 347; i <= 352; i++)
+ check_load(mt, i, xa_mk_value(347));
+ for (i = 353; i <= 361; i++)
+ check_load(mt, i, xa_mk_value(353));
+ check_load(mt, 362, xa_mk_value(362));
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(50);
+ check_seq(mt, 400, false);
+ check_store_range(mt, 352, 364, NULL, 0);
+ check_store_range(mt, 351, 363, xa_mk_value(352), 0);
+ check_load(mt, 350, xa_mk_value(350));
+ check_load(mt, 351, xa_mk_value(352));
+ for (i = 352; i <= 363; i++)
+ check_load(mt, i, xa_mk_value(352));
+ check_load(mt, 364, NULL);
+ check_load(mt, 365, xa_mk_value(365));
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ mt_set_non_kernel(5);
+ check_seq(mt, 400, false);
+ check_store_range(mt, 352, 364, NULL, 0);
+ check_store_range(mt, 351, 364, xa_mk_value(352), 0);
+ check_load(mt, 350, xa_mk_value(350));
+ check_load(mt, 351, xa_mk_value(352));
+ for (i = 352; i <= 364; i++)
+ check_load(mt, i, xa_mk_value(352));
+ check_load(mt, 365, xa_mk_value(365));
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+
+ mt_set_non_kernel(50);
+ check_seq(mt, 400, false);
+ check_store_range(mt, 362, 367, xa_mk_value(362), 0);
+ check_store_range(mt, 353, 361, xa_mk_value(353), 0);
+ mt_set_non_kernel(0);
+ mt_validate(mt);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+ /*
+ * Interesting cases:
+ * 1. Overwrite the end of a node and end in the first entry of the next
+ * node.
+ * 2. Split a single range
+ * 3. Overwrite the start of a range
+ * 4. Overwrite the end of a range
+ * 5. Overwrite the entire range
+ * 6. Overwrite a range that causes multiple parent nodes to be
+ * combined
+ * 7. Overwrite a range that causes multiple parent nodes and part of
+ * root to be combined
+ * 8. Overwrite the whole tree
+ * 9. Try to overwrite the zero entry of an alloc tree.
+ * 10. Write a range larger than a nodes current pivot
+ */
+
+ mt_set_non_kernel(50);
+ for (i = 0; i <= 500; i++) {
+ val = i*5;
+ val2 = (i+1)*5;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ }
+ check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0);
+ check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0);
+ check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0);
+ check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0);
+ check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0);
+ mtree_destroy(mt);
+ mt_set_non_kernel(0);
+
+ mt_set_non_kernel(50);
+ for (i = 0; i <= 500; i++) {
+ val = i*5;
+ val2 = (i+1)*5;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ }
+ check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0);
+ check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0);
+ check_store_range(mt, 2425, 2425, xa_mk_value(2), 0);
+ check_store_range(mt, 2460, 2470, NULL, 0);
+ check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0);
+ check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0);
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ /* Test rebalance gaps */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ mt_set_non_kernel(50);
+ for (i = 0; i <= 50; i++) {
+ val = i*10;
+ val2 = (i+1)*10;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ }
+ check_store_range(mt, 161, 161, xa_mk_value(161), 0);
+ check_store_range(mt, 162, 162, xa_mk_value(162), 0);
+ check_store_range(mt, 163, 163, xa_mk_value(163), 0);
+ check_store_range(mt, 240, 249, NULL, 0);
+ mtree_erase(mt, 200);
+ mtree_erase(mt, 210);
+ mtree_erase(mt, 220);
+ mtree_erase(mt, 230);
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= 500; i++) {
+ val = i*10;
+ val2 = (i+1)*10;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ }
+ check_store_range(mt, 4600, 4959, xa_mk_value(1), 0);
+ mt_validate(mt);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= 500; i++) {
+ val = i*10;
+ val2 = (i+1)*10;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ }
+ check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0);
+ check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0);
+ check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0);
+ check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0);
+ check_store_range(mt, 4842, 4849, NULL, 0);
+ mt_validate(mt);
+ MT_BUG_ON(mt, !mt_height(mt));
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= 1300; i++) {
+ val = i*10;
+ val2 = (i+1)*10;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ MT_BUG_ON(mt, mt_height(mt) >= 4);
+ }
+ /* Cause a 3 child split all the way up the tree. */
+ for (i = 5; i < 215; i += 10)
+ check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0);
+ for (i = 5; i < 65; i += 10)
+ check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0);
+
+ MT_BUG_ON(mt, mt_height(mt) >= 4);
+ for (i = 5; i < 45; i += 10)
+ check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0);
+ MT_BUG_ON(mt, mt_height(mt) < 4);
+ mtree_destroy(mt);
+
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= 1200; i++) {
+ val = i*10;
+ val2 = (i+1)*10;
+ check_store_range(mt, val, val2, xa_mk_value(val), 0);
+ MT_BUG_ON(mt, mt_height(mt) >= 4);
+ }
+ /* Fill parents and leaves before split. */
+ for (i = 5; i < 455; i += 10)
+ check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0);
+
+ for (i = 1; i < 16; i++)
+ check_store_range(mt, 8185 + i, 8185 + i + 1,
+ xa_mk_value(8185+i), 0);
+ MT_BUG_ON(mt, mt_height(mt) >= 4);
+ /* triple split across multiple levels. */
+ check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0);
+ MT_BUG_ON(mt, mt_height(mt) != 4);
+}
+
+static noinline void check_next_entry(struct maple_tree *mt)
+{
+ void *entry = NULL;
+ unsigned long limit = 30, i = 0;
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+ MA_STATE(mas, mt, i, i);
+
+ check_seq(mt, limit, false);
+ rcu_read_lock();
+
+ /* Check the first one and get ma_state in the correct state. */
+ MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++));
+ for ( ; i <= limit + 1; i++) {
+ entry = mas_next(&mas, limit);
+ if (i > limit)
+ MT_BUG_ON(mt, entry != NULL);
+ else
+ MT_BUG_ON(mt, xa_mk_value(i) != entry);
+ }
+ rcu_read_unlock();
+ mtree_destroy(mt);
+}
+
+static noinline void check_prev_entry(struct maple_tree *mt)
+{
+ unsigned long index = 16;
+ void *value;
+ int i;
+
+ MA_STATE(mas, mt, index, index);
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+ check_seq(mt, 30, false);
+
+ rcu_read_lock();
+ value = mas_find(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, value != xa_mk_value(index));
+ value = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, value != xa_mk_value(index - 1));
+ rcu_read_unlock();
+ mtree_destroy(mt);
+
+ /* Check limits on prev */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ mas_lock(&mas);
+ for (i = 0; i <= index; i++) {
+ mas_set_range(&mas, i*10, i*10+5);
+ mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL);
+ }
+
+ mas_set(&mas, 20);
+ value = mas_walk(&mas);
+ MT_BUG_ON(mt, value != xa_mk_value(2));
+
+ value = mas_prev(&mas, 19);
+ MT_BUG_ON(mt, value != NULL);
+
+ mas_set(&mas, 80);
+ value = mas_walk(&mas);
+ MT_BUG_ON(mt, value != xa_mk_value(8));
+
+ value = mas_prev(&mas, 76);
+ MT_BUG_ON(mt, value != NULL);
+
+ mas_unlock(&mas);
+}
+
+static noinline void check_root_expand(struct maple_tree *mt)
+{
+ MA_STATE(mas, mt, 0, 0);
+ void *ptr;
+
+
+ mas_lock(&mas);
+ mas_set(&mas, 3);
+ ptr = mas_walk(&mas);
+ MT_BUG_ON(mt, ptr != NULL);
+ MT_BUG_ON(mt, mas.index != 0);
+ MT_BUG_ON(mt, mas.last != ULONG_MAX);
+
+ ptr = &check_prev_entry;
+ mas_set(&mas, 1);
+ mas_store_gfp(&mas, ptr, GFP_KERNEL);
+
+ mas_set(&mas, 0);
+ ptr = mas_walk(&mas);
+ MT_BUG_ON(mt, ptr != NULL);
+
+ mas_set(&mas, 1);
+ ptr = mas_walk(&mas);
+ MT_BUG_ON(mt, ptr != &check_prev_entry);
+
+ mas_set(&mas, 2);
+ ptr = mas_walk(&mas);
+ MT_BUG_ON(mt, ptr != NULL);
+ mas_unlock(&mas);
+ mtree_destroy(mt);
+
+
+ mt_init_flags(mt, 0);
+ mas_lock(&mas);
+
+ mas_set(&mas, 0);
+ ptr = &check_prev_entry;
+ mas_store_gfp(&mas, ptr, GFP_KERNEL);
+
+ mas_set(&mas, 5);
+ ptr = mas_walk(&mas);
+ MT_BUG_ON(mt, ptr != NULL);
+ MT_BUG_ON(mt, mas.index != 1);
+ MT_BUG_ON(mt, mas.last != ULONG_MAX);
+
+ mas_set_range(&mas, 0, 100);
+ ptr = mas_walk(&mas);
+ MT_BUG_ON(mt, ptr != &check_prev_entry);
+ MT_BUG_ON(mt, mas.last != 0);
+ mas_unlock(&mas);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, 0);
+ mas_lock(&mas);
+
+ mas_set(&mas, 0);
+ ptr = (void *)((unsigned long) check_prev_entry | 1UL);
+ mas_store_gfp(&mas, ptr, GFP_KERNEL);
+ ptr = mas_next(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, ptr != NULL);
+ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX));
+
+ mas_set(&mas, 1);
+ ptr = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0));
+ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL));
+
+ mas_unlock(&mas);
+
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, 0);
+ mas_lock(&mas);
+ mas_set(&mas, 0);
+ ptr = (void *)((unsigned long) check_prev_entry | 2UL);
+ mas_store_gfp(&mas, ptr, GFP_KERNEL);
+ ptr = mas_next(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, ptr != NULL);
+ MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX));
+
+ mas_set(&mas, 1);
+ ptr = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0));
+ MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL));
+
+
+ mas_unlock(&mas);
+}
+
+static noinline void check_prealloc(struct maple_tree *mt)
+{
+ unsigned long i, max = 100;
+ unsigned long allocated;
+ unsigned char height;
+ struct maple_node *mn;
+ void *ptr = check_prealloc;
+ MA_STATE(mas, mt, 10, 20);
+
+ mt_set_non_kernel(1000);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mas_destroy(&mas);
+ allocated = mas_allocated(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ mas_destroy(&mas);
+ allocated = mas_allocated(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1);
+ ma_free_rcu(mn);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ mas_destroy(&mas);
+ allocated = mas_allocated(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ mas_destroy(&mas);
+ allocated = mas_allocated(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+ ma_free_rcu(mn);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mn = mas_pop_node(&mas);
+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1);
+ mas_push_node(&mas, mn);
+ MT_BUG_ON(mt, mas_allocated(&mas) != allocated);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ mas_destroy(&mas);
+ allocated = mas_allocated(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mas_store_prealloc(&mas, ptr);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mas_store_prealloc(&mas, ptr);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mas_store_prealloc(&mas, ptr);
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mas_store_prealloc(&mas, ptr);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ mt_set_non_kernel(1);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+ mas_destroy(&mas);
+
+
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated == 0);
+ MT_BUG_ON(mt, allocated != 1 + height * 3);
+ mas_store_prealloc(&mas, ptr);
+ MT_BUG_ON(mt, mas_allocated(&mas) != 0);
+ mt_set_non_kernel(1);
+ MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0);
+ allocated = mas_allocated(&mas);
+ height = mas_mt_height(&mas);
+ MT_BUG_ON(mt, allocated != 0);
+}
+
+static noinline void check_spanning_write(struct maple_tree *mt)
+{
+ unsigned long i, max = 5000;
+ MA_STATE(mas, mt, 1200, 2380);
+
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 1205);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ for (i = 1; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mtree_lock(mt);
+ mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 1205);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* Test spanning store that requires a right cousin rebalance */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 1205);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /* Test non-alloc tree spanning store */
+ mt_init_flags(mt, 0);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mas_set_range(&mas, 0, 300);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 15);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /* Test spanning store that requires a right sibling rebalance */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mas_set_range(&mas, 0, 12865);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 15);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /* Test spanning store that requires a left sibling rebalance */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mas_set_range(&mas, 90, 13665);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 95);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /* Test spanning store that requires a left cousin rebalance */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mas_set_range(&mas, 46805, 49995);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 46815);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /*
+ * Test spanning store that requires a left cousin rebalance all the way
+ * to root
+ */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mas_set_range(&mas, 32395, 49995);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 46815);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /*
+ * Test spanning store that requires a right cousin rebalance all the
+ * way to root
+ */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+ mas_set_range(&mas, 38875, 43190);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 38900);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /* Test spanning store ending at full node (depth 2)*/
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+ mtree_lock(mt);
+ mas_set(&mas, 47606);
+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL);
+ mas_set(&mas, 47607);
+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL);
+ mas_set(&mas, 47608);
+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL);
+ mas_set(&mas, 47609);
+ mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL);
+ /* Ensure the parent node is full */
+ mas_ascend(&mas);
+ MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1);
+ mas_set_range(&mas, 11516, 48940);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ /* Test spanning write with many levels of no siblings */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+ mas_set_range(&mas, 43200, 49999);
+ mtree_lock(mt);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mas_set(&mas, 43200);
+ MT_BUG_ON(mt, mas_walk(&mas) != NULL);
+ mtree_unlock(mt);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= 100; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+
+ mtree_lock(mt);
+ mas_set_range(&mas, 76, 875);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ mtree_unlock(mt);
+}
+
+static noinline void check_null_expand(struct maple_tree *mt)
+{
+ unsigned long i, max = 100;
+ unsigned char data_end;
+ MA_STATE(mas, mt, 959, 959);
+
+ for (i = 0; i <= max; i++)
+ mtree_test_store_range(mt, i * 10, i * 10 + 5, &i);
+ /* Test expanding null at start. */
+ mas_walk(&mas);
+ data_end = mas_data_end(&mas);
+ mas_set_range(&mas, 959, 963);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ MT_BUG_ON(mt, mtree_load(mt, 963) != NULL);
+ MT_BUG_ON(mt, data_end != mas_data_end(&mas));
+
+ /* Test expanding null at end. */
+ mas_set(&mas, 880);
+ mas_walk(&mas);
+ data_end = mas_data_end(&mas);
+ mas_set_range(&mas, 884, 887);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ MT_BUG_ON(mt, mtree_load(mt, 884) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 889) != NULL);
+ MT_BUG_ON(mt, data_end != mas_data_end(&mas));
+
+ /* Test expanding null at start and end. */
+ mas_set(&mas, 890);
+ mas_walk(&mas);
+ data_end = mas_data_end(&mas);
+ mas_set_range(&mas, 900, 905);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ MT_BUG_ON(mt, mtree_load(mt, 899) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 900) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 905) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 906) != NULL);
+ MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas));
+
+ /* Test expanding null across multiple slots. */
+ mas_set(&mas, 800);
+ mas_walk(&mas);
+ data_end = mas_data_end(&mas);
+ mas_set_range(&mas, 810, 825);
+ mas_store_gfp(&mas, NULL, GFP_KERNEL);
+ MT_BUG_ON(mt, mtree_load(mt, 809) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 810) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 825) != NULL);
+ MT_BUG_ON(mt, mtree_load(mt, 826) != NULL);
+ MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas));
+}
+
+static noinline void check_gap_combining(struct maple_tree *mt)
+{
+ struct maple_enode *mn1, *mn2;
+ void *entry;
+
+ unsigned long seq100[] = {
+ /* 0-5 */
+ 74, 75, 76,
+ 50, 100, 2,
+
+ /* 6-12 */
+ 44, 45, 46, 43,
+ 20, 50, 3,
+
+ /* 13-20*/
+ 80, 81, 82,
+ 76, 2, 79, 85, 4,
+ };
+ unsigned long seq2000[] = {
+ 1152, 1151,
+ 1100, 1200, 2,
+ };
+ unsigned long seq400[] = {
+ 286, 318,
+ 256, 260, 266, 270, 275, 280, 290, 398,
+ 286, 310,
+ };
+
+ unsigned long index = seq100[0];
+
+ MA_STATE(mas, mt, index, index);
+
+ MT_BUG_ON(mt, !mtree_empty(mt));
+ check_seq(mt, 100, false); /* create 100 singletons. */
+
+ mt_set_non_kernel(1);
+ mtree_test_erase(mt, seq100[2]);
+ check_load(mt, seq100[2], NULL);
+ mtree_test_erase(mt, seq100[1]);
+ check_load(mt, seq100[1], NULL);
+
+ rcu_read_lock();
+ entry = mas_find(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, entry != xa_mk_value(index));
+ mn1 = mas.node;
+ mas_next(&mas, ULONG_MAX);
+ entry = mas_next(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, entry != xa_mk_value(index + 4));
+ mn2 = mas.node;
+ MT_BUG_ON(mt, mn1 == mn2); /* test the test. */
+
+ /*
+ * At this point, there is a gap of 2 at index + 1 between seq100[3] and
+ * seq100[4]. Search for the gap.
+ */
+ mt_set_non_kernel(1);
+ mas_reset(&mas);
+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4],
+ seq100[5]));
+ MT_BUG_ON(mt, mas.index != index + 1);
+ rcu_read_unlock();
+
+ mtree_test_erase(mt, seq100[6]);
+ check_load(mt, seq100[6], NULL);
+ mtree_test_erase(mt, seq100[7]);
+ check_load(mt, seq100[7], NULL);
+ mtree_test_erase(mt, seq100[8]);
+ index = seq100[9];
+
+ rcu_read_lock();
+ mas.index = index;
+ mas.last = index;
+ mas_reset(&mas);
+ entry = mas_find(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, entry != xa_mk_value(index));
+ mn1 = mas.node;
+ entry = mas_next(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, entry != xa_mk_value(index + 4));
+ mas_next(&mas, ULONG_MAX); /* go to the next entry. */
+ mn2 = mas.node;
+ MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */
+
+ /*
+ * At this point, there is a gap of 3 at seq100[6]. Find it by
+ * searching 20 - 50 for size 3.
+ */
+ mas_reset(&mas);
+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11],
+ seq100[12]));
+ MT_BUG_ON(mt, mas.index != seq100[6]);
+ rcu_read_unlock();
+
+ mt_set_non_kernel(1);
+ mtree_store(mt, seq100[13], NULL, GFP_KERNEL);
+ check_load(mt, seq100[13], NULL);
+ check_load(mt, seq100[14], xa_mk_value(seq100[14]));
+ mtree_store(mt, seq100[14], NULL, GFP_KERNEL);
+ check_load(mt, seq100[13], NULL);
+ check_load(mt, seq100[14], NULL);
+
+ mas_reset(&mas);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15],
+ seq100[17]));
+ MT_BUG_ON(mt, mas.index != seq100[13]);
+ mt_validate(mt);
+ rcu_read_unlock();
+
+ /*
+ * *DEPRECATED: no retries anymore* Test retry entry in the start of a
+ * gap.
+ */
+ mt_set_non_kernel(2);
+ mtree_test_store_range(mt, seq100[18], seq100[14], NULL);
+ mtree_test_erase(mt, seq100[15]);
+ mas_reset(&mas);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19],
+ seq100[20]));
+ rcu_read_unlock();
+ MT_BUG_ON(mt, mas.index != seq100[18]);
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* seq 2000 tests are for multi-level tree gaps */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_seq(mt, 2000, false);
+ mt_set_non_kernel(1);
+ mtree_test_erase(mt, seq2000[0]);
+ mtree_test_erase(mt, seq2000[1]);
+
+ mt_set_non_kernel(2);
+ mas_reset(&mas);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3],
+ seq2000[4]));
+ MT_BUG_ON(mt, mas.index != seq2000[1]);
+ rcu_read_unlock();
+ mt_validate(mt);
+ mtree_destroy(mt);
+
+ /* seq 400 tests rebalancing over two levels. */
+ mt_set_non_kernel(99);
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_seq(mt, 400, false);
+ mtree_test_store_range(mt, seq400[0], seq400[1], NULL);
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_seq(mt, 400, false);
+ mt_set_non_kernel(50);
+ mtree_test_store_range(mt, seq400[2], seq400[9],
+ xa_mk_value(seq400[2]));
+ mtree_test_store_range(mt, seq400[3], seq400[9],
+ xa_mk_value(seq400[3]));
+ mtree_test_store_range(mt, seq400[4], seq400[9],
+ xa_mk_value(seq400[4]));
+ mtree_test_store_range(mt, seq400[5], seq400[9],
+ xa_mk_value(seq400[5]));
+ mtree_test_store_range(mt, seq400[0], seq400[9],
+ xa_mk_value(seq400[0]));
+ mtree_test_store_range(mt, seq400[6], seq400[9],
+ xa_mk_value(seq400[6]));
+ mtree_test_store_range(mt, seq400[7], seq400[9],
+ xa_mk_value(seq400[7]));
+ mtree_test_store_range(mt, seq400[8], seq400[9],
+ xa_mk_value(seq400[8]));
+ mtree_test_store_range(mt, seq400[10], seq400[11],
+ xa_mk_value(seq400[10]));
+ mt_validate(mt);
+ mt_set_non_kernel(0);
+ mtree_destroy(mt);
+}
+static noinline void check_node_overwrite(struct maple_tree *mt)
+{
+ int i, max = 4000;
+
+ for (i = 0; i < max; i++)
+ mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100));
+
+ mtree_test_store_range(mt, 319951, 367950, NULL);
+ /*mt_dump(mt); */
+ mt_validate(mt);
+}
+
+static void mas_dfs_preorder(struct ma_state *mas)
+{
+
+ struct maple_enode *prev;
+ unsigned char end, slot = 0;
+
+ if (mas_is_start(mas)) {
+ mas_start(mas);
+ return;
+ }
+
+ if (mte_is_leaf(mas->node) && mte_is_root(mas->node))
+ goto done;
+
+walk_up:
+ end = mas_data_end(mas);
+ if (mte_is_leaf(mas->node) ||
+ (slot > end)) {
+ if (mte_is_root(mas->node))
+ goto done;
+
+ slot = mte_parent_slot(mas->node) + 1;
+ mas_ascend(mas);
+ goto walk_up;
+ }
+
+ prev = mas->node;
+ mas->node = mas_get_slot(mas, slot);
+ if (!mas->node || slot > end) {
+ if (mte_is_root(prev))
+ goto done;
+
+ mas->node = prev;
+ slot = mte_parent_slot(mas->node) + 1;
+ mas_ascend(mas);
+ goto walk_up;
+ }
+
+ return;
+done:
+ mas->node = MAS_NONE;
+}
+
+
+static void check_dfs_preorder(struct maple_tree *mt)
+{
+ unsigned long count = 0, max = 1000;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ check_seq(mt, max, false);
+ do {
+ count++;
+ mas_dfs_preorder(&mas);
+ } while (!mas_is_none(&mas));
+ MT_BUG_ON(mt, count != 74);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ mas_reset(&mas);
+ count = 0;
+ check_seq(mt, max, false);
+ do {
+ count++;
+ mas_dfs_preorder(&mas);
+ } while (!mas_is_none(&mas));
+ /*printk("count %lu\n", count); */
+ MT_BUG_ON(mt, count != 77);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ mas_reset(&mas);
+ count = 0;
+ check_rev_seq(mt, max, false);
+ do {
+ count++;
+ mas_dfs_preorder(&mas);
+ } while (!mas_is_none(&mas));
+ /*printk("count %lu\n", count); */
+ MT_BUG_ON(mt, count != 77);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ mas_reset(&mas);
+ mt_zero_nr_tallocated();
+ mt_set_non_kernel(200);
+ mas_expected_entries(&mas, max);
+ for (count = 0; count <= max; count++) {
+ mas.index = mas.last = count;
+ mas_store(&mas, xa_mk_value(count));
+ MT_BUG_ON(mt, mas_is_err(&mas));
+ }
+ mas_destroy(&mas);
+ rcu_barrier();
+ /*
+ * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n",
+ * max, mt_get_alloc_size()/1024, mt_nr_allocated(),
+ * mt_nr_tallocated());
+ */
+
+}
+
+#if defined(BENCH_SLOT_STORE)
+static noinline void bench_slot_store(struct maple_tree *mt)
+{
+ int i, brk = 105, max = 1040, brk_start = 100, count = 20000000;
+
+ for (i = 0; i < max; i += 10)
+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL);
+
+ for (i = 0; i < count; i++) {
+ mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL);
+ mtree_store_range(mt, brk_start, brk, xa_mk_value(brk),
+ GFP_KERNEL);
+ }
+}
+#endif
+
+#if defined(BENCH_NODE_STORE)
+static noinline void bench_node_store(struct maple_tree *mt)
+{
+ int i, overwrite = 76, max = 240, count = 20000000;
+
+ for (i = 0; i < max; i += 10)
+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL);
+
+ for (i = 0; i < count; i++) {
+ mtree_store_range(mt, overwrite, overwrite + 15,
+ xa_mk_value(overwrite), GFP_KERNEL);
+
+ overwrite += 5;
+ if (overwrite >= 135)
+ overwrite = 76;
+ }
+}
+#endif
+
+#if defined(BENCH_AWALK)
+static noinline void bench_awalk(struct maple_tree *mt)
+{
+ int i, max = 2500, count = 50000000;
+ MA_STATE(mas, mt, 1470, 1470);
+
+ for (i = 0; i < max; i += 10)
+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL);
+
+ mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL);
+
+ for (i = 0; i < count; i++) {
+ mas_empty_area_rev(&mas, 0, 2000, 10);
+ mas_reset(&mas);
+ }
+}
+#endif
+#if defined(BENCH_WALK)
+static noinline void bench_walk(struct maple_tree *mt)
+{
+ int i, max = 2500, count = 550000000;
+ MA_STATE(mas, mt, 1470, 1470);
+
+ for (i = 0; i < max; i += 10)
+ mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL);
+
+ for (i = 0; i < count; i++) {
+ mas_walk(&mas);
+ mas_reset(&mas);
+ }
+
+}
+#endif
+
+#if defined(BENCH_MT_FOR_EACH)
+static noinline void bench_mt_for_each(struct maple_tree *mt)
+{
+ int i, count = 1000000;
+ unsigned long max = 2500, index = 0;
+ void *entry;
+
+ for (i = 0; i < max; i += 5)
+ mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL);
+
+ for (i = 0; i < count; i++) {
+ unsigned long j = 0;
+
+ mt_for_each(mt, entry, index, max) {
+ MT_BUG_ON(mt, entry != xa_mk_value(j));
+ j += 5;
+ }
+
+ index = 0;
+ }
+
+}
+#endif
+
+static noinline void check_forking(struct maple_tree *mt)
+{
+
+ struct maple_tree newmt;
+ int i, nr_entries = 134;
+ void *val;
+ MA_STATE(mas, mt, 0, 0);
+ MA_STATE(newmas, mt, 0, 0);
+
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ mt_set_non_kernel(99999);
+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE);
+ newmas.tree = &newmt;
+ mas_reset(&newmas);
+ mas_reset(&mas);
+ mas.index = 0;
+ mas.last = 0;
+ if (mas_expected_entries(&newmas, nr_entries)) {
+ pr_err("OOM!");
+ BUG_ON(1);
+ }
+ mas_for_each(&mas, val, ULONG_MAX) {
+ newmas.index = mas.index;
+ newmas.last = mas.last;
+ mas_store(&newmas, val);
+ }
+ mas_destroy(&newmas);
+ mt_validate(&newmt);
+ mt_set_non_kernel(0);
+ mtree_destroy(&newmt);
+}
+
+static noinline void check_mas_store_gfp(struct maple_tree *mt)
+{
+
+ struct maple_tree newmt;
+ int i, nr_entries = 135;
+ void *val;
+ MA_STATE(mas, mt, 0, 0);
+ MA_STATE(newmas, mt, 0, 0);
+
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ mt_set_non_kernel(99999);
+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE);
+ newmas.tree = &newmt;
+ mas_reset(&newmas);
+ mas_set(&mas, 0);
+ mas_for_each(&mas, val, ULONG_MAX) {
+ newmas.index = mas.index;
+ newmas.last = mas.last;
+ mas_store_gfp(&newmas, val, GFP_KERNEL);
+ }
+
+ mt_validate(&newmt);
+ mt_set_non_kernel(0);
+ mtree_destroy(&newmt);
+}
+
+#if defined(BENCH_FORK)
+static noinline void bench_forking(struct maple_tree *mt)
+{
+
+ struct maple_tree newmt;
+ int i, nr_entries = 134, nr_fork = 80000;
+ void *val;
+ MA_STATE(mas, mt, 0, 0);
+ MA_STATE(newmas, mt, 0, 0);
+
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ for (i = 0; i < nr_fork; i++) {
+ mt_set_non_kernel(99999);
+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE);
+ newmas.tree = &newmt;
+ mas_reset(&newmas);
+ mas_reset(&mas);
+ mas.index = 0;
+ mas.last = 0;
+ if (mas_expected_entries(&newmas, nr_entries)) {
+ printk("OOM!");
+ BUG_ON(1);
+ }
+ mas_for_each(&mas, val, ULONG_MAX) {
+ newmas.index = mas.index;
+ newmas.last = mas.last;
+ mas_store(&newmas, val);
+ }
+ mas_destroy(&newmas);
+ mt_validate(&newmt);
+ mt_set_non_kernel(0);
+ mtree_destroy(&newmt);
+ }
+}
+#endif
+
+static noinline void next_prev_test(struct maple_tree *mt)
+{
+ int i, nr_entries = 200;
+ void *val;
+ MA_STATE(mas, mt, 0, 0);
+ struct maple_enode *mn;
+
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ for (i = 0; i <= nr_entries / 2; i++) {
+ mas_next(&mas, 1000);
+ if (mas_is_none(&mas))
+ break;
+
+ }
+ mas_reset(&mas);
+ mas_set(&mas, 0);
+ i = 0;
+ mas_for_each(&mas, val, 1000) {
+ i++;
+ }
+
+ mas_reset(&mas);
+ mas_set(&mas, 0);
+ i = 0;
+ mas_for_each(&mas, val, 1000) {
+ mas_pause(&mas);
+ i++;
+ }
+
+ /*
+ * 680 - 685 = 0x61a00001930c
+ * 686 - 689 = NULL;
+ * 690 - 695 = 0x61a00001930c
+ * Check simple next/prev
+ */
+ mas_set(&mas, 686);
+ val = mas_walk(&mas);
+ MT_BUG_ON(mt, val != NULL);
+
+ val = mas_next(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(690 / 10));
+ MT_BUG_ON(mt, mas.index != 690);
+ MT_BUG_ON(mt, mas.last != 695);
+
+ val = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(680 / 10));
+ MT_BUG_ON(mt, mas.index != 680);
+ MT_BUG_ON(mt, mas.last != 685);
+
+ val = mas_next(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(690 / 10));
+ MT_BUG_ON(mt, mas.index != 690);
+ MT_BUG_ON(mt, mas.last != 695);
+
+ val = mas_next(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(700 / 10));
+ MT_BUG_ON(mt, mas.index != 700);
+ MT_BUG_ON(mt, mas.last != 705);
+
+ /* Check across node boundaries of the tree */
+ mas_set(&mas, 70);
+ val = mas_walk(&mas);
+ MT_BUG_ON(mt, val != xa_mk_value(70 / 10));
+ MT_BUG_ON(mt, mas.index != 70);
+ MT_BUG_ON(mt, mas.last != 75);
+
+ val = mas_next(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(80 / 10));
+ MT_BUG_ON(mt, mas.index != 80);
+ MT_BUG_ON(mt, mas.last != 85);
+
+ val = mas_prev(&mas, 70);
+ MT_BUG_ON(mt, val != xa_mk_value(70 / 10));
+ MT_BUG_ON(mt, mas.index != 70);
+ MT_BUG_ON(mt, mas.last != 75);
+
+ /* Check across two levels of the tree */
+ mas_reset(&mas);
+ mas_set(&mas, 707);
+ val = mas_walk(&mas);
+ MT_BUG_ON(mt, val != NULL);
+ val = mas_next(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(710 / 10));
+ MT_BUG_ON(mt, mas.index != 710);
+ MT_BUG_ON(mt, mas.last != 715);
+ mn = mas.node;
+
+ val = mas_next(&mas, 1000);
+ MT_BUG_ON(mt, val != xa_mk_value(720 / 10));
+ MT_BUG_ON(mt, mas.index != 720);
+ MT_BUG_ON(mt, mas.last != 725);
+ MT_BUG_ON(mt, mn == mas.node);
+
+ val = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(710 / 10));
+ MT_BUG_ON(mt, mas.index != 710);
+ MT_BUG_ON(mt, mas.last != 715);
+
+ /* Check running off the end and back on */
+ mas_reset(&mas);
+ mas_set(&mas, 2000);
+ val = mas_walk(&mas);
+ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10));
+ MT_BUG_ON(mt, mas.index != 2000);
+ MT_BUG_ON(mt, mas.last != 2005);
+
+ val = mas_next(&mas, ULONG_MAX);
+ MT_BUG_ON(mt, val != NULL);
+ MT_BUG_ON(mt, mas.index != ULONG_MAX);
+ MT_BUG_ON(mt, mas.last != ULONG_MAX);
+
+ val = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(2000 / 10));
+ MT_BUG_ON(mt, mas.index != 2000);
+ MT_BUG_ON(mt, mas.last != 2005);
+
+ /* Check running off the start and back on */
+ mas_reset(&mas);
+ mas_set(&mas, 10);
+ val = mas_walk(&mas);
+ MT_BUG_ON(mt, val != xa_mk_value(1));
+ MT_BUG_ON(mt, mas.index != 10);
+ MT_BUG_ON(mt, mas.last != 15);
+
+ val = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, val != xa_mk_value(0));
+ MT_BUG_ON(mt, mas.index != 0);
+ MT_BUG_ON(mt, mas.last != 5);
+
+ val = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, val != NULL);
+ MT_BUG_ON(mt, mas.index != 0);
+ MT_BUG_ON(mt, mas.last != 0);
+
+ mas.index = 0;
+ mas.last = 5;
+ mas_store(&mas, NULL);
+ mas_reset(&mas);
+ mas_set(&mas, 10);
+ mas_walk(&mas);
+
+ val = mas_prev(&mas, 0);
+ MT_BUG_ON(mt, val != NULL);
+ MT_BUG_ON(mt, mas.index != 0);
+ MT_BUG_ON(mt, mas.last != 0);
+
+ mtree_destroy(mt);
+
+ mt_init(mt);
+ mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL);
+ mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL);
+ mas_set(&mas, 5);
+ val = mas_prev(&mas, 4);
+ MT_BUG_ON(mt, val != NULL);
+}
+
+#define RCU_RANGE_COUNT 1000
+#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true;} MT_BUG_ON(test->mt, y);}
+struct rcu_test_struct2 {
+ struct maple_tree *mt;
+
+ bool start;
+ bool stop;
+ unsigned int thread_count;
+
+ unsigned int seen_toggle;
+ unsigned int seen_added;
+ unsigned int seen_modified;
+ unsigned int seen_deleted;
+ int pause;
+
+ unsigned long index[RCU_RANGE_COUNT];
+ unsigned long last[RCU_RANGE_COUNT];
+};
+
+struct rcu_reader_struct {
+ unsigned int id;
+ int mod;
+ int del;
+ int flip;
+ int add;
+ int next;
+ struct rcu_test_struct2 *test;
+};
+
+/* RCU reader helper function */
+static void rcu_reader_register(struct rcu_test_struct2 *test)
+{
+ rcu_register_thread();
+ uatomic_inc(&test->thread_count);
+
+ while (!test->start)
+ usleep(test->pause * 100);
+}
+
+static void rcu_reader_setup(struct rcu_reader_struct *reader,
+ unsigned int id, struct rcu_test_struct2 *test)
+{
+ reader->id = id;
+ reader->test = test;
+ reader->mod = reader->id % 10;
+ reader->del = (reader->mod + 1) % 10;
+ reader->flip = (reader->mod + 2) % 10;
+ reader->add = (reader->mod + 3) % 10;
+ reader->next = (reader->mod + 4) % 10;
+}
+
+/* RCU reader in increasing index */
+static void *rcu_reader_fwd(void *ptr)
+{
+ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr;
+ struct rcu_test_struct2 *test = reader->test;
+ unsigned long index = reader->id;
+ bool toggled, modified, deleted, added;
+ int i;
+ void *entry, *prev = NULL;
+ MA_STATE(mas, test->mt, 0, 0);
+
+ rcu_reader_register(test);
+ toggled = modified = deleted = added = false;
+
+ while (!test->stop) {
+ i = 0;
+ /* mas_for_each ?*/
+ rcu_read_lock();
+ mas_set(&mas, test->index[index]);
+ mas_for_each(&mas, entry, test->last[index + 9]) {
+ unsigned long r_start, r_end, alt_start;
+ void *expected, *alt;
+
+ r_start = test->index[index + i];
+ r_end = test->last[index + i];
+ expected = xa_mk_value(r_start);
+
+ if (i == reader->del) {
+ if (!deleted) {
+ alt_start = test->index[index + reader->flip];
+ /* delete occurred. */
+ if (mas.index == alt_start) {
+ uatomic_inc(&test->seen_deleted);
+ deleted = true;
+ }
+ }
+ if (deleted) {
+ i = reader->flip;
+ r_start = test->index[index + i];
+ r_end = test->last[index + i];
+ expected = xa_mk_value(r_start);
+ }
+ }
+
+ if (!added && (i == reader->add)) {
+ alt_start = test->index[index + reader->next];
+ if (mas.index == r_start) {
+ uatomic_inc(&test->seen_added);
+ added = true;
+ } else if (mas.index == alt_start) {
+ i = reader->next;
+ r_start = test->index[index + i];
+ r_end = test->last[index + i];
+ expected = xa_mk_value(r_start);
+ }
+ }
+
+ RCU_MT_BUG_ON(test, mas.index != r_start);
+ RCU_MT_BUG_ON(test, mas.last != r_end);
+
+ if (i == reader->flip) {
+ alt = xa_mk_value(index + i + RCU_RANGE_COUNT);
+ if (prev) {
+ if (toggled && entry == expected)
+ uatomic_inc(&test->seen_toggle);
+ else if (!toggled && entry == alt)
+ uatomic_inc(&test->seen_toggle);
+ }
+
+ if (entry == expected)
+ toggled = false;
+ else if (entry == alt)
+ toggled = true;
+ else {
+ printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt);
+ RCU_MT_BUG_ON(test, 1);
+ }
+
+ prev = entry;
+ } else if (i == reader->mod) {
+ alt = xa_mk_value(index + i * 2 + 1 +
+ RCU_RANGE_COUNT);
+ if (entry != expected) {
+ if (!modified)
+ uatomic_inc(&test->seen_modified);
+ modified = true;
+ } else {
+ if (modified)
+ uatomic_inc(&test->seen_modified);
+ modified = false;
+ }
+
+ if (modified)
+ RCU_MT_BUG_ON(test, entry != alt);
+
+ } else {
+ if (entry != expected)
+ printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected);
+ RCU_MT_BUG_ON(test, entry != expected);
+ }
+
+ i++;
+ }
+ rcu_read_unlock();
+ usleep(test->pause);
+ }
+
+ rcu_unregister_thread();
+ return NULL;
+}
+
+/* RCU reader in decreasing index */
+static void *rcu_reader_rev(void *ptr)
+{
+ struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr;
+ struct rcu_test_struct2 *test = reader->test;
+ unsigned long index = reader->id;
+ bool toggled, modified, deleted, added;
+ int i;
+ void *prev = NULL;
+ MA_STATE(mas, test->mt, 0, 0);
+
+ rcu_reader_register(test);
+ toggled = modified = deleted = added = false;
+
+
+ while (!test->stop) {
+ void *entry;
+
+ i = 9;
+ mas_set(&mas, test->index[index + i]);
+
+ rcu_read_lock();
+ while (i--) {
+ unsigned long r_start, r_end, alt_start;
+ void *expected, *alt;
+ int line = __LINE__;
+
+ entry = mas_prev(&mas, test->index[index]);
+ r_start = test->index[index + i];
+ r_end = test->last[index + i];
+ expected = xa_mk_value(r_start);
+
+ if (i == reader->del) {
+ alt_start = test->index[index + reader->mod];
+ if (mas.index == alt_start) {
+ line = __LINE__;
+ if (!deleted)
+ uatomic_inc(&test->seen_deleted);
+ deleted = true;
+ }
+ if (deleted) {
+ line = __LINE__;
+ i = reader->mod;
+ r_start = test->index[index + i];
+ r_end = test->last[index + i];
+ expected = xa_mk_value(r_start);
+ }
+ }
+ if (!added && (i == reader->add)) {
+ alt_start = test->index[index + reader->flip];
+ if (mas.index == r_start) {
+ line = __LINE__;
+ uatomic_inc(&test->seen_added);
+ added = true;
+ } else if (mas.index == alt_start) {
+ line = __LINE__;
+ i = reader->flip;
+ r_start = test->index[index + i];
+ r_end = test->last[index + i];
+ expected = xa_mk_value(r_start);
+ }
+ }
+
+ if (i == reader->mod)
+ line = __LINE__;
+ else if (i == reader->flip)
+ line = __LINE__;
+
+ if (mas.index != r_start) {
+ alt = xa_mk_value(index + i * 2 + 1 +
+ RCU_RANGE_COUNT);
+ mt_dump(test->mt);
+ printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n",
+ mas.index, mas.last, entry,
+ r_start, r_end, expected, alt,
+ line, i);
+ }
+ RCU_MT_BUG_ON(test, mas.index != r_start);
+ RCU_MT_BUG_ON(test, mas.last != r_end);
+
+ if (i == reader->mod) {
+ alt = xa_mk_value(index + i * 2 + 1 +
+ RCU_RANGE_COUNT);
+
+ if (entry != expected) {
+ if (!modified)
+ uatomic_inc(&test->seen_modified);
+ modified = true;
+ } else {
+ if (modified)
+ uatomic_inc(&test->seen_modified);
+ modified = false;
+ }
+ if (modified)
+ RCU_MT_BUG_ON(test, entry != alt);
+
+
+ } else if (i == reader->flip) {
+ alt = xa_mk_value(index + i +
+ RCU_RANGE_COUNT);
+ if (prev) {
+ if (toggled && entry == expected)
+ uatomic_inc(&test->seen_toggle);
+ else if (!toggled && entry == alt)
+ uatomic_inc(&test->seen_toggle);
+ }
+
+ if (entry == expected)
+ toggled = false;
+ else if (entry == alt)
+ toggled = true;
+ else {
+ printk("%lu-%lu %p != %p or %p\n",
+ mas.index, mas.last, entry,
+ expected, alt);
+ RCU_MT_BUG_ON(test, 1);
+ }
+
+ prev = entry;
+ } else {
+ if (entry != expected)
+ printk("%lu-%lu %p != %p\n", mas.index,
+ mas.last, entry, expected);
+ RCU_MT_BUG_ON(test, entry != expected);
+ }
+ }
+ rcu_read_unlock();
+ usleep(test->pause);
+ }
+
+ rcu_unregister_thread();
+ return NULL;
+}
+
+static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test,
+ int count, struct rcu_reader_struct *test_reader)
+{
+ int i, j = 10000;
+ bool toggle = true;
+
+ test->start = true; /* Release the hounds! */
+ usleep(5);
+
+ while (j--) {
+ toggle = !toggle;
+ i = count;
+ while (i--) {
+ unsigned long start, end;
+ struct rcu_reader_struct *this = &test_reader[i];
+
+ /* Mod offset */
+ if (j == 600) {
+ start = test->index[this->id + this->mod];
+ end = test->last[this->id + this->mod];
+ mtree_store_range(mt, start, end,
+ xa_mk_value(this->id + this->mod * 2 +
+ 1 + RCU_RANGE_COUNT),
+ GFP_KERNEL);
+ }
+
+ /* Toggle */
+ if (!(j % 5)) {
+ start = test->index[this->id + this->flip];
+ end = test->last[this->id + this->flip];
+ mtree_store_range(mt, start, end,
+ xa_mk_value((toggle ? start :
+ this->id + this->flip +
+ RCU_RANGE_COUNT)),
+ GFP_KERNEL);
+ }
+
+ /* delete */
+ if (j == 400) {
+ start = test->index[this->id + this->del];
+ end = test->last[this->id + this->del];
+ mtree_store_range(mt, start, end, NULL, GFP_KERNEL);
+ }
+
+ /* add */
+ if (j == 500) {
+ start = test->index[this->id + this->add];
+ end = test->last[this->id + this->add];
+ mtree_store_range(mt, start, end,
+ xa_mk_value(start), GFP_KERNEL);
+ }
+ }
+ usleep(test->pause);
+ /* If a test fails, don't flood the console */
+ if (test->stop)
+ break;
+ }
+}
+
+static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test,
+ int count, struct rcu_reader_struct *test_reader)
+{
+ int j, i;
+ bool toggle = true;
+
+ test->start = true; /* Release the hounds! */
+ usleep(5);
+ for (j = 0; j < 10000; j++) {
+ toggle = !toggle;
+ for (i = 0; i < count; i++) {
+ unsigned long start, end;
+ struct rcu_reader_struct *this = &test_reader[i];
+
+ /* Mod offset */
+ if (j == 600) {
+ start = test->index[this->id + this->mod];
+ end = test->last[this->id + this->mod];
+ mtree_store_range(mt, start, end,
+ xa_mk_value(this->id + this->mod * 2 +
+ 1 + RCU_RANGE_COUNT),
+ GFP_KERNEL);
+ }
+
+ /* Toggle */
+ if (!(j % 5)) {
+ start = test->index[this->id + this->flip];
+ end = test->last[this->id + this->flip];
+ mtree_store_range(mt, start, end,
+ xa_mk_value((toggle ? start :
+ this->id + this->flip +
+ RCU_RANGE_COUNT)),
+ GFP_KERNEL);
+ }
+
+ /* delete */
+ if (j == 400) {
+ start = test->index[this->id + this->del];
+ end = test->last[this->id + this->del];
+ mtree_store_range(mt, start, end, NULL, GFP_KERNEL);
+ }
+
+ /* add */
+ if (j == 500) {
+ start = test->index[this->id + this->add];
+ end = test->last[this->id + this->add];
+ mtree_store_range(mt, start, end,
+ xa_mk_value(start), GFP_KERNEL);
+ }
+ }
+ usleep(test->pause);
+ /* If a test fails, don't flood the console */
+ if (test->stop)
+ break;
+ }
+}
+
+/*
+ * This is to check:
+ * 1. Range that is not ever present
+ * 2. Range that is always present
+ * 3. Things being added but not removed.
+ * 4. Things being removed but not added.
+ * 5. Things are being added and removed, searches my succeed or fail
+ *
+ * This sets up two readers for every 10 entries; one forward and one reverse
+ * reading.
+ */
+static void rcu_stress(struct maple_tree *mt, bool forward)
+{
+ unsigned int count, i;
+ unsigned long r, seed;
+ pthread_t readers[RCU_RANGE_COUNT / 5];
+ struct rcu_test_struct2 test;
+ struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5];
+ void *(*function)(void *);
+
+ /* Test setup */
+ test.mt = mt;
+ test.pause = 5;
+ test.seen_toggle = 0;
+ test.seen_deleted = 0;
+ test.seen_added = 0;
+ test.seen_modified = 0;
+ test.thread_count = 0;
+ test.start = test.stop = false;
+ seed = time(NULL);
+ srand(seed);
+ for (i = 0; i < RCU_RANGE_COUNT; i++) {
+ r = seed + rand();
+ mtree_store_range(mt, seed, r,
+ xa_mk_value(seed), GFP_KERNEL);
+
+ /* Record start and end of entry */
+ test.index[i] = seed;
+ test.last[i] = r;
+ seed = 1 + r + rand() % 10;
+ }
+
+ i = count = ARRAY_SIZE(readers);
+ while (i--) {
+ unsigned long id;
+
+ id = i / 2 * 10;
+ if (i % 2)
+ function = rcu_reader_fwd;
+ else
+ function = rcu_reader_rev;
+
+ rcu_reader_setup(&test_reader[i], id, &test);
+ if (pthread_create(&readers[i], NULL, *function,
+ &test_reader[i])) {
+ perror("creating reader thread");
+ exit(1);
+ }
+ }
+
+ for (i = 0; i < ARRAY_SIZE(readers); i++) {
+ struct rcu_reader_struct *this = &test_reader[i];
+ int add = this->id + this->add;
+
+ /* Remove add entries from the tree for later addition */
+ mtree_store_range(mt, test.index[add], test.last[add],
+ NULL, GFP_KERNEL);
+ }
+
+ mt_set_in_rcu(mt);
+ do {
+ usleep(5);
+ } while (test.thread_count > ARRAY_SIZE(readers));
+
+ if (forward)
+ rcu_stress_fwd(mt, &test, count, test_reader);
+ else
+ rcu_stress_rev(mt, &test, count, test_reader);
+
+ test.stop = true;
+ while (count--)
+ pthread_join(readers[count], NULL);
+
+ mt_validate(mt);
+}
+
+
+struct rcu_test_struct {
+ struct maple_tree *mt; /* the maple tree */
+ int count; /* Number of times to check value(s) */
+ unsigned long index; /* The first index to check */
+ void *entry1; /* The first entry value */
+ void *entry2; /* The second entry value */
+ void *entry3; /* The third entry value */
+
+ bool update_2;
+ bool update_3;
+ unsigned long range_start;
+ unsigned long range_end;
+ unsigned int loop_sleep;
+ unsigned int val_sleep;
+
+ unsigned int failed; /* failed detection for other threads */
+ unsigned int seen_entry2; /* Number of threads that have seen the new value */
+ unsigned int seen_entry3; /* Number of threads that have seen the new value */
+ unsigned int seen_both; /* Number of threads that have seen both new values */
+ unsigned int seen_toggle;
+ unsigned int seen_added;
+ unsigned int seen_removed;
+ unsigned long last; /* The end of the range to write. */
+
+ unsigned long removed; /* The index of the removed entry */
+ unsigned long added; /* The index of the removed entry */
+ unsigned long toggle; /* The index of the removed entry */
+};
+
+static inline
+int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2,
+ bool *update_3)
+{
+ if (entry == test->entry1)
+ return 0;
+
+ if (entry == test->entry2) {
+ if (!(*update_2)) {
+ uatomic_inc(&test->seen_entry2);
+ *update_2 = true;
+ if (update_3)
+ uatomic_inc(&test->seen_both);
+ }
+ return 0;
+ }
+
+ if (entry == test->entry3) {
+ if (!(*update_3)) {
+ uatomic_inc(&test->seen_entry3);
+ *update_3 = true;
+ if (update_2)
+ uatomic_inc(&test->seen_both);
+ }
+ return 0;
+ }
+
+ return 1;
+}
+
+/*
+ * rcu_val() - Read a given value in the tree test->count times using the
+ * regular API
+ *
+ * @ptr: The pointer to the rcu_test_struct
+ */
+static void *rcu_val(void *ptr)
+{
+ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr;
+ unsigned long count = test->count;
+ bool update_2 = false;
+ bool update_3 = false;
+ void *entry;
+
+ rcu_register_thread();
+ while (count--) {
+ usleep(test->val_sleep);
+ /*
+ * No locking required, regular API locking is handled in the
+ * maple tree code
+ */
+ entry = mtree_load(test->mt, test->index);
+ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2,
+ &update_3));
+ }
+ rcu_unregister_thread();
+ return NULL;
+}
+
+/*
+ * rcu_loop() - Loop over a section of the maple tree, checking for an expected
+ * value using the advanced API
+ *
+ * @ptr - The pointer to the rcu_test_struct
+ */
+static void *rcu_loop(void *ptr)
+{
+ struct rcu_test_struct *test = (struct rcu_test_struct *)ptr;
+ unsigned long count = test->count;
+ void *entry, *expected;
+ bool update_2 = false;
+ bool update_3 = false;
+ MA_STATE(mas, test->mt, test->range_start, test->range_start);
+
+ rcu_register_thread();
+
+ /*
+ * Loop through the test->range_start - test->range_end test->count
+ * times
+ */
+ while (count--) {
+ usleep(test->loop_sleep);
+ rcu_read_lock();
+ mas_for_each(&mas, entry, test->range_end) {
+ /* The expected value is based on the start range. */
+ expected = xa_mk_value(mas.index ? mas.index / 10 : 0);
+
+ /* Out of the interesting range */
+ if (mas.index < test->index || mas.index > test->last) {
+ if (entry != expected) {
+ printk("%lx - %lx = %p not %p\n",
+ mas.index, mas.last, entry, expected);
+ }
+ MT_BUG_ON(test->mt, entry != expected);
+ continue;
+ }
+
+ if (entry == expected)
+ continue; /* Not seen. */
+
+ /* In the interesting range */
+ MT_BUG_ON(test->mt, eval_rcu_entry(test, entry,
+ &update_2,
+ &update_3));
+ }
+ rcu_read_unlock();
+ mas_set(&mas, test->range_start);
+ }
+
+ rcu_unregister_thread();
+ return NULL;
+}
+
+static noinline
+void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals)
+{
+
+ int i;
+ void *(*function)(void *);
+ pthread_t readers[20];
+
+ mt_set_in_rcu(mt);
+ MT_BUG_ON(mt, !mt_in_rcu(mt));
+
+ for (i = 0; i < ARRAY_SIZE(readers); i++) {
+ if (i % 2)
+ function = rcu_loop;
+ else
+ function = rcu_val;
+
+ if (pthread_create(&readers[i], NULL, *function, vals)) {
+ perror("creating reader thread");
+ exit(1);
+ }
+ }
+
+ usleep(5); /* small yield to ensure all threads are at least started. */
+ mtree_store_range(mt, vals->index, vals->last, vals->entry2,
+ GFP_KERNEL);
+ while (i--)
+ pthread_join(readers[i], NULL);
+
+ /* Make sure the test caught at least one update. */
+ MT_BUG_ON(mt, !vals->seen_entry2);
+}
+
+static noinline
+void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals)
+{
+
+ int i;
+ void *(*function)(void *);
+ pthread_t readers[20];
+ unsigned int index = vals->index;
+
+ mt_set_in_rcu(mt);
+ MT_BUG_ON(mt, !mt_in_rcu(mt));
+
+ for (i = 0; i < ARRAY_SIZE(readers); i++) {
+ if (i % 2)
+ function = rcu_loop;
+ else
+ function = rcu_val;
+
+ if (pthread_create(&readers[i], NULL, *function, vals)) {
+ perror("creating reader thread");
+ exit(1);
+ }
+ }
+
+ usleep(5); /* small yield to ensure all threads are at least started. */
+
+ while (index <= vals->last) {
+ mtree_store(mt, index,
+ (index % 2 ? vals->entry2 : vals->entry3),
+ GFP_KERNEL);
+ index++;
+ usleep(5);
+ }
+
+ while (i--)
+ pthread_join(readers[i], NULL);
+
+ /* Make sure the test caught at least one update. */
+ MT_BUG_ON(mt, !vals->seen_entry2);
+ MT_BUG_ON(mt, !vals->seen_entry3);
+ MT_BUG_ON(mt, !vals->seen_both);
+}
+static noinline void check_rcu_simulated(struct maple_tree *mt)
+{
+ unsigned long i, nr_entries = 1000;
+ unsigned long target = 4320;
+ unsigned long val = 0xDEAD;
+
+ MA_STATE(mas_writer, mt, 0, 0);
+ MA_STATE(mas_reader, mt, target, target);
+
+ rcu_register_thread();
+
+ mt_set_in_rcu(mt);
+ mas_lock(&mas_writer);
+ for (i = 0; i <= nr_entries; i++) {
+ mas_writer.index = i * 10;
+ mas_writer.last = i * 10 + 5;
+ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL);
+ }
+ mas_unlock(&mas_writer);
+
+ /* Overwrite one entry with a new value. */
+ mas_set_range(&mas_writer, target, target + 5);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val));
+ rcu_read_unlock();
+
+ /* Restore value. */
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ mas_reset(&mas_reader);
+
+
+ /* Overwrite 1/2 the entry */
+ mas_set_range(&mas_writer, target, target + 2);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val));
+ rcu_read_unlock();
+
+
+ /* Restore value. */
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ mas_reset(&mas_reader);
+
+ /* Overwrite last 1/2 the entry */
+ mas_set_range(&mas_writer, target + 2, target + 5);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ rcu_read_unlock();
+
+
+ /* Restore value. */
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ mas_reset(&mas_reader);
+
+ /* Overwrite more than the entry */
+ mas_set_range(&mas_writer, target - 5, target + 15);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val));
+ rcu_read_unlock();
+
+ /* Restore value. */
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ mas_reset(&mas_reader);
+
+ /* Overwrite more than the node. */
+ mas_set_range(&mas_writer, target - 400, target + 400);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val));
+ rcu_read_unlock();
+
+ /* Restore value. */
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ mas_reset(&mas_reader);
+
+ /* Overwrite the tree */
+ mas_set_range(&mas_writer, 0, ULONG_MAX);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val));
+ rcu_read_unlock();
+
+ /* Clear out tree & recreate it */
+ mas_lock(&mas_writer);
+ mas_set_range(&mas_writer, 0, ULONG_MAX);
+ mas_store_gfp(&mas_writer, NULL, GFP_KERNEL);
+ mas_set_range(&mas_writer, 0, 0);
+ for (i = 0; i <= nr_entries; i++) {
+ mas_writer.index = i * 10;
+ mas_writer.last = i * 10 + 5;
+ mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL);
+ }
+ mas_unlock(&mas_writer);
+
+ /* next check */
+ /* Overwrite one entry with a new value. */
+ mas_reset(&mas_reader);
+ mas_set_range(&mas_writer, target, target + 5);
+ mas_set_range(&mas_reader, target, target);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_prev(&mas_reader, 0);
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val));
+ rcu_read_unlock();
+
+ /* Restore value. */
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+
+ /* prev check */
+ /* Overwrite one entry with a new value. */
+ mas_reset(&mas_reader);
+ mas_set_range(&mas_writer, target, target + 5);
+ mas_set_range(&mas_reader, target, target);
+ rcu_read_lock();
+ MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10));
+ mas_next(&mas_reader, ULONG_MAX);
+ mas_lock(&mas_writer);
+ mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL);
+ mas_unlock(&mas_writer);
+ MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val));
+ rcu_read_unlock();
+
+ rcu_unregister_thread();
+}
+
+static noinline void check_rcu_threaded(struct maple_tree *mt)
+{
+ unsigned long i, nr_entries = 1000;
+ struct rcu_test_struct vals;
+
+ vals.val_sleep = 200;
+ vals.loop_sleep = 110;
+
+ rcu_register_thread();
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+ /* Store across several slots. */
+ vals.count = 1000;
+ vals.mt = mt;
+ vals.index = 8650;
+ vals.last = 8666;
+ vals.entry1 = xa_mk_value(865);
+ vals.entry2 = xa_mk_value(8650);
+ vals.entry3 = xa_mk_value(8650);
+ vals.range_start = 0;
+ vals.range_end = ULONG_MAX;
+ vals.seen_entry2 = 0;
+ vals.seen_entry3 = 0;
+
+ run_check_rcu(mt, &vals);
+ mtree_destroy(mt);
+
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ /* 4390-4395: value 439 (0x1b7) [0x36f] */
+ /* Store across several slots. */
+ /* Spanning store. */
+ vals.count = 10000;
+ vals.mt = mt;
+ vals.index = 4390;
+ vals.last = 4398;
+ vals.entry1 = xa_mk_value(4390);
+ vals.entry2 = xa_mk_value(439);
+ vals.entry3 = xa_mk_value(439);
+ vals.seen_entry2 = 0;
+ vals.range_start = 4316;
+ vals.range_end = 5035;
+ run_check_rcu(mt, &vals);
+ mtree_destroy(mt);
+
+
+ /* Forward writer for rcu stress */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ rcu_stress(mt, true);
+ mtree_destroy(mt);
+
+ /* Reverse writer for rcu stress */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ rcu_stress(mt, false);
+ mtree_destroy(mt);
+
+ /* Slow reader test with spanning store. */
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+ /* 4390-4395: value 439 (0x1b7) [0x36f] */
+ /* Store across several slots. */
+ /* Spanning store. */
+ vals.count = 15000;
+ vals.mt = mt;
+ vals.index = 4390;
+ vals.last = 4398;
+ vals.entry1 = xa_mk_value(4390);
+ vals.entry2 = xa_mk_value(439);
+ vals.entry3 = xa_mk_value(4391);
+ vals.seen_toggle = 0;
+ vals.seen_added = 0;
+ vals.seen_removed = 0;
+ vals.range_start = 4316;
+ vals.range_end = 5035;
+ vals.removed = 4360;
+ vals.added = 4396;
+ vals.toggle = 4347;
+ vals.val_sleep = 400;
+ vals.loop_sleep = 200;
+ vals.seen_entry2 = 0;
+ vals.seen_entry3 = 0;
+ vals.seen_both = 0;
+ vals.entry3 = xa_mk_value(438);
+
+ run_check_rcu_slowread(mt, &vals);
+ rcu_unregister_thread();
+}
+
+extern void test_kmem_cache_bulk(void);
+
+/* Test spanning writes that require balancing right sibling or right cousin */
+static noinline void check_spanning_relatives(struct maple_tree *mt)
+{
+
+ unsigned long i, nr_entries = 1000;
+
+ for (i = 0; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, i*10 + 5,
+ xa_mk_value(i), GFP_KERNEL);
+
+
+ mtree_store_range(mt, 9365, 9955, NULL, GFP_KERNEL);
+}
+
+static noinline void check_fuzzer(struct maple_tree *mt)
+{
+ /*
+ * 1. Causes a spanning rebalance of a single root node.
+ * Fixed by setting the correct limit in mast_cp_to_nodes() when the
+ * entire right side is consumed.
+ */
+ mtree_test_insert(mt, 88, (void *)0xb1);
+ mtree_test_insert(mt, 84, (void *)0xa9);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 4, (void *)0x9);
+ mtree_test_insert(mt, 14, (void *)0x1d);
+ mtree_test_insert(mt, 7, (void *)0xf);
+ mtree_test_insert(mt, 12, (void *)0x19);
+ mtree_test_insert(mt, 18, (void *)0x25);
+ mtree_test_store_range(mt, 8, 18, (void *)0x11);
+ mtree_destroy(mt);
+
+
+ /*
+ * 2. Cause a spanning rebalance of two nodes in root.
+ * Fixed by setting mast->r->max correctly.
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_store(mt, 87, (void *)0xaf);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_load(mt, 4);
+ mtree_test_insert(mt, 4, (void *)0x9);
+ mtree_test_store(mt, 8, (void *)0x11);
+ mtree_test_store(mt, 44, (void *)0x59);
+ mtree_test_store(mt, 68, (void *)0x89);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 43, (void *)0x57);
+ mtree_test_insert(mt, 24, (void *)0x31);
+ mtree_test_insert(mt, 844, (void *)0x699);
+ mtree_test_store(mt, 84, (void *)0xa9);
+ mtree_test_store(mt, 4, (void *)0x9);
+ mtree_test_erase(mt, 4);
+ mtree_test_load(mt, 5);
+ mtree_test_erase(mt, 0);
+ mtree_destroy(mt);
+
+ /*
+ * 3. Cause a node overflow on copy
+ * Fixed by using the correct check for node size in mas_wr_modify()
+ * Also discovered issue with metadata setting.
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_store_range(mt, 0, 18446744073709551615UL, (void *)0x1);
+ mtree_test_store(mt, 4, (void *)0x9);
+ mtree_test_erase(mt, 5);
+ mtree_test_erase(mt, 0);
+ mtree_test_erase(mt, 4);
+ mtree_test_store(mt, 5, (void *)0xb);
+ mtree_test_erase(mt, 5);
+ mtree_test_store(mt, 5, (void *)0xb);
+ mtree_test_erase(mt, 5);
+ mtree_test_erase(mt, 4);
+ mtree_test_store(mt, 4, (void *)0x9);
+ mtree_test_store(mt, 444, (void *)0x379);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_load(mt, 0);
+ mtree_test_store(mt, 5, (void *)0xb);
+ mtree_test_erase(mt, 0);
+ mtree_destroy(mt);
+
+ /*
+ * 4. spanning store failure due to writing incorrect pivot value at
+ * last slot.
+ * Fixed by setting mast->r->max correctly in mast_cp_to_nodes()
+ *
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_insert(mt, 261, (void *)0x20b);
+ mtree_test_store(mt, 516, (void *)0x409);
+ mtree_test_store(mt, 6, (void *)0xd);
+ mtree_test_insert(mt, 5, (void *)0xb);
+ mtree_test_insert(mt, 1256, (void *)0x9d1);
+ mtree_test_store(mt, 4, (void *)0x9);
+ mtree_test_erase(mt, 1);
+ mtree_test_store(mt, 56, (void *)0x71);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_store(mt, 24, (void *)0x31);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 2263, (void *)0x11af);
+ mtree_test_insert(mt, 446, (void *)0x37d);
+ mtree_test_store_range(mt, 6, 45, (void *)0xd);
+ mtree_test_store_range(mt, 3, 446, (void *)0x7);
+ mtree_destroy(mt);
+
+ /*
+ * 5. mas_wr_extend_null() may overflow slots.
+ * Fix by checking against wr_mas->node_end.
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_store(mt, 48, (void *)0x61);
+ mtree_test_store(mt, 3, (void *)0x7);
+ mtree_test_load(mt, 0);
+ mtree_test_store(mt, 88, (void *)0xb1);
+ mtree_test_store(mt, 81, (void *)0xa3);
+ mtree_test_insert(mt, 0, (void *)0x1);
+ mtree_test_insert(mt, 8, (void *)0x11);
+ mtree_test_insert(mt, 4, (void *)0x9);
+ mtree_test_insert(mt, 2480, (void *)0x1361);
+ mtree_test_insert(mt, 18446744073709551615UL,
+ (void *)0xffffffffffffffff);
+ mtree_test_erase(mt, 18446744073709551615UL);
+ mtree_destroy(mt);
+
+ /*
+ * 6. When reusing a node with an implied pivot and the node is
+ * shrinking, old data would be left in the implied slot
+ * Fixed by checking the last pivot for the mas->max and clear
+ * accordingly. This only affected the left-most node as that node is
+ * the only one allowed to end in NULL.
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_erase(mt, 3);
+ mtree_test_insert(mt, 22, (void *)0x2d);
+ mtree_test_insert(mt, 15, (void *)0x1f);
+ mtree_test_load(mt, 2);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 5, (void *)0xb);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 4, (void *)0x9);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 3);
+ mtree_test_insert(mt, 22, (void *)0x2d);
+ mtree_test_insert(mt, 15, (void *)0x1f);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 8, (void *)0x11);
+ mtree_test_load(mt, 2);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 5, (void *)0xb);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 4, (void *)0x9);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 3);
+ mtree_test_insert(mt, 22, (void *)0x2d);
+ mtree_test_insert(mt, 15, (void *)0x1f);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 8, (void *)0x11);
+ mtree_test_insert(mt, 12, (void *)0x19);
+ mtree_test_erase(mt, 1);
+ mtree_test_store_range(mt, 4, 62, (void *)0x9);
+ mtree_test_erase(mt, 62);
+ mtree_test_store_range(mt, 1, 0, (void *)0x3);
+ mtree_test_insert(mt, 11, (void *)0x17);
+ mtree_test_insert(mt, 3, (void *)0x7);
+ mtree_test_insert(mt, 3, (void *)0x7);
+ mtree_test_store(mt, 62, (void *)0x7d);
+ mtree_test_erase(mt, 62);
+ mtree_test_store_range(mt, 1, 15, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 22, (void *)0x2d);
+ mtree_test_insert(mt, 12, (void *)0x19);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 3, (void *)0x7);
+ mtree_test_store(mt, 62, (void *)0x7d);
+ mtree_test_erase(mt, 62);
+ mtree_test_insert(mt, 122, (void *)0xf5);
+ mtree_test_store(mt, 3, (void *)0x7);
+ mtree_test_insert(mt, 0, (void *)0x1);
+ mtree_test_store_range(mt, 0, 1, (void *)0x1);
+ mtree_test_insert(mt, 85, (void *)0xab);
+ mtree_test_insert(mt, 72, (void *)0x91);
+ mtree_test_insert(mt, 81, (void *)0xa3);
+ mtree_test_insert(mt, 726, (void *)0x5ad);
+ mtree_test_insert(mt, 0, (void *)0x1);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_store(mt, 51, (void *)0x67);
+ mtree_test_insert(mt, 611, (void *)0x4c7);
+ mtree_test_insert(mt, 485, (void *)0x3cb);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 0, (void *)0x1);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert_range(mt, 26, 1, (void *)0x35);
+ mtree_test_load(mt, 1);
+ mtree_test_store_range(mt, 1, 22, (void *)0x3);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_load(mt, 53);
+ mtree_test_load(mt, 1);
+ mtree_test_store_range(mt, 1, 1, (void *)0x3);
+ mtree_test_insert(mt, 222, (void *)0x1bd);
+ mtree_test_insert(mt, 485, (void *)0x3cb);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_load(mt, 0);
+ mtree_test_insert(mt, 21, (void *)0x2b);
+ mtree_test_insert(mt, 3, (void *)0x7);
+ mtree_test_store(mt, 621, (void *)0x4db);
+ mtree_test_insert(mt, 0, (void *)0x1);
+ mtree_test_erase(mt, 5);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_store(mt, 62, (void *)0x7d);
+ mtree_test_erase(mt, 62);
+ mtree_test_store_range(mt, 1, 0, (void *)0x3);
+ mtree_test_insert(mt, 22, (void *)0x2d);
+ mtree_test_insert(mt, 12, (void *)0x19);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_store_range(mt, 4, 62, (void *)0x9);
+ mtree_test_erase(mt, 62);
+ mtree_test_erase(mt, 1);
+ mtree_test_load(mt, 1);
+ mtree_test_store_range(mt, 1, 22, (void *)0x3);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_load(mt, 53);
+ mtree_test_load(mt, 1);
+ mtree_test_store_range(mt, 1, 1, (void *)0x3);
+ mtree_test_insert(mt, 222, (void *)0x1bd);
+ mtree_test_insert(mt, 485, (void *)0x3cb);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_load(mt, 0);
+ mtree_test_load(mt, 0);
+ mtree_destroy(mt);
+
+ /*
+ * 7. Previous fix was incomplete, fix mas_resuse_node() clearing of old
+ * data by overwriting it first - that way metadata is of no concern.
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_load(mt, 1);
+ mtree_test_insert(mt, 102, (void *)0xcd);
+ mtree_test_erase(mt, 2);
+ mtree_test_erase(mt, 0);
+ mtree_test_load(mt, 0);
+ mtree_test_insert(mt, 4, (void *)0x9);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 110, (void *)0xdd);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_insert_range(mt, 5, 0, (void *)0xb);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_store(mt, 112, (void *)0xe1);
+ mtree_test_insert(mt, 21, (void *)0x2b);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_insert_range(mt, 110, 2, (void *)0xdd);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_load(mt, 22);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 210, (void *)0x1a5);
+ mtree_test_store_range(mt, 0, 2, (void *)0x1);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_erase(mt, 2);
+ mtree_test_erase(mt, 22);
+ mtree_test_erase(mt, 1);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_load(mt, 112);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_insert_range(mt, 1, 2, (void *)0x3);
+ mtree_test_erase(mt, 0);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_erase(mt, 0);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_erase(mt, 2);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert_range(mt, 1, 2, (void *)0x3);
+ mtree_test_erase(mt, 0);
+ mtree_test_erase(mt, 2);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_load(mt, 112);
+ mtree_test_store_range(mt, 110, 12, (void *)0xdd);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_load(mt, 110);
+ mtree_test_insert_range(mt, 4, 71, (void *)0x9);
+ mtree_test_load(mt, 2);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_insert_range(mt, 11, 22, (void *)0x17);
+ mtree_test_erase(mt, 12);
+ mtree_test_store(mt, 2, (void *)0x5);
+ mtree_test_load(mt, 22);
+ mtree_destroy(mt);
+
+
+ /*
+ * 8. When rebalancing or spanning_rebalance(), the max of the new node
+ * may be set incorrectly to the final pivot and not the right max.
+ * Fix by setting the left max to orig right max if the entire node is
+ * consumed.
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_store(mt, 6, (void *)0xd);
+ mtree_test_store(mt, 67, (void *)0x87);
+ mtree_test_insert(mt, 15, (void *)0x1f);
+ mtree_test_insert(mt, 6716, (void *)0x3479);
+ mtree_test_store(mt, 61, (void *)0x7b);
+ mtree_test_insert(mt, 13, (void *)0x1b);
+ mtree_test_store(mt, 8, (void *)0x11);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_load(mt, 0);
+ mtree_test_erase(mt, 67167);
+ mtree_test_insert_range(mt, 6, 7167, (void *)0xd);
+ mtree_test_insert(mt, 6, (void *)0xd);
+ mtree_test_erase(mt, 67);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 667167);
+ mtree_test_insert(mt, 6, (void *)0xd);
+ mtree_test_store(mt, 67, (void *)0x87);
+ mtree_test_insert(mt, 5, (void *)0xb);
+ mtree_test_erase(mt, 1);
+ mtree_test_insert(mt, 6, (void *)0xd);
+ mtree_test_erase(mt, 67);
+ mtree_test_insert(mt, 15, (void *)0x1f);
+ mtree_test_insert(mt, 67167, (void *)0x20cbf);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_load(mt, 7);
+ mtree_test_insert(mt, 16, (void *)0x21);
+ mtree_test_insert(mt, 36, (void *)0x49);
+ mtree_test_store(mt, 67, (void *)0x87);
+ mtree_test_store(mt, 6, (void *)0xd);
+ mtree_test_insert(mt, 367, (void *)0x2df);
+ mtree_test_insert(mt, 115, (void *)0xe7);
+ mtree_test_store(mt, 0, (void *)0x1);
+ mtree_test_store_range(mt, 1, 3, (void *)0x3);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 67167);
+ mtree_test_insert_range(mt, 6, 47, (void *)0xd);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_insert_range(mt, 1, 67, (void *)0x3);
+ mtree_test_load(mt, 67);
+ mtree_test_insert(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 67167);
+ mtree_destroy(mt);
+
+ /*
+ * 9. spanning store to the end of data caused an invalid metadata
+ * length which resulted in a crash eventually.
+ * Fix by checking if there is a value in pivot before incrementing the
+ * metadata end in mab_mas_cp(). To ensure this doesn't happen again,
+ * abstract the two locations this happens into a function called
+ * mas_leaf_set_meta().
+ */
+ mt_init_flags(mt, 0);
+ mtree_test_insert(mt, 21, (void *)0x2b);
+ mtree_test_insert(mt, 12, (void *)0x19);
+ mtree_test_insert(mt, 6, (void *)0xd);
+ mtree_test_insert(mt, 8, (void *)0x11);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 91, (void *)0xb7);
+ mtree_test_insert(mt, 18, (void *)0x25);
+ mtree_test_insert(mt, 81, (void *)0xa3);
+ mtree_test_store_range(mt, 0, 128, (void *)0x1);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_erase(mt, 8);
+ mtree_test_insert(mt, 11, (void *)0x17);
+ mtree_test_insert(mt, 8, (void *)0x11);
+ mtree_test_insert(mt, 21, (void *)0x2b);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb);
+ mtree_test_erase(mt, 18446744073709551605UL);
+ mtree_test_store_range(mt, 0, 281, (void *)0x1);
+ mtree_test_erase(mt, 2);
+ mtree_test_insert(mt, 1211, (void *)0x977);
+ mtree_test_insert(mt, 111, (void *)0xdf);
+ mtree_test_insert(mt, 13, (void *)0x1b);
+ mtree_test_insert(mt, 211, (void *)0x1a7);
+ mtree_test_insert(mt, 11, (void *)0x17);
+ mtree_test_insert(mt, 5, (void *)0xb);
+ mtree_test_insert(mt, 1218, (void *)0x985);
+ mtree_test_insert(mt, 61, (void *)0x7b);
+ mtree_test_store(mt, 1, (void *)0x3);
+ mtree_test_insert(mt, 121, (void *)0xf3);
+ mtree_test_insert(mt, 8, (void *)0x11);
+ mtree_test_insert(mt, 21, (void *)0x2b);
+ mtree_test_insert(mt, 2, (void *)0x5);
+ mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb);
+ mtree_test_erase(mt, 18446744073709551605UL);
+}
+static noinline void check_dup_gaps(struct maple_tree *mt,
+ unsigned long nr_entries, bool zero_start,
+ unsigned long gap)
+{
+ unsigned long i = 0;
+ struct maple_tree newmt;
+ int ret;
+ void *tmp;
+ MA_STATE(mas, mt, 0, 0);
+ MA_STATE(newmas, &newmt, 0, 0);
+
+
+ if (!zero_start)
+ i = 1;
+
+ mt_zero_nr_tallocated();
+ for (; i <= nr_entries; i++)
+ mtree_store_range(mt, i*10, (i+1)*10 - gap,
+ xa_mk_value(i), GFP_KERNEL);
+
+ mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE);
+ mt_set_non_kernel(99999);
+ ret = mas_expected_entries(&newmas, nr_entries);
+ mt_set_non_kernel(0);
+ MT_BUG_ON(mt, ret != 0);
+
+ mas_for_each(&mas, tmp, ULONG_MAX) {
+ newmas.index = mas.index;
+ newmas.last = mas.last;
+ mas_store(&newmas, tmp);
+ }
+
+ mas_destroy(&mas);
+ mas_destroy(&newmas);
+ mtree_destroy(&newmt);
+}
+
+static noinline void check_dup(struct maple_tree *mt)
+{
+ int i;
+
+ /* Check with a value at zero */
+ for (i = 10; i < 1000; i++) {
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_dup_gaps(mt, i, true, 5);
+ mtree_destroy(mt);
+ }
+
+ /* Check with a value at zero, no gap */
+ for (i = 1000; i < 2000; i++) {
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_dup_gaps(mt, i, true, 0);
+ mtree_destroy(mt);
+ }
+
+ /* Check with a value at zero and unreasonably large */
+ for (i = 100010; i < 100020; i++) {
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_dup_gaps(mt, i, true, 5);
+ mtree_destroy(mt);
+ }
+
+ /* Small to medium size not starting at zero*/
+ for (i = 200; i < 1000; i++) {
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_dup_gaps(mt, i, false, 5);
+ mtree_destroy(mt);
+ }
+
+ /* Unreasonably large not starting at zero*/
+ for (i = 100010; i < 100020; i++) {
+ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+ check_dup_gaps(mt, i, false, 5);
+ mtree_destroy(mt);
+ }
+
+ /* Check non-allocation tree not starting at zero */
+ for (i = 1500; i < 3000; i++) {
+ mt_init_flags(mt, 0);
+ check_dup_gaps(mt, i, false, 5);
+ mtree_destroy(mt);
+ }
+
+ /* Check non-allocation tree starting at zero */
+ for (i = 200; i < 1000; i++) {
+ mt_init_flags(mt, 0);
+ check_dup_gaps(mt, i, true, 5);
+ mtree_destroy(mt);
+ }
+
+ /* Unreasonably large */
+ for (i = 100015; i < 100020; i++) {
+ mt_init_flags(mt, 0);
+ check_dup_gaps(mt, i, true, 5);
+ mtree_destroy(mt);
+ }
+
+}
+
+static DEFINE_MTREE(tree);
+static int maple_tree_seed(void)
+{
+ unsigned long set[] = {5015, 5014, 5017, 25, 1000,
+ 1001, 1002, 1003, 1005, 0,
+ 5003, 5002};
+ void *ptr = &set;
+
+ pr_info("\nTEST STARTING\n\n");
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_root_expand(&tree);
+ mtree_destroy(&tree);
+
+#if defined(BENCH_SLOT_STORE)
+#define BENCH
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ bench_slot_store(&tree);
+ mtree_destroy(&tree);
+ goto skip;
+#endif
+#if defined(BENCH_NODE_STORE)
+#define BENCH
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ bench_node_store(&tree);
+ mtree_destroy(&tree);
+ goto skip;
+#endif
+#if defined(BENCH_AWALK)
+#define BENCH
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ bench_awalk(&tree);
+ mtree_destroy(&tree);
+ goto skip;
+#endif
+#if defined(BENCH_WALK)
+#define BENCH
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ bench_walk(&tree);
+ mtree_destroy(&tree);
+ goto skip;
+#endif
+#if defined(BENCH_FORK)
+#define BENCH
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ bench_forking(&tree);
+ mtree_destroy(&tree);
+ goto skip;
+#endif
+#if defined(BENCH_MT_FOR_EACH)
+#define BENCH
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ bench_mt_for_each(&tree);
+ mtree_destroy(&tree);
+ goto skip;
+#endif
+
+ test_kmem_cache_bulk();
+
+ mt_init_flags(&tree, 0);
+ check_new_node(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_prealloc(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_spanning_write(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_null_expand(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ check_dfs_preorder(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_forking(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_mas_store_gfp(&tree);
+ mtree_destroy(&tree);
+
+ /* Test ranges (store and insert) */
+ mt_init_flags(&tree, 0);
+ check_ranges(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_alloc_range(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_alloc_rev_range(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+
+ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */
+
+ check_insert(&tree, set[9], &tree); /* Insert 0 */
+ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */
+ check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */
+
+ check_insert(&tree, set[10], ptr); /* Insert 5003 */
+ check_load(&tree, set[9], &tree); /* See if 0 -> &tree */
+ check_load(&tree, set[11], NULL); /* See if 5002 -> NULL */
+ check_load(&tree, set[10], ptr); /* See if 5003 -> ptr */
+
+ /* Clear out the tree */
+ mtree_destroy(&tree);
+
+ /* Try to insert, insert a dup, and load back what was inserted. */
+ mt_init_flags(&tree, 0);
+ check_insert(&tree, set[0], &tree); /* Insert 5015 */
+ check_dup_insert(&tree, set[0], &tree); /* Insert 5015 again */
+ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */
+
+ /*
+ * Second set of tests try to load a value that doesn't exist, inserts
+ * a second value, then loads the value again
+ */
+ check_load(&tree, set[1], NULL); /* See if 5014 -> NULL */
+ check_insert(&tree, set[1], ptr); /* insert 5014 -> ptr */
+ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */
+ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */
+ /*
+ * Tree currently contains:
+ * p[0]: 14 -> (nil) p[1]: 15 -> ptr p[2]: 16 -> &tree p[3]: 0 -> (nil)
+ */
+ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */
+ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */
+
+ check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */
+ check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */
+ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */
+ check_load(&tree, set[7], &tree); /* 1003 = &tree ? */
+
+ /* Clear out tree */
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ /* Test inserting into a NULL hole. */
+ check_insert(&tree, set[5], ptr); /* insert 1001 -> ptr */
+ check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */
+ check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */
+ check_load(&tree, set[5], ptr); /* See if 1001 -> ptr */
+ check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */
+ check_load(&tree, set[7], &tree); /* See if 1003 -> &tree */
+
+ /* Clear out the tree */
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ check_erase_testset(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ /*
+ * set[] = {5015, 5014, 5017, 25, 1000,
+ * 1001, 1002, 1003, 1005, 0,
+ * 5003, 5002};
+ */
+
+ check_insert(&tree, set[0], ptr); /* 5015 */
+ check_insert(&tree, set[1], &tree); /* 5014 */
+ check_insert(&tree, set[2], ptr); /* 5017 */
+ check_insert(&tree, set[3], &tree); /* 25 */
+ check_load(&tree, set[0], ptr);
+ check_load(&tree, set[1], &tree);
+ check_load(&tree, set[2], ptr);
+ check_load(&tree, set[3], &tree);
+ check_insert(&tree, set[4], ptr); /* 1000 < Should split. */
+ check_load(&tree, set[0], ptr);
+ check_load(&tree, set[1], &tree);
+ check_load(&tree, set[2], ptr);
+ check_load(&tree, set[3], &tree); /*25 */
+ check_load(&tree, set[4], ptr);
+ check_insert(&tree, set[5], &tree); /* 1001 */
+ check_load(&tree, set[0], ptr);
+ check_load(&tree, set[1], &tree);
+ check_load(&tree, set[2], ptr);
+ check_load(&tree, set[3], &tree);
+ check_load(&tree, set[4], ptr);
+ check_load(&tree, set[5], &tree);
+ check_insert(&tree, set[6], ptr);
+ check_load(&tree, set[0], ptr);
+ check_load(&tree, set[1], &tree);
+ check_load(&tree, set[2], ptr);
+ check_load(&tree, set[3], &tree);
+ check_load(&tree, set[4], ptr);
+ check_load(&tree, set[5], &tree);
+ check_load(&tree, set[6], ptr);
+ check_insert(&tree, set[7], &tree);
+ check_load(&tree, set[0], ptr);
+ check_insert(&tree, set[8], ptr);
+
+ check_insert(&tree, set[9], &tree);
+
+ check_load(&tree, set[0], ptr);
+ check_load(&tree, set[1], &tree);
+ check_load(&tree, set[2], ptr);
+ check_load(&tree, set[3], &tree);
+ check_load(&tree, set[4], ptr);
+ check_load(&tree, set[5], &tree);
+ check_load(&tree, set[6], ptr);
+ check_load(&tree, set[9], &tree);
+ mtree_destroy(&tree);
+
+ check_nomem(&tree);
+ mt_init_flags(&tree, 0);
+ check_seq(&tree, 16, false);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ check_seq(&tree, 1000, true);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_rev_seq(&tree, 1000, true);
+ mtree_destroy(&tree);
+
+ check_lower_bound_split(&tree);
+ check_upper_bound_split(&tree);
+ check_mid_split(&tree);
+
+ mt_init_flags(&tree, 0);
+ check_next_entry(&tree);
+ check_find(&tree);
+ check_find_2(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_prev_entry(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ check_erase2_sets(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_gap_combining(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_node_overwrite(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ next_prev_test(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_rcu_simulated(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_rcu_threaded(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_spanning_relatives(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_rev_find(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, 0);
+ check_fuzzer(&tree);
+ mtree_destroy(&tree);
+
+ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
+ check_dup(&tree);
+ mtree_destroy(&tree);
+
+#if defined(BENCH)
+skip:
+#endif
+ rcu_barrier();
+ pr_info("maple_tree: %u of %u tests passed\n",
+ atomic_read(&maple_tree_tests_passed),
+ atomic_read(&maple_tree_tests_run));
+ if (atomic_read(&maple_tree_tests_run) ==
+ atomic_read(&maple_tree_tests_passed))
+ return 0;
+
+ return -EINVAL;
+}
+
+static void maple_tree_harvest(void)
+{
+
+}
+
+module_init(maple_tree_seed);
+module_exit(maple_tree_harvest);
+MODULE_AUTHOR("Liam R. Howlett <Liam.Howlett@Oracle.com>");
+MODULE_LICENSE("GPL");
size_t size = PAGE_SIZE << order;
page = alloc_pages(GFP_KERNEL, order);
+ if (!page)
+ goto err;
buf = page_address(page);
fill_with_garbage(buf, size);
__free_pages(page, order);
page = alloc_pages(GFP_KERNEL, order);
+ if (!page)
+ goto err;
buf = page_address(page);
if (count_nonzero_bytes(buf, size))
(*total_failures)++;
fill_with_garbage(buf, size);
__free_pages(page, order);
return 1;
+err:
+ (*total_failures)++;
+ return 1;
}
/* Test the page allocator by calling alloc_pages with different orders. */
void *buf;
buf = kmalloc(size, GFP_KERNEL);
+ if (!buf)
+ goto err;
fill_with_garbage(buf, size);
kfree(buf);
buf = kmalloc(size, GFP_KERNEL);
+ if (!buf)
+ goto err;
if (count_nonzero_bytes(buf, size))
(*total_failures)++;
fill_with_garbage(buf, size);
kfree(buf);
return 1;
+err:
+ (*total_failures)++;
+ return 1;
}
/* Test vmalloc() with given parameters. */
void *buf;
buf = vmalloc(size);
+ if (!buf)
+ goto err;
fill_with_garbage(buf, size);
vfree(buf);
buf = vmalloc(size);
+ if (!buf)
+ goto err;
if (count_nonzero_bytes(buf, size))
(*total_failures)++;
fill_with_garbage(buf, size);
vfree(buf);
return 1;
+err:
+ (*total_failures)++;
+ return 1;
}
/* Test kmalloc()/vmalloc() by allocating objects of different sizes. */
/* Test with randomly generated values. */
heap.nr = ARRAY_SIZE(values);
for (i = 0; i < heap.nr; i++)
- values[i] = get_random_int();
+ values[i] = get_random_u32();
min_heapify_all(&heap, &funcs);
err += pop_verify_heap(min_heap, &heap, &funcs);
/* Test with randomly generated values. */
while (heap.nr < heap.size) {
- temp = get_random_int();
+ temp = get_random_u32();
min_heap_push(&heap, &temp, &funcs);
}
err += pop_verify_heap(min_heap, &heap, &funcs);
/* Test with randomly generated values. */
for (i = 0; i < ARRAY_SIZE(data); i++) {
- temp = get_random_int();
+ temp = get_random_u32();
min_heap_pop_push(&heap, &temp, &funcs);
}
err += pop_verify_heap(min_heap, &heap, &funcs);
int err;
if (should_create_root)
- prandom_bytes(world->next_root_buf,
+ get_random_bytes(world->next_root_buf,
sizeof(world->next_root_buf));
objagg_obj = world_obj_get(world, objagg, key_id);
if (WARN_ON(err))
goto out_free;
- k = prandom_u32();
+ k = get_random_u32();
ret = 0;
for (i = 0; i < entries; i++) {
rhl_test_objects[i].value.id = k;
pr_info("test %d random rhlist add/delete operations\n", entries);
for (j = 0; j < entries; j++) {
u32 i = prandom_u32_max(entries);
- u32 prand = prandom_u32();
+ u32 prand = get_random_u32();
cond_resched();
if (prand == 0)
- prand = prandom_u32();
+ prand = get_random_u32();
if (prand & 1) {
prand >>= 1;
int i;
for (i = 0; i < test_loop_count; i++) {
- rnd = prandom_u32();
+ rnd = get_random_u8();
/*
* Maximum 1024 pages, if PAGE_SIZE is 4096.
int i;
for (i = 0; i < test_loop_count; i++) {
- n = prandom_u32();
- n = (n % 100) + 1;
-
+ n = prandom_u32_max(100) + 1;
p = vmalloc(n * PAGE_SIZE);
if (!p)
return -1;
for (i = 0; i < 35000; i++) {
- unsigned int r;
-
- r = prandom_u32();
- size = (r % (PAGE_SIZE / 4)) + 1;
+ size = prandom_u32_max(PAGE_SIZE / 4) + 1;
/*
* Maximum PAGE_SIZE
*/
- r = prandom_u32();
- align = 1 << ((r % 11) + 1);
+ align = 1 << (prandom_u32_max(11) + 1);
pcpu[i] = __alloc_percpu(size, align);
if (!pcpu[i])
static void shuffle_array(int *arr, int n)
{
- unsigned int rnd;
int i, j;
for (i = n - 1; i > 0; i--) {
- rnd = prandom_u32();
-
/* Cut the range. */
- j = rnd % i;
+ j = prandom_u32_max(i);
/* Swap indexes. */
swap(arr[i], arr[j]);
unsigned long res = n;
might_fault();
if (!should_fail_usercopy() && likely(access_ok(from, n))) {
- instrument_copy_from_user(to, from, n);
+ instrument_copy_from_user_before(to, from, n);
res = raw_copy_from_user(to, from, n);
+ instrument_copy_from_user_after(to, from, n, res);
}
if (unlikely(res))
memset(to + (n - res), 0, res);
static void __uuid_gen_common(__u8 b[16])
{
- prandom_bytes(b, 16);
+ get_random_bytes(b, 16);
/* reversion 0b10 */
b[8] = (b[8] & 0x3F) | 0x80;
}
in your computer. If unsure say Y.
config ZSWAP
- bool "Compressed cache for swap pages (EXPERIMENTAL)"
+ bool "Compressed cache for swap pages"
depends on SWAP
select FRONTSWAP
select CRYPTO
in the case where decompressing from RAM is faster than swap device
reads, can also improve workload performance.
- This is marked experimental because it is a new feature (as of
- v3.11) that interacts heavily with memory reclaim. While these
- interactions don't cause any known issues on simple memory setups,
- they have not be fully explored on the large set of potential
- configurations and workloads that exist.
-
config ZSWAP_DEFAULT_ON
bool "Enable the compressed cache for swap pages by default"
depends on ZSWAP
purposes. It is required to enable userfaultfd write protection on
file-backed memory types like shmem and hugetlbfs.
+# multi-gen LRU {
+config LRU_GEN
+ bool "Multi-Gen LRU"
+ depends on MMU
+ # make sure folio->flags has enough spare bits
+ depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP
+ help
+ A high performance LRU implementation to overcommit memory. See
+ Documentation/admin-guide/mm/multigen_lru.rst for details.
+
+config LRU_GEN_ENABLED
+ bool "Enable by default"
+ depends on LRU_GEN
+ help
+ This option enables the multi-gen LRU by default.
+
+config LRU_GEN_STATS
+ bool "Full stats for debugging"
+ depends on LRU_GEN
+ help
+ Do not enable this option unless you plan to look at historical stats
+ from evicted generations for debugging purpose.
+
+ This option has a per-memcg and per-node memory overhead.
+# }
+
source "mm/damon/Kconfig"
endmenu
readahead.o swap.o truncate.o vmscan.o shmem.o \
util.o mmzone.o vmstat.o backing-dev.o \
mm_init.o percpu.o slab_common.o \
- compaction.o vmacache.o \
+ compaction.o \
interval_tree.o list_lru.o workingset.o \
debug.o gup.o mmap_lock.o $(mmu-y)
obj-$(CONFIG_SLUB) += slub.o
obj-$(CONFIG_KASAN) += kasan/
obj-$(CONFIG_KFENCE) += kfence/
+obj-$(CONFIG_KMSAN) += kmsan/
obj-$(CONFIG_FAILSLAB) += failslab.o
obj-$(CONFIG_MEMTEST) += memtest.o
obj-$(CONFIG_MIGRATION) += migrate.o
+obj-$(CONFIG_NUMA) += memory-tiers.o
obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o
obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
obj-$(CONFIG_PAGE_COUNTER) += page_counter.o
obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o
-obj-$(CONFIG_MEMCG_SWAP) += swap_cgroup.o
+ifdef CONFIG_SWAP
+obj-$(CONFIG_MEMCG) += swap_cgroup.o
+endif
obj-$(CONFIG_CGROUP_HUGETLB) += hugetlb_cgroup.o
obj-$(CONFIG_GUP_TEST) += gup_test.o
obj-$(CONFIG_MEMORY_FAILURE) += memory-failure.o
int bdi_init(struct backing_dev_info *bdi)
{
- int ret;
-
bdi->dev = NULL;
kref_init(&bdi->refcnt);
INIT_LIST_HEAD(&bdi->wb_list);
init_waitqueue_head(&bdi->wb_waitq);
- ret = cgwb_bdi_init(bdi);
-
- return ret;
+ return cgwb_bdi_init(bdi);
}
struct backing_dev_info *bdi_alloc(int node_id)
static void cma_debugfs_add_one(struct cma *cma, struct dentry *root_dentry)
{
struct dentry *tmp;
- char name[CMA_MAX_NAME];
- scnprintf(name, sizeof(name), "cma-%s", cma->name);
-
- tmp = debugfs_create_dir(name, root_dentry);
+ tmp = debugfs_create_dir(cma->name, root_dentry);
debugfs_create_file("alloc", 0200, tmp, cma, &cma_alloc_fops);
debugfs_create_file("free", 0200, tmp, cma, &cma_free_fops);
#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
-#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
-#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
/*
* Page order with-respect-to which proactive compaction
if (cc->ignore_skip_hint)
return false;
- if (!IS_ALIGNED(pfn, pageblock_nr_pages))
+ if (!pageblock_aligned(pfn))
return false;
skip = get_pageblock_skip(page);
* COMPACT_CLUSTER_MAX at a time so the second call must
* not falsely conclude that the block should be skipped.
*/
- if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) {
+ if (!valid_page && pageblock_aligned(low_pfn)) {
if (!isolation_suitable(cc, page)) {
low_pfn = end_pfn;
page = NULL;
pfn = cc->zone->zone_start_pfn;
cc->fast_search_fail = 0;
found_block = true;
- set_pageblock_skip(freepage);
break;
}
}
* before making it "skip" so other compaction instances do
* not scan the same block.
*/
- if (IS_ALIGNED(low_pfn, pageblock_nr_pages) &&
+ if (pageblock_aligned(low_pfn) &&
!fast_find_block && !isolation_suitable(cc, page))
continue;
return order == -1;
}
+/*
+ * Determine whether kswapd is (or recently was!) running on this node.
+ *
+ * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't
+ * zero it.
+ */
static bool kswapd_is_running(pg_data_t *pgdat)
{
- return pgdat->kswapd && task_is_running(pgdat->kswapd);
+ bool running;
+
+ pgdat_kswapd_lock(pgdat);
+ running = pgdat->kswapd && task_is_running(pgdat->kswapd);
+ pgdat_kswapd_unlock(pgdat);
+
+ return running;
}
/*
* migration source is unmovable/reclaimable but it's not worth
* special casing.
*/
- if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
+ if (!pageblock_aligned(cc->migrate_pfn))
return COMPACT_CONTINUE;
/* Direct compactor: Is a suitable page free? */
If unsure, say N.
+ This will be removed after >5.15.y LTS kernel is released, so users
+ should move to the sysfs interface (DAMON_SYSFS).
+
config DAMON_DBGFS_KUNIT_TEST
bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS
depends on DAMON_DBGFS && KUNIT=y
t = damon_new_target();
r = damon_new_region(0, 100);
damon_add_region(r, t);
- damon_split_region_at(c, t, r, 25);
+ damon_split_region_at(t, r, 25);
KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
KUNIT_EXPECT_EQ(test, r->ar.end, 25ul);
t = damon_new_target();
r = damon_new_region(0, 22);
damon_add_region(r, t);
- damon_split_regions_of(c, t, 2);
+ damon_split_regions_of(t, 2);
KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u);
damon_free_target(t);
t = damon_new_target();
r = damon_new_region(0, 220);
damon_add_region(r, t);
- damon_split_regions_of(c, t, 4);
+ damon_split_regions_of(t, 4);
KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u);
damon_free_target(t);
damon_destroy_ctx(c);
KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL);
}
+static void damon_test_set_regions(struct kunit *test)
+{
+ struct damon_target *t = damon_new_target();
+ struct damon_region *r1 = damon_new_region(4, 16);
+ struct damon_region *r2 = damon_new_region(24, 32);
+ struct damon_addr_range range = {.start = 8, .end = 28};
+ unsigned long expects[] = {8, 16, 16, 24, 24, 28};
+ int expect_idx = 0;
+ struct damon_region *r;
+
+ damon_add_region(r1, t);
+ damon_add_region(r2, t);
+ damon_set_regions(t, &range, 1);
+
+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3);
+ damon_for_each_region(r, t) {
+ KUNIT_EXPECT_EQ(test, r->ar.start, expects[expect_idx++]);
+ KUNIT_EXPECT_EQ(test, r->ar.end, expects[expect_idx++]);
+ }
+ damon_destroy_target(t);
+}
+
static struct kunit_case damon_test_cases[] = {
KUNIT_CASE(damon_test_target),
KUNIT_CASE(damon_test_regions),
KUNIT_CASE(damon_test_merge_regions_of),
KUNIT_CASE(damon_test_split_regions_of),
KUNIT_CASE(damon_test_ops_registration),
+ KUNIT_CASE(damon_test_set_regions),
{},
};
static DEFINE_MUTEX(damon_ops_lock);
static struct damon_operations damon_registered_ops[NR_DAMON_OPS];
+static struct kmem_cache *damon_region_cache __ro_after_init;
+
/* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */
static bool __damon_is_registered_ops(enum damon_ops_id id)
{
{
struct damon_region *region;
- region = kmalloc(sizeof(*region), GFP_KERNEL);
+ region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL);
if (!region)
return NULL;
static void damon_free_region(struct damon_region *r)
{
- kfree(r);
+ kmem_cache_free(damon_region_cache, r);
}
void damon_destroy_region(struct damon_region *r, struct damon_target *t)
return !(r->ar.end <= re->start || re->end <= r->ar.start);
}
+/*
+ * Fill holes in regions with new regions.
+ */
+static int damon_fill_regions_holes(struct damon_region *first,
+ struct damon_region *last, struct damon_target *t)
+{
+ struct damon_region *r = first;
+
+ damon_for_each_region_from(r, t) {
+ struct damon_region *next, *newr;
+
+ if (r == last)
+ break;
+ next = damon_next_region(r);
+ if (r->ar.end != next->ar.start) {
+ newr = damon_new_region(r->ar.end, next->ar.start);
+ if (!newr)
+ return -ENOMEM;
+ damon_insert_region(newr, r, next, t);
+ }
+ }
+ return 0;
+}
+
/*
* damon_set_regions() - Set regions of a target for given address ranges.
* @t: the given target.
{
struct damon_region *r, *next;
unsigned int i;
+ int err;
/* Remove regions which are not in the new ranges */
damon_for_each_region_safe(r, next, t) {
damon_destroy_region(r, t);
}
+ r = damon_first_region(t);
/* Add new regions or resize existing regions to fit in the ranges */
for (i = 0; i < nr_ranges; i++) {
struct damon_region *first = NULL, *last, *newr;
range = &ranges[i];
/* Get the first/last regions intersecting with the range */
- damon_for_each_region(r, t) {
+ damon_for_each_region_from(r, t) {
if (damon_intersect(r, range)) {
if (!first)
first = r;
first->ar.start = ALIGN_DOWN(range->start,
DAMON_MIN_REGION);
last->ar.end = ALIGN(range->end, DAMON_MIN_REGION);
+
+ /* fill possible holes in the range */
+ err = damon_fill_regions_holes(first, last, t);
+ if (err)
+ return err;
}
}
return 0;
}
-struct damos *damon_new_scheme(
- unsigned long min_sz_region, unsigned long max_sz_region,
- unsigned int min_nr_accesses, unsigned int max_nr_accesses,
- unsigned int min_age_region, unsigned int max_age_region,
- enum damos_action action, struct damos_quota *quota,
- struct damos_watermarks *wmarks)
+/* initialize private fields of damos_quota and return the pointer */
+static struct damos_quota *damos_quota_init_priv(struct damos_quota *quota)
+{
+ quota->total_charged_sz = 0;
+ quota->total_charged_ns = 0;
+ quota->esz = 0;
+ quota->charged_sz = 0;
+ quota->charged_from = 0;
+ quota->charge_target_from = NULL;
+ quota->charge_addr_from = 0;
+ return quota;
+}
+
+struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
+ enum damos_action action, struct damos_quota *quota,
+ struct damos_watermarks *wmarks)
{
struct damos *scheme;
scheme = kmalloc(sizeof(*scheme), GFP_KERNEL);
if (!scheme)
return NULL;
- scheme->min_sz_region = min_sz_region;
- scheme->max_sz_region = max_sz_region;
- scheme->min_nr_accesses = min_nr_accesses;
- scheme->max_nr_accesses = max_nr_accesses;
- scheme->min_age_region = min_age_region;
- scheme->max_age_region = max_age_region;
+ scheme->pattern = *pattern;
scheme->action = action;
scheme->stat = (struct damos_stat){};
INIT_LIST_HEAD(&scheme->list);
- scheme->quota.ms = quota->ms;
- scheme->quota.sz = quota->sz;
- scheme->quota.reset_interval = quota->reset_interval;
- scheme->quota.weight_sz = quota->weight_sz;
- scheme->quota.weight_nr_accesses = quota->weight_nr_accesses;
- scheme->quota.weight_age = quota->weight_age;
- scheme->quota.total_charged_sz = 0;
- scheme->quota.total_charged_ns = 0;
- scheme->quota.esz = 0;
- scheme->quota.charged_sz = 0;
- scheme->quota.charged_from = 0;
- scheme->quota.charge_target_from = NULL;
- scheme->quota.charge_addr_from = 0;
-
- scheme->wmarks.metric = wmarks->metric;
- scheme->wmarks.interval = wmarks->interval;
- scheme->wmarks.high = wmarks->high;
- scheme->wmarks.mid = wmarks->mid;
- scheme->wmarks.low = wmarks->low;
+ scheme->quota = *(damos_quota_init_priv(quota));
+
+ scheme->wmarks = *wmarks;
scheme->wmarks.activated = true;
return scheme;
t->pid = NULL;
t->nr_regions = 0;
INIT_LIST_HEAD(&t->regions_list);
+ INIT_LIST_HEAD(&t->list);
return t;
}
if (!ctx)
return NULL;
- ctx->sample_interval = 5 * 1000;
- ctx->aggr_interval = 100 * 1000;
- ctx->ops_update_interval = 60 * 1000 * 1000;
+ ctx->attrs.sample_interval = 5 * 1000;
+ ctx->attrs.aggr_interval = 100 * 1000;
+ ctx->attrs.ops_update_interval = 60 * 1000 * 1000;
ktime_get_coarse_ts64(&ctx->last_aggregation);
ctx->last_ops_update = ctx->last_aggregation;
mutex_init(&ctx->kdamond_lock);
- ctx->min_nr_regions = 10;
- ctx->max_nr_regions = 1000;
+ ctx->attrs.min_nr_regions = 10;
+ ctx->attrs.max_nr_regions = 1000;
INIT_LIST_HEAD(&ctx->adaptive_targets);
INIT_LIST_HEAD(&ctx->schemes);
/**
* damon_set_attrs() - Set attributes for the monitoring.
* @ctx: monitoring context
- * @sample_int: time interval between samplings
- * @aggr_int: time interval between aggregations
- * @ops_upd_int: time interval between monitoring operations updates
- * @min_nr_reg: minimal number of regions
- * @max_nr_reg: maximum number of regions
+ * @attrs: monitoring attributes
*
* This function should not be called while the kdamond is running.
* Every time interval is in micro-seconds.
*
* Return: 0 on success, negative error code otherwise.
*/
-int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int,
- unsigned long aggr_int, unsigned long ops_upd_int,
- unsigned long min_nr_reg, unsigned long max_nr_reg)
+int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs)
{
- if (min_nr_reg < 3)
+ if (attrs->min_nr_regions < 3)
return -EINVAL;
- if (min_nr_reg > max_nr_reg)
+ if (attrs->min_nr_regions > attrs->max_nr_regions)
return -EINVAL;
- ctx->sample_interval = sample_int;
- ctx->aggr_interval = aggr_int;
- ctx->ops_update_interval = ops_upd_int;
- ctx->min_nr_regions = min_nr_reg;
- ctx->max_nr_regions = max_nr_reg;
-
+ ctx->attrs = *attrs;
return 0;
}
*
* This function should not be called while the kdamond of the context is
* running.
- *
- * Return: 0 if success, or negative error code otherwise.
*/
-int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes,
+void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes,
ssize_t nr_schemes)
{
struct damos *s, *next;
damon_destroy_scheme(s);
for (i = 0; i < nr_schemes; i++)
damon_add_scheme(ctx, schemes[i]);
- return 0;
}
/**
damon_for_each_target(t, ctx) {
damon_for_each_region(r, t)
- sz += r->ar.end - r->ar.start;
+ sz += damon_sz_region(r);
}
- if (ctx->min_nr_regions)
- sz /= ctx->min_nr_regions;
+ if (ctx->attrs.min_nr_regions)
+ sz /= ctx->attrs.min_nr_regions;
if (sz < DAMON_MIN_REGION)
sz = DAMON_MIN_REGION;
static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx)
{
return damon_check_reset_time_interval(&ctx->last_aggregation,
- ctx->aggr_interval);
+ ctx->attrs.aggr_interval);
}
/*
}
}
-static void damon_split_region_at(struct damon_ctx *ctx,
- struct damon_target *t, struct damon_region *r,
- unsigned long sz_r);
+static void damon_split_region_at(struct damon_target *t,
+ struct damon_region *r, unsigned long sz_r);
static bool __damos_valid_target(struct damon_region *r, struct damos *s)
{
unsigned long sz;
- sz = r->ar.end - r->ar.start;
- return s->min_sz_region <= sz && sz <= s->max_sz_region &&
- s->min_nr_accesses <= r->nr_accesses &&
- r->nr_accesses <= s->max_nr_accesses &&
- s->min_age_region <= r->age && r->age <= s->max_age_region;
+ sz = damon_sz_region(r);
+ return s->pattern.min_sz_region <= sz &&
+ sz <= s->pattern.max_sz_region &&
+ s->pattern.min_nr_accesses <= r->nr_accesses &&
+ r->nr_accesses <= s->pattern.max_nr_accesses &&
+ s->pattern.min_age_region <= r->age &&
+ r->age <= s->pattern.max_age_region;
}
static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t,
damon_for_each_scheme(s, c) {
struct damos_quota *quota = &s->quota;
- unsigned long sz = r->ar.end - r->ar.start;
+ unsigned long sz = damon_sz_region(r);
struct timespec64 begin, end;
unsigned long sz_applied = 0;
sz = ALIGN_DOWN(quota->charge_addr_from -
r->ar.start, DAMON_MIN_REGION);
if (!sz) {
- if (r->ar.end - r->ar.start <=
- DAMON_MIN_REGION)
+ if (damon_sz_region(r) <=
+ DAMON_MIN_REGION)
continue;
sz = DAMON_MIN_REGION;
}
- damon_split_region_at(c, t, r, sz);
+ damon_split_region_at(t, r, sz);
r = damon_next_region(r);
- sz = r->ar.end - r->ar.start;
+ sz = damon_sz_region(r);
}
quota->charge_target_from = NULL;
quota->charge_addr_from = 0;
DAMON_MIN_REGION);
if (!sz)
goto update_stat;
- damon_split_region_at(c, t, r, sz);
+ damon_split_region_at(t, r, sz);
}
ktime_get_coarse_ts64(&begin);
sz_applied = c->ops.apply_scheme(c, t, r, s);
continue;
score = c->ops.get_scheme_score(
c, t, r, s);
- quota->histogram[score] +=
- r->ar.end - r->ar.start;
+ quota->histogram[score] += damon_sz_region(r);
if (score > max_score)
max_score = score;
}
}
}
-static inline unsigned long sz_damon_region(struct damon_region *r)
-{
- return r->ar.end - r->ar.start;
-}
-
/*
* Merge two adjacent regions into one region
*/
static void damon_merge_two_regions(struct damon_target *t,
struct damon_region *l, struct damon_region *r)
{
- unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r);
+ unsigned long sz_l = damon_sz_region(l), sz_r = damon_sz_region(r);
l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) /
(sz_l + sz_r);
if (prev && prev->ar.end == r->ar.start &&
abs(prev->nr_accesses - r->nr_accesses) <= thres &&
- sz_damon_region(prev) + sz_damon_region(r) <= sz_limit)
+ damon_sz_region(prev) + damon_sz_region(r) <= sz_limit)
damon_merge_two_regions(t, prev, r);
else
prev = r;
* r the region to be split
* sz_r size of the first sub-region that will be made
*/
-static void damon_split_region_at(struct damon_ctx *ctx,
- struct damon_target *t, struct damon_region *r,
- unsigned long sz_r)
+static void damon_split_region_at(struct damon_target *t,
+ struct damon_region *r, unsigned long sz_r)
{
struct damon_region *new;
}
/* Split every region in the given target into 'nr_subs' regions */
-static void damon_split_regions_of(struct damon_ctx *ctx,
- struct damon_target *t, int nr_subs)
+static void damon_split_regions_of(struct damon_target *t, int nr_subs)
{
struct damon_region *r, *next;
unsigned long sz_region, sz_sub = 0;
int i;
damon_for_each_region_safe(r, next, t) {
- sz_region = r->ar.end - r->ar.start;
+ sz_region = damon_sz_region(r);
for (i = 0; i < nr_subs - 1 &&
sz_region > 2 * DAMON_MIN_REGION; i++) {
if (sz_sub == 0 || sz_sub >= sz_region)
continue;
- damon_split_region_at(ctx, t, r, sz_sub);
+ damon_split_region_at(t, r, sz_sub);
sz_region = sz_sub;
}
}
damon_for_each_target(t, ctx)
nr_regions += damon_nr_regions(t);
- if (nr_regions > ctx->max_nr_regions / 2)
+ if (nr_regions > ctx->attrs.max_nr_regions / 2)
return;
/* Maybe the middle of the region has different access frequency */
if (last_nr_regions == nr_regions &&
- nr_regions < ctx->max_nr_regions / 3)
+ nr_regions < ctx->attrs.max_nr_regions / 3)
nr_subregions = 3;
damon_for_each_target(t, ctx)
- damon_split_regions_of(ctx, t, nr_subregions);
+ damon_split_regions_of(t, nr_subregions);
last_nr_regions = nr_regions;
}
static bool kdamond_need_update_operations(struct damon_ctx *ctx)
{
return damon_check_reset_time_interval(&ctx->last_ops_update,
- ctx->ops_update_interval);
+ ctx->attrs.ops_update_interval);
}
/*
struct damon_region *r, *next;
unsigned int max_nr_accesses = 0;
unsigned long sz_limit = 0;
- bool done = false;
pr_debug("kdamond (%d) starts\n", current->pid);
if (ctx->ops.init)
ctx->ops.init(ctx);
if (ctx->callback.before_start && ctx->callback.before_start(ctx))
- done = true;
+ goto done;
sz_limit = damon_region_sz_limit(ctx);
- while (!kdamond_need_stop(ctx) && !done) {
- if (kdamond_wait_activation(ctx)) {
- done = true;
- continue;
- }
+ while (!kdamond_need_stop(ctx)) {
+ if (kdamond_wait_activation(ctx))
+ break;
if (ctx->ops.prepare_access_checks)
ctx->ops.prepare_access_checks(ctx);
if (ctx->callback.after_sampling &&
- ctx->callback.after_sampling(ctx)) {
- done = true;
- continue;
- }
+ ctx->callback.after_sampling(ctx))
+ break;
- kdamond_usleep(ctx->sample_interval);
+ kdamond_usleep(ctx->attrs.sample_interval);
if (ctx->ops.check_accesses)
max_nr_accesses = ctx->ops.check_accesses(ctx);
max_nr_accesses / 10,
sz_limit);
if (ctx->callback.after_aggregation &&
- ctx->callback.after_aggregation(ctx)) {
- done = true;
- continue;
- }
+ ctx->callback.after_aggregation(ctx))
+ break;
kdamond_apply_schemes(ctx);
kdamond_reset_aggregated(ctx);
kdamond_split_regions(ctx);
sz_limit = damon_region_sz_limit(ctx);
}
}
+done:
damon_for_each_target(t, ctx) {
damon_for_each_region_safe(r, next, t)
damon_destroy_region(r, t);
return 0;
}
+/*
+ * struct damon_system_ram_region - System RAM resource address region of
+ * [@start, @end).
+ * @start: Start address of the region (inclusive).
+ * @end: End address of the region (exclusive).
+ */
+struct damon_system_ram_region {
+ unsigned long start;
+ unsigned long end;
+};
+
+static int walk_system_ram(struct resource *res, void *arg)
+{
+ struct damon_system_ram_region *a = arg;
+
+ if (a->end - a->start < resource_size(res)) {
+ a->start = res->start;
+ a->end = res->end;
+ }
+ return 0;
+}
+
+/*
+ * Find biggest 'System RAM' resource and store its start and end address in
+ * @start and @end, respectively. If no System RAM is found, returns false.
+ */
+static bool damon_find_biggest_system_ram(unsigned long *start,
+ unsigned long *end)
+
+{
+ struct damon_system_ram_region arg = {};
+
+ walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
+ if (arg.end <= arg.start)
+ return false;
+
+ *start = arg.start;
+ *end = arg.end;
+ return true;
+}
+
+/**
+ * damon_set_region_biggest_system_ram_default() - Set the region of the given
+ * monitoring target as requested, or biggest 'System RAM'.
+ * @t: The monitoring target to set the region.
+ * @start: The pointer to the start address of the region.
+ * @end: The pointer to the end address of the region.
+ *
+ * This function sets the region of @t as requested by @start and @end. If the
+ * values of @start and @end are zero, however, this function finds the biggest
+ * 'System RAM' resource and sets the region to cover the resource. In the
+ * latter case, this function saves the start and end addresses of the resource
+ * in @start and @end, respectively.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int damon_set_region_biggest_system_ram_default(struct damon_target *t,
+ unsigned long *start, unsigned long *end)
+{
+ struct damon_addr_range addr_range;
+
+ if (*start > *end)
+ return -EINVAL;
+
+ if (!*start && !*end &&
+ !damon_find_biggest_system_ram(start, end))
+ return -EINVAL;
+
+ addr_range.start = *start;
+ addr_range.end = *end;
+ return damon_set_regions(t, &addr_range, 1);
+}
+
+static int __init damon_init(void)
+{
+ damon_region_cache = KMEM_CACHE(damon_region, 0);
+ if (unlikely(!damon_region_cache)) {
+ pr_err("creating damon_region_cache fails\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+subsys_initcall(damon_init);
+
#include "core-test.h"
mutex_lock(&ctx->kdamond_lock);
ret = scnprintf(kbuf, ARRAY_SIZE(kbuf), "%lu %lu %lu %lu %lu\n",
- ctx->sample_interval, ctx->aggr_interval,
- ctx->ops_update_interval, ctx->min_nr_regions,
- ctx->max_nr_regions);
+ ctx->attrs.sample_interval, ctx->attrs.aggr_interval,
+ ctx->attrs.ops_update_interval,
+ ctx->attrs.min_nr_regions, ctx->attrs.max_nr_regions);
mutex_unlock(&ctx->kdamond_lock);
return simple_read_from_buffer(buf, count, ppos, kbuf, ret);
const char __user *buf, size_t count, loff_t *ppos)
{
struct damon_ctx *ctx = file->private_data;
- unsigned long s, a, r, minr, maxr;
+ struct damon_attrs attrs;
char *kbuf;
ssize_t ret;
return PTR_ERR(kbuf);
if (sscanf(kbuf, "%lu %lu %lu %lu %lu",
- &s, &a, &r, &minr, &maxr) != 5) {
+ &attrs.sample_interval, &attrs.aggr_interval,
+ &attrs.ops_update_interval,
+ &attrs.min_nr_regions,
+ &attrs.max_nr_regions) != 5) {
ret = -EINVAL;
goto out;
}
goto unlock_out;
}
- ret = damon_set_attrs(ctx, s, a, r, minr, maxr);
+ ret = damon_set_attrs(ctx, &attrs);
if (!ret)
ret = count;
unlock_out:
damon_for_each_scheme(s, c) {
rc = scnprintf(&buf[written], len - written,
"%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu %lu %lu %lu\n",
- s->min_sz_region, s->max_sz_region,
- s->min_nr_accesses, s->max_nr_accesses,
- s->min_age_region, s->max_age_region,
+ s->pattern.min_sz_region,
+ s->pattern.max_sz_region,
+ s->pattern.min_nr_accesses,
+ s->pattern.max_nr_accesses,
+ s->pattern.min_age_region,
+ s->pattern.max_age_region,
damos_action_to_dbgfs_scheme_action(s->action),
s->quota.ms, s->quota.sz,
s->quota.reset_interval,
struct damos *scheme, **schemes;
const int max_nr_schemes = 256;
int pos = 0, parsed, ret;
- unsigned long min_sz, max_sz;
- unsigned int min_nr_a, max_nr_a, min_age, max_age;
unsigned int action_input;
enum damos_action action;
*nr_schemes = 0;
while (pos < len && *nr_schemes < max_nr_schemes) {
+ struct damos_access_pattern pattern = {};
struct damos_quota quota = {};
struct damos_watermarks wmarks;
ret = sscanf(&str[pos],
"%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u %u %lu %lu %lu %lu%n",
- &min_sz, &max_sz, &min_nr_a, &max_nr_a,
- &min_age, &max_age, &action_input, "a.ms,
+ &pattern.min_sz_region, &pattern.max_sz_region,
+ &pattern.min_nr_accesses,
+ &pattern.max_nr_accesses,
+ &pattern.min_age_region,
+ &pattern.max_age_region,
+ &action_input, "a.ms,
"a.sz, "a.reset_interval,
"a.weight_sz, "a.weight_nr_accesses,
"a.weight_age, &wmarks.metric,
if ((int)action < 0)
goto fail;
- if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age)
+ if (pattern.min_sz_region > pattern.max_sz_region ||
+ pattern.min_nr_accesses > pattern.max_nr_accesses ||
+ pattern.min_age_region > pattern.max_age_region)
goto fail;
if (wmarks.high < wmarks.mid || wmarks.high < wmarks.low ||
goto fail;
pos += parsed;
- scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a,
- min_age, max_age, action, "a, &wmarks);
+ scheme = damon_new_scheme(&pattern, action, "a, &wmarks);
if (!scheme)
goto fail;
goto unlock_out;
}
- ret = damon_set_schemes(ctx, schemes, nr_schemes);
- if (!ret) {
- ret = count;
- nr_schemes = 0;
- }
+ damon_set_schemes(ctx, schemes, nr_schemes);
+ ret = count;
+ nr_schemes = 0;
unlock_out:
mutex_unlock(&ctx->kdamond_lock);
fops[i]);
dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]);
- dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL);
+ dbgfs_dirs = kmalloc(sizeof(dbgfs_root), GFP_KERNEL);
if (!dbgfs_dirs) {
debugfs_remove(dbgfs_root);
return -ENOMEM;
#include <linux/sched.h>
#include <linux/workqueue.h>
+#include "modules-common.h"
+
#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
static unsigned long cold_min_age __read_mostly = 120000000;
module_param(cold_min_age, ulong, 0600);
-/*
- * Limit of time for trying the LRU lists sorting in milliseconds.
- *
- * DAMON_LRU_SORT tries to use only up to this time within a time window
- * (quota_reset_interval_ms) for trying LRU lists sorting. This can be used
- * for limiting CPU consumption of DAMON_LRU_SORT. If the value is zero, the
- * limit is disabled.
- *
- * 10 ms by default.
- */
-static unsigned long quota_ms __read_mostly = 10;
-module_param(quota_ms, ulong, 0600);
-
-/*
- * The time quota charge reset interval in milliseconds.
- *
- * The charge reset interval for the quota of time (quota_ms). That is,
- * DAMON_LRU_SORT does not try LRU-lists sorting for more than quota_ms
- * milliseconds or quota_sz bytes within quota_reset_interval_ms milliseconds.
- *
- * 1 second by default.
- */
-static unsigned long quota_reset_interval_ms __read_mostly = 1000;
-module_param(quota_reset_interval_ms, ulong, 0600);
-
-/*
- * The watermarks check time interval in microseconds.
- *
- * Minimal time to wait before checking the watermarks, when DAMON_LRU_SORT is
- * enabled but inactive due to its watermarks rule. 5 seconds by default.
- */
-static unsigned long wmarks_interval __read_mostly = 5000000;
-module_param(wmarks_interval, ulong, 0600);
-
-/*
- * Free memory rate (per thousand) for the high watermark.
- *
- * If free memory of the system in bytes per thousand bytes is higher than
- * this, DAMON_LRU_SORT becomes inactive, so it does nothing but periodically
- * checks the watermarks. 200 (20%) by default.
- */
-static unsigned long wmarks_high __read_mostly = 200;
-module_param(wmarks_high, ulong, 0600);
-
-/*
- * Free memory rate (per thousand) for the middle watermark.
- *
- * If free memory of the system in bytes per thousand bytes is between this and
- * the low watermark, DAMON_LRU_SORT becomes active, so starts the monitoring
- * and the LRU-lists sorting. 150 (15%) by default.
- */
-static unsigned long wmarks_mid __read_mostly = 150;
-module_param(wmarks_mid, ulong, 0600);
-
-/*
- * Free memory rate (per thousand) for the low watermark.
- *
- * If free memory of the system in bytes per thousand bytes is lower than this,
- * DAMON_LRU_SORT becomes inactive, so it does nothing but periodically checks
- * the watermarks. 50 (5%) by default.
- */
-static unsigned long wmarks_low __read_mostly = 50;
-module_param(wmarks_low, ulong, 0600);
-
-/*
- * Sampling interval for the monitoring in microseconds.
- *
- * The sampling interval of DAMON for the hot/cold memory monitoring. Please
- * refer to the DAMON documentation for more detail. 5 ms by default.
- */
-static unsigned long sample_interval __read_mostly = 5000;
-module_param(sample_interval, ulong, 0600);
-
-/*
- * Aggregation interval for the monitoring in microseconds.
- *
- * The aggregation interval of DAMON for the hot/cold memory monitoring.
- * Please refer to the DAMON documentation for more detail. 100 ms by default.
- */
-static unsigned long aggr_interval __read_mostly = 100000;
-module_param(aggr_interval, ulong, 0600);
-
-/*
- * Minimum number of monitoring regions.
- *
- * The minimal number of monitoring regions of DAMON for the hot/cold memory
- * monitoring. This can be used to set lower-bound of the monitoring quality.
- * But, setting this too high could result in increased monitoring overhead.
- * Please refer to the DAMON documentation for more detail. 10 by default.
- */
-static unsigned long min_nr_regions __read_mostly = 10;
-module_param(min_nr_regions, ulong, 0600);
-
-/*
- * Maximum number of monitoring regions.
- *
- * The maximum number of monitoring regions of DAMON for the hot/cold memory
- * monitoring. This can be used to set upper-bound of the monitoring overhead.
- * However, setting this too low could result in bad monitoring quality.
- * Please refer to the DAMON documentation for more detail. 1000 by default.
- */
-static unsigned long max_nr_regions __read_mostly = 1000;
-module_param(max_nr_regions, ulong, 0600);
+static struct damos_quota damon_lru_sort_quota = {
+ /* Use up to 10 ms per 1 sec, by default */
+ .ms = 10,
+ .sz = 0,
+ .reset_interval = 1000,
+ /* Within the quota, mark hotter regions accessed first. */
+ .weight_sz = 0,
+ .weight_nr_accesses = 1,
+ .weight_age = 0,
+};
+DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota);
+
+static struct damos_watermarks damon_lru_sort_wmarks = {
+ .metric = DAMOS_WMARK_FREE_MEM_RATE,
+ .interval = 5000000, /* 5 seconds */
+ .high = 200, /* 20 percent */
+ .mid = 150, /* 15 percent */
+ .low = 50, /* 5 percent */
+};
+DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks);
+
+static struct damon_attrs damon_lru_sort_mon_attrs = {
+ .sample_interval = 5000, /* 5 ms */
+ .aggr_interval = 100000, /* 100 ms */
+ .ops_update_interval = 0,
+ .min_nr_regions = 10,
+ .max_nr_regions = 1000,
+};
+DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs);
/*
* Start of the target memory region in physical address.
static int kdamond_pid __read_mostly = -1;
module_param(kdamond_pid, int, 0400);
-/*
- * Number of hot memory regions that tried to be LRU-sorted.
- */
-static unsigned long nr_lru_sort_tried_hot_regions __read_mostly;
-module_param(nr_lru_sort_tried_hot_regions, ulong, 0400);
-
-/*
- * Total bytes of hot memory regions that tried to be LRU-sorted.
- */
-static unsigned long bytes_lru_sort_tried_hot_regions __read_mostly;
-module_param(bytes_lru_sort_tried_hot_regions, ulong, 0400);
-
-/*
- * Number of hot memory regions that successfully be LRU-sorted.
- */
-static unsigned long nr_lru_sorted_hot_regions __read_mostly;
-module_param(nr_lru_sorted_hot_regions, ulong, 0400);
-
-/*
- * Total bytes of hot memory regions that successfully be LRU-sorted.
- */
-static unsigned long bytes_lru_sorted_hot_regions __read_mostly;
-module_param(bytes_lru_sorted_hot_regions, ulong, 0400);
-
-/*
- * Number of times that the time quota limit for hot regions have exceeded
- */
-static unsigned long nr_hot_quota_exceeds __read_mostly;
-module_param(nr_hot_quota_exceeds, ulong, 0400);
-
-/*
- * Number of cold memory regions that tried to be LRU-sorted.
- */
-static unsigned long nr_lru_sort_tried_cold_regions __read_mostly;
-module_param(nr_lru_sort_tried_cold_regions, ulong, 0400);
-
-/*
- * Total bytes of cold memory regions that tried to be LRU-sorted.
- */
-static unsigned long bytes_lru_sort_tried_cold_regions __read_mostly;
-module_param(bytes_lru_sort_tried_cold_regions, ulong, 0400);
-
-/*
- * Number of cold memory regions that successfully be LRU-sorted.
- */
-static unsigned long nr_lru_sorted_cold_regions __read_mostly;
-module_param(nr_lru_sorted_cold_regions, ulong, 0400);
-
-/*
- * Total bytes of cold memory regions that successfully be LRU-sorted.
- */
-static unsigned long bytes_lru_sorted_cold_regions __read_mostly;
-module_param(bytes_lru_sorted_cold_regions, ulong, 0400);
-
-/*
- * Number of times that the time quota limit for cold regions have exceeded
- */
-static unsigned long nr_cold_quota_exceeds __read_mostly;
-module_param(nr_cold_quota_exceeds, ulong, 0400);
+static struct damos_stat damon_lru_sort_hot_stat;
+DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat,
+ lru_sort_tried_hot_regions, lru_sorted_hot_regions,
+ hot_quota_exceeds);
+
+static struct damos_stat damon_lru_sort_cold_stat;
+DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat,
+ lru_sort_tried_cold_regions, lru_sorted_cold_regions,
+ cold_quota_exceeds);
+
+static struct damos_access_pattern damon_lru_sort_stub_pattern = {
+ /* Find regions having PAGE_SIZE or larger size */
+ .min_sz_region = PAGE_SIZE,
+ .max_sz_region = ULONG_MAX,
+ /* no matter its access frequency */
+ .min_nr_accesses = 0,
+ .max_nr_accesses = UINT_MAX,
+ /* no matter its age */
+ .min_age_region = 0,
+ .max_age_region = UINT_MAX,
+};
static struct damon_ctx *ctx;
static struct damon_target *target;
-struct damon_lru_sort_ram_walk_arg {
- unsigned long start;
- unsigned long end;
-};
-
-static int walk_system_ram(struct resource *res, void *arg)
-{
- struct damon_lru_sort_ram_walk_arg *a = arg;
-
- if (a->end - a->start < resource_size(res)) {
- a->start = res->start;
- a->end = res->end;
- }
- return 0;
-}
-
-/*
- * Find biggest 'System RAM' resource and store its start and end address in
- * @start and @end, respectively. If no System RAM is found, returns false.
- */
-static bool get_monitoring_region(unsigned long *start, unsigned long *end)
+static struct damos *damon_lru_sort_new_scheme(
+ struct damos_access_pattern *pattern, enum damos_action action)
{
- struct damon_lru_sort_ram_walk_arg arg = {};
+ struct damos_quota quota = damon_lru_sort_quota;
- walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
- if (arg.end <= arg.start)
- return false;
+ /* Use half of total quota for hot/cold pages sorting */
+ quota.ms = quota.ms / 2;
- *start = arg.start;
- *end = arg.end;
- return true;
+ return damon_new_scheme(
+ /* find the pattern, and */
+ pattern,
+ /* (de)prioritize on LRU-lists */
+ action,
+ /* under the quota. */
+ "a,
+ /* (De)activate this according to the watermarks. */
+ &damon_lru_sort_wmarks);
}
/* Create a DAMON-based operation scheme for hot memory regions */
static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres)
{
- struct damos_watermarks wmarks = {
- .metric = DAMOS_WMARK_FREE_MEM_RATE,
- .interval = wmarks_interval,
- .high = wmarks_high,
- .mid = wmarks_mid,
- .low = wmarks_low,
- };
- struct damos_quota quota = {
- /*
- * Do not try LRU-lists sorting of hot pages for more than half
- * of quota_ms milliseconds within quota_reset_interval_ms.
- */
- .ms = quota_ms / 2,
- .sz = 0,
- .reset_interval = quota_reset_interval_ms,
- /* Within the quota, mark hotter regions accessed first. */
- .weight_sz = 0,
- .weight_nr_accesses = 1,
- .weight_age = 0,
- };
- struct damos *scheme = damon_new_scheme(
- /* Find regions having PAGE_SIZE or larger size */
- PAGE_SIZE, ULONG_MAX,
- /* and accessed for more than the threshold */
- hot_thres, UINT_MAX,
- /* no matter its age */
- 0, UINT_MAX,
- /* prioritize those on LRU lists, as soon as found */
- DAMOS_LRU_PRIO,
- /* under the quota. */
- "a,
- /* (De)activate this according to the watermarks. */
- &wmarks);
+ struct damos_access_pattern pattern = damon_lru_sort_stub_pattern;
- return scheme;
+ pattern.min_nr_accesses = hot_thres;
+ return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO);
}
/* Create a DAMON-based operation scheme for cold memory regions */
static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres)
{
- struct damos_watermarks wmarks = {
- .metric = DAMOS_WMARK_FREE_MEM_RATE,
- .interval = wmarks_interval,
- .high = wmarks_high,
- .mid = wmarks_mid,
- .low = wmarks_low,
- };
- struct damos_quota quota = {
- /*
- * Do not try LRU-lists sorting of cold pages for more than
- * half of quota_ms milliseconds within
- * quota_reset_interval_ms.
- */
- .ms = quota_ms / 2,
- .sz = 0,
- .reset_interval = quota_reset_interval_ms,
- /* Within the quota, mark colder regions not accessed first. */
- .weight_sz = 0,
- .weight_nr_accesses = 0,
- .weight_age = 1,
- };
- struct damos *scheme = damon_new_scheme(
- /* Find regions having PAGE_SIZE or larger size */
- PAGE_SIZE, ULONG_MAX,
- /* and not accessed at all */
- 0, 0,
- /* for cold_thres or more micro-seconds, and */
- cold_thres, UINT_MAX,
- /* mark those as not accessed, as soon as found */
- DAMOS_LRU_DEPRIO,
- /* under the quota. */
- "a,
- /* (De)activate this according to the watermarks. */
- &wmarks);
+ struct damos_access_pattern pattern = damon_lru_sort_stub_pattern;
- return scheme;
+ pattern.max_nr_accesses = 0;
+ pattern.min_age_region = cold_thres;
+ return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO);
}
static int damon_lru_sort_apply_parameters(void)
{
- struct damos *scheme, *next_scheme;
- struct damon_addr_range addr_range;
+ struct damos *scheme;
unsigned int hot_thres, cold_thres;
int err = 0;
- err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0,
- min_nr_regions, max_nr_regions);
+ err = damon_set_attrs(ctx, &damon_lru_sort_mon_attrs);
if (err)
return err;
- /* free previously set schemes */
- damon_for_each_scheme_safe(scheme, next_scheme, ctx)
- damon_destroy_scheme(scheme);
-
/* aggr_interval / sample_interval is the maximum nr_accesses */
- hot_thres = aggr_interval / sample_interval * hot_thres_access_freq /
- 1000;
+ hot_thres = damon_lru_sort_mon_attrs.aggr_interval /
+ damon_lru_sort_mon_attrs.sample_interval *
+ hot_thres_access_freq / 1000;
scheme = damon_lru_sort_new_hot_scheme(hot_thres);
if (!scheme)
return -ENOMEM;
- damon_add_scheme(ctx, scheme);
+ damon_set_schemes(ctx, &scheme, 1);
- cold_thres = cold_min_age / aggr_interval;
+ cold_thres = cold_min_age / damon_lru_sort_mon_attrs.aggr_interval;
scheme = damon_lru_sort_new_cold_scheme(cold_thres);
if (!scheme)
return -ENOMEM;
damon_add_scheme(ctx, scheme);
- if (monitor_region_start > monitor_region_end)
- return -EINVAL;
- if (!monitor_region_start && !monitor_region_end &&
- !get_monitoring_region(&monitor_region_start,
- &monitor_region_end))
- return -EINVAL;
- addr_range.start = monitor_region_start;
- addr_range.end = monitor_region_end;
- return damon_set_regions(target, &addr_range, 1);
+ return damon_set_region_biggest_system_ram_default(target,
+ &monitor_region_start,
+ &monitor_region_end);
}
static int damon_lru_sort_turn(bool on)
/* update the stats parameter */
damon_for_each_scheme(s, c) {
- if (s->action == DAMOS_LRU_PRIO) {
- nr_lru_sort_tried_hot_regions = s->stat.nr_tried;
- bytes_lru_sort_tried_hot_regions = s->stat.sz_tried;
- nr_lru_sorted_hot_regions = s->stat.nr_applied;
- bytes_lru_sorted_hot_regions = s->stat.sz_applied;
- nr_hot_quota_exceeds = s->stat.qt_exceeds;
- } else if (s->action == DAMOS_LRU_DEPRIO) {
- nr_lru_sort_tried_cold_regions = s->stat.nr_tried;
- bytes_lru_sort_tried_cold_regions = s->stat.sz_tried;
- nr_lru_sorted_cold_regions = s->stat.nr_applied;
- bytes_lru_sorted_cold_regions = s->stat.sz_applied;
- nr_cold_quota_exceeds = s->stat.qt_exceeds;
- }
+ if (s->action == DAMOS_LRU_PRIO)
+ damon_lru_sort_hot_stat = s->stat;
+ else if (s->action == DAMOS_LRU_DEPRIO)
+ damon_lru_sort_cold_stat = s->stat;
}
return damon_lru_sort_handle_commit_inputs();
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Common Primitives for DAMON Modules
+ *
+ * Author: SeongJae Park <sj@kernel.org>
+ */
+
+#include <linux/moduleparam.h>
+
+#define DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(attrs) \
+ module_param_named(sample_interval, attrs.sample_interval, \
+ ulong, 0600); \
+ module_param_named(aggr_interval, attrs.aggr_interval, ulong, \
+ 0600); \
+ module_param_named(min_nr_regions, attrs.min_nr_regions, ulong, \
+ 0600); \
+ module_param_named(max_nr_regions, attrs.max_nr_regions, ulong, \
+ 0600);
+
+#define DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(quota) \
+ module_param_named(quota_ms, quota.ms, ulong, 0600); \
+ module_param_named(quota_reset_interval_ms, \
+ quota.reset_interval, ulong, 0600);
+
+#define DEFINE_DAMON_MODULES_DAMOS_QUOTAS(quota) \
+ DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(quota) \
+ module_param_named(quota_sz, quota.sz, ulong, 0600);
+
+#define DEFINE_DAMON_MODULES_WMARKS_PARAMS(wmarks) \
+ module_param_named(wmarks_interval, wmarks.interval, ulong, \
+ 0600); \
+ module_param_named(wmarks_high, wmarks.high, ulong, 0600); \
+ module_param_named(wmarks_mid, wmarks.mid, ulong, 0600); \
+ module_param_named(wmarks_low, wmarks.low, ulong, 0600);
+
+#define DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(stat, try_name, \
+ succ_name, qt_exceed_name) \
+ module_param_named(nr_##try_name, stat.nr_tried, ulong, 0400); \
+ module_param_named(bytes_##try_name, stat.sz_tried, ulong, \
+ 0400); \
+ module_param_named(nr_##succ_name, stat.nr_applied, ulong, \
+ 0400); \
+ module_param_named(bytes_##succ_name, stat.sz_applied, ulong, \
+ 0400); \
+ module_param_named(nr_##qt_exceed_name, stat.qt_exceeds, ulong, \
+ 0400);
#define DAMON_MAX_SUBSCORE (100)
#define DAMON_MAX_AGE_IN_LOG (32)
-int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
+int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s)
{
unsigned int max_nr_accesses;
unsigned int age_weight = s->quota.weight_age;
int hotness;
- max_nr_accesses = c->aggr_interval / c->sample_interval;
+ max_nr_accesses = c->attrs.aggr_interval / c->attrs.sample_interval;
freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
- age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
+ age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
age_in_log++, age_in_sec >>= 1)
;
*/
hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
- /* Return coldness of the region */
- return DAMOS_MAX_SCORE - hotness;
+ return hotness;
}
-int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
+int damon_cold_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s)
{
- unsigned int max_nr_accesses;
- int freq_subscore;
- unsigned int age_in_sec;
- int age_in_log, age_subscore;
- unsigned int freq_weight = s->quota.weight_nr_accesses;
- unsigned int age_weight = s->quota.weight_age;
- int hotness;
-
- max_nr_accesses = c->aggr_interval / c->sample_interval;
- freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
-
- age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
- for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
- age_in_log++, age_in_sec >>= 1)
- ;
+ int hotness = damon_hot_score(c, r, s);
- /* If frequency is 0, higher age means it's colder */
- if (freq_subscore == 0)
- age_in_log *= -1;
-
- /*
- * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
- * Scale it to be in [0, 100] and set it as age subscore.
- */
- age_in_log += DAMON_MAX_AGE_IN_LOG;
- age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
- DAMON_MAX_AGE_IN_LOG / 2;
-
- hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
- if (freq_weight + age_weight)
- hotness /= freq_weight + age_weight;
- /*
- * Transform it to fit in [0, DAMOS_MAX_SCORE]
- */
- hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
-
- return hotness;
+ /* Return coldness of the region */
+ return DAMOS_MAX_SCORE - hotness;
}
void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr);
void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr);
-int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
+int damon_cold_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s);
int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
struct damos *s);
folio_put(folio);
}
-static void __damon_pa_prepare_access_check(struct damon_ctx *ctx,
- struct damon_region *r)
+static void __damon_pa_prepare_access_check(struct damon_region *r)
{
r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
damon_for_each_target(t, ctx) {
damon_for_each_region(r, t)
- __damon_pa_prepare_access_check(ctx, r);
+ __damon_pa_prepare_access_check(r);
}
}
return result.accessed;
}
-static void __damon_pa_check_access(struct damon_ctx *ctx,
- struct damon_region *r)
+static void __damon_pa_check_access(struct damon_region *r)
{
static unsigned long last_addr;
static unsigned long last_page_sz = PAGE_SIZE;
damon_for_each_target(t, ctx) {
damon_for_each_region(r, t) {
- __damon_pa_check_access(ctx, r);
+ __damon_pa_check_access(r);
max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
}
}
return applied * PAGE_SIZE;
}
-static unsigned long damon_pa_mark_accessed(struct damon_region *r)
+static inline unsigned long damon_pa_mark_accessed_or_deactivate(
+ struct damon_region *r, bool mark_accessed)
{
unsigned long addr, applied = 0;
if (!page)
continue;
- mark_page_accessed(page);
+ if (mark_accessed)
+ mark_page_accessed(page);
+ else
+ deactivate_page(page);
put_page(page);
applied++;
}
return applied * PAGE_SIZE;
}
-static unsigned long damon_pa_deactivate_pages(struct damon_region *r)
+static unsigned long damon_pa_mark_accessed(struct damon_region *r)
{
- unsigned long addr, applied = 0;
-
- for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
- struct page *page = damon_get_page(PHYS_PFN(addr));
+ return damon_pa_mark_accessed_or_deactivate(r, true);
+}
- if (!page)
- continue;
- deactivate_page(page);
- put_page(page);
- applied++;
- }
- return applied * PAGE_SIZE;
+static unsigned long damon_pa_deactivate_pages(struct damon_region *r)
+{
+ return damon_pa_mark_accessed_or_deactivate(r, false);
}
static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
return damon_pa_mark_accessed(r);
case DAMOS_LRU_DEPRIO:
return damon_pa_deactivate_pages(r);
+ case DAMOS_STAT:
+ break;
default:
+ /* DAMOS actions that not yet supported by 'paddr'. */
break;
}
return 0;
{
switch (scheme->action) {
case DAMOS_PAGEOUT:
- return damon_pageout_score(context, r, scheme);
+ return damon_cold_score(context, r, scheme);
case DAMOS_LRU_PRIO:
return damon_hot_score(context, r, scheme);
case DAMOS_LRU_DEPRIO:
- return damon_pageout_score(context, r, scheme);
+ return damon_cold_score(context, r, scheme);
default:
break;
}
#include <linux/sched.h>
#include <linux/workqueue.h>
+#include "modules-common.h"
+
#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
static unsigned long min_age __read_mostly = 120000000;
module_param(min_age, ulong, 0600);
-/*
- * Limit of time for trying the reclamation in milliseconds.
- *
- * DAMON_RECLAIM tries to use only up to this time within a time window
- * (quota_reset_interval_ms) for trying reclamation of cold pages. This can be
- * used for limiting CPU consumption of DAMON_RECLAIM. If the value is zero,
- * the limit is disabled.
- *
- * 10 ms by default.
- */
-static unsigned long quota_ms __read_mostly = 10;
-module_param(quota_ms, ulong, 0600);
-
-/*
- * Limit of size of memory for the reclamation in bytes.
- *
- * DAMON_RECLAIM charges amount of memory which it tried to reclaim within a
- * time window (quota_reset_interval_ms) and makes no more than this limit is
- * tried. This can be used for limiting consumption of CPU and IO. If this
- * value is zero, the limit is disabled.
- *
- * 128 MiB by default.
- */
-static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024;
-module_param(quota_sz, ulong, 0600);
-
-/*
- * The time/size quota charge reset interval in milliseconds.
- *
- * The charge reset interval for the quota of time (quota_ms) and size
- * (quota_sz). That is, DAMON_RECLAIM does not try reclamation for more than
- * quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms
- * milliseconds.
- *
- * 1 second by default.
- */
-static unsigned long quota_reset_interval_ms __read_mostly = 1000;
-module_param(quota_reset_interval_ms, ulong, 0600);
-
-/*
- * The watermarks check time interval in microseconds.
- *
- * Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is
- * enabled but inactive due to its watermarks rule. 5 seconds by default.
- */
-static unsigned long wmarks_interval __read_mostly = 5000000;
-module_param(wmarks_interval, ulong, 0600);
-
-/*
- * Free memory rate (per thousand) for the high watermark.
- *
- * If free memory of the system in bytes per thousand bytes is higher than
- * this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically
- * checks the watermarks. 500 (50%) by default.
- */
-static unsigned long wmarks_high __read_mostly = 500;
-module_param(wmarks_high, ulong, 0600);
-
-/*
- * Free memory rate (per thousand) for the middle watermark.
- *
- * If free memory of the system in bytes per thousand bytes is between this and
- * the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring
- * and the reclaiming. 400 (40%) by default.
- */
-static unsigned long wmarks_mid __read_mostly = 400;
-module_param(wmarks_mid, ulong, 0600);
-
-/*
- * Free memory rate (per thousand) for the low watermark.
- *
- * If free memory of the system in bytes per thousand bytes is lower than this,
- * DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks
- * the watermarks. In the case, the system falls back to the LRU-based page
- * granularity reclamation logic. 200 (20%) by default.
- */
-static unsigned long wmarks_low __read_mostly = 200;
-module_param(wmarks_low, ulong, 0600);
-
-/*
- * Sampling interval for the monitoring in microseconds.
- *
- * The sampling interval of DAMON for the cold memory monitoring. Please refer
- * to the DAMON documentation for more detail. 5 ms by default.
- */
-static unsigned long sample_interval __read_mostly = 5000;
-module_param(sample_interval, ulong, 0600);
-
-/*
- * Aggregation interval for the monitoring in microseconds.
- *
- * The aggregation interval of DAMON for the cold memory monitoring. Please
- * refer to the DAMON documentation for more detail. 100 ms by default.
- */
-static unsigned long aggr_interval __read_mostly = 100000;
-module_param(aggr_interval, ulong, 0600);
-
-/*
- * Minimum number of monitoring regions.
- *
- * The minimal number of monitoring regions of DAMON for the cold memory
- * monitoring. This can be used to set lower-bound of the monitoring quality.
- * But, setting this too high could result in increased monitoring overhead.
- * Please refer to the DAMON documentation for more detail. 10 by default.
- */
-static unsigned long min_nr_regions __read_mostly = 10;
-module_param(min_nr_regions, ulong, 0600);
-
-/*
- * Maximum number of monitoring regions.
- *
- * The maximum number of monitoring regions of DAMON for the cold memory
- * monitoring. This can be used to set upper-bound of the monitoring overhead.
- * However, setting this too low could result in bad monitoring quality.
- * Please refer to the DAMON documentation for more detail. 1000 by default.
- */
-static unsigned long max_nr_regions __read_mostly = 1000;
-module_param(max_nr_regions, ulong, 0600);
+static struct damos_quota damon_reclaim_quota = {
+ /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
+ .ms = 10,
+ .sz = 128 * 1024 * 1024,
+ .reset_interval = 1000,
+ /* Within the quota, page out older regions first. */
+ .weight_sz = 0,
+ .weight_nr_accesses = 0,
+ .weight_age = 1
+};
+DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
+
+static struct damos_watermarks damon_reclaim_wmarks = {
+ .metric = DAMOS_WMARK_FREE_MEM_RATE,
+ .interval = 5000000, /* 5 seconds */
+ .high = 500, /* 50 percent */
+ .mid = 400, /* 40 percent */
+ .low = 200, /* 20 percent */
+};
+DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
+
+static struct damon_attrs damon_reclaim_mon_attrs = {
+ .sample_interval = 5000, /* 5 ms */
+ .aggr_interval = 100000, /* 100 ms */
+ .ops_update_interval = 0,
+ .min_nr_regions = 10,
+ .max_nr_regions = 1000,
+};
+DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
/*
* Start of the target memory region in physical address.
static int kdamond_pid __read_mostly = -1;
module_param(kdamond_pid, int, 0400);
-/*
- * Number of memory regions that tried to be reclaimed.
- */
-static unsigned long nr_reclaim_tried_regions __read_mostly;
-module_param(nr_reclaim_tried_regions, ulong, 0400);
-
-/*
- * Total bytes of memory regions that tried to be reclaimed.
- */
-static unsigned long bytes_reclaim_tried_regions __read_mostly;
-module_param(bytes_reclaim_tried_regions, ulong, 0400);
-
-/*
- * Number of memory regions that successfully be reclaimed.
- */
-static unsigned long nr_reclaimed_regions __read_mostly;
-module_param(nr_reclaimed_regions, ulong, 0400);
-
-/*
- * Total bytes of memory regions that successfully be reclaimed.
- */
-static unsigned long bytes_reclaimed_regions __read_mostly;
-module_param(bytes_reclaimed_regions, ulong, 0400);
-
-/*
- * Number of times that the time/space quota limits have exceeded
- */
-static unsigned long nr_quota_exceeds __read_mostly;
-module_param(nr_quota_exceeds, ulong, 0400);
+static struct damos_stat damon_reclaim_stat;
+DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
+ reclaim_tried_regions, reclaimed_regions, quota_exceeds);
static struct damon_ctx *ctx;
static struct damon_target *target;
-struct damon_reclaim_ram_walk_arg {
- unsigned long start;
- unsigned long end;
-};
-
-static int walk_system_ram(struct resource *res, void *arg)
-{
- struct damon_reclaim_ram_walk_arg *a = arg;
-
- if (a->end - a->start < resource_size(res)) {
- a->start = res->start;
- a->end = res->end;
- }
- return 0;
-}
-
-/*
- * Find biggest 'System RAM' resource and store its start and end address in
- * @start and @end, respectively. If no System RAM is found, returns false.
- */
-static bool get_monitoring_region(unsigned long *start, unsigned long *end)
-{
- struct damon_reclaim_ram_walk_arg arg = {};
-
- walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
- if (arg.end <= arg.start)
- return false;
-
- *start = arg.start;
- *end = arg.end;
- return true;
-}
-
static struct damos *damon_reclaim_new_scheme(void)
{
- struct damos_watermarks wmarks = {
- .metric = DAMOS_WMARK_FREE_MEM_RATE,
- .interval = wmarks_interval,
- .high = wmarks_high,
- .mid = wmarks_mid,
- .low = wmarks_low,
- };
- struct damos_quota quota = {
- /*
- * Do not try reclamation for more than quota_ms milliseconds
- * or quota_sz bytes within quota_reset_interval_ms.
- */
- .ms = quota_ms,
- .sz = quota_sz,
- .reset_interval = quota_reset_interval_ms,
- /* Within the quota, page out older regions first. */
- .weight_sz = 0,
- .weight_nr_accesses = 0,
- .weight_age = 1
+ struct damos_access_pattern pattern = {
+ /* Find regions having PAGE_SIZE or larger size */
+ .min_sz_region = PAGE_SIZE,
+ .max_sz_region = ULONG_MAX,
+ /* and not accessed at all */
+ .min_nr_accesses = 0,
+ .max_nr_accesses = 0,
+ /* for min_age or more micro-seconds */
+ .min_age_region = min_age /
+ damon_reclaim_mon_attrs.aggr_interval,
+ .max_age_region = UINT_MAX,
};
- struct damos *scheme = damon_new_scheme(
- /* Find regions having PAGE_SIZE or larger size */
- PAGE_SIZE, ULONG_MAX,
- /* and not accessed at all */
- 0, 0,
- /* for min_age or more micro-seconds, and */
- min_age / aggr_interval, UINT_MAX,
+
+ return damon_new_scheme(
+ &pattern,
/* page out those, as soon as found */
DAMOS_PAGEOUT,
/* under the quota. */
- "a,
+ &damon_reclaim_quota,
/* (De)activate this according to the watermarks. */
- &wmarks);
-
- return scheme;
+ &damon_reclaim_wmarks);
}
static int damon_reclaim_apply_parameters(void)
{
struct damos *scheme;
- struct damon_addr_range addr_range;
int err = 0;
- err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0,
- min_nr_regions, max_nr_regions);
+ err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
if (err)
return err;
scheme = damon_reclaim_new_scheme();
if (!scheme)
return -ENOMEM;
- err = damon_set_schemes(ctx, &scheme, 1);
- if (err)
- return err;
+ damon_set_schemes(ctx, &scheme, 1);
- if (monitor_region_start > monitor_region_end)
- return -EINVAL;
- if (!monitor_region_start && !monitor_region_end &&
- !get_monitoring_region(&monitor_region_start,
- &monitor_region_end))
- return -EINVAL;
- addr_range.start = monitor_region_start;
- addr_range.end = monitor_region_end;
- return damon_set_regions(target, &addr_range, 1);
+ return damon_set_region_biggest_system_ram_default(target,
+ &monitor_region_start,
+ &monitor_region_end);
}
static int damon_reclaim_turn(bool on)
struct damos *s;
/* update the stats parameter */
- damon_for_each_scheme(s, c) {
- nr_reclaim_tried_regions = s->stat.nr_tried;
- bytes_reclaim_tried_regions = s->stat.sz_tried;
- nr_reclaimed_regions = s->stat.nr_applied;
- bytes_reclaimed_regions = s->stat.sz_applied;
- nr_quota_exceeds = s->stat.qt_exceeds;
- }
+ damon_for_each_scheme(s, c)
+ damon_reclaim_stat = s->stat;
return damon_reclaim_handle_commit_inputs();
}
err = kstrtoul(buf, 0, &min);
if (err)
- return -EINVAL;
+ return err;
range->min = min;
return count;
err = kstrtoul(buf, 0, &max);
if (err)
- return -EINVAL;
+ return err;
range->max = max;
return count;
struct damon_sysfs_watermarks, kobj);
int err = kstrtoul(buf, 0, &watermarks->interval_us);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static ssize_t high_show(struct kobject *kobj,
struct damon_sysfs_watermarks, kobj);
int err = kstrtoul(buf, 0, &watermarks->high);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static ssize_t mid_show(struct kobject *kobj,
struct damon_sysfs_watermarks, kobj);
int err = kstrtoul(buf, 0, &watermarks->mid);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static ssize_t low_show(struct kobject *kobj,
struct damon_sysfs_watermarks, kobj);
int err = kstrtoul(buf, 0, &watermarks->low);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static void damon_sysfs_watermarks_release(struct kobject *kobj)
struct damon_sysfs_weights, kobj);
int err = kstrtouint(buf, 0, &weights->sz);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static ssize_t nr_accesses_permil_show(struct kobject *kobj,
struct damon_sysfs_weights, kobj);
int err = kstrtouint(buf, 0, &weights->nr_accesses);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static ssize_t age_permil_show(struct kobject *kobj,
struct damon_sysfs_weights, kobj);
int err = kstrtouint(buf, 0, &weights->age);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static void damon_sysfs_weights_release(struct kobject *kobj)
static ssize_t nr_schemes_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
- struct damon_sysfs_schemes *schemes = container_of(kobj,
- struct damon_sysfs_schemes, kobj);
+ struct damon_sysfs_schemes *schemes;
int nr, err = kstrtoint(buf, 0, &nr);
if (err)
if (nr < 0)
return -EINVAL;
+ schemes = container_of(kobj, struct damon_sysfs_schemes, kobj);
+
if (!mutex_trylock(&damon_sysfs_lock))
return -EBUSY;
err = damon_sysfs_schemes_add_dirs(schemes, nr);
struct damon_sysfs_region, kobj);
int err = kstrtoul(buf, 0, ®ion->start);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static ssize_t end_show(struct kobject *kobj, struct kobj_attribute *attr,
struct damon_sysfs_region, kobj);
int err = kstrtoul(buf, 0, ®ion->end);
- if (err)
- return -EINVAL;
- return count;
+ return err ? err : count;
}
static void damon_sysfs_region_release(struct kobject *kobj)
static ssize_t nr_regions_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
- struct damon_sysfs_regions *regions = container_of(kobj,
- struct damon_sysfs_regions, kobj);
+ struct damon_sysfs_regions *regions;
int nr, err = kstrtoint(buf, 0, &nr);
if (err)
if (nr < 0)
return -EINVAL;
+ regions = container_of(kobj, struct damon_sysfs_regions, kobj);
+
if (!mutex_trylock(&damon_sysfs_lock))
return -EBUSY;
err = damon_sysfs_regions_add_dirs(regions, nr);
static ssize_t nr_targets_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
- struct damon_sysfs_targets *targets = container_of(kobj,
- struct damon_sysfs_targets, kobj);
+ struct damon_sysfs_targets *targets;
int nr, err = kstrtoint(buf, 0, &nr);
if (err)
if (nr < 0)
return -EINVAL;
+ targets = container_of(kobj, struct damon_sysfs_targets, kobj);
+
if (!mutex_trylock(&damon_sysfs_lock))
return -EBUSY;
err = damon_sysfs_targets_add_dirs(targets, nr);
int err = kstrtoul(buf, 0, &us);
if (err)
- return -EINVAL;
+ return err;
intervals->sample_us = us;
return count;
int err = kstrtoul(buf, 0, &us);
if (err)
- return -EINVAL;
+ return err;
intervals->aggr_us = us;
return count;
int err = kstrtoul(buf, 0, &us);
if (err)
- return -EINVAL;
+ return err;
intervals->update_us = us;
return count;
static ssize_t nr_contexts_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
- struct damon_sysfs_contexts *contexts = container_of(kobj,
- struct damon_sysfs_contexts, kobj);
+ struct damon_sysfs_contexts *contexts;
int nr, err;
err = kstrtoint(buf, 0, &nr);
if (nr < 0 || 1 < nr)
return -EINVAL;
+ contexts = container_of(kobj, struct damon_sysfs_contexts, kobj);
if (!mutex_trylock(&damon_sysfs_lock))
return -EBUSY;
err = damon_sysfs_contexts_add_dirs(contexts, nr);
struct damon_sysfs_intervals *sys_intervals = sys_attrs->intervals;
struct damon_sysfs_ul_range *sys_nr_regions =
sys_attrs->nr_regions_range;
-
- return damon_set_attrs(ctx, sys_intervals->sample_us,
- sys_intervals->aggr_us, sys_intervals->update_us,
- sys_nr_regions->min, sys_nr_regions->max);
+ struct damon_attrs attrs = {
+ .sample_interval = sys_intervals->sample_us,
+ .aggr_interval = sys_intervals->aggr_us,
+ .ops_update_interval = sys_intervals->update_us,
+ .min_nr_regions = sys_nr_regions->min,
+ .max_nr_regions = sys_nr_regions->max,
+ };
+ return damon_set_attrs(ctx, &attrs);
}
static void damon_sysfs_destroy_targets(struct damon_ctx *ctx)
{
struct damon_target *t, *next;
+ bool has_pid = damon_target_has_pid(ctx);
damon_for_each_target_safe(t, next, ctx) {
- if (damon_target_has_pid(ctx))
+ if (has_pid)
put_pid(t->pid);
damon_destroy_target(t);
}
static struct damos *damon_sysfs_mk_scheme(
struct damon_sysfs_scheme *sysfs_scheme)
{
- struct damon_sysfs_access_pattern *pattern =
+ struct damon_sysfs_access_pattern *access_pattern =
sysfs_scheme->access_pattern;
struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
+
+ struct damos_access_pattern pattern = {
+ .min_sz_region = access_pattern->sz->min,
+ .max_sz_region = access_pattern->sz->max,
+ .min_nr_accesses = access_pattern->nr_accesses->min,
+ .max_nr_accesses = access_pattern->nr_accesses->max,
+ .min_age_region = access_pattern->age->min,
+ .max_age_region = access_pattern->age->max,
+ };
struct damos_quota quota = {
.ms = sysfs_quotas->ms,
.sz = sysfs_quotas->sz,
.low = sysfs_wmarks->low,
};
- return damon_new_scheme(pattern->sz->min, pattern->sz->max,
- pattern->nr_accesses->min, pattern->nr_accesses->max,
- pattern->age->min, pattern->age->max,
- sysfs_scheme->action, "a, &wmarks);
+ return damon_new_scheme(&pattern, sysfs_scheme->action, "a,
+ &wmarks);
}
static int damon_sysfs_set_schemes(struct damon_ctx *ctx,
{
struct damon_target *t, *next;
- if (ctx->ops.id != DAMON_OPS_VADDR && ctx->ops.id != DAMON_OPS_FVADDR)
+ if (!damon_target_has_pid(ctx))
return;
mutex_lock(&ctx->kdamond_lock);
struct damon_ctx *ctx;
int err;
- if (kdamond->damon_ctx &&
- damon_sysfs_ctx_running(kdamond->damon_ctx))
+ if (damon_sysfs_kdamond_running(kdamond))
return -EBUSY;
if (damon_sysfs_cmd_request.kdamond == kdamond)
return -EBUSY;
struct damon_sysfs_kdamond *kdamond = container_of(kobj,
struct damon_sysfs_kdamond, kobj);
struct damon_ctx *ctx;
- int pid;
+ int pid = -1;
if (!mutex_trylock(&damon_sysfs_lock))
return -EBUSY;
ctx = kdamond->damon_ctx;
- if (!ctx) {
- pid = -1;
+ if (!ctx)
goto out;
- }
+
mutex_lock(&ctx->kdamond_lock);
- if (!ctx->kdamond)
- pid = -1;
- else
+ if (ctx->kdamond)
pid = ctx->kdamond->pid;
mutex_unlock(&ctx->kdamond_lock);
out:
kdamonds->kdamonds_arr = NULL;
}
-static int damon_sysfs_nr_running_ctxs(struct damon_sysfs_kdamond **kdamonds,
+static bool damon_sysfs_kdamonds_busy(struct damon_sysfs_kdamond **kdamonds,
int nr_kdamonds)
{
- int nr_running_ctxs = 0;
int i;
for (i = 0; i < nr_kdamonds; i++) {
- struct damon_ctx *ctx = kdamonds[i]->damon_ctx;
-
- if (!ctx)
- continue;
- mutex_lock(&ctx->kdamond_lock);
- if (ctx->kdamond)
- nr_running_ctxs++;
- mutex_unlock(&ctx->kdamond_lock);
+ if (damon_sysfs_kdamond_running(kdamonds[i]) ||
+ damon_sysfs_cmd_request.kdamond == kdamonds[i])
+ return true;
}
- return nr_running_ctxs;
+
+ return false;
}
static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds,
struct damon_sysfs_kdamond **kdamonds_arr, *kdamond;
int err, i;
- if (damon_sysfs_nr_running_ctxs(kdamonds->kdamonds_arr, kdamonds->nr))
+ if (damon_sysfs_kdamonds_busy(kdamonds->kdamonds_arr, kdamonds->nr))
return -EBUSY;
- for (i = 0; i < kdamonds->nr; i++) {
- if (damon_sysfs_cmd_request.kdamond ==
- kdamonds->kdamonds_arr[i])
- return -EBUSY;
- }
-
damon_sysfs_kdamonds_rm_dirs(kdamonds);
if (!nr_kdamonds)
return 0;
static ssize_t nr_kdamonds_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
- struct damon_sysfs_kdamonds *kdamonds = container_of(kobj,
- struct damon_sysfs_kdamonds, kobj);
+ struct damon_sysfs_kdamonds *kdamonds;
int nr, err;
err = kstrtoint(buf, 0, &nr);
if (nr < 0)
return -EINVAL;
+ kdamonds = container_of(kobj, struct damon_sysfs_kdamonds, kobj);
+
if (!mutex_trylock(&damon_sysfs_lock))
return -EBUSY;
err = damon_sysfs_kdamonds_add_dirs(kdamonds, nr);
#include <kunit/test.h>
-static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas)
+static void __link_vmas(struct maple_tree *mt, struct vm_area_struct *vmas,
+ ssize_t nr_vmas)
{
- int i, j;
- unsigned long largest_gap, gap;
+ int i;
+ MA_STATE(mas, mt, 0, 0);
if (!nr_vmas)
return;
- for (i = 0; i < nr_vmas - 1; i++) {
- vmas[i].vm_next = &vmas[i + 1];
-
- vmas[i].vm_rb.rb_left = NULL;
- vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb;
-
- largest_gap = 0;
- for (j = i; j < nr_vmas; j++) {
- if (j == 0)
- continue;
- gap = vmas[j].vm_start - vmas[j - 1].vm_end;
- if (gap > largest_gap)
- largest_gap = gap;
- }
- vmas[i].rb_subtree_gap = largest_gap;
- }
- vmas[i].vm_next = NULL;
- vmas[i].vm_rb.rb_right = NULL;
- vmas[i].rb_subtree_gap = 0;
+ mas_lock(&mas);
+ for (i = 0; i < nr_vmas; i++)
+ vma_mas_store(&vmas[i], &mas);
+ mas_unlock(&mas);
}
/*
*/
static void damon_test_three_regions_in_vmas(struct kunit *test)
{
+ static struct mm_struct mm;
struct damon_addr_range regions[3] = {0,};
/* 10-20-25, 200-210-220, 300-305, 307-330 */
struct vm_area_struct vmas[] = {
(struct vm_area_struct) {.vm_start = 307, .vm_end = 330},
};
- __link_vmas(vmas, 6);
+ mt_init_flags(&mm.mm_mt, MM_MT_FLAGS);
+ __link_vmas(&mm.mm_mt, vmas, ARRAY_SIZE(vmas));
- __damon_va_three_regions(&vmas[0], regions);
+ __damon_va_three_regions(&mm, regions);
KUNIT_EXPECT_EQ(test, 10ul, regions[0].start);
KUNIT_EXPECT_EQ(test, 25ul, regions[0].end);
return -EINVAL;
orig_end = r->ar.end;
- sz_orig = r->ar.end - r->ar.start;
+ sz_orig = damon_sz_region(r);
sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION);
if (!sz_piece)
*
* Returns 0 if success, or negative error code otherwise.
*/
-static int __damon_va_three_regions(struct vm_area_struct *vma,
+static int __damon_va_three_regions(struct mm_struct *mm,
struct damon_addr_range regions[3])
{
- struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0};
- struct vm_area_struct *last_vma = NULL;
- unsigned long start = 0;
- struct rb_root rbroot;
-
- /* Find two biggest gaps so that first_gap > second_gap > others */
- for (; vma; vma = vma->vm_next) {
- if (!last_vma) {
- start = vma->vm_start;
- goto next;
- }
+ struct damon_addr_range first_gap = {0}, second_gap = {0};
+ VMA_ITERATOR(vmi, mm, 0);
+ struct vm_area_struct *vma, *prev = NULL;
+ unsigned long start;
- if (vma->rb_subtree_gap <= sz_range(&second_gap)) {
- rbroot.rb_node = &vma->vm_rb;
- vma = rb_entry(rb_last(&rbroot),
- struct vm_area_struct, vm_rb);
+ /*
+ * Find the two biggest gaps so that first_gap > second_gap > others.
+ * If this is too slow, it can be optimised to examine the maple
+ * tree gaps.
+ */
+ for_each_vma(vmi, vma) {
+ unsigned long gap;
+
+ if (!prev) {
+ start = vma->vm_start;
goto next;
}
-
- gap.start = last_vma->vm_end;
- gap.end = vma->vm_start;
- if (sz_range(&gap) > sz_range(&second_gap)) {
- swap(gap, second_gap);
- if (sz_range(&second_gap) > sz_range(&first_gap))
- swap(second_gap, first_gap);
+ gap = vma->vm_start - prev->vm_end;
+
+ if (gap > sz_range(&first_gap)) {
+ second_gap = first_gap;
+ first_gap.start = prev->vm_end;
+ first_gap.end = vma->vm_start;
+ } else if (gap > sz_range(&second_gap)) {
+ second_gap.start = prev->vm_end;
+ second_gap.end = vma->vm_start;
}
next:
- last_vma = vma;
+ prev = vma;
}
if (!sz_range(&second_gap) || !sz_range(&first_gap))
regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION);
regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION);
regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION);
- regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION);
+ regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION);
return 0;
}
return -EINVAL;
mmap_read_lock(mm);
- rc = __damon_va_three_regions(mm->mmap, regions);
+ rc = __damon_va_three_regions(mm, regions);
mmap_read_unlock(mm);
mmput(mm);
for (i = 0; i < 3; i++)
sz += regions[i].end - regions[i].start;
- if (ctx->min_nr_regions)
- sz /= ctx->min_nr_regions;
+ if (ctx->attrs.min_nr_regions)
+ sz /= ctx->attrs.min_nr_regions;
if (sz < DAMON_MIN_REGION)
sz = DAMON_MIN_REGION;
pte_t *pte;
spinlock_t *ptl;
- if (pmd_huge(*pmd)) {
+ if (pmd_trans_huge(*pmd)) {
ptl = pmd_lock(walk->mm, pmd);
- if (pmd_huge(*pmd)) {
+ if (!pmd_present(*pmd)) {
+ spin_unlock(ptl);
+ return 0;
+ }
+
+ if (pmd_trans_huge(*pmd)) {
damon_pmdp_mkold(pmd, walk->mm, addr);
spin_unlock(ptl);
return 0;
* Functions for the access checking of the regions
*/
-static void __damon_va_prepare_access_check(struct damon_ctx *ctx,
- struct mm_struct *mm, struct damon_region *r)
+static void __damon_va_prepare_access_check(struct mm_struct *mm,
+ struct damon_region *r)
{
r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
if (!mm)
continue;
damon_for_each_region(r, t)
- __damon_va_prepare_access_check(ctx, mm, r);
+ __damon_va_prepare_access_check(mm, r);
mmput(mm);
}
}
struct damon_young_walk_private *priv = walk->private;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
- if (pmd_huge(*pmd)) {
+ if (pmd_trans_huge(*pmd)) {
ptl = pmd_lock(walk->mm, pmd);
- if (!pmd_huge(*pmd)) {
+ if (!pmd_present(*pmd)) {
+ spin_unlock(ptl);
+ return 0;
+ }
+
+ if (!pmd_trans_huge(*pmd)) {
spin_unlock(ptl);
goto regular_page;
}
* mm 'mm_struct' for the given virtual address space
* r the region to be checked
*/
-static void __damon_va_check_access(struct damon_ctx *ctx,
- struct mm_struct *mm, struct damon_region *r)
+static void __damon_va_check_access(struct mm_struct *mm,
+ struct damon_region *r, bool same_target)
{
- static struct mm_struct *last_mm;
static unsigned long last_addr;
static unsigned long last_page_sz = PAGE_SIZE;
static bool last_accessed;
/* If the region is in the last checked page, reuse the result */
- if (mm == last_mm && (ALIGN_DOWN(last_addr, last_page_sz) ==
+ if (same_target && (ALIGN_DOWN(last_addr, last_page_sz) ==
ALIGN_DOWN(r->sampling_addr, last_page_sz))) {
if (last_accessed)
r->nr_accesses++;
if (last_accessed)
r->nr_accesses++;
- last_mm = mm;
last_addr = r->sampling_addr;
}
struct mm_struct *mm;
struct damon_region *r;
unsigned int max_nr_accesses = 0;
+ bool same_target;
damon_for_each_target(t, ctx) {
mm = damon_get_mm(t);
if (!mm)
continue;
+ same_target = false;
damon_for_each_region(r, t) {
- __damon_va_check_access(ctx, mm, r);
+ __damon_va_check_access(mm, r, same_target);
max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
+ same_target = true;
}
mmput(mm);
}
* Functions for the target validity check and cleanup
*/
-static bool damon_va_target_valid(void *target)
+static bool damon_va_target_valid(struct damon_target *t)
{
- struct damon_target *t = target;
struct task_struct *task;
task = damon_get_task_struct(t);
{
struct mm_struct *mm;
unsigned long start = PAGE_ALIGN(r->ar.start);
- unsigned long len = PAGE_ALIGN(r->ar.end - r->ar.start);
+ unsigned long len = PAGE_ALIGN(damon_sz_region(r));
unsigned long applied;
mm = damon_get_mm(target);
case DAMOS_STAT:
return 0;
default:
+ /*
+ * DAMOS actions that are not yet supported by 'vaddr'.
+ */
return 0;
}
switch (scheme->action) {
case DAMOS_PAGEOUT:
- return damon_pageout_score(context, r, scheme);
+ return damon_cold_score(context, r, scheme);
default:
break;
}
void dump_vma(const struct vm_area_struct *vma)
{
- pr_emerg("vma %px start %px end %px\n"
- "next %px prev %px mm %px\n"
+ pr_emerg("vma %px start %px end %px mm %px\n"
"prot %lx anon_vma %px vm_ops %px\n"
"pgoff %lx file %px private_data %px\n"
"flags: %#lx(%pGv)\n",
- vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_next,
- vma->vm_prev, vma->vm_mm,
+ vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm,
(unsigned long)pgprot_val(vma->vm_page_prot),
vma->anon_vma, vma->vm_ops, vma->vm_pgoff,
vma->vm_file, vma->vm_private_data,
void dump_mm(const struct mm_struct *mm)
{
- pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n"
+ pr_emerg("mm %px task_size %lu\n"
#ifdef CONFIG_MMU
"get_unmapped_area %px\n"
#endif
- "mmap_base %lu mmap_legacy_base %lu highest_vm_end %lu\n"
+ "mmap_base %lu mmap_legacy_base %lu\n"
"pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n"
"hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n"
"pinned_vm %llx data_vm %lx exec_vm %lx stack_vm %lx\n"
"tlb_flush_pending %d\n"
"def_flags: %#lx(%pGv)\n",
- mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size,
+ mm, mm->task_size,
#ifdef CONFIG_MMU
mm->get_unmapped_area,
#endif
- mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end,
+ mm->mmap_base, mm->mmap_legacy_base,
mm->pgd, atomic_read(&mm->mm_users),
atomic_read(&mm->mm_count),
mm_pgtables_bytes(mm),
{
XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
pgoff_t max = end_byte >> PAGE_SHIFT;
- struct page *page;
+ struct folio *folio;
if (end_byte < start_byte)
return false;
rcu_read_lock();
- xas_for_each(&xas, page, max) {
- if (xas_retry(&xas, page))
+ xas_for_each(&xas, folio, max) {
+ if (xas_retry(&xas, folio))
continue;
- if (xa_is_value(page))
+ if (xa_is_value(folio))
continue;
- if (PageDirty(page) || PageLocked(page) || PageWriteback(page))
+ if (folio_test_dirty(folio) || folio_test_locked(folio) ||
+ folio_test_writeback(folio))
break;
}
rcu_read_unlock();
- return page != NULL;
+ return folio != NULL;
}
EXPORT_SYMBOL_GPL(filemap_range_has_writeback);
struct wait_page_queue wait_page;
wait_queue_entry_t *wait = &wait_page.wait;
bool thrashing = false;
- bool delayacct = false;
unsigned long pflags;
+ bool in_thrashing;
if (bit_nr == PG_locked &&
!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
- if (!folio_test_swapbacked(folio)) {
- delayacct_thrashing_start();
- delayacct = true;
- }
+ delayacct_thrashing_start(&in_thrashing);
psi_memstall_enter(&pflags);
thrashing = true;
}
finish_wait(q, wait);
if (thrashing) {
- if (delayacct)
- delayacct_thrashing_end();
+ delayacct_thrashing_end(&in_thrashing);
psi_memstall_leave(&pflags);
}
struct wait_page_queue wait_page;
wait_queue_entry_t *wait = &wait_page.wait;
bool thrashing = false;
- bool delayacct = false;
unsigned long pflags;
+ bool in_thrashing;
wait_queue_head_t *q;
struct folio *folio = page_folio(pfn_swap_entry_to_page(entry));
q = folio_waitqueue(folio);
if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
- if (!folio_test_swapbacked(folio)) {
- delayacct_thrashing_start();
- delayacct = true;
- }
+ delayacct_thrashing_start(&in_thrashing);
psi_memstall_enter(&pflags);
thrashing = true;
}
finish_wait(q, wait);
if (thrashing) {
- if (delayacct)
- delayacct_thrashing_end();
+ delayacct_thrashing_end(&in_thrashing);
psi_memstall_leave(&pflags);
}
}
*
* Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal.
*/
-int folio_put_wait_locked(struct folio *folio, int state)
+static int folio_put_wait_locked(struct folio *folio, int state)
{
return folio_wait_bit_common(folio, PG_locked, state, DROP);
}
*/
void page_endio(struct page *page, bool is_write, int err)
{
+ struct folio *folio = page_folio(page);
+
if (!is_write) {
if (!err) {
- SetPageUptodate(page);
+ folio_mark_uptodate(folio);
} else {
- ClearPageUptodate(page);
- SetPageError(page);
+ folio_clear_uptodate(folio);
+ folio_set_error(folio);
}
- unlock_page(page);
+ folio_unlock(folio);
} else {
if (err) {
struct address_space *mapping;
- SetPageError(page);
- mapping = page_mapping(page);
+ folio_set_error(folio);
+ mapping = folio_mapping(folio);
if (mapping)
mapping_set_error(mapping, err);
}
- end_page_writeback(page);
+ folio_end_writeback(folio);
}
}
EXPORT_SYMBOL_GPL(page_endio);
}
/**
- * find_get_pages_contig - gang contiguous pagecache lookup
+ * filemap_get_folios_contig - Get a batch of contiguous folios
* @mapping: The address_space to search
- * @index: The starting page index
- * @nr_pages: The maximum number of pages
- * @pages: Where the resulting pages are placed
+ * @start: The starting page index
+ * @end: The final page index (inclusive)
+ * @fbatch: The batch to fill
*
- * find_get_pages_contig() works exactly like find_get_pages_range(),
- * except that the returned number of pages are guaranteed to be
- * contiguous.
+ * filemap_get_folios_contig() works exactly like filemap_get_folios(),
+ * except the returned folios are guaranteed to be contiguous. This may
+ * not return all contiguous folios if the batch gets filled up.
*
- * Return: the number of pages which were found.
+ * Return: The number of folios found.
+ * Also update @start to be positioned for traversal of the next folio.
*/
-unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
- unsigned int nr_pages, struct page **pages)
+
+unsigned filemap_get_folios_contig(struct address_space *mapping,
+ pgoff_t *start, pgoff_t end, struct folio_batch *fbatch)
{
- XA_STATE(xas, &mapping->i_pages, index);
+ XA_STATE(xas, &mapping->i_pages, *start);
+ unsigned long nr;
struct folio *folio;
- unsigned int ret = 0;
-
- if (unlikely(!nr_pages))
- return 0;
rcu_read_lock();
- for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
+
+ for (folio = xas_load(&xas); folio && xas.xa_index <= end;
+ folio = xas_next(&xas)) {
if (xas_retry(&xas, folio))
continue;
/*
* No current caller is looking for DAX entries.
*/
if (xa_is_value(folio))
- break;
+ goto update_start;
if (!folio_try_get_rcu(folio))
goto retry;
if (unlikely(folio != xas_reload(&xas)))
- goto put_page;
+ goto put_folio;
-again:
- pages[ret] = folio_file_page(folio, xas.xa_index);
- if (++ret == nr_pages)
- break;
- if (folio_more_pages(folio, xas.xa_index, ULONG_MAX)) {
- xas.xa_index++;
- folio_ref_inc(folio);
- goto again;
+ if (!folio_batch_add(fbatch, folio)) {
+ nr = folio_nr_pages(folio);
+
+ if (folio_test_hugetlb(folio))
+ nr = 1;
+ *start = folio->index + nr;
+ goto out;
}
continue;
-put_page:
+put_folio:
folio_put(folio);
+
retry:
xas_reset(&xas);
}
+
+update_start:
+ nr = folio_batch_count(fbatch);
+
+ if (nr) {
+ folio = fbatch->folios[nr - 1];
+ if (folio_test_hugetlb(folio))
+ *start = folio->index + 1;
+ else
+ *start = folio->index + folio_nr_pages(folio);
+ }
+out:
rcu_read_unlock();
- return ret;
+ return folio_batch_count(fbatch);
}
-EXPORT_SYMBOL(find_get_pages_contig);
+EXPORT_SYMBOL(filemap_get_folios_contig);
/**
* find_get_pages_range_tag - Find and return head pages matching @tag.
unsigned long offset; /* Offset into pagecache page */
unsigned long bytes; /* Bytes to write to page */
size_t copied; /* Bytes copied from user */
- void *fsdata;
+ void *fsdata = NULL;
offset = (pos & (PAGE_SIZE - 1));
bytes = min_t(unsigned long, PAGE_SIZE - offset,
}
EXPORT_SYMBOL(lru_cache_add);
+void lru_cache_add_inactive_or_unevictable(struct page *page,
+ struct vm_area_struct *vma)
+{
+ folio_add_lru_vma(page_folio(page), vma);
+}
+
int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
pgoff_t index, gfp_t gfp)
{
else
folio_ref_add(folio,
refs * (GUP_PIN_COUNTING_BIAS - 1));
+ /*
+ * Adjust the pincount before re-checking the PTE for changes.
+ * This is essentially a smp_mb() and is paired with a memory
+ * barrier in page_try_share_anon_rmap().
+ */
+ smp_mb__after_atomic();
+
node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
return folio;
if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
(FOLL_PIN | FOLL_GET)))
return ERR_PTR(-EINVAL);
+
+ /*
+ * Considering PTE level hugetlb, like continuous-PTE hugetlb on
+ * ARM64 architecture.
+ */
+ if (is_vm_hugetlb_page(vma)) {
+ page = follow_huge_pmd_pte(vma, address, flags);
+ if (page)
+ return page;
+ return no_page_table(vma, flags);
+ }
+
retry:
if (unlikely(pmd_bad(*pmd)))
return no_page_table(vma, flags);
migration_entry_wait(mm, pmd, address);
goto retry;
}
- if ((flags & FOLL_NUMA) && pte_protnone(pte))
+ if (pte_protnone(pte) && !gup_can_follow_protnone(flags))
goto no_page;
page = vm_normal_page(vma, address, pte);
if (pmd_none(pmdval))
return no_page_table(vma, flags);
if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
- page = follow_huge_pmd(mm, address, pmd, flags);
+ page = follow_huge_pmd_pte(vma, address, flags);
if (page)
return page;
return no_page_table(vma, flags);
if (likely(!pmd_trans_huge(pmdval)))
return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
- if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
+ if (pmd_protnone(pmdval) && !gup_can_follow_protnone(flags))
return no_page_table(vma, flags);
retry_locked:
VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
- /*
- * If FOLL_FORCE is set then do not force a full fault as the hinting
- * fault information is unrelated to the reference behaviour of a task
- * using the address space
- */
- if (!(gup_flags & FOLL_FORCE))
- gup_flags |= FOLL_NUMA;
-
do {
struct page *page;
unsigned int foll_flags = gup_flags;
if (!locked) {
locked = 1;
mmap_read_lock(mm);
- vma = find_vma(mm, nstart);
+ vma = find_vma_intersection(mm, nstart, end);
} else if (nstart >= vma->vm_end)
- vma = vma->vm_next;
- if (!vma || vma->vm_start >= end)
+ vma = find_vma_intersection(mm, vma->vm_end, end);
+
+ if (!vma)
break;
/*
* Set [nstart; nend) to intersection of desired address
#ifdef CONFIG_MIGRATION
/*
- * Check whether all pages are pinnable, if so return number of pages. If some
- * pages are not pinnable, migrate them, and unpin all pages. Return zero if
- * pages were migrated, or if some pages were not successfully isolated.
- * Return negative error if migration fails.
+ * Returns the number of collected pages. Return value is always >= 0.
*/
-static long check_and_migrate_movable_pages(unsigned long nr_pages,
- struct page **pages,
- unsigned int gup_flags)
+static unsigned long collect_longterm_unpinnable_pages(
+ struct list_head *movable_page_list,
+ unsigned long nr_pages,
+ struct page **pages)
{
- unsigned long isolation_error_count = 0, i;
+ unsigned long i, collected = 0;
struct folio *prev_folio = NULL;
- LIST_HEAD(movable_page_list);
- bool drain_allow = true, coherent_pages = false;
- int ret = 0;
+ bool drain_allow = true;
for (i = 0; i < nr_pages; i++) {
struct folio *folio = page_folio(pages[i]);
continue;
prev_folio = folio;
- /*
- * Device coherent pages are managed by a driver and should not
- * be pinned indefinitely as it prevents the driver moving the
- * page. So when trying to pin with FOLL_LONGTERM instead try
- * to migrate the page out of device memory.
- */
- if (folio_is_device_coherent(folio)) {
- /*
- * We always want a new GUP lookup with device coherent
- * pages.
- */
- pages[i] = 0;
- coherent_pages = true;
-
- /*
- * Migration will fail if the page is pinned, so convert
- * the pin on the source page to a normal reference.
- */
- if (gup_flags & FOLL_PIN) {
- get_page(&folio->page);
- unpin_user_page(&folio->page);
- }
+ if (folio_is_longterm_pinnable(folio))
+ continue;
- ret = migrate_device_coherent_page(&folio->page);
- if (ret)
- goto unpin_pages;
+ collected++;
+ if (folio_is_device_coherent(folio))
continue;
- }
- if (folio_is_longterm_pinnable(folio))
- continue;
- /*
- * Try to move out any movable page before pinning the range.
- */
if (folio_test_hugetlb(folio)) {
- if (isolate_hugetlb(&folio->page,
- &movable_page_list))
- isolation_error_count++;
+ isolate_hugetlb(&folio->page, movable_page_list);
continue;
}
drain_allow = false;
}
- if (folio_isolate_lru(folio)) {
- isolation_error_count++;
+ if (!folio_isolate_lru(folio))
continue;
- }
- list_add_tail(&folio->lru, &movable_page_list);
+
+ list_add_tail(&folio->lru, movable_page_list);
node_stat_mod_folio(folio,
NR_ISOLATED_ANON + folio_is_file_lru(folio),
folio_nr_pages(folio));
}
- if (!list_empty(&movable_page_list) || isolation_error_count ||
- coherent_pages)
- goto unpin_pages;
+ return collected;
+}
- /*
- * If list is empty, and no isolation errors, means that all pages are
- * in the correct zone.
- */
- return nr_pages;
+/*
+ * Unpins all pages and migrates device coherent pages and movable_page_list.
+ * Returns -EAGAIN if all pages were successfully migrated or -errno for failure
+ * (or partial success).
+ */
+static int migrate_longterm_unpinnable_pages(
+ struct list_head *movable_page_list,
+ unsigned long nr_pages,
+ struct page **pages)
+{
+ int ret;
+ unsigned long i;
-unpin_pages:
- /*
- * pages[i] might be NULL if any device coherent pages were found.
- */
for (i = 0; i < nr_pages; i++) {
- if (!pages[i])
+ struct folio *folio = page_folio(pages[i]);
+
+ if (folio_is_device_coherent(folio)) {
+ /*
+ * Migration will fail if the page is pinned, so convert
+ * the pin on the source page to a normal reference.
+ */
+ pages[i] = NULL;
+ folio_get(folio);
+ gup_put_folio(folio, 1, FOLL_PIN);
+
+ if (migrate_device_coherent_page(&folio->page)) {
+ ret = -EBUSY;
+ goto err;
+ }
+
continue;
+ }
- if (gup_flags & FOLL_PIN)
- unpin_user_page(pages[i]);
- else
- put_page(pages[i]);
+ /*
+ * We can't migrate pages with unexpected references, so drop
+ * the reference obtained by __get_user_pages_locked().
+ * Migrating pages have been added to movable_page_list after
+ * calling folio_isolate_lru() which takes a reference so the
+ * page won't be freed if it's migrating.
+ */
+ unpin_user_page(pages[i]);
+ pages[i] = NULL;
}
- if (!list_empty(&movable_page_list)) {
+ if (!list_empty(movable_page_list)) {
struct migration_target_control mtc = {
.nid = NUMA_NO_NODE,
.gfp_mask = GFP_USER | __GFP_NOWARN,
};
- ret = migrate_pages(&movable_page_list, alloc_migration_target,
- NULL, (unsigned long)&mtc, MIGRATE_SYNC,
- MR_LONGTERM_PIN, NULL);
- if (ret > 0) /* number of pages not migrated */
+ if (migrate_pages(movable_page_list, alloc_migration_target,
+ NULL, (unsigned long)&mtc, MIGRATE_SYNC,
+ MR_LONGTERM_PIN, NULL)) {
ret = -ENOMEM;
+ goto err;
+ }
}
- if (ret && !list_empty(&movable_page_list))
- putback_movable_pages(&movable_page_list);
+ putback_movable_pages(movable_page_list);
+
+ return -EAGAIN;
+
+err:
+ for (i = 0; i < nr_pages; i++)
+ if (pages[i])
+ unpin_user_page(pages[i]);
+ putback_movable_pages(movable_page_list);
+
return ret;
}
+
+/*
+ * Check whether all pages are *allowed* to be pinned. Rather confusingly, all
+ * pages in the range are required to be pinned via FOLL_PIN, before calling
+ * this routine.
+ *
+ * If any pages in the range are not allowed to be pinned, then this routine
+ * will migrate those pages away, unpin all the pages in the range and return
+ * -EAGAIN. The caller should re-pin the entire range with FOLL_PIN and then
+ * call this routine again.
+ *
+ * If an error other than -EAGAIN occurs, this indicates a migration failure.
+ * The caller should give up, and propagate the error back up the call stack.
+ *
+ * If everything is OK and all pages in the range are allowed to be pinned, then
+ * this routine leaves all pages pinned and returns zero for success.
+ */
+static long check_and_migrate_movable_pages(unsigned long nr_pages,
+ struct page **pages)
+{
+ unsigned long collected;
+ LIST_HEAD(movable_page_list);
+
+ collected = collect_longterm_unpinnable_pages(&movable_page_list,
+ nr_pages, pages);
+ if (!collected)
+ return 0;
+
+ return migrate_longterm_unpinnable_pages(&movable_page_list, nr_pages,
+ pages);
+}
#else
static long check_and_migrate_movable_pages(unsigned long nr_pages,
- struct page **pages,
- unsigned int gup_flags)
+ struct page **pages)
{
- return nr_pages;
+ return 0;
}
#endif /* CONFIG_MIGRATION */
unsigned int gup_flags)
{
unsigned int flags;
- long rc;
+ long rc, nr_pinned_pages;
if (!(gup_flags & FOLL_LONGTERM))
return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
NULL, gup_flags);
+
+ /*
+ * If we get to this point then FOLL_LONGTERM is set, and FOLL_LONGTERM
+ * implies FOLL_PIN (although the reverse is not true). Therefore it is
+ * correct to unconditionally call check_and_migrate_movable_pages()
+ * which assumes pages have been pinned via FOLL_PIN.
+ *
+ * Enforce the above reasoning by asserting that FOLL_PIN is set.
+ */
+ if (WARN_ON(!(gup_flags & FOLL_PIN)))
+ return -EINVAL;
flags = memalloc_pin_save();
do {
- rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
- NULL, gup_flags);
- if (rc <= 0)
+ nr_pinned_pages = __get_user_pages_locked(mm, start, nr_pages,
+ pages, vmas, NULL,
+ gup_flags);
+ if (nr_pinned_pages <= 0) {
+ rc = nr_pinned_pages;
break;
- rc = check_and_migrate_movable_pages(rc, pages, gup_flags);
- } while (!rc);
+ }
+ rc = check_and_migrate_movable_pages(nr_pinned_pages, pages);
+ } while (rc == -EAGAIN);
memalloc_pin_restore(flags);
- return rc;
+ return rc ? rc : nr_pinned_pages;
}
static bool is_valid_gup_flags(unsigned int gup_flags)
struct page *page;
struct folio *folio;
- /*
- * Similar to the PMD case below, NUMA hinting must take slow
- * path using the pte_protnone check.
- */
- if (pte_protnone(pte))
+ if (pte_protnone(pte) && !gup_can_follow_protnone(flags))
goto pte_unmap;
if (!pte_access_permitted(pte, flags & FOLL_WRITE))
if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
pmd_devmap(pmd))) {
- /*
- * NUMA hinting faults need to be handled in the GUP
- * slowpath for accounting purposes and so that they
- * can be serialised against THP migration.
- */
- if (pmd_protnone(pmd))
+ if (pmd_protnone(pmd) &&
+ !gup_can_follow_protnone(flags))
return 0;
if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
#include <asm/tlbflush.h>
#include <linux/vmalloc.h>
+#ifdef CONFIG_KMAP_LOCAL
+static inline int kmap_local_calc_idx(int idx)
+{
+ return idx + KM_MAX_IDX * smp_processor_id();
+}
+
+#ifndef arch_kmap_local_map_idx
+#define arch_kmap_local_map_idx(idx, pfn) kmap_local_calc_idx(idx)
+#endif
+#endif /* CONFIG_KMAP_LOCAL */
+
/*
* Virtual_count is not a pure "count".
* 0 means that it is not mapped, and has not been mapped
struct page *__kmap_to_page(void *vaddr)
{
+ unsigned long base = (unsigned long) vaddr & PAGE_MASK;
+ struct kmap_ctrl *kctrl = ¤t->kmap_ctrl;
unsigned long addr = (unsigned long)vaddr;
+ int i;
+
+ /* kmap() mappings */
+ if (WARN_ON_ONCE(addr >= PKMAP_ADDR(0) &&
+ addr < PKMAP_ADDR(LAST_PKMAP)))
+ return pte_page(pkmap_page_table[PKMAP_NR(addr)]);
- if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) {
- int i = PKMAP_NR(addr);
+ /* kmap_local_page() mappings */
+ if (WARN_ON_ONCE(base >= __fix_to_virt(FIX_KMAP_END) &&
+ base < __fix_to_virt(FIX_KMAP_BEGIN))) {
+ for (i = 0; i < kctrl->idx; i++) {
+ unsigned long base_addr;
+ int idx;
- return pte_page(pkmap_page_table[i]);
+ idx = arch_kmap_local_map_idx(i, pte_pfn(pteval));
+ base_addr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+
+ if (base_addr == base)
+ return pte_page(kctrl->pteval[i]);
+ }
}
return virt_to_page(vaddr);
# define arch_kmap_local_post_unmap(vaddr) do { } while (0)
#endif
-#ifndef arch_kmap_local_map_idx
-#define arch_kmap_local_map_idx(idx, pfn) kmap_local_calc_idx(idx)
-#endif
-
#ifndef arch_kmap_local_unmap_idx
#define arch_kmap_local_unmap_idx(idx, vaddr) kmap_local_calc_idx(idx)
#endif
return false;
}
-static inline int kmap_local_calc_idx(int idx)
-{
- return idx + KM_MAX_IDX * smp_processor_id();
-}
-
static pte_t *__kmap_pte;
static pte_t *kmap_get_pte(unsigned long vaddr, int idx)
cpu_flags = HMM_PFN_VALID;
if (is_writable_device_private_entry(entry))
cpu_flags |= HMM_PFN_WRITE;
- *hmm_pfn = swp_offset(entry) | cpu_flags;
+ *hmm_pfn = swp_offset_pfn(entry) | cpu_flags;
return 0;
}
#include <linux/numa.h>
#include <linux/page_owner.h>
#include <linux/sched/sysctl.h>
+#include <linux/memory-tiers.h>
#include <asm/tlb.h>
#include <asm/pgalloc.h>
struct page *huge_zero_page __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
-bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags,
- bool smaps, bool in_pf)
+bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags,
+ bool smaps, bool in_pf, bool enforce_sysfs)
{
if (!vma->vm_mm) /* vdso */
return false;
* own flags.
*/
if (!in_pf && shmem_file(vma->vm_file))
- return shmem_huge_enabled(vma);
+ return shmem_huge_enabled(vma, !enforce_sysfs);
- if (!hugepage_flags_enabled())
- return false;
-
- /* THP settings require madvise. */
- if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always())
+ /* Enforce sysfs THP requirements as necessary */
+ if (enforce_sysfs &&
+ (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
+ !hugepage_flags_always())))
return false;
/* Only regular file is valid */
count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
return false;
}
- count_vm_event(THP_ZERO_PAGE_ALLOC);
preempt_disable();
if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
preempt_enable();
/* We take additional reference here. It will be put back by shrinker */
atomic_set(&huge_zero_refcount, 2);
preempt_enable();
+ count_vm_event(THP_ZERO_PAGE_ALLOC);
return true;
}
return;
entry = mk_pmd(zero_page, vma->vm_page_prot);
entry = pmd_mkhuge(entry);
- if (pgtable)
- pgtable_trans_huge_deposit(mm, pmd, pgtable);
+ pgtable_trans_huge_deposit(mm, pmd, pgtable);
set_pmd_at(mm, haddr, pmd, entry);
mm_inc_nr_ptes(mm);
}
{
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
struct vm_area_struct *vma = vmf->vma;
+ struct folio *folio;
struct page *page;
unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
pmd_t orig_pmd = vmf->orig_pmd;
}
page = pmd_page(orig_pmd);
+ folio = page_folio(page);
VM_BUG_ON_PAGE(!PageHead(page), page);
/* Early check when only holding the PT lock. */
if (PageAnonExclusive(page))
goto reuse;
- if (!trylock_page(page)) {
- get_page(page);
+ if (!folio_trylock(folio)) {
+ folio_get(folio);
spin_unlock(vmf->ptl);
- lock_page(page);
+ folio_lock(folio);
spin_lock(vmf->ptl);
if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
spin_unlock(vmf->ptl);
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
return 0;
}
- put_page(page);
+ folio_put(folio);
}
/* Recheck after temporarily dropping the PT lock. */
if (PageAnonExclusive(page)) {
- unlock_page(page);
+ folio_unlock(folio);
goto reuse;
}
/*
- * See do_wp_page(): we can only reuse the page exclusively if there are
- * no additional references. Note that we always drain the LRU
- * pagevecs immediately after adding a THP.
+ * See do_wp_page(): we can only reuse the folio exclusively if
+ * there are no additional references. Note that we always drain
+ * the LRU pagevecs immediately after adding a THP.
*/
- if (page_count(page) > 1 + PageSwapCache(page) * thp_nr_pages(page))
+ if (folio_ref_count(folio) >
+ 1 + folio_test_swapcache(folio) * folio_nr_pages(folio))
goto unlock_fallback;
- if (PageSwapCache(page))
- try_to_free_swap(page);
- if (page_count(page) == 1) {
+ if (folio_test_swapcache(folio))
+ folio_free_swap(folio);
+ if (folio_ref_count(folio) == 1) {
pmd_t entry;
page_move_anon_rmap(page, vma);
- unlock_page(page);
+ folio_unlock(folio);
reuse:
if (unlikely(unshare)) {
spin_unlock(vmf->ptl);
}
unlock_fallback:
- unlock_page(page);
+ folio_unlock(folio);
spin_unlock(vmf->ptl);
fallback:
__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
return ERR_PTR(-EFAULT);
/* Full NUMA hinting faults to serialise migration in fault paths */
- if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
+ if (pmd_protnone(*pmd) && !gup_can_follow_protnone(flags))
return NULL;
if (!pmd_write(*pmd) && gup_must_unshare(flags, page))
struct page *page;
unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
int page_nid = NUMA_NO_NODE;
- int target_nid, last_cpupid = -1;
+ int target_nid, last_cpupid = (-1 & LAST_CPUPID_MASK);
bool migrated = false;
bool was_writable = pmd_savedwrite(oldpmd);
int flags = 0;
flags |= TNF_NO_GROUP;
page_nid = page_to_nid(page);
- last_cpupid = page_cpupid_last(page);
+ /*
+ * For memory tiering mode, cpupid of slow memory page is used
+ * to record page access time. So use default value.
+ */
+ if (node_is_toptier(page_nid))
+ last_cpupid = page_cpupid_last(page);
target_nid = numa_migrate_prep(page, vma, haddr, page_nid,
&flags);
if (prot_numa) {
struct page *page;
+ bool toptier;
/*
* Avoid trapping faults against the zero page. The read-only
* data is likely to be read-cached on the local CPU and
goto unlock;
page = pmd_page(*pmd);
+ toptier = node_is_toptier(page_to_nid(page));
/*
* Skip scanning top tier node if normal numa
* balancing is disabled
*/
if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
- node_is_toptier(page_to_nid(page)))
+ toptier)
goto unlock;
+
+ if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
+ !toptier)
+ xchg_page_access_time(page, jiffies_to_msecs(jiffies));
}
/*
* In case prot_numa, we are under mmap_read_lock(mm). It's critical
pgtable_t pgtable;
pmd_t old_pmd, _pmd;
bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false;
- bool anon_exclusive = false;
+ bool anon_exclusive = false, dirty = false;
unsigned long addr;
int i;
write = is_writable_migration_entry(entry);
if (PageAnon(page))
anon_exclusive = is_readable_exclusive_migration_entry(entry);
- young = false;
+ young = is_migration_entry_young(entry);
+ dirty = is_migration_entry_dirty(entry);
soft_dirty = pmd_swp_soft_dirty(old_pmd);
uffd_wp = pmd_swp_uffd_wp(old_pmd);
} else {
page = pmd_page(old_pmd);
- if (pmd_dirty(old_pmd))
+ if (pmd_dirty(old_pmd)) {
+ dirty = true;
SetPageDirty(page);
+ }
write = pmd_write(old_pmd);
young = pmd_young(old_pmd);
soft_dirty = pmd_soft_dirty(old_pmd);
*
* In case we cannot clear PageAnonExclusive(), split the PMD
* only and let try_to_migrate_one() fail later.
+ *
+ * See page_try_share_anon_rmap(): invalidate PMD first.
*/
anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
if (freeze && anon_exclusive && page_try_share_anon_rmap(page))
else
swp_entry = make_readable_migration_entry(
page_to_pfn(page + i));
+ if (young)
+ swp_entry = make_migration_entry_young(swp_entry);
+ if (dirty)
+ swp_entry = make_migration_entry_dirty(swp_entry);
entry = swp_entry_to_pte(swp_entry);
if (soft_dirty)
entry = pte_swp_mksoft_dirty(entry);
entry = pte_wrprotect(entry);
if (!young)
entry = pte_mkold(entry);
+ /* NOTE: this may set soft-dirty too on some archs */
+ if (dirty)
+ entry = pte_mkdirty(entry);
if (soft_dirty)
entry = pte_mksoft_dirty(entry);
if (uffd_wp)
void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
bool freeze, struct folio *folio)
{
- pgd_t *pgd;
- p4d_t *p4d;
- pud_t *pud;
- pmd_t *pmd;
+ pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);
- pgd = pgd_offset(vma->vm_mm, address);
- if (!pgd_present(*pgd))
+ if (!pmd)
return;
- p4d = p4d_offset(pgd, address);
- if (!p4d_present(*p4d))
- return;
-
- pud = pud_offset(p4d, address);
- if (!pud_present(*pud))
- return;
-
- pmd = pmd_offset(pud, address);
-
__split_huge_pmd(vma, pmd, address, freeze, folio);
}
split_huge_pmd_if_needed(vma, end);
/*
- * If we're also updating the vma->vm_next->vm_start,
+ * If we're also updating the next vma vm_start,
* check if we need to split it.
*/
if (adjust_next > 0) {
- struct vm_area_struct *next = vma->vm_next;
+ struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end);
unsigned long nstart = next->vm_start;
nstart += adjust_next;
split_huge_pmd_if_needed(next, nstart);
}
}
-static void unmap_page(struct page *page)
+static void unmap_folio(struct folio *folio)
{
- struct folio *folio = page_folio(page);
enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
TTU_SYNC;
- VM_BUG_ON_PAGE(!PageHead(page), page);
+ VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
/*
* Anon pages need migration entries to preserve them, but file
{
int i = 0;
- /* If unmap_page() uses try_to_migrate() on file, remove this check */
+ /* If unmap_folio() uses try_to_migrate() on file, remove this check */
if (!folio_test_anon(folio))
return;
for (;;) {
* for example lock_page() which set PG_waiters.
*
* Note that for mapped sub-pages of an anonymous THP,
- * PG_anon_exclusive has been cleared in unmap_page() and is stored in
+ * PG_anon_exclusive has been cleared in unmap_folio() and is stored in
* the migration entry instead from where remap_page() will restore it.
* We can still have PG_anon_exclusive set on effectively unmapped and
* unreferenced sub-pages of an anonymous THP: we can simply drop
#ifdef CONFIG_64BIT
(1L << PG_arch_2) |
#endif
- (1L << PG_dirty)));
+ (1L << PG_dirty) |
+ LRU_GEN_MASK | LRU_REFS_MASK));
/* ->mapping in first tail page is compound_mapcount */
VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
int split_huge_page_to_list(struct page *page, struct list_head *list)
{
struct folio *folio = page_folio(page);
- struct page *head = &folio->page;
- struct deferred_split *ds_queue = get_deferred_split_queue(head);
- XA_STATE(xas, &head->mapping->i_pages, head->index);
+ struct deferred_split *ds_queue = get_deferred_split_queue(&folio->page);
+ XA_STATE(xas, &folio->mapping->i_pages, folio->index);
struct anon_vma *anon_vma = NULL;
struct address_space *mapping = NULL;
int extra_pins, ret;
pgoff_t end;
bool is_hzp;
- VM_BUG_ON_PAGE(!PageLocked(head), head);
- VM_BUG_ON_PAGE(!PageCompound(head), head);
+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
+ VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
- is_hzp = is_huge_zero_page(head);
- VM_WARN_ON_ONCE_PAGE(is_hzp, head);
+ is_hzp = is_huge_zero_page(&folio->page);
+ VM_WARN_ON_ONCE_FOLIO(is_hzp, folio);
if (is_hzp)
return -EBUSY;
- if (PageWriteback(head))
+ if (folio_test_writeback(folio))
return -EBUSY;
- if (PageAnon(head)) {
+ if (folio_test_anon(folio)) {
/*
* The caller does not necessarily hold an mmap_lock that would
* prevent the anon_vma disappearing so we first we take a
* is taken to serialise against parallel split or collapse
* operations.
*/
- anon_vma = page_get_anon_vma(head);
+ anon_vma = folio_get_anon_vma(folio);
if (!anon_vma) {
ret = -EBUSY;
goto out;
mapping = NULL;
anon_vma_lock_write(anon_vma);
} else {
- mapping = head->mapping;
+ gfp_t gfp;
+
+ mapping = folio->mapping;
/* Truncated ? */
if (!mapping) {
goto out;
}
- xas_split_alloc(&xas, head, compound_order(head),
- mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK);
+ gfp = current_gfp_context(mapping_gfp_mask(mapping) &
+ GFP_RECLAIM_MASK);
+
+ if (folio_test_private(folio) &&
+ !filemap_release_folio(folio, gfp)) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ xas_split_alloc(&xas, folio, folio_order(folio), gfp);
if (xas_error(&xas)) {
ret = xas_error(&xas);
goto out;
* but on 32-bit, i_size_read() takes an irq-unsafe seqlock,
* which cannot be nested inside the page tree lock. So note
* end now: i_size itself may be changed at any moment, but
- * head page lock is good enough to serialize the trimming.
+ * folio lock is good enough to serialize the trimming.
*/
end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
if (shmem_mapping(mapping))
}
/*
- * Racy check if we can split the page, before unmap_page() will
+ * Racy check if we can split the page, before unmap_folio() will
* split PMDs
*/
if (!can_split_folio(folio, &extra_pins)) {
goto out_unlock;
}
- unmap_page(head);
+ unmap_folio(folio);
/* block interrupt reentry in xa_lock and spinlock */
local_irq_disable();
if (mapping) {
/*
- * Check if the head page is present in page cache.
- * We assume all tail are present too, if head is there.
+ * Check if the folio is present in page cache.
+ * We assume all tail are present too, if folio is there.
*/
xas_lock(&xas);
xas_reset(&xas);
- if (xas_load(&xas) != head)
+ if (xas_load(&xas) != folio)
goto fail;
}
/* Prevent deferred_split_scan() touching ->_refcount */
spin_lock(&ds_queue->split_queue_lock);
- if (page_ref_freeze(head, 1 + extra_pins)) {
- if (!list_empty(page_deferred_list(head))) {
+ if (folio_ref_freeze(folio, 1 + extra_pins)) {
+ if (!list_empty(page_deferred_list(&folio->page))) {
ds_queue->split_queue_len--;
- list_del(page_deferred_list(head));
+ list_del(page_deferred_list(&folio->page));
}
spin_unlock(&ds_queue->split_queue_lock);
if (mapping) {
- int nr = thp_nr_pages(head);
+ int nr = folio_nr_pages(folio);
- xas_split(&xas, head, thp_order(head));
- if (PageSwapBacked(head)) {
- __mod_lruvec_page_state(head, NR_SHMEM_THPS,
+ xas_split(&xas, folio, folio_order(folio));
+ if (folio_test_swapbacked(folio)) {
+ __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS,
-nr);
} else {
- __mod_lruvec_page_state(head, NR_FILE_THPS,
+ __lruvec_stat_mod_folio(folio, NR_FILE_THPS,
-nr);
filemap_nr_thps_dec(mapping);
}
/* FOLL_DUMP to ignore special (like zero) pages */
page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
- if (IS_ERR_OR_NULL(page) || is_zone_device_page(page))
+ if (IS_ERR_OR_NULL(page))
continue;
if (!is_transparent_hugepage(page))
flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
pmdval = pmdp_invalidate(vma, address, pvmw->pmd);
+ /* See page_try_share_anon_rmap(): invalidate PMD first. */
anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
if (anon_exclusive && page_try_share_anon_rmap(page)) {
set_pmd_at(mm, address, pvmw->pmd, pmdval);
entry = make_readable_exclusive_migration_entry(page_to_pfn(page));
else
entry = make_readable_migration_entry(page_to_pfn(page));
+ if (pmd_young(pmdval))
+ entry = make_migration_entry_young(entry);
+ if (pmd_dirty(pmdval))
+ entry = make_migration_entry_dirty(entry);
pmdswp = swp_entry_to_pmd(entry);
if (pmd_soft_dirty(pmdval))
pmdswp = pmd_swp_mksoft_dirty(pmdswp);
entry = pmd_to_swp_entry(*pvmw->pmd);
get_page(new);
- pmde = pmd_mkold(mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot)));
+ pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot));
if (pmd_swp_soft_dirty(*pvmw->pmd))
pmde = pmd_mksoft_dirty(pmde);
if (is_writable_migration_entry(entry))
pmde = maybe_pmd_mkwrite(pmde, vma);
if (pmd_swp_uffd_wp(*pvmw->pmd))
pmde = pmd_wrprotect(pmd_mkuffd_wp(pmde));
+ if (!is_migration_entry_young(entry))
+ pmde = pmd_mkold(pmde);
+ /* NOTE: this may contain setting soft-dirty on some archs */
+ if (PageDirty(new) && is_migration_entry_dirty(entry))
+ pmde = pmd_mkdirty(pmde);
if (PageAnon(new)) {
rmap_t rmap_flags = RMAP_COMPOUND;
#include <linux/migrate.h>
#include <linux/nospec.h>
#include <linux/delayacct.h>
+#include <linux/memory.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
/* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta);
+static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
+static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
static inline bool subpool_is_free(struct hugepage_subpool *spool)
{
static struct file_region *
get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
{
- struct file_region *nrg = NULL;
+ struct file_region *nrg;
VM_BUG_ON(resv->region_cache_count <= 0);
static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
{
- struct file_region *nrg = NULL, *prg = NULL;
+ struct file_region *nrg, *prg;
prg = list_prev_entry(rg, link);
if (&prg->link != &resv->regions && prg->to == rg->from &&
int regions_needed)
__must_hold(&resv->lock)
{
- struct list_head allocated_regions;
+ LIST_HEAD(allocated_regions);
int to_allocate = 0, i = 0;
struct file_region *trg = NULL, *rg = NULL;
VM_BUG_ON(regions_needed < 0);
- INIT_LIST_HEAD(&allocated_regions);
-
/*
* Check for sufficient descriptors in the cache to accommodate
* the number of in progress add operations plus regions_needed.
* faults in a MAP_PRIVATE mapping. Only the process that called mmap()
* is guaranteed to have their future faults succeed.
*
- * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
+ * With the exception of hugetlb_dup_vma_private() which is called at fork(),
* the reserve counters are updated with the hugetlb_lock held. It is safe
* to reset the VMA at fork() time as it is not in use yet and there is no
* chance of the global counters getting corrupted as a result of the values.
return (get_vma_private_data(vma) & flag) != 0;
}
-/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
-void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
+void hugetlb_dup_vma_private(struct vm_area_struct *vma)
{
VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
+ /*
+ * Clear vm_private_data
+ * - For MAP_PRIVATE mappings, this is the reserve map which does
+ * not apply to children. Faults generated by the children are
+ * not guaranteed to succeed, even if read-only.
+ * - For shared mappings this is a per-vma semaphore that may be
+ * allocated in a subsequent call to hugetlb_vm_op_open.
+ */
+ vma->vm_private_data = (void *)0;
if (!(vma->vm_flags & VM_MAYSHARE))
- vma->vm_private_data = (void *)0;
+ return;
}
/*
kref_put(&reservations->refs, resv_map_release);
}
- reset_vma_resv_huge_pages(vma);
+ hugetlb_dup_vma_private(vma);
}
/* Returns true if the VMA has associated reserve pages */
return NULL;
}
+static unsigned long available_huge_pages(struct hstate *h)
+{
+ return h->free_huge_pages - h->resv_huge_pages;
+}
+
static struct page *dequeue_huge_page_vma(struct hstate *h,
struct vm_area_struct *vma,
unsigned long address, int avoid_reserve,
* have no page reserves. This check ensures that reservations are
* not "stolen". The child may still get SIGKILLed
*/
- if (!vma_has_reserves(vma, chg) &&
- h->free_huge_pages - h->resv_huge_pages == 0)
+ if (!vma_has_reserves(vma, chg) && !available_huge_pages(h))
goto err;
/* If reserves cannot be used, ensure enough pages are in the pool */
- if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
+ if (avoid_reserve && !available_huge_pages(h))
goto err;
gfp_mask = htlb_alloc_mask(h);
{
int i;
int nr_pages = 1 << order;
- struct page *p = page + 1;
+ struct page *p;
atomic_set(compound_mapcount_ptr(page), 0);
atomic_set(compound_pincount_ptr(page), 0);
- for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
+ for (i = 1; i < nr_pages; i++) {
+ p = nth_page(page, i);
p->mapping = NULL;
clear_compound_head(p);
if (!demote)
set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
set_page_private(page, 0);
+ /*
+ * We have to set HPageVmemmapOptimized again as above
+ * set_page_private(page, 0) cleared it.
+ */
SetHPageVmemmapOptimized(page);
/*
static void __update_and_free_page(struct hstate *h, struct page *page)
{
int i;
- struct page *subpage = page;
+ struct page *subpage;
if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
return;
if (unlikely(PageHWPoison(page)))
hugetlb_clear_page_hwpoison(page);
- for (i = 0; i < pages_per_huge_page(h);
- i++, subpage = mem_map_next(subpage, page, i)) {
+ for (i = 0; i < pages_per_huge_page(h); i++) {
+ subpage = nth_page(page, i);
subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1 << PG_referenced | 1 << PG_dirty |
1 << PG_active | 1 << PG_private |
{
int i, j;
int nr_pages = 1 << order;
- struct page *p = page + 1;
+ struct page *p;
/* we rely on prep_new_huge_page to set the destructor */
set_compound_order(page, order);
- __ClearPageReserved(page);
__SetPageHead(page);
- for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
+ for (i = 0; i < nr_pages; i++) {
+ p = nth_page(page, i);
+
/*
* For gigantic hugepages allocated through bootmem at
* boot, it's safer to be consistent with the not-gigantic
} else {
VM_BUG_ON_PAGE(page_count(p), p);
}
- set_compound_head(p, page);
+ if (i != 0)
+ set_compound_head(p, page);
}
atomic_set(compound_mapcount_ptr(page), -1);
atomic_set(compound_pincount_ptr(page), 0);
return true;
out_error:
- /* undo tail page modifications made above */
- p = page + 1;
- for (j = 1; j < i; j++, p = mem_map_next(p, page, j)) {
- clear_compound_head(p);
+ /* undo page modifications made above */
+ for (j = 0; j < i; j++) {
+ p = nth_page(page, j);
+ if (j != 0)
+ clear_compound_head(p);
set_page_refcounted(p);
}
/* need to clear PG_reserved on remaining tail pages */
- for (; j < nr_pages; j++, p = mem_map_next(p, page, j))
+ for (; j < nr_pages; j++) {
+ p = nth_page(page, j);
__ClearPageReserved(p);
+ }
set_compound_order(page, 0);
#ifdef CONFIG_64BIT
page[1].compound_nr = 0;
int order = huge_page_order(h);
struct page *page;
bool alloc_try_hard = true;
+ bool retry = true;
/*
* By default we always try hard to allocate the page with
gfp_mask |= __GFP_RETRY_MAYFAIL;
if (nid == NUMA_NO_NODE)
nid = numa_mem_id();
+retry:
page = __alloc_pages(gfp_mask, order, nid, nmask);
+
+ /* Freeze head page */
+ if (page && !page_ref_freeze(page, 1)) {
+ __free_pages(page, order);
+ if (retry) { /* retry once */
+ retry = false;
+ goto retry;
+ }
+ /* WOW! twice in a row. */
+ pr_warn("HugeTLB head page unexpected inflated ref count\n");
+ page = NULL;
+ }
+
if (page)
__count_vm_event(HTLB_BUDDY_PGALLOC);
else
/*
* Common helper to allocate a fresh hugetlb page. All specific allocators
* should use this function to get new hugetlb pages
+ *
+ * Note that returned page is 'frozen': ref count of head page and all tail
+ * pages is zero.
*/
static struct page *alloc_fresh_huge_page(struct hstate *h,
gfp_t gfp_mask, int nid, nodemask_t *nmask,
if (!page)
return 0;
- put_page(page); /* free it into the hugepage allocator */
+ free_huge_page(page); /* free it into the hugepage allocator */
return 1;
}
if (!page_count(page)) {
struct page *head = compound_head(page);
struct hstate *h = page_hstate(head);
- if (h->free_huge_pages - h->resv_huge_pages == 0)
+ if (!available_huge_pages(h))
goto out;
/*
* Allocates a fresh surplus page from the page allocator.
*/
static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
- int nid, nodemask_t *nmask, bool zero_ref)
+ int nid, nodemask_t *nmask)
{
struct page *page = NULL;
- bool retry = false;
if (hstate_is_gigantic(h))
return NULL;
goto out_unlock;
spin_unlock_irq(&hugetlb_lock);
-retry:
page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
if (!page)
return NULL;
if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
SetHPageTemporary(page);
spin_unlock_irq(&hugetlb_lock);
- put_page(page);
+ free_huge_page(page);
return NULL;
}
- if (zero_ref) {
- /*
- * Caller requires a page with zero ref count.
- * We will drop ref count here. If someone else is holding
- * a ref, the page will be freed when they drop it. Abuse
- * temporary page flag to accomplish this.
- */
- SetHPageTemporary(page);
- if (!put_page_testzero(page)) {
- /*
- * Unexpected inflated ref count on freshly allocated
- * huge. Retry once.
- */
- pr_info("HugeTLB unexpected inflated ref count on freshly allocated page\n");
- spin_unlock_irq(&hugetlb_lock);
- if (retry)
- return NULL;
-
- retry = true;
- goto retry;
- }
- ClearHPageTemporary(page);
- }
-
h->surplus_huge_pages++;
h->surplus_huge_pages_node[page_to_nid(page)]++;
if (!page)
return NULL;
+ /* fresh huge pages are frozen */
+ set_page_refcounted(page);
+
/*
* We do not account these pages as surplus because they are only
* temporary and will be released properly on the last reference
gfp_t gfp = gfp_mask | __GFP_NOWARN;
gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
- page = alloc_surplus_huge_page(h, gfp, nid, nodemask, false);
+ page = alloc_surplus_huge_page(h, gfp, nid, nodemask);
/* Fallback to all nodes if page==NULL */
nodemask = NULL;
}
if (!page)
- page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask, false);
+ page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);
mpol_cond_put(mpol);
return page;
}
nodemask_t *nmask, gfp_t gfp_mask)
{
spin_lock_irq(&hugetlb_lock);
- if (h->free_huge_pages - h->resv_huge_pages > 0) {
+ if (available_huge_pages(h)) {
struct page *page;
page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
static int gather_surplus_pages(struct hstate *h, long delta)
__must_hold(&hugetlb_lock)
{
- struct list_head surplus_list;
+ LIST_HEAD(surplus_list);
struct page *page, *tmp;
int ret;
long i;
}
allocated = 0;
- INIT_LIST_HEAD(&surplus_list);
ret = -ENOMEM;
retry:
spin_unlock_irq(&hugetlb_lock);
for (i = 0; i < needed; i++) {
page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
- NUMA_NO_NODE, NULL, true);
+ NUMA_NO_NODE, NULL);
if (!page) {
alloc_ok = false;
break;
{
gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
int nid = page_to_nid(old_page);
- bool alloc_retry = false;
struct page *new_page;
int ret = 0;
* the pool. This simplifies and let us do most of the processing
* under the lock.
*/
-alloc_retry:
new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
if (!new_page)
return -ENOMEM;
- /*
- * If all goes well, this page will be directly added to the free
- * list in the pool. For this the ref count needs to be zero.
- * Attempt to drop now, and retry once if needed. It is VERY
- * unlikely there is another ref on the page.
- *
- * If someone else has a reference to the page, it will be freed
- * when they drop their ref. Abuse temporary page flag to accomplish
- * this. Retry once if there is an inflated ref count.
- */
- SetHPageTemporary(new_page);
- if (!put_page_testzero(new_page)) {
- if (alloc_retry)
- return -EBUSY;
-
- alloc_retry = true;
- goto alloc_retry;
- }
- ClearHPageTemporary(new_page);
-
__prep_new_huge_page(h, new_page);
retry:
}
spin_lock_irq(&hugetlb_lock);
list_add(&page->lru, &h->hugepage_activelist);
+ set_page_refcounted(page);
/* Fall through */
}
hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
if (prep_compound_gigantic_page(page, huge_page_order(h))) {
WARN_ON(PageReserved(page));
prep_new_huge_page(h, page, page_to_nid(page));
- put_page(page); /* add to the hugepage allocator */
+ free_huge_page(page); /* add to the hugepage allocator */
} else {
/* VERY unlikely inflated ref count on a tail page */
free_gigantic_page(page, huge_page_order(h));
&node_states[N_MEMORY], NULL);
if (!page)
break;
- put_page(page); /* free it into the hugepage allocator */
+ free_huge_page(page); /* free it into the hugepage allocator */
}
cond_resched();
}
else
prep_compound_page(subpage, target_hstate->order);
set_page_private(subpage, 0);
- set_page_refcounted(subpage);
prep_new_huge_page(target_hstate, subpage, nid);
- put_page(subpage);
+ free_huge_page(subpage);
}
mutex_unlock(&target_hstate->resize_lock);
* based on pool changes for the demoted page.
*/
h->max_huge_pages--;
- target_hstate->max_huge_pages += pages_per_huge_page(h);
+ target_hstate->max_huge_pages +=
+ pages_per_huge_page(h) / pages_per_huge_page(target_hstate);
return rc;
}
unsigned long nr_available;
nodemask_t nodes_allowed, *n_mask;
struct hstate *h;
- int err = 0;
+ int err;
int nid;
err = kstrtoul(buf, 10, &nr_demote);
static ssize_t demote_size_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
- int nid;
- struct hstate *h = kobj_to_hstate(kobj, &nid);
+ struct hstate *h = kobj_to_hstate(kobj, NULL);
unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
return sysfs_emit(buf, "%lukB\n", demote_size);
struct hstate *h, *demote_hstate;
unsigned long demote_size;
unsigned int demote_order;
- int nid;
demote_size = (unsigned long)memparse(buf, NULL);
return -EINVAL;
/* demote order must be smaller than hstate order */
- h = kobj_to_hstate(kobj, &nid);
+ h = kobj_to_hstate(kobj, NULL);
if (demote_order >= h->order)
return -EINVAL;
if (retval) {
kobject_put(hstate_kobjs[hi]);
hstate_kobjs[hi] = NULL;
+ return retval;
}
if (h->demote_order) {
- if (sysfs_create_group(hstate_kobjs[hi],
- &hstate_demote_attr_group))
+ retval = sysfs_create_group(hstate_kobjs[hi],
+ &hstate_demote_attr_group);
+ if (retval) {
pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
+ sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
+ kobject_put(hstate_kobjs[hi]);
+ hstate_kobjs[hi] = NULL;
+ return retval;
+ }
}
- return retval;
-}
-
-static void __init hugetlb_sysfs_init(void)
-{
- struct hstate *h;
- int err;
-
- hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
- if (!hugepages_kobj)
- return;
-
- for_each_hstate(h) {
- err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
- hstate_kobjs, &hstate_attr_group);
- if (err)
- pr_err("HugeTLB: Unable to add hstate %s", h->name);
- }
+ return 0;
}
#ifdef CONFIG_NUMA
+static bool hugetlb_sysfs_initialized __ro_after_init;
/*
* node_hstate/s - associate per node hstate attributes, via their kobjects,
* Unregister hstate attributes from a single node device.
* No-op if no hstate attributes attached.
*/
-static void hugetlb_unregister_node(struct node *node)
+void hugetlb_unregister_node(struct node *node)
{
struct hstate *h;
struct node_hstate *nhs = &node_hstates[node->dev.id];
for_each_hstate(h) {
int idx = hstate_index(h);
- if (nhs->hstate_kobjs[idx]) {
- kobject_put(nhs->hstate_kobjs[idx]);
- nhs->hstate_kobjs[idx] = NULL;
- }
+ struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];
+
+ if (!hstate_kobj)
+ continue;
+ if (h->demote_order)
+ sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
+ sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
+ kobject_put(hstate_kobj);
+ nhs->hstate_kobjs[idx] = NULL;
}
kobject_put(nhs->hugepages_kobj);
* Register hstate attributes for a single node device.
* No-op if attributes already registered.
*/
-static void hugetlb_register_node(struct node *node)
+void hugetlb_register_node(struct node *node)
{
struct hstate *h;
struct node_hstate *nhs = &node_hstates[node->dev.id];
int err;
+ if (!hugetlb_sysfs_initialized)
+ return;
+
if (nhs->hugepages_kobj)
return; /* already allocated */
{
int nid;
- for_each_node_state(nid, N_MEMORY) {
- struct node *node = node_devices[nid];
- if (node->dev.id == nid)
- hugetlb_register_node(node);
- }
-
- /*
- * Let the node device driver know we're here so it can
- * [un]register hstate attributes on node hotplug.
- */
- register_hugetlbfs_with_node(hugetlb_register_node,
- hugetlb_unregister_node);
+ for_each_online_node(nid)
+ hugetlb_register_node(node_devices[nid]);
}
#else /* !CONFIG_NUMA */
#endif
+#ifdef CONFIG_CMA
+static void __init hugetlb_cma_check(void);
+#else
+static inline __init void hugetlb_cma_check(void)
+{
+}
+#endif
+
+static void __init hugetlb_sysfs_init(void)
+{
+ struct hstate *h;
+ int err;
+
+ hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
+ if (!hugepages_kobj)
+ return;
+
+ for_each_hstate(h) {
+ err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
+ hstate_kobjs, &hstate_attr_group);
+ if (err)
+ pr_err("HugeTLB: Unable to add hstate %s", h->name);
+ }
+
+#ifdef CONFIG_NUMA
+ hugetlb_sysfs_initialized = true;
+#endif
+ hugetlb_register_all_nodes();
+}
+
static int __init hugetlb_init(void)
{
int i;
report_hugepages();
hugetlb_sysfs_init();
- hugetlb_register_all_nodes();
hugetlb_cgroup_file_init();
#ifdef CONFIG_SMP
h->next_nid_to_alloc = first_memory_node;
h->next_nid_to_free = first_memory_node;
snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
- huge_page_size(h)/1024);
+ huge_page_size(h)/SZ_1K);
parsed_hstate = h;
}
if (!hugetlb_max_hstate) {
default_hstate_max_huge_pages = 0;
memset(default_hugepages_in_node, 0,
- MAX_NUMNODES * sizeof(unsigned int));
+ sizeof(default_hugepages_in_node));
} else {
parsed_hstate->max_huge_pages = 0;
memset(parsed_hstate->max_huge_pages_node, 0,
- MAX_NUMNODES * sizeof(unsigned int));
+ sizeof(parsed_hstate->max_huge_pages_node));
}
}
}
__setup("default_hugepagesz=", default_hugepagesz_setup);
+static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
+{
+#ifdef CONFIG_NUMA
+ struct mempolicy *mpol = get_task_policy(current);
+
+ /*
+ * Only enforce MPOL_BIND policy which overlaps with cpuset policy
+ * (from policy_nodemask) specifically for hugetlb case
+ */
+ if (mpol->mode == MPOL_BIND &&
+ (apply_policy_zone(mpol, gfp_zone(gfp)) &&
+ cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
+ return &mpol->nodes;
+#endif
+ return NULL;
+}
+
static unsigned int allowed_mems_nr(struct hstate *h)
{
int node;
unsigned int nr = 0;
- nodemask_t *mpol_allowed;
+ nodemask_t *mbind_nodemask;
unsigned int *array = h->free_huge_pages_node;
gfp_t gfp_mask = htlb_alloc_mask(h);
- mpol_allowed = policy_nodemask_current(gfp_mask);
-
+ mbind_nodemask = policy_mbind_nodemask(gfp_mask);
for_each_node_mask(node, cpuset_current_mems_allowed) {
- if (!mpol_allowed || node_isset(node, *mpol_allowed))
+ if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
nr += array[node];
}
resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
kref_get(&resv->refs);
}
+
+ /*
+ * vma_lock structure for sharable mappings is vma specific.
+ * Clear old pointer (if copied via vm_area_dup) and create new.
+ */
+ if (vma->vm_flags & VM_MAYSHARE) {
+ vma->vm_private_data = NULL;
+ hugetlb_vma_lock_alloc(vma);
+ }
}
static void hugetlb_vm_op_close(struct vm_area_struct *vma)
{
struct hstate *h = hstate_vma(vma);
- struct resv_map *resv = vma_resv_map(vma);
+ struct resv_map *resv;
struct hugepage_subpool *spool = subpool_vma(vma);
unsigned long reserve, start, end;
long gbl_reserve;
+ hugetlb_vma_lock_free(vma);
+
+ resv = vma_resv_map(vma);
if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
return;
struct vm_area_struct *dst_vma,
struct vm_area_struct *src_vma)
{
- pte_t *src_pte, *dst_pte, entry, dst_entry;
+ pte_t *src_pte, *dst_pte, entry;
struct page *ptepage;
unsigned long addr;
bool cow = is_cow_mapping(src_vma->vm_flags);
struct hstate *h = hstate_vma(src_vma);
unsigned long sz = huge_page_size(h);
unsigned long npages = pages_per_huge_page(h);
- struct address_space *mapping = src_vma->vm_file->f_mapping;
struct mmu_notifier_range range;
unsigned long last_addr_mask;
int ret = 0;
raw_write_seqcount_begin(&src->write_protect_seq);
} else {
/*
- * For shared mappings i_mmap_rwsem must be held to call
- * huge_pte_alloc, otherwise the returned ptep could go
- * away if part of a shared pmd and another thread calls
- * huge_pmd_unshare.
+ * For shared mappings the vma lock must be held before
+ * calling huge_pte_offset in the src vma. Otherwise, the
+ * returned ptep could go away if part of a shared pmd and
+ * another thread calls huge_pmd_unshare.
*/
- i_mmap_lock_read(mapping);
+ hugetlb_vma_lock_read(src_vma);
}
last_addr_mask = hugetlb_mask_last_page(h);
/*
* If the pagetables are shared don't copy or take references.
- * dst_pte == src_pte is the common case of src/dest sharing.
*
+ * dst_pte == src_pte is the common case of src/dest sharing.
* However, src could have 'unshared' and dst shares with
- * another vma. If dst_pte !none, this implies sharing.
- * Check here before taking page table lock, and once again
- * after taking the lock below.
+ * another vma. So page_count of ptep page is checked instead
+ * to reliably determine whether pte is shared.
*/
- dst_entry = huge_ptep_get(dst_pte);
- if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) {
+ if (page_count(virt_to_page(dst_pte)) > 1) {
addr |= last_addr_mask;
continue;
}
src_ptl = huge_pte_lockptr(h, src, src_pte);
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
entry = huge_ptep_get(src_pte);
- dst_entry = huge_ptep_get(dst_pte);
again:
- if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
+ if (huge_pte_none(entry)) {
/*
- * Skip if src entry none. Also, skip in the
- * unlikely case dst entry !none as this implies
- * sharing with another vma.
+ * Skip if src entry none.
*/
;
} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
restore_reserve_on_error(h, dst_vma, addr,
new);
put_page(new);
- /* dst_entry won't change as in child */
+ /* huge_ptep of dst_pte won't change as in child */
goto again;
}
hugetlb_install_page(dst_vma, dst_pte, addr, new);
raw_write_seqcount_end(&src->write_protect_seq);
mmu_notifier_invalidate_range_end(&range);
} else {
- i_mmap_unlock_read(mapping);
+ hugetlb_vma_unlock_read(src_vma);
}
return ret;
mmu_notifier_invalidate_range_start(&range);
last_addr_mask = hugetlb_mask_last_page(h);
/* Prevent race with file truncation */
+ hugetlb_vma_lock_write(vma);
i_mmap_lock_write(mapping);
for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
src_pte = huge_pte_offset(mm, old_addr, sz);
flush_tlb_range(vma, old_end - len, old_end);
mmu_notifier_invalidate_range_end(&range);
i_mmap_unlock_write(mapping);
+ hugetlb_vma_unlock_write(vma);
return len + old_addr - old_end;
}
* unmapped and its refcount is dropped, so just clear pte here.
*/
if (unlikely(!pte_present(pte))) {
+#ifdef CONFIG_PTE_MARKER_UFFD_WP
/*
* If the pte was wr-protected by uffd-wp in any of the
* swap forms, meanwhile the caller does not want to
set_huge_pte_at(mm, address, ptep,
make_pte_marker(PTE_MARKER_UFFD_WP));
else
+#endif
huge_pte_clear(mm, address, ptep, sz);
spin_unlock(ptl);
continue;
tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
if (huge_pte_dirty(pte))
set_page_dirty(page);
+#ifdef CONFIG_PTE_MARKER_UFFD_WP
/* Leave a uffd-wp pte marker if needed */
if (huge_pte_uffd_wp(pte) &&
!(zap_flags & ZAP_FLAG_DROP_MARKER))
set_huge_pte_at(mm, address, ptep,
make_pte_marker(PTE_MARKER_UFFD_WP));
+#endif
hugetlb_count_sub(pages_per_huge_page(h), mm);
page_remove_rmap(page, vma, true);
unsigned long end, struct page *ref_page,
zap_flags_t zap_flags)
{
+ hugetlb_vma_lock_write(vma);
+ i_mmap_lock_write(vma->vm_file->f_mapping);
+
__unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags);
/*
- * Clear this flag so that x86's huge_pmd_share page_table_shareable
- * test will fail on a vma being torn down, and not grab a page table
- * on its way out. We're lucky that the flag has such an appropriate
- * name, and can in fact be safely cleared here. We could clear it
- * before the __unmap_hugepage_range above, but all that's necessary
- * is to clear it before releasing the i_mmap_rwsem. This works
- * because in the context this is called, the VMA is about to be
- * destroyed and the i_mmap_rwsem is held.
+ * Unlock and free the vma lock before releasing i_mmap_rwsem. When
+ * the vma_lock is freed, this makes the vma ineligible for pmd
+ * sharing. And, i_mmap_rwsem is required to set up pmd sharing.
+ * This is important as page tables for this unmapped range will
+ * be asynchrously deleted. If the page tables are shared, there
+ * will be issues when accessed by someone else.
*/
- vma->vm_flags &= ~VM_MAYSHARE;
+ __hugetlb_vma_unlock_write_free(vma);
+
+ i_mmap_unlock_write(vma->vm_file->f_mapping);
}
void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
u32 hash;
put_page(old_page);
- BUG_ON(huge_pte_none(pte));
/*
- * Drop hugetlb_fault_mutex and i_mmap_rwsem before
- * unmapping. unmapping needs to hold i_mmap_rwsem
- * in write mode. Dropping i_mmap_rwsem in read mode
+ * Drop hugetlb_fault_mutex and vma_lock before
+ * unmapping. unmapping needs to hold vma_lock
+ * in write mode. Dropping vma_lock in read mode
* here is OK as COW mappings do not interact with
* PMD sharing.
*
*/
idx = vma_hugecache_offset(h, vma, haddr);
hash = hugetlb_fault_mutex_hash(mapping, idx);
+ hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
unmap_ref_private(mm, vma, old_page, haddr);
- i_mmap_lock_read(mapping);
mutex_lock(&hugetlb_fault_mutex_table[hash]);
+ hugetlb_vma_lock_read(vma);
spin_lock(ptl);
ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
if (likely(ptep &&
return ret;
}
-/* Return the pagecache page at a given address within a VMA */
-static struct page *hugetlbfs_pagecache_page(struct hstate *h,
- struct vm_area_struct *vma, unsigned long address)
-{
- struct address_space *mapping;
- pgoff_t idx;
-
- mapping = vma->vm_file->f_mapping;
- idx = vma_hugecache_offset(h, vma, address);
-
- return find_lock_page(mapping, idx);
-}
-
/*
* Return whether there is a pagecache page to back given address within VMA.
* Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
return page != NULL;
}
-int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
+int hugetlb_add_to_page_cache(struct page *page, struct address_space *mapping,
pgoff_t idx)
{
struct folio *folio = page_folio(page);
unsigned long addr,
unsigned long reason)
{
- vm_fault_t ret;
u32 hash;
struct vm_fault vmf = {
.vma = vma,
};
/*
- * hugetlb_fault_mutex and i_mmap_rwsem must be
- * dropped before handling userfault. Reacquire
- * after handling fault to make calling code simpler.
+ * vma_lock and hugetlb_fault_mutex must be dropped before handling
+ * userfault. Also mmap_lock could be dropped due to handling
+ * userfault, any vma operation should be careful from here.
*/
+ hugetlb_vma_unlock_read(vma);
hash = hugetlb_fault_mutex_hash(mapping, idx);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
- ret = handle_userfault(&vmf, reason);
- i_mmap_lock_read(mapping);
- mutex_lock(&hugetlb_fault_mutex_table[hash]);
+ return handle_userfault(&vmf, reason);
+}
- return ret;
+/*
+ * Recheck pte with pgtable lock. Returns true if pte didn't change, or
+ * false if pte changed or is changing.
+ */
+static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm,
+ pte_t *ptep, pte_t old_pte)
+{
+ spinlock_t *ptl;
+ bool same;
+
+ ptl = huge_pte_lock(h, mm, ptep);
+ same = pte_same(huge_ptep_get(ptep), old_pte);
+ spin_unlock(ptl);
+
+ return same;
}
static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
spinlock_t *ptl;
unsigned long haddr = address & huge_page_mask(h);
bool new_page, new_pagecache_page = false;
+ u32 hash = hugetlb_fault_mutex_hash(mapping, idx);
/*
* Currently, we are forced to kill the process in the event the
if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
current->pid);
- return ret;
+ goto out;
}
/*
- * We can not race with truncation due to holding i_mmap_rwsem.
- * i_size is modified when holding i_mmap_rwsem, so check here
- * once for faults beyond end of file.
+ * Use page lock to guard against racing truncation
+ * before we get page_table_lock.
*/
- size = i_size_read(mapping->host) >> huge_page_shift(h);
- if (idx >= size)
- goto out;
-
-retry:
new_page = false;
page = find_lock_page(mapping, idx);
if (!page) {
+ size = i_size_read(mapping->host) >> huge_page_shift(h);
+ if (idx >= size)
+ goto out;
/* Check for page in userfault range */
if (userfaultfd_missing(vma)) {
- ret = hugetlb_handle_userfault(vma, mapping, idx,
- flags, haddr, address,
- VM_UFFD_MISSING);
- goto out;
+ /*
+ * Since hugetlb_no_page() was examining pte
+ * without pgtable lock, we need to re-test under
+ * lock because the pte may not be stable and could
+ * have changed from under us. Try to detect
+ * either changed or during-changing ptes and retry
+ * properly when needed.
+ *
+ * Note that userfaultfd is actually fine with
+ * false positives (e.g. caused by pte changed),
+ * but not wrong logical events (e.g. caused by
+ * reading a pte during changing). The latter can
+ * confuse the userspace, so the strictness is very
+ * much preferred. E.g., MISSING event should
+ * never happen on the page after UFFDIO_COPY has
+ * correctly installed the page and returned.
+ */
+ if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
+ ret = 0;
+ goto out;
+ }
+
+ return hugetlb_handle_userfault(vma, mapping, idx, flags,
+ haddr, address,
+ VM_UFFD_MISSING);
}
page = alloc_huge_page(vma, haddr, 0);
* here. Before returning error, get ptl and make
* sure there really is no pte entry.
*/
- ptl = huge_pte_lock(h, mm, ptep);
- ret = 0;
- if (huge_pte_none(huge_ptep_get(ptep)))
+ if (hugetlb_pte_stable(h, mm, ptep, old_pte))
ret = vmf_error(PTR_ERR(page));
- spin_unlock(ptl);
+ else
+ ret = 0;
goto out;
}
clear_huge_page(page, address, pages_per_huge_page(h));
new_page = true;
if (vma->vm_flags & VM_MAYSHARE) {
- int err = huge_add_to_page_cache(page, mapping, idx);
+ int err = hugetlb_add_to_page_cache(page, mapping, idx);
if (err) {
+ /*
+ * err can't be -EEXIST which implies someone
+ * else consumed the reservation since hugetlb
+ * fault mutex is held when add a hugetlb page
+ * to the page cache. So it's safe to call
+ * restore_reserve_on_error() here.
+ */
+ restore_reserve_on_error(h, vma, haddr, page);
put_page(page);
- if (err == -EEXIST)
- goto retry;
goto out;
}
new_pagecache_page = true;
if (userfaultfd_minor(vma)) {
unlock_page(page);
put_page(page);
- ret = hugetlb_handle_userfault(vma, mapping, idx,
- flags, haddr, address,
- VM_UFFD_MINOR);
- goto out;
+ /* See comment in userfaultfd_missing() block above */
+ if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
+ ret = 0;
+ goto out;
+ }
+ return hugetlb_handle_userfault(vma, mapping, idx, flags,
+ haddr, address,
+ VM_UFFD_MINOR);
}
}
unlock_page(page);
out:
+ hugetlb_vma_unlock_read(vma);
+ mutex_unlock(&hugetlb_fault_mutex_table[hash]);
return ret;
backout:
spin_unlock(ptl);
backout_unlocked:
- unlock_page(page);
- /* restore reserve for newly allocated pages not in page cache */
if (new_page && !new_pagecache_page)
restore_reserve_on_error(h, vma, haddr, page);
+
+ unlock_page(page);
put_page(page);
goto out;
}
}
/*
- * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold
- * until finished with ptep. This serves two purposes:
- * 1) It prevents huge_pmd_unshare from being called elsewhere
- * and making the ptep no longer valid.
- * 2) It synchronizes us with i_size modifications during truncation.
+ * Serialize hugepage allocation and instantiation, so that we don't
+ * get spurious allocation failures if two CPUs race to instantiate
+ * the same page in the page cache.
+ */
+ mapping = vma->vm_file->f_mapping;
+ idx = vma_hugecache_offset(h, vma, haddr);
+ hash = hugetlb_fault_mutex_hash(mapping, idx);
+ mutex_lock(&hugetlb_fault_mutex_table[hash]);
+
+ /*
+ * Acquire vma lock before calling huge_pte_alloc and hold
+ * until finished with ptep. This prevents huge_pmd_unshare from
+ * being called elsewhere and making the ptep no longer valid.
*
* ptep could have already be assigned via huge_pte_offset. That
* is OK, as huge_pte_alloc will return the same value unless
* something has changed.
*/
- mapping = vma->vm_file->f_mapping;
- i_mmap_lock_read(mapping);
+ hugetlb_vma_lock_read(vma);
ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
if (!ptep) {
- i_mmap_unlock_read(mapping);
+ hugetlb_vma_unlock_read(vma);
+ mutex_unlock(&hugetlb_fault_mutex_table[hash]);
return VM_FAULT_OOM;
}
- /*
- * Serialize hugepage allocation and instantiation, so that we don't
- * get spurious allocation failures if two CPUs race to instantiate
- * the same page in the page cache.
- */
- idx = vma_hugecache_offset(h, vma, haddr);
- hash = hugetlb_fault_mutex_hash(mapping, idx);
- mutex_lock(&hugetlb_fault_mutex_table[hash]);
-
entry = huge_ptep_get(ptep);
/* PTE markers should be handled the same way as none pte */
- if (huge_pte_none_mostly(entry)) {
- ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
+ if (huge_pte_none_mostly(entry))
+ /*
+ * hugetlb_no_page will drop vma lock and hugetlb fault
+ * mutex internally, which make us return immediately.
+ */
+ return hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
entry, flags);
- goto out_mutex;
- }
ret = 0;
/* Just decrements count, does not deallocate */
vma_end_reservation(h, vma, haddr);
- pagecache_page = hugetlbfs_pagecache_page(h, vma, haddr);
+ pagecache_page = find_lock_page(mapping, idx);
}
ptl = huge_pte_lock(h, mm, ptep);
unlock_page(pagecache_page);
put_page(pagecache_page);
}
+ hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
return handle_userfault(&vmf, VM_UFFD_WP);
}
put_page(pagecache_page);
}
out_mutex:
+ hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
/*
* Generally it's safe to hold refcount during waiting page lock. But
* here we just wait to defer the next page fault to avoid busy loop and
/*
* Serialization between remove_inode_hugepages() and
- * huge_add_to_page_cache() below happens through the
+ * hugetlb_add_to_page_cache() below happens through the
* hugetlb_fault_mutex_table that here must be hold by
* the caller.
*/
- ret = huge_add_to_page_cache(page, mapping, idx);
+ ret = hugetlb_add_to_page_cache(page, mapping, idx);
if (ret)
goto out_release_nounlock;
page_in_pagecache = true;
}
- ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
- spin_lock(ptl);
+ ptl = huge_pte_lock(h, dst_mm, dst_pte);
- /*
- * Recheck the i_size after holding PT lock to make sure not
- * to leave any page mapped (as page_mapped()) beyond the end
- * of the i_size (remove_inode_hugepages() is strict about
- * enforcing that). If we bail out here, we'll also leave a
- * page in the radix tree in the vm_shared case beyond the end
- * of the i_size, but remove_inode_hugepages() will take care
- * of it as soon as we drop the hugetlb_fault_mutex_table.
- */
- size = i_size_read(mapping->host) >> huge_page_shift(h);
- ret = -EFAULT;
- if (idx >= size)
- goto out_release_unlock;
-
- ret = -EEXIST;
/*
* We allow to overwrite a pte marker: consider when both MISSING|WP
* registered, we firstly wr-protect a none pte which has no page cache
* page backing it, then access the page.
*/
+ ret = -EEXIST;
if (!huge_pte_none_mostly(huge_ptep_get(dst_pte)))
goto out_release_unlock;
for (nr = 0; nr < refs; nr++) {
if (likely(pages))
- pages[nr] = mem_map_offset(page, nr);
+ pages[nr] = nth_page(page, nr);
if (vmas)
vmas[nr] = vma;
}
(vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT);
if (pages || vmas)
- record_subpages_vmas(mem_map_offset(page, pfn_offset),
+ record_subpages_vmas(nth_page(page, pfn_offset),
vma, refs,
likely(pages) ? pages + i : NULL,
vmas ? vmas + i : NULL);
flush_cache_range(vma, range.start, range.end);
mmu_notifier_invalidate_range_start(&range);
- last_addr_mask = hugetlb_mask_last_page(h);
+ hugetlb_vma_lock_write(vma);
i_mmap_lock_write(vma->vm_file->f_mapping);
+ last_addr_mask = hugetlb_mask_last_page(h);
for (; address < end; address += psize) {
spinlock_t *ptl;
ptep = huge_pte_offset(mm, address, psize);
* See Documentation/mm/mmu_notifier.rst
*/
i_mmap_unlock_write(vma->vm_file->f_mapping);
+ hugetlb_vma_unlock_write(vma);
mmu_notifier_invalidate_range_end(&range);
return pages << h->order;
return false;
}
+ /*
+ * vma specific semaphore used for pmd sharing synchronization
+ */
+ hugetlb_vma_lock_alloc(vma);
+
/*
* Only apply hugepage reservation if asked. At fault time, an
* attempt will be made for VM_NORESERVE to allocate a page
resv_map = inode_resv_map(inode);
chg = region_chg(resv_map, from, to, ®ions_needed);
-
} else {
/* Private mapping. */
resv_map = resv_map_alloc();
if (!resv_map)
- return false;
+ goto out_err;
chg = to - from;
hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
chg * pages_per_huge_page(h), h_cg);
out_err:
+ hugetlb_vma_lock_free(vma);
if (!vma || vma->vm_flags & VM_MAYSHARE)
/* Only call region_abort if the region_chg succeeded but the
* region_add failed or didn't run.
/*
* match the virtual addresses, permission and the alignment of the
* page table page.
+ *
+ * Also, vma_lock (vm_private_data) is required for sharing.
*/
if (pmd_index(addr) != pmd_index(saddr) ||
vm_flags != svm_flags ||
- !range_in_vma(svma, sbase, s_end))
+ !range_in_vma(svma, sbase, s_end) ||
+ !svma->vm_private_data)
return 0;
return saddr;
}
-static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
-{
- unsigned long base = addr & PUD_MASK;
- unsigned long end = base + PUD_SIZE;
-
- /*
- * check on proper vm_flags and page table alignment
- */
- if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
- return true;
- return false;
-}
-
bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
{
+ unsigned long start = addr & PUD_MASK;
+ unsigned long end = start + PUD_SIZE;
+
#ifdef CONFIG_USERFAULTFD
if (uffd_disable_huge_pmd_share(vma))
return false;
#endif
- return vma_shareable(vma, addr);
+ /*
+ * check on proper vm_flags and page table alignment
+ */
+ if (!(vma->vm_flags & VM_MAYSHARE))
+ return false;
+ if (!vma->vm_private_data) /* vma lock required for sharing */
+ return false;
+ if (!range_in_vma(vma, start, end))
+ return false;
+ return true;
}
/*
*end = ALIGN(*end, PUD_SIZE);
}
+static bool __vma_shareable_flags_pmd(struct vm_area_struct *vma)
+{
+ return vma->vm_flags & (VM_MAYSHARE | VM_SHARED) &&
+ vma->vm_private_data;
+}
+
+void hugetlb_vma_lock_read(struct vm_area_struct *vma)
+{
+ if (__vma_shareable_flags_pmd(vma)) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ down_read(&vma_lock->rw_sema);
+ }
+}
+
+void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
+{
+ if (__vma_shareable_flags_pmd(vma)) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ up_read(&vma_lock->rw_sema);
+ }
+}
+
+void hugetlb_vma_lock_write(struct vm_area_struct *vma)
+{
+ if (__vma_shareable_flags_pmd(vma)) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ down_write(&vma_lock->rw_sema);
+ }
+}
+
+void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
+{
+ if (__vma_shareable_flags_pmd(vma)) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ up_write(&vma_lock->rw_sema);
+ }
+}
+
+int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
+{
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ if (!__vma_shareable_flags_pmd(vma))
+ return 1;
+
+ return down_write_trylock(&vma_lock->rw_sema);
+}
+
+void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
+{
+ if (__vma_shareable_flags_pmd(vma)) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ lockdep_assert_held(&vma_lock->rw_sema);
+ }
+}
+
+void hugetlb_vma_lock_release(struct kref *kref)
+{
+ struct hugetlb_vma_lock *vma_lock = container_of(kref,
+ struct hugetlb_vma_lock, refs);
+
+ kfree(vma_lock);
+}
+
+static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
+{
+ struct vm_area_struct *vma = vma_lock->vma;
+
+ /*
+ * vma_lock structure may or not be released as a result of put,
+ * it certainly will no longer be attached to vma so clear pointer.
+ * Semaphore synchronizes access to vma_lock->vma field.
+ */
+ vma_lock->vma = NULL;
+ vma->vm_private_data = NULL;
+ up_write(&vma_lock->rw_sema);
+ kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
+}
+
+static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
+{
+ if (__vma_shareable_flags_pmd(vma)) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ __hugetlb_vma_unlock_write_put(vma_lock);
+ }
+}
+
+static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
+{
+ /*
+ * Only present in sharable vmas.
+ */
+ if (!vma || !__vma_shareable_flags_pmd(vma))
+ return;
+
+ if (vma->vm_private_data) {
+ struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
+
+ down_write(&vma_lock->rw_sema);
+ __hugetlb_vma_unlock_write_put(vma_lock);
+ }
+}
+
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
+{
+ struct hugetlb_vma_lock *vma_lock;
+
+ /* Only establish in (flags) sharable vmas */
+ if (!vma || !(vma->vm_flags & VM_MAYSHARE))
+ return;
+
+ /* Should never get here with non-NULL vm_private_data */
+ if (vma->vm_private_data)
+ return;
+
+ vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL);
+ if (!vma_lock) {
+ /*
+ * If we can not allocate structure, then vma can not
+ * participate in pmd sharing. This is only a possible
+ * performance enhancement and memory saving issue.
+ * However, the lock is also used to synchronize page
+ * faults with truncation. If the lock is not present,
+ * unlikely races could leave pages in a file past i_size
+ * until the file is removed. Warn in the unlikely case of
+ * allocation failure.
+ */
+ pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
+ return;
+ }
+
+ kref_init(&vma_lock->refs);
+ init_rwsem(&vma_lock->rw_sema);
+ vma_lock->vma = vma;
+ vma->vm_private_data = vma_lock;
+}
+
/*
* Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
* and returns the corresponding pte. While this is not necessary for the
* !shared pmd case because we can allocate the pmd later as well, it makes the
- * code much cleaner.
- *
- * This routine must be called with i_mmap_rwsem held in at least read mode if
- * sharing is possible. For hugetlbfs, this prevents removal of any page
- * table entries associated with the address space. This is important as we
- * are setting up sharing based on existing page table entries (mappings).
+ * code much cleaner. pmd allocation is essential for the shared case because
+ * pud has to be populated inside the same i_mmap_rwsem section - otherwise
+ * racing tasks could either miss the sharing (see huge_pte_offset) or select a
+ * bad pmd for sharing.
*/
pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long addr, pud_t *pud)
pte_t *pte;
spinlock_t *ptl;
- i_mmap_assert_locked(mapping);
+ i_mmap_lock_read(mapping);
vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
if (svma == vma)
continue;
spin_unlock(ptl);
out:
pte = (pte_t *)pmd_alloc(mm, pud, addr);
+ i_mmap_unlock_read(mapping);
return pte;
}
* indicated by page_count > 1, unmap is achieved by clearing pud and
* decrementing the ref count. If count == 1, the pte page is not shared.
*
- * Called with page table lock held and i_mmap_rwsem held in write mode.
+ * Called with page table lock held.
*
* returns: 1 successfully unmapped a shared pte page
* 0 the underlying pte page is not shared, or it is the last user
pud_t *pud = pud_offset(p4d, addr);
i_mmap_assert_write_locked(vma->vm_file->f_mapping);
+ hugetlb_vma_assert_locked(vma);
BUG_ON(page_count(virt_to_page(ptep)) == 0);
if (page_count(virt_to_page(ptep)) == 1)
return 0;
}
#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
+
+void hugetlb_vma_lock_read(struct vm_area_struct *vma)
+{
+}
+
+void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
+{
+}
+
+void hugetlb_vma_lock_write(struct vm_area_struct *vma)
+{
+}
+
+void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
+{
+}
+
+int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
+{
+ return 1;
+}
+
+void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
+{
+}
+
+void hugetlb_vma_lock_release(struct kref *kref)
+{
+}
+
+static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
+{
+}
+
+static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
+{
+}
+
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
+{
+}
+
pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long addr, pud_t *pud)
{
}
struct page * __weak
-follow_huge_pmd(struct mm_struct *mm, unsigned long address,
- pmd_t *pmd, int flags)
+follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address, int flags)
{
+ struct hstate *h = hstate_vma(vma);
+ struct mm_struct *mm = vma->vm_mm;
struct page *page = NULL;
spinlock_t *ptl;
- pte_t pte;
+ pte_t *ptep, pte;
/*
* FOLL_PIN is not supported for follow_page(). Ordinary GUP goes via
return NULL;
retry:
- ptl = pmd_lockptr(mm, pmd);
- spin_lock(ptl);
- /*
- * make sure that the address range covered by this pmd is not
- * unmapped from other threads.
- */
- if (!pmd_huge(*pmd))
- goto out;
- pte = huge_ptep_get((pte_t *)pmd);
+ ptep = huge_pte_offset(mm, address, huge_page_size(h));
+ if (!ptep)
+ return NULL;
+
+ ptl = huge_pte_lock(h, mm, ptep);
+ pte = huge_ptep_get(ptep);
if (pte_present(pte)) {
- page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
+ page = pte_page(pte) +
+ ((address & ~huge_page_mask(h)) >> PAGE_SHIFT);
/*
* try_grab_page() should always succeed here, because: a) we
* hold the pmd (ptl) lock, and b) we've just checked that the
} else {
if (is_hugetlb_entry_migration(pte)) {
spin_unlock(ptl);
- __migration_entry_wait_huge((pte_t *)pmd, ptl);
+ __migration_entry_wait_huge(ptep, ptl);
goto retry;
}
/*
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
start, end);
mmu_notifier_invalidate_range_start(&range);
+ hugetlb_vma_lock_write(vma);
i_mmap_lock_write(vma->vm_file->f_mapping);
for (address = start; address < end; address += PUD_SIZE) {
ptep = huge_pte_offset(mm, address, sz);
}
flush_hugetlb_tlb_range(vma, start, end);
i_mmap_unlock_write(vma->vm_file->f_mapping);
+ hugetlb_vma_unlock_write(vma);
/*
* No need to call mmu_notifier_invalidate_range(), see
* Documentation/mm/mmu_notifier.rst.
hugetlb_cma_size = 0;
}
-void __init hugetlb_cma_check(void)
+static void __init hugetlb_cma_check(void)
{
if (!hugetlb_cma_size || cma_reserve_called)
return;
static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)
{
- int idx;
+ struct hstate *h;
- for (idx = 0; idx < hugetlb_max_hstate; idx++) {
+ for_each_hstate(h) {
if (page_counter_read(
- hugetlb_cgroup_counter_from_cgroup(h_cg, idx)))
+ hugetlb_cgroup_counter_from_cgroup(h_cg, hstate_index(h))))
return true;
}
return false;
* function.
*/
for_each_node(node) {
- /* Set node_to_alloc to -1 for offline nodes. */
+ /* Set node_to_alloc to NUMA_NO_NODE for offline nodes. */
int node_to_alloc =
- node_state(node, N_NORMAL_MEMORY) ? node : -1;
+ node_state(node, N_NORMAL_MEMORY) ? node : NUMA_NO_NODE;
h_cgroup->nodeinfo[node] =
kzalloc_node(sizeof(struct hugetlb_cgroup_per_node),
GFP_KERNEL, node_to_alloc);
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
struct hstate *h;
struct page *page;
- int idx;
do {
- idx = 0;
for_each_hstate(h) {
spin_lock_irq(&hugetlb_lock);
list_for_each_entry(page, &h->hugepage_activelist, lru)
- hugetlb_cgroup_move_parent(idx, h_cg, page);
+ hugetlb_cgroup_move_parent(hstate_index(h), h_cg, page);
spin_unlock_irq(&hugetlb_lock);
- idx++;
}
cond_resched();
} while (hugetlb_cgroup_have_usage(h_cg));
if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages)
return;
- if (rg->reservation_counter && resv->pages_per_hpage && nr_pages > 0 &&
+ if (rg->reservation_counter && resv->pages_per_hpage &&
!resv->reservation_counter) {
page_counter_uncharge(rg->reservation_counter,
nr_pages * resv->pages_per_hpage);
static char *mem_fmt(char *buf, int size, unsigned long hsize)
{
- if (hsize >= (1UL << 30))
- snprintf(buf, size, "%luGB", hsize >> 30);
- else if (hsize >= (1UL << 20))
- snprintf(buf, size, "%luMB", hsize >> 20);
+ if (hsize >= SZ_1G)
+ snprintf(buf, size, "%luGB", hsize / SZ_1G);
+ else if (hsize >= SZ_1M)
+ snprintf(buf, size, "%luMB", hsize / SZ_1M);
else
- snprintf(buf, size, "%luKB", hsize >> 10);
+ snprintf(buf, size, "%luKB", hsize / SZ_1K);
return buf;
}
static inline void reset_struct_pages(struct page *start)
{
- int i;
struct page *from = start + NR_RESET_STRUCT_PAGE;
- for (i = 0; i < NR_RESET_STRUCT_PAGE; i++)
- memcpy(start + i, from, sizeof(*from));
+ BUILD_BUG_ON(NR_RESET_STRUCT_PAGE * 2 > PAGE_SIZE / sizeof(struct page));
+ memcpy(start, from, sizeof(*from) * NR_RESET_STRUCT_PAGE);
}
static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
copy_page(to, (void *)walk->reuse_addr);
reset_struct_pages(to);
+ /*
+ * Makes sure that preceding stores to the page contents become visible
+ * before the set_pte_at() write.
+ */
+ smp_wmb();
set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
}
DEFINE_DEBUGFS_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n");
DEFINE_DEBUGFS_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n");
-static void pfn_inject_exit(void)
+static void __exit pfn_inject_exit(void)
{
hwpoison_filter_enable = 0;
debugfs_remove_recursive(hwpoison_dir);
}
-static int pfn_inject_init(void)
+static int __init pfn_inject_init(void)
{
hwpoison_dir = debugfs_create_dir("hwpoison", NULL);
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm_types.h>
-#include <linux/rbtree.h>
+#include <linux/maple_tree.h>
#include <linux/rwsem.h>
#include <linux/spinlock.h>
#include <linux/list.h>
* and size this cpu_bitmask to NR_CPUS.
*/
struct mm_struct init_mm = {
- .mm_rb = RB_ROOT,
+ .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, init_mm.mmap_lock),
.pgd = swapper_pg_dir,
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),
void folio_rotate_reclaimable(struct folio *folio);
bool __folio_end_writeback(struct folio *folio);
void deactivate_file_folio(struct folio *folio);
+void folio_activate(struct folio *folio);
-void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
- unsigned long floor, unsigned long ceiling);
+void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt,
+ struct vm_area_struct *start_vma, unsigned long floor,
+ unsigned long ceiling);
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
struct zap_details;
/*
* in mm/rmap.c:
*/
-extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
+pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
/*
* in mm/page_alloc.c
extern void free_unref_page(struct page *page, unsigned int order);
extern void free_unref_page_list(struct list_head *list);
-extern void zone_pcp_update(struct zone *zone, int cpu_online);
extern void zone_pcp_reset(struct zone *zone);
extern void zone_pcp_disable(struct zone *zone);
extern void zone_pcp_enable(struct zone *zone);
}
/* mm/util.c */
-void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev);
-void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma);
struct anon_vma *folio_anon_vma(struct folio *folio);
#ifdef CONFIG_MMU
}
#endif /* !CONFIG_MMU */
-/*
- * Return the mem_map entry representing the 'offset' subpage within
- * the maximally aligned gigantic page 'base'. Handle any discontiguity
- * in the mem_map at MAX_ORDER_NR_PAGES boundaries.
- */
-static inline struct page *mem_map_offset(struct page *base, int offset)
-{
- if (unlikely(offset >= MAX_ORDER_NR_PAGES))
- return nth_page(base, offset);
- return base + offset;
-}
-
-/*
- * Iterator over all subpages within the maximally aligned gigantic
- * page 'base'. Handle any discontiguity in the mem_map.
- */
-static inline struct page *mem_map_next(struct page *iter,
- struct page *base, int offset)
-{
- if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
- unsigned long pfn = page_to_pfn(base) + offset;
- if (!pfn_valid(pfn))
- return NULL;
- return pfn_to_page(pfn);
- }
- return iter + 1;
-}
-
/* Memory initialisation debug and verification */
enum mminit_level {
MMINIT_WARNING,
}
#endif
+int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+ pgprot_t prot, struct page **pages,
+ unsigned int page_shift);
+
void vunmap_range_noflush(unsigned long start, unsigned long end);
+void __vunmap_range_noflush(unsigned long start, unsigned long end);
+
int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
unsigned long addr, int page_nid, int *flags);
*/
struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
-DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
-
extern bool mirrored_kernelcore;
static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
CFLAGS_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME)
CFLAGS_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME)
+CFLAGS_KASAN_TEST := $(CFLAGS_KASAN) -fno-builtin $(call cc-disable-warning, vla)
+
+CFLAGS_kasan_test.o := $(CFLAGS_KASAN_TEST)
+CFLAGS_kasan_test_module.o := $(CFLAGS_KASAN_TEST)
+
obj-y := common.o report.o
obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quarantine.o
obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o tags.o report_tags.o
obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o tags.o report_tags.o
+
+obj-$(CONFIG_KASAN_KUNIT_TEST) += kasan_test.o
+obj-$(CONFIG_KASAN_MODULE_TEST) += kasan_test_module.o
#include "kasan.h"
#include "../slab.h"
+struct slab *kasan_addr_to_slab(const void *addr)
+{
+ if (virt_addr_valid(addr))
+ return virt_to_slab(addr);
+ return NULL;
+}
+
depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc)
{
unsigned long entries[KASAN_STACK_DEPTH];
unsigned int nr_entries;
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
- return __stack_depot_save(entries, nr_entries, flags, can_alloc);
+ return __stack_depot_save(entries, nr_entries, 0, flags, can_alloc);
}
void kasan_set_track(struct kasan_track *track, gfp_t flags)
}
#endif /* CONFIG_KASAN_STACK */
-/*
- * Only allow cache merging when stack collection is disabled and no metadata
- * is present.
- */
-slab_flags_t __kasan_never_merge(void)
-{
- if (kasan_stack_collection_enabled())
- return SLAB_KASAN;
- return 0;
-}
-
void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init)
{
u8 tag;
KASAN_PAGE_FREE, init);
}
-/*
- * Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
- * For larger allocations larger redzones are used.
- */
-static inline unsigned int optimal_redzone(unsigned int object_size)
-{
- return
- object_size <= 64 - 16 ? 16 :
- object_size <= 128 - 32 ? 32 :
- object_size <= 512 - 64 ? 64 :
- object_size <= 4096 - 128 ? 128 :
- object_size <= (1 << 14) - 256 ? 256 :
- object_size <= (1 << 15) - 512 ? 512 :
- object_size <= (1 << 16) - 1024 ? 1024 : 2048;
-}
-
-void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
- slab_flags_t *flags)
-{
- unsigned int ok_size;
- unsigned int optimal_size;
-
- /*
- * SLAB_KASAN is used to mark caches as ones that are sanitized by
- * KASAN. Currently this flag is used in two places:
- * 1. In slab_ksize() when calculating the size of the accessible
- * memory within the object.
- * 2. In slab_common.c to prevent merging of sanitized caches.
- */
- *flags |= SLAB_KASAN;
-
- if (!kasan_stack_collection_enabled())
- return;
-
- ok_size = *size;
-
- /* Add alloc meta into redzone. */
- cache->kasan_info.alloc_meta_offset = *size;
- *size += sizeof(struct kasan_alloc_meta);
-
- /*
- * If alloc meta doesn't fit, don't add it.
- * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal
- * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for
- * larger sizes.
- */
- if (*size > KMALLOC_MAX_SIZE) {
- cache->kasan_info.alloc_meta_offset = 0;
- *size = ok_size;
- /* Continue, since free meta might still fit. */
- }
-
- /* Only the generic mode uses free meta or flexible redzones. */
- if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) {
- cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
- return;
- }
-
- /*
- * Add free meta into redzone when it's not possible to store
- * it in the object. This is the case when:
- * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can
- * be touched after it was freed, or
- * 2. Object has a constructor, which means it's expected to
- * retain its content until the next allocation, or
- * 3. Object is too small.
- * Otherwise cache->kasan_info.free_meta_offset = 0 is implied.
- */
- if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||
- cache->object_size < sizeof(struct kasan_free_meta)) {
- ok_size = *size;
-
- cache->kasan_info.free_meta_offset = *size;
- *size += sizeof(struct kasan_free_meta);
-
- /* If free meta doesn't fit, don't add it. */
- if (*size > KMALLOC_MAX_SIZE) {
- cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
- *size = ok_size;
- }
- }
-
- /* Calculate size with optimal redzone. */
- optimal_size = cache->object_size + optimal_redzone(cache->object_size);
- /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */
- if (optimal_size > KMALLOC_MAX_SIZE)
- optimal_size = KMALLOC_MAX_SIZE;
- /* Use optimal size if the size with added metas is not large enough. */
- if (*size < optimal_size)
- *size = optimal_size;
-}
-
void __kasan_cache_create_kmalloc(struct kmem_cache *cache)
{
cache->kasan_info.is_kmalloc = true;
}
-size_t __kasan_metadata_size(struct kmem_cache *cache)
-{
- if (!kasan_stack_collection_enabled())
- return 0;
- return (cache->kasan_info.alloc_meta_offset ?
- sizeof(struct kasan_alloc_meta) : 0) +
- (cache->kasan_info.free_meta_offset ?
- sizeof(struct kasan_free_meta) : 0);
-}
-
-struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
- const void *object)
-{
- if (!cache->kasan_info.alloc_meta_offset)
- return NULL;
- return kasan_reset_tag(object) + cache->kasan_info.alloc_meta_offset;
-}
-
-#ifdef CONFIG_KASAN_GENERIC
-struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
- const void *object)
-{
- BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);
- if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META)
- return NULL;
- return kasan_reset_tag(object) + cache->kasan_info.free_meta_offset;
-}
-#endif
-
void __kasan_poison_slab(struct slab *slab)
{
struct page *page = slab_page(slab);
void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
const void *object)
{
- struct kasan_alloc_meta *alloc_meta;
-
- if (kasan_stack_collection_enabled()) {
- alloc_meta = kasan_get_alloc_meta(cache, object);
- if (alloc_meta)
- __memset(alloc_meta, 0, sizeof(*alloc_meta));
- }
+ /* Initialize per-object metadata if it is present. */
+ if (kasan_requires_meta())
+ kasan_init_object_meta(cache, object);
/* Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS */
object = set_tag(object, assign_tag(cache, object, true));
static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object,
unsigned long ip, bool quarantine, bool init)
{
- u8 tag;
void *tagged_object;
if (!kasan_arch_is_ready())
return false;
- tag = get_tag(object);
tagged_object = object;
object = kasan_reset_tag(object);
return false;
if (kasan_stack_collection_enabled())
- kasan_set_free_info(cache, object, tag);
+ kasan_save_free_info(cache, tagged_object);
return kasan_quarantine_put(cache, object);
}
}
}
-static void set_alloc_info(struct kmem_cache *cache, void *object,
- gfp_t flags, bool is_kmalloc)
-{
- struct kasan_alloc_meta *alloc_meta;
-
- /* Don't save alloc info for kmalloc caches in kasan_slab_alloc(). */
- if (cache->kasan_info.is_kmalloc && !is_kmalloc)
- return;
-
- alloc_meta = kasan_get_alloc_meta(cache, object);
- if (alloc_meta)
- kasan_set_track(&alloc_meta->alloc_track, flags);
-}
-
void * __must_check __kasan_slab_alloc(struct kmem_cache *cache,
void *object, gfp_t flags, bool init)
{
kasan_unpoison(tagged_object, cache->object_size, init);
/* Save alloc info (if possible) for non-kmalloc() allocations. */
- if (kasan_stack_collection_enabled())
- set_alloc_info(cache, (void *)object, flags, false);
+ if (kasan_stack_collection_enabled() && !cache->kasan_info.is_kmalloc)
+ kasan_save_alloc_info(cache, tagged_object, flags);
return tagged_object;
}
* Save alloc info (if possible) for kmalloc() allocations.
* This also rewrites the alloc info when called from kasan_krealloc().
*/
- if (kasan_stack_collection_enabled())
- set_alloc_info(cache, (void *)object, flags, true);
+ if (kasan_stack_collection_enabled() && cache->kasan_info.is_kmalloc)
+ kasan_save_alloc_info(cache, (void *)object, flags);
/* Keep the tag that was set by kasan_slab_alloc(). */
return (void *)object;
DEFINE_ASAN_SET_SHADOW(f5);
DEFINE_ASAN_SET_SHADOW(f8);
+/* Only allow cache merging when no per-object metadata is present. */
+slab_flags_t kasan_never_merge(void)
+{
+ if (!kasan_requires_meta())
+ return 0;
+ return SLAB_KASAN;
+}
+
+/*
+ * Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
+ * For larger allocations larger redzones are used.
+ */
+static inline unsigned int optimal_redzone(unsigned int object_size)
+{
+ return
+ object_size <= 64 - 16 ? 16 :
+ object_size <= 128 - 32 ? 32 :
+ object_size <= 512 - 64 ? 64 :
+ object_size <= 4096 - 128 ? 128 :
+ object_size <= (1 << 14) - 256 ? 256 :
+ object_size <= (1 << 15) - 512 ? 512 :
+ object_size <= (1 << 16) - 1024 ? 1024 : 2048;
+}
+
+void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
+ slab_flags_t *flags)
+{
+ unsigned int ok_size;
+ unsigned int optimal_size;
+
+ if (!kasan_requires_meta())
+ return;
+
+ /*
+ * SLAB_KASAN is used to mark caches that are sanitized by KASAN
+ * and that thus have per-object metadata.
+ * Currently this flag is used in two places:
+ * 1. In slab_ksize() to account for per-object metadata when
+ * calculating the size of the accessible memory within the object.
+ * 2. In slab_common.c via kasan_never_merge() to prevent merging of
+ * caches with per-object metadata.
+ */
+ *flags |= SLAB_KASAN;
+
+ ok_size = *size;
+
+ /* Add alloc meta into redzone. */
+ cache->kasan_info.alloc_meta_offset = *size;
+ *size += sizeof(struct kasan_alloc_meta);
+
+ /*
+ * If alloc meta doesn't fit, don't add it.
+ * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal
+ * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for
+ * larger sizes.
+ */
+ if (*size > KMALLOC_MAX_SIZE) {
+ cache->kasan_info.alloc_meta_offset = 0;
+ *size = ok_size;
+ /* Continue, since free meta might still fit. */
+ }
+
+ /*
+ * Add free meta into redzone when it's not possible to store
+ * it in the object. This is the case when:
+ * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can
+ * be touched after it was freed, or
+ * 2. Object has a constructor, which means it's expected to
+ * retain its content until the next allocation, or
+ * 3. Object is too small.
+ * Otherwise cache->kasan_info.free_meta_offset = 0 is implied.
+ */
+ if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||
+ cache->object_size < sizeof(struct kasan_free_meta)) {
+ ok_size = *size;
+
+ cache->kasan_info.free_meta_offset = *size;
+ *size += sizeof(struct kasan_free_meta);
+
+ /* If free meta doesn't fit, don't add it. */
+ if (*size > KMALLOC_MAX_SIZE) {
+ cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
+ *size = ok_size;
+ }
+ }
+
+ /* Calculate size with optimal redzone. */
+ optimal_size = cache->object_size + optimal_redzone(cache->object_size);
+ /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */
+ if (optimal_size > KMALLOC_MAX_SIZE)
+ optimal_size = KMALLOC_MAX_SIZE;
+ /* Use optimal size if the size with added metas is not large enough. */
+ if (*size < optimal_size)
+ *size = optimal_size;
+}
+
+struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
+ const void *object)
+{
+ if (!cache->kasan_info.alloc_meta_offset)
+ return NULL;
+ return (void *)object + cache->kasan_info.alloc_meta_offset;
+}
+
+struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
+ const void *object)
+{
+ BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);
+ if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META)
+ return NULL;
+ return (void *)object + cache->kasan_info.free_meta_offset;
+}
+
+void kasan_init_object_meta(struct kmem_cache *cache, const void *object)
+{
+ struct kasan_alloc_meta *alloc_meta;
+
+ alloc_meta = kasan_get_alloc_meta(cache, object);
+ if (alloc_meta)
+ __memset(alloc_meta, 0, sizeof(*alloc_meta));
+}
+
+size_t kasan_metadata_size(struct kmem_cache *cache)
+{
+ if (!kasan_requires_meta())
+ return 0;
+ return (cache->kasan_info.alloc_meta_offset ?
+ sizeof(struct kasan_alloc_meta) : 0) +
+ ((cache->kasan_info.free_meta_offset &&
+ cache->kasan_info.free_meta_offset != KASAN_NO_FREE_META) ?
+ sizeof(struct kasan_free_meta) : 0);
+}
+
static void __kasan_record_aux_stack(void *addr, bool can_alloc)
{
struct slab *slab = kasan_addr_to_slab(addr);
return __kasan_record_aux_stack(addr, false);
}
-void kasan_set_free_info(struct kmem_cache *cache,
- void *object, u8 tag)
+void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
+{
+ struct kasan_alloc_meta *alloc_meta;
+
+ alloc_meta = kasan_get_alloc_meta(cache, object);
+ if (alloc_meta)
+ kasan_set_track(&alloc_meta->alloc_track, flags);
+}
+
+void kasan_save_free_info(struct kmem_cache *cache, void *object)
{
struct kasan_free_meta *free_meta;
/* The object was freed and has free track set. */
*(u8 *)kasan_mem_to_shadow(object) = KASAN_SLAB_FREETRACK;
}
-
-struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
- void *object, u8 tag)
-{
- if (*(u8 *)kasan_mem_to_shadow(object) != KASAN_SLAB_FREETRACK)
- return NULL;
- /* Free meta must be present with KASAN_SLAB_FREETRACK. */
- return &kasan_get_free_meta(cache, object)->free_track;
-}
KASAN_ARG_VMALLOC_ON,
};
-enum kasan_arg_stacktrace {
- KASAN_ARG_STACKTRACE_DEFAULT,
- KASAN_ARG_STACKTRACE_OFF,
- KASAN_ARG_STACKTRACE_ON,
-};
-
static enum kasan_arg kasan_arg __ro_after_init;
static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
-static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
/*
* Whether KASAN is enabled at all.
/* Whether to enable vmalloc tagging. */
DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
-/* Whether to collect alloc/free stack traces. */
-DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
-
/* kasan=off/on */
static int __init early_kasan_flag(char *arg)
{
}
early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
-/* kasan.stacktrace=off/on */
-static int __init early_kasan_flag_stacktrace(char *arg)
-{
- if (!arg)
- return -EINVAL;
-
- if (!strcmp(arg, "off"))
- kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
- else if (!strcmp(arg, "on"))
- kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
- else
- return -EINVAL;
-
- return 0;
-}
-early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
-
static inline const char *kasan_mode_info(void)
{
if (kasan_mode == KASAN_MODE_ASYNC)
break;
}
- switch (kasan_arg_stacktrace) {
- case KASAN_ARG_STACKTRACE_DEFAULT:
- /* Default is specified by kasan_flag_stacktrace definition. */
- break;
- case KASAN_ARG_STACKTRACE_OFF:
- static_branch_disable(&kasan_flag_stacktrace);
- break;
- case KASAN_ARG_STACKTRACE_ON:
- static_branch_enable(&kasan_flag_stacktrace);
- break;
- }
+ kasan_init_tags();
/* KASAN is now initialized, enable it. */
static_branch_enable(&kasan_flag_enabled);
#ifndef __MM_KASAN_KASAN_H
#define __MM_KASAN_KASAN_H
+#include <linux/atomic.h>
#include <linux/kasan.h>
#include <linux/kasan-tags.h>
#include <linux/kfence.h>
#include <linux/stackdepot.h>
-#ifdef CONFIG_KASAN_HW_TAGS
+#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
#include <linux/static_key.h>
+
+DECLARE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
+
+static inline bool kasan_stack_collection_enabled(void)
+{
+ return static_branch_unlikely(&kasan_flag_stacktrace);
+}
+
+#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
+
+static inline bool kasan_stack_collection_enabled(void)
+{
+ return true;
+}
+
+#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
+
+#ifdef CONFIG_KASAN_HW_TAGS
+
#include "../slab.h"
DECLARE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
-DECLARE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
enum kasan_mode {
KASAN_MODE_SYNC,
return static_branch_likely(&kasan_flag_vmalloc);
}
-static inline bool kasan_stack_collection_enabled(void)
-{
- return static_branch_unlikely(&kasan_flag_stacktrace);
-}
-
static inline bool kasan_async_fault_possible(void)
{
return kasan_mode == KASAN_MODE_ASYNC || kasan_mode == KASAN_MODE_ASYMM;
return kasan_mode == KASAN_MODE_SYNC || kasan_mode == KASAN_MODE_ASYMM;
}
-#else
-
-static inline bool kasan_stack_collection_enabled(void)
-{
- return true;
-}
+#else /* CONFIG_KASAN_HW_TAGS */
static inline bool kasan_async_fault_possible(void)
{
return true;
}
-#endif
+#endif /* CONFIG_KASAN_HW_TAGS */
+
+#ifdef CONFIG_KASAN_GENERIC
+
+/* Generic KASAN uses per-object metadata to store stack traces. */
+static inline bool kasan_requires_meta(void)
+{
+ /*
+ * Technically, Generic KASAN always collects stack traces right now.
+ * However, let's use kasan_stack_collection_enabled() in case the
+ * kasan.stacktrace command-line argument is changed to affect
+ * Generic KASAN.
+ */
+ return kasan_stack_collection_enabled();
+}
+
+#else /* CONFIG_KASAN_GENERIC */
+
+/* Tag-based KASAN modes do not use per-object metadata. */
+static inline bool kasan_requires_meta(void)
+{
+ return false;
+}
+
+#endif /* CONFIG_KASAN_GENERIC */
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
#define KASAN_GRANULE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
#define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE)
#define META_ROWS_AROUND_ADDR 2
+#define KASAN_STACK_DEPTH 64
+
+struct kasan_track {
+ u32 pid;
+ depot_stack_handle_t stack;
+};
+
enum kasan_report_type {
KASAN_REPORT_ACCESS,
KASAN_REPORT_INVALID_FREE,
};
struct kasan_report_info {
+ /* Filled in by kasan_report_*(). */
enum kasan_report_type type;
void *access_addr;
- void *first_bad_addr;
size_t access_size;
bool is_write;
unsigned long ip;
+
+ /* Filled in by the common reporting code. */
+ void *first_bad_addr;
+ struct kmem_cache *cache;
+ void *object;
+
+ /* Filled in by the mode-specific reporting code. */
+ const char *bug_type;
+ struct kasan_track alloc_track;
+ struct kasan_track free_track;
};
/* Do not change the struct layout: compiler ABI. */
#endif
};
-/* Structures for keeping alloc and free tracks. */
+/* Structures for keeping alloc and free meta. */
-#define KASAN_STACK_DEPTH 64
-
-struct kasan_track {
- u32 pid;
- depot_stack_handle_t stack;
-};
-
-#if defined(CONFIG_KASAN_TAGS_IDENTIFY) && defined(CONFIG_KASAN_SW_TAGS)
-#define KASAN_NR_FREE_STACKS 5
-#else
-#define KASAN_NR_FREE_STACKS 1
-#endif
+#ifdef CONFIG_KASAN_GENERIC
struct kasan_alloc_meta {
struct kasan_track alloc_track;
- /* Generic mode stores free track in kasan_free_meta. */
-#ifdef CONFIG_KASAN_GENERIC
+ /* Free track is stored in kasan_free_meta. */
depot_stack_handle_t aux_stack[2];
-#else
- struct kasan_track free_track[KASAN_NR_FREE_STACKS];
-#endif
-#ifdef CONFIG_KASAN_TAGS_IDENTIFY
- u8 free_pointer_tag[KASAN_NR_FREE_STACKS];
- u8 free_track_idx;
-#endif
};
struct qlist_node {
* After that, slab allocator stores the freelist pointer in the object.
*/
struct kasan_free_meta {
-#ifdef CONFIG_KASAN_GENERIC
struct qlist_node quarantine_link;
struct kasan_track free_track;
-#endif
};
+#endif /* CONFIG_KASAN_GENERIC */
+
+#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
+
+struct kasan_stack_ring_entry {
+ void *ptr;
+ size_t size;
+ u32 pid;
+ depot_stack_handle_t stack;
+ bool is_free;
+};
+
+struct kasan_stack_ring {
+ rwlock_t lock;
+ size_t size;
+ atomic64_t pos;
+ struct kasan_stack_ring_entry *entries;
+};
+
+#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
+
#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
/* Used in KUnit-compatible KASAN tests. */
struct kunit_kasan_status {
};
#endif
-struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
- const void *object);
-#ifdef CONFIG_KASAN_GENERIC
-struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
- const void *object);
-#endif
-
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
+void *kasan_find_first_bad_addr(void *addr, size_t size);
+void kasan_complete_mode_report_info(struct kasan_report_info *info);
+void kasan_metadata_fetch_row(char *buffer, void *row);
+
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
void kasan_print_tags(u8 addr_tag, const void *addr);
#else
static inline void kasan_print_tags(u8 addr_tag, const void *addr) { }
#endif
-void *kasan_find_first_bad_addr(void *addr, size_t size);
-const char *kasan_get_bug_type(struct kasan_report_info *info);
-void kasan_metadata_fetch_row(char *buffer, void *row);
-
#if defined(CONFIG_KASAN_STACK)
void kasan_print_address_stack_frame(const void *addr);
#else
static inline void kasan_print_address_stack_frame(const void *addr) { }
#endif
+#ifdef CONFIG_KASAN_GENERIC
+void kasan_print_aux_stacks(struct kmem_cache *cache, const void *object);
+#else
+static inline void kasan_print_aux_stacks(struct kmem_cache *cache, const void *object) { }
+#endif
+
bool kasan_report(unsigned long addr, size_t size,
bool is_write, unsigned long ip);
void kasan_report_invalid_free(void *object, unsigned long ip, enum kasan_report_type type);
-struct page *kasan_addr_to_page(const void *addr);
struct slab *kasan_addr_to_slab(const void *addr);
+#ifdef CONFIG_KASAN_GENERIC
+void kasan_init_cache_meta(struct kmem_cache *cache, unsigned int *size);
+void kasan_init_object_meta(struct kmem_cache *cache, const void *object);
+struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
+ const void *object);
+struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
+ const void *object);
+#else
+static inline void kasan_init_cache_meta(struct kmem_cache *cache, unsigned int *size) { }
+static inline void kasan_init_object_meta(struct kmem_cache *cache, const void *object) { }
+#endif
+
depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc);
void kasan_set_track(struct kasan_track *track, gfp_t flags);
-void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag);
-struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
- void *object, u8 tag);
+void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags);
+void kasan_save_free_info(struct kmem_cache *cache, void *object);
#if defined(CONFIG_KASAN_GENERIC) && \
(defined(CONFIG_SLAB) || defined(CONFIG_SLUB))
#endif /* CONFIG_KASAN_HW_TAGS */
+#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
+void __init kasan_init_tags(void);
+#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
+
#if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
void kasan_force_async_fault(void);
#include <kunit/test.h>
-#include "../mm/kasan/kasan.h"
+#include "kasan.h"
#define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE)
ptr2 = krealloc(ptr1, size2, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2);
+ /* Suppress -Warray-bounds warnings. */
+ OPTIMIZER_HIDE_VAR(ptr2);
+
/* All offsets up to size2 must be accessible. */
ptr2[size1 - 1] = 'x';
ptr2[size1] = 'x';
ptr2 = krealloc(ptr1, size2, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2);
+ /* Suppress -Warray-bounds warnings. */
+ OPTIMIZER_HIDE_VAR(ptr2);
+
/* Must be accessible for all modes. */
ptr2[size2 - 1] = 'x';
{
char *ptr;
size_t size = 64;
- volatile size_t invalid_size = size;
+ size_t invalid_size = size;
ptr = kmalloc(size, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr);
memset((char *)ptr, 0, 64);
OPTIMIZER_HIDE_VAR(ptr);
+ OPTIMIZER_HIDE_VAR(invalid_size);
KUNIT_EXPECT_KASAN_FAIL(test,
memmove((char *)ptr, (char *)ptr + 4, invalid_size));
kfree(ptr);
kfree(ptr2);
}
+/*
+ * Check that KASAN detects use-after-free when another object was allocated in
+ * the same slot. Relevant for the tag-based modes, which do not use quarantine.
+ */
+static void kmalloc_uaf3(struct kunit *test)
+{
+ char *ptr1, *ptr2;
+ size_t size = 100;
+
+ /* This test is specifically crafted for tag-based modes. */
+ KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC);
+
+ ptr1 = kmalloc(size, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1);
+ kfree(ptr1);
+
+ ptr2 = kmalloc(size, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2);
+ kfree(ptr2);
+
+ KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr1)[8]);
+}
+
static void kfree_via_page(struct kunit *test)
{
char *ptr;
KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC);
for (i = 0; i < 256; i++) {
- size = (get_random_int() % 1024) + 1;
+ size = prandom_u32_max(1024) + 1;
ptr = kmalloc(size, GFP_KERNEL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr);
KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN);
}
for (i = 0; i < 256; i++) {
- order = (get_random_int() % 4) + 1;
+ order = prandom_u32_max(4) + 1;
pages = alloc_pages(GFP_KERNEL, order);
ptr = page_address(pages);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr);
return;
for (i = 0; i < 256; i++) {
- size = (get_random_int() % 1024) + 1;
+ size = prandom_u32_max(1024) + 1;
ptr = vmalloc(size);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr);
KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN);
KUNIT_CASE(kmalloc_uaf),
KUNIT_CASE(kmalloc_uaf_memset),
KUNIT_CASE(kmalloc_uaf2),
+ KUNIT_CASE(kmalloc_uaf3),
KUNIT_CASE(kfree_via_page),
KUNIT_CASE(kfree_via_phys),
KUNIT_CASE(kmem_cache_oob),
#include <linux/slab.h>
#include <linux/uaccess.h>
-#include "../mm/kasan/kasan.h"
+#include "kasan.h"
static noinline void __init copy_user_test(void)
{
static void print_error_description(struct kasan_report_info *info)
{
- if (info->type == KASAN_REPORT_INVALID_FREE) {
- pr_err("BUG: KASAN: invalid-free in %pS\n", (void *)info->ip);
- return;
- }
+ pr_err("BUG: KASAN: %s in %pS\n", info->bug_type, (void *)info->ip);
- if (info->type == KASAN_REPORT_DOUBLE_FREE) {
- pr_err("BUG: KASAN: double-free in %pS\n", (void *)info->ip);
+ if (info->type != KASAN_REPORT_ACCESS) {
+ pr_err("Free of addr %px by task %s/%d\n",
+ info->access_addr, current->comm, task_pid_nr(current));
return;
}
- pr_err("BUG: KASAN: %s in %pS\n",
- kasan_get_bug_type(info), (void *)info->ip);
if (info->access_size)
pr_err("%s of size %zu at addr %px by task %s/%d\n",
info->is_write ? "Write" : "Read", info->access_size,
static void print_track(struct kasan_track *track, const char *prefix)
{
pr_err("%s by task %u:\n", prefix, track->pid);
- if (track->stack) {
+ if (track->stack)
stack_depot_print(track->stack);
- } else {
+ else
pr_err("(stack is not available)\n");
- }
}
-struct page *kasan_addr_to_page(const void *addr)
+static inline struct page *addr_to_page(const void *addr)
{
- if ((addr >= (void *)PAGE_OFFSET) &&
- (addr < high_memory))
+ if (virt_addr_valid(addr))
return virt_to_head_page(addr);
return NULL;
}
-struct slab *kasan_addr_to_slab(const void *addr)
-{
- if ((addr >= (void *)PAGE_OFFSET) &&
- (addr < high_memory))
- return virt_to_slab(addr);
- return NULL;
-}
-
-static void describe_object_addr(struct kmem_cache *cache, void *object,
- const void *addr)
+static void describe_object_addr(const void *addr, struct kmem_cache *cache,
+ void *object)
{
unsigned long access_addr = (unsigned long)addr;
unsigned long object_addr = (unsigned long)object;
(void *)(object_addr + cache->object_size));
}
-static void describe_object_stacks(struct kmem_cache *cache, void *object,
- const void *addr, u8 tag)
+static void describe_object_stacks(struct kasan_report_info *info)
{
- struct kasan_alloc_meta *alloc_meta;
- struct kasan_track *free_track;
-
- alloc_meta = kasan_get_alloc_meta(cache, object);
- if (alloc_meta) {
- print_track(&alloc_meta->alloc_track, "Allocated");
+ if (info->alloc_track.stack) {
+ print_track(&info->alloc_track, "Allocated");
pr_err("\n");
}
- free_track = kasan_get_free_track(cache, object, tag);
- if (free_track) {
- print_track(free_track, "Freed");
+ if (info->free_track.stack) {
+ print_track(&info->free_track, "Freed");
pr_err("\n");
}
-#ifdef CONFIG_KASAN_GENERIC
- if (!alloc_meta)
- return;
- if (alloc_meta->aux_stack[0]) {
- pr_err("Last potentially related work creation:\n");
- stack_depot_print(alloc_meta->aux_stack[0]);
- pr_err("\n");
- }
- if (alloc_meta->aux_stack[1]) {
- pr_err("Second to last potentially related work creation:\n");
- stack_depot_print(alloc_meta->aux_stack[1]);
- pr_err("\n");
- }
-#endif
+ kasan_print_aux_stacks(info->cache, info->object);
}
-static void describe_object(struct kmem_cache *cache, void *object,
- const void *addr, u8 tag)
+static void describe_object(const void *addr, struct kasan_report_info *info)
{
if (kasan_stack_collection_enabled())
- describe_object_stacks(cache, object, addr, tag);
- describe_object_addr(cache, object, addr);
+ describe_object_stacks(info);
+ describe_object_addr(addr, info->cache, info->object);
}
static inline bool kernel_or_module_addr(const void *addr)
sizeof(init_thread_union.stack));
}
-static void print_address_description(void *addr, u8 tag)
+static void print_address_description(void *addr, u8 tag,
+ struct kasan_report_info *info)
{
- struct page *page = kasan_addr_to_page(addr);
+ struct page *page = addr_to_page(addr);
dump_stack_lvl(KERN_ERR);
pr_err("\n");
- if (page && PageSlab(page)) {
- struct slab *slab = page_slab(page);
- struct kmem_cache *cache = slab->slab_cache;
- void *object = nearest_obj(cache, slab, addr);
-
- describe_object(cache, object, addr, tag);
+ if (info->cache && info->object) {
+ describe_object(addr, info);
pr_err("\n");
}
static void print_report(struct kasan_report_info *info)
{
- void *tagged_addr = info->access_addr;
- void *untagged_addr = kasan_reset_tag(tagged_addr);
- u8 tag = get_tag(tagged_addr);
+ void *addr = kasan_reset_tag(info->access_addr);
+ u8 tag = get_tag(info->access_addr);
print_error_description(info);
- if (addr_has_metadata(untagged_addr))
+ if (addr_has_metadata(addr))
kasan_print_tags(tag, info->first_bad_addr);
pr_err("\n");
- if (addr_has_metadata(untagged_addr)) {
- print_address_description(untagged_addr, tag);
+ if (addr_has_metadata(addr)) {
+ print_address_description(addr, tag, info);
print_memory_metadata(info->first_bad_addr);
} else {
dump_stack_lvl(KERN_ERR);
}
}
+static void complete_report_info(struct kasan_report_info *info)
+{
+ void *addr = kasan_reset_tag(info->access_addr);
+ struct slab *slab;
+
+ if (info->type == KASAN_REPORT_ACCESS)
+ info->first_bad_addr = kasan_find_first_bad_addr(
+ info->access_addr, info->access_size);
+ else
+ info->first_bad_addr = addr;
+
+ slab = kasan_addr_to_slab(addr);
+ if (slab) {
+ info->cache = slab->slab_cache;
+ info->object = nearest_obj(info->cache, slab, addr);
+ } else
+ info->cache = info->object = NULL;
+
+ switch (info->type) {
+ case KASAN_REPORT_INVALID_FREE:
+ info->bug_type = "invalid-free";
+ break;
+ case KASAN_REPORT_DOUBLE_FREE:
+ info->bug_type = "double-free";
+ break;
+ default:
+ /* bug_type filled in by kasan_complete_mode_report_info. */
+ break;
+ }
+
+ /* Fill in mode-specific report info fields. */
+ kasan_complete_mode_report_info(info);
+}
+
void kasan_report_invalid_free(void *ptr, unsigned long ip, enum kasan_report_type type)
{
unsigned long flags;
start_report(&flags, true);
+ memset(&info, 0, sizeof(info));
info.type = type;
info.access_addr = ptr;
- info.first_bad_addr = kasan_reset_tag(ptr);
info.access_size = 0;
info.is_write = false;
info.ip = ip;
+ complete_report_info(&info);
+
print_report(&info);
end_report(&flags, ptr);
start_report(&irq_flags, true);
+ memset(&info, 0, sizeof(info));
info.type = KASAN_REPORT_ACCESS;
info.access_addr = ptr;
- info.first_bad_addr = kasan_find_first_bad_addr(ptr, size);
info.access_size = size;
info.is_write = is_write;
info.ip = ip;
+ complete_report_info(&info);
+
print_report(&info);
end_report(&irq_flags, ptr);
return bug_type;
}
-const char *kasan_get_bug_type(struct kasan_report_info *info)
+static const char *get_bug_type(struct kasan_report_info *info)
{
/*
* If access_size is a negative number, then it has reason to be
return get_wild_bug_type(info);
}
+void kasan_complete_mode_report_info(struct kasan_report_info *info)
+{
+ struct kasan_alloc_meta *alloc_meta;
+ struct kasan_free_meta *free_meta;
+
+ if (!info->bug_type)
+ info->bug_type = get_bug_type(info);
+
+ if (!info->cache || !info->object)
+ return;
+
+ alloc_meta = kasan_get_alloc_meta(info->cache, info->object);
+ if (alloc_meta)
+ memcpy(&info->alloc_track, &alloc_meta->alloc_track,
+ sizeof(info->alloc_track));
+
+ if (*(u8 *)kasan_mem_to_shadow(info->object) == KASAN_SLAB_FREETRACK) {
+ /* Free meta must be present with KASAN_SLAB_FREETRACK. */
+ free_meta = kasan_get_free_meta(info->cache, info->object);
+ memcpy(&info->free_track, &free_meta->free_track,
+ sizeof(info->free_track));
+ }
+}
+
void kasan_metadata_fetch_row(char *buffer, void *row)
{
memcpy(buffer, kasan_mem_to_shadow(row), META_BYTES_PER_ROW);
}
+void kasan_print_aux_stacks(struct kmem_cache *cache, const void *object)
+{
+ struct kasan_alloc_meta *alloc_meta;
+
+ alloc_meta = kasan_get_alloc_meta(cache, object);
+ if (!alloc_meta)
+ return;
+
+ if (alloc_meta->aux_stack[0]) {
+ pr_err("Last potentially related work creation:\n");
+ stack_depot_print(alloc_meta->aux_stack[0]);
+ pr_err("\n");
+ }
+ if (alloc_meta->aux_stack[1]) {
+ pr_err("Second to last potentially related work creation:\n");
+ stack_depot_print(alloc_meta->aux_stack[1]);
+ pr_err("\n");
+ }
+}
+
#ifdef CONFIG_KASAN_STACK
static bool __must_check tokenize_frame_descr(const char **frame_descr,
char *token, size_t max_tok_len,
* Copyright (c) 2020 Google, Inc.
*/
+#include <linux/atomic.h>
+
#include "kasan.h"
-#include "../slab.h"
-const char *kasan_get_bug_type(struct kasan_report_info *info)
-{
-#ifdef CONFIG_KASAN_TAGS_IDENTIFY
- struct kasan_alloc_meta *alloc_meta;
- struct kmem_cache *cache;
- struct slab *slab;
- const void *addr;
- void *object;
- u8 tag;
- int i;
-
- tag = get_tag(info->access_addr);
- addr = kasan_reset_tag(info->access_addr);
- slab = kasan_addr_to_slab(addr);
- if (slab) {
- cache = slab->slab_cache;
- object = nearest_obj(cache, slab, (void *)addr);
- alloc_meta = kasan_get_alloc_meta(cache, object);
-
- if (alloc_meta) {
- for (i = 0; i < KASAN_NR_FREE_STACKS; i++) {
- if (alloc_meta->free_pointer_tag[i] == tag)
- return "use-after-free";
- }
- }
- return "out-of-bounds";
- }
-#endif
+extern struct kasan_stack_ring stack_ring;
+static const char *get_common_bug_type(struct kasan_report_info *info)
+{
/*
* If access_size is a negative number, then it has reason to be
* defined as out-of-bounds bug type.
return "invalid-access";
}
+
+void kasan_complete_mode_report_info(struct kasan_report_info *info)
+{
+ unsigned long flags;
+ u64 pos;
+ struct kasan_stack_ring_entry *entry;
+ void *ptr;
+ u32 pid;
+ depot_stack_handle_t stack;
+ bool is_free;
+ bool alloc_found = false, free_found = false;
+
+ if ((!info->cache || !info->object) && !info->bug_type) {
+ info->bug_type = get_common_bug_type(info);
+ return;
+ }
+
+ write_lock_irqsave(&stack_ring.lock, flags);
+
+ pos = atomic64_read(&stack_ring.pos);
+
+ /*
+ * The loop below tries to find stack ring entries relevant to the
+ * buggy object. This is a best-effort process.
+ *
+ * First, another object with the same tag can be allocated in place of
+ * the buggy object. Also, since the number of entries is limited, the
+ * entries relevant to the buggy object can be overwritten.
+ */
+
+ for (u64 i = pos - 1; i != pos - 1 - stack_ring.size; i--) {
+ if (alloc_found && free_found)
+ break;
+
+ entry = &stack_ring.entries[i % stack_ring.size];
+
+ /* Paired with smp_store_release() in save_stack_info(). */
+ ptr = (void *)smp_load_acquire(&entry->ptr);
+
+ if (kasan_reset_tag(ptr) != info->object ||
+ get_tag(ptr) != get_tag(info->access_addr))
+ continue;
+
+ pid = READ_ONCE(entry->pid);
+ stack = READ_ONCE(entry->stack);
+ is_free = READ_ONCE(entry->is_free);
+
+ if (is_free) {
+ /*
+ * Second free of the same object.
+ * Give up on trying to find the alloc entry.
+ */
+ if (free_found)
+ break;
+
+ info->free_track.pid = pid;
+ info->free_track.stack = stack;
+ free_found = true;
+
+ /*
+ * If a free entry is found first, the bug is likely
+ * a use-after-free.
+ */
+ if (!info->bug_type)
+ info->bug_type = "use-after-free";
+ } else {
+ /* Second alloc of the same object. Give up. */
+ if (alloc_found)
+ break;
+
+ info->alloc_track.pid = pid;
+ info->alloc_track.stack = stack;
+ alloc_found = true;
+
+ /*
+ * If an alloc entry is found first, the bug is likely
+ * an out-of-bounds.
+ */
+ if (!info->bug_type)
+ info->bug_type = "slab-out-of-bounds";
+ }
+ }
+
+ write_unlock_irqrestore(&stack_ring.lock, flags);
+
+ /* Assign the common bug type if no entries were found. */
+ if (!info->bug_type)
+ info->bug_type = get_common_bug_type(info);
+}
for_each_possible_cpu(cpu)
per_cpu(prng_state, cpu) = (u32)get_cycles();
- pr_info("KernelAddressSanitizer initialized (sw-tags)\n");
+ kasan_init_tags();
+
+ pr_info("KernelAddressSanitizer initialized (sw-tags, stacktrace=%s)\n",
+ kasan_stack_collection_enabled() ? "on" : "off");
}
/*
* Copyright (c) 2020 Google, Inc.
*/
+#include <linux/atomic.h>
#include <linux/init.h>
#include <linux/kasan.h>
#include <linux/kernel.h>
+#include <linux/memblock.h>
#include <linux/memory.h>
#include <linux/mm.h>
#include <linux/static_key.h>
#include <linux/types.h>
#include "kasan.h"
+#include "../slab.h"
-void kasan_set_free_info(struct kmem_cache *cache,
- void *object, u8 tag)
-{
- struct kasan_alloc_meta *alloc_meta;
- u8 idx = 0;
+#define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10)
+
+enum kasan_arg_stacktrace {
+ KASAN_ARG_STACKTRACE_DEFAULT,
+ KASAN_ARG_STACKTRACE_OFF,
+ KASAN_ARG_STACKTRACE_ON,
+};
+
+static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
+
+/* Whether to collect alloc/free stack traces. */
+DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
- alloc_meta = kasan_get_alloc_meta(cache, object);
- if (!alloc_meta)
- return;
+/* Non-zero, as initial pointer values are 0. */
+#define STACK_RING_BUSY_PTR ((void *)1)
+
+struct kasan_stack_ring stack_ring = {
+ .lock = __RW_LOCK_UNLOCKED(stack_ring.lock)
+};
+
+/* kasan.stacktrace=off/on */
+static int __init early_kasan_flag_stacktrace(char *arg)
+{
+ if (!arg)
+ return -EINVAL;
-#ifdef CONFIG_KASAN_TAGS_IDENTIFY
- idx = alloc_meta->free_track_idx;
- alloc_meta->free_pointer_tag[idx] = tag;
- alloc_meta->free_track_idx = (idx + 1) % KASAN_NR_FREE_STACKS;
-#endif
+ if (!strcmp(arg, "off"))
+ kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
+ else if (!strcmp(arg, "on"))
+ kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
+ else
+ return -EINVAL;
- kasan_set_track(&alloc_meta->free_track[idx], GFP_NOWAIT);
+ return 0;
}
+early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
-struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
- void *object, u8 tag)
+/* kasan.stack_ring_size=<number of entries> */
+static int __init early_kasan_flag_stack_ring_size(char *arg)
{
- struct kasan_alloc_meta *alloc_meta;
- int i = 0;
+ if (!arg)
+ return -EINVAL;
- alloc_meta = kasan_get_alloc_meta(cache, object);
- if (!alloc_meta)
- return NULL;
+ return kstrtoul(arg, 0, &stack_ring.size);
+}
+early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size);
+
+void __init kasan_init_tags(void)
+{
+ switch (kasan_arg_stacktrace) {
+ case KASAN_ARG_STACKTRACE_DEFAULT:
+ /* Default is specified by kasan_flag_stacktrace definition. */
+ break;
+ case KASAN_ARG_STACKTRACE_OFF:
+ static_branch_disable(&kasan_flag_stacktrace);
+ break;
+ case KASAN_ARG_STACKTRACE_ON:
+ static_branch_enable(&kasan_flag_stacktrace);
+ break;
+ }
-#ifdef CONFIG_KASAN_TAGS_IDENTIFY
- for (i = 0; i < KASAN_NR_FREE_STACKS; i++) {
- if (alloc_meta->free_pointer_tag[i] == tag)
- break;
+ if (kasan_stack_collection_enabled()) {
+ if (!stack_ring.size)
+ stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT;
+ stack_ring.entries = memblock_alloc(
+ sizeof(stack_ring.entries[0]) * stack_ring.size,
+ SMP_CACHE_BYTES);
+ if (WARN_ON(!stack_ring.entries))
+ static_branch_disable(&kasan_flag_stacktrace);
}
- if (i == KASAN_NR_FREE_STACKS)
- i = alloc_meta->free_track_idx;
-#endif
+}
+
+static void save_stack_info(struct kmem_cache *cache, void *object,
+ gfp_t gfp_flags, bool is_free)
+{
+ unsigned long flags;
+ depot_stack_handle_t stack;
+ u64 pos;
+ struct kasan_stack_ring_entry *entry;
+ void *old_ptr;
+
+ stack = kasan_save_stack(gfp_flags, true);
+
+ /*
+ * Prevent save_stack_info() from modifying stack ring
+ * when kasan_complete_mode_report_info() is walking it.
+ */
+ read_lock_irqsave(&stack_ring.lock, flags);
+
+next:
+ pos = atomic64_fetch_add(1, &stack_ring.pos);
+ entry = &stack_ring.entries[pos % stack_ring.size];
+
+ /* Detect stack ring entry slots that are being written to. */
+ old_ptr = READ_ONCE(entry->ptr);
+ if (old_ptr == STACK_RING_BUSY_PTR)
+ goto next; /* Busy slot. */
+ if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
+ goto next; /* Busy slot. */
+
+ WRITE_ONCE(entry->size, cache->object_size);
+ WRITE_ONCE(entry->pid, current->pid);
+ WRITE_ONCE(entry->stack, stack);
+ WRITE_ONCE(entry->is_free, is_free);
+
+ /*
+ * Paired with smp_load_acquire() in kasan_complete_mode_report_info().
+ */
+ smp_store_release(&entry->ptr, (s64)object);
- return &alloc_meta->free_track[i];
+ read_unlock_irqrestore(&stack_ring.lock, flags);
+}
+
+void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
+{
+ save_stack_info(cache, object, flags, false);
+}
+
+void kasan_save_free_info(struct kmem_cache *cache, void *object)
+{
+ save_stack_info(cache, object, GFP_NOWAIT, true);
}
return 0;
}
-static const struct seq_operations object_seqops = {
+static const struct seq_operations objects_sops = {
.start = start_object,
.next = next_object,
.stop = stop_object,
.show = show_object,
};
-
-static int open_objects(struct inode *inode, struct file *file)
-{
- return seq_open(file, &object_seqops);
-}
-
-static const struct file_operations objects_fops = {
- .open = open_objects,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release,
-};
+DEFINE_SEQ_ATTRIBUTE(objects);
static int __init kfence_debugfs_init(void)
{
return NULL;
}
+ /*
+ * Skip allocations for this slab, if KFENCE has been disabled for
+ * this slab.
+ */
+ if (s->flags & SLAB_SKIP_KFENCE)
+ return NULL;
+
if (atomic_inc_return(&kfence_allocation_gate) > 1)
return NULL;
#ifdef CONFIG_KFENCE_STATIC_KEYS
#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"
+#include "mm_slot.h"
enum scan_result {
SCAN_FAIL,
SCAN_SUCCEED,
SCAN_PMD_NULL,
+ SCAN_PMD_NONE,
+ SCAN_PMD_MAPPED,
SCAN_EXCEED_NONE_PTE,
SCAN_EXCEED_SWAP_PTE,
SCAN_EXCEED_SHARED_PTE,
SCAN_PTE_NON_PRESENT,
SCAN_PTE_UFFD_WP,
+ SCAN_PTE_MAPPED_HUGEPAGE,
SCAN_PAGE_RO,
SCAN_LACK_REFERENCED_PAGE,
SCAN_PAGE_NULL,
* default collapse hugepages if there is at least one pte mapped like
* it would have happened if the vma was large enough during page
* fault.
+ *
+ * Note that these are only respected if collapse was initiated by khugepaged.
*/
static unsigned int khugepaged_max_ptes_none __read_mostly;
static unsigned int khugepaged_max_ptes_swap __read_mostly;
#define MAX_PTE_MAPPED_THP 8
+struct collapse_control {
+ bool is_khugepaged;
+
+ /* Num pages scanned per node */
+ u32 node_load[MAX_NUMNODES];
+
+ /* Last target selected in hpage_collapse_find_target_node() */
+ int last_target_node;
+};
+
/**
- * struct mm_slot - hash lookup from mm to mm_slot
- * @hash: hash collision list
- * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
- * @mm: the mm that this information is valid for
+ * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned
+ * @slot: hash lookup from mm to mm_slot
* @nr_pte_mapped_thp: number of pte mapped THP
* @pte_mapped_thp: address array corresponding pte mapped THP
*/
-struct mm_slot {
- struct hlist_node hash;
- struct list_head mm_node;
- struct mm_struct *mm;
+struct khugepaged_mm_slot {
+ struct mm_slot slot;
/* pte-mapped THP in this mm */
int nr_pte_mapped_thp;
*/
struct khugepaged_scan {
struct list_head mm_head;
- struct mm_slot *mm_slot;
+ struct khugepaged_mm_slot *mm_slot;
unsigned long address;
};
int __init khugepaged_init(void)
{
mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
- sizeof(struct mm_slot),
- __alignof__(struct mm_slot), 0, NULL);
+ sizeof(struct khugepaged_mm_slot),
+ __alignof__(struct khugepaged_mm_slot),
+ 0, NULL);
if (!mm_slot_cache)
return -ENOMEM;
kmem_cache_destroy(mm_slot_cache);
}
-static inline struct mm_slot *alloc_mm_slot(void)
-{
- if (!mm_slot_cache) /* initialization failed */
- return NULL;
- return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
-}
-
-static inline void free_mm_slot(struct mm_slot *mm_slot)
-{
- kmem_cache_free(mm_slot_cache, mm_slot);
-}
-
-static struct mm_slot *get_mm_slot(struct mm_struct *mm)
-{
- struct mm_slot *mm_slot;
-
- hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
- if (mm == mm_slot->mm)
- return mm_slot;
-
- return NULL;
-}
-
-static void insert_to_mm_slots_hash(struct mm_struct *mm,
- struct mm_slot *mm_slot)
-{
- mm_slot->mm = mm;
- hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
-}
-
-static inline int khugepaged_test_exit(struct mm_struct *mm)
+static inline int hpage_collapse_test_exit(struct mm_struct *mm)
{
return atomic_read(&mm->mm_users) == 0;
}
void __khugepaged_enter(struct mm_struct *mm)
{
- struct mm_slot *mm_slot;
+ struct khugepaged_mm_slot *mm_slot;
+ struct mm_slot *slot;
int wakeup;
- mm_slot = alloc_mm_slot();
+ mm_slot = mm_slot_alloc(mm_slot_cache);
if (!mm_slot)
return;
+ slot = &mm_slot->slot;
+
/* __khugepaged_exit() must not run from under us */
- VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
+ VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm);
if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
- free_mm_slot(mm_slot);
+ mm_slot_free(mm_slot_cache, mm_slot);
return;
}
spin_lock(&khugepaged_mm_lock);
- insert_to_mm_slots_hash(mm, mm_slot);
+ mm_slot_insert(mm_slots_hash, mm, slot);
/*
* Insert just behind the scanning cursor, to let the area settle
* down a little.
*/
wakeup = list_empty(&khugepaged_scan.mm_head);
- list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
+ list_add_tail(&slot->mm_node, &khugepaged_scan.mm_head);
spin_unlock(&khugepaged_mm_lock);
mmgrab(mm);
{
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
hugepage_flags_enabled()) {
- if (hugepage_vma_check(vma, vm_flags, false, false))
+ if (hugepage_vma_check(vma, vm_flags, false, false, true))
__khugepaged_enter(vma->vm_mm);
}
}
void __khugepaged_exit(struct mm_struct *mm)
{
- struct mm_slot *mm_slot;
+ struct khugepaged_mm_slot *mm_slot;
+ struct mm_slot *slot;
int free = 0;
spin_lock(&khugepaged_mm_lock);
- mm_slot = get_mm_slot(mm);
+ slot = mm_slot_lookup(mm_slots_hash, mm);
+ mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
- hash_del(&mm_slot->hash);
- list_del(&mm_slot->mm_node);
+ hash_del(&slot->hash);
+ list_del(&slot->mm_node);
free = 1;
}
spin_unlock(&khugepaged_mm_lock);
if (free) {
clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
- free_mm_slot(mm_slot);
+ mm_slot_free(mm_slot_cache, mm_slot);
mmdrop(mm);
} else if (mm_slot) {
/*
* This is required to serialize against
- * khugepaged_test_exit() (which is guaranteed to run
- * under mmap sem read mode). Stop here (after we
- * return all pagetables will be destroyed) until
- * khugepaged has finished working on the pagetables
- * under the mmap_lock.
+ * hpage_collapse_test_exit() (which is guaranteed to run
+ * under mmap sem read mode). Stop here (after we return all
+ * pagetables will be destroyed) until khugepaged has finished
+ * working on the pagetables under the mmap_lock.
*/
mmap_write_lock(mm);
mmap_write_unlock(mm);
static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
unsigned long address,
pte_t *pte,
+ struct collapse_control *cc,
struct list_head *compound_pagelist)
{
struct page *page = NULL;
pte_t *_pte;
- int none_or_zero = 0, shared = 0, result = 0, referenced = 0;
+ int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0;
bool writable = false;
for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
pte_t pteval = *_pte;
if (pte_none(pteval) || (pte_present(pteval) &&
is_zero_pfn(pte_pfn(pteval)))) {
+ ++none_or_zero;
if (!userfaultfd_armed(vma) &&
- ++none_or_zero <= khugepaged_max_ptes_none) {
+ (!cc->is_khugepaged ||
+ none_or_zero <= khugepaged_max_ptes_none)) {
continue;
} else {
result = SCAN_EXCEED_NONE_PTE;
VM_BUG_ON_PAGE(!PageAnon(page), page);
- if (page_mapcount(page) > 1 &&
- ++shared > khugepaged_max_ptes_shared) {
- result = SCAN_EXCEED_SHARED_PTE;
- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
- goto out;
+ if (page_mapcount(page) > 1) {
+ ++shared;
+ if (cc->is_khugepaged &&
+ shared > khugepaged_max_ptes_shared) {
+ result = SCAN_EXCEED_SHARED_PTE;
+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+ goto out;
+ }
}
if (PageCompound(page)) {
if (PageCompound(page))
list_add_tail(&page->lru, compound_pagelist);
next:
- /* There should be enough young pte to collapse the page */
- if (pte_young(pteval) ||
- page_is_young(page) || PageReferenced(page) ||
- mmu_notifier_test_young(vma->vm_mm, address))
+ /*
+ * If collapse was initiated by khugepaged, check that there is
+ * enough young pte to justify collapsing the page
+ */
+ if (cc->is_khugepaged &&
+ (pte_young(pteval) || page_is_young(page) ||
+ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm,
+ address)))
referenced++;
if (pte_write(pteval))
if (unlikely(!writable)) {
result = SCAN_PAGE_RO;
- } else if (unlikely(!referenced)) {
+ } else if (unlikely(cc->is_khugepaged && !referenced)) {
result = SCAN_LACK_REFERENCED_PAGE;
} else {
result = SCAN_SUCCEED;
trace_mm_collapse_huge_page_isolate(page, none_or_zero,
referenced, writable, result);
- return 1;
+ return result;
}
out:
release_pte_pages(pte, _pte, compound_pagelist);
trace_mm_collapse_huge_page_isolate(page, none_or_zero,
referenced, writable, result);
- return 0;
+ return result;
}
static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
remove_wait_queue(&khugepaged_wait, &wait);
}
-static int khugepaged_node_load[MAX_NUMNODES];
+struct collapse_control khugepaged_collapse_control = {
+ .is_khugepaged = true,
+ .last_target_node = NUMA_NO_NODE,
+};
-static bool khugepaged_scan_abort(int nid)
+static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc)
{
int i;
return false;
/* If there is a count for this node already, it must be acceptable */
- if (khugepaged_node_load[nid])
+ if (cc->node_load[nid])
return false;
for (i = 0; i < MAX_NUMNODES; i++) {
- if (!khugepaged_node_load[i])
+ if (!cc->node_load[i])
continue;
if (node_distance(nid, i) > node_reclaim_distance)
return true;
}
#ifdef CONFIG_NUMA
-static int khugepaged_find_target_node(void)
+static int hpage_collapse_find_target_node(struct collapse_control *cc)
{
- static int last_khugepaged_target_node = NUMA_NO_NODE;
int nid, target_node = 0, max_value = 0;
/* find first node with max normal pages hit */
for (nid = 0; nid < MAX_NUMNODES; nid++)
- if (khugepaged_node_load[nid] > max_value) {
- max_value = khugepaged_node_load[nid];
+ if (cc->node_load[nid] > max_value) {
+ max_value = cc->node_load[nid];
target_node = nid;
}
/* do some balance if several nodes have the same hit record */
- if (target_node <= last_khugepaged_target_node)
- for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
- nid++)
- if (max_value == khugepaged_node_load[nid]) {
+ if (target_node <= cc->last_target_node)
+ for (nid = cc->last_target_node + 1; nid < MAX_NUMNODES;
+ nid++)
+ if (max_value == cc->node_load[nid]) {
target_node = nid;
break;
}
- last_khugepaged_target_node = target_node;
+ cc->last_target_node = target_node;
return target_node;
}
-
-static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
+#else
+static int hpage_collapse_find_target_node(struct collapse_control *cc)
{
- if (IS_ERR(*hpage)) {
- if (!*wait)
- return false;
-
- *wait = false;
- *hpage = NULL;
- khugepaged_alloc_sleep();
- } else if (*hpage) {
- put_page(*hpage);
- *hpage = NULL;
- }
-
- return true;
+ return 0;
}
+#endif
-static struct page *
-khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
+static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node)
{
- VM_BUG_ON_PAGE(*hpage, *hpage);
-
*hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
if (unlikely(!*hpage)) {
count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
- *hpage = ERR_PTR(-ENOMEM);
- return NULL;
+ return false;
}
prep_transhuge_page(*hpage);
count_vm_event(THP_COLLAPSE_ALLOC);
- return *hpage;
-}
-#else
-static int khugepaged_find_target_node(void)
-{
- return 0;
-}
-
-static inline struct page *alloc_khugepaged_hugepage(void)
-{
- struct page *page;
-
- page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
- HPAGE_PMD_ORDER);
- if (page)
- prep_transhuge_page(page);
- return page;
-}
-
-static struct page *khugepaged_alloc_hugepage(bool *wait)
-{
- struct page *hpage;
-
- do {
- hpage = alloc_khugepaged_hugepage();
- if (!hpage) {
- count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
- if (!*wait)
- return NULL;
-
- *wait = false;
- khugepaged_alloc_sleep();
- } else
- count_vm_event(THP_COLLAPSE_ALLOC);
- } while (unlikely(!hpage) && likely(hugepage_flags_enabled()));
-
- return hpage;
-}
-
-static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
-{
- /*
- * If the hpage allocated earlier was briefly exposed in page cache
- * before collapse_file() failed, it is possible that racing lookups
- * have not yet completed, and would then be unpleasantly surprised by
- * finding the hpage reused for the same mapping at a different offset.
- * Just release the previous allocation if there is any danger of that.
- */
- if (*hpage && page_count(*hpage) > 1) {
- put_page(*hpage);
- *hpage = NULL;
- }
-
- if (!*hpage)
- *hpage = khugepaged_alloc_hugepage(wait);
-
- if (unlikely(!*hpage))
- return false;
-
return true;
}
-static struct page *
-khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node)
-{
- VM_BUG_ON(!*hpage);
-
- return *hpage;
-}
-#endif
-
/*
* If mmap_lock temporarily dropped, revalidate vma
* before taking mmap_lock.
- * Return 0 if succeeds, otherwise return none-zero
- * value (scan code).
+ * Returns enum scan_result value.
*/
static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
- struct vm_area_struct **vmap)
+ bool expect_anon,
+ struct vm_area_struct **vmap,
+ struct collapse_control *cc)
{
struct vm_area_struct *vma;
- if (unlikely(khugepaged_test_exit(mm)))
+ if (unlikely(hpage_collapse_test_exit(mm)))
return SCAN_ANY_PROCESS;
*vmap = vma = find_vma(mm, address);
if (!transhuge_vma_suitable(vma, address))
return SCAN_ADDRESS_RANGE;
- if (!hugepage_vma_check(vma, vma->vm_flags, false, false))
+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false,
+ cc->is_khugepaged))
return SCAN_VMA_CHECK;
/*
* Anon VMA expected, the address may be unmapped then
* hugepage_vma_check may return true for qualified file
* vmas.
*/
- if (!vma->anon_vma || !vma_is_anonymous(vma))
- return SCAN_VMA_CHECK;
- return 0;
+ if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
+ return SCAN_PAGE_ANON;
+ return SCAN_SUCCEED;
+}
+
+static int find_pmd_or_thp_or_none(struct mm_struct *mm,
+ unsigned long address,
+ pmd_t **pmd)
+{
+ pmd_t pmde;
+
+ *pmd = mm_find_pmd(mm, address);
+ if (!*pmd)
+ return SCAN_PMD_NULL;
+
+ pmde = pmd_read_atomic(*pmd);
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ /* See comments in pmd_none_or_trans_huge_or_clear_bad() */
+ barrier();
+#endif
+ if (pmd_none(pmde))
+ return SCAN_PMD_NONE;
+ if (pmd_trans_huge(pmde))
+ return SCAN_PMD_MAPPED;
+ if (pmd_bad(pmde))
+ return SCAN_PMD_NULL;
+ return SCAN_SUCCEED;
+}
+
+static int check_pmd_still_valid(struct mm_struct *mm,
+ unsigned long address,
+ pmd_t *pmd)
+{
+ pmd_t *new_pmd;
+ int result = find_pmd_or_thp_or_none(mm, address, &new_pmd);
+
+ if (result != SCAN_SUCCEED)
+ return result;
+ if (new_pmd != pmd)
+ return SCAN_FAIL;
+ return SCAN_SUCCEED;
}
/*
* Bring missing pages in from swap, to complete THP collapse.
- * Only done if khugepaged_scan_pmd believes it is worthwhile.
+ * Only done if hpage_collapse_scan_pmd believes it is worthwhile.
*
* Called and returns without pte mapped or spinlocks held.
* Note that if false is returned, mmap_lock will be released.
*/
-static bool __collapse_huge_page_swapin(struct mm_struct *mm,
- struct vm_area_struct *vma,
- unsigned long haddr, pmd_t *pmd,
- int referenced)
+static int __collapse_huge_page_swapin(struct mm_struct *mm,
+ struct vm_area_struct *vma,
+ unsigned long haddr, pmd_t *pmd,
+ int referenced)
{
int swapped_in = 0;
vm_fault_t ret = 0;
*/
if (ret & VM_FAULT_RETRY) {
trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
- return false;
+ /* Likely, but not guaranteed, that page lock failed */
+ return SCAN_PAGE_LOCK;
}
if (ret & VM_FAULT_ERROR) {
mmap_read_unlock(mm);
trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
- return false;
+ return SCAN_FAIL;
}
swapped_in++;
}
lru_add_drain();
trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
- return true;
+ return SCAN_SUCCEED;
}
-static void collapse_huge_page(struct mm_struct *mm,
- unsigned long address,
- struct page **hpage,
- int node, int referenced, int unmapped)
+static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm,
+ struct collapse_control *cc)
+{
+ /* Only allocate from the target node */
+ gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :
+ GFP_TRANSHUGE) | __GFP_THISNODE;
+ int node = hpage_collapse_find_target_node(cc);
+
+ if (!hpage_collapse_alloc_page(hpage, gfp, node))
+ return SCAN_ALLOC_HUGE_PAGE_FAIL;
+ if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp)))
+ return SCAN_CGROUP_CHARGE_FAIL;
+ count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC);
+ return SCAN_SUCCEED;
+}
+
+static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
+ int referenced, int unmapped,
+ struct collapse_control *cc)
{
LIST_HEAD(compound_pagelist);
pmd_t *pmd, _pmd;
pte_t *pte;
pgtable_t pgtable;
- struct page *new_page;
+ struct page *hpage;
spinlock_t *pmd_ptl, *pte_ptl;
- int isolated = 0, result = 0;
+ int result = SCAN_FAIL;
struct vm_area_struct *vma;
struct mmu_notifier_range range;
- gfp_t gfp;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
- /* Only allocate from the target node */
- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
-
/*
* Before allocating the hugepage, release the mmap_lock read lock.
* The allocation can take potentially a long time if it involves
* that. We will recheck the vma after taking it again in write mode.
*/
mmap_read_unlock(mm);
- new_page = khugepaged_alloc_page(hpage, gfp, node);
- if (!new_page) {
- result = SCAN_ALLOC_HUGE_PAGE_FAIL;
- goto out_nolock;
- }
- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) {
- result = SCAN_CGROUP_CHARGE_FAIL;
+ result = alloc_charge_hpage(&hpage, mm, cc);
+ if (result != SCAN_SUCCEED)
goto out_nolock;
- }
- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC);
mmap_read_lock(mm);
- result = hugepage_vma_revalidate(mm, address, &vma);
- if (result) {
+ result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
+ if (result != SCAN_SUCCEED) {
mmap_read_unlock(mm);
goto out_nolock;
}
- pmd = mm_find_pmd(mm, address);
- if (!pmd) {
- result = SCAN_PMD_NULL;
+ result = find_pmd_or_thp_or_none(mm, address, &pmd);
+ if (result != SCAN_SUCCEED) {
mmap_read_unlock(mm);
goto out_nolock;
}
- /*
- * __collapse_huge_page_swapin will return with mmap_lock released
- * when it fails. So we jump out_nolock directly in that case.
- * Continuing to collapse causes inconsistency.
- */
- if (unmapped && !__collapse_huge_page_swapin(mm, vma, address,
- pmd, referenced)) {
- goto out_nolock;
+ if (unmapped) {
+ /*
+ * __collapse_huge_page_swapin will return with mmap_lock
+ * released when it fails. So we jump out_nolock directly in
+ * that case. Continuing to collapse causes inconsistency.
+ */
+ result = __collapse_huge_page_swapin(mm, vma, address, pmd,
+ referenced);
+ if (result != SCAN_SUCCEED)
+ goto out_nolock;
}
mmap_read_unlock(mm);
* handled by the anon_vma lock + PG_lock.
*/
mmap_write_lock(mm);
- result = hugepage_vma_revalidate(mm, address, &vma);
- if (result)
+ result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
+ if (result != SCAN_SUCCEED)
goto out_up_write;
/* check if the pmd is still valid */
- if (mm_find_pmd(mm, address) != pmd)
+ result = check_pmd_still_valid(mm, address, pmd);
+ if (result != SCAN_SUCCEED)
goto out_up_write;
anon_vma_lock_write(vma->anon_vma);
mmu_notifier_invalidate_range_end(&range);
spin_lock(pte_ptl);
- isolated = __collapse_huge_page_isolate(vma, address, pte,
- &compound_pagelist);
+ result = __collapse_huge_page_isolate(vma, address, pte, cc,
+ &compound_pagelist);
spin_unlock(pte_ptl);
- if (unlikely(!isolated)) {
+ if (unlikely(result != SCAN_SUCCEED)) {
pte_unmap(pte);
spin_lock(pmd_ptl);
BUG_ON(!pmd_none(*pmd));
pmd_populate(mm, pmd, pmd_pgtable(_pmd));
spin_unlock(pmd_ptl);
anon_vma_unlock_write(vma->anon_vma);
- result = SCAN_FAIL;
goto out_up_write;
}
*/
anon_vma_unlock_write(vma->anon_vma);
- __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl,
- &compound_pagelist);
+ __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl,
+ &compound_pagelist);
pte_unmap(pte);
/*
* spin_lock() below is not the equivalent of smp_wmb(), but
* avoid the copy_huge_page writes to become visible after
* the set_pmd_at() write.
*/
- __SetPageUptodate(new_page);
+ __SetPageUptodate(hpage);
pgtable = pmd_pgtable(_pmd);
- _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
+ _pmd = mk_huge_pmd(hpage, vma->vm_page_prot);
_pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
spin_lock(pmd_ptl);
BUG_ON(!pmd_none(*pmd));
- page_add_new_anon_rmap(new_page, vma, address);
- lru_cache_add_inactive_or_unevictable(new_page, vma);
+ page_add_new_anon_rmap(hpage, vma, address);
+ lru_cache_add_inactive_or_unevictable(hpage, vma);
pgtable_trans_huge_deposit(mm, pmd, pgtable);
set_pmd_at(mm, address, pmd, _pmd);
update_mmu_cache_pmd(vma, address, pmd);
spin_unlock(pmd_ptl);
- *hpage = NULL;
+ hpage = NULL;
- khugepaged_pages_collapsed++;
result = SCAN_SUCCEED;
out_up_write:
mmap_write_unlock(mm);
out_nolock:
- if (!IS_ERR_OR_NULL(*hpage))
- mem_cgroup_uncharge(page_folio(*hpage));
- trace_mm_collapse_huge_page(mm, isolated, result);
- return;
+ if (hpage) {
+ mem_cgroup_uncharge(page_folio(hpage));
+ put_page(hpage);
+ }
+ trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result);
+ return result;
}
-static int khugepaged_scan_pmd(struct mm_struct *mm,
- struct vm_area_struct *vma,
- unsigned long address,
- struct page **hpage)
+static int hpage_collapse_scan_pmd(struct mm_struct *mm,
+ struct vm_area_struct *vma,
+ unsigned long address, bool *mmap_locked,
+ struct collapse_control *cc)
{
pmd_t *pmd;
pte_t *pte, *_pte;
- int ret = 0, result = 0, referenced = 0;
+ int result = SCAN_FAIL, referenced = 0;
int none_or_zero = 0, shared = 0;
struct page *page = NULL;
unsigned long _address;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
- pmd = mm_find_pmd(mm, address);
- if (!pmd) {
- result = SCAN_PMD_NULL;
+ result = find_pmd_or_thp_or_none(mm, address, &pmd);
+ if (result != SCAN_SUCCEED)
goto out;
- }
- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
+ memset(cc->node_load, 0, sizeof(cc->node_load));
pte = pte_offset_map_lock(mm, pmd, address, &ptl);
for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR;
_pte++, _address += PAGE_SIZE) {
pte_t pteval = *_pte;
if (is_swap_pte(pteval)) {
- if (++unmapped <= khugepaged_max_ptes_swap) {
+ ++unmapped;
+ if (!cc->is_khugepaged ||
+ unmapped <= khugepaged_max_ptes_swap) {
/*
* Always be strict with uffd-wp
* enabled swap entries. Please see
}
}
if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
+ ++none_or_zero;
if (!userfaultfd_armed(vma) &&
- ++none_or_zero <= khugepaged_max_ptes_none) {
+ (!cc->is_khugepaged ||
+ none_or_zero <= khugepaged_max_ptes_none)) {
continue;
} else {
result = SCAN_EXCEED_NONE_PTE;
goto out_unmap;
}
- if (page_mapcount(page) > 1 &&
- ++shared > khugepaged_max_ptes_shared) {
- result = SCAN_EXCEED_SHARED_PTE;
- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
- goto out_unmap;
+ if (page_mapcount(page) > 1) {
+ ++shared;
+ if (cc->is_khugepaged &&
+ shared > khugepaged_max_ptes_shared) {
+ result = SCAN_EXCEED_SHARED_PTE;
+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+ goto out_unmap;
+ }
}
page = compound_head(page);
/*
* Record which node the original page is from and save this
- * information to khugepaged_node_load[].
+ * information to cc->node_load[].
* Khugepaged will allocate hugepage from the node has the max
* hit record.
*/
node = page_to_nid(page);
- if (khugepaged_scan_abort(node)) {
+ if (hpage_collapse_scan_abort(node, cc)) {
result = SCAN_SCAN_ABORT;
goto out_unmap;
}
- khugepaged_node_load[node]++;
+ cc->node_load[node]++;
if (!PageLRU(page)) {
result = SCAN_PAGE_LRU;
goto out_unmap;
result = SCAN_PAGE_COUNT;
goto out_unmap;
}
- if (pte_young(pteval) ||
- page_is_young(page) || PageReferenced(page) ||
- mmu_notifier_test_young(vma->vm_mm, address))
+
+ /*
+ * If collapse was initiated by khugepaged, check that there is
+ * enough young pte to justify collapsing the page
+ */
+ if (cc->is_khugepaged &&
+ (pte_young(pteval) || page_is_young(page) ||
+ PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm,
+ address)))
referenced++;
}
if (!writable) {
result = SCAN_PAGE_RO;
- } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) {
+ } else if (cc->is_khugepaged &&
+ (!referenced ||
+ (unmapped && referenced < HPAGE_PMD_NR / 2))) {
result = SCAN_LACK_REFERENCED_PAGE;
} else {
result = SCAN_SUCCEED;
- ret = 1;
}
out_unmap:
pte_unmap_unlock(pte, ptl);
- if (ret) {
- node = khugepaged_find_target_node();
+ if (result == SCAN_SUCCEED) {
+ result = collapse_huge_page(mm, address, referenced,
+ unmapped, cc);
/* collapse_huge_page will return with the mmap_lock released */
- collapse_huge_page(mm, address, hpage, node,
- referenced, unmapped);
+ *mmap_locked = false;
}
out:
trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
none_or_zero, result, unmapped);
- return ret;
+ return result;
}
-static void collect_mm_slot(struct mm_slot *mm_slot)
+static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot)
{
- struct mm_struct *mm = mm_slot->mm;
+ struct mm_slot *slot = &mm_slot->slot;
+ struct mm_struct *mm = slot->mm;
lockdep_assert_held(&khugepaged_mm_lock);
- if (khugepaged_test_exit(mm)) {
+ if (hpage_collapse_test_exit(mm)) {
/* free mm_slot */
- hash_del(&mm_slot->hash);
- list_del(&mm_slot->mm_node);
+ hash_del(&slot->hash);
+ list_del(&slot->mm_node);
/*
* Not strictly needed because the mm exited already.
*/
/* khugepaged_mm_lock actually not necessary for the below */
- free_mm_slot(mm_slot);
+ mm_slot_free(mm_slot_cache, mm_slot);
mmdrop(mm);
}
}
/*
* Notify khugepaged that given addr of the mm is pte-mapped THP. Then
* khugepaged should try to collapse the page table.
+ *
+ * Note that following race exists:
+ * (1) khugepaged calls khugepaged_collapse_pte_mapped_thps() for mm_struct A,
+ * emptying the A's ->pte_mapped_thp[] array.
+ * (2) MADV_COLLAPSE collapses some file extent with target mm_struct B, and
+ * retract_page_tables() finds a VMA in mm_struct A mapping the same extent
+ * (at virtual address X) and adds an entry (for X) into mm_struct A's
+ * ->pte-mapped_thp[] array.
+ * (3) khugepaged calls khugepaged_collapse_scan_file() for mm_struct A at X,
+ * sees a pte-mapped THP (SCAN_PTE_MAPPED_HUGEPAGE) and adds an entry
+ * (for X) into mm_struct A's ->pte-mapped_thp[] array.
+ * Thus, it's possible the same address is added multiple times for the same
+ * mm_struct. Should this happen, we'll simply attempt
+ * collapse_pte_mapped_thp() multiple times for the same address, under the same
+ * exclusive mmap_lock, and assuming the first call is successful, subsequent
+ * attempts will return quickly (without grabbing any additional locks) when
+ * a huge pmd is found in find_pmd_or_thp_or_none(). Since this is a cheap
+ * check, and since this is a rare occurrence, the cost of preventing this
+ * "multiple-add" is thought to be more expensive than just handling it, should
+ * it occur.
*/
-static void khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
+static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
unsigned long addr)
{
- struct mm_slot *mm_slot;
+ struct khugepaged_mm_slot *mm_slot;
+ struct mm_slot *slot;
+ bool ret = false;
VM_BUG_ON(addr & ~HPAGE_PMD_MASK);
spin_lock(&khugepaged_mm_lock);
- mm_slot = get_mm_slot(mm);
- if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP))
+ slot = mm_slot_lookup(mm_slots_hash, mm);
+ mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
+ if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) {
mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr;
+ ret = true;
+ }
spin_unlock(&khugepaged_mm_lock);
+ return ret;
+}
+
+/* hpage must be locked, and mmap_lock must be held in write */
+static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr,
+ pmd_t *pmdp, struct page *hpage)
+{
+ struct vm_fault vmf = {
+ .vma = vma,
+ .address = addr,
+ .flags = 0,
+ .pmd = pmdp,
+ };
+
+ VM_BUG_ON(!PageTransHuge(hpage));
+ mmap_assert_write_locked(vma->vm_mm);
+
+ if (do_set_pmd(&vmf, hpage))
+ return SCAN_FAIL;
+
+ get_page(hpage);
+ return SCAN_SUCCEED;
}
static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
*
* @mm: process address space where collapse happens
* @addr: THP collapse address
+ * @install_pmd: If a huge PMD should be installed
*
* This function checks whether all the PTEs in the PMD are pointing to the
* right THP. If so, retract the page table so the THP can refault in with
- * as pmd-mapped.
+ * as pmd-mapped. Possibly install a huge PMD mapping the THP.
*/
-void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
+int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr,
+ bool install_pmd)
{
unsigned long haddr = addr & HPAGE_PMD_MASK;
- struct vm_area_struct *vma = find_vma(mm, haddr);
+ struct vm_area_struct *vma = vma_lookup(mm, haddr);
struct page *hpage;
pte_t *start_pte, *pte;
pmd_t *pmd;
spinlock_t *ptl;
- int count = 0;
+ int count = 0, result = SCAN_FAIL;
int i;
+ mmap_assert_write_locked(mm);
+
+ /* Fast check before locking page if already PMD-mapped */
+ result = find_pmd_or_thp_or_none(mm, haddr, &pmd);
+ if (result == SCAN_PMD_MAPPED)
+ return result;
+
if (!vma || !vma->vm_file ||
!range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE))
- return;
+ return SCAN_VMA_CHECK;
/*
- * This vm_flags may not have VM_HUGEPAGE if the page was not
- * collapsed by this mm. But we can still collapse if the page is
- * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check()
- * will not fail the vma for missing VM_HUGEPAGE
+ * If we are here, we've succeeded in replacing all the native pages
+ * in the page cache with a single hugepage. If a mm were to fault-in
+ * this memory (mapped by a suitably aligned VMA), we'd get the hugepage
+ * and map it by a PMD, regardless of sysfs THP settings. As such, let's
+ * analogously elide sysfs THP settings here.
*/
- if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false))
- return;
+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false))
+ return SCAN_VMA_CHECK;
/* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */
if (userfaultfd_wp(vma))
- return;
+ return SCAN_PTE_UFFD_WP;
hpage = find_lock_page(vma->vm_file->f_mapping,
linear_page_index(vma, haddr));
if (!hpage)
- return;
+ return SCAN_PAGE_NULL;
+
+ if (!PageHead(hpage)) {
+ result = SCAN_FAIL;
+ goto drop_hpage;
+ }
- if (!PageHead(hpage))
+ if (compound_order(hpage) != HPAGE_PMD_ORDER) {
+ result = SCAN_PAGE_COMPOUND;
goto drop_hpage;
+ }
- pmd = mm_find_pmd(mm, haddr);
- if (!pmd)
+ switch (result) {
+ case SCAN_SUCCEED:
+ break;
+ case SCAN_PMD_NONE:
+ /*
+ * In MADV_COLLAPSE path, possible race with khugepaged where
+ * all pte entries have been removed and pmd cleared. If so,
+ * skip all the pte checks and just update the pmd mapping.
+ */
+ goto maybe_install_pmd;
+ default:
goto drop_hpage;
+ }
start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl);
+ result = SCAN_FAIL;
/* step 1: check all mapped PTEs are to the right huge page */
for (i = 0, addr = haddr, pte = start_pte;
continue;
/* page swapped out, abort */
- if (!pte_present(*pte))
+ if (!pte_present(*pte)) {
+ result = SCAN_PTE_NON_PRESENT;
goto abort;
+ }
page = vm_normal_page(vma, addr, *pte);
if (WARN_ON_ONCE(page && is_zone_device_page(page)))
add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count);
}
- /* step 4: collapse pmd */
+ /* step 4: remove pte entries */
collapse_and_free_pmd(mm, vma, haddr, pmd);
+
+maybe_install_pmd:
+ /* step 5: install pmd entry */
+ result = install_pmd
+ ? set_huge_pmd(vma, haddr, pmd, hpage)
+ : SCAN_SUCCEED;
+
drop_hpage:
unlock_page(hpage);
put_page(hpage);
- return;
+ return result;
abort:
pte_unmap_unlock(start_pte, ptl);
goto drop_hpage;
}
-static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
+static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot)
{
- struct mm_struct *mm = mm_slot->mm;
+ struct mm_slot *slot = &mm_slot->slot;
+ struct mm_struct *mm = slot->mm;
int i;
if (likely(mm_slot->nr_pte_mapped_thp == 0))
if (!mmap_write_trylock(mm))
return;
- if (unlikely(khugepaged_test_exit(mm)))
+ if (unlikely(hpage_collapse_test_exit(mm)))
goto out;
for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++)
- collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]);
+ collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i], false);
out:
mm_slot->nr_pte_mapped_thp = 0;
mmap_write_unlock(mm);
}
-static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
+static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff,
+ struct mm_struct *target_mm,
+ unsigned long target_addr, struct page *hpage,
+ struct collapse_control *cc)
{
struct vm_area_struct *vma;
- struct mm_struct *mm;
- unsigned long addr;
- pmd_t *pmd;
+ int target_result = SCAN_FAIL;
i_mmap_lock_write(mapping);
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
+ int result = SCAN_FAIL;
+ struct mm_struct *mm = NULL;
+ unsigned long addr = 0;
+ pmd_t *pmd;
+ bool is_target = false;
+
/*
* Check vma->anon_vma to exclude MAP_PRIVATE mappings that
* got written to. These VMAs are likely not worth investing
* ptl. It has higher chance to recover THP for the VMA, but
* has higher cost too.
*/
- if (vma->anon_vma)
- continue;
+ if (vma->anon_vma) {
+ result = SCAN_PAGE_ANON;
+ goto next;
+ }
addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
- if (addr & ~HPAGE_PMD_MASK)
- continue;
- if (vma->vm_end < addr + HPAGE_PMD_SIZE)
- continue;
+ if (addr & ~HPAGE_PMD_MASK ||
+ vma->vm_end < addr + HPAGE_PMD_SIZE) {
+ result = SCAN_VMA_CHECK;
+ goto next;
+ }
mm = vma->vm_mm;
- pmd = mm_find_pmd(mm, addr);
- if (!pmd)
- continue;
+ is_target = mm == target_mm && addr == target_addr;
+ result = find_pmd_or_thp_or_none(mm, addr, &pmd);
+ if (result != SCAN_SUCCEED)
+ goto next;
/*
* We need exclusive mmap_lock to retract page table.
*
* We use trylock due to lock inversion: we need to acquire
* mmap_lock while holding page lock. Fault path does it in
* reverse order. Trylock is a way to avoid deadlock.
+ *
+ * Also, it's not MADV_COLLAPSE's job to collapse other
+ * mappings - let khugepaged take care of them later.
*/
- if (mmap_write_trylock(mm)) {
+ result = SCAN_PTE_MAPPED_HUGEPAGE;
+ if ((cc->is_khugepaged || is_target) &&
+ mmap_write_trylock(mm)) {
/*
* When a vma is registered with uffd-wp, we can't
* recycle the pmd pgtable because there can be pte
* it'll always mapped in small page size for uffd-wp
* registered ranges.
*/
- if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma))
- collapse_and_free_pmd(mm, vma, addr, pmd);
+ if (hpage_collapse_test_exit(mm)) {
+ result = SCAN_ANY_PROCESS;
+ goto unlock_next;
+ }
+ if (userfaultfd_wp(vma)) {
+ result = SCAN_PTE_UFFD_WP;
+ goto unlock_next;
+ }
+ collapse_and_free_pmd(mm, vma, addr, pmd);
+ if (!cc->is_khugepaged && is_target)
+ result = set_huge_pmd(vma, addr, pmd, hpage);
+ else
+ result = SCAN_SUCCEED;
+
+unlock_next:
mmap_write_unlock(mm);
- } else {
- /* Try again later */
+ goto next;
+ }
+ /*
+ * Calling context will handle target mm/addr. Otherwise, let
+ * khugepaged try again later.
+ */
+ if (!is_target) {
khugepaged_add_pte_mapped_thp(mm, addr);
+ continue;
}
+next:
+ if (is_target)
+ target_result = result;
}
i_mmap_unlock_write(mapping);
+ return target_result;
}
/**
* collapse_file - collapse filemap/tmpfs/shmem pages into huge one.
*
* @mm: process address space where collapse happens
+ * @addr: virtual collapse start address
* @file: file that collapse on
* @start: collapse start address
- * @hpage: new allocated huge page for collapse
- * @node: appointed node the new huge page allocate from
+ * @cc: collapse context and scratchpad
*
* Basic scheme is simple, details are more complex:
* - allocate and lock a new huge page;
* + restore gaps in the page cache;
* + unlock and free huge page;
*/
-static void collapse_file(struct mm_struct *mm,
- struct file *file, pgoff_t start,
- struct page **hpage, int node)
+static int collapse_file(struct mm_struct *mm, unsigned long addr,
+ struct file *file, pgoff_t start,
+ struct collapse_control *cc)
{
struct address_space *mapping = file->f_mapping;
- gfp_t gfp;
- struct page *new_page;
+ struct page *hpage;
pgoff_t index, end = start + HPAGE_PMD_NR;
LIST_HEAD(pagelist);
XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
- /* Only allocate from the target node */
- gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE;
-
- new_page = khugepaged_alloc_page(hpage, gfp, node);
- if (!new_page) {
- result = SCAN_ALLOC_HUGE_PAGE_FAIL;
- goto out;
- }
-
- if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) {
- result = SCAN_CGROUP_CHARGE_FAIL;
+ result = alloc_charge_hpage(&hpage, mm, cc);
+ if (result != SCAN_SUCCEED)
goto out;
- }
- count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC);
/*
* Ensure we have slots for all the pages in the range. This is
}
} while (1);
- __SetPageLocked(new_page);
+ __SetPageLocked(hpage);
if (is_shmem)
- __SetPageSwapBacked(new_page);
- new_page->index = start;
- new_page->mapping = mapping;
+ __SetPageSwapBacked(hpage);
+ hpage->index = start;
+ hpage->mapping = mapping;
/*
- * At this point the new_page is locked and not up-to-date.
+ * At this point the hpage is locked and not up-to-date.
* It's safe to insert it into the page cache, because nobody would
* be able to map it or use it in another way until we unlock it.
*/
result = SCAN_FAIL;
goto xa_locked;
}
- xas_store(&xas, new_page);
+ xas_store(&xas, hpage);
nr_none++;
continue;
}
if (xa_is_value(page) || !PageUptodate(page)) {
+ struct folio *folio;
+
xas_unlock_irq(&xas);
/* swap in or instantiate fallocated page */
- if (shmem_getpage(mapping->host, index, &page,
- SGP_NOALLOC)) {
+ if (shmem_get_folio(mapping->host, index,
+ &folio, SGP_NOALLOC)) {
result = SCAN_FAIL;
goto xa_unlocked;
}
+ page = folio_file_page(folio, index);
} else if (trylock_page(page)) {
get_page(page);
xas_unlock_irq(&xas);
/*
* If file was truncated then extended, or hole-punched, before
* we locked the first page, then a THP might be there already.
+ * This will be discovered on the first iteration.
*/
if (PageTransCompound(page)) {
- result = SCAN_PAGE_COMPOUND;
+ struct page *head = compound_head(page);
+
+ result = compound_order(head) == HPAGE_PMD_ORDER &&
+ head->index == start
+ /* Maybe PMD-mapped */
+ ? SCAN_PTE_MAPPED_HUGEPAGE
+ : SCAN_PAGE_COMPOUND;
goto out_unlock;
}
list_add_tail(&page->lru, &pagelist);
/* Finally, replace with the new page. */
- xas_store(&xas, new_page);
+ xas_store(&xas, hpage);
continue;
out_unlock:
unlock_page(page);
put_page(page);
goto xa_unlocked;
}
- nr = thp_nr_pages(new_page);
+ nr = thp_nr_pages(hpage);
if (is_shmem)
- __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr);
+ __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr);
else {
- __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr);
+ __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr);
filemap_nr_thps_inc(mapping);
/*
* Paired with smp_mb() in do_dentry_open() to ensure
smp_mb();
if (inode_is_open_for_write(mapping->host)) {
result = SCAN_FAIL;
- __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr);
+ __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr);
filemap_nr_thps_dec(mapping);
goto xa_locked;
}
}
if (nr_none) {
- __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none);
+ __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none);
/* nr_none is always 0 for non-shmem. */
- __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none);
+ __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none);
}
/* Join all the small entries into a single multi-index entry */
xas_set_order(&xas, start, HPAGE_PMD_ORDER);
- xas_store(&xas, new_page);
+ xas_store(&xas, hpage);
xa_locked:
xas_unlock_irq(&xas);
xa_unlocked:
index = start;
list_for_each_entry_safe(page, tmp, &pagelist, lru) {
while (index < page->index) {
- clear_highpage(new_page + (index % HPAGE_PMD_NR));
+ clear_highpage(hpage + (index % HPAGE_PMD_NR));
index++;
}
- copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
- page);
+ copy_highpage(hpage + (page->index % HPAGE_PMD_NR),
+ page);
list_del(&page->lru);
page->mapping = NULL;
page_ref_unfreeze(page, 1);
index++;
}
while (index < end) {
- clear_highpage(new_page + (index % HPAGE_PMD_NR));
+ clear_highpage(hpage + (index % HPAGE_PMD_NR));
index++;
}
- SetPageUptodate(new_page);
- page_ref_add(new_page, HPAGE_PMD_NR - 1);
+ SetPageUptodate(hpage);
+ page_ref_add(hpage, HPAGE_PMD_NR - 1);
if (is_shmem)
- set_page_dirty(new_page);
- lru_cache_add(new_page);
+ set_page_dirty(hpage);
+ lru_cache_add(hpage);
/*
* Remove pte page tables, so we can re-fault the page as huge.
*/
- retract_page_tables(mapping, start);
- *hpage = NULL;
-
- khugepaged_pages_collapsed++;
+ result = retract_page_tables(mapping, start, mm, addr, hpage,
+ cc);
+ unlock_page(hpage);
+ hpage = NULL;
} else {
struct page *page;
VM_BUG_ON(nr_none);
xas_unlock_irq(&xas);
- new_page->mapping = NULL;
+ hpage->mapping = NULL;
}
- unlock_page(new_page);
+ if (hpage)
+ unlock_page(hpage);
out:
VM_BUG_ON(!list_empty(&pagelist));
- if (!IS_ERR_OR_NULL(*hpage))
- mem_cgroup_uncharge(page_folio(*hpage));
+ if (hpage) {
+ mem_cgroup_uncharge(page_folio(hpage));
+ put_page(hpage);
+ }
/* TODO: tracepoints */
+ return result;
}
-static void khugepaged_scan_file(struct mm_struct *mm,
- struct file *file, pgoff_t start, struct page **hpage)
+static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
+ struct file *file, pgoff_t start,
+ struct collapse_control *cc)
{
struct page *page = NULL;
struct address_space *mapping = file->f_mapping;
present = 0;
swap = 0;
- memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
+ memset(cc->node_load, 0, sizeof(cc->node_load));
rcu_read_lock();
xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) {
if (xas_retry(&xas, page))
continue;
if (xa_is_value(page)) {
- if (++swap > khugepaged_max_ptes_swap) {
+ ++swap;
+ if (cc->is_khugepaged &&
+ swap > khugepaged_max_ptes_swap) {
result = SCAN_EXCEED_SWAP_PTE;
count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
break;
}
/*
- * XXX: khugepaged should compact smaller compound pages
+ * TODO: khugepaged should compact smaller compound pages
* into a PMD sized page
*/
if (PageTransCompound(page)) {
- result = SCAN_PAGE_COMPOUND;
+ struct page *head = compound_head(page);
+
+ result = compound_order(head) == HPAGE_PMD_ORDER &&
+ head->index == start
+ /* Maybe PMD-mapped */
+ ? SCAN_PTE_MAPPED_HUGEPAGE
+ : SCAN_PAGE_COMPOUND;
+ /*
+ * For SCAN_PTE_MAPPED_HUGEPAGE, further processing
+ * by the caller won't touch the page cache, and so
+ * it's safe to skip LRU and refcount checks before
+ * returning.
+ */
break;
}
node = page_to_nid(page);
- if (khugepaged_scan_abort(node)) {
+ if (hpage_collapse_scan_abort(node, cc)) {
result = SCAN_SCAN_ABORT;
break;
}
- khugepaged_node_load[node]++;
+ cc->node_load[node]++;
if (!PageLRU(page)) {
result = SCAN_PAGE_LRU;
rcu_read_unlock();
if (result == SCAN_SUCCEED) {
- if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
+ if (cc->is_khugepaged &&
+ present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
result = SCAN_EXCEED_NONE_PTE;
count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
} else {
- node = khugepaged_find_target_node();
- collapse_file(mm, file, start, hpage, node);
+ result = collapse_file(mm, addr, file, start, cc);
}
}
- /* TODO: tracepoints */
+ trace_mm_khugepaged_scan_file(mm, page, file->f_path.dentry->d_iname,
+ present, swap, result);
+ return result;
}
#else
-static void khugepaged_scan_file(struct mm_struct *mm,
- struct file *file, pgoff_t start, struct page **hpage)
+static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr,
+ struct file *file, pgoff_t start,
+ struct collapse_control *cc)
{
BUILD_BUG();
}
-static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)
+static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot)
{
}
+
+static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm,
+ unsigned long addr)
+{
+ return false;
+}
#endif
-static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
- struct page **hpage)
+static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result,
+ struct collapse_control *cc)
__releases(&khugepaged_mm_lock)
__acquires(&khugepaged_mm_lock)
{
- struct mm_slot *mm_slot;
+ struct vma_iterator vmi;
+ struct khugepaged_mm_slot *mm_slot;
+ struct mm_slot *slot;
struct mm_struct *mm;
struct vm_area_struct *vma;
int progress = 0;
VM_BUG_ON(!pages);
lockdep_assert_held(&khugepaged_mm_lock);
+ *result = SCAN_FAIL;
- if (khugepaged_scan.mm_slot)
+ if (khugepaged_scan.mm_slot) {
mm_slot = khugepaged_scan.mm_slot;
- else {
- mm_slot = list_entry(khugepaged_scan.mm_head.next,
+ slot = &mm_slot->slot;
+ } else {
+ slot = list_entry(khugepaged_scan.mm_head.next,
struct mm_slot, mm_node);
+ mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
khugepaged_scan.address = 0;
khugepaged_scan.mm_slot = mm_slot;
}
spin_unlock(&khugepaged_mm_lock);
khugepaged_collapse_pte_mapped_thps(mm_slot);
- mm = mm_slot->mm;
+ mm = slot->mm;
/*
* Don't wait for semaphore (to avoid long wait times). Just move to
* the next mm on the list.
vma = NULL;
if (unlikely(!mmap_read_trylock(mm)))
goto breakouterloop_mmap_lock;
- if (likely(!khugepaged_test_exit(mm)))
- vma = find_vma(mm, khugepaged_scan.address);
progress++;
- for (; vma; vma = vma->vm_next) {
+ if (unlikely(hpage_collapse_test_exit(mm)))
+ goto breakouterloop;
+
+ vma_iter_init(&vmi, mm, khugepaged_scan.address);
+ for_each_vma(vmi, vma) {
unsigned long hstart, hend;
cond_resched();
- if (unlikely(khugepaged_test_exit(mm))) {
+ if (unlikely(hpage_collapse_test_exit(mm))) {
progress++;
break;
}
- if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) {
+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) {
skip:
progress++;
continue;
VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
while (khugepaged_scan.address < hend) {
- int ret;
+ bool mmap_locked = true;
+
cond_resched();
- if (unlikely(khugepaged_test_exit(mm)))
+ if (unlikely(hpage_collapse_test_exit(mm)))
goto breakouterloop;
VM_BUG_ON(khugepaged_scan.address < hstart ||
khugepaged_scan.address);
mmap_read_unlock(mm);
- ret = 1;
- khugepaged_scan_file(mm, file, pgoff, hpage);
+ *result = hpage_collapse_scan_file(mm,
+ khugepaged_scan.address,
+ file, pgoff, cc);
+ mmap_locked = false;
fput(file);
} else {
- ret = khugepaged_scan_pmd(mm, vma,
- khugepaged_scan.address,
- hpage);
+ *result = hpage_collapse_scan_pmd(mm, vma,
+ khugepaged_scan.address,
+ &mmap_locked,
+ cc);
+ }
+ switch (*result) {
+ case SCAN_PTE_MAPPED_HUGEPAGE: {
+ pmd_t *pmd;
+
+ *result = find_pmd_or_thp_or_none(mm,
+ khugepaged_scan.address,
+ &pmd);
+ if (*result != SCAN_SUCCEED)
+ break;
+ if (!khugepaged_add_pte_mapped_thp(mm,
+ khugepaged_scan.address))
+ break;
+ } fallthrough;
+ case SCAN_SUCCEED:
+ ++khugepaged_pages_collapsed;
+ break;
+ default:
+ break;
}
+
/* move to next address */
khugepaged_scan.address += HPAGE_PMD_SIZE;
progress += HPAGE_PMD_NR;
- if (ret)
- /* we released mmap_lock so break loop */
+ if (!mmap_locked)
+ /*
+ * We released mmap_lock so break loop. Note
+ * that we drop mmap_lock before all hugepage
+ * allocations, so if allocation fails, we are
+ * guaranteed to break here and report the
+ * correct result back to caller.
+ */
goto breakouterloop_mmap_lock;
if (progress >= pages)
goto breakouterloop;
* Release the current mm_slot if this mm is about to die, or
* if we scanned all vmas of this mm.
*/
- if (khugepaged_test_exit(mm) || !vma) {
+ if (hpage_collapse_test_exit(mm) || !vma) {
/*
* Make sure that if mm_users is reaching zero while
* khugepaged runs here, khugepaged_exit will find
* mm_slot not pointing to the exiting mm.
*/
- if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
- khugepaged_scan.mm_slot = list_entry(
- mm_slot->mm_node.next,
- struct mm_slot, mm_node);
+ if (slot->mm_node.next != &khugepaged_scan.mm_head) {
+ slot = list_entry(slot->mm_node.next,
+ struct mm_slot, mm_node);
+ khugepaged_scan.mm_slot =
+ mm_slot_entry(slot, struct khugepaged_mm_slot, slot);
khugepaged_scan.address = 0;
} else {
khugepaged_scan.mm_slot = NULL;
kthread_should_stop();
}
-static void khugepaged_do_scan(void)
+static void khugepaged_do_scan(struct collapse_control *cc)
{
- struct page *hpage = NULL;
unsigned int progress = 0, pass_through_head = 0;
unsigned int pages = READ_ONCE(khugepaged_pages_to_scan);
bool wait = true;
+ int result = SCAN_SUCCEED;
lru_add_drain_all();
- while (progress < pages) {
- if (!khugepaged_prealloc_page(&hpage, &wait))
- break;
-
+ while (true) {
cond_resched();
if (unlikely(kthread_should_stop() || try_to_freeze()))
if (khugepaged_has_work() &&
pass_through_head < 2)
progress += khugepaged_scan_mm_slot(pages - progress,
- &hpage);
+ &result, cc);
else
progress = pages;
spin_unlock(&khugepaged_mm_lock);
- }
- if (!IS_ERR_OR_NULL(hpage))
- put_page(hpage);
+ if (progress >= pages)
+ break;
+
+ if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) {
+ /*
+ * If fail to allocate the first time, try to sleep for
+ * a while. When hit again, cancel the scan.
+ */
+ if (!wait)
+ break;
+ wait = false;
+ khugepaged_alloc_sleep();
+ }
+ }
}
static bool khugepaged_should_wakeup(void)
static int khugepaged(void *none)
{
- struct mm_slot *mm_slot;
+ struct khugepaged_mm_slot *mm_slot;
set_freezable();
set_user_nice(current, MAX_NICE);
while (!kthread_should_stop()) {
- khugepaged_do_scan();
+ khugepaged_do_scan(&khugepaged_collapse_control);
khugepaged_wait_work();
}
set_recommended_min_free_kbytes();
mutex_unlock(&khugepaged_mutex);
}
+
+static int madvise_collapse_errno(enum scan_result r)
+{
+ /*
+ * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide
+ * actionable feedback to caller, so they may take an appropriate
+ * fallback measure depending on the nature of the failure.
+ */
+ switch (r) {
+ case SCAN_ALLOC_HUGE_PAGE_FAIL:
+ return -ENOMEM;
+ case SCAN_CGROUP_CHARGE_FAIL:
+ return -EBUSY;
+ /* Resource temporary unavailable - trying again might succeed */
+ case SCAN_PAGE_LOCK:
+ case SCAN_PAGE_LRU:
+ case SCAN_DEL_PAGE_LRU:
+ return -EAGAIN;
+ /*
+ * Other: Trying again likely not to succeed / error intrinsic to
+ * specified memory range. khugepaged likely won't be able to collapse
+ * either.
+ */
+ default:
+ return -EINVAL;
+ }
+}
+
+int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev,
+ unsigned long start, unsigned long end)
+{
+ struct collapse_control *cc;
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long hstart, hend, addr;
+ int thps = 0, last_fail = SCAN_FAIL;
+ bool mmap_locked = true;
+
+ BUG_ON(vma->vm_start > start);
+ BUG_ON(vma->vm_end < end);
+
+ *prev = vma;
+
+ if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false))
+ return -EINVAL;
+
+ cc = kmalloc(sizeof(*cc), GFP_KERNEL);
+ if (!cc)
+ return -ENOMEM;
+ cc->is_khugepaged = false;
+ cc->last_target_node = NUMA_NO_NODE;
+
+ mmgrab(mm);
+ lru_add_drain_all();
+
+ hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
+ hend = end & HPAGE_PMD_MASK;
+
+ for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
+ int result = SCAN_FAIL;
+
+ if (!mmap_locked) {
+ cond_resched();
+ mmap_read_lock(mm);
+ mmap_locked = true;
+ result = hugepage_vma_revalidate(mm, addr, false, &vma,
+ cc);
+ if (result != SCAN_SUCCEED) {
+ last_fail = result;
+ goto out_nolock;
+ }
+
+ hend = vma->vm_end & HPAGE_PMD_MASK;
+ }
+ mmap_assert_locked(mm);
+ memset(cc->node_load, 0, sizeof(cc->node_load));
+ if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) {
+ struct file *file = get_file(vma->vm_file);
+ pgoff_t pgoff = linear_page_index(vma, addr);
+
+ mmap_read_unlock(mm);
+ mmap_locked = false;
+ result = hpage_collapse_scan_file(mm, addr, file, pgoff,
+ cc);
+ fput(file);
+ } else {
+ result = hpage_collapse_scan_pmd(mm, vma, addr,
+ &mmap_locked, cc);
+ }
+ if (!mmap_locked)
+ *prev = NULL; /* Tell caller we dropped mmap_lock */
+
+handle_result:
+ switch (result) {
+ case SCAN_SUCCEED:
+ case SCAN_PMD_MAPPED:
+ ++thps;
+ break;
+ case SCAN_PTE_MAPPED_HUGEPAGE:
+ BUG_ON(mmap_locked);
+ BUG_ON(*prev);
+ mmap_write_lock(mm);
+ result = collapse_pte_mapped_thp(mm, addr, true);
+ mmap_write_unlock(mm);
+ goto handle_result;
+ /* Whitelisted set of results where continuing OK */
+ case SCAN_PMD_NULL:
+ case SCAN_PTE_NON_PRESENT:
+ case SCAN_PTE_UFFD_WP:
+ case SCAN_PAGE_RO:
+ case SCAN_LACK_REFERENCED_PAGE:
+ case SCAN_PAGE_NULL:
+ case SCAN_PAGE_COUNT:
+ case SCAN_PAGE_LOCK:
+ case SCAN_PAGE_COMPOUND:
+ case SCAN_PAGE_LRU:
+ case SCAN_DEL_PAGE_LRU:
+ last_fail = result;
+ break;
+ default:
+ last_fail = result;
+ /* Other error, exit */
+ goto out_maybelock;
+ }
+ }
+
+out_maybelock:
+ /* Caller expects us to hold mmap_lock on return */
+ if (!mmap_locked)
+ mmap_read_lock(mm);
+out_nolock:
+ mmap_assert_locked(mm);
+ mmdrop(mm);
+ kfree(cc);
+
+ return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0
+ : madvise_collapse_errno(last_fail);
+}
* memory block and add it to the object_list and object_tree_root (or
* object_phys_tree_root).
*/
-static struct kmemleak_object *__create_object(unsigned long ptr, size_t size,
- int min_count, gfp_t gfp,
- bool is_phys)
+static void __create_object(unsigned long ptr, size_t size,
+ int min_count, gfp_t gfp, bool is_phys)
{
unsigned long flags;
struct kmemleak_object *object, *parent;
if (!object) {
pr_warn("Cannot allocate a kmemleak_object structure\n");
kmemleak_disable();
- return NULL;
+ return;
}
INIT_LIST_HEAD(&object->object_list);
*/
dump_object_info(parent);
kmem_cache_free(object_cache, object);
- object = NULL;
goto out;
}
}
list_add_tail_rcu(&object->object_list, &object_list);
out:
raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
- return object;
}
/* Create kmemleak object which allocated with virtual address. */
-static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
- int min_count, gfp_t gfp)
+static void create_object(unsigned long ptr, size_t size,
+ int min_count, gfp_t gfp)
{
- return __create_object(ptr, size, min_count, gfp, false);
+ __create_object(ptr, size, min_count, gfp, false);
}
/* Create kmemleak object which allocated with physical address. */
-static struct kmemleak_object *create_object_phys(unsigned long ptr, size_t size,
- int min_count, gfp_t gfp)
+static void create_object_phys(unsigned long ptr, size_t size,
+ int min_count, gfp_t gfp)
{
- return __create_object(ptr, size, min_count, gfp, true);
+ __create_object(ptr, size, min_count, gfp, true);
}
/*
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for KernelMemorySanitizer (KMSAN).
+#
+#
+obj-y := core.o instrumentation.o init.o hooks.o report.o shadow.o
+
+KMSAN_SANITIZE := n
+KCOV_INSTRUMENT := n
+UBSAN_SANITIZE := n
+
+# Disable instrumentation of KMSAN runtime with other tools.
+CC_FLAGS_KMSAN_RUNTIME := -fno-stack-protector
+CC_FLAGS_KMSAN_RUNTIME += $(call cc-option,-fno-conserve-stack)
+CC_FLAGS_KMSAN_RUNTIME += -DDISABLE_BRANCH_PROFILING
+
+CFLAGS_REMOVE.o = $(CC_FLAGS_FTRACE)
+
+CFLAGS_core.o := $(CC_FLAGS_KMSAN_RUNTIME)
+CFLAGS_hooks.o := $(CC_FLAGS_KMSAN_RUNTIME)
+CFLAGS_init.o := $(CC_FLAGS_KMSAN_RUNTIME)
+CFLAGS_instrumentation.o := $(CC_FLAGS_KMSAN_RUNTIME)
+CFLAGS_report.o := $(CC_FLAGS_KMSAN_RUNTIME)
+CFLAGS_shadow.o := $(CC_FLAGS_KMSAN_RUNTIME)
+
+obj-$(CONFIG_KMSAN_KUNIT_TEST) += kmsan_test.o
+KMSAN_SANITIZE_kmsan_test.o := y
+CFLAGS_kmsan_test.o += $(call cc-disable-warning, uninitialized)
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN runtime library.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include <asm/page.h>
+#include <linux/compiler.h>
+#include <linux/export.h>
+#include <linux/highmem.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/kmsan_types.h>
+#include <linux/memory.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/mmzone.h>
+#include <linux/percpu-defs.h>
+#include <linux/preempt.h>
+#include <linux/slab.h>
+#include <linux/stackdepot.h>
+#include <linux/stacktrace.h>
+#include <linux/types.h>
+#include <linux/vmalloc.h>
+
+#include "../slab.h"
+#include "kmsan.h"
+
+bool kmsan_enabled __read_mostly;
+
+/*
+ * Per-CPU KMSAN context to be used in interrupts, where current->kmsan is
+ * unavaliable.
+ */
+DEFINE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx);
+
+void kmsan_internal_task_create(struct task_struct *task)
+{
+ struct kmsan_ctx *ctx = &task->kmsan_ctx;
+ struct thread_info *info = current_thread_info();
+
+ __memset(ctx, 0, sizeof(*ctx));
+ ctx->allow_reporting = true;
+ kmsan_internal_unpoison_memory(info, sizeof(*info), false);
+}
+
+void kmsan_internal_poison_memory(void *address, size_t size, gfp_t flags,
+ unsigned int poison_flags)
+{
+ u32 extra_bits =
+ kmsan_extra_bits(/*depth*/ 0, poison_flags & KMSAN_POISON_FREE);
+ bool checked = poison_flags & KMSAN_POISON_CHECK;
+ depot_stack_handle_t handle;
+
+ handle = kmsan_save_stack_with_flags(flags, extra_bits);
+ kmsan_internal_set_shadow_origin(address, size, -1, handle, checked);
+}
+
+void kmsan_internal_unpoison_memory(void *address, size_t size, bool checked)
+{
+ kmsan_internal_set_shadow_origin(address, size, 0, 0, checked);
+}
+
+depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags,
+ unsigned int extra)
+{
+ unsigned long entries[KMSAN_STACK_DEPTH];
+ unsigned int nr_entries;
+
+ nr_entries = stack_trace_save(entries, KMSAN_STACK_DEPTH, 0);
+
+ /* Don't sleep (see might_sleep_if() in __alloc_pages_nodemask()). */
+ flags &= ~__GFP_DIRECT_RECLAIM;
+
+ return __stack_depot_save(entries, nr_entries, extra, flags, true);
+}
+
+/* Copy the metadata following the memmove() behavior. */
+void kmsan_internal_memmove_metadata(void *dst, void *src, size_t n)
+{
+ depot_stack_handle_t old_origin = 0, new_origin = 0;
+ int src_slots, dst_slots, i, iter, step, skip_bits;
+ depot_stack_handle_t *origin_src, *origin_dst;
+ void *shadow_src, *shadow_dst;
+ u32 *align_shadow_src, shadow;
+ bool backwards;
+
+ shadow_dst = kmsan_get_metadata(dst, KMSAN_META_SHADOW);
+ if (!shadow_dst)
+ return;
+ KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(dst, n));
+
+ shadow_src = kmsan_get_metadata(src, KMSAN_META_SHADOW);
+ if (!shadow_src) {
+ /*
+ * @src is untracked: zero out destination shadow, ignore the
+ * origins, we're done.
+ */
+ __memset(shadow_dst, 0, n);
+ return;
+ }
+ KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(src, n));
+
+ __memmove(shadow_dst, shadow_src, n);
+
+ origin_dst = kmsan_get_metadata(dst, KMSAN_META_ORIGIN);
+ origin_src = kmsan_get_metadata(src, KMSAN_META_ORIGIN);
+ KMSAN_WARN_ON(!origin_dst || !origin_src);
+ src_slots = (ALIGN((u64)src + n, KMSAN_ORIGIN_SIZE) -
+ ALIGN_DOWN((u64)src, KMSAN_ORIGIN_SIZE)) /
+ KMSAN_ORIGIN_SIZE;
+ dst_slots = (ALIGN((u64)dst + n, KMSAN_ORIGIN_SIZE) -
+ ALIGN_DOWN((u64)dst, KMSAN_ORIGIN_SIZE)) /
+ KMSAN_ORIGIN_SIZE;
+ KMSAN_WARN_ON((src_slots < 1) || (dst_slots < 1));
+ KMSAN_WARN_ON((src_slots - dst_slots > 1) ||
+ (dst_slots - src_slots < -1));
+
+ backwards = dst > src;
+ i = backwards ? min(src_slots, dst_slots) - 1 : 0;
+ iter = backwards ? -1 : 1;
+
+ align_shadow_src =
+ (u32 *)ALIGN_DOWN((u64)shadow_src, KMSAN_ORIGIN_SIZE);
+ for (step = 0; step < min(src_slots, dst_slots); step++, i += iter) {
+ KMSAN_WARN_ON(i < 0);
+ shadow = align_shadow_src[i];
+ if (i == 0) {
+ /*
+ * If @src isn't aligned on KMSAN_ORIGIN_SIZE, don't
+ * look at the first @src % KMSAN_ORIGIN_SIZE bytes
+ * of the first shadow slot.
+ */
+ skip_bits = ((u64)src % KMSAN_ORIGIN_SIZE) * 8;
+ shadow = (shadow >> skip_bits) << skip_bits;
+ }
+ if (i == src_slots - 1) {
+ /*
+ * If @src + n isn't aligned on
+ * KMSAN_ORIGIN_SIZE, don't look at the last
+ * (@src + n) % KMSAN_ORIGIN_SIZE bytes of the
+ * last shadow slot.
+ */
+ skip_bits = (((u64)src + n) % KMSAN_ORIGIN_SIZE) * 8;
+ shadow = (shadow << skip_bits) >> skip_bits;
+ }
+ /*
+ * Overwrite the origin only if the corresponding
+ * shadow is nonempty.
+ */
+ if (origin_src[i] && (origin_src[i] != old_origin) && shadow) {
+ old_origin = origin_src[i];
+ new_origin = kmsan_internal_chain_origin(old_origin);
+ /*
+ * kmsan_internal_chain_origin() may return
+ * NULL, but we don't want to lose the previous
+ * origin value.
+ */
+ if (!new_origin)
+ new_origin = old_origin;
+ }
+ if (shadow)
+ origin_dst[i] = new_origin;
+ else
+ origin_dst[i] = 0;
+ }
+ /*
+ * If dst_slots is greater than src_slots (i.e.
+ * dst_slots == src_slots + 1), there is an extra origin slot at the
+ * beginning or end of the destination buffer, for which we take the
+ * origin from the previous slot.
+ * This is only done if the part of the source shadow corresponding to
+ * slot is non-zero.
+ *
+ * E.g. if we copy 8 aligned bytes that are marked as uninitialized
+ * and have origins o111 and o222, to an unaligned buffer with offset 1,
+ * these two origins are copied to three origin slots, so one of then
+ * needs to be duplicated, depending on the copy direction (@backwards)
+ *
+ * src shadow: |uuuu|uuuu|....|
+ * src origin: |o111|o222|....|
+ *
+ * backwards = 0:
+ * dst shadow: |.uuu|uuuu|u...|
+ * dst origin: |....|o111|o222| - fill the empty slot with o111
+ * backwards = 1:
+ * dst shadow: |.uuu|uuuu|u...|
+ * dst origin: |o111|o222|....| - fill the empty slot with o222
+ */
+ if (src_slots < dst_slots) {
+ if (backwards) {
+ shadow = align_shadow_src[src_slots - 1];
+ skip_bits = (((u64)dst + n) % KMSAN_ORIGIN_SIZE) * 8;
+ shadow = (shadow << skip_bits) >> skip_bits;
+ if (shadow)
+ /* src_slots > 0, therefore dst_slots is at least 2 */
+ origin_dst[dst_slots - 1] =
+ origin_dst[dst_slots - 2];
+ } else {
+ shadow = align_shadow_src[0];
+ skip_bits = ((u64)dst % KMSAN_ORIGIN_SIZE) * 8;
+ shadow = (shadow >> skip_bits) << skip_bits;
+ if (shadow)
+ origin_dst[0] = origin_dst[1];
+ }
+ }
+}
+
+depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id)
+{
+ unsigned long entries[3];
+ u32 extra_bits;
+ int depth;
+ bool uaf;
+
+ if (!id)
+ return id;
+ /*
+ * Make sure we have enough spare bits in @id to hold the UAF bit and
+ * the chain depth.
+ */
+ BUILD_BUG_ON(
+ (1 << STACK_DEPOT_EXTRA_BITS) <= (KMSAN_MAX_ORIGIN_DEPTH << 1));
+
+ extra_bits = stack_depot_get_extra_bits(id);
+ depth = kmsan_depth_from_eb(extra_bits);
+ uaf = kmsan_uaf_from_eb(extra_bits);
+
+ /*
+ * Stop chaining origins once the depth reached KMSAN_MAX_ORIGIN_DEPTH.
+ * This mostly happens in the case structures with uninitialized padding
+ * are copied around many times. Origin chains for such structures are
+ * usually periodic, and it does not make sense to fully store them.
+ */
+ if (depth == KMSAN_MAX_ORIGIN_DEPTH)
+ return id;
+
+ depth++;
+ extra_bits = kmsan_extra_bits(depth, uaf);
+
+ entries[0] = KMSAN_CHAIN_MAGIC_ORIGIN;
+ entries[1] = kmsan_save_stack_with_flags(GFP_ATOMIC, 0);
+ entries[2] = id;
+ /*
+ * @entries is a local var in non-instrumented code, so KMSAN does not
+ * know it is initialized. Explicitly unpoison it to avoid false
+ * positives when __stack_depot_save() passes it to instrumented code.
+ */
+ kmsan_internal_unpoison_memory(entries, sizeof(entries), false);
+ return __stack_depot_save(entries, ARRAY_SIZE(entries), extra_bits,
+ GFP_ATOMIC, true);
+}
+
+void kmsan_internal_set_shadow_origin(void *addr, size_t size, int b,
+ u32 origin, bool checked)
+{
+ u64 address = (u64)addr;
+ void *shadow_start;
+ u32 *origin_start;
+ size_t pad = 0;
+
+ KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(addr, size));
+ shadow_start = kmsan_get_metadata(addr, KMSAN_META_SHADOW);
+ if (!shadow_start) {
+ /*
+ * kmsan_metadata_is_contiguous() is true, so either all shadow
+ * and origin pages are NULL, or all are non-NULL.
+ */
+ if (checked) {
+ pr_err("%s: not memsetting %ld bytes starting at %px, because the shadow is NULL\n",
+ __func__, size, addr);
+ KMSAN_WARN_ON(true);
+ }
+ return;
+ }
+ __memset(shadow_start, b, size);
+
+ if (!IS_ALIGNED(address, KMSAN_ORIGIN_SIZE)) {
+ pad = address % KMSAN_ORIGIN_SIZE;
+ address -= pad;
+ size += pad;
+ }
+ size = ALIGN(size, KMSAN_ORIGIN_SIZE);
+ origin_start =
+ (u32 *)kmsan_get_metadata((void *)address, KMSAN_META_ORIGIN);
+
+ for (int i = 0; i < size / KMSAN_ORIGIN_SIZE; i++)
+ origin_start[i] = origin;
+}
+
+struct page *kmsan_vmalloc_to_page_or_null(void *vaddr)
+{
+ struct page *page;
+
+ if (!kmsan_internal_is_vmalloc_addr(vaddr) &&
+ !kmsan_internal_is_module_addr(vaddr))
+ return NULL;
+ page = vmalloc_to_page(vaddr);
+ if (pfn_valid(page_to_pfn(page)))
+ return page;
+ else
+ return NULL;
+}
+
+void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr,
+ int reason)
+{
+ depot_stack_handle_t cur_origin = 0, new_origin = 0;
+ unsigned long addr64 = (unsigned long)addr;
+ depot_stack_handle_t *origin = NULL;
+ unsigned char *shadow = NULL;
+ int cur_off_start = -1;
+ int chunk_size;
+ size_t pos = 0;
+
+ if (!size)
+ return;
+ KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(addr, size));
+ while (pos < size) {
+ chunk_size = min(size - pos,
+ PAGE_SIZE - ((addr64 + pos) % PAGE_SIZE));
+ shadow = kmsan_get_metadata((void *)(addr64 + pos),
+ KMSAN_META_SHADOW);
+ if (!shadow) {
+ /*
+ * This page is untracked. If there were uninitialized
+ * bytes before, report them.
+ */
+ if (cur_origin) {
+ kmsan_enter_runtime();
+ kmsan_report(cur_origin, addr, size,
+ cur_off_start, pos - 1, user_addr,
+ reason);
+ kmsan_leave_runtime();
+ }
+ cur_origin = 0;
+ cur_off_start = -1;
+ pos += chunk_size;
+ continue;
+ }
+ for (int i = 0; i < chunk_size; i++) {
+ if (!shadow[i]) {
+ /*
+ * This byte is unpoisoned. If there were
+ * poisoned bytes before, report them.
+ */
+ if (cur_origin) {
+ kmsan_enter_runtime();
+ kmsan_report(cur_origin, addr, size,
+ cur_off_start, pos + i - 1,
+ user_addr, reason);
+ kmsan_leave_runtime();
+ }
+ cur_origin = 0;
+ cur_off_start = -1;
+ continue;
+ }
+ origin = kmsan_get_metadata((void *)(addr64 + pos + i),
+ KMSAN_META_ORIGIN);
+ KMSAN_WARN_ON(!origin);
+ new_origin = *origin;
+ /*
+ * Encountered new origin - report the previous
+ * uninitialized range.
+ */
+ if (cur_origin != new_origin) {
+ if (cur_origin) {
+ kmsan_enter_runtime();
+ kmsan_report(cur_origin, addr, size,
+ cur_off_start, pos + i - 1,
+ user_addr, reason);
+ kmsan_leave_runtime();
+ }
+ cur_origin = new_origin;
+ cur_off_start = pos + i;
+ }
+ }
+ pos += chunk_size;
+ }
+ KMSAN_WARN_ON(pos != size);
+ if (cur_origin) {
+ kmsan_enter_runtime();
+ kmsan_report(cur_origin, addr, size, cur_off_start, pos - 1,
+ user_addr, reason);
+ kmsan_leave_runtime();
+ }
+}
+
+bool kmsan_metadata_is_contiguous(void *addr, size_t size)
+{
+ char *cur_shadow = NULL, *next_shadow = NULL, *cur_origin = NULL,
+ *next_origin = NULL;
+ u64 cur_addr = (u64)addr, next_addr = cur_addr + PAGE_SIZE;
+ depot_stack_handle_t *origin_p;
+ bool all_untracked = false;
+
+ if (!size)
+ return true;
+
+ /* The whole range belongs to the same page. */
+ if (ALIGN_DOWN(cur_addr + size - 1, PAGE_SIZE) ==
+ ALIGN_DOWN(cur_addr, PAGE_SIZE))
+ return true;
+
+ cur_shadow = kmsan_get_metadata((void *)cur_addr, /*is_origin*/ false);
+ if (!cur_shadow)
+ all_untracked = true;
+ cur_origin = kmsan_get_metadata((void *)cur_addr, /*is_origin*/ true);
+ if (all_untracked && cur_origin)
+ goto report;
+
+ for (; next_addr < (u64)addr + size;
+ cur_addr = next_addr, cur_shadow = next_shadow,
+ cur_origin = next_origin, next_addr += PAGE_SIZE) {
+ next_shadow = kmsan_get_metadata((void *)next_addr, false);
+ next_origin = kmsan_get_metadata((void *)next_addr, true);
+ if (all_untracked) {
+ if (next_shadow || next_origin)
+ goto report;
+ if (!next_shadow && !next_origin)
+ continue;
+ }
+ if (((u64)cur_shadow == ((u64)next_shadow - PAGE_SIZE)) &&
+ ((u64)cur_origin == ((u64)next_origin - PAGE_SIZE)))
+ continue;
+ goto report;
+ }
+ return true;
+
+report:
+ pr_err("%s: attempting to access two shadow page ranges.\n", __func__);
+ pr_err("Access of size %ld at %px.\n", size, addr);
+ pr_err("Addresses belonging to different ranges: %px and %px\n",
+ (void *)cur_addr, (void *)next_addr);
+ pr_err("page[0].shadow: %px, page[1].shadow: %px\n", cur_shadow,
+ next_shadow);
+ pr_err("page[0].origin: %px, page[1].origin: %px\n", cur_origin,
+ next_origin);
+ origin_p = kmsan_get_metadata(addr, KMSAN_META_ORIGIN);
+ if (origin_p) {
+ pr_err("Origin: %08x\n", *origin_p);
+ kmsan_print_origin(*origin_p);
+ } else {
+ pr_err("Origin: unavailable\n");
+ }
+ return false;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN hooks for kernel subsystems.
+ *
+ * These functions handle creation of KMSAN metadata for memory allocations.
+ *
+ * Copyright (C) 2018-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include <linux/cacheflush.h>
+#include <linux/dma-direction.h>
+#include <linux/gfp.h>
+#include <linux/kmsan.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/usb.h>
+
+#include "../internal.h"
+#include "../slab.h"
+#include "kmsan.h"
+
+/*
+ * Instrumented functions shouldn't be called under
+ * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
+ * skipping effects of functions like memset() inside instrumented code.
+ */
+
+void kmsan_task_create(struct task_struct *task)
+{
+ kmsan_enter_runtime();
+ kmsan_internal_task_create(task);
+ kmsan_leave_runtime();
+}
+
+void kmsan_task_exit(struct task_struct *task)
+{
+ struct kmsan_ctx *ctx = &task->kmsan_ctx;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ ctx->allow_reporting = false;
+}
+
+void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
+{
+ if (unlikely(object == NULL))
+ return;
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ /*
+ * There's a ctor or this is an RCU cache - do nothing. The memory
+ * status hasn't changed since last use.
+ */
+ if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
+ return;
+
+ kmsan_enter_runtime();
+ if (flags & __GFP_ZERO)
+ kmsan_internal_unpoison_memory(object, s->object_size,
+ KMSAN_POISON_CHECK);
+ else
+ kmsan_internal_poison_memory(object, s->object_size, flags,
+ KMSAN_POISON_CHECK);
+ kmsan_leave_runtime();
+}
+
+void kmsan_slab_free(struct kmem_cache *s, void *object)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ /* RCU slabs could be legally used after free within the RCU period */
+ if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
+ return;
+ /*
+ * If there's a constructor, freed memory must remain in the same state
+ * until the next allocation. We cannot save its state to detect
+ * use-after-free bugs, instead we just keep it unpoisoned.
+ */
+ if (s->ctor)
+ return;
+ kmsan_enter_runtime();
+ kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
+ KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
+ kmsan_leave_runtime();
+}
+
+void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
+{
+ if (unlikely(ptr == NULL))
+ return;
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ if (flags & __GFP_ZERO)
+ kmsan_internal_unpoison_memory((void *)ptr, size,
+ /*checked*/ true);
+ else
+ kmsan_internal_poison_memory((void *)ptr, size, flags,
+ KMSAN_POISON_CHECK);
+ kmsan_leave_runtime();
+}
+
+void kmsan_kfree_large(const void *ptr)
+{
+ struct page *page;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ page = virt_to_head_page((void *)ptr);
+ KMSAN_WARN_ON(ptr != page_address(page));
+ kmsan_internal_poison_memory((void *)ptr,
+ PAGE_SIZE << compound_order(page),
+ GFP_KERNEL,
+ KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
+ kmsan_leave_runtime();
+}
+
+static unsigned long vmalloc_shadow(unsigned long addr)
+{
+ return (unsigned long)kmsan_get_metadata((void *)addr,
+ KMSAN_META_SHADOW);
+}
+
+static unsigned long vmalloc_origin(unsigned long addr)
+{
+ return (unsigned long)kmsan_get_metadata((void *)addr,
+ KMSAN_META_ORIGIN);
+}
+
+void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
+{
+ __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
+ __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
+ flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
+ flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
+}
+
+/*
+ * This function creates new shadow/origin pages for the physical pages mapped
+ * into the virtual memory. If those physical pages already had shadow/origin,
+ * those are ignored.
+ */
+void kmsan_ioremap_page_range(unsigned long start, unsigned long end,
+ phys_addr_t phys_addr, pgprot_t prot,
+ unsigned int page_shift)
+{
+ gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;
+ struct page *shadow, *origin;
+ unsigned long off = 0;
+ int nr;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ nr = (end - start) / PAGE_SIZE;
+ kmsan_enter_runtime();
+ for (int i = 0; i < nr; i++, off += PAGE_SIZE) {
+ shadow = alloc_pages(gfp_mask, 1);
+ origin = alloc_pages(gfp_mask, 1);
+ __vmap_pages_range_noflush(
+ vmalloc_shadow(start + off),
+ vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
+ PAGE_SHIFT);
+ __vmap_pages_range_noflush(
+ vmalloc_origin(start + off),
+ vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
+ PAGE_SHIFT);
+ }
+ flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
+ flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
+ kmsan_leave_runtime();
+}
+
+void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
+{
+ unsigned long v_shadow, v_origin;
+ struct page *shadow, *origin;
+ int nr;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ nr = (end - start) / PAGE_SIZE;
+ kmsan_enter_runtime();
+ v_shadow = (unsigned long)vmalloc_shadow(start);
+ v_origin = (unsigned long)vmalloc_origin(start);
+ for (int i = 0; i < nr;
+ i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
+ shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
+ origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
+ __vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
+ __vunmap_range_noflush(v_origin, vmalloc_origin(end));
+ if (shadow)
+ __free_pages(shadow, 1);
+ if (origin)
+ __free_pages(origin, 1);
+ }
+ flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
+ flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
+ kmsan_leave_runtime();
+}
+
+void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
+ size_t left)
+{
+ unsigned long ua_flags;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ /*
+ * At this point we've copied the memory already. It's hard to check it
+ * before copying, as the size of actually copied buffer is unknown.
+ */
+
+ /* copy_to_user() may copy zero bytes. No need to check. */
+ if (!to_copy)
+ return;
+ /* Or maybe copy_to_user() failed to copy anything. */
+ if (to_copy <= left)
+ return;
+
+ ua_flags = user_access_save();
+ if ((u64)to < TASK_SIZE) {
+ /* This is a user memory access, check it. */
+ kmsan_internal_check_memory((void *)from, to_copy - left, to,
+ REASON_COPY_TO_USER);
+ } else {
+ /* Otherwise this is a kernel memory access. This happens when a
+ * compat syscall passes an argument allocated on the kernel
+ * stack to a real syscall.
+ * Don't check anything, just copy the shadow of the copied
+ * bytes.
+ */
+ kmsan_internal_memmove_metadata((void *)to, (void *)from,
+ to_copy - left);
+ }
+ user_access_restore(ua_flags);
+}
+EXPORT_SYMBOL(kmsan_copy_to_user);
+
+/* Helper function to check an URB. */
+void kmsan_handle_urb(const struct urb *urb, bool is_out)
+{
+ if (!urb)
+ return;
+ if (is_out)
+ kmsan_internal_check_memory(urb->transfer_buffer,
+ urb->transfer_buffer_length,
+ /*user_addr*/ 0, REASON_SUBMIT_URB);
+ else
+ kmsan_internal_unpoison_memory(urb->transfer_buffer,
+ urb->transfer_buffer_length,
+ /*checked*/ false);
+}
+
+static void kmsan_handle_dma_page(const void *addr, size_t size,
+ enum dma_data_direction dir)
+{
+ switch (dir) {
+ case DMA_BIDIRECTIONAL:
+ kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
+ REASON_ANY);
+ kmsan_internal_unpoison_memory((void *)addr, size,
+ /*checked*/ false);
+ break;
+ case DMA_TO_DEVICE:
+ kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
+ REASON_ANY);
+ break;
+ case DMA_FROM_DEVICE:
+ kmsan_internal_unpoison_memory((void *)addr, size,
+ /*checked*/ false);
+ break;
+ case DMA_NONE:
+ break;
+ }
+}
+
+/* Helper function to handle DMA data transfers. */
+void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
+ enum dma_data_direction dir)
+{
+ u64 page_offset, to_go, addr;
+
+ if (PageHighMem(page))
+ return;
+ addr = (u64)page_address(page) + offset;
+ /*
+ * The kernel may occasionally give us adjacent DMA pages not belonging
+ * to the same allocation. Process them separately to avoid triggering
+ * internal KMSAN checks.
+ */
+ while (size > 0) {
+ page_offset = addr % PAGE_SIZE;
+ to_go = min(PAGE_SIZE - page_offset, (u64)size);
+ kmsan_handle_dma_page((void *)addr, to_go, dir);
+ addr += to_go;
+ size -= to_go;
+ }
+}
+
+void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
+ enum dma_data_direction dir)
+{
+ struct scatterlist *item;
+ int i;
+
+ for_each_sg(sg, item, nents, i)
+ kmsan_handle_dma(sg_page(item), item->offset, item->length,
+ dir);
+}
+
+/* Functions from kmsan-checks.h follow. */
+void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ /* The users may want to poison/unpoison random memory. */
+ kmsan_internal_poison_memory((void *)address, size, flags,
+ KMSAN_POISON_NOCHECK);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(kmsan_poison_memory);
+
+void kmsan_unpoison_memory(const void *address, size_t size)
+{
+ unsigned long ua_flags;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ ua_flags = user_access_save();
+ kmsan_enter_runtime();
+ /* The users may want to poison/unpoison random memory. */
+ kmsan_internal_unpoison_memory((void *)address, size,
+ KMSAN_POISON_NOCHECK);
+ kmsan_leave_runtime();
+ user_access_restore(ua_flags);
+}
+EXPORT_SYMBOL(kmsan_unpoison_memory);
+
+/*
+ * Version of kmsan_unpoison_memory() that can be called from within the KMSAN
+ * runtime.
+ *
+ * Non-instrumented IRQ entry functions receive struct pt_regs from assembly
+ * code. Those regs need to be unpoisoned, otherwise using them will result in
+ * false positives.
+ * Using kmsan_unpoison_memory() is not an option in entry code, because the
+ * return value of in_task() is inconsistent - as a result, certain calls to
+ * kmsan_unpoison_memory() are ignored. kmsan_unpoison_entry_regs() ensures that
+ * the registers are unpoisoned even if kmsan_in_runtime() is true in the early
+ * entry code.
+ */
+void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
+{
+ unsigned long ua_flags;
+
+ if (!kmsan_enabled)
+ return;
+
+ ua_flags = user_access_save();
+ kmsan_internal_unpoison_memory((void *)regs, sizeof(*regs),
+ KMSAN_POISON_NOCHECK);
+ user_access_restore(ua_flags);
+}
+
+void kmsan_check_memory(const void *addr, size_t size)
+{
+ if (!kmsan_enabled)
+ return;
+ return kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
+ REASON_ANY);
+}
+EXPORT_SYMBOL(kmsan_check_memory);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN initialization routines.
+ *
+ * Copyright (C) 2017-2021 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include "kmsan.h"
+
+#include <asm/sections.h>
+#include <linux/mm.h>
+#include <linux/memblock.h>
+
+#include "../internal.h"
+
+#define NUM_FUTURE_RANGES 128
+struct start_end_pair {
+ u64 start, end;
+};
+
+static struct start_end_pair start_end_pairs[NUM_FUTURE_RANGES] __initdata;
+static int future_index __initdata;
+
+/*
+ * Record a range of memory for which the metadata pages will be created once
+ * the page allocator becomes available.
+ */
+static void __init kmsan_record_future_shadow_range(void *start, void *end)
+{
+ u64 nstart = (u64)start, nend = (u64)end, cstart, cend;
+ bool merged = false;
+
+ KMSAN_WARN_ON(future_index == NUM_FUTURE_RANGES);
+ KMSAN_WARN_ON((nstart >= nend) || !nstart || !nend);
+ nstart = ALIGN_DOWN(nstart, PAGE_SIZE);
+ nend = ALIGN(nend, PAGE_SIZE);
+
+ /*
+ * Scan the existing ranges to see if any of them overlaps with
+ * [start, end). In that case, merge the two ranges instead of
+ * creating a new one.
+ * The number of ranges is less than 20, so there is no need to organize
+ * them into a more intelligent data structure.
+ */
+ for (int i = 0; i < future_index; i++) {
+ cstart = start_end_pairs[i].start;
+ cend = start_end_pairs[i].end;
+ if ((cstart < nstart && cend < nstart) ||
+ (cstart > nend && cend > nend))
+ /* ranges are disjoint - do not merge */
+ continue;
+ start_end_pairs[i].start = min(nstart, cstart);
+ start_end_pairs[i].end = max(nend, cend);
+ merged = true;
+ break;
+ }
+ if (merged)
+ return;
+ start_end_pairs[future_index].start = nstart;
+ start_end_pairs[future_index].end = nend;
+ future_index++;
+}
+
+/*
+ * Initialize the shadow for existing mappings during kernel initialization.
+ * These include kernel text/data sections, NODE_DATA and future ranges
+ * registered while creating other data (e.g. percpu).
+ *
+ * Allocations via memblock can be only done before slab is initialized.
+ */
+void __init kmsan_init_shadow(void)
+{
+ const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
+ phys_addr_t p_start, p_end;
+ u64 loop;
+ int nid;
+
+ for_each_reserved_mem_range(loop, &p_start, &p_end)
+ kmsan_record_future_shadow_range(phys_to_virt(p_start),
+ phys_to_virt(p_end));
+ /* Allocate shadow for .data */
+ kmsan_record_future_shadow_range(_sdata, _edata);
+
+ for_each_online_node(nid)
+ kmsan_record_future_shadow_range(
+ NODE_DATA(nid), (char *)NODE_DATA(nid) + nd_size);
+
+ for (int i = 0; i < future_index; i++)
+ kmsan_init_alloc_meta_for_range(
+ (void *)start_end_pairs[i].start,
+ (void *)start_end_pairs[i].end);
+}
+
+struct metadata_page_pair {
+ struct page *shadow, *origin;
+};
+static struct metadata_page_pair held_back[MAX_ORDER] __initdata;
+
+/*
+ * Eager metadata allocation. When the memblock allocator is freeing pages to
+ * pagealloc, we use 2/3 of them as metadata for the remaining 1/3.
+ * We store the pointers to the returned blocks of pages in held_back[] grouped
+ * by their order: when kmsan_memblock_free_pages() is called for the first
+ * time with a certain order, it is reserved as a shadow block, for the second
+ * time - as an origin block. On the third time the incoming block receives its
+ * shadow and origin ranges from the previously saved shadow and origin blocks,
+ * after which held_back[order] can be used again.
+ *
+ * At the very end there may be leftover blocks in held_back[]. They are
+ * collected later by kmsan_memblock_discard().
+ */
+bool kmsan_memblock_free_pages(struct page *page, unsigned int order)
+{
+ struct page *shadow, *origin;
+
+ if (!held_back[order].shadow) {
+ held_back[order].shadow = page;
+ return false;
+ }
+ if (!held_back[order].origin) {
+ held_back[order].origin = page;
+ return false;
+ }
+ shadow = held_back[order].shadow;
+ origin = held_back[order].origin;
+ kmsan_setup_meta(page, shadow, origin, order);
+
+ held_back[order].shadow = NULL;
+ held_back[order].origin = NULL;
+ return true;
+}
+
+#define MAX_BLOCKS 8
+struct smallstack {
+ struct page *items[MAX_BLOCKS];
+ int index;
+ int order;
+};
+
+static struct smallstack collect = {
+ .index = 0,
+ .order = MAX_ORDER,
+};
+
+static void smallstack_push(struct smallstack *stack, struct page *pages)
+{
+ KMSAN_WARN_ON(stack->index == MAX_BLOCKS);
+ stack->items[stack->index] = pages;
+ stack->index++;
+}
+#undef MAX_BLOCKS
+
+static struct page *smallstack_pop(struct smallstack *stack)
+{
+ struct page *ret;
+
+ KMSAN_WARN_ON(stack->index == 0);
+ stack->index--;
+ ret = stack->items[stack->index];
+ stack->items[stack->index] = NULL;
+ return ret;
+}
+
+static void do_collection(void)
+{
+ struct page *page, *shadow, *origin;
+
+ while (collect.index >= 3) {
+ page = smallstack_pop(&collect);
+ shadow = smallstack_pop(&collect);
+ origin = smallstack_pop(&collect);
+ kmsan_setup_meta(page, shadow, origin, collect.order);
+ __free_pages_core(page, collect.order);
+ }
+}
+
+static void collect_split(void)
+{
+ struct smallstack tmp = {
+ .order = collect.order - 1,
+ .index = 0,
+ };
+ struct page *page;
+
+ if (!collect.order)
+ return;
+ while (collect.index) {
+ page = smallstack_pop(&collect);
+ smallstack_push(&tmp, &page[0]);
+ smallstack_push(&tmp, &page[1 << tmp.order]);
+ }
+ __memcpy(&collect, &tmp, sizeof(tmp));
+}
+
+/*
+ * Memblock is about to go away. Split the page blocks left over in held_back[]
+ * and return 1/3 of that memory to the system.
+ */
+static void kmsan_memblock_discard(void)
+{
+ /*
+ * For each order=N:
+ * - push held_back[N].shadow and .origin to @collect;
+ * - while there are >= 3 elements in @collect, do garbage collection:
+ * - pop 3 ranges from @collect;
+ * - use two of them as shadow and origin for the third one;
+ * - repeat;
+ * - split each remaining element from @collect into 2 ranges of
+ * order=N-1,
+ * - repeat.
+ */
+ collect.order = MAX_ORDER - 1;
+ for (int i = MAX_ORDER - 1; i >= 0; i--) {
+ if (held_back[i].shadow)
+ smallstack_push(&collect, held_back[i].shadow);
+ if (held_back[i].origin)
+ smallstack_push(&collect, held_back[i].origin);
+ held_back[i].shadow = NULL;
+ held_back[i].origin = NULL;
+ do_collection();
+ collect_split();
+ }
+}
+
+void __init kmsan_init_runtime(void)
+{
+ /* Assuming current is init_task */
+ kmsan_internal_task_create(current);
+ kmsan_memblock_discard();
+ pr_info("Starting KernelMemorySanitizer\n");
+ pr_info("ATTENTION: KMSAN is a debugging tool! Do not use it on production machines!\n");
+ kmsan_enabled = true;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN compiler API.
+ *
+ * This file implements __msan_XXX hooks that Clang inserts into the code
+ * compiled with -fsanitize=kernel-memory.
+ * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN
+ * instrumentation works.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include "kmsan.h"
+#include <linux/gfp.h>
+#include <linux/mm.h>
+#include <linux/uaccess.h>
+
+static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store)
+{
+ if ((u64)addr < TASK_SIZE)
+ return true;
+ if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW))
+ return true;
+ return false;
+}
+
+static inline struct shadow_origin_ptr
+get_shadow_origin_ptr(void *addr, u64 size, bool store)
+{
+ unsigned long ua_flags = user_access_save();
+ struct shadow_origin_ptr ret;
+
+ ret = kmsan_get_shadow_origin_ptr(addr, size, store);
+ user_access_restore(ua_flags);
+ return ret;
+}
+
+/* Get shadow and origin pointers for a memory load with non-standard size. */
+struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
+ uintptr_t size)
+{
+ return get_shadow_origin_ptr(addr, size, /*store*/ false);
+}
+EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);
+
+/* Get shadow and origin pointers for a memory store with non-standard size. */
+struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
+ uintptr_t size)
+{
+ return get_shadow_origin_ptr(addr, size, /*store*/ true);
+}
+EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);
+
+/*
+ * Declare functions that obtain shadow/origin pointers for loads and stores
+ * with fixed size.
+ */
+#define DECLARE_METADATA_PTR_GETTER(size) \
+ struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \
+ void *addr) \
+ { \
+ return get_shadow_origin_ptr(addr, size, /*store*/ false); \
+ } \
+ EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size); \
+ struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \
+ void *addr) \
+ { \
+ return get_shadow_origin_ptr(addr, size, /*store*/ true); \
+ } \
+ EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)
+
+DECLARE_METADATA_PTR_GETTER(1);
+DECLARE_METADATA_PTR_GETTER(2);
+DECLARE_METADATA_PTR_GETTER(4);
+DECLARE_METADATA_PTR_GETTER(8);
+
+/*
+ * Handle a memory store performed by inline assembly. KMSAN conservatively
+ * attempts to unpoison the outputs of asm() directives to prevent false
+ * positives caused by missed stores.
+ */
+void __msan_instrument_asm_store(void *addr, uintptr_t size)
+{
+ unsigned long ua_flags;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ ua_flags = user_access_save();
+ /*
+ * Most of the accesses are below 32 bytes. The two exceptions so far
+ * are clwb() (64 bytes) and FPU state (512 bytes).
+ * It's unlikely that the assembly will touch more than 512 bytes.
+ */
+ if (size > 512) {
+ WARN_ONCE(1, "assembly store size too big: %ld\n", size);
+ size = 8;
+ }
+ if (is_bad_asm_addr(addr, size, /*is_store*/ true)) {
+ user_access_restore(ua_flags);
+ return;
+ }
+ kmsan_enter_runtime();
+ /* Unpoisoning the memory on best effort. */
+ kmsan_internal_unpoison_memory(addr, size, /*checked*/ false);
+ kmsan_leave_runtime();
+ user_access_restore(ua_flags);
+}
+EXPORT_SYMBOL(__msan_instrument_asm_store);
+
+/*
+ * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset
+ * intrinsics with calls to respective __msan_ functions. We use
+ * get_param0_metadata() and set_retval_metadata() to store the shadow/origin
+ * values for the destination argument of these functions and use them for the
+ * functions' return values.
+ */
+static inline void get_param0_metadata(u64 *shadow,
+ depot_stack_handle_t *origin)
+{
+ struct kmsan_ctx *ctx = kmsan_get_context();
+
+ *shadow = *(u64 *)(ctx->cstate.param_tls);
+ *origin = ctx->cstate.param_origin_tls[0];
+}
+
+static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin)
+{
+ struct kmsan_ctx *ctx = kmsan_get_context();
+
+ *(u64 *)(ctx->cstate.retval_tls) = shadow;
+ ctx->cstate.retval_origin_tls = origin;
+}
+
+/* Handle llvm.memmove intrinsic. */
+void *__msan_memmove(void *dst, const void *src, uintptr_t n)
+{
+ depot_stack_handle_t origin;
+ void *result;
+ u64 shadow;
+
+ get_param0_metadata(&shadow, &origin);
+ result = __memmove(dst, src, n);
+ if (!n)
+ /* Some people call memmove() with zero length. */
+ return result;
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return result;
+
+ kmsan_enter_runtime();
+ kmsan_internal_memmove_metadata(dst, (void *)src, n);
+ kmsan_leave_runtime();
+
+ set_retval_metadata(shadow, origin);
+ return result;
+}
+EXPORT_SYMBOL(__msan_memmove);
+
+/* Handle llvm.memcpy intrinsic. */
+void *__msan_memcpy(void *dst, const void *src, uintptr_t n)
+{
+ depot_stack_handle_t origin;
+ void *result;
+ u64 shadow;
+
+ get_param0_metadata(&shadow, &origin);
+ result = __memcpy(dst, src, n);
+ if (!n)
+ /* Some people call memcpy() with zero length. */
+ return result;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return result;
+
+ kmsan_enter_runtime();
+ /* Using memmove instead of memcpy doesn't affect correctness. */
+ kmsan_internal_memmove_metadata(dst, (void *)src, n);
+ kmsan_leave_runtime();
+
+ set_retval_metadata(shadow, origin);
+ return result;
+}
+EXPORT_SYMBOL(__msan_memcpy);
+
+/* Handle llvm.memset intrinsic. */
+void *__msan_memset(void *dst, int c, uintptr_t n)
+{
+ depot_stack_handle_t origin;
+ void *result;
+ u64 shadow;
+
+ get_param0_metadata(&shadow, &origin);
+ result = __memset(dst, c, n);
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return result;
+
+ kmsan_enter_runtime();
+ /*
+ * Clang doesn't pass parameter metadata here, so it is impossible to
+ * use shadow of @c to set up the shadow for @dst.
+ */
+ kmsan_internal_unpoison_memory(dst, n, /*checked*/ false);
+ kmsan_leave_runtime();
+
+ set_retval_metadata(shadow, origin);
+ return result;
+}
+EXPORT_SYMBOL(__msan_memset);
+
+/*
+ * Create a new origin from an old one. This is done when storing an
+ * uninitialized value to memory. When reporting an error, KMSAN unrolls and
+ * prints the whole chain of stores that preceded the use of this value.
+ */
+depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)
+{
+ depot_stack_handle_t ret = 0;
+ unsigned long ua_flags;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return ret;
+
+ ua_flags = user_access_save();
+
+ /* Creating new origins may allocate memory. */
+ kmsan_enter_runtime();
+ ret = kmsan_internal_chain_origin(origin);
+ kmsan_leave_runtime();
+ user_access_restore(ua_flags);
+ return ret;
+}
+EXPORT_SYMBOL(__msan_chain_origin);
+
+/* Poison a local variable when entering a function. */
+void __msan_poison_alloca(void *address, uintptr_t size, char *descr)
+{
+ depot_stack_handle_t handle;
+ unsigned long entries[4];
+ unsigned long ua_flags;
+
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ ua_flags = user_access_save();
+ entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;
+ entries[1] = (u64)descr;
+ entries[2] = (u64)__builtin_return_address(0);
+ /*
+ * With frame pointers enabled, it is possible to quickly fetch the
+ * second frame of the caller stack without calling the unwinder.
+ * Without them, simply do not bother.
+ */
+ if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER))
+ entries[3] = (u64)__builtin_return_address(1);
+ else
+ entries[3] = 0;
+
+ /* stack_depot_save() may allocate memory. */
+ kmsan_enter_runtime();
+ handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_ATOMIC);
+ kmsan_leave_runtime();
+
+ kmsan_internal_set_shadow_origin(address, size, -1, handle,
+ /*checked*/ true);
+ user_access_restore(ua_flags);
+}
+EXPORT_SYMBOL(__msan_poison_alloca);
+
+/* Unpoison a local variable. */
+void __msan_unpoison_alloca(void *address, uintptr_t size)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+
+ kmsan_enter_runtime();
+ kmsan_internal_unpoison_memory(address, size, /*checked*/ true);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(__msan_unpoison_alloca);
+
+/*
+ * Report that an uninitialized value with the given origin was used in a way
+ * that constituted undefined behavior.
+ */
+void __msan_warning(u32 origin)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ kmsan_report(origin, /*address*/ 0, /*size*/ 0,
+ /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ 0,
+ REASON_ANY);
+ kmsan_leave_runtime();
+}
+EXPORT_SYMBOL(__msan_warning);
+
+/*
+ * At the beginning of an instrumented function, obtain the pointer to
+ * `struct kmsan_context_state` holding the metadata for function parameters.
+ */
+struct kmsan_context_state *__msan_get_context_state(void)
+{
+ return &kmsan_get_context()->cstate;
+}
+EXPORT_SYMBOL(__msan_get_context_state);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Functions used by the KMSAN runtime.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#ifndef __MM_KMSAN_KMSAN_H
+#define __MM_KMSAN_KMSAN_H
+
+#include <asm/pgtable_64_types.h>
+#include <linux/irqflags.h>
+#include <linux/sched.h>
+#include <linux/stackdepot.h>
+#include <linux/stacktrace.h>
+#include <linux/nmi.h>
+#include <linux/mm.h>
+#include <linux/printk.h>
+
+#define KMSAN_ALLOCA_MAGIC_ORIGIN 0xabcd0100
+#define KMSAN_CHAIN_MAGIC_ORIGIN 0xabcd0200
+
+#define KMSAN_POISON_NOCHECK 0x0
+#define KMSAN_POISON_CHECK 0x1
+#define KMSAN_POISON_FREE 0x2
+
+#define KMSAN_ORIGIN_SIZE 4
+#define KMSAN_MAX_ORIGIN_DEPTH 7
+
+#define KMSAN_STACK_DEPTH 64
+
+#define KMSAN_META_SHADOW (false)
+#define KMSAN_META_ORIGIN (true)
+
+extern bool kmsan_enabled;
+extern int panic_on_kmsan;
+
+/*
+ * KMSAN performs a lot of consistency checks that are currently enabled by
+ * default. BUG_ON is normally discouraged in the kernel, unless used for
+ * debugging, but KMSAN itself is a debugging tool, so it makes little sense to
+ * recover if something goes wrong.
+ */
+#define KMSAN_WARN_ON(cond) \
+ ({ \
+ const bool __cond = WARN_ON(cond); \
+ if (unlikely(__cond)) { \
+ WRITE_ONCE(kmsan_enabled, false); \
+ if (panic_on_kmsan) { \
+ /* Can't call panic() here because */ \
+ /* of uaccess checks. */ \
+ BUG(); \
+ } \
+ } \
+ __cond; \
+ })
+
+/*
+ * A pair of metadata pointers to be returned by the instrumentation functions.
+ */
+struct shadow_origin_ptr {
+ void *shadow, *origin;
+};
+
+struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size,
+ bool store);
+void *kmsan_get_metadata(void *addr, bool is_origin);
+void __init kmsan_init_alloc_meta_for_range(void *start, void *end);
+
+enum kmsan_bug_reason {
+ REASON_ANY,
+ REASON_COPY_TO_USER,
+ REASON_SUBMIT_URB,
+};
+
+void kmsan_print_origin(depot_stack_handle_t origin);
+
+/**
+ * kmsan_report() - Report a use of uninitialized value.
+ * @origin: Stack ID of the uninitialized value.
+ * @address: Address at which the memory access happens.
+ * @size: Memory access size.
+ * @off_first: Offset (from @address) of the first byte to be reported.
+ * @off_last: Offset (from @address) of the last byte to be reported.
+ * @user_addr: When non-NULL, denotes the userspace address to which the kernel
+ * is leaking data.
+ * @reason: Error type from enum kmsan_bug_reason.
+ *
+ * kmsan_report() prints an error message for a consequent group of bytes
+ * sharing the same origin. If an uninitialized value is used in a comparison,
+ * this function is called once without specifying the addresses. When checking
+ * a memory range, KMSAN may call kmsan_report() multiple times with the same
+ * @address, @size, @user_addr and @reason, but different @off_first and
+ * @off_last corresponding to different @origin values.
+ */
+void kmsan_report(depot_stack_handle_t origin, void *address, int size,
+ int off_first, int off_last, const void *user_addr,
+ enum kmsan_bug_reason reason);
+
+DECLARE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx);
+
+static __always_inline struct kmsan_ctx *kmsan_get_context(void)
+{
+ return in_task() ? ¤t->kmsan_ctx : raw_cpu_ptr(&kmsan_percpu_ctx);
+}
+
+/*
+ * When a compiler hook or KMSAN runtime function is invoked, it may make a
+ * call to instrumented code and eventually call itself recursively. To avoid
+ * that, we guard the runtime entry regions with
+ * kmsan_enter_runtime()/kmsan_leave_runtime() and exit the hook if
+ * kmsan_in_runtime() is true.
+ *
+ * Non-runtime code may occasionally get executed in nested IRQs from the
+ * runtime code (e.g. when called via smp_call_function_single()). Because some
+ * KMSAN routines may take locks (e.g. for memory allocation), we conservatively
+ * bail out instead of calling them. To minimize the effect of this (potentially
+ * missing initialization events) kmsan_in_runtime() is not checked in
+ * non-blocking runtime functions.
+ */
+static __always_inline bool kmsan_in_runtime(void)
+{
+ if ((hardirq_count() >> HARDIRQ_SHIFT) > 1)
+ return true;
+ return kmsan_get_context()->kmsan_in_runtime;
+}
+
+static __always_inline void kmsan_enter_runtime(void)
+{
+ struct kmsan_ctx *ctx;
+
+ ctx = kmsan_get_context();
+ KMSAN_WARN_ON(ctx->kmsan_in_runtime++);
+}
+
+static __always_inline void kmsan_leave_runtime(void)
+{
+ struct kmsan_ctx *ctx = kmsan_get_context();
+
+ KMSAN_WARN_ON(--ctx->kmsan_in_runtime);
+}
+
+depot_stack_handle_t kmsan_save_stack(void);
+depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags,
+ unsigned int extra_bits);
+
+/*
+ * Pack and unpack the origin chain depth and UAF flag to/from the extra bits
+ * provided by the stack depot.
+ * The UAF flag is stored in the lowest bit, followed by the depth in the upper
+ * bits.
+ * set_dsh_extra_bits() is responsible for clamping the value.
+ */
+static __always_inline unsigned int kmsan_extra_bits(unsigned int depth,
+ bool uaf)
+{
+ return (depth << 1) | uaf;
+}
+
+static __always_inline bool kmsan_uaf_from_eb(unsigned int extra_bits)
+{
+ return extra_bits & 1;
+}
+
+static __always_inline unsigned int kmsan_depth_from_eb(unsigned int extra_bits)
+{
+ return extra_bits >> 1;
+}
+
+/*
+ * kmsan_internal_ functions are supposed to be very simple and not require the
+ * kmsan_in_runtime() checks.
+ */
+void kmsan_internal_memmove_metadata(void *dst, void *src, size_t n);
+void kmsan_internal_poison_memory(void *address, size_t size, gfp_t flags,
+ unsigned int poison_flags);
+void kmsan_internal_unpoison_memory(void *address, size_t size, bool checked);
+void kmsan_internal_set_shadow_origin(void *address, size_t size, int b,
+ u32 origin, bool checked);
+depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id);
+
+void kmsan_internal_task_create(struct task_struct *task);
+
+bool kmsan_metadata_is_contiguous(void *addr, size_t size);
+void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr,
+ int reason);
+
+struct page *kmsan_vmalloc_to_page_or_null(void *vaddr);
+void kmsan_setup_meta(struct page *page, struct page *shadow,
+ struct page *origin, int order);
+
+/*
+ * kmsan_internal_is_module_addr() and kmsan_internal_is_vmalloc_addr() are
+ * non-instrumented versions of is_module_address() and is_vmalloc_addr() that
+ * are safe to call from KMSAN runtime without recursion.
+ */
+static inline bool kmsan_internal_is_module_addr(void *vaddr)
+{
+ return ((u64)vaddr >= MODULES_VADDR) && ((u64)vaddr < MODULES_END);
+}
+
+static inline bool kmsan_internal_is_vmalloc_addr(void *addr)
+{
+ return ((u64)addr >= VMALLOC_START) && ((u64)addr < VMALLOC_END);
+}
+
+#endif /* __MM_KMSAN_KMSAN_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test cases for KMSAN.
+ * For each test case checks the presence (or absence) of generated reports.
+ * Relies on 'console' tracepoint to capture reports as they appear in the
+ * kernel log.
+ *
+ * Copyright (C) 2021-2022, Google LLC.
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include <kunit/test.h>
+#include "kmsan.h"
+
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/kmsan.h>
+#include <linux/mm.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/tracepoint.h>
+#include <trace/events/printk.h>
+
+static DEFINE_PER_CPU(int, per_cpu_var);
+
+/* Report as observed from console. */
+static struct {
+ spinlock_t lock;
+ bool available;
+ bool ignore; /* Stop console output collection. */
+ char header[256];
+} observed = {
+ .lock = __SPIN_LOCK_UNLOCKED(observed.lock),
+};
+
+/* Probe for console output: obtains observed lines of interest. */
+static void probe_console(void *ignore, const char *buf, size_t len)
+{
+ unsigned long flags;
+
+ if (observed.ignore)
+ return;
+ spin_lock_irqsave(&observed.lock, flags);
+
+ if (strnstr(buf, "BUG: KMSAN: ", len)) {
+ /*
+ * KMSAN report and related to the test.
+ *
+ * The provided @buf is not NUL-terminated; copy no more than
+ * @len bytes and let strscpy() add the missing NUL-terminator.
+ */
+ strscpy(observed.header, buf,
+ min(len + 1, sizeof(observed.header)));
+ WRITE_ONCE(observed.available, true);
+ observed.ignore = true;
+ }
+ spin_unlock_irqrestore(&observed.lock, flags);
+}
+
+/* Check if a report related to the test exists. */
+static bool report_available(void)
+{
+ return READ_ONCE(observed.available);
+}
+
+/* Information we expect in a report. */
+struct expect_report {
+ const char *error_type; /* Error type. */
+ /*
+ * Kernel symbol from the error header, or NULL if no report is
+ * expected.
+ */
+ const char *symbol;
+};
+
+/* Check observed report matches information in @r. */
+static bool report_matches(const struct expect_report *r)
+{
+ typeof(observed.header) expected_header;
+ unsigned long flags;
+ bool ret = false;
+ const char *end;
+ char *cur;
+
+ /* Doubled-checked locking. */
+ if (!report_available() || !r->symbol)
+ return (!report_available() && !r->symbol);
+
+ /* Generate expected report contents. */
+
+ /* Title */
+ cur = expected_header;
+ end = &expected_header[sizeof(expected_header) - 1];
+
+ cur += scnprintf(cur, end - cur, "BUG: KMSAN: %s", r->error_type);
+
+ scnprintf(cur, end - cur, " in %s", r->symbol);
+ /* The exact offset won't match, remove it; also strip module name. */
+ cur = strchr(expected_header, '+');
+ if (cur)
+ *cur = '\0';
+
+ spin_lock_irqsave(&observed.lock, flags);
+ if (!report_available())
+ goto out; /* A new report is being captured. */
+
+ /* Finally match expected output to what we actually observed. */
+ ret = strstr(observed.header, expected_header);
+out:
+ spin_unlock_irqrestore(&observed.lock, flags);
+
+ return ret;
+}
+
+/* ===== Test cases ===== */
+
+/* Prevent replacing branch with select in LLVM. */
+static noinline void check_true(char *arg)
+{
+ pr_info("%s is true\n", arg);
+}
+
+static noinline void check_false(char *arg)
+{
+ pr_info("%s is false\n", arg);
+}
+
+#define USE(x) \
+ do { \
+ if (x) \
+ check_true(#x); \
+ else \
+ check_false(#x); \
+ } while (0)
+
+#define EXPECTATION_ETYPE_FN(e, reason, fn) \
+ struct expect_report e = { \
+ .error_type = reason, \
+ .symbol = fn, \
+ }
+
+#define EXPECTATION_NO_REPORT(e) EXPECTATION_ETYPE_FN(e, NULL, NULL)
+#define EXPECTATION_UNINIT_VALUE_FN(e, fn) \
+ EXPECTATION_ETYPE_FN(e, "uninit-value", fn)
+#define EXPECTATION_UNINIT_VALUE(e) EXPECTATION_UNINIT_VALUE_FN(e, __func__)
+#define EXPECTATION_USE_AFTER_FREE(e) \
+ EXPECTATION_ETYPE_FN(e, "use-after-free", __func__)
+
+/* Test case: ensure that kmalloc() returns uninitialized memory. */
+static void test_uninit_kmalloc(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE(expect);
+ int *ptr;
+
+ kunit_info(test, "uninitialized kmalloc test (UMR report)\n");
+ ptr = kmalloc(sizeof(*ptr), GFP_KERNEL);
+ USE(*ptr);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that kmalloc'ed memory becomes initialized after memset().
+ */
+static void test_init_kmalloc(struct kunit *test)
+{
+ EXPECTATION_NO_REPORT(expect);
+ int *ptr;
+
+ kunit_info(test, "initialized kmalloc test (no reports)\n");
+ ptr = kmalloc(sizeof(*ptr), GFP_KERNEL);
+ memset(ptr, 0, sizeof(*ptr));
+ USE(*ptr);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/* Test case: ensure that kzalloc() returns initialized memory. */
+static void test_init_kzalloc(struct kunit *test)
+{
+ EXPECTATION_NO_REPORT(expect);
+ int *ptr;
+
+ kunit_info(test, "initialized kzalloc test (no reports)\n");
+ ptr = kzalloc(sizeof(*ptr), GFP_KERNEL);
+ USE(*ptr);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/* Test case: ensure that local variables are uninitialized by default. */
+static void test_uninit_stack_var(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE(expect);
+ volatile int cond;
+
+ kunit_info(test, "uninitialized stack variable (UMR report)\n");
+ USE(cond);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/* Test case: ensure that local variables with initializers are initialized. */
+static void test_init_stack_var(struct kunit *test)
+{
+ EXPECTATION_NO_REPORT(expect);
+ volatile int cond = 1;
+
+ kunit_info(test, "initialized stack variable (no reports)\n");
+ USE(cond);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+static noinline void two_param_fn_2(int arg1, int arg2)
+{
+ USE(arg1);
+ USE(arg2);
+}
+
+static noinline void one_param_fn(int arg)
+{
+ two_param_fn_2(arg, arg);
+ USE(arg);
+}
+
+static noinline void two_param_fn(int arg1, int arg2)
+{
+ int init = 0;
+
+ one_param_fn(init);
+ USE(arg1);
+ USE(arg2);
+}
+
+static void test_params(struct kunit *test)
+{
+#ifdef CONFIG_KMSAN_CHECK_PARAM_RETVAL
+ /*
+ * With eager param/retval checking enabled, KMSAN will report an error
+ * before the call to two_param_fn().
+ */
+ EXPECTATION_UNINIT_VALUE_FN(expect, "test_params");
+#else
+ EXPECTATION_UNINIT_VALUE_FN(expect, "two_param_fn");
+#endif
+ volatile int uninit, init = 1;
+
+ kunit_info(test,
+ "uninit passed through a function parameter (UMR report)\n");
+ two_param_fn(uninit, init);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+static int signed_sum3(int a, int b, int c)
+{
+ return a + b + c;
+}
+
+/*
+ * Test case: ensure that uninitialized values are tracked through function
+ * arguments.
+ */
+static void test_uninit_multiple_params(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE(expect);
+ volatile char b = 3, c;
+ volatile int a;
+
+ kunit_info(test, "uninitialized local passed to fn (UMR report)\n");
+ USE(signed_sum3(a, b, c));
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/* Helper function to make an array uninitialized. */
+static noinline void do_uninit_local_array(char *array, int start, int stop)
+{
+ volatile char uninit;
+
+ for (int i = start; i < stop; i++)
+ array[i] = uninit;
+}
+
+/*
+ * Test case: ensure kmsan_check_memory() reports an error when checking
+ * uninitialized memory.
+ */
+static void test_uninit_kmsan_check_memory(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE_FN(expect, "test_uninit_kmsan_check_memory");
+ volatile char local_array[8];
+
+ kunit_info(
+ test,
+ "kmsan_check_memory() called on uninit local (UMR report)\n");
+ do_uninit_local_array((char *)local_array, 5, 7);
+
+ kmsan_check_memory((char *)local_array, 8);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: check that a virtual memory range created with vmap() from
+ * initialized pages is still considered as initialized.
+ */
+static void test_init_kmsan_vmap_vunmap(struct kunit *test)
+{
+ EXPECTATION_NO_REPORT(expect);
+ const int npages = 2;
+ struct page **pages;
+ void *vbuf;
+
+ kunit_info(test, "pages initialized via vmap (no reports)\n");
+
+ pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+ for (int i = 0; i < npages; i++)
+ pages[i] = alloc_page(GFP_KERNEL);
+ vbuf = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
+ memset(vbuf, 0xfe, npages * PAGE_SIZE);
+ for (int i = 0; i < npages; i++)
+ kmsan_check_memory(page_address(pages[i]), PAGE_SIZE);
+
+ if (vbuf)
+ vunmap(vbuf);
+ for (int i = 0; i < npages; i++) {
+ if (pages[i])
+ __free_page(pages[i]);
+ }
+ kfree(pages);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that memset() can initialize a buffer allocated via
+ * vmalloc().
+ */
+static void test_init_vmalloc(struct kunit *test)
+{
+ EXPECTATION_NO_REPORT(expect);
+ int npages = 8;
+ char *buf;
+
+ kunit_info(test, "vmalloc buffer can be initialized (no reports)\n");
+ buf = vmalloc(PAGE_SIZE * npages);
+ buf[0] = 1;
+ memset(buf, 0xfe, PAGE_SIZE * npages);
+ USE(buf[0]);
+ for (int i = 0; i < npages; i++)
+ kmsan_check_memory(&buf[PAGE_SIZE * i], PAGE_SIZE);
+ vfree(buf);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/* Test case: ensure that use-after-free reporting works. */
+static void test_uaf(struct kunit *test)
+{
+ EXPECTATION_USE_AFTER_FREE(expect);
+ volatile int value;
+ volatile int *var;
+
+ kunit_info(test, "use-after-free in kmalloc-ed buffer (UMR report)\n");
+ var = kmalloc(80, GFP_KERNEL);
+ var[3] = 0xfeedface;
+ kfree((int *)var);
+ /* Copy the invalid value before checking it. */
+ value = var[3];
+ USE(value);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that uninitialized values are propagated through per-CPU
+ * memory.
+ */
+static void test_percpu_propagate(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE(expect);
+ volatile int uninit, check;
+
+ kunit_info(test,
+ "uninit local stored to per_cpu memory (UMR report)\n");
+
+ this_cpu_write(per_cpu_var, uninit);
+ check = this_cpu_read(per_cpu_var);
+ USE(check);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that passing uninitialized values to printk() leads to an
+ * error report.
+ */
+static void test_printk(struct kunit *test)
+{
+#ifdef CONFIG_KMSAN_CHECK_PARAM_RETVAL
+ /*
+ * With eager param/retval checking enabled, KMSAN will report an error
+ * before the call to pr_info().
+ */
+ EXPECTATION_UNINIT_VALUE_FN(expect, "test_printk");
+#else
+ EXPECTATION_UNINIT_VALUE_FN(expect, "number");
+#endif
+ volatile int uninit;
+
+ kunit_info(test, "uninit local passed to pr_info() (UMR report)\n");
+ pr_info("%px contains %d\n", &uninit, uninit);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that memcpy() correctly copies uninitialized values between
+ * aligned `src` and `dst`.
+ */
+static void test_memcpy_aligned_to_aligned(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE_FN(expect, "test_memcpy_aligned_to_aligned");
+ volatile int uninit_src;
+ volatile int dst = 0;
+
+ kunit_info(
+ test,
+ "memcpy()ing aligned uninit src to aligned dst (UMR report)\n");
+ memcpy((void *)&dst, (void *)&uninit_src, sizeof(uninit_src));
+ kmsan_check_memory((void *)&dst, sizeof(dst));
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that memcpy() correctly copies uninitialized values between
+ * aligned `src` and unaligned `dst`.
+ *
+ * Copying aligned 4-byte value to an unaligned one leads to touching two
+ * aligned 4-byte values. This test case checks that KMSAN correctly reports an
+ * error on the first of the two values.
+ */
+static void test_memcpy_aligned_to_unaligned(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE_FN(expect, "test_memcpy_aligned_to_unaligned");
+ volatile int uninit_src;
+ volatile char dst[8] = { 0 };
+
+ kunit_info(
+ test,
+ "memcpy()ing aligned uninit src to unaligned dst (UMR report)\n");
+ memcpy((void *)&dst[1], (void *)&uninit_src, sizeof(uninit_src));
+ kmsan_check_memory((void *)dst, 4);
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+/*
+ * Test case: ensure that memcpy() correctly copies uninitialized values between
+ * aligned `src` and unaligned `dst`.
+ *
+ * Copying aligned 4-byte value to an unaligned one leads to touching two
+ * aligned 4-byte values. This test case checks that KMSAN correctly reports an
+ * error on the second of the two values.
+ */
+static void test_memcpy_aligned_to_unaligned2(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE_FN(expect,
+ "test_memcpy_aligned_to_unaligned2");
+ volatile int uninit_src;
+ volatile char dst[8] = { 0 };
+
+ kunit_info(
+ test,
+ "memcpy()ing aligned uninit src to unaligned dst - part 2 (UMR report)\n");
+ memcpy((void *)&dst[1], (void *)&uninit_src, sizeof(uninit_src));
+ kmsan_check_memory((void *)&dst[4], sizeof(uninit_src));
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+static noinline void fibonacci(int *array, int size, int start) {
+ if (start < 2 || (start == size))
+ return;
+ array[start] = array[start - 1] + array[start - 2];
+ fibonacci(array, size, start + 1);
+}
+
+static void test_long_origin_chain(struct kunit *test)
+{
+ EXPECTATION_UNINIT_VALUE_FN(expect,
+ "test_long_origin_chain");
+ /* (KMSAN_MAX_ORIGIN_DEPTH * 2) recursive calls to fibonacci(). */
+ volatile int accum[KMSAN_MAX_ORIGIN_DEPTH * 2 + 2];
+ int last = ARRAY_SIZE(accum) - 1;
+
+ kunit_info(
+ test,
+ "origin chain exceeding KMSAN_MAX_ORIGIN_DEPTH (UMR report)\n");
+ /*
+ * We do not set accum[1] to 0, so the uninitializedness will be carried
+ * over to accum[2..last].
+ */
+ accum[0] = 1;
+ fibonacci((int *)accum, ARRAY_SIZE(accum), 2);
+ kmsan_check_memory((void *)&accum[last], sizeof(int));
+ KUNIT_EXPECT_TRUE(test, report_matches(&expect));
+}
+
+static struct kunit_case kmsan_test_cases[] = {
+ KUNIT_CASE(test_uninit_kmalloc),
+ KUNIT_CASE(test_init_kmalloc),
+ KUNIT_CASE(test_init_kzalloc),
+ KUNIT_CASE(test_uninit_stack_var),
+ KUNIT_CASE(test_init_stack_var),
+ KUNIT_CASE(test_params),
+ KUNIT_CASE(test_uninit_multiple_params),
+ KUNIT_CASE(test_uninit_kmsan_check_memory),
+ KUNIT_CASE(test_init_kmsan_vmap_vunmap),
+ KUNIT_CASE(test_init_vmalloc),
+ KUNIT_CASE(test_uaf),
+ KUNIT_CASE(test_percpu_propagate),
+ KUNIT_CASE(test_printk),
+ KUNIT_CASE(test_memcpy_aligned_to_aligned),
+ KUNIT_CASE(test_memcpy_aligned_to_unaligned),
+ KUNIT_CASE(test_memcpy_aligned_to_unaligned2),
+ KUNIT_CASE(test_long_origin_chain),
+ {},
+};
+
+/* ===== End test cases ===== */
+
+static int test_init(struct kunit *test)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&observed.lock, flags);
+ observed.header[0] = '\0';
+ observed.ignore = false;
+ observed.available = false;
+ spin_unlock_irqrestore(&observed.lock, flags);
+
+ return 0;
+}
+
+static void test_exit(struct kunit *test)
+{
+}
+
+static void register_tracepoints(struct tracepoint *tp, void *ignore)
+{
+ check_trace_callback_type_console(probe_console);
+ if (!strcmp(tp->name, "console"))
+ WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
+}
+
+static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
+{
+ if (!strcmp(tp->name, "console"))
+ tracepoint_probe_unregister(tp, probe_console, NULL);
+}
+
+static int kmsan_suite_init(struct kunit_suite *suite)
+{
+ /*
+ * Because we want to be able to build the test as a module, we need to
+ * iterate through all known tracepoints, since the static registration
+ * won't work here.
+ */
+ for_each_kernel_tracepoint(register_tracepoints, NULL);
+ return 0;
+}
+
+static void kmsan_suite_exit(struct kunit_suite *suite)
+{
+ for_each_kernel_tracepoint(unregister_tracepoints, NULL);
+ tracepoint_synchronize_unregister();
+}
+
+static struct kunit_suite kmsan_test_suite = {
+ .name = "kmsan",
+ .test_cases = kmsan_test_cases,
+ .init = test_init,
+ .exit = test_exit,
+ .suite_init = kmsan_suite_init,
+ .suite_exit = kmsan_suite_exit,
+};
+kunit_test_suites(&kmsan_test_suite);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Alexander Potapenko <glider@google.com>");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN error reporting routines.
+ *
+ * Copyright (C) 2019-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include <linux/console.h>
+#include <linux/moduleparam.h>
+#include <linux/stackdepot.h>
+#include <linux/stacktrace.h>
+#include <linux/uaccess.h>
+
+#include "kmsan.h"
+
+static DEFINE_RAW_SPINLOCK(kmsan_report_lock);
+#define DESCR_SIZE 128
+/* Protected by kmsan_report_lock */
+static char report_local_descr[DESCR_SIZE];
+int panic_on_kmsan __read_mostly;
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "kmsan."
+module_param_named(panic, panic_on_kmsan, int, 0);
+
+/*
+ * Skip internal KMSAN frames.
+ */
+static int get_stack_skipnr(const unsigned long stack_entries[],
+ int num_entries)
+{
+ int len, skip;
+ char buf[64];
+
+ for (skip = 0; skip < num_entries; ++skip) {
+ len = scnprintf(buf, sizeof(buf), "%ps",
+ (void *)stack_entries[skip]);
+
+ /* Never show __msan_* or kmsan_* functions. */
+ if ((strnstr(buf, "__msan_", len) == buf) ||
+ (strnstr(buf, "kmsan_", len) == buf))
+ continue;
+
+ /*
+ * No match for runtime functions -- @skip entries to skip to
+ * get to first frame of interest.
+ */
+ break;
+ }
+
+ return skip;
+}
+
+/*
+ * Currently the descriptions of locals generated by Clang look as follows:
+ * ----local_name@function_name
+ * We want to print only the name of the local, as other information in that
+ * description can be confusing.
+ * The meaningful part of the description is copied to a global buffer to avoid
+ * allocating memory.
+ */
+static char *pretty_descr(char *descr)
+{
+ int pos = 0, len = strlen(descr);
+
+ for (int i = 0; i < len; i++) {
+ if (descr[i] == '@')
+ break;
+ if (descr[i] == '-')
+ continue;
+ report_local_descr[pos] = descr[i];
+ if (pos + 1 == DESCR_SIZE)
+ break;
+ pos++;
+ }
+ report_local_descr[pos] = 0;
+ return report_local_descr;
+}
+
+void kmsan_print_origin(depot_stack_handle_t origin)
+{
+ unsigned long *entries = NULL, *chained_entries = NULL;
+ unsigned int nr_entries, chained_nr_entries, skipnr;
+ void *pc1 = NULL, *pc2 = NULL;
+ depot_stack_handle_t head;
+ unsigned long magic;
+ char *descr = NULL;
+ unsigned int depth;
+
+ if (!origin)
+ return;
+
+ while (true) {
+ nr_entries = stack_depot_fetch(origin, &entries);
+ depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin));
+ magic = nr_entries ? entries[0] : 0;
+ if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) {
+ descr = (char *)entries[1];
+ pc1 = (void *)entries[2];
+ pc2 = (void *)entries[3];
+ pr_err("Local variable %s created at:\n",
+ pretty_descr(descr));
+ if (pc1)
+ pr_err(" %pSb\n", pc1);
+ if (pc2)
+ pr_err(" %pSb\n", pc2);
+ break;
+ }
+ if ((nr_entries == 3) && (magic == KMSAN_CHAIN_MAGIC_ORIGIN)) {
+ /*
+ * Origin chains deeper than KMSAN_MAX_ORIGIN_DEPTH are
+ * not stored, so the output may be incomplete.
+ */
+ if (depth == KMSAN_MAX_ORIGIN_DEPTH)
+ pr_err("<Zero or more stacks not recorded to save memory>\n\n");
+ head = entries[1];
+ origin = entries[2];
+ pr_err("Uninit was stored to memory at:\n");
+ chained_nr_entries =
+ stack_depot_fetch(head, &chained_entries);
+ kmsan_internal_unpoison_memory(
+ chained_entries,
+ chained_nr_entries * sizeof(*chained_entries),
+ /*checked*/ false);
+ skipnr = get_stack_skipnr(chained_entries,
+ chained_nr_entries);
+ stack_trace_print(chained_entries + skipnr,
+ chained_nr_entries - skipnr, 0);
+ pr_err("\n");
+ continue;
+ }
+ pr_err("Uninit was created at:\n");
+ if (nr_entries) {
+ skipnr = get_stack_skipnr(entries, nr_entries);
+ stack_trace_print(entries + skipnr, nr_entries - skipnr,
+ 0);
+ } else {
+ pr_err("(stack is not available)\n");
+ }
+ break;
+ }
+}
+
+void kmsan_report(depot_stack_handle_t origin, void *address, int size,
+ int off_first, int off_last, const void *user_addr,
+ enum kmsan_bug_reason reason)
+{
+ unsigned long stack_entries[KMSAN_STACK_DEPTH];
+ int num_stack_entries, skipnr;
+ char *bug_type = NULL;
+ unsigned long ua_flags;
+ bool is_uaf;
+
+ if (!kmsan_enabled)
+ return;
+ if (!current->kmsan_ctx.allow_reporting)
+ return;
+ if (!origin)
+ return;
+
+ current->kmsan_ctx.allow_reporting = false;
+ ua_flags = user_access_save();
+ raw_spin_lock(&kmsan_report_lock);
+ pr_err("=====================================================\n");
+ is_uaf = kmsan_uaf_from_eb(stack_depot_get_extra_bits(origin));
+ switch (reason) {
+ case REASON_ANY:
+ bug_type = is_uaf ? "use-after-free" : "uninit-value";
+ break;
+ case REASON_COPY_TO_USER:
+ bug_type = is_uaf ? "kernel-infoleak-after-free" :
+ "kernel-infoleak";
+ break;
+ case REASON_SUBMIT_URB:
+ bug_type = is_uaf ? "kernel-usb-infoleak-after-free" :
+ "kernel-usb-infoleak";
+ break;
+ }
+
+ num_stack_entries =
+ stack_trace_save(stack_entries, KMSAN_STACK_DEPTH, 1);
+ skipnr = get_stack_skipnr(stack_entries, num_stack_entries);
+
+ pr_err("BUG: KMSAN: %s in %pSb\n", bug_type,
+ (void *)stack_entries[skipnr]);
+ stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr,
+ 0);
+ pr_err("\n");
+
+ kmsan_print_origin(origin);
+
+ if (size) {
+ pr_err("\n");
+ if (off_first == off_last)
+ pr_err("Byte %d of %d is uninitialized\n", off_first,
+ size);
+ else
+ pr_err("Bytes %d-%d of %d are uninitialized\n",
+ off_first, off_last, size);
+ }
+ if (address)
+ pr_err("Memory access of size %d starts at %px\n", size,
+ address);
+ if (user_addr && reason == REASON_COPY_TO_USER)
+ pr_err("Data copied to user address %px\n", user_addr);
+ pr_err("\n");
+ dump_stack_print_info(KERN_ERR);
+ pr_err("=====================================================\n");
+ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
+ raw_spin_unlock(&kmsan_report_lock);
+ if (panic_on_kmsan)
+ panic("kmsan.panic set ...\n");
+ user_access_restore(ua_flags);
+ current->kmsan_ctx.allow_reporting = true;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN shadow implementation.
+ *
+ * Copyright (C) 2017-2022 Google LLC
+ * Author: Alexander Potapenko <glider@google.com>
+ *
+ */
+
+#include <asm/kmsan.h>
+#include <asm/tlbflush.h>
+#include <linux/cacheflush.h>
+#include <linux/memblock.h>
+#include <linux/mm_types.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/stddef.h>
+
+#include "../internal.h"
+#include "kmsan.h"
+
+#define shadow_page_for(page) ((page)->kmsan_shadow)
+
+#define origin_page_for(page) ((page)->kmsan_origin)
+
+static void *shadow_ptr_for(struct page *page)
+{
+ return page_address(shadow_page_for(page));
+}
+
+static void *origin_ptr_for(struct page *page)
+{
+ return page_address(origin_page_for(page));
+}
+
+static bool page_has_metadata(struct page *page)
+{
+ return shadow_page_for(page) && origin_page_for(page);
+}
+
+static void set_no_shadow_origin_page(struct page *page)
+{
+ shadow_page_for(page) = NULL;
+ origin_page_for(page) = NULL;
+}
+
+/*
+ * Dummy load and store pages to be used when the real metadata is unavailable.
+ * There are separate pages for loads and stores, so that every load returns a
+ * zero, and every store doesn't affect other loads.
+ */
+static char dummy_load_page[PAGE_SIZE] __aligned(PAGE_SIZE);
+static char dummy_store_page[PAGE_SIZE] __aligned(PAGE_SIZE);
+
+static unsigned long vmalloc_meta(void *addr, bool is_origin)
+{
+ unsigned long addr64 = (unsigned long)addr, off;
+
+ KMSAN_WARN_ON(is_origin && !IS_ALIGNED(addr64, KMSAN_ORIGIN_SIZE));
+ if (kmsan_internal_is_vmalloc_addr(addr)) {
+ off = addr64 - VMALLOC_START;
+ return off + (is_origin ? KMSAN_VMALLOC_ORIGIN_START :
+ KMSAN_VMALLOC_SHADOW_START);
+ }
+ if (kmsan_internal_is_module_addr(addr)) {
+ off = addr64 - MODULES_VADDR;
+ return off + (is_origin ? KMSAN_MODULES_ORIGIN_START :
+ KMSAN_MODULES_SHADOW_START);
+ }
+ return 0;
+}
+
+static struct page *virt_to_page_or_null(void *vaddr)
+{
+ if (kmsan_virt_addr_valid(vaddr))
+ return virt_to_page(vaddr);
+ else
+ return NULL;
+}
+
+struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size,
+ bool store)
+{
+ struct shadow_origin_ptr ret;
+ void *shadow;
+
+ /*
+ * Even if we redirect this memory access to the dummy page, it will
+ * go out of bounds.
+ */
+ KMSAN_WARN_ON(size > PAGE_SIZE);
+
+ if (!kmsan_enabled)
+ goto return_dummy;
+
+ KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(address, size));
+ shadow = kmsan_get_metadata(address, KMSAN_META_SHADOW);
+ if (!shadow)
+ goto return_dummy;
+
+ ret.shadow = shadow;
+ ret.origin = kmsan_get_metadata(address, KMSAN_META_ORIGIN);
+ return ret;
+
+return_dummy:
+ if (store) {
+ /* Ignore this store. */
+ ret.shadow = dummy_store_page;
+ ret.origin = dummy_store_page;
+ } else {
+ /* This load will return zero. */
+ ret.shadow = dummy_load_page;
+ ret.origin = dummy_load_page;
+ }
+ return ret;
+}
+
+/*
+ * Obtain the shadow or origin pointer for the given address, or NULL if there's
+ * none. The caller must check the return value for being non-NULL if needed.
+ * The return value of this function should not depend on whether we're in the
+ * runtime or not.
+ */
+void *kmsan_get_metadata(void *address, bool is_origin)
+{
+ u64 addr = (u64)address, pad, off;
+ struct page *page;
+ void *ret;
+
+ if (is_origin && !IS_ALIGNED(addr, KMSAN_ORIGIN_SIZE)) {
+ pad = addr % KMSAN_ORIGIN_SIZE;
+ addr -= pad;
+ }
+ address = (void *)addr;
+ if (kmsan_internal_is_vmalloc_addr(address) ||
+ kmsan_internal_is_module_addr(address))
+ return (void *)vmalloc_meta(address, is_origin);
+
+ ret = arch_kmsan_get_meta_or_null(address, is_origin);
+ if (ret)
+ return ret;
+
+ page = virt_to_page_or_null(address);
+ if (!page)
+ return NULL;
+ if (!page_has_metadata(page))
+ return NULL;
+ off = addr % PAGE_SIZE;
+
+ return (is_origin ? origin_ptr_for(page) : shadow_ptr_for(page)) + off;
+}
+
+void kmsan_copy_page_meta(struct page *dst, struct page *src)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ if (!dst || !page_has_metadata(dst))
+ return;
+ if (!src || !page_has_metadata(src)) {
+ kmsan_internal_unpoison_memory(page_address(dst), PAGE_SIZE,
+ /*checked*/ false);
+ return;
+ }
+
+ kmsan_enter_runtime();
+ __memcpy(shadow_ptr_for(dst), shadow_ptr_for(src), PAGE_SIZE);
+ __memcpy(origin_ptr_for(dst), origin_ptr_for(src), PAGE_SIZE);
+ kmsan_leave_runtime();
+}
+
+void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags)
+{
+ bool initialized = (flags & __GFP_ZERO) || !kmsan_enabled;
+ struct page *shadow, *origin;
+ depot_stack_handle_t handle;
+ int pages = 1 << order;
+
+ if (!page)
+ return;
+
+ shadow = shadow_page_for(page);
+ origin = origin_page_for(page);
+
+ if (initialized) {
+ __memset(page_address(shadow), 0, PAGE_SIZE * pages);
+ __memset(page_address(origin), 0, PAGE_SIZE * pages);
+ return;
+ }
+
+ /* Zero pages allocated by the runtime should also be initialized. */
+ if (kmsan_in_runtime())
+ return;
+
+ __memset(page_address(shadow), -1, PAGE_SIZE * pages);
+ kmsan_enter_runtime();
+ handle = kmsan_save_stack_with_flags(flags, /*extra_bits*/ 0);
+ kmsan_leave_runtime();
+ /*
+ * Addresses are page-aligned, pages are contiguous, so it's ok
+ * to just fill the origin pages with @handle.
+ */
+ for (int i = 0; i < PAGE_SIZE * pages / sizeof(handle); i++)
+ ((depot_stack_handle_t *)page_address(origin))[i] = handle;
+}
+
+void kmsan_free_page(struct page *page, unsigned int order)
+{
+ if (!kmsan_enabled || kmsan_in_runtime())
+ return;
+ kmsan_enter_runtime();
+ kmsan_internal_poison_memory(page_address(page),
+ PAGE_SIZE << compound_order(page),
+ GFP_KERNEL,
+ KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
+ kmsan_leave_runtime();
+}
+
+void kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,
+ pgprot_t prot, struct page **pages,
+ unsigned int page_shift)
+{
+ unsigned long shadow_start, origin_start, shadow_end, origin_end;
+ struct page **s_pages, **o_pages;
+ int nr, mapped;
+
+ if (!kmsan_enabled)
+ return;
+
+ shadow_start = vmalloc_meta((void *)start, KMSAN_META_SHADOW);
+ shadow_end = vmalloc_meta((void *)end, KMSAN_META_SHADOW);
+ if (!shadow_start)
+ return;
+
+ nr = (end - start) / PAGE_SIZE;
+ s_pages = kcalloc(nr, sizeof(*s_pages), GFP_KERNEL);
+ o_pages = kcalloc(nr, sizeof(*o_pages), GFP_KERNEL);
+ if (!s_pages || !o_pages)
+ goto ret;
+ for (int i = 0; i < nr; i++) {
+ s_pages[i] = shadow_page_for(pages[i]);
+ o_pages[i] = origin_page_for(pages[i]);
+ }
+ prot = __pgprot(pgprot_val(prot) | _PAGE_NX);
+ prot = PAGE_KERNEL;
+
+ origin_start = vmalloc_meta((void *)start, KMSAN_META_ORIGIN);
+ origin_end = vmalloc_meta((void *)end, KMSAN_META_ORIGIN);
+ kmsan_enter_runtime();
+ mapped = __vmap_pages_range_noflush(shadow_start, shadow_end, prot,
+ s_pages, page_shift);
+ KMSAN_WARN_ON(mapped);
+ mapped = __vmap_pages_range_noflush(origin_start, origin_end, prot,
+ o_pages, page_shift);
+ KMSAN_WARN_ON(mapped);
+ kmsan_leave_runtime();
+ flush_tlb_kernel_range(shadow_start, shadow_end);
+ flush_tlb_kernel_range(origin_start, origin_end);
+ flush_cache_vmap(shadow_start, shadow_end);
+ flush_cache_vmap(origin_start, origin_end);
+
+ret:
+ kfree(s_pages);
+ kfree(o_pages);
+}
+
+/* Allocate metadata for pages allocated at boot time. */
+void __init kmsan_init_alloc_meta_for_range(void *start, void *end)
+{
+ struct page *shadow_p, *origin_p;
+ void *shadow, *origin;
+ struct page *page;
+ u64 size;
+
+ start = (void *)ALIGN_DOWN((u64)start, PAGE_SIZE);
+ size = ALIGN((u64)end - (u64)start, PAGE_SIZE);
+ shadow = memblock_alloc(size, PAGE_SIZE);
+ origin = memblock_alloc(size, PAGE_SIZE);
+ for (u64 addr = 0; addr < size; addr += PAGE_SIZE) {
+ page = virt_to_page_or_null((char *)start + addr);
+ shadow_p = virt_to_page_or_null((char *)shadow + addr);
+ set_no_shadow_origin_page(shadow_p);
+ shadow_page_for(page) = shadow_p;
+ origin_p = virt_to_page_or_null((char *)origin + addr);
+ set_no_shadow_origin_page(origin_p);
+ origin_page_for(page) = origin_p;
+ }
+}
+
+void kmsan_setup_meta(struct page *page, struct page *shadow,
+ struct page *origin, int order)
+{
+ for (int i = 0; i < (1 << order); i++) {
+ set_no_shadow_origin_page(&shadow[i]);
+ set_no_shadow_origin_page(&origin[i]);
+ shadow_page_for(&page[i]) = &shadow[i];
+ origin_page_for(&page[i]) = &origin[i];
+ }
+}
#include <asm/tlbflush.h>
#include "internal.h"
+#include "mm_slot.h"
#ifdef CONFIG_NUMA
#define NUMA(x) (x)
* different KSM page copy of that content
*
* Internally, the regular nodes, "dups" and "chains" are represented
- * using the same struct stable_node structure.
+ * using the same struct ksm_stable_node structure.
*
* In addition to the stable tree, KSM uses a second data structure called the
* unstable tree: this tree holds pointers to pages which have been found to
*/
/**
- * struct mm_slot - ksm information per mm that is being scanned
- * @link: link to the mm_slots hash list
- * @mm_list: link into the mm_slots list, rooted in ksm_mm_head
+ * struct ksm_mm_slot - ksm information per mm that is being scanned
+ * @slot: hash lookup from mm to mm_slot
* @rmap_list: head for this mm_slot's singly-linked list of rmap_items
- * @mm: the mm that this information is valid for
*/
-struct mm_slot {
- struct hlist_node link;
- struct list_head mm_list;
- struct rmap_item *rmap_list;
- struct mm_struct *mm;
+struct ksm_mm_slot {
+ struct mm_slot slot;
+ struct ksm_rmap_item *rmap_list;
};
/**
* There is only the one ksm_scan instance of this cursor structure.
*/
struct ksm_scan {
- struct mm_slot *mm_slot;
+ struct ksm_mm_slot *mm_slot;
unsigned long address;
- struct rmap_item **rmap_list;
+ struct ksm_rmap_item **rmap_list;
unsigned long seqnr;
};
/**
- * struct stable_node - node of the stable rbtree
+ * struct ksm_stable_node - node of the stable rbtree
* @node: rb node of this ksm page in the stable tree
* @head: (overlaying parent) &migrate_nodes indicates temporarily on that list
* @hlist_dup: linked into the stable_node->hlist with a stable_node chain
* @rmap_hlist_len: number of rmap_item entries in hlist or STABLE_NODE_CHAIN
* @nid: NUMA node id of stable tree in which linked (may not match kpfn)
*/
-struct stable_node {
+struct ksm_stable_node {
union {
struct rb_node node; /* when node of stable tree */
struct { /* when listed for migration */
};
/**
- * struct rmap_item - reverse mapping item for virtual addresses
+ * struct ksm_rmap_item - reverse mapping item for virtual addresses
* @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
* @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
* @nid: NUMA node id of unstable tree in which linked (may not match page)
* @head: pointer to stable_node heading this list in the stable tree
* @hlist: link into hlist of rmap_items hanging off that stable_node
*/
-struct rmap_item {
- struct rmap_item *rmap_list;
+struct ksm_rmap_item {
+ struct ksm_rmap_item *rmap_list;
union {
struct anon_vma *anon_vma; /* when stable */
#ifdef CONFIG_NUMA
union {
struct rb_node node; /* when node of unstable tree */
struct { /* when listed from stable tree */
- struct stable_node *head;
+ struct ksm_stable_node *head;
struct hlist_node hlist;
};
};
#define MM_SLOTS_HASH_BITS 10
static DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
-static struct mm_slot ksm_mm_head = {
- .mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list),
+static struct ksm_mm_slot ksm_mm_head = {
+ .slot.mm_node = LIST_HEAD_INIT(ksm_mm_head.slot.mm_node),
};
static struct ksm_scan ksm_scan = {
.mm_slot = &ksm_mm_head,
static DEFINE_MUTEX(ksm_thread_mutex);
static DEFINE_SPINLOCK(ksm_mmlist_lock);
-#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
+#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create(#__struct,\
sizeof(struct __struct), __alignof__(struct __struct),\
(__flags), NULL)
static int __init ksm_slab_init(void)
{
- rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
+ rmap_item_cache = KSM_KMEM_CACHE(ksm_rmap_item, 0);
if (!rmap_item_cache)
goto out;
- stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);
+ stable_node_cache = KSM_KMEM_CACHE(ksm_stable_node, 0);
if (!stable_node_cache)
goto out_free1;
- mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0);
+ mm_slot_cache = KSM_KMEM_CACHE(ksm_mm_slot, 0);
if (!mm_slot_cache)
goto out_free2;
mm_slot_cache = NULL;
}
-static __always_inline bool is_stable_node_chain(struct stable_node *chain)
+static __always_inline bool is_stable_node_chain(struct ksm_stable_node *chain)
{
return chain->rmap_hlist_len == STABLE_NODE_CHAIN;
}
-static __always_inline bool is_stable_node_dup(struct stable_node *dup)
+static __always_inline bool is_stable_node_dup(struct ksm_stable_node *dup)
{
return dup->head == STABLE_NODE_DUP_HEAD;
}
-static inline void stable_node_chain_add_dup(struct stable_node *dup,
- struct stable_node *chain)
+static inline void stable_node_chain_add_dup(struct ksm_stable_node *dup,
+ struct ksm_stable_node *chain)
{
VM_BUG_ON(is_stable_node_dup(dup));
dup->head = STABLE_NODE_DUP_HEAD;
ksm_stable_node_dups++;
}
-static inline void __stable_node_dup_del(struct stable_node *dup)
+static inline void __stable_node_dup_del(struct ksm_stable_node *dup)
{
VM_BUG_ON(!is_stable_node_dup(dup));
hlist_del(&dup->hlist_dup);
ksm_stable_node_dups--;
}
-static inline void stable_node_dup_del(struct stable_node *dup)
+static inline void stable_node_dup_del(struct ksm_stable_node *dup)
{
VM_BUG_ON(is_stable_node_chain(dup));
if (is_stable_node_dup(dup))
#endif
}
-static inline struct rmap_item *alloc_rmap_item(void)
+static inline struct ksm_rmap_item *alloc_rmap_item(void)
{
- struct rmap_item *rmap_item;
+ struct ksm_rmap_item *rmap_item;
rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
__GFP_NORETRY | __GFP_NOWARN);
return rmap_item;
}
-static inline void free_rmap_item(struct rmap_item *rmap_item)
+static inline void free_rmap_item(struct ksm_rmap_item *rmap_item)
{
ksm_rmap_items--;
+ rmap_item->mm->ksm_rmap_items--;
rmap_item->mm = NULL; /* debug safety */
kmem_cache_free(rmap_item_cache, rmap_item);
}
-static inline struct stable_node *alloc_stable_node(void)
+static inline struct ksm_stable_node *alloc_stable_node(void)
{
/*
* The allocation can take too long with GFP_KERNEL when memory is under
return kmem_cache_alloc(stable_node_cache, GFP_KERNEL | __GFP_HIGH);
}
-static inline void free_stable_node(struct stable_node *stable_node)
+static inline void free_stable_node(struct ksm_stable_node *stable_node)
{
VM_BUG_ON(stable_node->rmap_hlist_len &&
!is_stable_node_chain(stable_node));
kmem_cache_free(stable_node_cache, stable_node);
}
-static inline struct mm_slot *alloc_mm_slot(void)
-{
- if (!mm_slot_cache) /* initialization failed */
- return NULL;
- return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
-}
-
-static inline void free_mm_slot(struct mm_slot *mm_slot)
-{
- kmem_cache_free(mm_slot_cache, mm_slot);
-}
-
-static struct mm_slot *get_mm_slot(struct mm_struct *mm)
-{
- struct mm_slot *slot;
-
- hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)
- if (slot->mm == mm)
- return slot;
-
- return NULL;
-}
-
-static void insert_to_mm_slots_hash(struct mm_struct *mm,
- struct mm_slot *mm_slot)
-{
- mm_slot->mm = mm;
- hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm);
-}
-
/*
* ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's
* page tables after it has passed through ksm_exit() - which, if necessary,
cond_resched();
page = follow_page(vma, addr,
FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE);
- if (IS_ERR_OR_NULL(page) || is_zone_device_page(page))
+ if (IS_ERR_OR_NULL(page))
break;
if (PageKsm(page))
ret = handle_mm_fault(vma, addr,
return vma;
}
-static void break_cow(struct rmap_item *rmap_item)
+static void break_cow(struct ksm_rmap_item *rmap_item)
{
struct mm_struct *mm = rmap_item->mm;
unsigned long addr = rmap_item->address;
mmap_read_unlock(mm);
}
-static struct page *get_mergeable_page(struct rmap_item *rmap_item)
+static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item)
{
struct mm_struct *mm = rmap_item->mm;
unsigned long addr = rmap_item->address;
goto out;
page = follow_page(vma, addr, FOLL_GET);
- if (IS_ERR_OR_NULL(page) || is_zone_device_page(page))
+ if (IS_ERR_OR_NULL(page))
goto out;
+ if (is_zone_device_page(page))
+ goto out_putpage;
if (PageAnon(page)) {
flush_anon_page(vma, page, addr);
flush_dcache_page(page);
} else {
+out_putpage:
put_page(page);
out:
page = NULL;
return ksm_merge_across_nodes ? 0 : NUMA(pfn_to_nid(kpfn));
}
-static struct stable_node *alloc_stable_node_chain(struct stable_node *dup,
+static struct ksm_stable_node *alloc_stable_node_chain(struct ksm_stable_node *dup,
struct rb_root *root)
{
- struct stable_node *chain = alloc_stable_node();
+ struct ksm_stable_node *chain = alloc_stable_node();
VM_BUG_ON(is_stable_node_chain(dup));
if (likely(chain)) {
INIT_HLIST_HEAD(&chain->hlist);
return chain;
}
-static inline void free_stable_node_chain(struct stable_node *chain,
+static inline void free_stable_node_chain(struct ksm_stable_node *chain,
struct rb_root *root)
{
rb_erase(&chain->node, root);
ksm_stable_node_chains--;
}
-static void remove_node_from_stable_tree(struct stable_node *stable_node)
+static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node)
{
- struct rmap_item *rmap_item;
+ struct ksm_rmap_item *rmap_item;
/* check it's not STABLE_NODE_CHAIN or negative */
BUG_ON(stable_node->rmap_hlist_len < 0);
* a page to put something that might look like our key in page->mapping.
* is on its way to being freed; but it is an anomaly to bear in mind.
*/
-static struct page *get_ksm_page(struct stable_node *stable_node,
+static struct page *get_ksm_page(struct ksm_stable_node *stable_node,
enum get_ksm_page_flags flags)
{
struct page *page;
* Removing rmap_item from stable or unstable tree.
* This function will clean the information from the stable/unstable tree.
*/
-static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
+static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item)
{
if (rmap_item->address & STABLE_FLAG) {
- struct stable_node *stable_node;
+ struct ksm_stable_node *stable_node;
struct page *page;
stable_node = rmap_item->head;
cond_resched(); /* we're called from many long loops */
}
-static void remove_trailing_rmap_items(struct rmap_item **rmap_list)
+static void remove_trailing_rmap_items(struct ksm_rmap_item **rmap_list)
{
while (*rmap_list) {
- struct rmap_item *rmap_item = *rmap_list;
+ struct ksm_rmap_item *rmap_item = *rmap_list;
*rmap_list = rmap_item->rmap_list;
remove_rmap_item_from_tree(rmap_item);
free_rmap_item(rmap_item);
return err;
}
-static inline struct stable_node *folio_stable_node(struct folio *folio)
+static inline struct ksm_stable_node *folio_stable_node(struct folio *folio)
{
return folio_test_ksm(folio) ? folio_raw_mapping(folio) : NULL;
}
-static inline struct stable_node *page_stable_node(struct page *page)
+static inline struct ksm_stable_node *page_stable_node(struct page *page)
{
return folio_stable_node(page_folio(page));
}
static inline void set_page_stable_node(struct page *page,
- struct stable_node *stable_node)
+ struct ksm_stable_node *stable_node)
{
VM_BUG_ON_PAGE(PageAnon(page) && PageAnonExclusive(page), page);
page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM);
/*
* Only called through the sysfs control interface:
*/
-static int remove_stable_node(struct stable_node *stable_node)
+static int remove_stable_node(struct ksm_stable_node *stable_node)
{
struct page *page;
int err;
return err;
}
-static int remove_stable_node_chain(struct stable_node *stable_node,
+static int remove_stable_node_chain(struct ksm_stable_node *stable_node,
struct rb_root *root)
{
- struct stable_node *dup;
+ struct ksm_stable_node *dup;
struct hlist_node *hlist_safe;
if (!is_stable_node_chain(stable_node)) {
static int remove_all_stable_nodes(void)
{
- struct stable_node *stable_node, *next;
+ struct ksm_stable_node *stable_node, *next;
int nid;
int err = 0;
for (nid = 0; nid < ksm_nr_node_ids; nid++) {
while (root_stable_tree[nid].rb_node) {
stable_node = rb_entry(root_stable_tree[nid].rb_node,
- struct stable_node, node);
+ struct ksm_stable_node, node);
if (remove_stable_node_chain(stable_node,
root_stable_tree + nid)) {
err = -EBUSY;
static int unmerge_and_remove_all_rmap_items(void)
{
- struct mm_slot *mm_slot;
+ struct ksm_mm_slot *mm_slot;
+ struct mm_slot *slot;
struct mm_struct *mm;
struct vm_area_struct *vma;
int err = 0;
spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
- struct mm_slot, mm_list);
+ slot = list_entry(ksm_mm_head.slot.mm_node.next,
+ struct mm_slot, mm_node);
+ ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot);
spin_unlock(&ksm_mmlist_lock);
- for (mm_slot = ksm_scan.mm_slot;
- mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
- mm = mm_slot->mm;
+ for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head;
+ mm_slot = ksm_scan.mm_slot) {
+ VMA_ITERATOR(vmi, mm_slot->slot.mm, 0);
+
+ mm = mm_slot->slot.mm;
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (ksm_test_exit(mm))
break;
if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
mmap_read_unlock(mm);
spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next,
- struct mm_slot, mm_list);
+ slot = list_entry(mm_slot->slot.mm_node.next,
+ struct mm_slot, mm_node);
+ ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot);
if (ksm_test_exit(mm)) {
- hash_del(&mm_slot->link);
- list_del(&mm_slot->mm_list);
+ hash_del(&mm_slot->slot.hash);
+ list_del(&mm_slot->slot.mm_node);
spin_unlock(&ksm_mmlist_lock);
- free_mm_slot(mm_slot);
+ mm_slot_free(mm_slot_cache, mm_slot);
clear_bit(MMF_VM_MERGEABLE, &mm->flags);
mmdrop(mm);
} else
goto out_unlock;
}
+ /* See page_try_share_anon_rmap(): clear PTE first. */
if (anon_exclusive && page_try_share_anon_rmap(page)) {
set_pte_at(mm, pvmw.address, pvmw.pte, entry);
goto out_unlock;
struct page *kpage, pte_t orig_pte)
{
struct mm_struct *mm = vma->vm_mm;
+ struct folio *folio;
pmd_t *pmd;
+ pmd_t pmde;
pte_t *ptep;
pte_t newpte;
spinlock_t *ptl;
pmd = mm_find_pmd(mm, addr);
if (!pmd)
goto out;
+ /*
+ * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
+ * without holding anon_vma lock for write. So when looking for a
+ * genuine pmde (in which to find pte), test present and !THP together.
+ */
+ pmde = *pmd;
+ barrier();
+ if (!pmd_present(pmde) || pmd_trans_huge(pmde))
+ goto out;
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
addr + PAGE_SIZE);
ptep_clear_flush(vma, addr, ptep);
set_pte_at_notify(mm, addr, ptep, newpte);
+ folio = page_folio(page);
page_remove_rmap(page, vma, false);
- if (!page_mapped(page))
- try_to_free_swap(page);
- put_page(page);
+ if (!folio_mapped(folio))
+ folio_free_swap(folio);
+ folio_put(folio);
pte_unmap_unlock(ptep, ptl);
err = 0;
*
* This function returns 0 if the pages were merged, -EFAULT otherwise.
*/
-static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
+static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item,
struct page *page, struct page *kpage)
{
struct mm_struct *mm = rmap_item->mm;
* Note that this function upgrades page to ksm page: if one of the pages
* is already a ksm page, try_to_merge_with_ksm_page should be used.
*/
-static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
+static struct page *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item,
struct page *page,
- struct rmap_item *tree_rmap_item,
+ struct ksm_rmap_item *tree_rmap_item,
struct page *tree_page)
{
int err;
}
static __always_inline
-bool __is_page_sharing_candidate(struct stable_node *stable_node, int offset)
+bool __is_page_sharing_candidate(struct ksm_stable_node *stable_node, int offset)
{
VM_BUG_ON(stable_node->rmap_hlist_len < 0);
/*
}
static __always_inline
-bool is_page_sharing_candidate(struct stable_node *stable_node)
+bool is_page_sharing_candidate(struct ksm_stable_node *stable_node)
{
return __is_page_sharing_candidate(stable_node, 0);
}
-static struct page *stable_node_dup(struct stable_node **_stable_node_dup,
- struct stable_node **_stable_node,
+static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup,
+ struct ksm_stable_node **_stable_node,
struct rb_root *root,
bool prune_stale_stable_nodes)
{
- struct stable_node *dup, *found = NULL, *stable_node = *_stable_node;
+ struct ksm_stable_node *dup, *found = NULL, *stable_node = *_stable_node;
struct hlist_node *hlist_safe;
struct page *_tree_page, *tree_page = NULL;
int nr = 0;
return tree_page;
}
-static struct stable_node *stable_node_dup_any(struct stable_node *stable_node,
+static struct ksm_stable_node *stable_node_dup_any(struct ksm_stable_node *stable_node,
struct rb_root *root)
{
if (!is_stable_node_chain(stable_node))
* function and will be overwritten in all cases, the caller doesn't
* need to initialize it.
*/
-static struct page *__stable_node_chain(struct stable_node **_stable_node_dup,
- struct stable_node **_stable_node,
+static struct page *__stable_node_chain(struct ksm_stable_node **_stable_node_dup,
+ struct ksm_stable_node **_stable_node,
struct rb_root *root,
bool prune_stale_stable_nodes)
{
- struct stable_node *stable_node = *_stable_node;
+ struct ksm_stable_node *stable_node = *_stable_node;
if (!is_stable_node_chain(stable_node)) {
if (is_page_sharing_candidate(stable_node)) {
*_stable_node_dup = stable_node;
prune_stale_stable_nodes);
}
-static __always_inline struct page *chain_prune(struct stable_node **s_n_d,
- struct stable_node **s_n,
+static __always_inline struct page *chain_prune(struct ksm_stable_node **s_n_d,
+ struct ksm_stable_node **s_n,
struct rb_root *root)
{
return __stable_node_chain(s_n_d, s_n, root, true);
}
-static __always_inline struct page *chain(struct stable_node **s_n_d,
- struct stable_node *s_n,
+static __always_inline struct page *chain(struct ksm_stable_node **s_n_d,
+ struct ksm_stable_node *s_n,
struct rb_root *root)
{
- struct stable_node *old_stable_node = s_n;
+ struct ksm_stable_node *old_stable_node = s_n;
struct page *tree_page;
tree_page = __stable_node_chain(s_n_d, &s_n, root, false);
struct rb_root *root;
struct rb_node **new;
struct rb_node *parent;
- struct stable_node *stable_node, *stable_node_dup, *stable_node_any;
- struct stable_node *page_node;
+ struct ksm_stable_node *stable_node, *stable_node_dup, *stable_node_any;
+ struct ksm_stable_node *page_node;
page_node = page_stable_node(page);
if (page_node && page_node->head != &migrate_nodes) {
int ret;
cond_resched();
- stable_node = rb_entry(*new, struct stable_node, node);
+ stable_node = rb_entry(*new, struct ksm_stable_node, node);
stable_node_any = NULL;
tree_page = chain_prune(&stable_node_dup, &stable_node, root);
/*
* This function returns the stable tree node just allocated on success,
* NULL otherwise.
*/
-static struct stable_node *stable_tree_insert(struct page *kpage)
+static struct ksm_stable_node *stable_tree_insert(struct page *kpage)
{
int nid;
unsigned long kpfn;
struct rb_root *root;
struct rb_node **new;
struct rb_node *parent;
- struct stable_node *stable_node, *stable_node_dup, *stable_node_any;
+ struct ksm_stable_node *stable_node, *stable_node_dup, *stable_node_any;
bool need_chain = false;
kpfn = page_to_pfn(kpage);
int ret;
cond_resched();
- stable_node = rb_entry(*new, struct stable_node, node);
+ stable_node = rb_entry(*new, struct ksm_stable_node, node);
stable_node_any = NULL;
tree_page = chain(&stable_node_dup, stable_node, root);
if (!stable_node_dup) {
rb_insert_color(&stable_node_dup->node, root);
} else {
if (!is_stable_node_chain(stable_node)) {
- struct stable_node *orig = stable_node;
+ struct ksm_stable_node *orig = stable_node;
/* chain is missing so create it */
stable_node = alloc_stable_node_chain(orig, root);
if (!stable_node) {
* the same walking algorithm in an rbtree.
*/
static
-struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
+struct ksm_rmap_item *unstable_tree_search_insert(struct ksm_rmap_item *rmap_item,
struct page *page,
struct page **tree_pagep)
{
new = &root->rb_node;
while (*new) {
- struct rmap_item *tree_rmap_item;
+ struct ksm_rmap_item *tree_rmap_item;
struct page *tree_page;
int ret;
cond_resched();
- tree_rmap_item = rb_entry(*new, struct rmap_item, node);
+ tree_rmap_item = rb_entry(*new, struct ksm_rmap_item, node);
tree_page = get_mergeable_page(tree_rmap_item);
if (!tree_page)
return NULL;
* rmap_items hanging off a given node of the stable tree, all sharing
* the same ksm page.
*/
-static void stable_tree_append(struct rmap_item *rmap_item,
- struct stable_node *stable_node,
+static void stable_tree_append(struct ksm_rmap_item *rmap_item,
+ struct ksm_stable_node *stable_node,
bool max_page_sharing_bypass)
{
/*
* @page: the page that we are searching identical page to.
* @rmap_item: the reverse mapping into the virtual address of this page
*/
-static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
+static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_item)
{
struct mm_struct *mm = rmap_item->mm;
- struct rmap_item *tree_rmap_item;
+ struct ksm_rmap_item *tree_rmap_item;
struct page *tree_page = NULL;
- struct stable_node *stable_node;
+ struct ksm_stable_node *stable_node;
struct page *kpage;
unsigned int checksum;
int err;
}
}
-static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,
- struct rmap_item **rmap_list,
+static struct ksm_rmap_item *get_next_rmap_item(struct ksm_mm_slot *mm_slot,
+ struct ksm_rmap_item **rmap_list,
unsigned long addr)
{
- struct rmap_item *rmap_item;
+ struct ksm_rmap_item *rmap_item;
while (*rmap_list) {
rmap_item = *rmap_list;
rmap_item = alloc_rmap_item();
if (rmap_item) {
/* It has already been zeroed */
- rmap_item->mm = mm_slot->mm;
+ rmap_item->mm = mm_slot->slot.mm;
+ rmap_item->mm->ksm_rmap_items++;
rmap_item->address = addr;
rmap_item->rmap_list = *rmap_list;
*rmap_list = rmap_item;
return rmap_item;
}
-static struct rmap_item *scan_get_next_rmap_item(struct page **page)
+static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page)
{
struct mm_struct *mm;
+ struct ksm_mm_slot *mm_slot;
struct mm_slot *slot;
struct vm_area_struct *vma;
- struct rmap_item *rmap_item;
+ struct ksm_rmap_item *rmap_item;
+ struct vma_iterator vmi;
int nid;
- if (list_empty(&ksm_mm_head.mm_list))
+ if (list_empty(&ksm_mm_head.slot.mm_node))
return NULL;
- slot = ksm_scan.mm_slot;
- if (slot == &ksm_mm_head) {
+ mm_slot = ksm_scan.mm_slot;
+ if (mm_slot == &ksm_mm_head) {
/*
* A number of pages can hang around indefinitely on per-cpu
* pagevecs, raised page count preventing write_protect_page
* so prune them once before each full scan.
*/
if (!ksm_merge_across_nodes) {
- struct stable_node *stable_node, *next;
+ struct ksm_stable_node *stable_node, *next;
struct page *page;
list_for_each_entry_safe(stable_node, next,
root_unstable_tree[nid] = RB_ROOT;
spin_lock(&ksm_mmlist_lock);
- slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
- ksm_scan.mm_slot = slot;
+ slot = list_entry(mm_slot->slot.mm_node.next,
+ struct mm_slot, mm_node);
+ mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot);
+ ksm_scan.mm_slot = mm_slot;
spin_unlock(&ksm_mmlist_lock);
/*
* Although we tested list_empty() above, a racing __ksm_exit
* of the last mm on the list may have removed it since then.
*/
- if (slot == &ksm_mm_head)
+ if (mm_slot == &ksm_mm_head)
return NULL;
next_mm:
ksm_scan.address = 0;
- ksm_scan.rmap_list = &slot->rmap_list;
+ ksm_scan.rmap_list = &mm_slot->rmap_list;
}
+ slot = &mm_slot->slot;
mm = slot->mm;
+ vma_iter_init(&vmi, mm, ksm_scan.address);
+
mmap_read_lock(mm);
if (ksm_test_exit(mm))
- vma = NULL;
- else
- vma = find_vma(mm, ksm_scan.address);
+ goto no_vmas;
- for (; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (!(vma->vm_flags & VM_MERGEABLE))
continue;
if (ksm_scan.address < vma->vm_start)
if (ksm_test_exit(mm))
break;
*page = follow_page(vma, ksm_scan.address, FOLL_GET);
- if (IS_ERR_OR_NULL(*page) || is_zone_device_page(*page)) {
+ if (IS_ERR_OR_NULL(*page)) {
ksm_scan.address += PAGE_SIZE;
cond_resched();
continue;
}
+ if (is_zone_device_page(*page))
+ goto next_page;
if (PageAnon(*page)) {
flush_anon_page(vma, *page, ksm_scan.address);
flush_dcache_page(*page);
- rmap_item = get_next_rmap_item(slot,
+ rmap_item = get_next_rmap_item(mm_slot,
ksm_scan.rmap_list, ksm_scan.address);
if (rmap_item) {
ksm_scan.rmap_list =
mmap_read_unlock(mm);
return rmap_item;
}
+next_page:
put_page(*page);
ksm_scan.address += PAGE_SIZE;
cond_resched();
}
if (ksm_test_exit(mm)) {
+no_vmas:
ksm_scan.address = 0;
- ksm_scan.rmap_list = &slot->rmap_list;
+ ksm_scan.rmap_list = &mm_slot->rmap_list;
}
/*
* Nuke all the rmap_items that are above this current rmap:
remove_trailing_rmap_items(ksm_scan.rmap_list);
spin_lock(&ksm_mmlist_lock);
- ksm_scan.mm_slot = list_entry(slot->mm_list.next,
- struct mm_slot, mm_list);
+ slot = list_entry(mm_slot->slot.mm_node.next,
+ struct mm_slot, mm_node);
+ ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot);
if (ksm_scan.address == 0) {
/*
* We've completed a full scan of all vmas, holding mmap_lock
* or when all VM_MERGEABLE areas have been unmapped (and
* mmap_lock then protects against race with MADV_MERGEABLE).
*/
- hash_del(&slot->link);
- list_del(&slot->mm_list);
+ hash_del(&mm_slot->slot.hash);
+ list_del(&mm_slot->slot.mm_node);
spin_unlock(&ksm_mmlist_lock);
- free_mm_slot(slot);
+ mm_slot_free(mm_slot_cache, mm_slot);
clear_bit(MMF_VM_MERGEABLE, &mm->flags);
mmap_read_unlock(mm);
mmdrop(mm);
}
/* Repeat until we've completed scanning the whole list */
- slot = ksm_scan.mm_slot;
- if (slot != &ksm_mm_head)
+ mm_slot = ksm_scan.mm_slot;
+ if (mm_slot != &ksm_mm_head)
goto next_mm;
ksm_scan.seqnr++;
*/
static void ksm_do_scan(unsigned int scan_npages)
{
- struct rmap_item *rmap_item;
+ struct ksm_rmap_item *rmap_item;
struct page *page;
while (scan_npages-- && likely(!freezing(current))) {
static int ksmd_should_run(void)
{
- return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list);
+ return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.slot.mm_node);
}
static int ksm_scan_thread(void *nothing)
int __ksm_enter(struct mm_struct *mm)
{
- struct mm_slot *mm_slot;
+ struct ksm_mm_slot *mm_slot;
+ struct mm_slot *slot;
int needs_wakeup;
- mm_slot = alloc_mm_slot();
+ mm_slot = mm_slot_alloc(mm_slot_cache);
if (!mm_slot)
return -ENOMEM;
+ slot = &mm_slot->slot;
+
/* Check ksm_run too? Would need tighter locking */
- needs_wakeup = list_empty(&ksm_mm_head.mm_list);
+ needs_wakeup = list_empty(&ksm_mm_head.slot.mm_node);
spin_lock(&ksm_mmlist_lock);
- insert_to_mm_slots_hash(mm, mm_slot);
+ mm_slot_insert(mm_slots_hash, mm, slot);
/*
* When KSM_RUN_MERGE (or KSM_RUN_STOP),
* insert just behind the scanning cursor, to let the area settle
* missed: then we might as well insert at the end of the list.
*/
if (ksm_run & KSM_RUN_UNMERGE)
- list_add_tail(&mm_slot->mm_list, &ksm_mm_head.mm_list);
+ list_add_tail(&slot->mm_node, &ksm_mm_head.slot.mm_node);
else
- list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list);
+ list_add_tail(&slot->mm_node, &ksm_scan.mm_slot->slot.mm_node);
spin_unlock(&ksm_mmlist_lock);
set_bit(MMF_VM_MERGEABLE, &mm->flags);
void __ksm_exit(struct mm_struct *mm)
{
- struct mm_slot *mm_slot;
+ struct ksm_mm_slot *mm_slot;
+ struct mm_slot *slot;
int easy_to_free = 0;
/*
*/
spin_lock(&ksm_mmlist_lock);
- mm_slot = get_mm_slot(mm);
+ slot = mm_slot_lookup(mm_slots_hash, mm);
+ mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot);
if (mm_slot && ksm_scan.mm_slot != mm_slot) {
if (!mm_slot->rmap_list) {
- hash_del(&mm_slot->link);
- list_del(&mm_slot->mm_list);
+ hash_del(&slot->hash);
+ list_del(&slot->mm_node);
easy_to_free = 1;
} else {
- list_move(&mm_slot->mm_list,
- &ksm_scan.mm_slot->mm_list);
+ list_move(&slot->mm_node,
+ &ksm_scan.mm_slot->slot.mm_node);
}
}
spin_unlock(&ksm_mmlist_lock);
if (easy_to_free) {
- free_mm_slot(mm_slot);
+ mm_slot_free(mm_slot_cache, mm_slot);
clear_bit(MMF_VM_MERGEABLE, &mm->flags);
mmdrop(mm);
} else if (mm_slot) {
void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
{
- struct stable_node *stable_node;
- struct rmap_item *rmap_item;
+ struct ksm_stable_node *stable_node;
+ struct ksm_rmap_item *rmap_item;
int search_new_forks = 0;
VM_BUG_ON_FOLIO(!folio_test_ksm(folio), folio);
#ifdef CONFIG_MIGRATION
void folio_migrate_ksm(struct folio *newfolio, struct folio *folio)
{
- struct stable_node *stable_node;
+ struct ksm_stable_node *stable_node;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_FOLIO(!folio_test_locked(newfolio), newfolio);
}
}
-static bool stable_node_dup_remove_range(struct stable_node *stable_node,
+static bool stable_node_dup_remove_range(struct ksm_stable_node *stable_node,
unsigned long start_pfn,
unsigned long end_pfn)
{
return false;
}
-static bool stable_node_chain_remove_range(struct stable_node *stable_node,
+static bool stable_node_chain_remove_range(struct ksm_stable_node *stable_node,
unsigned long start_pfn,
unsigned long end_pfn,
struct rb_root *root)
{
- struct stable_node *dup;
+ struct ksm_stable_node *dup;
struct hlist_node *hlist_safe;
if (!is_stable_node_chain(stable_node)) {
static void ksm_check_stable_tree(unsigned long start_pfn,
unsigned long end_pfn)
{
- struct stable_node *stable_node, *next;
+ struct ksm_stable_node *stable_node, *next;
struct rb_node *node;
int nid;
for (nid = 0; nid < ksm_nr_node_ids; nid++) {
node = rb_first(root_stable_tree + nid);
while (node) {
- stable_node = rb_entry(node, struct stable_node, node);
+ stable_node = rb_entry(node, struct ksm_stable_node, node);
if (stable_node_chain_remove_range(stable_node,
start_pfn, end_pfn,
root_stable_tree +
case MADV_FREE:
case MADV_POPULATE_READ:
case MADV_POPULATE_WRITE:
+ case MADV_COLLAPSE:
return 0;
default:
/* be safe, default to 1. list exceptions explicitly */
struct vm_area_struct *vma = walk->vma;
spinlock_t *ptl;
pte_t *orig_pte, *pte, ptent;
+ struct folio *folio;
struct page *page;
int nr_swap = 0;
unsigned long next;
page = vm_normal_page(vma, addr, ptent);
if (!page || is_zone_device_page(page))
continue;
+ folio = page_folio(page);
/*
- * If pmd isn't transhuge but the page is THP and
+ * If pmd isn't transhuge but the folio is large and
* is owned by only this process, split it and
* deactivate all pages.
*/
- if (PageTransCompound(page)) {
- if (page_mapcount(page) != 1)
+ if (folio_test_large(folio)) {
+ if (folio_mapcount(folio) != 1)
goto out;
- get_page(page);
- if (!trylock_page(page)) {
- put_page(page);
+ folio_get(folio);
+ if (!folio_trylock(folio)) {
+ folio_put(folio);
goto out;
}
pte_unmap_unlock(orig_pte, ptl);
- if (split_huge_page(page)) {
- unlock_page(page);
- put_page(page);
+ if (split_folio(folio)) {
+ folio_unlock(folio);
+ folio_put(folio);
orig_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
goto out;
}
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
pte--;
addr -= PAGE_SIZE;
continue;
}
- VM_BUG_ON_PAGE(PageTransCompound(page), page);
-
- if (PageSwapCache(page) || PageDirty(page)) {
- if (!trylock_page(page))
+ if (folio_test_swapcache(folio) || folio_test_dirty(folio)) {
+ if (!folio_trylock(folio))
continue;
/*
- * If page is shared with others, we couldn't clear
- * PG_dirty of the page.
+ * If folio is shared with others, we mustn't clear
+ * the folio's dirty flag.
*/
- if (page_mapcount(page) != 1) {
- unlock_page(page);
+ if (folio_mapcount(folio) != 1) {
+ folio_unlock(folio);
continue;
}
- if (PageSwapCache(page) && !try_to_free_swap(page)) {
- unlock_page(page);
+ if (folio_test_swapcache(folio) &&
+ !folio_free_swap(folio)) {
+ folio_unlock(folio);
continue;
}
- ClearPageDirty(page);
- unlock_page(page);
+ folio_clear_dirty(folio);
+ folio_unlock(folio);
}
if (pte_young(ptent) || pte_dirty(ptent)) {
set_pte_at(mm, addr, pte, ptent);
tlb_remove_tlb_entry(tlb, pte, addr);
}
- mark_page_lazyfree(page);
+ mark_page_lazyfree(&folio->page);
}
out:
if (nr_swap) {
if (error)
goto out;
break;
+ case MADV_COLLAPSE:
+ return madvise_collapse(vma, prev, start, end);
}
anon_name = anon_vma_name(vma);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
case MADV_HUGEPAGE:
case MADV_NOHUGEPAGE:
+ case MADV_COLLAPSE:
#endif
case MADV_DONTDUMP:
case MADV_DODUMP:
}
}
-static bool
-process_madvise_behavior_valid(int behavior)
+static bool process_madvise_behavior_valid(int behavior)
{
switch (behavior) {
case MADV_COLD:
case MADV_PAGEOUT:
case MADV_WILLNEED:
+ case MADV_COLLAPSE:
return true;
default:
return false;
if (start >= end)
break;
if (prev)
- vma = prev->vm_next;
+ vma = find_vma(mm, prev->vm_end);
else /* madvise_remove dropped mmap_lock */
vma = find_vma(mm, start);
}
* MADV_NOHUGEPAGE - mark the given range as not worth being backed by
* transparent huge pages so the existing pages will not be
* coalesced into THP and new pages will not be allocated as THP.
+ * MADV_COLLAPSE - synchronously coalesce pages into new THP.
* MADV_DONTDUMP - the application wants to prevent pages in the given range
* from being included in its core dump.
* MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
* presume that there are no holes in the memory map inside
* a pageblock
*/
- start = round_down(start, pageblock_nr_pages);
+ start = pageblock_start_pfn(start);
/*
* If we had a previous bank, and there is a space
* presume that there are no holes in the memory map inside
* a pageblock
*/
- prev_end = ALIGN(end, pageblock_nr_pages);
+ prev_end = pageblock_align(end);
}
#ifdef CONFIG_SPARSEMEM
if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) {
- prev_end = ALIGN(end, pageblock_nr_pages);
+ prev_end = pageblock_align(end);
free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
}
#endif
/* Kernel memory accounting disabled? */
static bool cgroup_memory_nokmem __ro_after_init;
-/* Whether the swap controller is active */
-#ifdef CONFIG_MEMCG_SWAP
-static bool cgroup_memory_noswap __ro_after_init;
-#else
-#define cgroup_memory_noswap 1
-#endif
-
#ifdef CONFIG_CGROUP_WRITEBACK
static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
#endif
/* Whether legacy memory+swap accounting is active */
static bool do_memsw_account(void)
{
- return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
+ return !cgroup_subsys_on_dfl(memory_cgrp_subsys);
}
#define THRESHOLDS_EVENTS_TARGET 128
queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME);
}
+/* Subset of vm_event_item to report for memcg event stats */
+static const unsigned int memcg_vm_event_stat[] = {
+ PGPGIN,
+ PGPGOUT,
+ PGSCAN_KSWAPD,
+ PGSCAN_DIRECT,
+ PGSTEAL_KSWAPD,
+ PGSTEAL_DIRECT,
+ PGFAULT,
+ PGMAJFAULT,
+ PGREFILL,
+ PGACTIVATE,
+ PGDEACTIVATE,
+ PGLAZYFREE,
+ PGLAZYFREED,
+#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
+ ZSWPIN,
+ ZSWPOUT,
+#endif
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ THP_FAULT_ALLOC,
+ THP_COLLAPSE_ALLOC,
+#endif
+};
+
+#define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat)
+static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly;
+
+static void init_memcg_events(void)
+{
+ int i;
+
+ for (i = 0; i < NR_MEMCG_EVENTS; ++i)
+ mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1;
+}
+
+static inline int memcg_events_index(enum vm_event_item idx)
+{
+ return mem_cgroup_events_index[idx] - 1;
+}
+
+struct memcg_vmstats_percpu {
+ /* Local (CPU and cgroup) page state & events */
+ long state[MEMCG_NR_STAT];
+ unsigned long events[NR_MEMCG_EVENTS];
+
+ /* Delta calculation for lockless upward propagation */
+ long state_prev[MEMCG_NR_STAT];
+ unsigned long events_prev[NR_MEMCG_EVENTS];
+
+ /* Cgroup1: threshold notifications & softlimit tree updates */
+ unsigned long nr_page_events;
+ unsigned long targets[MEM_CGROUP_NTARGETS];
+};
+
+struct memcg_vmstats {
+ /* Aggregated (CPU and subtree) page state & events */
+ long state[MEMCG_NR_STAT];
+ unsigned long events[NR_MEMCG_EVENTS];
+
+ /* Pending child counts during tree propagation */
+ long state_pending[MEMCG_NR_STAT];
+ unsigned long events_pending[NR_MEMCG_EVENTS];
+};
+
+unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
+{
+ long x = READ_ONCE(memcg->vmstats->state[idx]);
+#ifdef CONFIG_SMP
+ if (x < 0)
+ x = 0;
+#endif
+ return x;
+}
+
/**
* __mod_memcg_state - update cgroup memory statistics
* @memcg: the memory cgroup
void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
unsigned long count)
{
- if (mem_cgroup_disabled())
+ int index = memcg_events_index(idx);
+
+ if (mem_cgroup_disabled() || index < 0)
return;
memcg_stats_lock();
- __this_cpu_add(memcg->vmstats_percpu->events[idx], count);
+ __this_cpu_add(memcg->vmstats_percpu->events[index], count);
memcg_rstat_updated(memcg, count);
memcg_stats_unlock();
}
static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
{
- return READ_ONCE(memcg->vmstats.events[event]);
+ int index = memcg_events_index(event);
+
+ if (index < 0)
+ return 0;
+ return READ_ONCE(memcg->vmstats->events[index]);
}
static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
{
long x = 0;
int cpu;
+ int index = memcg_events_index(event);
+
+ if (index < 0)
+ return 0;
for_each_possible_cpu(cpu)
- x += per_cpu(memcg->vmstats_percpu->events[event], cpu);
+ x += per_cpu(memcg->vmstats_percpu->events[index], cpu);
return x;
}
} while ((memcg = parent_mem_cgroup(memcg)));
/*
- * When cgruop1 non-hierarchy mode is used,
+ * When cgroup1 non-hierarchy mode is used,
* parent_mem_cgroup() does not walk all the way up to the
* cgroup root (root_mem_cgroup). So we have to handle
* dead_memcg from cgroup root separately.
return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
}
-/* Subset of vm_event_item to report for memcg event stats */
-static const unsigned int memcg_vm_event_stat[] = {
- PGSCAN_KSWAPD,
- PGSCAN_DIRECT,
- PGSTEAL_KSWAPD,
- PGSTEAL_DIRECT,
- PGFAULT,
- PGMAJFAULT,
- PGREFILL,
- PGACTIVATE,
- PGDEACTIVATE,
- PGLAZYFREE,
- PGLAZYFREED,
-#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
- ZSWPIN,
- ZSWPOUT,
-#endif
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
- THP_FAULT_ALLOC,
- THP_COLLAPSE_ALLOC,
-#endif
-};
-
static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize)
{
struct seq_buf s;
memcg_events(memcg, PGSTEAL_KSWAPD) +
memcg_events(memcg, PGSTEAL_DIRECT));
- for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++)
+ for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {
+ if (memcg_vm_event_stat[i] == PGPGIN ||
+ memcg_vm_event_stat[i] == PGPGOUT)
+ continue;
+
seq_buf_printf(&s, "%s %lu\n",
vm_event_name(memcg_vm_event_stat[i]),
memcg_events(memcg, memcg_vm_event_stat[i]));
+ }
/* The above should easily fit into one page */
WARN_ON_ONCE(seq_buf_has_overflowed(&s));
{
unsigned long max = READ_ONCE(memcg->memory.max);
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
- if (mem_cgroup_swappiness(memcg))
- max += min(READ_ONCE(memcg->swap.max),
- (unsigned long)total_swap_pages);
- } else { /* v1 */
+ if (do_memsw_account()) {
if (mem_cgroup_swappiness(memcg)) {
/* Calculate swap excess capacity from memsw limit */
unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
max += min(swap, (unsigned long)total_swap_pages);
}
+ } else {
+ if (mem_cgroup_swappiness(memcg))
+ max += min(READ_ONCE(memcg->swap.max),
+ (unsigned long)total_swap_pages);
}
return max;
}
* - LRU isolation
* - lock_page_memcg()
* - exclusive reference
+ * - mem_cgroup_trylock_pages()
*/
folio->memcg_data = (unsigned long)memcg;
}
css_get_many(&memcg->css, nr - 1);
}
-#ifdef CONFIG_MEMCG_SWAP
+#ifdef CONFIG_SWAP
/**
* mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
* @entry: swap entry to be moved
NR_FILE_MAPPED,
NR_FILE_DIRTY,
NR_WRITEBACK,
+ WORKINGSET_REFAULT_ANON,
+ WORKINGSET_REFAULT_FILE,
MEMCG_SWAP,
};
"mapped_file",
"dirty",
"writeback",
+ "workingset_refault_anon",
+ "workingset_refault_file",
"swap",
};
if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
continue;
nr = memcg_page_state_local(memcg, memcg1_stats[i]);
- seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
+ seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
+ nr * memcg_page_state_unit(memcg1_stats[i]));
}
for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
continue;
nr = memcg_page_state(memcg, memcg1_stats[i]);
seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
- (u64)nr * PAGE_SIZE);
+ (u64)nr * memcg_page_state_unit(memcg1_stats[i]));
}
for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
for_each_node(node)
free_mem_cgroup_per_node_info(memcg, node);
+ kfree(memcg->vmstats);
free_percpu(memcg->vmstats_percpu);
kfree(memcg);
}
static void mem_cgroup_free(struct mem_cgroup *memcg)
{
+ lru_gen_exit_memcg(memcg);
memcg_wb_domain_exit(memcg);
__mem_cgroup_free(memcg);
}
goto fail;
}
+ memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL);
+ if (!memcg->vmstats)
+ goto fail;
+
memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
GFP_KERNEL_ACCOUNT);
if (!memcg->vmstats_percpu)
memcg->deferred_split_queue.split_queue_len = 0;
#endif
idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
+ lru_gen_init_memcg(memcg);
return memcg;
fail:
mem_cgroup_id_remove(memcg);
page_counter_init(&memcg->kmem, &parent->kmem);
page_counter_init(&memcg->tcpmem, &parent->tcpmem);
} else {
+ init_memcg_events();
page_counter_init(&memcg->memory, NULL);
page_counter_init(&memcg->swap, NULL);
page_counter_init(&memcg->kmem, NULL);
* below us. We're in a per-cpu loop here and this is
* a global counter, so the first cycle will get them.
*/
- delta = memcg->vmstats.state_pending[i];
+ delta = memcg->vmstats->state_pending[i];
if (delta)
- memcg->vmstats.state_pending[i] = 0;
+ memcg->vmstats->state_pending[i] = 0;
/* Add CPU changes on this level since the last flush */
v = READ_ONCE(statc->state[i]);
continue;
/* Aggregate counts on this level and propagate upwards */
- memcg->vmstats.state[i] += delta;
+ memcg->vmstats->state[i] += delta;
if (parent)
- parent->vmstats.state_pending[i] += delta;
+ parent->vmstats->state_pending[i] += delta;
}
- for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
- delta = memcg->vmstats.events_pending[i];
+ for (i = 0; i < NR_MEMCG_EVENTS; i++) {
+ delta = memcg->vmstats->events_pending[i];
if (delta)
- memcg->vmstats.events_pending[i] = 0;
+ memcg->vmstats->events_pending[i] = 0;
v = READ_ONCE(statc->events[i]);
if (v != statc->events_prev[i]) {
if (!delta)
continue;
- memcg->vmstats.events[i] += delta;
+ memcg->vmstats->events[i] += delta;
if (parent)
- parent->vmstats.events_pending[i] += delta;
+ parent->vmstats->events_pending[i] += delta;
}
for_each_node_state(nid, N_MEMORY) {
return NULL;
/*
- * Because lookup_swap_cache() updates some statistics counter,
+ * Because swap_cache_get_folio() updates some statistics counter,
* we call find_get_page() with swapper_space directly.
*/
page = find_get_page(swap_address_space(ent), swp_offset(ent));
unsigned long precharge;
mmap_read_lock(mm);
- walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
+ walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL);
mmap_read_unlock(mm);
precharge = mc.precharge;
* When we have consumed all precharges and failed in doing
* additional charge, the page walk just aborts.
*/
- walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
- NULL);
-
+ walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL);
mmap_read_unlock(mc.mm);
atomic_dec(&mc.from->moving_account);
}
}
#endif
+#ifdef CONFIG_LRU_GEN
+static void mem_cgroup_attach(struct cgroup_taskset *tset)
+{
+ struct task_struct *task;
+ struct cgroup_subsys_state *css;
+
+ /* find the first leader if there is any */
+ cgroup_taskset_for_each_leader(task, css, tset)
+ break;
+
+ if (!task)
+ return;
+
+ task_lock(task);
+ if (task->mm && READ_ONCE(task->mm->owner) == task)
+ lru_gen_migrate_mm(task->mm);
+ task_unlock(task);
+}
+#else
+static void mem_cgroup_attach(struct cgroup_taskset *tset)
+{
+}
+#endif /* CONFIG_LRU_GEN */
+
static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
{
if (value == PAGE_COUNTER_MAX)
.css_reset = mem_cgroup_css_reset,
.css_rstat_flush = mem_cgroup_css_rstat_flush,
.can_attach = mem_cgroup_can_attach,
+ .attach = mem_cgroup_attach,
.cancel_attach = mem_cgroup_cancel_attach,
.post_attach = mem_cgroup_move_task,
.dfl_cftypes = memory_files,
}
/**
- * mem_cgroup_swapin_charge_page - charge a newly allocated page for swapin
- * @page: page to charge
+ * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin.
+ * @folio: folio to charge.
* @mm: mm context of the victim
* @gfp: reclaim mode
- * @entry: swap entry for which the page is allocated
+ * @entry: swap entry for which the folio is allocated
*
- * This function charges a page allocated for swapin. Please call this before
- * adding the page to the swapcache.
+ * This function charges a folio allocated for swapin. Please call this before
+ * adding the folio to the swapcache.
*
* Returns 0 on success. Otherwise, an error code is returned.
*/
-int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
+int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm,
gfp_t gfp, swp_entry_t entry)
{
- struct folio *folio = page_folio(page);
struct mem_cgroup *memcg;
unsigned short id;
int ret;
}
subsys_initcall(mem_cgroup_init);
-#ifdef CONFIG_MEMCG_SWAP
+#ifdef CONFIG_SWAP
static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
{
while (!refcount_inc_not_zero(&memcg->id.ref)) {
if (mem_cgroup_disabled())
return;
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
+ if (!do_memsw_account())
return;
memcg = folio_memcg(folio);
if (!mem_cgroup_is_root(memcg))
page_counter_uncharge(&memcg->memory, nr_entries);
- if (!cgroup_memory_noswap && memcg != swap_memcg) {
+ if (memcg != swap_memcg) {
if (!mem_cgroup_is_root(swap_memcg))
page_counter_charge(&swap_memcg->memsw, nr_entries);
page_counter_uncharge(&memcg->memsw, nr_entries);
struct mem_cgroup *memcg;
unsigned short oldid;
- if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
+ if (do_memsw_account())
return 0;
memcg = folio_memcg(folio);
memcg = mem_cgroup_id_get_online(memcg);
- if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
+ if (!mem_cgroup_is_root(memcg) &&
!page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
memcg_memory_event(memcg, MEMCG_SWAP_MAX);
memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
struct mem_cgroup *memcg;
unsigned short id;
+ if (mem_cgroup_disabled())
+ return;
+
id = swap_cgroup_record(entry, 0, nr_pages);
rcu_read_lock();
memcg = mem_cgroup_from_id(id);
if (memcg) {
- if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
- page_counter_uncharge(&memcg->swap, nr_pages);
- else
+ if (!mem_cgroup_is_root(memcg)) {
+ if (do_memsw_account())
page_counter_uncharge(&memcg->memsw, nr_pages);
+ else
+ page_counter_uncharge(&memcg->swap, nr_pages);
}
mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
mem_cgroup_id_put_many(memcg, nr_pages);
{
long nr_swap_pages = get_nr_swap_pages();
- if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
+ if (mem_cgroup_disabled() || do_memsw_account())
return nr_swap_pages;
for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
nr_swap_pages = min_t(long, nr_swap_pages,
return nr_swap_pages;
}
-bool mem_cgroup_swap_full(struct page *page)
+bool mem_cgroup_swap_full(struct folio *folio)
{
struct mem_cgroup *memcg;
- VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (vm_swap_full())
return true;
- if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
+ if (do_memsw_account())
return false;
- memcg = page_memcg(page);
+ memcg = folio_memcg(folio);
if (!memcg)
return false;
static int __init setup_swap_account(char *s)
{
- if (!strcmp(s, "1"))
- cgroup_memory_noswap = false;
- else if (!strcmp(s, "0"))
- cgroup_memory_noswap = true;
+ pr_warn_once("The swapaccount= commandline option is deprecated. "
+ "Please report your usecase to linux-mm@kvack.org if you "
+ "depend on this functionality.\n");
return 1;
}
__setup("swapaccount=", setup_swap_account);
};
#endif /* CONFIG_MEMCG_KMEM && CONFIG_ZSWAP */
-/*
- * If mem_cgroup_swap_init() is implemented as a subsys_initcall()
- * instead of a core_initcall(), this could mean cgroup_memory_noswap still
- * remains set to false even when memcg is disabled via "cgroup_disable=memory"
- * boot parameter. This may result in premature OOPS inside
- * mem_cgroup_get_nr_swap_pages() function in corner cases.
- */
static int __init mem_cgroup_swap_init(void)
{
- /* No memory control -> no swap control */
if (mem_cgroup_disabled())
- cgroup_memory_noswap = true;
-
- if (cgroup_memory_noswap)
return 0;
WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
#endif
return 0;
}
-core_initcall(mem_cgroup_swap_init);
+subsys_initcall(mem_cgroup_swap_init);
-#endif /* CONFIG_MEMCG_SWAP */
+#endif /* CONFIG_SWAP */
* to SIG_IGN, but hopefully no one will do that?
*/
ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr,
- addr_lsb, t); /* synchronous? */
+ addr_lsb, t);
if (ret < 0)
pr_info("Error sending signal to %s:%d: %d\n",
t->comm, t->pid, ret);
{
struct to_kill *tk, *next;
- list_for_each_entry_safe (tk, next, to_kill, nd) {
+ list_for_each_entry_safe(tk, next, to_kill, nd) {
if (forcekill) {
/*
* In case something went wrong with munmapping
pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n",
pfn, tk->tsk->comm, tk->tsk->pid);
}
+ list_del(&tk->nd);
put_task_struct(tk->tsk);
kfree(tk);
}
anon_vma_interval_tree_foreach(vmac, &av->rb_root,
pgoff, pgoff) {
vma = vmac->vma;
+ if (vma->vm_mm != t->mm)
+ continue;
if (!page_mapped_in_vma(page, vma))
continue;
- if (vma->vm_mm == t->mm)
- add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma,
- to_kill);
+ add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma, to_kill);
}
}
read_unlock(&tasklist_lock);
- page_unlock_anon_vma_read(av);
+ anon_vma_unlock_read(av);
}
/*
swp_entry_t swp = pte_to_swp_entry(pte);
if (is_hwpoison_entry(swp))
- pfn = hwpoison_entry_to_pfn(swp);
+ pfn = swp_offset_pfn(swp);
}
if (!pfn || pfn != poisoned_pfn)
return ret;
/*
- * This check prevents from calling get_hwpoison_unless_zero()
- * for any unsupported type of page in order to reduce the risk of
- * unexpected races caused by taking a page refcount.
+ * This check prevents from calling get_page_unless_zero() for any
+ * unsupported type of page in order to reduce the risk of unexpected
+ * races caused by taking a page refcount.
*/
if (!HWPoisonHandlable(head, flags))
return -EBUSY;
struct address_space *mapping;
LIST_HEAD(tokill);
bool unmap_success;
- int kill = 1, forcekill;
+ int forcekill;
bool mlocked = PageMlocked(hpage);
/*
* Here we are interested only in user-mapped pages, so skip any
* other types of pages.
*/
- if (PageReserved(p) || PageSlab(p))
+ if (PageReserved(p) || PageSlab(p) || PageTable(p))
return true;
if (!(PageLRU(hpage) || PageHuge(p)))
return true;
if (page_mkclean(hpage)) {
SetPageDirty(hpage);
} else {
- kill = 0;
ttu |= TTU_IGNORE_HWPOISON;
pr_info("%#lx: corrupted page was clean: dropped without side effects\n",
pfn);
* First collect all the processes that have the page
* mapped in dirty form. This has to be done before try_to_unmap,
* because ttu takes the rmap data structures down.
- *
- * Error handling: We ignore errors here because
- * there's nothing that can be done.
*/
- if (kill)
- collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);
+ collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);
if (PageHuge(hpage) && !PageAnon(hpage)) {
/*
* use a more force-full uncatchable kill to prevent
* any accesses to the poisoned memory.
*/
- forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL);
+ forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL) ||
+ !unmap_success;
kill_procs(&tokill, forcekill, !unmap_success, pfn, flags);
return unmap_success;
return page_action(ps, p, pfn);
}
-static int try_to_split_thp_page(struct page *page, const char *msg)
+static int try_to_split_thp_page(struct page *page)
{
+ int ret;
+
lock_page(page);
- if (unlikely(split_huge_page(page))) {
- unsigned long pfn = page_to_pfn(page);
+ ret = split_huge_page(page);
+ unlock_page(page);
- unlock_page(page);
- pr_info("%s: %#lx: thp split failed\n", msg, pfn);
+ if (unlikely(ret))
put_page(page);
- return -EBUSY;
- }
- unlock_page(page);
- return 0;
+ return ret;
}
static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,
if (hwpoison_filter(p)) {
hugetlb_clear_page_hwpoison(head);
- res = -EOPNOTSUPP;
- goto out;
+ unlock_page(head);
+ if (res == 1)
+ put_page(head);
+ return -EOPNOTSUPP;
}
/*
/*
* We need/can do nothing about count=0 pages.
* 1) it's a free page, and therefore in safe hand:
- * prep_new_page() will be the gate keeper.
+ * check_new_page() will be the gate keeper.
* 2) it's part of a non-compound high order page.
* Implies some kernel user: cannot stop them from
* R/W the page; let's pray that the page has been
* page is a valid handlable page.
*/
SetPageHasHWPoisoned(hpage);
- if (try_to_split_thp_page(p, "Memory Failure") < 0) {
+ if (try_to_split_thp_page(p) < 0) {
action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED);
res = -EBUSY;
goto unlock_mutex;
page_flags = p->flags;
if (hwpoison_filter(p)) {
- TestClearPageHWPoison(p);
+ ClearPageHWPoison(p);
unlock_page(p);
put_page(p);
res = -EOPNOTSUPP;
goto unlock_mutex;
}
- if (PageSlab(page) || PageTable(page))
+ if (PageSlab(page) || PageTable(page) || PageReserved(page))
goto unlock_mutex;
ret = get_hwpoison_page(p, MF_UNPOISON);
count = free_raw_hwp_pages(page, false);
if (count == 0) {
ret = -EBUSY;
+ put_page(page);
goto unlock_mutex;
}
}
freeit = !!TestClearPageHWPoison(p);
put_page(page);
- if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) {
+ if (freeit) {
put_page(page);
ret = 0;
}
static bool isolate_page(struct page *page, struct list_head *pagelist)
{
bool isolated = false;
- bool lru = PageLRU(page);
if (PageHuge(page)) {
isolated = !isolate_hugetlb(page, pagelist);
} else {
+ bool lru = !__PageMovable(page);
+
if (lru)
isolated = !isolate_lru_page(page);
else
- isolated = !isolate_movable_page(page, ISOLATE_UNEVICTABLE);
+ isolated = !isolate_movable_page(page,
+ ISOLATE_UNEVICTABLE);
- if (isolated)
+ if (isolated) {
list_add(&page->lru, pagelist);
+ if (lru)
+ inc_node_page_state(page, NR_ISOLATED_ANON +
+ page_is_file_lru(page));
+ }
}
- if (isolated && lru)
- inc_node_page_state(page, NR_ISOLATED_ANON +
- page_is_file_lru(page));
-
/*
* If we succeed to isolate the page, we grabbed another refcount on
* the page, so we can safely drop the one we got from get_any_pages().
}
/*
- * __soft_offline_page handles hugetlb-pages and non-hugetlb pages.
+ * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages.
* If the page is a non-dirty unmapped page-cache page, it simply invalidates.
* If the page is mapped, it migrates the contents over.
*/
-static int __soft_offline_page(struct page *page)
+static int soft_offline_in_use_page(struct page *page)
{
long ret = 0;
unsigned long pfn = page_to_pfn(page);
.gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
};
+ if (!huge && PageTransHuge(hpage)) {
+ if (try_to_split_thp_page(page)) {
+ pr_info("soft offline: %#lx: thp split failed\n", pfn);
+ return -EBUSY;
+ }
+ hpage = page;
+ }
+
lock_page(page);
if (!PageHuge(page))
wait_on_page_writeback(page);
return ret;
}
-static int soft_offline_in_use_page(struct page *page)
-{
- struct page *hpage = compound_head(page);
-
- if (!PageHuge(page) && PageTransHuge(hpage))
- if (try_to_split_thp_page(page, "soft offline") < 0)
- return -EBUSY;
- return __soft_offline_page(page);
-}
-
-static int soft_offline_free_page(struct page *page)
-{
- int rc = 0;
-
- if (!page_handle_poison(page, true, false))
- rc = -EBUSY;
-
- return rc;
-}
-
static void put_ref_page(struct page *page)
{
if (page)
if (hwpoison_filter(page)) {
if (ret > 0)
put_page(page);
- else
- put_ref_page(ref_page);
mutex_unlock(&mf_mutex);
return -EOPNOTSUPP;
if (ret > 0) {
ret = soft_offline_in_use_page(page);
} else if (ret == 0) {
- if (soft_offline_free_page(page) && try_again) {
+ if (!page_handle_poison(page, true, false) && try_again) {
try_again = false;
flags &= ~MF_COUNT_INCREASED;
goto retry;
void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
{
- int i;
+ int i, total = 0;
/*
* A further optimization is to have per section refcounted
for (i = 0; i < nr_pages; i++) {
if (PageHWPoison(&memmap[i])) {
- num_poisoned_pages_dec();
+ total++;
ClearPageHWPoison(&memmap[i]);
}
}
+ if (total)
+ num_poisoned_pages_sub(total);
}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/slab.h>
+#include <linux/lockdep.h>
+#include <linux/sysfs.h>
+#include <linux/kobject.h>
+#include <linux/memory.h>
+#include <linux/memory-tiers.h>
+
+#include "internal.h"
+
+struct memory_tier {
+ /* hierarchy of memory tiers */
+ struct list_head list;
+ /* list of all memory types part of this tier */
+ struct list_head memory_types;
+ /*
+ * start value of abstract distance. memory tier maps
+ * an abstract distance range,
+ * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
+ */
+ int adistance_start;
+ struct device dev;
+ /* All the nodes that are part of all the lower memory tiers. */
+ nodemask_t lower_tier_mask;
+};
+
+struct demotion_nodes {
+ nodemask_t preferred;
+};
+
+struct node_memory_type_map {
+ struct memory_dev_type *memtype;
+ int map_count;
+};
+
+static DEFINE_MUTEX(memory_tier_lock);
+static LIST_HEAD(memory_tiers);
+static struct node_memory_type_map node_memory_types[MAX_NUMNODES];
+static struct memory_dev_type *default_dram_type;
+
+static struct bus_type memory_tier_subsys = {
+ .name = "memory_tiering",
+ .dev_name = "memory_tier",
+};
+
+#ifdef CONFIG_MIGRATION
+static int top_tier_adistance;
+/*
+ * node_demotion[] examples:
+ *
+ * Example 1:
+ *
+ * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes.
+ *
+ * node distances:
+ * node 0 1 2 3
+ * 0 10 20 30 40
+ * 1 20 10 40 30
+ * 2 30 40 10 40
+ * 3 40 30 40 10
+ *
+ * memory_tiers0 = 0-1
+ * memory_tiers1 = 2-3
+ *
+ * node_demotion[0].preferred = 2
+ * node_demotion[1].preferred = 3
+ * node_demotion[2].preferred = <empty>
+ * node_demotion[3].preferred = <empty>
+ *
+ * Example 2:
+ *
+ * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node.
+ *
+ * node distances:
+ * node 0 1 2
+ * 0 10 20 30
+ * 1 20 10 30
+ * 2 30 30 10
+ *
+ * memory_tiers0 = 0-2
+ *
+ * node_demotion[0].preferred = <empty>
+ * node_demotion[1].preferred = <empty>
+ * node_demotion[2].preferred = <empty>
+ *
+ * Example 3:
+ *
+ * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node.
+ *
+ * node distances:
+ * node 0 1 2
+ * 0 10 20 30
+ * 1 20 10 40
+ * 2 30 40 10
+ *
+ * memory_tiers0 = 1
+ * memory_tiers1 = 0
+ * memory_tiers2 = 2
+ *
+ * node_demotion[0].preferred = 2
+ * node_demotion[1].preferred = 0
+ * node_demotion[2].preferred = <empty>
+ *
+ */
+static struct demotion_nodes *node_demotion __read_mostly;
+#endif /* CONFIG_MIGRATION */
+
+static inline struct memory_tier *to_memory_tier(struct device *device)
+{
+ return container_of(device, struct memory_tier, dev);
+}
+
+static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier)
+{
+ nodemask_t nodes = NODE_MASK_NONE;
+ struct memory_dev_type *memtype;
+
+ list_for_each_entry(memtype, &memtier->memory_types, tier_sibiling)
+ nodes_or(nodes, nodes, memtype->nodes);
+
+ return nodes;
+}
+
+static void memory_tier_device_release(struct device *dev)
+{
+ struct memory_tier *tier = to_memory_tier(dev);
+ /*
+ * synchronize_rcu in clear_node_memory_tier makes sure
+ * we don't have rcu access to this memory tier.
+ */
+ kfree(tier);
+}
+
+static ssize_t nodes_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ int ret;
+ nodemask_t nmask;
+
+ mutex_lock(&memory_tier_lock);
+ nmask = get_memtier_nodemask(to_memory_tier(dev));
+ ret = sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&nmask));
+ mutex_unlock(&memory_tier_lock);
+ return ret;
+}
+static DEVICE_ATTR_RO(nodes);
+
+static struct attribute *memtier_dev_attrs[] = {
+ &dev_attr_nodes.attr,
+ NULL
+};
+
+static const struct attribute_group memtier_dev_group = {
+ .attrs = memtier_dev_attrs,
+};
+
+static const struct attribute_group *memtier_dev_groups[] = {
+ &memtier_dev_group,
+ NULL
+};
+
+static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype)
+{
+ int ret;
+ bool found_slot = false;
+ struct memory_tier *memtier, *new_memtier;
+ int adistance = memtype->adistance;
+ unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE;
+
+ lockdep_assert_held_once(&memory_tier_lock);
+
+ adistance = round_down(adistance, memtier_adistance_chunk_size);
+ /*
+ * If the memtype is already part of a memory tier,
+ * just return that.
+ */
+ if (!list_empty(&memtype->tier_sibiling)) {
+ list_for_each_entry(memtier, &memory_tiers, list) {
+ if (adistance == memtier->adistance_start)
+ return memtier;
+ }
+ WARN_ON(1);
+ return ERR_PTR(-EINVAL);
+ }
+
+ list_for_each_entry(memtier, &memory_tiers, list) {
+ if (adistance == memtier->adistance_start) {
+ goto link_memtype;
+ } else if (adistance < memtier->adistance_start) {
+ found_slot = true;
+ break;
+ }
+ }
+
+ new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL);
+ if (!new_memtier)
+ return ERR_PTR(-ENOMEM);
+
+ new_memtier->adistance_start = adistance;
+ INIT_LIST_HEAD(&new_memtier->list);
+ INIT_LIST_HEAD(&new_memtier->memory_types);
+ if (found_slot)
+ list_add_tail(&new_memtier->list, &memtier->list);
+ else
+ list_add_tail(&new_memtier->list, &memory_tiers);
+
+ new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS;
+ new_memtier->dev.bus = &memory_tier_subsys;
+ new_memtier->dev.release = memory_tier_device_release;
+ new_memtier->dev.groups = memtier_dev_groups;
+
+ ret = device_register(&new_memtier->dev);
+ if (ret) {
+ list_del(&memtier->list);
+ put_device(&memtier->dev);
+ return ERR_PTR(ret);
+ }
+ memtier = new_memtier;
+
+link_memtype:
+ list_add(&memtype->tier_sibiling, &memtier->memory_types);
+ return memtier;
+}
+
+static struct memory_tier *__node_get_memory_tier(int node)
+{
+ pg_data_t *pgdat;
+
+ pgdat = NODE_DATA(node);
+ if (!pgdat)
+ return NULL;
+ /*
+ * Since we hold memory_tier_lock, we can avoid
+ * RCU read locks when accessing the details. No
+ * parallel updates are possible here.
+ */
+ return rcu_dereference_check(pgdat->memtier,
+ lockdep_is_held(&memory_tier_lock));
+}
+
+#ifdef CONFIG_MIGRATION
+bool node_is_toptier(int node)
+{
+ bool toptier;
+ pg_data_t *pgdat;
+ struct memory_tier *memtier;
+
+ pgdat = NODE_DATA(node);
+ if (!pgdat)
+ return false;
+
+ rcu_read_lock();
+ memtier = rcu_dereference(pgdat->memtier);
+ if (!memtier) {
+ toptier = true;
+ goto out;
+ }
+ if (memtier->adistance_start <= top_tier_adistance)
+ toptier = true;
+ else
+ toptier = false;
+out:
+ rcu_read_unlock();
+ return toptier;
+}
+
+void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets)
+{
+ struct memory_tier *memtier;
+
+ /*
+ * pg_data_t.memtier updates includes a synchronize_rcu()
+ * which ensures that we either find NULL or a valid memtier
+ * in NODE_DATA. protect the access via rcu_read_lock();
+ */
+ rcu_read_lock();
+ memtier = rcu_dereference(pgdat->memtier);
+ if (memtier)
+ *targets = memtier->lower_tier_mask;
+ else
+ *targets = NODE_MASK_NONE;
+ rcu_read_unlock();
+}
+
+/**
+ * next_demotion_node() - Get the next node in the demotion path
+ * @node: The starting node to lookup the next node
+ *
+ * Return: node id for next memory node in the demotion path hierarchy
+ * from @node; NUMA_NO_NODE if @node is terminal. This does not keep
+ * @node online or guarantee that it *continues* to be the next demotion
+ * target.
+ */
+int next_demotion_node(int node)
+{
+ struct demotion_nodes *nd;
+ int target;
+
+ if (!node_demotion)
+ return NUMA_NO_NODE;
+
+ nd = &node_demotion[node];
+
+ /*
+ * node_demotion[] is updated without excluding this
+ * function from running.
+ *
+ * Make sure to use RCU over entire code blocks if
+ * node_demotion[] reads need to be consistent.
+ */
+ rcu_read_lock();
+ /*
+ * If there are multiple target nodes, just select one
+ * target node randomly.
+ *
+ * In addition, we can also use round-robin to select
+ * target node, but we should introduce another variable
+ * for node_demotion[] to record last selected target node,
+ * that may cause cache ping-pong due to the changing of
+ * last target node. Or introducing per-cpu data to avoid
+ * caching issue, which seems more complicated. So selecting
+ * target node randomly seems better until now.
+ */
+ target = node_random(&nd->preferred);
+ rcu_read_unlock();
+
+ return target;
+}
+
+static void disable_all_demotion_targets(void)
+{
+ struct memory_tier *memtier;
+ int node;
+
+ for_each_node_state(node, N_MEMORY) {
+ node_demotion[node].preferred = NODE_MASK_NONE;
+ /*
+ * We are holding memory_tier_lock, it is safe
+ * to access pgda->memtier.
+ */
+ memtier = __node_get_memory_tier(node);
+ if (memtier)
+ memtier->lower_tier_mask = NODE_MASK_NONE;
+ }
+ /*
+ * Ensure that the "disable" is visible across the system.
+ * Readers will see either a combination of before+disable
+ * state or disable+after. They will never see before and
+ * after state together.
+ */
+ synchronize_rcu();
+}
+
+/*
+ * Find an automatic demotion target for all memory
+ * nodes. Failing here is OK. It might just indicate
+ * being at the end of a chain.
+ */
+static void establish_demotion_targets(void)
+{
+ struct memory_tier *memtier;
+ struct demotion_nodes *nd;
+ int target = NUMA_NO_NODE, node;
+ int distance, best_distance;
+ nodemask_t tier_nodes, lower_tier;
+
+ lockdep_assert_held_once(&memory_tier_lock);
+
+ if (!node_demotion || !IS_ENABLED(CONFIG_MIGRATION))
+ return;
+
+ disable_all_demotion_targets();
+
+ for_each_node_state(node, N_MEMORY) {
+ best_distance = -1;
+ nd = &node_demotion[node];
+
+ memtier = __node_get_memory_tier(node);
+ if (!memtier || list_is_last(&memtier->list, &memory_tiers))
+ continue;
+ /*
+ * Get the lower memtier to find the demotion node list.
+ */
+ memtier = list_next_entry(memtier, list);
+ tier_nodes = get_memtier_nodemask(memtier);
+ /*
+ * find_next_best_node, use 'used' nodemask as a skip list.
+ * Add all memory nodes except the selected memory tier
+ * nodelist to skip list so that we find the best node from the
+ * memtier nodelist.
+ */
+ nodes_andnot(tier_nodes, node_states[N_MEMORY], tier_nodes);
+
+ /*
+ * Find all the nodes in the memory tier node list of same best distance.
+ * add them to the preferred mask. We randomly select between nodes
+ * in the preferred mask when allocating pages during demotion.
+ */
+ do {
+ target = find_next_best_node(node, &tier_nodes);
+ if (target == NUMA_NO_NODE)
+ break;
+
+ distance = node_distance(node, target);
+ if (distance == best_distance || best_distance == -1) {
+ best_distance = distance;
+ node_set(target, nd->preferred);
+ } else {
+ break;
+ }
+ } while (1);
+ }
+ /*
+ * Promotion is allowed from a memory tier to higher
+ * memory tier only if the memory tier doesn't include
+ * compute. We want to skip promotion from a memory tier,
+ * if any node that is part of the memory tier have CPUs.
+ * Once we detect such a memory tier, we consider that tier
+ * as top tiper from which promotion is not allowed.
+ */
+ list_for_each_entry_reverse(memtier, &memory_tiers, list) {
+ tier_nodes = get_memtier_nodemask(memtier);
+ nodes_and(tier_nodes, node_states[N_CPU], tier_nodes);
+ if (!nodes_empty(tier_nodes)) {
+ /*
+ * abstract distance below the max value of this memtier
+ * is considered toptier.
+ */
+ top_tier_adistance = memtier->adistance_start +
+ MEMTIER_CHUNK_SIZE - 1;
+ break;
+ }
+ }
+ /*
+ * Now build the lower_tier mask for each node collecting node mask from
+ * all memory tier below it. This allows us to fallback demotion page
+ * allocation to a set of nodes that is closer the above selected
+ * perferred node.
+ */
+ lower_tier = node_states[N_MEMORY];
+ list_for_each_entry(memtier, &memory_tiers, list) {
+ /*
+ * Keep removing current tier from lower_tier nodes,
+ * This will remove all nodes in current and above
+ * memory tier from the lower_tier mask.
+ */
+ tier_nodes = get_memtier_nodemask(memtier);
+ nodes_andnot(lower_tier, lower_tier, tier_nodes);
+ memtier->lower_tier_mask = lower_tier;
+ }
+}
+
+#else
+static inline void disable_all_demotion_targets(void) {}
+static inline void establish_demotion_targets(void) {}
+#endif /* CONFIG_MIGRATION */
+
+static inline void __init_node_memory_type(int node, struct memory_dev_type *memtype)
+{
+ if (!node_memory_types[node].memtype)
+ node_memory_types[node].memtype = memtype;
+ /*
+ * for each device getting added in the same NUMA node
+ * with this specific memtype, bump the map count. We
+ * Only take memtype device reference once, so that
+ * changing a node memtype can be done by droping the
+ * only reference count taken here.
+ */
+
+ if (node_memory_types[node].memtype == memtype) {
+ if (!node_memory_types[node].map_count++)
+ kref_get(&memtype->kref);
+ }
+}
+
+static struct memory_tier *set_node_memory_tier(int node)
+{
+ struct memory_tier *memtier;
+ struct memory_dev_type *memtype;
+ pg_data_t *pgdat = NODE_DATA(node);
+
+
+ lockdep_assert_held_once(&memory_tier_lock);
+
+ if (!node_state(node, N_MEMORY))
+ return ERR_PTR(-EINVAL);
+
+ __init_node_memory_type(node, default_dram_type);
+
+ memtype = node_memory_types[node].memtype;
+ node_set(node, memtype->nodes);
+ memtier = find_create_memory_tier(memtype);
+ if (!IS_ERR(memtier))
+ rcu_assign_pointer(pgdat->memtier, memtier);
+ return memtier;
+}
+
+static void destroy_memory_tier(struct memory_tier *memtier)
+{
+ list_del(&memtier->list);
+ device_unregister(&memtier->dev);
+}
+
+static bool clear_node_memory_tier(int node)
+{
+ bool cleared = false;
+ pg_data_t *pgdat;
+ struct memory_tier *memtier;
+
+ pgdat = NODE_DATA(node);
+ if (!pgdat)
+ return false;
+
+ /*
+ * Make sure that anybody looking at NODE_DATA who finds
+ * a valid memtier finds memory_dev_types with nodes still
+ * linked to the memtier. We achieve this by waiting for
+ * rcu read section to finish using synchronize_rcu.
+ * This also enables us to free the destroyed memory tier
+ * with kfree instead of kfree_rcu
+ */
+ memtier = __node_get_memory_tier(node);
+ if (memtier) {
+ struct memory_dev_type *memtype;
+
+ rcu_assign_pointer(pgdat->memtier, NULL);
+ synchronize_rcu();
+ memtype = node_memory_types[node].memtype;
+ node_clear(node, memtype->nodes);
+ if (nodes_empty(memtype->nodes)) {
+ list_del_init(&memtype->tier_sibiling);
+ if (list_empty(&memtier->memory_types))
+ destroy_memory_tier(memtier);
+ }
+ cleared = true;
+ }
+ return cleared;
+}
+
+static void release_memtype(struct kref *kref)
+{
+ struct memory_dev_type *memtype;
+
+ memtype = container_of(kref, struct memory_dev_type, kref);
+ kfree(memtype);
+}
+
+struct memory_dev_type *alloc_memory_type(int adistance)
+{
+ struct memory_dev_type *memtype;
+
+ memtype = kmalloc(sizeof(*memtype), GFP_KERNEL);
+ if (!memtype)
+ return ERR_PTR(-ENOMEM);
+
+ memtype->adistance = adistance;
+ INIT_LIST_HEAD(&memtype->tier_sibiling);
+ memtype->nodes = NODE_MASK_NONE;
+ kref_init(&memtype->kref);
+ return memtype;
+}
+EXPORT_SYMBOL_GPL(alloc_memory_type);
+
+void destroy_memory_type(struct memory_dev_type *memtype)
+{
+ kref_put(&memtype->kref, release_memtype);
+}
+EXPORT_SYMBOL_GPL(destroy_memory_type);
+
+void init_node_memory_type(int node, struct memory_dev_type *memtype)
+{
+
+ mutex_lock(&memory_tier_lock);
+ __init_node_memory_type(node, memtype);
+ mutex_unlock(&memory_tier_lock);
+}
+EXPORT_SYMBOL_GPL(init_node_memory_type);
+
+void clear_node_memory_type(int node, struct memory_dev_type *memtype)
+{
+ mutex_lock(&memory_tier_lock);
+ if (node_memory_types[node].memtype == memtype)
+ node_memory_types[node].map_count--;
+ /*
+ * If we umapped all the attached devices to this node,
+ * clear the node memory type.
+ */
+ if (!node_memory_types[node].map_count) {
+ node_memory_types[node].memtype = NULL;
+ kref_put(&memtype->kref, release_memtype);
+ }
+ mutex_unlock(&memory_tier_lock);
+}
+EXPORT_SYMBOL_GPL(clear_node_memory_type);
+
+static int __meminit memtier_hotplug_callback(struct notifier_block *self,
+ unsigned long action, void *_arg)
+{
+ struct memory_tier *memtier;
+ struct memory_notify *arg = _arg;
+
+ /*
+ * Only update the node migration order when a node is
+ * changing status, like online->offline.
+ */
+ if (arg->status_change_nid < 0)
+ return notifier_from_errno(0);
+
+ switch (action) {
+ case MEM_OFFLINE:
+ mutex_lock(&memory_tier_lock);
+ if (clear_node_memory_tier(arg->status_change_nid))
+ establish_demotion_targets();
+ mutex_unlock(&memory_tier_lock);
+ break;
+ case MEM_ONLINE:
+ mutex_lock(&memory_tier_lock);
+ memtier = set_node_memory_tier(arg->status_change_nid);
+ if (!IS_ERR(memtier))
+ establish_demotion_targets();
+ mutex_unlock(&memory_tier_lock);
+ break;
+ }
+
+ return notifier_from_errno(0);
+}
+
+static int __init memory_tier_init(void)
+{
+ int ret, node;
+ struct memory_tier *memtier;
+
+ ret = subsys_virtual_register(&memory_tier_subsys, NULL);
+ if (ret)
+ panic("%s() failed to register memory tier subsystem\n", __func__);
+
+#ifdef CONFIG_MIGRATION
+ node_demotion = kcalloc(nr_node_ids, sizeof(struct demotion_nodes),
+ GFP_KERNEL);
+ WARN_ON(!node_demotion);
+#endif
+ mutex_lock(&memory_tier_lock);
+ /*
+ * For now we can have 4 faster memory tiers with smaller adistance
+ * than default DRAM tier.
+ */
+ default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM);
+ if (!default_dram_type)
+ panic("%s() failed to allocate default DRAM tier\n", __func__);
+
+ /*
+ * Look at all the existing N_MEMORY nodes and add them to
+ * default memory tier or to a tier if we already have memory
+ * types assigned.
+ */
+ for_each_node_state(node, N_MEMORY) {
+ memtier = set_node_memory_tier(node);
+ if (IS_ERR(memtier))
+ /*
+ * Continue with memtiers we are able to setup
+ */
+ break;
+ }
+ establish_demotion_targets();
+ mutex_unlock(&memory_tier_lock);
+
+ hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRIO);
+ return 0;
+}
+subsys_initcall(memory_tier_init);
+
+bool numa_demotion_enabled = false;
+
+#ifdef CONFIG_MIGRATION
+#ifdef CONFIG_SYSFS
+static ssize_t numa_demotion_enabled_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ return sysfs_emit(buf, "%s\n",
+ numa_demotion_enabled ? "true" : "false");
+}
+
+static ssize_t numa_demotion_enabled_store(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ ssize_t ret;
+
+ ret = kstrtobool(buf, &numa_demotion_enabled);
+ if (ret)
+ return ret;
+
+ return count;
+}
+
+static struct kobj_attribute numa_demotion_enabled_attr =
+ __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show,
+ numa_demotion_enabled_store);
+
+static struct attribute *numa_attrs[] = {
+ &numa_demotion_enabled_attr.attr,
+ NULL,
+};
+
+static const struct attribute_group numa_attr_group = {
+ .attrs = numa_attrs,
+};
+
+static int __init numa_init_sysfs(void)
+{
+ int err;
+ struct kobject *numa_kobj;
+
+ numa_kobj = kobject_create_and_add("numa", mm_kobj);
+ if (!numa_kobj) {
+ pr_err("failed to create numa kobject\n");
+ return -ENOMEM;
+ }
+ err = sysfs_create_group(numa_kobj, &numa_attr_group);
+ if (err) {
+ pr_err("failed to register numa group\n");
+ goto delete_obj;
+ }
+ return 0;
+
+delete_obj:
+ kobject_put(numa_kobj);
+ return err;
+}
+subsys_initcall(numa_init_sysfs);
+#endif /* CONFIG_SYSFS */
+#endif
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
+#include <linux/kmsan.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
#include <linux/gfp.h>
#include <linux/migrate.h>
#include <linux/string.h>
+#include <linux/memory-tiers.h>
#include <linux/debugfs.h>
#include <linux/userfaultfd_k.h>
#include <linux/dax.h>
#include <linux/perf_event.h>
#include <linux/ptrace.h>
#include <linux/vmalloc.h>
+#include <linux/sched/sysctl.h>
#include <trace/events/kmem.h>
2;
#endif
-#ifndef arch_faults_on_old_pte
-static inline bool arch_faults_on_old_pte(void)
-{
- /*
- * Those arches which don't have hw access flag feature need to
- * implement their own helper. By default, "true" means pagefault
- * will be hit on old pte.
- */
- return true;
-}
-#endif
-
#ifndef arch_wants_old_prefaulted_pte
static inline bool arch_wants_old_prefaulted_pte(void)
{
} while (pgd++, addr = next, addr != end);
}
-void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
- unsigned long floor, unsigned long ceiling)
+void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt,
+ struct vm_area_struct *vma, unsigned long floor,
+ unsigned long ceiling)
{
- while (vma) {
- struct vm_area_struct *next = vma->vm_next;
+ MA_STATE(mas, mt, vma->vm_end, vma->vm_end);
+
+ do {
unsigned long addr = vma->vm_start;
+ struct vm_area_struct *next;
+
+ /*
+ * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
+ * be 0. This will underflow and is okay.
+ */
+ next = mas_find(&mas, ceiling - 1);
/*
* Hide vma from rmap and truncate_pagecache before freeing
while (next && next->vm_start <= vma->vm_end + PMD_SIZE
&& !is_vm_hugetlb_page(next)) {
vma = next;
- next = vma->vm_next;
+ next = mas_find(&mas, ceiling - 1);
unlink_anon_vmas(vma);
unlink_file_vma(vma);
}
floor, next ? next->vm_start : ceiling);
}
vma = next;
- }
+ } while (vma);
}
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
unsigned long addr, pte_t *pte,
struct zap_details *details, pte_t pteval)
{
+#ifdef CONFIG_PTE_MARKER_UFFD_WP
if (zap_drop_file_uffd_wp(details))
return;
pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
+#endif
}
static unsigned long zap_pte_range(struct mmu_gather *tlb,
if (vma->vm_file) {
zap_flags_t zap_flags = details ?
details->zap_flags : 0;
- i_mmap_lock_write(vma->vm_file->f_mapping);
__unmap_hugepage_range_final(tlb, vma, start, end,
NULL, zap_flags);
- i_mmap_unlock_write(vma->vm_file->f_mapping);
}
} else
unmap_page_range(tlb, vma, start, end, details);
/**
* unmap_vmas - unmap a range of memory covered by a list of vma's
* @tlb: address of the caller's struct mmu_gather
+ * @mt: the maple tree
* @vma: the starting vma
* @start_addr: virtual address at which to start unmapping
* @end_addr: virtual address at which to end unmapping
* ensure that any thus-far unmapped pages are flushed before unmap_vmas()
* drops the lock and schedules.
*/
-void unmap_vmas(struct mmu_gather *tlb,
+void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt,
struct vm_area_struct *vma, unsigned long start_addr,
unsigned long end_addr)
{
/* Careful - we need to zap private pages too! */
.even_cows = true,
};
+ MA_STATE(mas, mt, vma->vm_end, vma->vm_end);
mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
start_addr, end_addr);
mmu_notifier_invalidate_range_start(&range);
- for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
+ do {
unmap_single_vma(tlb, vma, start_addr, end_addr, &details);
+ } while ((vma = mas_find(&mas, end_addr - 1)) != NULL);
mmu_notifier_invalidate_range_end(&range);
}
void zap_page_range(struct vm_area_struct *vma, unsigned long start,
unsigned long size)
{
+ struct maple_tree *mt = &vma->vm_mm->mm_mt;
+ unsigned long end = start + size;
struct mmu_notifier_range range;
struct mmu_gather tlb;
+ MA_STATE(mas, mt, vma->vm_end, vma->vm_end);
lru_add_drain();
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
tlb_gather_mmu(&tlb, vma->vm_mm);
update_hiwater_rss(vma->vm_mm);
mmu_notifier_invalidate_range_start(&range);
- for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
+ do {
unmap_single_vma(&tlb, vma, start, range.end, NULL);
+ } while ((vma = mas_find(&mas, end - 1)) != NULL);
mmu_notifier_invalidate_range_end(&range);
tlb_finish_mmu(&tlb);
}
* On architectures with software "accessed" bits, we would
* take a double page fault, so mark it accessed here.
*/
- if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) {
+ if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
pte_t entry;
vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
delayacct_wpcopy_end();
return 0;
}
+ kmsan_copy_page_meta(new_page, old_page);
}
if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL))
{
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
struct vm_area_struct *vma = vmf->vma;
+ struct folio *folio;
VM_BUG_ON(unshare && (vmf->flags & FAULT_FLAG_WRITE));
VM_BUG_ON(!unshare && !(vmf->flags & FAULT_FLAG_WRITE));
* Take out anonymous pages first, anonymous shared vmas are
* not dirty accountable.
*/
- if (PageAnon(vmf->page)) {
- struct page *page = vmf->page;
-
+ folio = page_folio(vmf->page);
+ if (folio_test_anon(folio)) {
/*
* If the page is exclusive to this process we must reuse the
* page without further checks.
*/
- if (PageAnonExclusive(page))
+ if (PageAnonExclusive(vmf->page))
goto reuse;
/*
- * We have to verify under page lock: these early checks are
- * just an optimization to avoid locking the page and freeing
+ * We have to verify under folio lock: these early checks are
+ * just an optimization to avoid locking the folio and freeing
* the swapcache if there is little hope that we can reuse.
*
- * PageKsm() doesn't necessarily raise the page refcount.
+ * KSM doesn't necessarily raise the folio refcount.
*/
- if (PageKsm(page) || page_count(page) > 3)
+ if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
goto copy;
- if (!PageLRU(page))
+ if (!folio_test_lru(folio))
/*
* Note: We cannot easily detect+handle references from
- * remote LRU pagevecs or references to PageLRU() pages.
+ * remote LRU pagevecs or references to LRU folios.
*/
lru_add_drain();
- if (page_count(page) > 1 + PageSwapCache(page))
+ if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
goto copy;
- if (!trylock_page(page))
+ if (!folio_trylock(folio))
goto copy;
- if (PageSwapCache(page))
- try_to_free_swap(page);
- if (PageKsm(page) || page_count(page) != 1) {
- unlock_page(page);
+ if (folio_test_swapcache(folio))
+ folio_free_swap(folio);
+ if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
+ folio_unlock(folio);
goto copy;
}
/*
- * Ok, we've got the only page reference from our mapping
- * and the page is locked, it's dark out, and we're wearing
+ * Ok, we've got the only folio reference from our mapping
+ * and the folio is locked, it's dark out, and we're wearing
* sunglasses. Hit it.
*/
- page_move_anon_rmap(page, vma);
- unlock_page(page);
+ page_move_anon_rmap(vmf->page, vma);
+ folio_unlock(folio);
reuse:
if (unlikely(unshare)) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
*/
static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
{
- struct page *page = vmf->page;
+ struct folio *folio = page_folio(vmf->page);
struct vm_area_struct *vma = vmf->vma;
struct mmu_notifier_range range;
- if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags))
+ if (!folio_lock_or_retry(folio, vma->vm_mm, vmf->flags))
return VM_FAULT_RETRY;
mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,
vma->vm_mm, vmf->address & PAGE_MASK,
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
&vmf->ptl);
if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
- restore_exclusive_pte(vma, page, vmf->address, vmf->pte);
+ restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
pte_unmap_unlock(vmf->pte, vmf->ptl);
- unlock_page(page);
+ folio_unlock(folio);
mmu_notifier_invalidate_range_end(&range);
return 0;
}
-static inline bool should_try_to_free_swap(struct page *page,
+static inline bool should_try_to_free_swap(struct folio *folio,
struct vm_area_struct *vma,
unsigned int fault_flags)
{
- if (!PageSwapCache(page))
+ if (!folio_test_swapcache(folio))
return false;
- if (mem_cgroup_swap_full(page) || (vma->vm_flags & VM_LOCKED) ||
- PageMlocked(page))
+ if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
+ folio_test_mlocked(folio))
return true;
/*
* If we want to map a page that's in the swapcache writable, we
* user. Try freeing the swapcache to get rid of the swapcache
* reference only in case it's likely that we'll be the exlusive user.
*/
- return (fault_flags & FAULT_FLAG_WRITE) && !PageKsm(page) &&
- page_count(page) == 2;
+ return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
+ folio_ref_count(folio) == 2;
}
static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
vm_fault_t do_swap_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
- struct page *page = NULL, *swapcache;
+ struct folio *swapcache, *folio = NULL;
+ struct page *page;
struct swap_info_struct *si = NULL;
rmap_t rmap_flags = RMAP_NONE;
bool exclusive = false;
ret = remove_device_exclusive_entry(vmf);
} else if (is_device_private_entry(entry)) {
vmf->page = pfn_swap_entry_to_page(entry);
- ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
+ vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
+ vmf->address, &vmf->ptl);
+ if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
+ spin_unlock(vmf->ptl);
+ goto out;
+ }
+
+ /*
+ * Get a page reference while we know the page can't be
+ * freed.
+ */
+ get_page(vmf->page);
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
+ vmf->page->pgmap->ops->migrate_to_ram(vmf);
+ put_page(vmf->page);
} else if (is_hwpoison_entry(entry)) {
ret = VM_FAULT_HWPOISON;
} else if (is_swapin_error_entry(entry)) {
if (unlikely(!si))
goto out;
- page = lookup_swap_cache(entry, vma, vmf->address);
- swapcache = page;
+ folio = swap_cache_get_folio(entry, vma, vmf->address);
+ if (folio)
+ page = folio_file_page(folio, swp_offset(entry));
+ swapcache = folio;
- if (!page) {
+ if (!folio) {
if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
__swap_count(entry) == 1) {
/* skip swapcache */
- page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
- vmf->address);
- if (page) {
- __SetPageLocked(page);
- __SetPageSwapBacked(page);
-
- if (mem_cgroup_swapin_charge_page(page,
- vma->vm_mm, GFP_KERNEL, entry)) {
+ folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0,
+ vma, vmf->address, false);
+ page = &folio->page;
+ if (folio) {
+ __folio_set_locked(folio);
+ __folio_set_swapbacked(folio);
+
+ if (mem_cgroup_swapin_charge_folio(folio,
+ vma->vm_mm, GFP_KERNEL,
+ entry)) {
ret = VM_FAULT_OOM;
goto out_page;
}
shadow = get_shadow_from_swap_cache(entry);
if (shadow)
- workingset_refault(page_folio(page),
- shadow);
+ workingset_refault(folio, shadow);
- lru_cache_add(page);
+ folio_add_lru(folio);
/* To provide entry to swap_readpage() */
- set_page_private(page, entry.val);
+ folio_set_swap_entry(folio, entry);
swap_readpage(page, true, NULL);
- set_page_private(page, 0);
+ folio->private = NULL;
}
} else {
page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
vmf);
- swapcache = page;
+ if (page)
+ folio = page_folio(page);
+ swapcache = folio;
}
- if (!page) {
+ if (!folio) {
/*
* Back out if somebody else faulted in this pte
* while we released the pte lock.
goto out_release;
}
- locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
+ locked = folio_lock_or_retry(folio, vma->vm_mm, vmf->flags);
if (!locked) {
ret |= VM_FAULT_RETRY;
if (swapcache) {
/*
- * Make sure try_to_free_swap or swapoff did not release the
+ * Make sure folio_free_swap() or swapoff did not release the
* swapcache from under us. The page pin, and pte_same test
* below, are not enough to exclude that. Even if it is still
* swapcache, we need to check that the page's swap has not
* changed.
*/
- if (unlikely(!PageSwapCache(page) ||
+ if (unlikely(!folio_test_swapcache(folio) ||
page_private(page) != entry.val))
goto out_page;
page = ksm_might_need_to_copy(page, vma, vmf->address);
if (unlikely(!page)) {
ret = VM_FAULT_OOM;
- page = swapcache;
goto out_page;
}
+ folio = page_folio(page);
/*
* If we want to map a page that's in the swapcache writable, we
* owner. Try removing the extra reference from the local LRU
* pagevecs if required.
*/
- if ((vmf->flags & FAULT_FLAG_WRITE) && page == swapcache &&
- !PageKsm(page) && !PageLRU(page))
+ if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
+ !folio_test_ksm(folio) && !folio_test_lru(folio))
lru_add_drain();
}
if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
goto out_nomap;
- if (unlikely(!PageUptodate(page))) {
+ if (unlikely(!folio_test_uptodate(folio))) {
ret = VM_FAULT_SIGBUS;
goto out_nomap;
}
* check after taking the PT lock and making sure that nobody
* concurrently faulted in this page and set PG_anon_exclusive.
*/
- BUG_ON(!PageAnon(page) && PageMappedToDisk(page));
- BUG_ON(PageAnon(page) && PageAnonExclusive(page));
+ BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
+ BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
/*
* Check under PT lock (to protect against concurrent fork() sharing
* the swap entry concurrently) for certainly exclusive pages.
*/
- if (!PageKsm(page)) {
+ if (!folio_test_ksm(folio)) {
/*
* Note that pte_swp_exclusive() == false for architectures
* without __HAVE_ARCH_PTE_SWP_EXCLUSIVE.
*/
exclusive = pte_swp_exclusive(vmf->orig_pte);
- if (page != swapcache) {
+ if (folio != swapcache) {
/*
* We have a fresh page that is not exposed to the
* swapcache -> certainly exclusive.
*/
exclusive = true;
- } else if (exclusive && PageWriteback(page) &&
+ } else if (exclusive && folio_test_writeback(folio) &&
data_race(si->flags & SWP_STABLE_WRITES)) {
/*
* This is tricky: not all swap backends support
* yet.
*/
swap_free(entry);
- if (should_try_to_free_swap(page, vma, vmf->flags))
- try_to_free_swap(page);
+ if (should_try_to_free_swap(folio, vma, vmf->flags))
+ folio_free_swap(folio);
inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
* exposing them to the swapcache or because the swap entry indicates
* exclusivity.
*/
- if (!PageKsm(page) && (exclusive || page_count(page) == 1)) {
+ if (!folio_test_ksm(folio) &&
+ (exclusive || folio_ref_count(folio) == 1)) {
if (vmf->flags & FAULT_FLAG_WRITE) {
pte = maybe_mkwrite(pte_mkdirty(pte), vma);
vmf->flags &= ~FAULT_FLAG_WRITE;
vmf->orig_pte = pte;
/* ksm created a completely new copy */
- if (unlikely(page != swapcache && swapcache)) {
+ if (unlikely(folio != swapcache && swapcache)) {
page_add_new_anon_rmap(page, vma, vmf->address);
- lru_cache_add_inactive_or_unevictable(page, vma);
+ folio_add_lru_vma(folio, vma);
} else {
page_add_anon_rmap(page, vma, vmf->address, rmap_flags);
}
- VM_BUG_ON(!PageAnon(page) || (pte_write(pte) && !PageAnonExclusive(page)));
+ VM_BUG_ON(!folio_test_anon(folio) ||
+ (pte_write(pte) && !PageAnonExclusive(page)));
set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
- unlock_page(page);
- if (page != swapcache && swapcache) {
+ folio_unlock(folio);
+ if (folio != swapcache && swapcache) {
/*
* Hold the lock to avoid the swap entry to be reused
* until we take the PT lock for the pte_same() check
* so that the swap count won't change under a
* parallel locked swapcache.
*/
- unlock_page(swapcache);
- put_page(swapcache);
+ folio_unlock(swapcache);
+ folio_put(swapcache);
}
if (vmf->flags & FAULT_FLAG_WRITE) {
out_nomap:
pte_unmap_unlock(vmf->pte, vmf->ptl);
out_page:
- unlock_page(page);
+ folio_unlock(folio);
out_release:
- put_page(page);
- if (page != swapcache && swapcache) {
- unlock_page(swapcache);
- put_page(swapcache);
+ folio_put(folio);
+ if (folio != swapcache && swapcache) {
+ folio_unlock(swapcache);
+ folio_put(swapcache);
}
if (si)
put_swap_device(si);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
&vmf->ptl);
if (!pte_none(*vmf->pte)) {
- update_mmu_cache(vma, vmf->address, vmf->pte);
+ update_mmu_tlb(vma, vmf->address, vmf->pte);
goto release;
}
if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
flags |= TNF_SHARED;
- last_cpupid = page_cpupid_last(page);
page_nid = page_to_nid(page);
+ /*
+ * For memory tiering mode, cpupid of slow memory page is used
+ * to record page access time. So use default value.
+ */
+ if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+ !node_is_toptier(page_nid))
+ last_cpupid = (-1 & LAST_CPUPID_MASK);
+ else
+ last_cpupid = page_cpupid_last(page);
target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
&flags);
if (target_nid == NUMA_NO_NODE) {
return VM_FAULT_OOM;
retry_pud:
if (pud_none(*vmf.pud) &&
- hugepage_vma_check(vma, vm_flags, false, true)) {
+ hugepage_vma_check(vma, vm_flags, false, true, true)) {
ret = create_huge_pud(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
goto retry_pud;
if (pmd_none(*vmf.pmd) &&
- hugepage_vma_check(vma, vm_flags, false, true)) {
+ hugepage_vma_check(vma, vm_flags, false, true, true)) {
ret = create_huge_pmd(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
}
+#ifdef CONFIG_LRU_GEN
+static void lru_gen_enter_fault(struct vm_area_struct *vma)
+{
+ /* the LRU algorithm doesn't apply to sequential or random reads */
+ current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ));
+}
+
+static void lru_gen_exit_fault(void)
+{
+ current->in_lru_fault = false;
+}
+#else
+static void lru_gen_enter_fault(struct vm_area_struct *vma)
+{
+}
+
+static void lru_gen_exit_fault(void)
+{
+}
+#endif /* CONFIG_LRU_GEN */
+
/*
* By the time we get here, we already hold the mm semaphore
*
if (flags & FAULT_FLAG_USER)
mem_cgroup_enter_user_fault();
+ lru_gen_enter_fault(vma);
+
if (unlikely(is_vm_hugetlb_page(vma)))
ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
else
ret = __handle_mm_fault(vma, address, flags);
+ lru_gen_exit_fault();
+
if (flags & FAULT_FLAG_USER) {
mem_cgroup_exit_user_fault();
/*
unsigned int pages_per_huge_page)
{
int i;
- struct page *p = page;
+ struct page *p;
might_sleep();
- for (i = 0; i < pages_per_huge_page;
- i++, p = mem_map_next(p, page, i)) {
+ for (i = 0; i < pages_per_huge_page; i++) {
+ p = nth_page(page, i);
cond_resched();
clear_user_highpage(p, addr + i * PAGE_SIZE);
}
struct page *dst_base = dst;
struct page *src_base = src;
- for (i = 0; i < pages_per_huge_page; ) {
+ for (i = 0; i < pages_per_huge_page; i++) {
+ dst = nth_page(dst_base, i);
+ src = nth_page(src_base, i);
+
cond_resched();
copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
-
- i++;
- dst = mem_map_next(dst, dst_base, i);
- src = mem_map_next(src, src_base, i);
}
}
void *page_kaddr;
unsigned long i, rc = 0;
unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
- struct page *subpage = dst_page;
+ struct page *subpage;
- for (i = 0; i < pages_per_huge_page;
- i++, subpage = mem_map_next(subpage, dst_page, i)) {
+ for (i = 0; i < pages_per_huge_page; i++) {
+ subpage = nth_page(dst_page, i);
if (allow_pagefault)
page_kaddr = kmap(subpage);
else
* of the physical memory space for vmemmaps. That space is pageblock
* aligned.
*/
- if (WARN_ON_ONCE(!nr_pages ||
- !IS_ALIGNED(pfn, pageblock_nr_pages) ||
+ if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(pfn) ||
!IS_ALIGNED(pfn + nr_pages, PAGES_PER_SECTION)))
return -EINVAL;
* of the physical memory space for vmemmaps. That space is pageblock
* aligned.
*/
- if (WARN_ON_ONCE(!nr_pages ||
- !IS_ALIGNED(start_pfn, pageblock_nr_pages) ||
+ if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(start_pfn) ||
!IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION)))
return -EINVAL;
node_states_clear_node(node, &arg);
if (arg.status_change_nid >= 0) {
- kswapd_stop(node);
kcompactd_stop(node);
+ kswapd_stop(node);
}
writeback_set_ratelimit();
static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
{
- int ret = !is_memblock_offlined(mem);
int *nid = arg;
*nid = mem->nid;
- if (unlikely(ret)) {
+ if (unlikely(mem->state != MEM_OFFLINE)) {
phys_addr_t beginpa, endpa;
beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ for_each_vma(vmi, vma)
mpol_rebind_policy(vma->vm_policy, new);
mmap_write_unlock(mm);
}
static int queue_pages_test_walk(unsigned long start, unsigned long end,
struct mm_walk *walk)
{
- struct vm_area_struct *vma = walk->vma;
+ struct vm_area_struct *next, *vma = walk->vma;
struct queue_pages *qp = walk->private;
unsigned long endvma = vma->vm_end;
unsigned long flags = qp->flags;
/* hole at head side of range */
return -EFAULT;
}
+ next = find_vma(vma->vm_mm, vma->vm_end);
if (!(flags & MPOL_MF_DISCONTIG_OK) &&
((vma->vm_end < qp->end) &&
- (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
+ (!next || vma->vm_end < next->vm_start)))
/* hole at middle or tail of range */
return -EFAULT;
static int mbind_range(struct mm_struct *mm, unsigned long start,
unsigned long end, struct mempolicy *new_pol)
{
+ MA_STATE(mas, &mm->mm_mt, start - 1, start - 1);
struct vm_area_struct *prev;
struct vm_area_struct *vma;
int err = 0;
pgoff_t pgoff;
- unsigned long vmstart;
- unsigned long vmend;
- vma = find_vma(mm, start);
- VM_BUG_ON(!vma);
-
- prev = vma->vm_prev;
- if (start > vma->vm_start)
- prev = vma;
+ prev = mas_find_rev(&mas, 0);
+ if (prev && (start < prev->vm_end))
+ vma = prev;
+ else
+ vma = mas_next(&mas, end - 1);
- for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) {
- vmstart = max(start, vma->vm_start);
- vmend = min(end, vma->vm_end);
+ for (; vma; vma = mas_next(&mas, end - 1)) {
+ unsigned long vmstart = max(start, vma->vm_start);
+ unsigned long vmend = min(end, vma->vm_end);
if (mpol_equal(vma_policy(vma), new_pol))
- continue;
+ goto next;
pgoff = vma->vm_pgoff +
((vmstart - vma->vm_start) >> PAGE_SHIFT);
new_pol, vma->vm_userfaultfd_ctx,
anon_vma_name(vma));
if (prev) {
+ /* vma_merge() invalidated the mas */
+ mas_pause(&mas);
vma = prev;
goto replace;
}
err = split_vma(vma->vm_mm, vma, vmstart, 1);
if (err)
goto out;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
if (vma->vm_end != vmend) {
err = split_vma(vma->vm_mm, vma, vmend, 0);
if (err)
goto out;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
- replace:
+replace:
err = vma_replace_policy(vma, new_pol);
if (err)
goto out;
+next:
+ prev = vma;
}
- out:
+out:
return err;
}
goto out;
}
+ task_lock(current);
ret = mpol_set_nodemask(new, nodes, scratch);
if (ret) {
+ task_unlock(current);
mpol_put(new);
goto out;
}
- task_lock(current);
+
old = current->mempolicy;
current->mempolicy = new;
if (new && new->mode == MPOL_INTERLEAVE)
int flags)
{
nodemask_t nmask;
+ struct vm_area_struct *vma;
LIST_HEAD(pagelist);
int err = 0;
struct migration_target_control mtc = {
* need migration. Between passing in the full user address
* space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
*/
+ vma = find_vma(mm, 0);
VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
- queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
+ queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
flags | MPOL_MF_DISCONTIG_OK, &pagelist);
if (!list_empty(&pagelist)) {
struct folio *dst, *src = page_folio(page);
struct vm_area_struct *vma;
unsigned long address;
+ VMA_ITERATOR(vmi, current->mm, start);
gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL;
- vma = find_vma(current->mm, start);
- while (vma) {
+ for_each_vma(vmi, vma) {
address = page_address_in_vma(page, vma);
if (address != -EFAULT)
break;
- vma = vma->vm_next;
}
if (folio_test_hugetlb(src))
if (mode == MPOL_DEFAULT)
flags &= ~MPOL_MF_STRICT;
- len = (len + PAGE_SIZE - 1) & PAGE_MASK;
+ len = PAGE_ALIGN(len);
end = start + len;
if (end < start)
unsigned long vmend;
unsigned long end;
int err = -ENOENT;
+ VMA_ITERATOR(vmi, mm, start);
start = untagged_addr(start);
if (start & ~PAGE_MASK)
if (home_node >= MAX_NUMNODES || !node_online(home_node))
return -EINVAL;
- len = (len + PAGE_SIZE - 1) & PAGE_MASK;
+ len = PAGE_ALIGN(len);
end = start + len;
if (end < start)
if (end == start)
return 0;
mmap_write_lock(mm);
- vma = find_vma(mm, start);
- for (; vma && vma->vm_start < end; vma = vma->vm_next) {
-
+ for_each_vma_range(vmi, vma, end) {
vmstart = max(start, vma->vm_start);
vmend = min(end, vma->vm_end);
new = mpol_dup(vma_policy(vma));
return pol->flags & MPOL_F_MOF;
}
-static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
+bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
{
enum zone_type dynamic_policy_zone = policy_zone;
int i;
percpu_ref_kill(&pgmap->ref);
- for (i = 0; i < pgmap->nr_range; i++)
- percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
+ if (pgmap->type != MEMORY_DEVICE_PRIVATE &&
+ pgmap->type != MEMORY_DEVICE_COHERENT)
+ for (i = 0; i < pgmap->nr_range; i++)
+ percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
+
wait_for_completion(&pgmap->done);
for (i = 0; i < pgmap->nr_range; i++)
memmap_init_zone_device(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
PHYS_PFN(range->start),
PHYS_PFN(range_len(range)), pgmap);
- percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id));
+ if (pgmap->type != MEMORY_DEVICE_PRIVATE &&
+ pgmap->type != MEMORY_DEVICE_COHERENT)
+ percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id));
return 0;
err_add_memory:
/* fall back to slow path lookup */
rcu_read_lock();
pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));
- if (pgmap && !percpu_ref_tryget_live(&pgmap->ref))
+ if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref))
pgmap = NULL;
rcu_read_unlock();
page->mapping = NULL;
page->pgmap->ops->page_free(page);
+ if (page->pgmap->type != MEMORY_DEVICE_PRIVATE &&
+ page->pgmap->type != MEMORY_DEVICE_COHERENT)
+ /*
+ * Reset the page count to 1 to prepare for handing out the page
+ * again.
+ */
+ set_page_count(page, 1);
+ else
+ put_dev_pagemap(page->pgmap);
+}
+
+void zone_device_page_init(struct page *page)
+{
/*
- * Reset the page count to 1 to prepare for handing out the page again.
+ * Drivers shouldn't be allocating pages after calling
+ * memunmap_pages().
*/
+ WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref));
set_page_count(page, 1);
+ lock_page(page);
}
+EXPORT_SYMBOL_GPL(zone_device_page_init);
#ifdef CONFIG_FS_DAX
bool __put_devmap_managed_page_refs(struct page *page, int refs)
#include <linux/memory.h>
#include <linux/random.h>
#include <linux/sched/sysctl.h>
+#include <linux/memory-tiers.h>
#include <asm/tlbflush.h>
#endif
folio_get(folio);
- pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
+ pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
if (pte_swp_soft_dirty(*pvmw.pte))
pte = pte_mksoft_dirty(pte);
* Recheck VMA as permissions can change since migration started
*/
entry = pte_to_swp_entry(*pvmw.pte);
+ if (!is_migration_entry_young(entry))
+ pte = pte_mkold(pte);
+ if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))
+ pte = pte_mkdirty(pte);
if (is_writable_migration_entry(entry))
pte = maybe_mkwrite(pte, vma);
else if (pte_swp_uffd_wp(*pvmw.pte))
* future migrations of this same page.
*/
cpupid = page_cpupid_xchg_last(&folio->page, -1);
+ /*
+ * For memory tiering mode, when migrate between slow and fast
+ * memory node, reset cpupid, because that is used to record
+ * page access time in slow memory node.
+ */
+ if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) {
+ bool f_toptier = node_is_toptier(page_to_nid(&folio->page));
+ bool t_toptier = node_is_toptier(page_to_nid(&newfolio->page));
+
+ if (f_toptier != t_toptier)
+ cpupid = -1;
+ }
page_cpupid_xchg_last(&newfolio->page, cpupid);
folio_migrate_ksm(newfolio, folio);
* Migration functions
***********************************************************/
+int migrate_folio_extra(struct address_space *mapping, struct folio *dst,
+ struct folio *src, enum migrate_mode mode, int extra_count)
+{
+ int rc;
+
+ BUG_ON(folio_test_writeback(src)); /* Writeback must be complete */
+
+ rc = folio_migrate_mapping(mapping, dst, src, extra_count);
+
+ if (rc != MIGRATEPAGE_SUCCESS)
+ return rc;
+
+ if (mode != MIGRATE_SYNC_NO_COPY)
+ folio_migrate_copy(dst, src);
+ else
+ folio_migrate_flags(dst, src);
+ return MIGRATEPAGE_SUCCESS;
+}
+
/**
* migrate_folio() - Simple folio migration.
* @mapping: The address_space containing the folio.
int migrate_folio(struct address_space *mapping, struct folio *dst,
struct folio *src, enum migrate_mode mode)
{
- int rc;
-
- BUG_ON(folio_test_writeback(src)); /* Writeback must be complete */
-
- rc = folio_migrate_mapping(mapping, dst, src, 0);
-
- if (rc != MIGRATEPAGE_SUCCESS)
- return rc;
-
- if (mode != MIGRATE_SYNC_NO_COPY)
- folio_migrate_copy(dst, src);
- else
- folio_migrate_flags(dst, src);
- return MIGRATEPAGE_SUCCESS;
+ return migrate_folio_extra(mapping, dst, src, mode, 0);
}
EXPORT_SYMBOL(migrate_folio);
return rc;
}
-static int __unmap_and_move(struct page *page, struct page *newpage,
+static int __unmap_and_move(struct folio *src, struct folio *dst,
int force, enum migrate_mode mode)
{
- struct folio *folio = page_folio(page);
- struct folio *dst = page_folio(newpage);
int rc = -EAGAIN;
bool page_was_mapped = false;
struct anon_vma *anon_vma = NULL;
- bool is_lru = !__PageMovable(page);
+ bool is_lru = !__PageMovable(&src->page);
- if (!trylock_page(page)) {
+ if (!folio_trylock(src)) {
if (!force || mode == MIGRATE_ASYNC)
goto out;
if (current->flags & PF_MEMALLOC)
goto out;
- lock_page(page);
+ folio_lock(src);
}
- if (PageWriteback(page)) {
+ if (folio_test_writeback(src)) {
/*
* Only in the case of a full synchronous migration is it
* necessary to wait for PageWriteback. In the async case,
}
if (!force)
goto out_unlock;
- wait_on_page_writeback(page);
+ folio_wait_writeback(src);
}
/*
- * By try_to_migrate(), page->mapcount goes down to 0 here. In this case,
- * we cannot notice that anon_vma is freed while we migrates a page.
+ * By try_to_migrate(), src->mapcount goes down to 0 here. In this case,
+ * we cannot notice that anon_vma is freed while we migrate a page.
* This get_anon_vma() delays freeing anon_vma pointer until the end
* of migration. File cache pages are no problem because of page_lock()
* File Caches may use write_page() or lock_page() in migration, then,
* just care Anon page here.
*
- * Only page_get_anon_vma() understands the subtleties of
+ * Only folio_get_anon_vma() understands the subtleties of
* getting a hold on an anon_vma from outside one of its mms.
* But if we cannot get anon_vma, then we won't need it anyway,
* because that implies that the anon page is no longer mapped
* (and cannot be remapped so long as we hold the page lock).
*/
- if (PageAnon(page) && !PageKsm(page))
- anon_vma = page_get_anon_vma(page);
+ if (folio_test_anon(src) && !folio_test_ksm(src))
+ anon_vma = folio_get_anon_vma(src);
/*
* Block others from accessing the new page when we get around to
* establishing additional references. We are usually the only one
- * holding a reference to newpage at this point. We used to have a BUG
- * here if trylock_page(newpage) fails, but would like to allow for
- * cases where there might be a race with the previous use of newpage.
+ * holding a reference to dst at this point. We used to have a BUG
+ * here if folio_trylock(dst) fails, but would like to allow for
+ * cases where there might be a race with the previous use of dst.
* This is much like races on refcount of oldpage: just don't BUG().
*/
- if (unlikely(!trylock_page(newpage)))
+ if (unlikely(!folio_trylock(dst)))
goto out_unlock;
if (unlikely(!is_lru)) {
- rc = move_to_new_folio(dst, folio, mode);
+ rc = move_to_new_folio(dst, src, mode);
goto out_unlock_both;
}
* Corner case handling:
* 1. When a new swap-cache page is read into, it is added to the LRU
* and treated as swapcache but it has no rmap yet.
- * Calling try_to_unmap() against a page->mapping==NULL page will
+ * Calling try_to_unmap() against a src->mapping==NULL page will
* trigger a BUG. So handle it here.
* 2. An orphaned page (see truncate_cleanup_page) might have
* fs-private metadata. The page can be picked up due to memory
* invisible to the vm, so the page can not be migrated. So try to
* free the metadata, so the page can be freed.
*/
- if (!page->mapping) {
- VM_BUG_ON_PAGE(PageAnon(page), page);
- if (page_has_private(page)) {
- try_to_free_buffers(folio);
+ if (!src->mapping) {
+ if (folio_test_private(src)) {
+ try_to_free_buffers(src);
goto out_unlock_both;
}
- } else if (page_mapped(page)) {
+ } else if (folio_mapped(src)) {
/* Establish migration ptes */
- VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
- page);
- try_to_migrate(folio, 0);
+ VM_BUG_ON_FOLIO(folio_test_anon(src) &&
+ !folio_test_ksm(src) && !anon_vma, src);
+ try_to_migrate(src, 0);
page_was_mapped = true;
}
- if (!page_mapped(page))
- rc = move_to_new_folio(dst, folio, mode);
+ if (!folio_mapped(src))
+ rc = move_to_new_folio(dst, src, mode);
/*
- * When successful, push newpage to LRU immediately: so that if it
+ * When successful, push dst to LRU immediately: so that if it
* turns out to be an mlocked page, remove_migration_ptes() will
- * automatically build up the correct newpage->mlock_count for it.
+ * automatically build up the correct dst->mlock_count for it.
*
* We would like to do something similar for the old page, when
* unsuccessful, and other cases when a page has been temporarily
* isolated from the unevictable LRU: but this case is the easiest.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
- lru_cache_add(newpage);
+ folio_add_lru(dst);
if (page_was_mapped)
lru_add_drain();
}
if (page_was_mapped)
- remove_migration_ptes(folio,
- rc == MIGRATEPAGE_SUCCESS ? dst : folio, false);
+ remove_migration_ptes(src,
+ rc == MIGRATEPAGE_SUCCESS ? dst : src, false);
out_unlock_both:
- unlock_page(newpage);
+ folio_unlock(dst);
out_unlock:
/* Drop an anon_vma reference if we took one */
if (anon_vma)
put_anon_vma(anon_vma);
- unlock_page(page);
+ folio_unlock(src);
out:
/*
- * If migration is successful, decrease refcount of the newpage,
+ * If migration is successful, decrease refcount of dst,
* which will not free the page because new page owner increased
* refcounter.
*/
if (rc == MIGRATEPAGE_SUCCESS)
- put_page(newpage);
+ folio_put(dst);
return rc;
}
enum migrate_reason reason,
struct list_head *ret)
{
+ struct folio *dst, *src = page_folio(page);
int rc = MIGRATEPAGE_SUCCESS;
struct page *newpage = NULL;
newpage = get_new_page(page, private);
if (!newpage)
return -ENOMEM;
+ dst = page_folio(newpage);
newpage->private = 0;
- rc = __unmap_and_move(page, newpage, force, mode);
+ rc = __unmap_and_move(src, dst, force, mode);
if (rc == MIGRATEPAGE_SUCCESS)
set_page_owner_migrate_reason(newpage, reason);
* tables or check whether the hugepage is pmd-based or not before
* kicking migration.
*/
- if (!hugepage_migration_supported(page_hstate(hpage))) {
- list_move_tail(&hpage->lru, ret);
+ if (!hugepage_migration_supported(page_hstate(hpage)))
return -ENOSYS;
- }
- if (page_count(hpage) == 1) {
+ if (folio_ref_count(src) == 1) {
/* page was freed from under us. So we are done. */
putback_active_hugepage(hpage);
return MIGRATEPAGE_SUCCESS;
return -ENOMEM;
dst = page_folio(new_hpage);
- if (!trylock_page(hpage)) {
+ if (!folio_trylock(src)) {
if (!force)
goto out;
switch (mode) {
default:
goto out;
}
- lock_page(hpage);
+ folio_lock(src);
}
/*
* Check for pages which are in the process of being freed. Without
- * page_mapping() set, hugetlbfs specific move page routine will not
+ * folio_mapping() set, hugetlbfs specific move page routine will not
* be called and we could leak usage counts for subpools.
*/
- if (hugetlb_page_subpool(hpage) && !page_mapping(hpage)) {
+ if (hugetlb_page_subpool(hpage) && !folio_mapping(src)) {
rc = -EBUSY;
goto out_unlock;
}
- if (PageAnon(hpage))
- anon_vma = page_get_anon_vma(hpage);
+ if (folio_test_anon(src))
+ anon_vma = folio_get_anon_vma(src);
- if (unlikely(!trylock_page(new_hpage)))
+ if (unlikely(!folio_trylock(dst)))
goto put_anon;
- if (page_mapped(hpage)) {
+ if (folio_mapped(src)) {
enum ttu_flags ttu = 0;
- if (!PageAnon(hpage)) {
+ if (!folio_test_anon(src)) {
/*
* In shared mappings, try_to_unmap could potentially
* call huge_pmd_unshare. Because of this, take
i_mmap_unlock_write(mapping);
}
- if (!page_mapped(hpage))
+ if (!folio_mapped(src))
rc = move_to_new_folio(dst, src, mode);
if (page_was_mapped)
rc == MIGRATEPAGE_SUCCESS ? dst : src, false);
unlock_put_anon:
- unlock_page(new_hpage);
+ folio_unlock(dst);
put_anon:
if (anon_vma)
}
out_unlock:
- unlock_page(hpage);
+ folio_unlock(src);
out:
if (rc == MIGRATEPAGE_SUCCESS)
putback_active_hugepage(hpage);
else if (rc != -EAGAIN)
- list_move_tail(&hpage->lru, ret);
+ list_move_tail(&src->lru, ret);
/*
* If migration was not successful and there's a freeing callback, use
return rc;
}
-static inline int try_split_thp(struct page *page, struct page **page2,
- struct list_head *from)
+static inline int try_split_thp(struct page *page, struct list_head *split_pages)
{
- int rc = 0;
+ int rc;
lock_page(page);
- rc = split_huge_page_to_list(page, from);
+ rc = split_huge_page_to_list(page, split_pages);
unlock_page(page);
if (!rc)
- list_safe_reset_next(page, *page2, lru);
+ list_move_tail(&page->lru, split_pages);
return rc;
}
int thp_retry = 1;
int nr_failed = 0;
int nr_failed_pages = 0;
+ int nr_retry_pages = 0;
int nr_succeeded = 0;
int nr_thp_succeeded = 0;
int nr_thp_failed = 0;
for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
retry = 0;
thp_retry = 0;
+ nr_retry_pages = 0;
list_for_each_entry_safe(page, page2, from, lru) {
-retry:
/*
* THP statistics is based on the source huge page.
* Capture required information that might get lost
* page will be put back
* -EAGAIN: stay on the from list
* -ENOMEM: stay on the from list
+ * -ENOSYS: stay on the from list
* Other errno: put on ret_pages list then splice to
* from list
*/
* retry on the same page with the THP split
* to base pages.
*
- * Head page is retried immediately and tail
- * pages are added to the tail of the list so
- * we encounter them after the rest of the list
- * is processed.
+ * Sub-pages are put in thp_split_pages, and
+ * we will migrate them after the rest of the
+ * list is processed.
*/
case -ENOSYS:
/* THP migration is unsupported */
if (is_thp) {
nr_thp_failed++;
- if (!try_split_thp(page, &page2, &thp_split_pages)) {
+ if (!try_split_thp(page, &thp_split_pages)) {
nr_thp_split++;
- goto retry;
+ break;
}
/* Hugetlb migration is unsupported */
} else if (!no_subpage_counting) {
}
nr_failed_pages += nr_subpages;
+ list_move_tail(&page->lru, &ret_pages);
break;
case -ENOMEM:
/*
* When memory is low, don't bother to try to migrate
* other pages, just exit.
- * THP NUMA faulting doesn't split THP to retry.
*/
- if (is_thp && !nosplit) {
+ if (is_thp) {
nr_thp_failed++;
- if (!try_split_thp(page, &page2, &thp_split_pages)) {
+ /* THP NUMA faulting doesn't split THP to retry. */
+ if (!nosplit && !try_split_thp(page, &thp_split_pages)) {
nr_thp_split++;
- goto retry;
+ break;
}
} else if (!no_subpage_counting) {
nr_failed++;
}
- nr_failed_pages += nr_subpages;
+ nr_failed_pages += nr_subpages + nr_retry_pages;
/*
* There might be some subpages of fail-to-migrate THPs
* left in thp_split_pages list. Move them back to migration
* the caller otherwise the page refcnt will be leaked.
*/
list_splice_init(&thp_split_pages, from);
+ /* nr_failed isn't updated for not used */
nr_thp_failed += thp_retry;
goto out;
case -EAGAIN:
if (is_thp)
thp_retry++;
- else
+ else if (!no_subpage_counting)
retry++;
+ nr_retry_pages += nr_subpages;
break;
case MIGRATEPAGE_SUCCESS:
nr_succeeded += nr_subpages;
}
nr_failed += retry;
nr_thp_failed += thp_retry;
+ nr_failed_pages += nr_retry_pages;
/*
* Try to migrate subpages of fail-to-migrate THPs, no nr_failed
* counting in this round, since all subpages of a THP is counted
goto out;
err = -ENOENT;
- if (!page || is_zone_device_page(page))
+ if (!page)
goto out;
+ if (is_zone_device_page(page))
+ goto out_putpage;
+
err = 0;
if (page_to_nid(page) == node)
goto out_putpage;
* well.
*/
if (err > 0)
- err += nr_pages - i - 1;
+ err += nr_pages - i;
return err;
}
return store_status(status, start, node, i - start);
err = move_pages_and_store_status(mm, current_node, &pagelist,
status, start, i, nr_pages);
- if (err)
+ if (err) {
+ /* We have accounted for page i */
+ if (err > 0)
+ err--;
goto out;
+ }
current_node = NUMA_NO_NODE;
}
out_flush:
for (i = 0; i < nr_pages; i++) {
unsigned long addr = (unsigned long)(*pages);
+ unsigned int foll_flags = FOLL_DUMP;
struct vm_area_struct *vma;
struct page *page;
int err = -EFAULT;
if (!vma)
goto set_status;
+ /* Not all huge page follow APIs support 'FOLL_GET' */
+ if (!is_vm_hugetlb_page(vma))
+ foll_flags |= FOLL_GET;
+
/* FOLL_DUMP to ignore special (like zero) pages */
- page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
+ page = follow_page(vma, addr, foll_flags);
err = PTR_ERR(page);
if (IS_ERR(page))
goto set_status;
- if (page && !is_zone_device_page(page)) {
+ err = -ENOENT;
+ if (!page)
+ goto set_status;
+
+ if (!is_zone_device_page(page))
err = page_to_nid(page);
+
+ if (foll_flags & FOLL_GET)
put_page(page);
- } else {
- err = -ENOENT;
- }
set_status:
*status = err;
return 0;
}
#endif /* CONFIG_NUMA_BALANCING */
-
-/*
- * node_demotion[] example:
- *
- * Consider a system with two sockets. Each socket has
- * three classes of memory attached: fast, medium and slow.
- * Each memory class is placed in its own NUMA node. The
- * CPUs are placed in the node with the "fast" memory. The
- * 6 NUMA nodes (0-5) might be split among the sockets like
- * this:
- *
- * Socket A: 0, 1, 2
- * Socket B: 3, 4, 5
- *
- * When Node 0 fills up, its memory should be migrated to
- * Node 1. When Node 1 fills up, it should be migrated to
- * Node 2. The migration path start on the nodes with the
- * processors (since allocations default to this node) and
- * fast memory, progress through medium and end with the
- * slow memory:
- *
- * 0 -> 1 -> 2 -> stop
- * 3 -> 4 -> 5 -> stop
- *
- * This is represented in the node_demotion[] like this:
- *
- * { nr=1, nodes[0]=1 }, // Node 0 migrates to 1
- * { nr=1, nodes[0]=2 }, // Node 1 migrates to 2
- * { nr=0, nodes[0]=-1 }, // Node 2 does not migrate
- * { nr=1, nodes[0]=4 }, // Node 3 migrates to 4
- * { nr=1, nodes[0]=5 }, // Node 4 migrates to 5
- * { nr=0, nodes[0]=-1 }, // Node 5 does not migrate
- *
- * Moreover some systems may have multiple slow memory nodes.
- * Suppose a system has one socket with 3 memory nodes, node 0
- * is fast memory type, and node 1/2 both are slow memory
- * type, and the distance between fast memory node and slow
- * memory node is same. So the migration path should be:
- *
- * 0 -> 1/2 -> stop
- *
- * This is represented in the node_demotion[] like this:
- * { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2
- * { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate
- * { nr=0, nodes[0]=-1, }, // Node 2 does not migrate
- */
-
-/*
- * Writes to this array occur without locking. Cycles are
- * not allowed: Node X demotes to Y which demotes to X...
- *
- * If multiple reads are performed, a single rcu_read_lock()
- * must be held over all reads to ensure that no cycles are
- * observed.
- */
-#define DEFAULT_DEMOTION_TARGET_NODES 15
-
-#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES
-#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1)
-#else
-#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES
-#endif
-
-struct demotion_nodes {
- unsigned short nr;
- short nodes[DEMOTION_TARGET_NODES];
-};
-
-static struct demotion_nodes *node_demotion __read_mostly;
-
-/**
- * next_demotion_node() - Get the next node in the demotion path
- * @node: The starting node to lookup the next node
- *
- * Return: node id for next memory node in the demotion path hierarchy
- * from @node; NUMA_NO_NODE if @node is terminal. This does not keep
- * @node online or guarantee that it *continues* to be the next demotion
- * target.
- */
-int next_demotion_node(int node)
-{
- struct demotion_nodes *nd;
- unsigned short target_nr, index;
- int target;
-
- if (!node_demotion)
- return NUMA_NO_NODE;
-
- nd = &node_demotion[node];
-
- /*
- * node_demotion[] is updated without excluding this
- * function from running. RCU doesn't provide any
- * compiler barriers, so the READ_ONCE() is required
- * to avoid compiler reordering or read merging.
- *
- * Make sure to use RCU over entire code blocks if
- * node_demotion[] reads need to be consistent.
- */
- rcu_read_lock();
- target_nr = READ_ONCE(nd->nr);
-
- switch (target_nr) {
- case 0:
- target = NUMA_NO_NODE;
- goto out;
- case 1:
- index = 0;
- break;
- default:
- /*
- * If there are multiple target nodes, just select one
- * target node randomly.
- *
- * In addition, we can also use round-robin to select
- * target node, but we should introduce another variable
- * for node_demotion[] to record last selected target node,
- * that may cause cache ping-pong due to the changing of
- * last target node. Or introducing per-cpu data to avoid
- * caching issue, which seems more complicated. So selecting
- * target node randomly seems better until now.
- */
- index = get_random_int() % target_nr;
- break;
- }
-
- target = READ_ONCE(nd->nodes[index]);
-
-out:
- rcu_read_unlock();
- return target;
-}
-
-/* Disable reclaim-based migration. */
-static void __disable_all_migrate_targets(void)
-{
- int node, i;
-
- if (!node_demotion)
- return;
-
- for_each_online_node(node) {
- node_demotion[node].nr = 0;
- for (i = 0; i < DEMOTION_TARGET_NODES; i++)
- node_demotion[node].nodes[i] = NUMA_NO_NODE;
- }
-}
-
-static void disable_all_migrate_targets(void)
-{
- __disable_all_migrate_targets();
-
- /*
- * Ensure that the "disable" is visible across the system.
- * Readers will see either a combination of before+disable
- * state or disable+after. They will never see before and
- * after state together.
- *
- * The before+after state together might have cycles and
- * could cause readers to do things like loop until this
- * function finishes. This ensures they can only see a
- * single "bad" read and would, for instance, only loop
- * once.
- */
- synchronize_rcu();
-}
-
-/*
- * Find an automatic demotion target for 'node'.
- * Failing here is OK. It might just indicate
- * being at the end of a chain.
- */
-static int establish_migrate_target(int node, nodemask_t *used,
- int best_distance)
-{
- int migration_target, index, val;
- struct demotion_nodes *nd;
-
- if (!node_demotion)
- return NUMA_NO_NODE;
-
- nd = &node_demotion[node];
-
- migration_target = find_next_best_node(node, used);
- if (migration_target == NUMA_NO_NODE)
- return NUMA_NO_NODE;
-
- /*
- * If the node has been set a migration target node before,
- * which means it's the best distance between them. Still
- * check if this node can be demoted to other target nodes
- * if they have a same best distance.
- */
- if (best_distance != -1) {
- val = node_distance(node, migration_target);
- if (val > best_distance)
- goto out_clear;
- }
-
- index = nd->nr;
- if (WARN_ONCE(index >= DEMOTION_TARGET_NODES,
- "Exceeds maximum demotion target nodes\n"))
- goto out_clear;
-
- nd->nodes[index] = migration_target;
- nd->nr++;
-
- return migration_target;
-out_clear:
- node_clear(migration_target, *used);
- return NUMA_NO_NODE;
-}
-
-/*
- * When memory fills up on a node, memory contents can be
- * automatically migrated to another node instead of
- * discarded at reclaim.
- *
- * Establish a "migration path" which will start at nodes
- * with CPUs and will follow the priorities used to build the
- * page allocator zonelists.
- *
- * The difference here is that cycles must be avoided. If
- * node0 migrates to node1, then neither node1, nor anything
- * node1 migrates to can migrate to node0. Also one node can
- * be migrated to multiple nodes if the target nodes all have
- * a same best-distance against the source node.
- *
- * This function can run simultaneously with readers of
- * node_demotion[]. However, it can not run simultaneously
- * with itself. Exclusion is provided by memory hotplug events
- * being single-threaded.
- */
-static void __set_migration_target_nodes(void)
-{
- nodemask_t next_pass;
- nodemask_t this_pass;
- nodemask_t used_targets = NODE_MASK_NONE;
- int node, best_distance;
-
- /*
- * Avoid any oddities like cycles that could occur
- * from changes in the topology. This will leave
- * a momentary gap when migration is disabled.
- */
- disable_all_migrate_targets();
-
- /*
- * Allocations go close to CPUs, first. Assume that
- * the migration path starts at the nodes with CPUs.
- */
- next_pass = node_states[N_CPU];
-again:
- this_pass = next_pass;
- next_pass = NODE_MASK_NONE;
- /*
- * To avoid cycles in the migration "graph", ensure
- * that migration sources are not future targets by
- * setting them in 'used_targets'. Do this only
- * once per pass so that multiple source nodes can
- * share a target node.
- *
- * 'used_targets' will become unavailable in future
- * passes. This limits some opportunities for
- * multiple source nodes to share a destination.
- */
- nodes_or(used_targets, used_targets, this_pass);
-
- for_each_node_mask(node, this_pass) {
- best_distance = -1;
-
- /*
- * Try to set up the migration path for the node, and the target
- * migration nodes can be multiple, so doing a loop to find all
- * the target nodes if they all have a best node distance.
- */
- do {
- int target_node =
- establish_migrate_target(node, &used_targets,
- best_distance);
-
- if (target_node == NUMA_NO_NODE)
- break;
-
- if (best_distance == -1)
- best_distance = node_distance(node, target_node);
-
- /*
- * Visit targets from this pass in the next pass.
- * Eventually, every node will have been part of
- * a pass, and will become set in 'used_targets'.
- */
- node_set(target_node, next_pass);
- } while (1);
- }
- /*
- * 'next_pass' contains nodes which became migration
- * targets in this pass. Make additional passes until
- * no more migrations targets are available.
- */
- if (!nodes_empty(next_pass))
- goto again;
-}
-
-/*
- * For callers that do not hold get_online_mems() already.
- */
-void set_migration_target_nodes(void)
-{
- get_online_mems();
- __set_migration_target_nodes();
- put_online_mems();
-}
-
-/*
- * This leaves migrate-on-reclaim transiently disabled between
- * the MEM_GOING_OFFLINE and MEM_OFFLINE events. This runs
- * whether reclaim-based migration is enabled or not, which
- * ensures that the user can turn reclaim-based migration at
- * any time without needing to recalculate migration targets.
- *
- * These callbacks already hold get_online_mems(). That is why
- * __set_migration_target_nodes() can be used as opposed to
- * set_migration_target_nodes().
- */
-#ifdef CONFIG_MEMORY_HOTPLUG
-static int __meminit migrate_on_reclaim_callback(struct notifier_block *self,
- unsigned long action, void *_arg)
-{
- struct memory_notify *arg = _arg;
-
- /*
- * Only update the node migration order when a node is
- * changing status, like online->offline. This avoids
- * the overhead of synchronize_rcu() in most cases.
- */
- if (arg->status_change_nid < 0)
- return notifier_from_errno(0);
-
- switch (action) {
- case MEM_GOING_OFFLINE:
- /*
- * Make sure there are not transient states where
- * an offline node is a migration target. This
- * will leave migration disabled until the offline
- * completes and the MEM_OFFLINE case below runs.
- */
- disable_all_migrate_targets();
- break;
- case MEM_OFFLINE:
- case MEM_ONLINE:
- /*
- * Recalculate the target nodes once the node
- * reaches its final state (online or offline).
- */
- __set_migration_target_nodes();
- break;
- case MEM_CANCEL_OFFLINE:
- /*
- * MEM_GOING_OFFLINE disabled all the migration
- * targets. Reenable them.
- */
- __set_migration_target_nodes();
- break;
- case MEM_GOING_ONLINE:
- case MEM_CANCEL_ONLINE:
- break;
- }
-
- return notifier_from_errno(0);
-}
-#endif
-
-void __init migrate_on_reclaim_init(void)
-{
- node_demotion = kcalloc(nr_node_ids,
- sizeof(struct demotion_nodes),
- GFP_KERNEL);
- WARN_ON(!node_demotion);
-#ifdef CONFIG_MEMORY_HOTPLUG
- hotplug_memory_notifier(migrate_on_reclaim_callback, 100);
-#endif
- /*
- * At this point, all numa nodes with memory/CPus have their state
- * properly set, so we can build the demotion order now.
- * Let us hold the cpu_hotplug lock just, as we could possibily have
- * CPU hotplug events during boot.
- */
- cpus_read_lock();
- set_migration_target_nodes();
- cpus_read_unlock();
-}
-
-bool numa_demotion_enabled = false;
-
-#ifdef CONFIG_SYSFS
-static ssize_t numa_demotion_enabled_show(struct kobject *kobj,
- struct kobj_attribute *attr, char *buf)
-{
- return sysfs_emit(buf, "%s\n",
- numa_demotion_enabled ? "true" : "false");
-}
-
-static ssize_t numa_demotion_enabled_store(struct kobject *kobj,
- struct kobj_attribute *attr,
- const char *buf, size_t count)
-{
- ssize_t ret;
-
- ret = kstrtobool(buf, &numa_demotion_enabled);
- if (ret)
- return ret;
-
- return count;
-}
-
-static struct kobj_attribute numa_demotion_enabled_attr =
- __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show,
- numa_demotion_enabled_store);
-
-static struct attribute *numa_attrs[] = {
- &numa_demotion_enabled_attr.attr,
- NULL,
-};
-
-static const struct attribute_group numa_attr_group = {
- .attrs = numa_attrs,
-};
-
-static int __init numa_init_sysfs(void)
-{
- int err;
- struct kobject *numa_kobj;
-
- numa_kobj = kobject_create_and_add("numa", mm_kobj);
- if (!numa_kobj) {
- pr_err("failed to create numa kobject\n");
- return -ENOMEM;
- }
- err = sysfs_create_group(numa_kobj, &numa_attr_group);
- if (err) {
- pr_err("failed to register numa group\n");
- goto delete_obj;
- }
- return 0;
-
-delete_obj:
- kobject_put(numa_kobj);
- return err;
-}
-subsys_initcall(numa_init_sysfs);
-#endif /* CONFIG_SYSFS */
#endif /* CONFIG_NUMA */
get_page(page);
/*
- * Optimize for the common case where page is only mapped once
- * in one process. If we can lock the page, then we can safely
- * set up a special migration page table entry now.
+ * We rely on trylock_page() to avoid deadlock between
+ * concurrent migrations where each is waiting on the others
+ * page lock. If we can't immediately lock the page we fail this
+ * migration as it is only best effort anyway.
+ *
+ * If we can lock the page it's safe to set up a migration entry
+ * now. In the common case where the page is mapped once in a
+ * single process setting up the migration entry now is an
+ * optimisation to avoid walking the rmap later with
+ * try_to_migrate().
*/
if (trylock_page(page)) {
bool anon_exclusive;
else
entry = make_readable_migration_entry(
page_to_pfn(page));
+ if (pte_present(pte)) {
+ if (pte_young(pte))
+ entry = make_migration_entry_young(entry);
+ if (pte_dirty(pte))
+ entry = make_migration_entry_dirty(entry);
+ }
swp_pte = swp_entry_to_pte(entry);
if (pte_present(pte)) {
if (pte_soft_dirty(pte))
* folio_migrate_mapping(), except that here we allow migration of a
* ZONE_DEVICE page.
*/
-static bool migrate_vma_check_page(struct page *page)
+static bool migrate_vma_check_page(struct page *page, struct page *fault_page)
{
/*
* One extra ref because caller holds an extra reference, either from
* isolate_lru_page() for a regular page, or migrate_vma_collect() for
* a device page.
*/
- int extra = 1;
+ int extra = 1 + (page == fault_page);
/*
* FIXME support THP (transparent huge page), it is bit more complex to
}
/*
- * migrate_vma_unmap() - replace page mapping with special migration pte entry
- * @migrate: migrate struct containing all migration information
- *
- * Isolate pages from the LRU and replace mappings (CPU page table pte) with a
- * special migration pte entry and check if it has been pinned. Pinned pages are
- * restored because we cannot migrate them.
- *
- * This is the last step before we call the device driver callback to allocate
- * destination memory and copy contents of original page over to new page.
+ * Unmaps pages for migration. Returns number of unmapped pages.
*/
-static void migrate_vma_unmap(struct migrate_vma *migrate)
+static unsigned long migrate_device_unmap(unsigned long *src_pfns,
+ unsigned long npages,
+ struct page *fault_page)
{
- const unsigned long npages = migrate->npages;
unsigned long i, restore = 0;
bool allow_drain = true;
+ unsigned long unmapped = 0;
lru_add_drain();
for (i = 0; i < npages; i++) {
- struct page *page = migrate_pfn_to_page(migrate->src[i]);
+ struct page *page = migrate_pfn_to_page(src_pfns[i]);
struct folio *folio;
if (!page)
}
if (isolate_lru_page(page)) {
- migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
- migrate->cpages--;
+ src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
restore++;
continue;
}
if (folio_mapped(folio))
try_to_migrate(folio, 0);
- if (page_mapped(page) || !migrate_vma_check_page(page)) {
+ if (page_mapped(page) ||
+ !migrate_vma_check_page(page, fault_page)) {
if (!is_zone_device_page(page)) {
get_page(page);
putback_lru_page(page);
}
- migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
- migrate->cpages--;
+ src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
restore++;
continue;
}
+
+ unmapped++;
}
for (i = 0; i < npages && restore; i++) {
- struct page *page = migrate_pfn_to_page(migrate->src[i]);
+ struct page *page = migrate_pfn_to_page(src_pfns[i]);
struct folio *folio;
- if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
+ if (!page || (src_pfns[i] & MIGRATE_PFN_MIGRATE))
continue;
folio = page_folio(page);
remove_migration_ptes(folio, folio, false);
- migrate->src[i] = 0;
+ src_pfns[i] = 0;
folio_unlock(folio);
folio_put(folio);
restore--;
}
+
+ return unmapped;
+}
+
+/*
+ * migrate_vma_unmap() - replace page mapping with special migration pte entry
+ * @migrate: migrate struct containing all migration information
+ *
+ * Isolate pages from the LRU and replace mappings (CPU page table pte) with a
+ * special migration pte entry and check if it has been pinned. Pinned pages are
+ * restored because we cannot migrate them.
+ *
+ * This is the last step before we call the device driver callback to allocate
+ * destination memory and copy contents of original page over to new page.
+ */
+static void migrate_vma_unmap(struct migrate_vma *migrate)
+{
+ migrate->cpages = migrate_device_unmap(migrate->src, migrate->npages,
+ migrate->fault_page);
}
/**
return -EINVAL;
if (!args->src || !args->dst)
return -EINVAL;
+ if (args->fault_page && !is_device_private_page(args->fault_page))
+ return -EINVAL;
memset(args->src, 0, sizeof(*args->src) * nr_pages);
args->cpages = 0;
*src &= ~MIGRATE_PFN_MIGRATE;
}
-/**
- * migrate_vma_pages() - migrate meta-data from src page to dst page
- * @migrate: migrate struct containing all migration information
- *
- * This migrates struct page meta-data from source struct page to destination
- * struct page. This effectively finishes the migration from source page to the
- * destination page.
- */
-void migrate_vma_pages(struct migrate_vma *migrate)
+static void __migrate_device_pages(unsigned long *src_pfns,
+ unsigned long *dst_pfns, unsigned long npages,
+ struct migrate_vma *migrate)
{
- const unsigned long npages = migrate->npages;
- const unsigned long start = migrate->start;
struct mmu_notifier_range range;
- unsigned long addr, i;
+ unsigned long i;
bool notified = false;
- for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
- struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
- struct page *page = migrate_pfn_to_page(migrate->src[i]);
+ for (i = 0; i < npages; i++) {
+ struct page *newpage = migrate_pfn_to_page(dst_pfns[i]);
+ struct page *page = migrate_pfn_to_page(src_pfns[i]);
struct address_space *mapping;
int r;
if (!newpage) {
- migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
+ src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
if (!page) {
+ unsigned long addr;
+
+ if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE))
+ continue;
+
/*
* The only time there is no vma is when called from
* migrate_device_coherent_page(). However this isn't
* called if the page could not be unmapped.
*/
- VM_BUG_ON(!migrate->vma);
- if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
- continue;
+ VM_BUG_ON(!migrate);
+ addr = migrate->start + i*PAGE_SIZE;
if (!notified) {
notified = true;
mmu_notifier_invalidate_range_start(&range);
}
migrate_vma_insert_page(migrate, addr, newpage,
- &migrate->src[i]);
+ &src_pfns[i]);
continue;
}
* device private or coherent memory.
*/
if (mapping) {
- migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
+ src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
} else if (is_zone_device_page(newpage)) {
/*
* Other types of ZONE_DEVICE page are not supported.
*/
- migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
+ src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
- r = migrate_folio(mapping, page_folio(newpage),
- page_folio(page), MIGRATE_SYNC_NO_COPY);
+ if (migrate && migrate->fault_page == page)
+ r = migrate_folio_extra(mapping, page_folio(newpage),
+ page_folio(page),
+ MIGRATE_SYNC_NO_COPY, 1);
+ else
+ r = migrate_folio(mapping, page_folio(newpage),
+ page_folio(page), MIGRATE_SYNC_NO_COPY);
if (r != MIGRATEPAGE_SUCCESS)
- migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
+ src_pfns[i] &= ~MIGRATE_PFN_MIGRATE;
}
/*
if (notified)
mmu_notifier_invalidate_range_only_end(&range);
}
-EXPORT_SYMBOL(migrate_vma_pages);
/**
- * migrate_vma_finalize() - restore CPU page table entry
+ * migrate_device_pages() - migrate meta-data from src page to dst page
+ * @src_pfns: src_pfns returned from migrate_device_range()
+ * @dst_pfns: array of pfns allocated by the driver to migrate memory to
+ * @npages: number of pages in the range
+ *
+ * Equivalent to migrate_vma_pages(). This is called to migrate struct page
+ * meta-data from source struct page to destination.
+ */
+void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns,
+ unsigned long npages)
+{
+ __migrate_device_pages(src_pfns, dst_pfns, npages, NULL);
+}
+EXPORT_SYMBOL(migrate_device_pages);
+
+/**
+ * migrate_vma_pages() - migrate meta-data from src page to dst page
* @migrate: migrate struct containing all migration information
*
- * This replaces the special migration pte entry with either a mapping to the
- * new page if migration was successful for that page, or to the original page
- * otherwise.
+ * This migrates struct page meta-data from source struct page to destination
+ * struct page. This effectively finishes the migration from source page to the
+ * destination page.
+ */
+void migrate_vma_pages(struct migrate_vma *migrate)
+{
+ __migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate);
+}
+EXPORT_SYMBOL(migrate_vma_pages);
+
+/*
+ * migrate_device_finalize() - complete page migration
+ * @src_pfns: src_pfns returned from migrate_device_range()
+ * @dst_pfns: array of pfns allocated by the driver to migrate memory to
+ * @npages: number of pages in the range
*
- * This also unlocks the pages and puts them back on the lru, or drops the extra
- * refcount, for device pages.
+ * Completes migration of the page by removing special migration entries.
+ * Drivers must ensure copying of page data is complete and visible to the CPU
+ * before calling this.
*/
-void migrate_vma_finalize(struct migrate_vma *migrate)
+void migrate_device_finalize(unsigned long *src_pfns,
+ unsigned long *dst_pfns, unsigned long npages)
{
- const unsigned long npages = migrate->npages;
unsigned long i;
for (i = 0; i < npages; i++) {
struct folio *dst, *src;
- struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
- struct page *page = migrate_pfn_to_page(migrate->src[i]);
+ struct page *newpage = migrate_pfn_to_page(dst_pfns[i]);
+ struct page *page = migrate_pfn_to_page(src_pfns[i]);
if (!page) {
if (newpage) {
continue;
}
- if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
+ if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
if (newpage) {
unlock_page(newpage);
put_page(newpage);
}
}
}
+EXPORT_SYMBOL(migrate_device_finalize);
+
+/**
+ * migrate_vma_finalize() - restore CPU page table entry
+ * @migrate: migrate struct containing all migration information
+ *
+ * This replaces the special migration pte entry with either a mapping to the
+ * new page if migration was successful for that page, or to the original page
+ * otherwise.
+ *
+ * This also unlocks the pages and puts them back on the lru, or drops the extra
+ * refcount, for device pages.
+ */
+void migrate_vma_finalize(struct migrate_vma *migrate)
+{
+ migrate_device_finalize(migrate->src, migrate->dst, migrate->npages);
+}
EXPORT_SYMBOL(migrate_vma_finalize);
+/**
+ * migrate_device_range() - migrate device private pfns to normal memory.
+ * @src_pfns: array large enough to hold migrating source device private pfns.
+ * @start: starting pfn in the range to migrate.
+ * @npages: number of pages to migrate.
+ *
+ * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that
+ * instead of looking up pages based on virtual address mappings a range of
+ * device pfns that should be migrated to system memory is used instead.
+ *
+ * This is useful when a driver needs to free device memory but doesn't know the
+ * virtual mappings of every page that may be in device memory. For example this
+ * is often the case when a driver is being unloaded or unbound from a device.
+ *
+ * Like migrate_vma_setup() this function will take a reference and lock any
+ * migrating pages that aren't free before unmapping them. Drivers may then
+ * allocate destination pages and start copying data from the device to CPU
+ * memory before calling migrate_device_pages().
+ */
+int migrate_device_range(unsigned long *src_pfns, unsigned long start,
+ unsigned long npages)
+{
+ unsigned long i, pfn;
+
+ for (pfn = start, i = 0; i < npages; pfn++, i++) {
+ struct page *page = pfn_to_page(pfn);
+
+ if (!get_page_unless_zero(page)) {
+ src_pfns[i] = 0;
+ continue;
+ }
+
+ if (!trylock_page(page)) {
+ src_pfns[i] = 0;
+ put_page(page);
+ continue;
+ }
+
+ src_pfns[i] = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
+ }
+
+ migrate_device_unmap(src_pfns, npages, NULL);
+
+ return 0;
+}
+EXPORT_SYMBOL(migrate_device_range);
+
/*
* Migrate a device coherent page back to normal memory. The caller should have
* a reference on page which will be copied to the new page if migration is
int migrate_device_coherent_page(struct page *page)
{
unsigned long src_pfn, dst_pfn = 0;
- struct migrate_vma args;
struct page *dpage;
WARN_ON_ONCE(PageCompound(page));
lock_page(page);
src_pfn = migrate_pfn(page_to_pfn(page)) | MIGRATE_PFN_MIGRATE;
- args.src = &src_pfn;
- args.dst = &dst_pfn;
- args.cpages = 1;
- args.npages = 1;
- args.vma = NULL;
/*
* We don't have a VMA and don't need to walk the page tables to find
* the source page. So call migrate_vma_unmap() directly to unmap the
* page as migrate_vma_setup() will fail if args.vma == NULL.
*/
- migrate_vma_unmap(&args);
+ migrate_device_unmap(&src_pfn, 1, NULL);
if (!(src_pfn & MIGRATE_PFN_MIGRATE))
return -EBUSY;
dst_pfn = migrate_pfn(page_to_pfn(dpage));
}
- migrate_vma_pages(&args);
+ migrate_device_pages(&src_pfn, &dst_pfn, 1);
if (src_pfn & MIGRATE_PFN_MIGRATE)
copy_highpage(dpage, page);
- migrate_vma_finalize(&args);
+ migrate_device_finalize(&src_pfn, &dst_pfn, 1);
if (src_pfn & MIGRATE_PFN_MIGRATE)
return 0;
unsigned long nstart, end, tmp;
struct vm_area_struct *vma, *prev;
int error;
+ MA_STATE(mas, ¤t->mm->mm_mt, start, start);
VM_BUG_ON(offset_in_page(start));
VM_BUG_ON(len != PAGE_ALIGN(len));
return -EINVAL;
if (end == start)
return 0;
- vma = find_vma(current->mm, start);
- if (!vma || vma->vm_start > start)
+ vma = mas_walk(&mas);
+ if (!vma)
return -ENOMEM;
- prev = vma->vm_prev;
if (start > vma->vm_start)
prev = vma;
+ else
+ prev = mas_prev(&mas, 0);
for (nstart = start ; ; ) {
vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
if (nstart >= end)
break;
- vma = prev->vm_next;
+ vma = find_vma(prev->vm_mm, prev->vm_end);
if (!vma || vma->vm_start != nstart) {
error = -ENOMEM;
break;
{
struct vm_area_struct *vma;
unsigned long count = 0;
+ unsigned long end;
+ VMA_ITERATOR(vmi, mm, start);
- if (mm == NULL)
- mm = current->mm;
+ /* Don't overflow past ULONG_MAX */
+ if (unlikely(ULONG_MAX - len < start))
+ end = ULONG_MAX;
+ else
+ end = start + len;
- vma = find_vma(mm, start);
- if (vma == NULL)
- return 0;
-
- for (; vma ; vma = vma->vm_next) {
- if (start >= vma->vm_end)
- continue;
- if (start + len <= vma->vm_start)
- break;
+ for_each_vma_range(vmi, vma, end) {
if (vma->vm_flags & VM_LOCKED) {
if (start > vma->vm_start)
count -= (start - vma->vm_start);
- if (start + len < vma->vm_end) {
- count += start + len - vma->vm_start;
+ if (end < vma->vm_end) {
+ count += end - vma->vm_start;
break;
}
count += vma->vm_end - vma->vm_start;
*/
static int apply_mlockall_flags(int flags)
{
+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
struct vm_area_struct *vma, *prev = NULL;
vm_flags_t to_add = 0;
to_add |= VM_LOCKONFAULT;
}
- for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
vm_flags_t newflags;
newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
/* Ignore errors */
mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
+ mas_pause(&mas);
cond_resched();
}
out:
shift = 8 * sizeof(unsigned long);
width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
- - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH;
+ - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
- "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n",
+ "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
SECTIONS_WIDTH,
NODES_WIDTH,
ZONES_WIDTH,
LAST_CPUPID_WIDTH,
KASAN_TAG_WIDTH,
+ LRU_GEN_WIDTH,
+ LRU_REFS_WIDTH,
NR_PAGEFLAGS);
mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+#ifndef _LINUX_MM_SLOT_H
+#define _LINUX_MM_SLOT_H
+
+#include <linux/hashtable.h>
+#include <linux/slab.h>
+
+/*
+ * struct mm_slot - hash lookup from mm to mm_slot
+ * @hash: link to the mm_slots hash list
+ * @mm_node: link into the mm_slots list
+ * @mm: the mm that this information is valid for
+ */
+struct mm_slot {
+ struct hlist_node hash;
+ struct list_head mm_node;
+ struct mm_struct *mm;
+};
+
+#define mm_slot_entry(ptr, type, member) \
+ container_of(ptr, type, member)
+
+static inline void *mm_slot_alloc(struct kmem_cache *cache)
+{
+ if (!cache) /* initialization failed */
+ return NULL;
+ return kmem_cache_zalloc(cache, GFP_KERNEL);
+}
+
+static inline void mm_slot_free(struct kmem_cache *cache, void *objp)
+{
+ kmem_cache_free(cache, objp);
+}
+
+#define mm_slot_lookup(_hashtable, _mm) \
+({ \
+ struct mm_slot *tmp_slot, *mm_slot = NULL; \
+ \
+ hash_for_each_possible(_hashtable, tmp_slot, hash, (unsigned long)_mm) \
+ if (_mm == tmp_slot->mm) { \
+ mm_slot = tmp_slot; \
+ break; \
+ } \
+ \
+ mm_slot; \
+})
+
+#define mm_slot_insert(_hashtable, _mm, _mm_slot) \
+({ \
+ _mm_slot->mm = _mm; \
+ hash_add(_hashtable, &_mm_slot->hash, (unsigned long)_mm); \
+})
+
+#endif /* _LINUX_MM_SLOT_H */
#include <linux/backing-dev.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
-#include <linux/vmacache.h>
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/audit.h>
#include <linux/khugepaged.h>
#include <linux/uprobes.h>
-#include <linux/rbtree_augmented.h>
#include <linux/notifier.h>
#include <linux/memory.h>
#include <linux/printk.h>
static bool ignore_rlimit_data;
core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
-static void unmap_region(struct mm_struct *mm,
+static void unmap_region(struct mm_struct *mm, struct maple_tree *mt,
struct vm_area_struct *vma, struct vm_area_struct *prev,
- unsigned long start, unsigned long end);
+ struct vm_area_struct *next, unsigned long start,
+ unsigned long end);
static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
{
}
/*
- * Close a vm structure and free it, returning the next.
+ * Close a vm structure and free it.
*/
-static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
+static void remove_vma(struct vm_area_struct *vma)
{
- struct vm_area_struct *next = vma->vm_next;
-
might_sleep();
if (vma->vm_ops && vma->vm_ops->close)
vma->vm_ops->close(vma);
fput(vma->vm_file);
mpol_put(vma_policy(vma));
vm_area_free(vma);
- return next;
}
-static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags,
- struct list_head *uf);
+/*
+ * check_brk_limits() - Use platform specific check of range & verify mlock
+ * limits.
+ * @addr: The address to check
+ * @len: The size of increase.
+ *
+ * Return: 0 on success.
+ */
+static int check_brk_limits(unsigned long addr, unsigned long len)
+{
+ unsigned long mapped_addr;
+
+ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+ if (IS_ERR_VALUE(mapped_addr))
+ return mapped_addr;
+
+ return mlock_future_check(current->mm, current->mm->def_flags, len);
+}
+static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
+ unsigned long newbrk, unsigned long oldbrk,
+ struct list_head *uf);
+static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma,
+ unsigned long addr, unsigned long request, unsigned long flags);
SYSCALL_DEFINE1(brk, unsigned long, brk)
{
unsigned long newbrk, oldbrk, origbrk;
struct mm_struct *mm = current->mm;
- struct vm_area_struct *next;
+ struct vm_area_struct *brkvma, *next = NULL;
unsigned long min_brk;
bool populate;
bool downgraded = false;
LIST_HEAD(uf);
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
if (mmap_write_lock_killable(mm))
return -EINTR;
/*
* Always allow shrinking brk.
- * __do_munmap() may downgrade mmap_lock to read.
+ * do_brk_munmap() may downgrade mmap_lock to read.
*/
if (brk <= mm->brk) {
int ret;
+ /* Search one past newbrk */
+ mas_set(&mas, newbrk);
+ brkvma = mas_find(&mas, oldbrk);
+ BUG_ON(brkvma == NULL);
+ if (brkvma->vm_start >= oldbrk)
+ goto out; /* mapping intersects with an existing non-brk vma. */
/*
- * mm->brk must to be protected by write mmap_lock so update it
- * before downgrading mmap_lock. When __do_munmap() fails,
- * mm->brk will be restored from origbrk.
+ * mm->brk must be protected by write mmap_lock.
+ * do_brk_munmap() may downgrade the lock, so update it
+ * before calling do_brk_munmap().
*/
mm->brk = brk;
- ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true);
- if (ret < 0) {
- mm->brk = origbrk;
- goto out;
- } else if (ret == 1) {
+ ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf);
+ if (ret == 1) {
downgraded = true;
- }
- goto success;
+ goto success;
+ } else if (!ret)
+ goto success;
+
+ mm->brk = origbrk;
+ goto out;
}
- /* Check against existing mmap mappings. */
- next = find_vma(mm, oldbrk);
+ if (check_brk_limits(oldbrk, newbrk - oldbrk))
+ goto out;
+
+ /*
+ * Only check if the next VMA is within the stack_guard_gap of the
+ * expansion area
+ */
+ mas_set(&mas, oldbrk);
+ next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap);
if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
goto out;
+ brkvma = mas_prev(&mas, mm->start_brk);
/* Ok, looks good - let it rip. */
- if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0)
+ if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
goto out;
+
mm->brk = brk;
success:
return origbrk;
}
-static inline unsigned long vma_compute_gap(struct vm_area_struct *vma)
-{
- unsigned long gap, prev_end;
-
- /*
- * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we
- * allow two stack_guard_gaps between them here, and when choosing
- * an unmapped area; whereas when expanding we only require one.
- * That's a little inconsistent, but keeps the code here simpler.
- */
- gap = vm_start_gap(vma);
- if (vma->vm_prev) {
- prev_end = vm_end_gap(vma->vm_prev);
- if (gap > prev_end)
- gap -= prev_end;
- else
- gap = 0;
- }
- return gap;
-}
-
-#ifdef CONFIG_DEBUG_VM_RB
-static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma)
-{
- unsigned long max = vma_compute_gap(vma), subtree_gap;
- if (vma->vm_rb.rb_left) {
- subtree_gap = rb_entry(vma->vm_rb.rb_left,
- struct vm_area_struct, vm_rb)->rb_subtree_gap;
- if (subtree_gap > max)
- max = subtree_gap;
- }
- if (vma->vm_rb.rb_right) {
- subtree_gap = rb_entry(vma->vm_rb.rb_right,
- struct vm_area_struct, vm_rb)->rb_subtree_gap;
- if (subtree_gap > max)
- max = subtree_gap;
- }
- return max;
-}
-
-static int browse_rb(struct mm_struct *mm)
-{
- struct rb_root *root = &mm->mm_rb;
- int i = 0, j, bug = 0;
- struct rb_node *nd, *pn = NULL;
- unsigned long prev = 0, pend = 0;
-
- for (nd = rb_first(root); nd; nd = rb_next(nd)) {
- struct vm_area_struct *vma;
- vma = rb_entry(nd, struct vm_area_struct, vm_rb);
- if (vma->vm_start < prev) {
- pr_emerg("vm_start %lx < prev %lx\n",
- vma->vm_start, prev);
- bug = 1;
- }
- if (vma->vm_start < pend) {
- pr_emerg("vm_start %lx < pend %lx\n",
- vma->vm_start, pend);
- bug = 1;
- }
- if (vma->vm_start > vma->vm_end) {
- pr_emerg("vm_start %lx > vm_end %lx\n",
- vma->vm_start, vma->vm_end);
- bug = 1;
- }
- spin_lock(&mm->page_table_lock);
- if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
- pr_emerg("free gap %lx, correct %lx\n",
- vma->rb_subtree_gap,
- vma_compute_subtree_gap(vma));
- bug = 1;
+#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
+extern void mt_validate(struct maple_tree *mt);
+extern void mt_dump(const struct maple_tree *mt);
+
+/* Validate the maple tree */
+static void validate_mm_mt(struct mm_struct *mm)
+{
+ struct maple_tree *mt = &mm->mm_mt;
+ struct vm_area_struct *vma_mt;
+
+ MA_STATE(mas, mt, 0, 0);
+
+ mt_validate(&mm->mm_mt);
+ mas_for_each(&mas, vma_mt, ULONG_MAX) {
+ if ((vma_mt->vm_start != mas.index) ||
+ (vma_mt->vm_end - 1 != mas.last)) {
+ pr_emerg("issue in %s\n", current->comm);
+ dump_stack();
+ dump_vma(vma_mt);
+ pr_emerg("mt piv: %p %lu - %lu\n", vma_mt,
+ mas.index, mas.last);
+ pr_emerg("mt vma: %p %lu - %lu\n", vma_mt,
+ vma_mt->vm_start, vma_mt->vm_end);
+
+ mt_dump(mas.tree);
+ if (vma_mt->vm_end != mas.last + 1) {
+ pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n",
+ mm, vma_mt->vm_start, vma_mt->vm_end,
+ mas.index, mas.last);
+ mt_dump(mas.tree);
+ }
+ VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm);
+ if (vma_mt->vm_start != mas.index) {
+ pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n",
+ mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end);
+ mt_dump(mas.tree);
+ }
+ VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm);
}
- spin_unlock(&mm->page_table_lock);
- i++;
- pn = nd;
- prev = vma->vm_start;
- pend = vma->vm_end;
- }
- j = 0;
- for (nd = pn; nd; nd = rb_prev(nd))
- j++;
- if (i != j) {
- pr_emerg("backwards %d, forwards %d\n", j, i);
- bug = 1;
- }
- return bug ? -1 : i;
-}
-
-static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
-{
- struct rb_node *nd;
-
- for (nd = rb_first(root); nd; nd = rb_next(nd)) {
- struct vm_area_struct *vma;
- vma = rb_entry(nd, struct vm_area_struct, vm_rb);
- VM_BUG_ON_VMA(vma != ignore &&
- vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
- vma);
}
}
{
int bug = 0;
int i = 0;
- unsigned long highest_address = 0;
- struct vm_area_struct *vma = mm->mmap;
+ struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
- while (vma) {
+ validate_mm_mt(mm);
+
+ mas_for_each(&mas, vma, ULONG_MAX) {
+#ifdef CONFIG_DEBUG_VM_RB
struct anon_vma *anon_vma = vma->anon_vma;
struct anon_vma_chain *avc;
anon_vma_interval_tree_verify(avc);
anon_vma_unlock_read(anon_vma);
}
-
- highest_address = vm_end_gap(vma);
- vma = vma->vm_next;
+#endif
i++;
}
if (i != mm->map_count) {
- pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
- bug = 1;
- }
- if (highest_address != mm->highest_vm_end) {
- pr_emerg("mm->highest_vm_end %lx, found %lx\n",
- mm->highest_vm_end, highest_address);
- bug = 1;
- }
- i = browse_rb(mm);
- if (i != mm->map_count) {
- if (i != -1)
- pr_emerg("map_count %d rb %d\n", mm->map_count, i);
+ pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i);
bug = 1;
}
VM_BUG_ON_MM(bug, mm);
}
-#else
-#define validate_mm_rb(root, ignore) do { } while (0)
-#define validate_mm(mm) do { } while (0)
-#endif
-
-RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks,
- struct vm_area_struct, vm_rb,
- unsigned long, rb_subtree_gap, vma_compute_gap)
-
-/*
- * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
- * vma->vm_prev->vm_end values changed, without modifying the vma's position
- * in the rbtree.
- */
-static void vma_gap_update(struct vm_area_struct *vma)
-{
- /*
- * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created
- * a callback function that does exactly what we want.
- */
- vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
-}
-
-static inline void vma_rb_insert(struct vm_area_struct *vma,
- struct rb_root *root)
-{
- /* All rb_subtree_gap values must be consistent prior to insertion */
- validate_mm_rb(root, NULL);
-
- rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
-}
-
-static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
-{
- /*
- * Note rb_erase_augmented is a fairly large inline function,
- * so make sure we instantiate it only once with our desired
- * augmented rbtree callbacks.
- */
- rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
-}
-
-static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma,
- struct rb_root *root,
- struct vm_area_struct *ignore)
-{
- /*
- * All rb_subtree_gap values must be consistent prior to erase,
- * with the possible exception of
- *
- * a. the "next" vma being erased if next->vm_start was reduced in
- * __vma_adjust() -> __vma_unlink()
- * b. the vma being erased in detach_vmas_to_be_unmapped() ->
- * vma_rb_erase()
- */
- validate_mm_rb(root, ignore);
-
- __vma_rb_erase(vma, root);
-}
-static __always_inline void vma_rb_erase(struct vm_area_struct *vma,
- struct rb_root *root)
-{
- vma_rb_erase_ignore(vma, root, vma);
-}
+#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */
+#define validate_mm_mt(root) do { } while (0)
+#define validate_mm(mm) do { } while (0)
+#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
/*
* vma has some anon_vma assigned, and is already inserted on that
anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
}
-static int find_vma_links(struct mm_struct *mm, unsigned long addr,
- unsigned long end, struct vm_area_struct **pprev,
- struct rb_node ***rb_link, struct rb_node **rb_parent)
+static unsigned long count_vma_pages_range(struct mm_struct *mm,
+ unsigned long addr, unsigned long end)
{
- struct rb_node **__rb_link, *__rb_parent, *rb_prev;
+ VMA_ITERATOR(vmi, mm, addr);
+ struct vm_area_struct *vma;
+ unsigned long nr_pages = 0;
- mmap_assert_locked(mm);
- __rb_link = &mm->mm_rb.rb_node;
- rb_prev = __rb_parent = NULL;
+ for_each_vma_range(vmi, vma, end) {
+ unsigned long vm_start = max(addr, vma->vm_start);
+ unsigned long vm_end = min(end, vma->vm_end);
- while (*__rb_link) {
- struct vm_area_struct *vma_tmp;
+ nr_pages += PHYS_PFN(vm_end - vm_start);
+ }
- __rb_parent = *__rb_link;
- vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
+ return nr_pages;
+}
- if (vma_tmp->vm_end > addr) {
- /* Fail if an existing vma overlaps the area */
- if (vma_tmp->vm_start < end)
- return -ENOMEM;
- __rb_link = &__rb_parent->rb_left;
- } else {
- rb_prev = __rb_parent;
- __rb_link = &__rb_parent->rb_right;
- }
- }
+static void __vma_link_file(struct vm_area_struct *vma,
+ struct address_space *mapping)
+{
+ if (vma->vm_flags & VM_SHARED)
+ mapping_allow_writable(mapping);
- *pprev = NULL;
- if (rb_prev)
- *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
- *rb_link = __rb_link;
- *rb_parent = __rb_parent;
- return 0;
+ flush_dcache_mmap_lock(mapping);
+ vma_interval_tree_insert(vma, &mapping->i_mmap);
+ flush_dcache_mmap_unlock(mapping);
}
/*
- * vma_next() - Get the next VMA.
- * @mm: The mm_struct.
- * @vma: The current vma.
+ * vma_mas_store() - Store a VMA in the maple tree.
+ * @vma: The vm_area_struct
+ * @mas: The maple state
*
- * If @vma is NULL, return the first vma in the mm.
+ * Efficient way to store a VMA in the maple tree when the @mas has already
+ * walked to the correct location.
*
- * Returns: The next VMA after @vma.
+ * Note: the end address is inclusive in the maple tree.
*/
-static inline struct vm_area_struct *vma_next(struct mm_struct *mm,
- struct vm_area_struct *vma)
+void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas)
{
- if (!vma)
- return mm->mmap;
-
- return vma->vm_next;
+ trace_vma_store(mas->tree, vma);
+ mas_set_range(mas, vma->vm_start, vma->vm_end - 1);
+ mas_store_prealloc(mas, vma);
}
/*
- * munmap_vma_range() - munmap VMAs that overlap a range.
- * @mm: The mm struct
- * @start: The start of the range.
- * @len: The length of the range.
- * @pprev: pointer to the pointer that will be set to previous vm_area_struct
- * @rb_link: the rb_node
- * @rb_parent: the parent rb_node
- *
- * Find all the vm_area_struct that overlap from @start to
- * @end and munmap them. Set @pprev to the previous vm_area_struct.
+ * vma_mas_remove() - Remove a VMA from the maple tree.
+ * @vma: The vm_area_struct
+ * @mas: The maple state
*
- * Returns: -ENOMEM on munmap failure or 0 on success.
+ * Efficient way to remove a VMA from the maple tree when the @mas has already
+ * been established and points to the correct location.
+ * Note: the end address is inclusive in the maple tree.
*/
-static inline int
-munmap_vma_range(struct mm_struct *mm, unsigned long start, unsigned long len,
- struct vm_area_struct **pprev, struct rb_node ***link,
- struct rb_node **parent, struct list_head *uf)
+void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas)
{
-
- while (find_vma_links(mm, start, start + len, pprev, link, parent))
- if (do_munmap(mm, start, len, uf))
- return -ENOMEM;
-
- return 0;
+ trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1);
+ mas->index = vma->vm_start;
+ mas->last = vma->vm_end - 1;
+ mas_store_prealloc(mas, NULL);
}
-static unsigned long count_vma_pages_range(struct mm_struct *mm,
- unsigned long addr, unsigned long end)
+
+/*
+ * vma_mas_szero() - Set a given range to zero. Used when modifying a
+ * vm_area_struct start or end.
+ *
+ * @mm: The struct_mm
+ * @start: The start address to zero
+ * @end: The end address to zero.
+ */
+static inline void vma_mas_szero(struct ma_state *mas, unsigned long start,
+ unsigned long end)
{
- unsigned long nr_pages = 0;
- struct vm_area_struct *vma;
+ trace_vma_mas_szero(mas->tree, start, end - 1);
+ mas_set_range(mas, start, end - 1);
+ mas_store_prealloc(mas, NULL);
+}
- /* Find first overlapping mapping */
- vma = find_vma_intersection(mm, addr, end);
- if (!vma)
- return 0;
+static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
+{
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+ struct address_space *mapping = NULL;
- nr_pages = (min(end, vma->vm_end) -
- max(addr, vma->vm_start)) >> PAGE_SHIFT;
+ if (mas_preallocate(&mas, vma, GFP_KERNEL))
+ return -ENOMEM;
- /* Iterate over the rest of the overlaps */
- for (vma = vma->vm_next; vma; vma = vma->vm_next) {
- unsigned long overlap_len;
+ if (vma->vm_file) {
+ mapping = vma->vm_file->f_mapping;
+ i_mmap_lock_write(mapping);
+ }
- if (vma->vm_start > end)
- break;
+ vma_mas_store(vma, &mas);
- overlap_len = min(end, vma->vm_end) - vma->vm_start;
- nr_pages += overlap_len >> PAGE_SHIFT;
+ if (mapping) {
+ __vma_link_file(vma, mapping);
+ i_mmap_unlock_write(mapping);
}
- return nr_pages;
+ mm->map_count++;
+ validate_mm(mm);
+ return 0;
}
-void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
- struct rb_node **rb_link, struct rb_node *rb_parent)
+/*
+ * vma_expand - Expand an existing VMA
+ *
+ * @mas: The maple state
+ * @vma: The vma to expand
+ * @start: The start of the vma
+ * @end: The exclusive end of the vma
+ * @pgoff: The page offset of vma
+ * @next: The current of next vma.
+ *
+ * Expand @vma to @start and @end. Can expand off the start and end. Will
+ * expand over @next if it's different from @vma and @end == @next->vm_end.
+ * Checking if the @vma can expand and merge with @next needs to be handled by
+ * the caller.
+ *
+ * Returns: 0 on success
+ */
+inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma,
+ unsigned long start, unsigned long end, pgoff_t pgoff,
+ struct vm_area_struct *next)
{
- /* Update tracking information for the gap following the new vma. */
- if (vma->vm_next)
- vma_gap_update(vma->vm_next);
- else
- mm->highest_vm_end = vm_end_gap(vma);
+ struct mm_struct *mm = vma->vm_mm;
+ struct address_space *mapping = NULL;
+ struct rb_root_cached *root = NULL;
+ struct anon_vma *anon_vma = vma->anon_vma;
+ struct file *file = vma->vm_file;
+ bool remove_next = false;
- /*
- * vma->vm_prev wasn't known when we followed the rbtree to find the
- * correct insertion point for that vma. As a result, we could not
- * update the vma vm_rb parents rb_subtree_gap values on the way down.
- * So, we first insert the vma with a zero rb_subtree_gap value
- * (to be consistent with what we did on the way down), and then
- * immediately update the gap to the correct value. Finally we
- * rebalance the rbtree after all augmented values have been set.
- */
- rb_link_node(&vma->vm_rb, rb_parent, rb_link);
- vma->rb_subtree_gap = 0;
- vma_gap_update(vma);
- vma_rb_insert(vma, &mm->mm_rb);
-}
+ if (next && (vma != next) && (end == next->vm_end)) {
+ remove_next = true;
+ if (next->anon_vma && !vma->anon_vma) {
+ int error;
-static void __vma_link_file(struct vm_area_struct *vma)
-{
- struct file *file;
+ anon_vma = next->anon_vma;
+ vma->anon_vma = anon_vma;
+ error = anon_vma_clone(vma, next);
+ if (error)
+ return error;
+ }
+ }
+
+ /* Not merging but overwriting any part of next is not handled. */
+ VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start);
+ /* Only handles expanding */
+ VM_BUG_ON(vma->vm_start < start || vma->vm_end > end);
+
+ if (mas_preallocate(mas, vma, GFP_KERNEL))
+ goto nomem;
+
+ vma_adjust_trans_huge(vma, start, end, 0);
- file = vma->vm_file;
if (file) {
- struct address_space *mapping = file->f_mapping;
+ mapping = file->f_mapping;
+ root = &mapping->i_mmap;
+ uprobe_munmap(vma, vma->vm_start, vma->vm_end);
+ i_mmap_lock_write(mapping);
+ }
- if (vma->vm_flags & VM_SHARED)
- mapping_allow_writable(mapping);
+ if (anon_vma) {
+ anon_vma_lock_write(anon_vma);
+ anon_vma_interval_tree_pre_update_vma(vma);
+ }
+ if (file) {
flush_dcache_mmap_lock(mapping);
- vma_interval_tree_insert(vma, &mapping->i_mmap);
- flush_dcache_mmap_unlock(mapping);
+ vma_interval_tree_remove(vma, root);
}
-}
-static void
-__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev, struct rb_node **rb_link,
- struct rb_node *rb_parent)
-{
- __vma_link_list(mm, vma, prev);
- __vma_link_rb(mm, vma, rb_link, rb_parent);
-}
+ vma->vm_start = start;
+ vma->vm_end = end;
+ vma->vm_pgoff = pgoff;
+ /* Note: mas must be pointing to the expanding VMA */
+ vma_mas_store(vma, mas);
-static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev, struct rb_node **rb_link,
- struct rb_node *rb_parent)
-{
- struct address_space *mapping = NULL;
+ if (file) {
+ vma_interval_tree_insert(vma, root);
+ flush_dcache_mmap_unlock(mapping);
+ }
- if (vma->vm_file) {
- mapping = vma->vm_file->f_mapping;
- i_mmap_lock_write(mapping);
+ /* Expanding over the next vma */
+ if (remove_next && file) {
+ __remove_shared_vm_struct(next, file, mapping);
}
- __vma_link(mm, vma, prev, rb_link, rb_parent);
- __vma_link_file(vma);
+ if (anon_vma) {
+ anon_vma_interval_tree_post_update_vma(vma);
+ anon_vma_unlock_write(anon_vma);
+ }
- if (mapping)
+ if (file) {
i_mmap_unlock_write(mapping);
+ uprobe_mmap(vma);
+ }
- mm->map_count++;
- validate_mm(mm);
-}
-
-/*
- * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
- * mm's list and rbtree. It has already been inserted into the interval tree.
- */
-static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
-{
- struct vm_area_struct *prev;
- struct rb_node **rb_link, *rb_parent;
+ if (remove_next) {
+ if (file) {
+ uprobe_munmap(next, next->vm_start, next->vm_end);
+ fput(file);
+ }
+ if (next->anon_vma)
+ anon_vma_merge(vma, next);
+ mm->map_count--;
+ mpol_put(vma_policy(next));
+ vm_area_free(next);
+ }
- if (find_vma_links(mm, vma->vm_start, vma->vm_end,
- &prev, &rb_link, &rb_parent))
- BUG();
- __vma_link(mm, vma, prev, rb_link, rb_parent);
- mm->map_count++;
-}
+ validate_mm(mm);
+ return 0;
-static __always_inline void __vma_unlink(struct mm_struct *mm,
- struct vm_area_struct *vma,
- struct vm_area_struct *ignore)
-{
- vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
- __vma_unlink_list(mm, vma);
- /* Kill the cache */
- vmacache_invalidate(mm);
+nomem:
+ return -ENOMEM;
}
/*
struct vm_area_struct *expand)
{
struct mm_struct *mm = vma->vm_mm;
- struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
+ struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end);
+ struct vm_area_struct *orig_vma = vma;
struct address_space *mapping = NULL;
struct rb_root_cached *root = NULL;
struct anon_vma *anon_vma = NULL;
struct file *file = vma->vm_file;
- bool start_changed = false, end_changed = false;
+ bool vma_changed = false;
long adjust_next = 0;
int remove_next = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+ struct vm_area_struct *exporter = NULL, *importer = NULL;
if (next && !insert) {
- struct vm_area_struct *exporter = NULL, *importer = NULL;
-
if (end >= next->vm_end) {
/*
* vma expands, overlapping all the next, and
* remove_next == 1 is case 1 or 7.
*/
remove_next = 1 + (end > next->vm_end);
+ if (remove_next == 2)
+ next_next = find_vma(mm, next->vm_end);
+
VM_WARN_ON(remove_next == 2 &&
- end != next->vm_next->vm_end);
- /* trim end to next, for case 6 first pass */
- end = next->vm_end;
+ end != next_next->vm_end);
}
exporter = next;
* next, if the vma overlaps with it.
*/
if (remove_next == 2 && !next->anon_vma)
- exporter = next->vm_next;
+ exporter = next_next;
} else if (end > next->vm_start) {
/*
return error;
}
}
-again:
- vma_adjust_trans_huge(orig_vma, start, end, adjust_next);
+ if (mas_preallocate(&mas, vma, GFP_KERNEL))
+ return -ENOMEM;
+
+ vma_adjust_trans_huge(orig_vma, start, end, adjust_next);
if (file) {
mapping = file->f_mapping;
root = &mapping->i_mmap;
uprobe_munmap(next, next->vm_start, next->vm_end);
i_mmap_lock_write(mapping);
- if (insert) {
+ if (insert && insert->vm_file) {
/*
* Put into interval tree now, so instantiated pages
* are visible to arm/parisc __flush_dcache_page
* throughout; but we cannot insert into address
* space until vma start or end is updated.
*/
- __vma_link_file(insert);
+ __vma_link_file(insert, insert->vm_file->f_mapping);
}
}
}
if (start != vma->vm_start) {
+ if ((vma->vm_start < start) &&
+ (!insert || (insert->vm_end != start))) {
+ vma_mas_szero(&mas, vma->vm_start, start);
+ VM_WARN_ON(insert && insert->vm_start > vma->vm_start);
+ } else {
+ vma_changed = true;
+ }
vma->vm_start = start;
- start_changed = true;
}
if (end != vma->vm_end) {
+ if (vma->vm_end > end) {
+ if (!insert || (insert->vm_start != end)) {
+ vma_mas_szero(&mas, end, vma->vm_end);
+ mas_reset(&mas);
+ VM_WARN_ON(insert &&
+ insert->vm_end < vma->vm_end);
+ }
+ } else {
+ vma_changed = true;
+ }
vma->vm_end = end;
- end_changed = true;
}
+
+ if (vma_changed)
+ vma_mas_store(vma, &mas);
+
vma->vm_pgoff = pgoff;
if (adjust_next) {
next->vm_start += adjust_next;
next->vm_pgoff += adjust_next >> PAGE_SHIFT;
+ vma_mas_store(next, &mas);
}
if (file) {
flush_dcache_mmap_unlock(mapping);
}
- if (remove_next) {
- /*
- * vma_merge has merged next into vma, and needs
- * us to remove next before dropping the locks.
- */
- if (remove_next != 3)
- __vma_unlink(mm, next, next);
- else
- /*
- * vma is not before next if they've been
- * swapped.
- *
- * pre-swap() next->vm_start was reduced so
- * tell validate_mm_rb to ignore pre-swap()
- * "next" (which is stored in post-swap()
- * "vma").
- */
- __vma_unlink(mm, next, vma);
- if (file)
- __remove_shared_vm_struct(next, file, mapping);
+ if (remove_next && file) {
+ __remove_shared_vm_struct(next, file, mapping);
+ if (remove_next == 2)
+ __remove_shared_vm_struct(next_next, file, mapping);
} else if (insert) {
/*
* split_vma has split insert from vma, and needs
* us to insert it before dropping the locks
* (it may either follow vma or precede it).
*/
- __insert_vm_struct(mm, insert);
- } else {
- if (start_changed)
- vma_gap_update(vma);
- if (end_changed) {
- if (!next)
- mm->highest_vm_end = vm_end_gap(vma);
- else if (!adjust_next)
- vma_gap_update(next);
- }
+ mas_reset(&mas);
+ vma_mas_store(insert, &mas);
+ mm->map_count++;
}
if (anon_vma) {
}
if (remove_next) {
+again:
if (file) {
uprobe_munmap(next, next->vm_start, next->vm_end);
fput(file);
anon_vma_merge(vma, next);
mm->map_count--;
mpol_put(vma_policy(next));
+ if (remove_next != 2)
+ BUG_ON(vma->vm_end < next->vm_end);
vm_area_free(next);
+
/*
* In mprotect's case 6 (see comments on vma_merge),
- * we must remove another next too. It would clutter
- * up the code too much to do both in one go.
+ * we must remove next_next too.
*/
- if (remove_next != 3) {
- /*
- * If "next" was removed and vma->vm_end was
- * expanded (up) over it, in turn
- * "next->vm_prev->vm_end" changed and the
- * "vma->vm_next" gap must be updated.
- */
- next = vma->vm_next;
- } else {
- /*
- * For the scope of the comment "next" and
- * "vma" considered pre-swap(): if "vma" was
- * removed, next->vm_start was expanded (down)
- * over it and the "next" gap must be updated.
- * Because of the swap() the post-swap() "vma"
- * actually points to pre-swap() "next"
- * (post-swap() "next" as opposed is now a
- * dangling pointer).
- */
- next = vma;
- }
if (remove_next == 2) {
remove_next = 1;
- end = next->vm_end;
+ next = next_next;
goto again;
}
- else if (next)
- vma_gap_update(next);
- else {
- /*
- * If remove_next == 2 we obviously can't
- * reach this path.
- *
- * If remove_next == 3 we can't reach this
- * path because pre-swap() next is always not
- * NULL. pre-swap() "next" is not being
- * removed and its next->vm_end is not altered
- * (and furthermore "end" already matches
- * next->vm_end in remove_next == 3).
- *
- * We reach this only in the remove_next == 1
- * case if the "next" vma that was removed was
- * the highest vma of the mm. However in such
- * case next->vm_end == "end" and the extended
- * "vma" has vma->vm_end == next->vm_end so
- * mm->highest_vm_end doesn't need any update
- * in remove_next == 1 case.
- */
- VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma));
- }
}
if (insert && file)
uprobe_mmap(insert);
+ mas_destroy(&mas);
validate_mm(mm);
return 0;
struct anon_vma_name *anon_name)
{
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
- struct vm_area_struct *area, *next;
- int err;
+ struct vm_area_struct *mid, *next, *res;
+ int err = -1;
+ bool merge_prev = false;
+ bool merge_next = false;
/*
* We later require that vma->vm_flags == vm_flags,
if (vm_flags & VM_SPECIAL)
return NULL;
- next = vma_next(mm, prev);
- area = next;
- if (area && area->vm_end == end) /* cases 6, 7, 8 */
- next = next->vm_next;
+ next = find_vma(mm, prev ? prev->vm_end : 0);
+ mid = next;
+ if (next && next->vm_end == end) /* cases 6, 7, 8 */
+ next = find_vma(mm, next->vm_end);
/* verify some invariant that must be enforced by the caller */
VM_WARN_ON(prev && addr <= prev->vm_start);
- VM_WARN_ON(area && end > area->vm_end);
+ VM_WARN_ON(mid && end > mid->vm_end);
VM_WARN_ON(addr >= end);
- /*
- * Can it merge with the predecessor?
- */
+ /* Can we merge the predecessor? */
if (prev && prev->vm_end == addr &&
mpol_equal(vma_policy(prev), policy) &&
can_vma_merge_after(prev, vm_flags,
anon_vma, file, pgoff,
vm_userfaultfd_ctx, anon_name)) {
- /*
- * OK, it can. Can we now merge in the successor as well?
- */
- if (next && end == next->vm_start &&
- mpol_equal(policy, vma_policy(next)) &&
- can_vma_merge_before(next, vm_flags,
- anon_vma, file,
- pgoff+pglen,
- vm_userfaultfd_ctx, anon_name) &&
- is_mergeable_anon_vma(prev->anon_vma,
- next->anon_vma, NULL)) {
- /* cases 1, 6 */
- err = __vma_adjust(prev, prev->vm_start,
- next->vm_end, prev->vm_pgoff, NULL,
- prev);
- } else /* cases 2, 5, 7 */
- err = __vma_adjust(prev, prev->vm_start,
- end, prev->vm_pgoff, NULL, prev);
- if (err)
- return NULL;
- khugepaged_enter_vma(prev, vm_flags);
- return prev;
+ merge_prev = true;
}
-
- /*
- * Can this new request be merged in front of next?
- */
+ /* Can we merge the successor? */
if (next && end == next->vm_start &&
mpol_equal(policy, vma_policy(next)) &&
can_vma_merge_before(next, vm_flags,
anon_vma, file, pgoff+pglen,
vm_userfaultfd_ctx, anon_name)) {
+ merge_next = true;
+ }
+ /* Can we merge both the predecessor and the successor? */
+ if (merge_prev && merge_next &&
+ is_mergeable_anon_vma(prev->anon_vma,
+ next->anon_vma, NULL)) { /* cases 1, 6 */
+ err = __vma_adjust(prev, prev->vm_start,
+ next->vm_end, prev->vm_pgoff, NULL,
+ prev);
+ res = prev;
+ } else if (merge_prev) { /* cases 2, 5, 7 */
+ err = __vma_adjust(prev, prev->vm_start,
+ end, prev->vm_pgoff, NULL, prev);
+ res = prev;
+ } else if (merge_next) {
if (prev && addr < prev->vm_end) /* case 4 */
err = __vma_adjust(prev, prev->vm_start,
- addr, prev->vm_pgoff, NULL, next);
- else { /* cases 3, 8 */
- err = __vma_adjust(area, addr, next->vm_end,
- next->vm_pgoff - pglen, NULL, next);
- /*
- * In case 3 area is already equal to next and
- * this is a noop, but in case 8 "area" has
- * been removed and next was expanded over it.
- */
- area = next;
- }
- if (err)
- return NULL;
- khugepaged_enter_vma(area, vm_flags);
- return area;
+ addr, prev->vm_pgoff, NULL, next);
+ else /* cases 3, 8 */
+ err = __vma_adjust(mid, addr, next->vm_end,
+ next->vm_pgoff - pglen, NULL, next);
+ res = next;
}
- return NULL;
+ /*
+ * Cannot merge with predecessor or successor or error in __vma_adjust?
+ */
+ if (err)
+ return NULL;
+ khugepaged_enter_vma(res, vm_flags);
+ return res;
}
/*
*/
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
{
+ MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end);
struct anon_vma *anon_vma = NULL;
+ struct vm_area_struct *prev, *next;
/* Try next first. */
- if (vma->vm_next) {
- anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next);
+ next = mas_walk(&mas);
+ if (next) {
+ anon_vma = reusable_anon_vma(next, vma, next);
if (anon_vma)
return anon_vma;
}
+ prev = mas_prev(&mas, 0);
+ VM_BUG_ON_VMA(prev != vma, vma);
+ prev = mas_prev(&mas, 0);
/* Try prev next. */
- if (vma->vm_prev)
- anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma);
+ if (prev)
+ anon_vma = reusable_anon_vma(prev, prev, vma);
/*
* We might reach here with anon_vma == NULL if we can't find
vm_flags_t vm_flags;
int pkey = 0;
+ validate_mm(mm);
*populate = 0;
if (!len)
return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
}
-unsigned long mmap_region(struct file *file, unsigned long addr,
- unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
- struct list_head *uf)
-{
- struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma, *prev, *merge;
- int error;
- struct rb_node **rb_link, *rb_parent;
- unsigned long charged = 0;
-
- /* Check against address space limit. */
- if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
- unsigned long nr_pages;
-
- /*
- * MAP_FIXED may remove pages of mappings that intersects with
- * requested mapping. Account for the pages it would unmap.
- */
- nr_pages = count_vma_pages_range(mm, addr, addr + len);
-
- if (!may_expand_vm(mm, vm_flags,
- (len >> PAGE_SHIFT) - nr_pages))
- return -ENOMEM;
- }
-
- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */
- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf))
- return -ENOMEM;
- /*
- * Private writable mapping: check memory availability
- */
- if (accountable_mapping(file, vm_flags)) {
- charged = len >> PAGE_SHIFT;
- if (security_vm_enough_memory_mm(mm, charged))
- return -ENOMEM;
- vm_flags |= VM_ACCOUNT;
- }
-
- /*
- * Can we just expand an old mapping?
- */
- vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
- NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
- if (vma)
- goto out;
-
- /*
- * Determine the object being mapped and call the appropriate
- * specific mapper. the address has already been validated, but
- * not unmapped, but the maps are removed from the list.
- */
- vma = vm_area_alloc(mm);
- if (!vma) {
- error = -ENOMEM;
- goto unacct_error;
- }
-
- vma->vm_start = addr;
- vma->vm_end = addr + len;
- vma->vm_flags = vm_flags;
- vma->vm_page_prot = vm_get_page_prot(vm_flags);
- vma->vm_pgoff = pgoff;
-
- if (file) {
- if (vm_flags & VM_SHARED) {
- error = mapping_map_writable(file->f_mapping);
- if (error)
- goto free_vma;
- }
-
- vma->vm_file = get_file(file);
- error = call_mmap(file, vma);
- if (error)
- goto unmap_and_free_vma;
-
- /* Can addr have changed??
- *
- * Answer: Yes, several device drivers can do it in their
- * f_op->mmap method. -DaveM
- * Bug: If addr is changed, prev, rb_link, rb_parent should
- * be updated for vma_link()
- */
- WARN_ON_ONCE(addr != vma->vm_start);
-
- addr = vma->vm_start;
-
- /* If vm_flags changed after call_mmap(), we should try merge vma again
- * as we may succeed this time.
- */
- if (unlikely(vm_flags != vma->vm_flags && prev)) {
- merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags,
- NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
- if (merge) {
- /* ->mmap() can change vma->vm_file and fput the original file. So
- * fput the vma->vm_file here or we would add an extra fput for file
- * and cause general protection fault ultimately.
- */
- fput(vma->vm_file);
- vm_area_free(vma);
- vma = merge;
- /* Update vm_flags to pick up the change. */
- vm_flags = vma->vm_flags;
- goto unmap_writable;
- }
- }
-
- vm_flags = vma->vm_flags;
- } else if (vm_flags & VM_SHARED) {
- error = shmem_zero_setup(vma);
- if (error)
- goto free_vma;
- } else {
- vma_set_anonymous(vma);
- }
-
- /* Allow architectures to sanity-check the vm_flags */
- if (!arch_validate_flags(vma->vm_flags)) {
- error = -EINVAL;
- if (file)
- goto unmap_and_free_vma;
- else
- goto free_vma;
- }
-
- vma_link(mm, vma, prev, rb_link, rb_parent);
-
- /*
- * vma_merge() calls khugepaged_enter_vma() either, the below
- * call covers the non-merge case.
- */
- khugepaged_enter_vma(vma, vma->vm_flags);
-
- /* Once vma denies write, undo our temporary denial count */
-unmap_writable:
- if (file && vm_flags & VM_SHARED)
- mapping_unmap_writable(file->f_mapping);
- file = vma->vm_file;
-out:
- perf_event_mmap(vma);
-
- vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
- if (vm_flags & VM_LOCKED) {
- if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
- is_vm_hugetlb_page(vma) ||
- vma == get_gate_vma(current->mm))
- vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
- else
- mm->locked_vm += (len >> PAGE_SHIFT);
- }
-
- if (file)
- uprobe_mmap(vma);
-
- /*
- * New (or expanded) vma always get soft dirty status.
- * Otherwise user-space soft-dirty page tracker won't
- * be able to distinguish situation when vma area unmapped,
- * then new mapped in-place (which must be aimed as
- * a completely new data area).
- */
- vma->vm_flags |= VM_SOFTDIRTY;
-
- vma_set_page_prot(vma);
-
- return addr;
-
-unmap_and_free_vma:
- fput(vma->vm_file);
- vma->vm_file = NULL;
-
- /* Undo any partial mapping done by a device driver. */
- unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
- if (vm_flags & VM_SHARED)
- mapping_unmap_writable(file->f_mapping);
-free_vma:
- vm_area_free(vma);
-unacct_error:
- if (charged)
- vm_unacct_memory(charged);
- return error;
-}
-
+/**
+ * unmapped_area() - Find an area between the low_limit and the high_limit with
+ * the correct alignment and offset, all from @info. Note: current->mm is used
+ * for the search.
+ *
+ * @info: The unmapped area information including the range (low_limit -
+ * hight_limit), the alignment offset and mask.
+ *
+ * Return: A memory address or -ENOMEM.
+ */
static unsigned long unmapped_area(struct vm_unmapped_area_info *info)
{
- /*
- * We implement the search by looking for an rbtree node that
- * immediately follows a suitable gap. That is,
- * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
- * - gap_end = vma->vm_start >= info->low_limit + length;
- * - gap_end - gap_start >= length
- */
+ unsigned long length, gap;
- struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma;
- unsigned long length, low_limit, high_limit, gap_start, gap_end;
+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
/* Adjust search length to account for worst case alignment overhead */
length = info->length + info->align_mask;
if (length < info->length)
return -ENOMEM;
- /* Adjust search limits by the desired length */
- if (info->high_limit < length)
- return -ENOMEM;
- high_limit = info->high_limit - length;
-
- if (info->low_limit > high_limit)
- return -ENOMEM;
- low_limit = info->low_limit + length;
-
- /* Check if rbtree root looks promising */
- if (RB_EMPTY_ROOT(&mm->mm_rb))
- goto check_highest;
- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
- if (vma->rb_subtree_gap < length)
- goto check_highest;
-
- while (true) {
- /* Visit left subtree if it looks promising */
- gap_end = vm_start_gap(vma);
- if (gap_end >= low_limit && vma->vm_rb.rb_left) {
- struct vm_area_struct *left =
- rb_entry(vma->vm_rb.rb_left,
- struct vm_area_struct, vm_rb);
- if (left->rb_subtree_gap >= length) {
- vma = left;
- continue;
- }
- }
-
- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
-check_current:
- /* Check if current node has a suitable gap */
- if (gap_start > high_limit)
- return -ENOMEM;
- if (gap_end >= low_limit &&
- gap_end > gap_start && gap_end - gap_start >= length)
- goto found;
-
- /* Visit right subtree if it looks promising */
- if (vma->vm_rb.rb_right) {
- struct vm_area_struct *right =
- rb_entry(vma->vm_rb.rb_right,
- struct vm_area_struct, vm_rb);
- if (right->rb_subtree_gap >= length) {
- vma = right;
- continue;
- }
- }
-
- /* Go back up the rbtree to find next candidate node */
- while (true) {
- struct rb_node *prev = &vma->vm_rb;
- if (!rb_parent(prev))
- goto check_highest;
- vma = rb_entry(rb_parent(prev),
- struct vm_area_struct, vm_rb);
- if (prev == vma->vm_rb.rb_left) {
- gap_start = vm_end_gap(vma->vm_prev);
- gap_end = vm_start_gap(vma);
- goto check_current;
- }
- }
- }
-
-check_highest:
- /* Check highest gap, which does not precede any rbtree node */
- gap_start = mm->highest_vm_end;
- gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */
- if (gap_start > high_limit)
+ if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1,
+ length))
return -ENOMEM;
-found:
- /* We found a suitable gap. Clip it with the original low_limit. */
- if (gap_start < info->low_limit)
- gap_start = info->low_limit;
-
- /* Adjust gap address to the desired alignment */
- gap_start += (info->align_offset - gap_start) & info->align_mask;
-
- VM_BUG_ON(gap_start + info->length > info->high_limit);
- VM_BUG_ON(gap_start + info->length > gap_end);
- return gap_start;
+ gap = mas.index;
+ gap += (info->align_offset - gap) & info->align_mask;
+ return gap;
}
+/**
+ * unmapped_area_topdown() - Find an area between the low_limit and the
+ * high_limit with * the correct alignment and offset at the highest available
+ * address, all from @info. Note: current->mm is used for the search.
+ *
+ * @info: The unmapped area information including the range (low_limit -
+ * hight_limit), the alignment offset and mask.
+ *
+ * Return: A memory address or -ENOMEM.
+ */
static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
{
- struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma;
- unsigned long length, low_limit, high_limit, gap_start, gap_end;
+ unsigned long length, gap;
+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
/* Adjust search length to account for worst case alignment overhead */
length = info->length + info->align_mask;
if (length < info->length)
return -ENOMEM;
- /*
- * Adjust search limits by the desired length.
- * See implementation comment at top of unmapped_area().
- */
- gap_end = info->high_limit;
- if (gap_end < length)
- return -ENOMEM;
- high_limit = gap_end - length;
-
- if (info->low_limit > high_limit)
- return -ENOMEM;
- low_limit = info->low_limit + length;
-
- /* Check highest gap, which does not precede any rbtree node */
- gap_start = mm->highest_vm_end;
- if (gap_start <= high_limit)
- goto found_highest;
-
- /* Check if rbtree root looks promising */
- if (RB_EMPTY_ROOT(&mm->mm_rb))
- return -ENOMEM;
- vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
- if (vma->rb_subtree_gap < length)
+ if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1,
+ length))
return -ENOMEM;
- while (true) {
- /* Visit right subtree if it looks promising */
- gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
- if (gap_start <= high_limit && vma->vm_rb.rb_right) {
- struct vm_area_struct *right =
- rb_entry(vma->vm_rb.rb_right,
- struct vm_area_struct, vm_rb);
- if (right->rb_subtree_gap >= length) {
- vma = right;
- continue;
- }
- }
-
-check_current:
- /* Check if current node has a suitable gap */
- gap_end = vm_start_gap(vma);
- if (gap_end < low_limit)
- return -ENOMEM;
- if (gap_start <= high_limit &&
- gap_end > gap_start && gap_end - gap_start >= length)
- goto found;
-
- /* Visit left subtree if it looks promising */
- if (vma->vm_rb.rb_left) {
- struct vm_area_struct *left =
- rb_entry(vma->vm_rb.rb_left,
- struct vm_area_struct, vm_rb);
- if (left->rb_subtree_gap >= length) {
- vma = left;
- continue;
- }
- }
-
- /* Go back up the rbtree to find next candidate node */
- while (true) {
- struct rb_node *prev = &vma->vm_rb;
- if (!rb_parent(prev))
- return -ENOMEM;
- vma = rb_entry(rb_parent(prev),
- struct vm_area_struct, vm_rb);
- if (prev == vma->vm_rb.rb_right) {
- gap_start = vma->vm_prev ?
- vm_end_gap(vma->vm_prev) : 0;
- goto check_current;
- }
- }
- }
-
-found:
- /* We found a suitable gap. Clip it with the original high_limit. */
- if (gap_end > info->high_limit)
- gap_end = info->high_limit;
-
-found_highest:
- /* Compute highest gap address at the desired alignment */
- gap_end -= info->length;
- gap_end -= (gap_end - info->align_offset) & info->align_mask;
-
- VM_BUG_ON(gap_end < info->low_limit);
- VM_BUG_ON(gap_end < gap_start);
- return gap_end;
+ gap = mas.last + 1 - info->length;
+ gap -= (gap - info->align_offset) & info->align_mask;
+ return gap;
}
/*
*/
pgoff = 0;
get_area = shmem_get_unmapped_area;
+ } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
+ /* Ensures that larger anonymous mappings are THP aligned. */
+ get_area = thp_get_unmapped_area;
}
addr = get_area(file, addr, len, pgoff, flags);
EXPORT_SYMBOL(get_unmapped_area);
-/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
-struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
+/**
+ * find_vma_intersection() - Look up the first VMA which intersects the interval
+ * @mm: The process address space.
+ * @start_addr: The inclusive start user address.
+ * @end_addr: The exclusive end user address.
+ *
+ * Returns: The first VMA within the provided range, %NULL otherwise. Assumes
+ * start_addr < end_addr.
+ */
+struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
+ unsigned long start_addr,
+ unsigned long end_addr)
{
- struct rb_node *rb_node;
- struct vm_area_struct *vma;
+ unsigned long index = start_addr;
mmap_assert_locked(mm);
- /* Check the cache first. */
- vma = vmacache_find(mm, addr);
- if (likely(vma))
- return vma;
-
- rb_node = mm->mm_rb.rb_node;
-
- while (rb_node) {
- struct vm_area_struct *tmp;
-
- tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
+ return mt_find(&mm->mm_mt, &index, end_addr - 1);
+}
+EXPORT_SYMBOL(find_vma_intersection);
- if (tmp->vm_end > addr) {
- vma = tmp;
- if (tmp->vm_start <= addr)
- break;
- rb_node = rb_node->rb_left;
- } else
- rb_node = rb_node->rb_right;
- }
+/**
+ * find_vma() - Find the VMA for a given address, or the next VMA.
+ * @mm: The mm_struct to check
+ * @addr: The address
+ *
+ * Returns: The VMA associated with addr, or the next VMA.
+ * May return %NULL in the case of no VMA at addr or above.
+ */
+struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
+{
+ unsigned long index = addr;
- if (vma)
- vmacache_update(addr, vma);
- return vma;
+ mmap_assert_locked(mm);
+ return mt_find(&mm->mm_mt, &index, ULONG_MAX);
}
-
EXPORT_SYMBOL(find_vma);
-/*
- * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
+/**
+ * find_vma_prev() - Find the VMA for a given address, or the next vma and
+ * set %pprev to the previous VMA, if any.
+ * @mm: The mm_struct to check
+ * @addr: The address
+ * @pprev: The pointer to set to the previous VMA
+ *
+ * Note that RCU lock is missing here since the external mmap_lock() is used
+ * instead.
+ *
+ * Returns: The VMA associated with @addr, or the next vma.
+ * May return %NULL in the case of no vma at addr or above.
*/
struct vm_area_struct *
find_vma_prev(struct mm_struct *mm, unsigned long addr,
struct vm_area_struct **pprev)
{
struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
- vma = find_vma(mm, addr);
- if (vma) {
- *pprev = vma->vm_prev;
- } else {
- struct rb_node *rb_node = rb_last(&mm->mm_rb);
-
- *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL;
- }
+ vma = mas_walk(&mas);
+ *pprev = mas_prev(&mas, 0);
+ if (!vma)
+ vma = mas_next(&mas, ULONG_MAX);
return vma;
}
struct vm_area_struct *next;
unsigned long gap_addr;
int error = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
if (!(vma->vm_flags & VM_GROWSUP))
return -EFAULT;
if (gap_addr < address || gap_addr > TASK_SIZE)
gap_addr = TASK_SIZE;
- next = vma->vm_next;
- if (next && next->vm_start < gap_addr && vma_is_accessible(next)) {
+ next = find_vma_intersection(mm, vma->vm_end, gap_addr);
+ if (next && vma_is_accessible(next)) {
if (!(next->vm_flags & VM_GROWSUP))
return -ENOMEM;
/* Check that both stack segments have the same anon_vma? */
}
+ if (mas_preallocate(&mas, vma, GFP_KERNEL))
+ return -ENOMEM;
+
/* We must make sure the anon_vma is allocated. */
- if (unlikely(anon_vma_prepare(vma)))
+ if (unlikely(anon_vma_prepare(vma))) {
+ mas_destroy(&mas);
return -ENOMEM;
+ }
/*
* vma->vm_start/vm_end cannot change under us because the caller
error = acct_stack_growth(vma, size, grow);
if (!error) {
/*
- * vma_gap_update() doesn't support concurrent
- * updates, but we only hold a shared mmap_lock
- * lock here, so we need to protect against
- * concurrent vma expansions.
- * anon_vma_lock_write() doesn't help here, as
- * we don't guarantee that all growable vmas
- * in a mm share the same root anon vma.
- * So, we reuse mm->page_table_lock to guard
- * against concurrent vma expansions.
+ * We only hold a shared mmap_lock lock here, so
+ * we need to protect against concurrent vma
+ * expansions. anon_vma_lock_write() doesn't
+ * help here, as we don't guarantee that all
+ * growable vmas in a mm share the same root
+ * anon vma. So, we reuse mm->page_table_lock
+ * to guard against concurrent vma expansions.
*/
spin_lock(&mm->page_table_lock);
if (vma->vm_flags & VM_LOCKED)
vm_stat_account(mm, vma->vm_flags, grow);
anon_vma_interval_tree_pre_update_vma(vma);
vma->vm_end = address;
+ /* Overwrite old entry in mtree. */
+ vma_mas_store(vma, &mas);
anon_vma_interval_tree_post_update_vma(vma);
- if (vma->vm_next)
- vma_gap_update(vma->vm_next);
- else
- mm->highest_vm_end = vm_end_gap(vma);
spin_unlock(&mm->page_table_lock);
perf_event_mmap(vma);
}
anon_vma_unlock_write(vma->anon_vma);
khugepaged_enter_vma(vma, vma->vm_flags);
- validate_mm(mm);
+ mas_destroy(&mas);
return error;
}
#endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
/*
* vma is the first one with address < vma->vm_start. Have to extend vma.
*/
-int expand_downwards(struct vm_area_struct *vma,
- unsigned long address)
+int expand_downwards(struct vm_area_struct *vma, unsigned long address)
{
struct mm_struct *mm = vma->vm_mm;
+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
struct vm_area_struct *prev;
int error = 0;
return -EPERM;
/* Enforce stack_guard_gap */
- prev = vma->vm_prev;
+ prev = mas_prev(&mas, 0);
/* Check that both stack segments have the same anon_vma? */
if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
vma_is_accessible(prev)) {
return -ENOMEM;
}
+ if (mas_preallocate(&mas, vma, GFP_KERNEL))
+ return -ENOMEM;
+
/* We must make sure the anon_vma is allocated. */
- if (unlikely(anon_vma_prepare(vma)))
+ if (unlikely(anon_vma_prepare(vma))) {
+ mas_destroy(&mas);
return -ENOMEM;
+ }
/*
* vma->vm_start/vm_end cannot change under us because the caller
error = acct_stack_growth(vma, size, grow);
if (!error) {
/*
- * vma_gap_update() doesn't support concurrent
- * updates, but we only hold a shared mmap_lock
- * lock here, so we need to protect against
- * concurrent vma expansions.
- * anon_vma_lock_write() doesn't help here, as
- * we don't guarantee that all growable vmas
- * in a mm share the same root anon vma.
- * So, we reuse mm->page_table_lock to guard
- * against concurrent vma expansions.
+ * We only hold a shared mmap_lock lock here, so
+ * we need to protect against concurrent vma
+ * expansions. anon_vma_lock_write() doesn't
+ * help here, as we don't guarantee that all
+ * growable vmas in a mm share the same root
+ * anon vma. So, we reuse mm->page_table_lock
+ * to guard against concurrent vma expansions.
*/
spin_lock(&mm->page_table_lock);
if (vma->vm_flags & VM_LOCKED)
anon_vma_interval_tree_pre_update_vma(vma);
vma->vm_start = address;
vma->vm_pgoff -= grow;
+ /* Overwrite old entry in mtree. */
+ vma_mas_store(vma, &mas);
anon_vma_interval_tree_post_update_vma(vma);
- vma_gap_update(vma);
spin_unlock(&mm->page_table_lock);
perf_event_mmap(vma);
}
anon_vma_unlock_write(vma->anon_vma);
khugepaged_enter_vma(vma, vma->vm_flags);
- validate_mm(mm);
+ mas_destroy(&mas);
return error;
}
EXPORT_SYMBOL_GPL(find_extend_vma);
/*
- * Ok - we have the memory areas we should free on the vma list,
- * so release them, and do the vma updates.
+ * Ok - we have the memory areas we should free on a maple tree so release them,
+ * and do the vma updates.
*
* Called with the mm semaphore held.
*/
-static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
+static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
{
unsigned long nr_accounted = 0;
+ struct vm_area_struct *vma;
/* Update high watermark before we lower total_vm */
update_hiwater_vm(mm);
- do {
+ mas_for_each(mas, vma, ULONG_MAX) {
long nrpages = vma_pages(vma);
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += nrpages;
vm_stat_account(mm, vma->vm_flags, -nrpages);
- vma = remove_vma(vma);
- } while (vma);
+ remove_vma(vma);
+ }
vm_unacct_memory(nr_accounted);
validate_mm(mm);
}
*
* Called with the mm semaphore held.
*/
-static void unmap_region(struct mm_struct *mm,
+static void unmap_region(struct mm_struct *mm, struct maple_tree *mt,
struct vm_area_struct *vma, struct vm_area_struct *prev,
+ struct vm_area_struct *next,
unsigned long start, unsigned long end)
{
- struct vm_area_struct *next = vma_next(mm, prev);
struct mmu_gather tlb;
lru_add_drain();
tlb_gather_mmu(&tlb, mm);
update_hiwater_rss(mm);
- unmap_vmas(&tlb, vma, start, end);
- free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
+ unmap_vmas(&tlb, mt, vma, start, end);
+ free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
next ? next->vm_start : USER_PGTABLES_CEILING);
tlb_finish_mmu(&tlb);
}
-/*
- * Create a list of vma's touched by the unmap, removing them from the mm's
- * vma list as we go..
- */
-static bool
-detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev, unsigned long end)
-{
- struct vm_area_struct **insertion_point;
- struct vm_area_struct *tail_vma = NULL;
-
- insertion_point = (prev ? &prev->vm_next : &mm->mmap);
- vma->vm_prev = NULL;
- do {
- vma_rb_erase(vma, &mm->mm_rb);
- if (vma->vm_flags & VM_LOCKED)
- mm->locked_vm -= vma_pages(vma);
- mm->map_count--;
- tail_vma = vma;
- vma = vma->vm_next;
- } while (vma && vma->vm_start < end);
- *insertion_point = vma;
- if (vma) {
- vma->vm_prev = prev;
- vma_gap_update(vma);
- } else
- mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
- tail_vma->vm_next = NULL;
-
- /* Kill the cache */
- vmacache_invalidate(mm);
-
- /*
- * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
- * VM_GROWSUP VMA. Such VMAs can change their size under
- * down_read(mmap_lock) and collide with the VMA we are about to unmap.
- */
- if (vma && (vma->vm_flags & VM_GROWSDOWN))
- return false;
- if (prev && (prev->vm_flags & VM_GROWSUP))
- return false;
- return true;
-}
-
/*
* __split_vma() bypasses sysctl_max_map_count checking. We use this where it
* has already been checked or doesn't make sense to fail.
{
struct vm_area_struct *new;
int err;
+ validate_mm_mt(mm);
if (vma->vm_ops && vma->vm_ops->may_split) {
err = vma->vm_ops->may_split(vma, addr);
if (!err)
return 0;
+ /* Avoid vm accounting in close() operation */
+ new->vm_start = new->vm_end;
+ new->vm_pgoff = 0;
/* Clean everything up if vma_adjust failed. */
if (new->vm_ops && new->vm_ops->close)
new->vm_ops->close(new);
mpol_put(vma_policy(new));
out_free_vma:
vm_area_free(new);
+ validate_mm_mt(mm);
return err;
}
return __split_vma(mm, vma, addr, new_below);
}
-/* Munmap is split into 2 main parts -- this part which finds
- * what needs doing, and the areas themselves, which do the
- * work. This now handles partial unmappings.
- * Jeremy Fitzhardinge <jeremy@goop.org>
- */
-int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
- struct list_head *uf, bool downgrade)
+static inline int munmap_sidetree(struct vm_area_struct *vma,
+ struct ma_state *mas_detach)
{
- unsigned long end;
- struct vm_area_struct *vma, *prev, *last;
-
- if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
- return -EINVAL;
+ mas_set_range(mas_detach, vma->vm_start, vma->vm_end - 1);
+ if (mas_store_gfp(mas_detach, vma, GFP_KERNEL))
+ return -ENOMEM;
- len = PAGE_ALIGN(len);
- end = start + len;
- if (len == 0)
- return -EINVAL;
+ if (vma->vm_flags & VM_LOCKED)
+ vma->vm_mm->locked_vm -= vma_pages(vma);
- /*
- * arch_unmap() might do unmaps itself. It must be called
- * and finish any rbtree manipulation before this code
- * runs and also starts to manipulate the rbtree.
- */
- arch_unmap(mm, start, end);
+ return 0;
+}
- /* Find the first overlapping VMA where start < vma->vm_end */
- vma = find_vma_intersection(mm, start, end);
- if (!vma)
- return 0;
- prev = vma->vm_prev;
+/*
+ * do_mas_align_munmap() - munmap the aligned region from @start to @end.
+ * @mas: The maple_state, ideally set up to alter the correct tree location.
+ * @vma: The starting vm_area_struct
+ * @mm: The mm_struct
+ * @start: The aligned start address to munmap.
+ * @end: The aligned end address to munmap.
+ * @uf: The userfaultfd list_head
+ * @downgrade: Set to true to attempt a write downgrade of the mmap_sem
+ *
+ * If @downgrade is true, check return code for potential release of the lock.
+ */
+static int
+do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma,
+ struct mm_struct *mm, unsigned long start,
+ unsigned long end, struct list_head *uf, bool downgrade)
+{
+ struct vm_area_struct *prev, *next = NULL;
+ struct maple_tree mt_detach;
+ int count = 0;
+ int error = -ENOMEM;
+ MA_STATE(mas_detach, &mt_detach, 0, 0);
+ mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN);
+ mt_set_external_lock(&mt_detach, &mm->mmap_lock);
+
+ if (mas_preallocate(mas, vma, GFP_KERNEL))
+ return -ENOMEM;
+ mas->last = end - 1;
/*
* If we need to split any vma, do it now to save pain later.
*
* unmapped vm_area_struct will remain in use: so lower split_vma
* places tmp vma above, and higher split_vma places tmp vma below.
*/
+
+ /* Does it split the first one? */
if (start > vma->vm_start) {
- int error;
/*
* Make sure that map_count on return from munmap() will
* its limit temporarily, to help free resources as expected.
*/
if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
- return -ENOMEM;
+ goto map_count_exceeded;
+ /*
+ * mas_pause() is not needed since mas->index needs to be set
+ * differently than vma->vm_end anyways.
+ */
error = __split_vma(mm, vma, start, 0);
if (error)
- return error;
- prev = vma;
+ goto start_split_failed;
+
+ mas_set(mas, start);
+ vma = mas_walk(mas);
}
- /* Does it split the last one? */
- last = find_vma(mm, end);
- if (last && end > last->vm_start) {
- int error = __split_vma(mm, last, end, 1);
+ prev = mas_prev(mas, 0);
+ if (unlikely((!prev)))
+ mas_set(mas, start);
+
+ /*
+ * Detach a range of VMAs from the mm. Using next as a temp variable as
+ * it is always overwritten.
+ */
+ mas_for_each(mas, next, end - 1) {
+ /* Does it split the end? */
+ if (next->vm_end > end) {
+ struct vm_area_struct *split;
+
+ error = __split_vma(mm, next, end, 1);
+ if (error)
+ goto end_split_failed;
+
+ mas_set(mas, end);
+ split = mas_prev(mas, 0);
+ error = munmap_sidetree(split, &mas_detach);
+ if (error)
+ goto munmap_sidetree_failed;
+
+ count++;
+ if (vma == next)
+ vma = split;
+ break;
+ }
+ error = munmap_sidetree(next, &mas_detach);
if (error)
- return error;
+ goto munmap_sidetree_failed;
+
+ count++;
+#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
+ BUG_ON(next->vm_start < start);
+ BUG_ON(next->vm_start > end);
+#endif
}
- vma = vma_next(mm, prev);
+
+ if (!next)
+ next = mas_next(mas, ULONG_MAX);
if (unlikely(uf)) {
/*
* split, despite we could. This is unlikely enough
* failure that it's not worth optimizing it for.
*/
- int error = userfaultfd_unmap_prep(vma, start, end, uf);
+ error = userfaultfd_unmap_prep(mm, start, end, uf);
+
if (error)
- return error;
+ goto userfaultfd_error;
+ }
+
+ /* Point of no return */
+ mas_set_range(mas, start, end - 1);
+#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
+ /* Make sure no VMAs are about to be lost. */
+ {
+ MA_STATE(test, &mt_detach, start, end - 1);
+ struct vm_area_struct *vma_mas, *vma_test;
+ int test_count = 0;
+
+ rcu_read_lock();
+ vma_test = mas_find(&test, end - 1);
+ mas_for_each(mas, vma_mas, end - 1) {
+ BUG_ON(vma_mas != vma_test);
+ test_count++;
+ vma_test = mas_next(&test, end - 1);
+ }
+ rcu_read_unlock();
+ BUG_ON(count != test_count);
+ mas_set_range(mas, start, end - 1);
+ }
+#endif
+ mas_store_prealloc(mas, NULL);
+ mm->map_count -= count;
+ /*
+ * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or
+ * VM_GROWSUP VMA. Such VMAs can change their size under
+ * down_read(mmap_lock) and collide with the VMA we are about to unmap.
+ */
+ if (downgrade) {
+ if (next && (next->vm_flags & VM_GROWSDOWN))
+ downgrade = false;
+ else if (prev && (prev->vm_flags & VM_GROWSUP))
+ downgrade = false;
+ else
+ mmap_write_downgrade(mm);
}
- /* Detach vmas from rbtree */
- if (!detach_vmas_to_be_unmapped(mm, vma, prev, end))
- downgrade = false;
+ unmap_region(mm, &mt_detach, vma, prev, next, start, end);
+ /* Statistics and freeing VMAs */
+ mas_set(&mas_detach, start);
+ remove_mt(mm, &mas_detach);
+ __mt_destroy(&mt_detach);
- if (downgrade)
- mmap_write_downgrade(mm);
- unmap_region(mm, vma, prev, start, end);
+ validate_mm(mm);
+ return downgrade ? 1 : 0;
- /* Fix up all other VM information */
- remove_vma_list(mm, vma);
+userfaultfd_error:
+munmap_sidetree_failed:
+end_split_failed:
+ __mt_destroy(&mt_detach);
+start_split_failed:
+map_count_exceeded:
+ mas_destroy(mas);
+ return error;
+}
- return downgrade ? 1 : 0;
+/*
+ * do_mas_munmap() - munmap a given range.
+ * @mas: The maple state
+ * @mm: The mm_struct
+ * @start: The start address to munmap
+ * @len: The length of the range to munmap
+ * @uf: The userfaultfd list_head
+ * @downgrade: set to true if the user wants to attempt to write_downgrade the
+ * mmap_sem
+ *
+ * This function takes a @mas that is either pointing to the previous VMA or set
+ * to MA_START and sets it up to remove the mapping(s). The @len will be
+ * aligned and any arch_unmap work will be preformed.
+ *
+ * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise.
+ */
+int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm,
+ unsigned long start, size_t len, struct list_head *uf,
+ bool downgrade)
+{
+ unsigned long end;
+ struct vm_area_struct *vma;
+
+ if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
+ return -EINVAL;
+
+ end = start + PAGE_ALIGN(len);
+ if (end == start)
+ return -EINVAL;
+
+ /* arch_unmap() might do unmaps itself. */
+ arch_unmap(mm, start, end);
+
+ /* Find the first overlapping VMA */
+ vma = mas_find(mas, end - 1);
+ if (!vma)
+ return 0;
+
+ return do_mas_align_munmap(mas, vma, mm, start, end, uf, downgrade);
}
+/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
+ * @mm: The mm_struct
+ * @start: The start address to munmap
+ * @len: The length to be munmapped.
+ * @uf: The userfaultfd list_head
+ */
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
struct list_head *uf)
{
- return __do_munmap(mm, start, len, uf, false);
+ MA_STATE(mas, &mm->mm_mt, start, start);
+
+ return do_mas_munmap(&mas, mm, start, len, uf, false);
+}
+
+unsigned long mmap_region(struct file *file, unsigned long addr,
+ unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
+ struct list_head *uf)
+{
+ struct mm_struct *mm = current->mm;
+ struct vm_area_struct *vma = NULL;
+ struct vm_area_struct *next, *prev, *merge;
+ pgoff_t pglen = len >> PAGE_SHIFT;
+ unsigned long charged = 0;
+ unsigned long end = addr + len;
+ unsigned long merge_start = addr, merge_end = end;
+ pgoff_t vm_pgoff;
+ int error;
+ MA_STATE(mas, &mm->mm_mt, addr, end - 1);
+
+ /* Check against address space limit. */
+ if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
+ unsigned long nr_pages;
+
+ /*
+ * MAP_FIXED may remove pages of mappings that intersects with
+ * requested mapping. Account for the pages it would unmap.
+ */
+ nr_pages = count_vma_pages_range(mm, addr, end);
+
+ if (!may_expand_vm(mm, vm_flags,
+ (len >> PAGE_SHIFT) - nr_pages))
+ return -ENOMEM;
+ }
+
+ /* Unmap any existing mapping in the area */
+ if (do_mas_munmap(&mas, mm, addr, len, uf, false))
+ return -ENOMEM;
+
+ /*
+ * Private writable mapping: check memory availability
+ */
+ if (accountable_mapping(file, vm_flags)) {
+ charged = len >> PAGE_SHIFT;
+ if (security_vm_enough_memory_mm(mm, charged))
+ return -ENOMEM;
+ vm_flags |= VM_ACCOUNT;
+ }
+
+ next = mas_next(&mas, ULONG_MAX);
+ prev = mas_prev(&mas, 0);
+ if (vm_flags & VM_SPECIAL)
+ goto cannot_expand;
+
+ /* Attempt to expand an old mapping */
+ /* Check next */
+ if (next && next->vm_start == end && !vma_policy(next) &&
+ can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen,
+ NULL_VM_UFFD_CTX, NULL)) {
+ merge_end = next->vm_end;
+ vma = next;
+ vm_pgoff = next->vm_pgoff - pglen;
+ }
+
+ /* Check prev */
+ if (prev && prev->vm_end == addr && !vma_policy(prev) &&
+ (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file,
+ pgoff, vma->vm_userfaultfd_ctx, NULL) :
+ can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,
+ NULL_VM_UFFD_CTX, NULL))) {
+ merge_start = prev->vm_start;
+ vma = prev;
+ vm_pgoff = prev->vm_pgoff;
+ }
+
+
+ /* Actually expand, if possible */
+ if (vma &&
+ !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) {
+ khugepaged_enter_vma(vma, vm_flags);
+ goto expanded;
+ }
+
+ mas.index = addr;
+ mas.last = end - 1;
+cannot_expand:
+ /*
+ * Determine the object being mapped and call the appropriate
+ * specific mapper. the address has already been validated, but
+ * not unmapped, but the maps are removed from the list.
+ */
+ vma = vm_area_alloc(mm);
+ if (!vma) {
+ error = -ENOMEM;
+ goto unacct_error;
+ }
+
+ vma->vm_start = addr;
+ vma->vm_end = end;
+ vma->vm_flags = vm_flags;
+ vma->vm_page_prot = vm_get_page_prot(vm_flags);
+ vma->vm_pgoff = pgoff;
+
+ if (file) {
+ if (vm_flags & VM_SHARED) {
+ error = mapping_map_writable(file->f_mapping);
+ if (error)
+ goto free_vma;
+ }
+
+ vma->vm_file = get_file(file);
+ error = call_mmap(file, vma);
+ if (error)
+ goto unmap_and_free_vma;
+
+ /* Can addr have changed??
+ *
+ * Answer: Yes, several device drivers can do it in their
+ * f_op->mmap method. -DaveM
+ */
+ WARN_ON_ONCE(addr != vma->vm_start);
+
+ addr = vma->vm_start;
+ mas_reset(&mas);
+
+ /*
+ * If vm_flags changed after call_mmap(), we should try merge
+ * vma again as we may succeed this time.
+ */
+ if (unlikely(vm_flags != vma->vm_flags && prev)) {
+ merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags,
+ NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
+ if (merge) {
+ /*
+ * ->mmap() can change vma->vm_file and fput
+ * the original file. So fput the vma->vm_file
+ * here or we would add an extra fput for file
+ * and cause general protection fault
+ * ultimately.
+ */
+ fput(vma->vm_file);
+ vm_area_free(vma);
+ vma = merge;
+ /* Update vm_flags to pick up the change. */
+ addr = vma->vm_start;
+ vm_flags = vma->vm_flags;
+ goto unmap_writable;
+ }
+ }
+
+ vm_flags = vma->vm_flags;
+ } else if (vm_flags & VM_SHARED) {
+ error = shmem_zero_setup(vma);
+ if (error)
+ goto free_vma;
+ } else {
+ vma_set_anonymous(vma);
+ }
+
+ /* Allow architectures to sanity-check the vm_flags */
+ if (!arch_validate_flags(vma->vm_flags)) {
+ error = -EINVAL;
+ if (file)
+ goto close_and_free_vma;
+ else
+ goto free_vma;
+ }
+
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ error = -ENOMEM;
+ if (file)
+ goto unmap_and_free_vma;
+ else
+ goto free_vma;
+ }
+
+ if (vma->vm_file)
+ i_mmap_lock_write(vma->vm_file->f_mapping);
+
+ vma_mas_store(vma, &mas);
+ mm->map_count++;
+ if (vma->vm_file) {
+ if (vma->vm_flags & VM_SHARED)
+ mapping_allow_writable(vma->vm_file->f_mapping);
+
+ flush_dcache_mmap_lock(vma->vm_file->f_mapping);
+ vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap);
+ flush_dcache_mmap_unlock(vma->vm_file->f_mapping);
+ i_mmap_unlock_write(vma->vm_file->f_mapping);
+ }
+
+ /*
+ * vma_merge() calls khugepaged_enter_vma() either, the below
+ * call covers the non-merge case.
+ */
+ khugepaged_enter_vma(vma, vma->vm_flags);
+
+ /* Once vma denies write, undo our temporary denial count */
+unmap_writable:
+ if (file && vm_flags & VM_SHARED)
+ mapping_unmap_writable(file->f_mapping);
+ file = vma->vm_file;
+expanded:
+ perf_event_mmap(vma);
+
+ vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
+ if (vm_flags & VM_LOCKED) {
+ if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
+ is_vm_hugetlb_page(vma) ||
+ vma == get_gate_vma(current->mm))
+ vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
+ else
+ mm->locked_vm += (len >> PAGE_SHIFT);
+ }
+
+ if (file)
+ uprobe_mmap(vma);
+
+ /*
+ * New (or expanded) vma always get soft dirty status.
+ * Otherwise user-space soft-dirty page tracker won't
+ * be able to distinguish situation when vma area unmapped,
+ * then new mapped in-place (which must be aimed as
+ * a completely new data area).
+ */
+ vma->vm_flags |= VM_SOFTDIRTY;
+
+ vma_set_page_prot(vma);
+
+ validate_mm(mm);
+ return addr;
+
+close_and_free_vma:
+ if (vma->vm_ops && vma->vm_ops->close)
+ vma->vm_ops->close(vma);
+unmap_and_free_vma:
+ fput(vma->vm_file);
+ vma->vm_file = NULL;
+
+ /* Undo any partial mapping done by a device driver. */
+ unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end);
+ if (vm_flags & VM_SHARED)
+ mapping_unmap_writable(file->f_mapping);
+free_vma:
+ vm_area_free(vma);
+unacct_error:
+ if (charged)
+ vm_unacct_memory(charged);
+ validate_mm(mm);
+ return error;
}
static int __vm_munmap(unsigned long start, size_t len, bool downgrade)
int ret;
struct mm_struct *mm = current->mm;
LIST_HEAD(uf);
+ MA_STATE(mas, &mm->mm_mt, start, start);
if (mmap_write_lock_killable(mm))
return -EINTR;
- ret = __do_munmap(mm, start, len, &uf, downgrade);
+ ret = do_mas_munmap(&mas, mm, start, len, &uf, downgrade);
/*
* Returning 1 indicates mmap_lock is downgraded.
* But 1 is not legal return value of vm_munmap() and munmap(), reset
goto out;
if (start + size > vma->vm_end) {
- struct vm_area_struct *next;
+ VMA_ITERATOR(vmi, mm, vma->vm_end);
+ struct vm_area_struct *next, *prev = vma;
- for (next = vma->vm_next; next; next = next->vm_next) {
+ for_each_vma_range(vmi, next, start + size) {
/* hole between vmas ? */
- if (next->vm_start != next->vm_prev->vm_end)
+ if (next->vm_start != prev->vm_end)
goto out;
if (next->vm_file != vma->vm_file)
if (next->vm_flags != vma->vm_flags)
goto out;
- if (start + size <= next->vm_end)
- break;
+ prev = next;
}
if (!next)
}
/*
- * this is really a simplified "do_mmap". it only handles
- * anonymous maps. eventually we may be able to do some
- * brk-specific accounting here.
+ * brk_munmap() - Unmap a parital vma.
+ * @mas: The maple tree state.
+ * @vma: The vma to be modified
+ * @newbrk: the start of the address to unmap
+ * @oldbrk: The end of the address to unmap
+ * @uf: The userfaultfd list_head
+ *
+ * Returns: 1 on success.
+ * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if
+ * possible.
*/
-static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf)
+static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma,
+ unsigned long newbrk, unsigned long oldbrk,
+ struct list_head *uf)
{
- struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma, *prev;
- struct rb_node **rb_link, *rb_parent;
- pgoff_t pgoff = addr >> PAGE_SHIFT;
- int error;
- unsigned long mapped_addr;
-
- /* Until we need other flags, refuse anything except VM_EXEC. */
- if ((flags & (~VM_EXEC)) != 0)
- return -EINVAL;
- flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
-
- mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
- if (IS_ERR_VALUE(mapped_addr))
- return mapped_addr;
+ struct mm_struct *mm = vma->vm_mm;
+ int ret;
- error = mlock_future_check(mm, mm->def_flags, len);
- if (error)
- return error;
+ arch_unmap(mm, newbrk, oldbrk);
+ ret = do_mas_align_munmap(mas, vma, mm, newbrk, oldbrk, uf, true);
+ validate_mm_mt(mm);
+ return ret;
+}
- /* Clear old maps, set up prev, rb_link, rb_parent, and uf */
- if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf))
- return -ENOMEM;
+/*
+ * do_brk_flags() - Increase the brk vma if the flags match.
+ * @mas: The maple tree state.
+ * @addr: The start address
+ * @len: The length of the increase
+ * @vma: The vma,
+ * @flags: The VMA Flags
+ *
+ * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags
+ * do not match then create a new anonymous VMA. Eventually we may be able to
+ * do some brk-specific accounting here.
+ */
+static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma,
+ unsigned long addr, unsigned long len, unsigned long flags)
+{
+ struct mm_struct *mm = current->mm;
- /* Check against address space limits *after* clearing old maps... */
+ validate_mm_mt(mm);
+ /*
+ * Check against address space limits by the changed size
+ * Note: This happens *after* clearing old mappings in some code paths.
+ */
+ flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
return -ENOMEM;
if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
return -ENOMEM;
- /* Can we just expand an old private anonymous mapping? */
- vma = vma_merge(mm, prev, addr, addr + len, flags,
- NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
- if (vma)
- goto out;
-
/*
- * create a vma struct for an anonymous mapping
+ * Expand the existing vma if possible; Note that singular lists do not
+ * occur after forking, so the expand will only happen on new VMAs.
*/
- vma = vm_area_alloc(mm);
- if (!vma) {
- vm_unacct_memory(len >> PAGE_SHIFT);
- return -ENOMEM;
+ if (vma &&
+ (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) &&
+ ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) {
+ mas_set_range(mas, vma->vm_start, addr + len - 1);
+ if (mas_preallocate(mas, vma, GFP_KERNEL))
+ return -ENOMEM;
+
+ vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0);
+ if (vma->anon_vma) {
+ anon_vma_lock_write(vma->anon_vma);
+ anon_vma_interval_tree_pre_update_vma(vma);
+ }
+ vma->vm_end = addr + len;
+ vma->vm_flags |= VM_SOFTDIRTY;
+ mas_store_prealloc(mas, vma);
+
+ if (vma->anon_vma) {
+ anon_vma_interval_tree_post_update_vma(vma);
+ anon_vma_unlock_write(vma->anon_vma);
+ }
+ khugepaged_enter_vma(vma, flags);
+ goto out;
}
+ /* create a vma struct for an anonymous mapping */
+ vma = vm_area_alloc(mm);
+ if (!vma)
+ goto vma_alloc_fail;
+
vma_set_anonymous(vma);
vma->vm_start = addr;
vma->vm_end = addr + len;
- vma->vm_pgoff = pgoff;
+ vma->vm_pgoff = addr >> PAGE_SHIFT;
vma->vm_flags = flags;
vma->vm_page_prot = vm_get_page_prot(flags);
- vma_link(mm, vma, prev, rb_link, rb_parent);
+ mas_set_range(mas, vma->vm_start, addr + len - 1);
+ if (mas_store_gfp(mas, vma, GFP_KERNEL))
+ goto mas_store_fail;
+
+ mm->map_count++;
out:
perf_event_mmap(vma);
mm->total_vm += len >> PAGE_SHIFT;
if (flags & VM_LOCKED)
mm->locked_vm += (len >> PAGE_SHIFT);
vma->vm_flags |= VM_SOFTDIRTY;
+ validate_mm(mm);
return 0;
+
+mas_store_fail:
+ vm_area_free(vma);
+vma_alloc_fail:
+ vm_unacct_memory(len >> PAGE_SHIFT);
+ return -ENOMEM;
}
int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
{
struct mm_struct *mm = current->mm;
+ struct vm_area_struct *vma = NULL;
unsigned long len;
int ret;
bool populate;
LIST_HEAD(uf);
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
len = PAGE_ALIGN(request);
if (len < request)
if (mmap_write_lock_killable(mm))
return -EINTR;
- ret = do_brk_flags(addr, len, flags, &uf);
+ /* Until we need other flags, refuse anything except VM_EXEC. */
+ if ((flags & (~VM_EXEC)) != 0)
+ return -EINVAL;
+
+ ret = check_brk_limits(addr, len);
+ if (ret)
+ goto limits_failed;
+
+ ret = do_mas_munmap(&mas, mm, addr, len, &uf, 0);
+ if (ret)
+ goto munmap_failed;
+
+ vma = mas_prev(&mas, 0);
+ if (!vma || vma->vm_end != addr || vma_policy(vma) ||
+ !can_vma_merge_after(vma, flags, NULL, NULL,
+ addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL))
+ vma = NULL;
+
+ ret = do_brk_flags(&mas, vma, addr, len, flags);
populate = ((mm->def_flags & VM_LOCKED) != 0);
mmap_write_unlock(mm);
userfaultfd_unmap_complete(mm, &uf);
if (populate && !ret)
mm_populate(addr, len);
return ret;
+
+munmap_failed:
+limits_failed:
+ mmap_write_unlock(mm);
+ return ret;
}
EXPORT_SYMBOL(vm_brk_flags);
struct mmu_gather tlb;
struct vm_area_struct *vma;
unsigned long nr_accounted = 0;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
+ int count = 0;
/* mm's last user has gone, and its about to be pulled down */
mmu_notifier_release(mm);
- if (unlikely(mm_is_oom_victim(mm))) {
- /*
- * Manually reap the mm to free as much memory as possible.
- * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard
- * this mm from further consideration. Taking mm->mmap_lock for
- * write after setting MMF_OOM_SKIP will guarantee that the oom
- * reaper will not run on this mm again after mmap_lock is
- * dropped.
- *
- * Nothing can be holding mm->mmap_lock here and the above call
- * to mmu_notifier_release(mm) ensures mmu notifier callbacks in
- * __oom_reap_task_mm() will not block.
- */
- (void)__oom_reap_task_mm(mm);
- set_bit(MMF_OOM_SKIP, &mm->flags);
- }
-
- mmap_write_lock(mm);
+ mmap_read_lock(mm);
arch_exit_mmap(mm);
- vma = mm->mmap;
+ vma = mas_find(&mas, ULONG_MAX);
if (!vma) {
/* Can happen if dup_mmap() received an OOM */
- mmap_write_unlock(mm);
+ mmap_read_unlock(mm);
return;
}
flush_cache_mm(mm);
tlb_gather_mmu_fullmm(&tlb, mm);
/* update_hiwater_rss(mm) here? but nobody should be looking */
- /* Use -1 here to ensure all VMAs in the mm are unmapped */
- unmap_vmas(&tlb, vma, 0, -1);
- free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
+ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
+ unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX);
+ mmap_read_unlock(mm);
+
+ /*
+ * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper
+ * because the memory has been already freed.
+ */
+ set_bit(MMF_OOM_SKIP, &mm->flags);
+ mmap_write_lock(mm);
+ free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS,
+ USER_PGTABLES_CEILING);
tlb_finish_mmu(&tlb);
- /* Walk the list again, actually closing and freeing it. */
- while (vma) {
+ /*
+ * Walk the list again, actually closing and freeing it, with preemption
+ * enabled, without holding any MM locks besides the unreachable
+ * mmap_write_lock.
+ */
+ do {
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += vma_pages(vma);
- vma = remove_vma(vma);
+ remove_vma(vma);
+ count++;
cond_resched();
- }
- mm->mmap = NULL;
+ } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL);
+
+ BUG_ON(count != mm->map_count);
+
+ trace_exit_mmap(mm);
+ __mt_destroy(&mm->mm_mt);
mmap_write_unlock(mm);
vm_unacct_memory(nr_accounted);
}
*/
int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
{
- struct vm_area_struct *prev;
- struct rb_node **rb_link, *rb_parent;
+ unsigned long charged = vma_pages(vma);
+
- if (find_vma_links(mm, vma->vm_start, vma->vm_end,
- &prev, &rb_link, &rb_parent))
+ if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
return -ENOMEM;
+
if ((vma->vm_flags & VM_ACCOUNT) &&
- security_vm_enough_memory_mm(mm, vma_pages(vma)))
+ security_vm_enough_memory_mm(mm, charged))
return -ENOMEM;
/*
vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
}
- vma_link(mm, vma, prev, rb_link, rb_parent);
+ if (vma_link(mm, vma)) {
+ vm_unacct_memory(charged);
+ return -ENOMEM;
+ }
+
return 0;
}
unsigned long vma_start = vma->vm_start;
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *new_vma, *prev;
- struct rb_node **rb_link, *rb_parent;
bool faulted_in_anon_vma = true;
+ validate_mm_mt(mm);
/*
* If anonymous vma has not yet been faulted, update new pgoff
* to match new location, to increase its chance of merging.
faulted_in_anon_vma = false;
}
- if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
+ new_vma = find_vma_prev(mm, addr, &prev);
+ if (new_vma && new_vma->vm_start < addr + len)
return NULL; /* should never get here */
+
new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_vma_name(vma));
get_file(new_vma->vm_file);
if (new_vma->vm_ops && new_vma->vm_ops->open)
new_vma->vm_ops->open(new_vma);
- vma_link(mm, new_vma, prev, rb_link, rb_parent);
+ if (vma_link(mm, new_vma))
+ goto out_vma_link;
*need_rmap_locks = false;
}
+ validate_mm_mt(mm);
return new_vma;
+out_vma_link:
+ if (new_vma->vm_ops && new_vma->vm_ops->close)
+ new_vma->vm_ops->close(new_vma);
+
+ if (new_vma->vm_file)
+ fput(new_vma->vm_file);
+
+ unlink_anon_vmas(new_vma);
out_free_mempol:
mpol_put(vma_policy(new_vma));
out_free_vma:
vm_area_free(new_vma);
out:
+ validate_mm_mt(mm);
return NULL;
}
int ret;
struct vm_area_struct *vma;
+ validate_mm_mt(mm);
vma = vm_area_alloc(mm);
if (unlikely(vma == NULL))
return ERR_PTR(-ENOMEM);
perf_event_mmap(vma);
+ validate_mm_mt(mm);
return vma;
out:
vm_area_free(vma);
+ validate_mm_mt(mm);
return ERR_PTR(ret);
}
{
struct vm_area_struct *vma;
struct anon_vma_chain *avc;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_assert_write_locked(mm);
mutex_lock(&mm_all_locks_mutex);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
if (vma->vm_file && vma->vm_file->f_mapping &&
vm_lock_mapping(mm, vma->vm_file->f_mapping);
}
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_set(&mas, 0);
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
if (vma->vm_file && vma->vm_file->f_mapping &&
vm_lock_mapping(mm, vma->vm_file->f_mapping);
}
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_set(&mas, 0);
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (signal_pending(current))
goto out_unlock;
if (vma->anon_vma)
{
struct vm_area_struct *vma;
struct anon_vma_chain *avc;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
mmap_assert_write_locked(mm);
BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->anon_vma)
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
vm_unlock_anon_vma(avc->anon_vma);
#include <linux/gfp.h>
#include <linux/highmem.h>
#include <linux/kernel.h>
+#include <linux/kmsan-checks.h>
#include <linux/mmdebug.h>
#include <linux/mm_types.h>
#include <linux/mm_inline.h>
static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
bool fullmm)
{
+ /*
+ * struct mmu_gather contains 7 1-bit fields packed into a 32-bit
+ * unsigned int value. The remaining 25 bits remain uninitialized
+ * and are never used, but KMSAN updates the origin for them in
+ * zap_pXX_range() in mm/memory.c, thus creating very long origin
+ * chains. This is technically correct, but consumes too much memory.
+ * Unpoisoning the whole structure will prevent creating such chains.
+ */
+ kmsan_unpoison_memory(tlb, sizeof(*tlb));
tlb->mm = mm;
tlb->fullmm = fullmm;
* Poison its list head, so that any operations on it would crash.
*/
list_del(&lruvec->lists[LRU_UNEVICTABLE]);
+
+ lru_gen_init_lruvec(lruvec);
}
#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)
#include <linux/pgtable.h>
#include <linux/sched/sysctl.h>
#include <linux/userfaultfd_k.h>
+#include <linux/memory-tiers.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
if (prot_numa) {
struct page *page;
int nid;
+ bool toptier;
/* Avoid TLB flush if possible */
if (pte_protnone(oldpte))
nid = page_to_nid(page);
if (target_node == nid)
continue;
+ toptier = node_is_toptier(nid);
/*
* Skip scanning top tier node if normal numa
* balancing is disabled
*/
if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
- node_is_toptier(nid))
+ toptier)
continue;
+ if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
+ !toptier)
+ xchg_page_access_time(page,
+ jiffies_to_msecs(jiffies));
}
oldpte = ptep_modify_prot_start(vma, addr, pte);
} else {
/* It must be an none page, or what else?.. */
WARN_ON_ONCE(!pte_none(oldpte));
+#ifdef CONFIG_PTE_MARKER_UFFD_WP
if (unlikely(uffd_wp && !vma_is_anonymous(vma))) {
/*
* For file-backed mem, we need to be able to
make_pte_marker(PTE_MARKER_UFFD_WP));
pages++;
}
+#endif
}
} while (pte++, addr += PAGE_SIZE, addr != end);
arch_leave_lazy_mmu_mode();
const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
(prot & PROT_READ);
struct mmu_gather tlb;
+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
start = untagged_addr(start);
if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
goto out;
- vma = find_vma(current->mm, start);
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
error = -ENOMEM;
if (!vma)
goto out;
if (start > vma->vm_start)
prev = vma;
else
- prev = vma->vm_prev;
+ prev = mas_prev(&mas, 0);
tlb_gather_mmu(&tlb, current->mm);
for (nstart = start ; ; ) {
if (nstart >= end)
break;
- vma = prev->vm_next;
+ vma = find_vma(current->mm, prev->vm_end);
if (!vma || vma->vm_start != nstart) {
error = -ENOMEM;
break;
*/
#include <linux/mm.h>
+#include <linux/mm_inline.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/ksm.h>
#include <linux/mmu_notifier.h>
#include <linux/uaccess.h>
#include <linux/userfaultfd_k.h>
+#include <linux/mempolicy.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>
if (excess) {
vma->vm_flags |= VM_ACCOUNT;
if (split)
- vma->vm_next->vm_flags |= VM_ACCOUNT;
+ find_vma(mm, vma->vm_end)->vm_flags |= VM_ACCOUNT;
}
return new_addr;
static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
{
unsigned long end = vma->vm_end + delta;
+
if (end < vma->vm_end) /* overflow */
return 0;
- if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */
+ if (find_vma_intersection(vma->vm_mm, vma->vm_end, end))
return 0;
if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start,
0, MAP_FIXED) & ~PAGE_MASK)
/*
* Always allow a shrinking remap: that just unmaps
* the unnecessary pages..
- * __do_munmap does all the needed commit accounting, and
+ * do_mas_munmap does all the needed commit accounting, and
* downgrades mmap_lock to read if so directed.
*/
if (old_len >= new_len) {
int retval;
+ MA_STATE(mas, &mm->mm_mt, addr + new_len, addr + new_len);
- retval = __do_munmap(mm, addr+new_len, old_len - new_len,
- &uf_unmap, true);
- if (retval < 0 && old_len != new_len) {
- ret = retval;
- goto out;
+ retval = do_mas_munmap(&mas, mm, addr + new_len,
+ old_len - new_len, &uf_unmap, true);
/* Returning 1 indicates mmap_lock is downgraded to read. */
- } else if (retval == 1)
+ if (retval == 1) {
downgraded = true;
+ } else if (retval < 0 && old_len != new_len) {
+ ret = retval;
+ goto out;
+ }
+
ret = addr;
goto out;
}
/* can we just expand the current mapping? */
if (vma_expandable(vma, new_len - old_len)) {
long pages = (new_len - old_len) >> PAGE_SHIFT;
+ unsigned long extension_start = addr + old_len;
+ unsigned long extension_end = addr + new_len;
+ pgoff_t extension_pgoff = vma->vm_pgoff + (old_len >> PAGE_SHIFT);
if (vma->vm_flags & VM_ACCOUNT) {
if (security_vm_enough_memory_mm(mm, pages)) {
}
}
- if (vma_adjust(vma, vma->vm_start, addr + new_len,
- vma->vm_pgoff, NULL)) {
+ /*
+ * Function vma_merge() is called on the extension we are adding to
+ * the already existing vma, vma_merge() will merge this extension with
+ * the already existing vma (expand operation itself) and possibly also
+ * with the next vma if it becomes adjacent to the expanded vma and
+ * otherwise compatible.
+ */
+ vma = vma_merge(mm, vma, extension_start, extension_end,
+ vma->vm_flags, vma->anon_vma, vma->vm_file,
+ extension_pgoff, vma_policy(vma),
+ vma->vm_userfaultfd_ctx, anon_vma_name(vma));
+ if (!vma) {
vm_unacct_memory(pages);
ret = -ENOMEM;
goto out;
error = 0;
goto out_unlock;
}
- vma = vma->vm_next;
+ vma = find_vma(mm, vma->vm_end);
}
}
out_unlock:
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
-#include <linux/vmacache.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/file.h>
__put_nommu_region(region);
}
-/*
- * add a VMA into a process's mm_struct in the appropriate place in the list
- * and tree and add to the address space's page tree also if not an anonymous
- * page
- * - should be called with mm->mmap_lock held writelocked
- */
-static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
+void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas)
{
- struct vm_area_struct *pvma, *prev;
- struct address_space *mapping;
- struct rb_node **p, *parent, *rb_prev;
+ mas_set_range(mas, vma->vm_start, vma->vm_end - 1);
+ mas_store_prealloc(mas, vma);
+}
- BUG_ON(!vma->vm_region);
+void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas)
+{
+ mas->index = vma->vm_start;
+ mas->last = vma->vm_end - 1;
+ mas_store_prealloc(mas, NULL);
+}
+static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm)
+{
mm->map_count++;
vma->vm_mm = mm;
/* add the VMA to the mapping */
if (vma->vm_file) {
- mapping = vma->vm_file->f_mapping;
+ struct address_space *mapping = vma->vm_file->f_mapping;
i_mmap_lock_write(mapping);
flush_dcache_mmap_lock(mapping);
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
+}
- /* add the VMA to the tree */
- parent = rb_prev = NULL;
- p = &mm->mm_rb.rb_node;
- while (*p) {
- parent = *p;
- pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
-
- /* sort by: start addr, end addr, VMA struct addr in that order
- * (the latter is necessary as we may get identical VMAs) */
- if (vma->vm_start < pvma->vm_start)
- p = &(*p)->rb_left;
- else if (vma->vm_start > pvma->vm_start) {
- rb_prev = parent;
- p = &(*p)->rb_right;
- } else if (vma->vm_end < pvma->vm_end)
- p = &(*p)->rb_left;
- else if (vma->vm_end > pvma->vm_end) {
- rb_prev = parent;
- p = &(*p)->rb_right;
- } else if (vma < pvma)
- p = &(*p)->rb_left;
- else if (vma > pvma) {
- rb_prev = parent;
- p = &(*p)->rb_right;
- } else
- BUG();
- }
-
- rb_link_node(&vma->vm_rb, parent, p);
- rb_insert_color(&vma->vm_rb, &mm->mm_rb);
+/*
+ * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm().
+ * @mas: The maple state with preallocations.
+ * @mm: The mm_struct
+ * @vma: The vma to add
+ *
+ */
+static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm,
+ struct vm_area_struct *vma)
+{
+ BUG_ON(!vma->vm_region);
- /* add VMA to the VMA list also */
- prev = NULL;
- if (rb_prev)
- prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
+ setup_vma_to_mm(vma, mm);
- __vma_link_list(mm, vma, prev);
+ /* add the VMA to the tree */
+ vma_mas_store(vma, mas);
}
/*
- * delete a VMA from its owning mm_struct and address space
+ * add a VMA into a process's mm_struct in the appropriate place in the list
+ * and tree and add to the address space's page tree also if not an anonymous
+ * page
+ * - should be called with mm->mmap_lock held writelocked
*/
-static void delete_vma_from_mm(struct vm_area_struct *vma)
-{
- int i;
- struct address_space *mapping;
- struct mm_struct *mm = vma->vm_mm;
- struct task_struct *curr = current;
-
- mm->map_count--;
- for (i = 0; i < VMACACHE_SIZE; i++) {
- /* if the vma is cached, invalidate the entire cache */
- if (curr->vmacache.vmas[i] == vma) {
- vmacache_invalidate(mm);
- break;
- }
+static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
+{
+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
+
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ pr_warn("Allocation of vma tree for process %d failed\n",
+ current->pid);
+ return -ENOMEM;
}
+ mas_add_vma_to_mm(&mas, mm, vma);
+ return 0;
+}
+static void cleanup_vma_from_mm(struct vm_area_struct *vma)
+{
+ vma->vm_mm->map_count--;
/* remove the VMA from the mapping */
if (vma->vm_file) {
+ struct address_space *mapping;
mapping = vma->vm_file->f_mapping;
i_mmap_lock_write(mapping);
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
+}
+/*
+ * delete a VMA from its owning mm_struct and address space
+ */
+static int delete_vma_from_mm(struct vm_area_struct *vma)
+{
+ MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
- /* remove from the MM's tree and list */
- rb_erase(&vma->vm_rb, &mm->mm_rb);
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ pr_warn("Allocation of vma tree for process %d failed\n",
+ current->pid);
+ return -ENOMEM;
+ }
+ cleanup_vma_from_mm(vma);
- __vma_unlink_list(mm, vma);
+ /* remove from the MM's tree and list */
+ vma_mas_remove(vma, &mas);
+ return 0;
}
/*
vm_area_free(vma);
}
+struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
+ unsigned long start_addr,
+ unsigned long end_addr)
+{
+ unsigned long index = start_addr;
+
+ mmap_assert_locked(mm);
+ return mt_find(&mm->mm_mt, &index, end_addr - 1);
+}
+EXPORT_SYMBOL(find_vma_intersection);
+
/*
* look up the first VMA in which addr resides, NULL if none
* - should be called with mm->mmap_lock at least held readlocked
*/
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
- struct vm_area_struct *vma;
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
- /* check the cache first */
- vma = vmacache_find(mm, addr);
- if (likely(vma))
- return vma;
-
- /* trawl the list (there may be multiple mappings in which addr
- * resides) */
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
- if (vma->vm_start > addr)
- return NULL;
- if (vma->vm_end > addr) {
- vmacache_update(addr, vma);
- return vma;
- }
- }
-
- return NULL;
+ return mas_walk(&mas);
}
EXPORT_SYMBOL(find_vma);
{
struct vm_area_struct *vma;
unsigned long end = addr + len;
+ MA_STATE(mas, &mm->mm_mt, addr, addr);
- /* check the cache first */
- vma = vmacache_find_exact(mm, addr, end);
- if (vma)
- return vma;
-
- /* trawl the list (there may be multiple mappings in which addr
- * resides) */
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
- if (vma->vm_start < addr)
- continue;
- if (vma->vm_start > addr)
- return NULL;
- if (vma->vm_end == end) {
- vmacache_update(addr, vma);
- return vma;
- }
- }
+ vma = mas_walk(&mas);
+ if (!vma)
+ return NULL;
+ if (vma->vm_start != addr)
+ return NULL;
+ if (vma->vm_end != end)
+ return NULL;
- return NULL;
+ return vma;
}
/*
vm_flags_t vm_flags;
unsigned long capabilities, result;
int ret;
+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
*populate = 0;
* now know into VMA flags */
vm_flags = determine_vm_flags(file, prot, flags, capabilities);
+
/* we're going to need to record the mapping */
region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
if (!region)
if (!vma)
goto error_getting_vma;
+ if (mas_preallocate(&mas, vma, GFP_KERNEL))
+ goto error_maple_preallocate;
+
region->vm_usage = 1;
region->vm_flags = vm_flags;
region->vm_pgoff = pgoff;
current->mm->total_vm += len >> PAGE_SHIFT;
share:
- add_vma_to_mm(current->mm, vma);
+ mas_add_vma_to_mm(&mas, current->mm, vma);
/* we flush the region from the icache only when the first executable
* mapping of it is made */
sharing_violation:
up_write(&nommu_region_sem);
+ mas_destroy(&mas);
pr_warn("Attempt to share mismatched mappings\n");
ret = -EINVAL;
goto error;
len, current->pid);
show_free_areas(0, NULL);
return -ENOMEM;
+
+error_maple_preallocate:
+ kmem_cache_free(vm_region_jar, region);
+ vm_area_free(vma);
+ pr_warn("Allocation of vma tree for process %d failed\n", current->pid);
+ show_free_areas(0, NULL);
+ return -ENOMEM;
+
}
unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
struct vm_area_struct *new;
struct vm_region *region;
unsigned long npages;
+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
/* we're only permitted to split anonymous regions (these should have
* only a single usage on the region) */
return -ENOMEM;
new = vm_area_dup(vma);
- if (!new) {
- kmem_cache_free(vm_region_jar, region);
- return -ENOMEM;
+ if (!new)
+ goto err_vma_dup;
+
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ pr_warn("Allocation of vma tree for process %d failed\n",
+ current->pid);
+ goto err_mas_preallocate;
}
/* most fields are the same, copy all, and then fixup */
if (new->vm_ops && new->vm_ops->open)
new->vm_ops->open(new);
- delete_vma_from_mm(vma);
down_write(&nommu_region_sem);
delete_nommu_region(vma->vm_region);
if (new_below) {
add_nommu_region(vma->vm_region);
add_nommu_region(new->vm_region);
up_write(&nommu_region_sem);
- add_vma_to_mm(mm, vma);
- add_vma_to_mm(mm, new);
+
+ setup_vma_to_mm(vma, mm);
+ setup_vma_to_mm(new, mm);
+ mas_set_range(&mas, vma->vm_start, vma->vm_end - 1);
+ mas_store(&mas, vma);
+ vma_mas_store(new, &mas);
return 0;
+
+err_mas_preallocate:
+ vm_area_free(new);
+err_vma_dup:
+ kmem_cache_free(vm_region_jar, region);
+ return -ENOMEM;
}
/*
/* adjust the VMA's pointers, which may reposition it in the MM's tree
* and list */
- delete_vma_from_mm(vma);
+ if (delete_vma_from_mm(vma))
+ return -ENOMEM;
if (from > vma->vm_start)
vma->vm_end = from;
else
vma->vm_start = to;
- add_vma_to_mm(mm, vma);
+ if (add_vma_to_mm(mm, vma))
+ return -ENOMEM;
/* cut the backing region down to size */
region = vma->vm_region;
*/
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
{
+ MA_STATE(mas, &mm->mm_mt, start, start);
struct vm_area_struct *vma;
unsigned long end;
- int ret;
+ int ret = 0;
len = PAGE_ALIGN(len);
if (len == 0)
end = start + len;
/* find the first potentially overlapping VMA */
- vma = find_vma(mm, start);
+ vma = mas_find(&mas, end - 1);
if (!vma) {
static int limit;
if (limit < 5) {
return -EINVAL;
if (end == vma->vm_end)
goto erase_whole_vma;
- vma = vma->vm_next;
+ vma = mas_next(&mas, end - 1);
} while (vma);
return -EINVAL;
} else {
}
erase_whole_vma:
- delete_vma_from_mm(vma);
+ if (delete_vma_from_mm(vma))
+ ret = -ENOMEM;
delete_vma(mm, vma);
- return 0;
+ return ret;
}
int vm_munmap(unsigned long addr, size_t len)
*/
void exit_mmap(struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
if (!mm)
mm->total_vm = 0;
- while ((vma = mm->mmap)) {
- mm->mmap = vma->vm_next;
- delete_vma_from_mm(vma);
+ /*
+ * Lock the mm to avoid assert complaining even though this is the only
+ * user of the mm
+ */
+ mmap_write_lock(mm);
+ for_each_vma(vmi, vma) {
+ cleanup_vma_from_mm(vma);
delete_vma(mm, vma);
cond_resched();
}
+ __mt_destroy(&mm->mm_mt);
+ mmap_write_unlock(mm);
}
int vm_brk(unsigned long addr, unsigned long len)
if (is_memcg_oom(oc))
mem_cgroup_print_oom_meminfo(oc->memcg);
else {
- show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask);
+ __show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask, gfp_zone(oc->gfp_mask));
if (should_dump_unreclaim_slab())
dump_unreclaimable_slab();
}
static struct task_struct *oom_reaper_list;
static DEFINE_SPINLOCK(oom_reaper_lock);
-bool __oom_reap_task_mm(struct mm_struct *mm)
+static bool __oom_reap_task_mm(struct mm_struct *mm)
{
struct vm_area_struct *vma;
bool ret = true;
+ VMA_ITERATOR(vmi, mm, 0);
/*
* Tell all users of get_user/copy_from_user etc... that the content
*/
set_bit(MMF_UNSTABLE, &mm->flags);
- for (vma = mm->mmap ; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP))
continue;
return;
/* oom_mm is bound to the signal struct life time. */
- if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) {
+ if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm))
mmgrab(tsk->signal->oom_mm);
- set_bit(MMF_OOM_VICTIM, &mm->flags);
- }
/*
* Make sure that the task is woken up from uninterruptible sleep
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/kasan.h>
+#include <linux/kmsan.h>
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
{
static unsigned long prev_end_pfn, nr_initialised;
+ if (early_page_ext_enabled())
+ return false;
/*
* prev_end_pfn static that contains the end of previous zone
* No need to protect because called very early in boot before smp_init.
#ifdef CONFIG_SPARSEMEM
pfn &= (PAGES_PER_SECTION-1);
#else
- pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
+ pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn);
#endif /* CONFIG_SPARSEMEM */
return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
}
INIT_LIST_HEAD(&page->buddy_list);
set_page_private(page, order);
/* Guard pages are not available for any usage */
- __mod_zone_freepage_state(zone, -(1 << order), migratetype);
+ if (!is_migrate_isolate(migratetype))
+ __mod_zone_freepage_state(zone, -(1 << order), migratetype);
return true;
}
* order of appearance. So we need to first gather the full picture of what was
* enabled, and then make decisions.
*/
-void init_mem_debugging_and_hardening(void)
+void __init init_mem_debugging_and_hardening(void)
{
bool page_poisoning_requested = false;
else
static_branch_disable(&init_on_free);
+ if (IS_ENABLED(CONFIG_KMSAN) &&
+ (_init_on_alloc_enabled_early || _init_on_free_enabled_early))
+ pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");
+
#ifdef CONFIG_DEBUG_PAGEALLOC
if (!debug_pagealloc_enabled())
return;
int migratetype, fpi_t fpi_flags)
{
struct capture_control *capc = task_capc(zone);
- unsigned long buddy_pfn;
+ unsigned long buddy_pfn = 0;
unsigned long combined_pfn;
struct page *buddy;
bool to_tail;
return bad_reason;
}
-static void check_free_page_bad(struct page *page)
+static void free_page_is_bad_report(struct page *page)
{
bad_page(page,
page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
}
-static inline int check_free_page(struct page *page)
+static inline bool free_page_is_bad(struct page *page)
{
if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
- return 0;
+ return false;
/* Something has gone sideways, find it */
- check_free_page_bad(page);
- return 1;
+ free_page_is_bad_report(page);
+ return true;
}
static int free_tail_pages_check(struct page *head_page, struct page *page)
VM_BUG_ON_PAGE(PageTail(page), page);
trace_mm_page_free(page, order);
+ kmsan_free_page(page, order);
if (unlikely(PageHWPoison(page)) && !order) {
/*
for (i = 1; i < (1 << order); i++) {
if (compound)
bad += free_tail_pages_check(page, page + i);
- if (unlikely(check_free_page(page + i))) {
+ if (unlikely(free_page_is_bad(page + i))) {
bad++;
continue;
}
page->mapping = NULL;
if (memcg_kmem_enabled() && PageMemcgKmem(page))
__memcg_kmem_uncharge_page(page, order);
- if (check_free)
- bad += check_free_page(page);
+ if (check_free && free_page_is_bad(page))
+ bad++;
if (bad)
return false;
return free_pages_prepare(page, order, true, FPI_NONE);
}
+/* return true if this page has an inappropriate state */
static bool bulkfree_pcp_prepare(struct page *page)
{
if (debug_pagealloc_enabled_static())
- return check_free_page(page);
+ return free_page_is_bad(page);
else
return false;
}
static bool bulkfree_pcp_prepare(struct page *page)
{
- return check_free_page(page);
+ return free_page_is_bad(page);
}
#endif /* CONFIG_DEBUG_VM */
order = pindex_to_order(pindex);
nr_pages = 1 << order;
- BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH));
do {
int mt;
{
if (early_page_uninitialised(pfn))
return;
+ if (!kmsan_memblock_free_pages(page, order)) {
+ /* KMSAN will take care of these pages. */
+ return;
+ }
__free_pages_core(page, order);
}
unsigned long block_start_pfn = zone->zone_start_pfn;
unsigned long block_end_pfn;
- block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
+ block_end_pfn = pageblock_end_pfn(block_start_pfn);
for (; block_start_pfn < zone_end_pfn(zone);
block_start_pfn = block_end_pfn,
block_end_pfn += pageblock_nr_pages) {
page = pfn_to_page(pfn);
/* Free a large naturally-aligned chunk if possible */
- if (nr_pages == pageblock_nr_pages &&
- (pfn & (pageblock_nr_pages - 1)) == 0) {
+ if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) {
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
__free_pages_core(page, pageblock_order);
return;
}
for (i = 0; i < nr_pages; i++, page++, pfn++) {
- if ((pfn & (pageblock_nr_pages - 1)) == 0)
+ if (pageblock_aligned(pfn))
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
__free_pages_core(page, 0);
}
/*
* Returns true if page needs to be initialized or freed to buddy allocator.
*
- * First we check if pfn is valid on architectures where it is possible to have
- * holes within pageblock_nr_pages. On systems where it is not possible, this
- * function is optimized out.
- *
- * Then, we check if a current large page is valid by only checking the validity
+ * We check if a current large page is valid by only checking the validity
* of the head pfn.
*/
static inline bool __init deferred_pfn_valid(unsigned long pfn)
{
- if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
+ if (pageblock_aligned(pfn) && !pfn_valid(pfn))
return false;
return true;
}
static void __init deferred_free_pages(unsigned long pfn,
unsigned long end_pfn)
{
- unsigned long nr_pgmask = pageblock_nr_pages - 1;
unsigned long nr_free = 0;
for (; pfn < end_pfn; pfn++) {
if (!deferred_pfn_valid(pfn)) {
deferred_free_range(pfn - nr_free, nr_free);
nr_free = 0;
- } else if (!(pfn & nr_pgmask)) {
+ } else if (pageblock_aligned(pfn)) {
deferred_free_range(pfn - nr_free, nr_free);
nr_free = 1;
} else {
unsigned long pfn,
unsigned long end_pfn)
{
- unsigned long nr_pgmask = pageblock_nr_pages - 1;
int nid = zone_to_nid(zone);
unsigned long nr_pages = 0;
int zid = zone_idx(zone);
if (!deferred_pfn_valid(pfn)) {
page = NULL;
continue;
- } else if (!page || !(pfn & nr_pgmask)) {
+ } else if (!page || pageblock_aligned(pfn)) {
page = pfn_to_page(pfn);
} else {
page++;
*num_movable = 0;
pfn = page_to_pfn(page);
- start_pfn = pfn & ~(pageblock_nr_pages - 1);
- end_pfn = start_pfn + pageblock_nr_pages - 1;
+ start_pfn = pageblock_start_pfn(pfn);
+ end_pfn = pageblock_end_pfn(pfn) - 1;
/* Do not cross zone boundaries */
if (!zone_spans_pfn(zone, start_pfn))
* i.e. orders < pageblock_order. If there are no local zones free,
* the zonelists will be reiterated without ALLOC_NOFRAGMENT.
*/
- if (alloc_flags & ALLOC_NOFRAGMENT)
+ if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT)
min_order = pageblock_order;
/*
int pindex;
bool free_high;
- __count_vm_event(PGFREE);
+ __count_vm_events(PGFREE, 1 << order);
pindex = order_to_pindex(migratetype, order);
list_add(&page->pcp_list, &pcp->lists[pindex]);
pcp->count += 1 << order;
int __isolate_free_page(struct page *page, unsigned int order)
{
- unsigned long watermark;
- struct zone *zone;
- int mt;
-
- BUG_ON(!PageBuddy(page));
-
- zone = page_zone(page);
- mt = get_pageblock_migratetype(page);
+ struct zone *zone = page_zone(page);
+ int mt = get_pageblock_migratetype(page);
if (!is_migrate_isolate(mt)) {
+ unsigned long watermark;
/*
* Obey watermarks as if the page was being allocated. We can
* emulate a high-order watermark check with a raised order-0
__mod_zone_freepage_state(zone, -(1UL << order), mt);
}
- /* Remove page from free list */
-
del_page_from_free_list(page, zone, order);
/*
}
}
-
return 1UL << order;
}
/*
* Update NUMA hit/miss statistics
- *
- * Must be called with interrupts disabled.
*/
static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
long nr_account)
/* Lock and remove page from the per-cpu list */
static struct page *rmqueue_pcplist(struct zone *preferred_zone,
struct zone *zone, unsigned int order,
- gfp_t gfp_flags, int migratetype,
- unsigned int alloc_flags)
+ int migratetype, unsigned int alloc_flags)
{
struct per_cpu_pages *pcp;
struct list_head *list;
pcp_spin_unlock_irqrestore(pcp, flags);
pcp_trylock_finish(UP_flags);
if (page) {
- __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
+ __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
zone_statistics(preferred_zone, zone, 1);
}
return page;
}
/*
- * Allocate a page from the given zone. Use pcplists for order-0 allocations.
+ * Allocate a page from the given zone.
+ * Use pcplists for THP or "cheap" high-order allocations.
+ */
+
+/*
+ * Do not instrument rmqueue() with KMSAN. This function may call
+ * __msan_poison_alloca() through a call to set_pfnblock_flags_mask().
+ * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it
+ * may call rmqueue() again, which will result in a deadlock.
*/
+__no_sanitize_memory
static inline
struct page *rmqueue(struct zone *preferred_zone,
struct zone *zone, unsigned int order,
if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
migratetype != MIGRATE_MOVABLE) {
page = rmqueue_pcplist(preferred_zone, zone, order,
- gfp_flags, migratetype, alloc_flags);
+ migratetype, alloc_flags);
if (likely(page))
goto out;
}
if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
filter &= ~SHOW_MEM_FILTER_NODES;
- show_mem(filter, nodemask);
+ __show_mem(filter, nodemask, gfp_zone(gfp_mask));
}
void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
if (reserve_flags)
- alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags);
+ alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) |
+ (alloc_flags & ALLOC_KSWAPD);
/*
* Reset the nodemask and zonelist iterators if memory policies can be
* so that we can identify them and convert them to something
* else.
*/
- WARN_ON_ONCE_GFP(order > PAGE_ALLOC_COSTLY_ORDER, gfp_mask);
+ WARN_ON_ONCE_GFP(costly_order, gfp_mask);
/*
* Help non-failing allocations by giving them access to memory
}
trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
+ kmsan_alloc_page(page, order, alloc_gfp);
return page;
}
printk(KERN_CONT "(%s) ", tmp);
}
+static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx)
+{
+ int zone_idx;
+ for (zone_idx = 0; zone_idx <= max_zone_idx; zone_idx++)
+ if (zone_managed_pages(pgdat->node_zones + zone_idx))
+ return true;
+ return false;
+}
+
/*
* Show free area list (used inside shift_scroll-lock stuff)
* We also calculate the percentage fragmentation. We do this by counting the
* SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
* cpuset.
*/
-void show_free_areas(unsigned int filter, nodemask_t *nodemask)
+void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx)
{
unsigned long free_pcp = 0;
int cpu, nid;
pg_data_t *pgdat;
for_each_populated_zone(zone) {
+ if (zone_idx(zone) > max_zone_idx)
+ continue;
if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
continue;
for_each_online_pgdat(pgdat) {
if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
continue;
+ if (!node_has_managed_zones(pgdat, max_zone_idx))
+ continue;
printk("Node %d"
" active_anon:%lukB"
for_each_populated_zone(zone) {
int i;
+ if (zone_idx(zone) > max_zone_idx)
+ continue;
if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
continue;
unsigned long nr[MAX_ORDER], flags, total = 0;
unsigned char types[MAX_ORDER];
+ if (zone_idx(zone) > max_zone_idx)
+ continue;
if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
continue;
show_node(zone);
#define BOOT_PAGESET_BATCH 1
static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset);
static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats);
-DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
+static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
static void __build_all_zonelists(void *data)
{
* such that unmovable allocations won't be scattered all
* over the place during system boot.
*/
- if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
+ if (pageblock_aligned(pfn)) {
set_pageblock_migratetype(page, migratetype);
cond_resched();
}
* Please note that MEMINIT_HOTPLUG path doesn't clear memmap
* because this is done early in section_activate()
*/
- if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
+ if (pageblock_aligned(pfn)) {
set_pageblock_migratetype(page, MIGRATE_MOVABLE);
cond_resched();
}
+
+ /*
+ * ZONE_DEVICE pages are released directly to the driver page allocator
+ * which will set the page count to 1 when allocating the page.
+ */
+ if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
+ pgmap->type == MEMORY_DEVICE_COHERENT)
+ set_page_count(page, 0);
}
/*
unsigned long start = jiffies;
int nid = pgdat->node_id;
- if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE))
+ if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
return;
/*
u64 pgcnt = 0;
for (pfn = spfn; pfn < epfn; pfn++) {
- if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
- pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
- + pageblock_nr_pages - 1;
+ if (!pfn_valid(pageblock_start_pfn(pfn))) {
+ pfn = pageblock_end_pfn(pfn) - 1;
continue;
}
__init_single_page(pfn_to_page(pfn), pfn, zone, node);
* size is striking a balance between allocation latency
* and zone lock contention.
*/
- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE);
+ batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE);
batch /= 4; /* We effectively *= 4 below */
if (batch < 1)
batch = 1;
zone_set_pageset_high_and_batch(zone, 0);
}
+/*
+ * The zone indicated has a new number of managed_pages; batch sizes and percpu
+ * page high values need to be recalculated.
+ */
+static void zone_pcp_update(struct zone *zone, int cpu_online)
+{
+ mutex_lock(&pcp_batch_high_lock);
+ zone_set_pageset_high_and_batch(zone, cpu_online);
+ mutex_unlock(&pcp_batch_high_lock);
+}
+
/*
* Allocate per cpu pagesets and initialize them.
* Before this call only boot pagesets were available.
int i;
pgdat_resize_init(pgdat);
+ pgdat_kswapd_lock_init(pgdat);
pgdat_init_split_queue(pgdat);
pgdat_init_kcompactd(pgdat);
return ~accl_mask + 1;
}
-/**
- * find_min_pfn_with_active_regions - Find the minimum PFN registered
- *
- * Return: the minimum PFN based on information provided via
- * memblock_set_node().
- */
-unsigned long __init find_min_pfn_with_active_regions(void)
-{
- return PHYS_PFN(memblock_start_of_DRAM());
-}
-
/*
* early_calculate_totalpages()
* Sum pages in active regions for movable zone.
memset(arch_zone_highest_possible_pfn, 0,
sizeof(arch_zone_highest_possible_pfn));
- start_pfn = find_min_pfn_with_active_regions();
+ start_pfn = PHYS_PFN(memblock_start_of_DRAM());
descending = arch_has_descending_max_zone_pfns();
for (i = 0; i < MAX_NR_ZONES; i++) {
#endif
")\n",
K(nr_free_pages()), K(physpages),
- codesize >> 10, datasize >> 10, rosize >> 10,
- (init_data_size + init_code_size) >> 10, bss_size >> 10,
+ codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,
+ (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,
K(physpages - totalram_pages() - totalcma_pages),
K(totalcma_pages)
#ifdef CONFIG_HIGHMEM
{
unsigned long long max = high_limit;
unsigned long log2qty, size;
- void *table = NULL;
+ void *table;
gfp_t gfp_flags;
bool virt;
bool huge;
numentries -= arch_reserved_kernel_pages();
/* It isn't necessary when PAGE_SIZE >= 1MB */
- if (PAGE_SHIFT < 20)
- numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
+ if (PAGE_SIZE < SZ_1M)
+ numentries = round_up(numentries, SZ_1M / PAGE_SIZE);
#if __BITS_PER_LONG > 32
if (!high_limit) {
}
EXPORT_SYMBOL(free_contig_range);
-/*
- * The zone indicated has a new number of managed_pages; batch sizes and percpu
- * page high values need to be recalculated.
- */
-void zone_pcp_update(struct zone *zone, int cpu_online)
-{
- mutex_lock(&pcp_batch_high_lock);
- zone_set_pageset_high_and_batch(zone, cpu_online);
- mutex_unlock(&pcp_batch_high_lock);
-}
-
/*
* Effectively disable pcplists for the zone by setting the high limit to 0
* and draining all cpus. A concurrent page freeing on another CPU that's about
drain_zonestat(zone, pzstats);
}
free_percpu(zone->per_cpu_pageset);
- free_percpu(zone->per_cpu_zonestats);
zone->per_cpu_pageset = &boot_pageset;
- zone->per_cpu_zonestats = &boot_zonestats;
+ if (zone->per_cpu_zonestats != &boot_zonestats) {
+ free_percpu(zone->per_cpu_zonestats);
+ zone->per_cpu_zonestats = &boot_zonestats;
+ }
}
}
unsigned long usage)
{
unsigned long protected, old_protected;
- unsigned long low, min;
long delta;
if (!c->parent)
return;
- min = READ_ONCE(c->min);
- if (min || atomic_long_read(&c->min_usage)) {
- protected = min(usage, min);
+ protected = min(usage, READ_ONCE(c->min));
+ old_protected = atomic_long_read(&c->min_usage);
+ if (protected != old_protected) {
old_protected = atomic_long_xchg(&c->min_usage, protected);
delta = protected - old_protected;
if (delta)
atomic_long_add(delta, &c->parent->children_min_usage);
}
- low = READ_ONCE(c->low);
- if (low || atomic_long_read(&c->low_usage)) {
- protected = min(usage, low);
+ protected = min(usage, READ_ONCE(c->low));
+ old_protected = atomic_long_read(&c->low_usage);
+ if (protected != old_protected) {
old_protected = atomic_long_xchg(&c->low_usage, protected);
delta = protected - old_protected;
if (delta)
old = xchg(&counter->max, nr_pages);
- if (page_counter_read(counter) <= usage)
+ if (page_counter_read(counter) <= usage || nr_pages >= old)
return 0;
counter->max = old;
#include <linux/page_owner.h>
#include <linux/page_idle.h>
#include <linux/page_table_check.h>
+#include <linux/rcupdate.h>
/*
* struct page extension
* can utilize this callback to initialize the state of it correctly.
*/
+#ifdef CONFIG_SPARSEMEM
+#define PAGE_EXT_INVALID (0x1)
+#endif
+
#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
static bool need_page_idle(void)
{
unsigned long page_ext_size = sizeof(struct page_ext);
static unsigned long total_usage;
+static struct page_ext *lookup_page_ext(const struct page *page);
+
+bool early_page_ext;
+static int __init setup_early_page_ext(char *str)
+{
+ early_page_ext = true;
+ return 0;
+}
+early_param("early_page_ext", setup_early_page_ext);
static bool __init invoke_need_callbacks(void)
{
return base + page_ext_size * index;
}
+/**
+ * page_ext_get() - Get the extended information for a page.
+ * @page: The page we're interested in.
+ *
+ * Ensures that the page_ext will remain valid until page_ext_put()
+ * is called.
+ *
+ * Return: NULL if no page_ext exists for this page.
+ * Context: Any context. Caller may not sleep until they have called
+ * page_ext_put().
+ */
+struct page_ext *page_ext_get(struct page *page)
+{
+ struct page_ext *page_ext;
+
+ rcu_read_lock();
+ page_ext = lookup_page_ext(page);
+ if (!page_ext) {
+ rcu_read_unlock();
+ return NULL;
+ }
+
+ return page_ext;
+}
+
+/**
+ * page_ext_put() - Working with page extended information is done.
+ * @page_ext - Page extended information received from page_ext_get().
+ *
+ * The page extended information of the page may not be valid after this
+ * function is called.
+ *
+ * Return: None.
+ * Context: Any context with corresponding page_ext_get() is called.
+ */
+void page_ext_put(struct page_ext *page_ext)
+{
+ if (unlikely(!page_ext))
+ return;
+
+ rcu_read_unlock();
+}
#ifndef CONFIG_SPARSEMEM
pgdat->node_page_ext = NULL;
}
-struct page_ext *lookup_page_ext(const struct page *page)
+static struct page_ext *lookup_page_ext(const struct page *page)
{
unsigned long pfn = page_to_pfn(page);
unsigned long index;
struct page_ext *base;
+ WARN_ON_ONCE(!rcu_read_lock_held());
base = NODE_DATA(page_to_nid(page))->node_page_ext;
/*
* The sanity checks the page allocator does upon freeing a
}
#else /* CONFIG_SPARSEMEM */
+static bool page_ext_invalid(struct page_ext *page_ext)
+{
+ return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);
+}
-struct page_ext *lookup_page_ext(const struct page *page)
+static struct page_ext *lookup_page_ext(const struct page *page)
{
unsigned long pfn = page_to_pfn(page);
struct mem_section *section = __pfn_to_section(pfn);
+ struct page_ext *page_ext = READ_ONCE(section->page_ext);
+
+ WARN_ON_ONCE(!rcu_read_lock_held());
/*
* The sanity checks the page allocator does upon freeing a
* page can reach here before the page_ext arrays are
* allocated when feeding a range of pages to the allocator
* for the first time during bootup or memory hotplug.
*/
- if (!section->page_ext)
+ if (page_ext_invalid(page_ext))
return NULL;
- return get_entry(section->page_ext, pfn);
+ return get_entry(page_ext, pfn);
}
static void *__meminit alloc_page_ext(size_t size, int nid)
ms = __pfn_to_section(pfn);
if (!ms || !ms->page_ext)
return;
- base = get_entry(ms->page_ext, pfn);
+
+ base = READ_ONCE(ms->page_ext);
+ /*
+ * page_ext here can be valid while doing the roll back
+ * operation in online_page_ext().
+ */
+ if (page_ext_invalid(base))
+ base = (void *)base - PAGE_EXT_INVALID;
+ WRITE_ONCE(ms->page_ext, NULL);
+
+ base = get_entry(base, pfn);
free_page_ext(base);
- ms->page_ext = NULL;
+}
+
+static void __invalidate_page_ext(unsigned long pfn)
+{
+ struct mem_section *ms;
+ void *val;
+
+ ms = __pfn_to_section(pfn);
+ if (!ms || !ms->page_ext)
+ return;
+ val = (void *)ms->page_ext + PAGE_EXT_INVALID;
+ WRITE_ONCE(ms->page_ext, val);
}
static int __meminit online_page_ext(unsigned long start_pfn,
}
static int __meminit offline_page_ext(unsigned long start_pfn,
- unsigned long nr_pages, int nid)
+ unsigned long nr_pages)
{
unsigned long start, end, pfn;
start = SECTION_ALIGN_DOWN(start_pfn);
end = SECTION_ALIGN_UP(start_pfn + nr_pages);
+ /*
+ * Freeing of page_ext is done in 3 steps to avoid
+ * use-after-free of it:
+ * 1) Traverse all the sections and mark their page_ext
+ * as invalid.
+ * 2) Wait for all the existing users of page_ext who
+ * started before invalidation to finish.
+ * 3) Free the page_ext.
+ */
+ for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
+ __invalidate_page_ext(pfn);
+
+ synchronize_rcu();
+
for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
__free_page_ext(pfn);
return 0;
break;
case MEM_OFFLINE:
offline_page_ext(mn->start_pfn,
- mn->nr_pages, mn->status_change_nid);
+ mn->nr_pages);
break;
case MEM_CANCEL_ONLINE:
offline_page_ext(mn->start_pfn,
- mn->nr_pages, mn->status_change_nid);
+ mn->nr_pages);
break;
case MEM_GOING_OFFLINE:
break;
#include <linux/delayacct.h>
#include "swap.h"
-void end_swap_bio_write(struct bio *bio)
+static void end_swap_bio_write(struct bio *bio)
{
struct page *page = bio_first_page_all(bio);
*/
int swap_writepage(struct page *page, struct writeback_control *wbc)
{
+ struct folio *folio = page_folio(page);
int ret = 0;
- if (try_to_free_swap(page)) {
- unlock_page(page);
+ if (folio_free_swap(folio)) {
+ folio_unlock(folio);
goto out;
}
/*
* Arch code may have to preserve more data than just the page
* contents, e.g. memory tags.
*/
- ret = arch_prepare_to_swap(page);
+ ret = arch_prepare_to_swap(&folio->page);
if (ret) {
- set_page_dirty(page);
- unlock_page(page);
+ folio_mark_dirty(folio);
+ folio_unlock(folio);
goto out;
}
- if (frontswap_store(page) == 0) {
- set_page_writeback(page);
- unlock_page(page);
- end_page_writeback(page);
+ if (frontswap_store(&folio->page) == 0) {
+ folio_start_writeback(folio);
+ folio_unlock(folio);
+ folio_end_writeback(folio);
goto out;
}
- ret = __swap_writepage(page, wbc, end_swap_bio_write);
+ ret = __swap_writepage(&folio->page, wbc);
out:
return ret;
}
return 0;
}
-int __swap_writepage(struct page *page, struct writeback_control *wbc,
- bio_end_io_t end_write_func)
+int __swap_writepage(struct page *page, struct writeback_control *wbc)
{
struct bio *bio;
int ret;
REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc),
GFP_NOIO);
bio->bi_iter.bi_sector = swap_page_sector(page);
- bio->bi_end_io = end_write_func;
+ bio->bi_end_io = end_swap_bio_write;
bio_add_page(bio, page, thp_size(page), 0);
bio_associate_blkg_from_page(bio, page);
struct swap_info_struct *sis = page_swap_info(page);
bool workingset = PageWorkingset(page);
unsigned long pflags;
+ bool in_thrashing;
VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(PageUptodate(page), page);
/*
- * Count submission time as memory stall. When the device is congested,
- * or the submitting cgroup IO-throttled, submission can be a
- * significant part of overall IO time.
+ * Count submission time as memory stall and delay. When the device
+ * is congested, or the submitting cgroup IO-throttled, submission
+ * can be a significant part of overall IO time.
*/
- if (workingset)
+ if (workingset) {
+ delayacct_thrashing_start(&in_thrashing);
psi_memstall_enter(&pflags);
+ }
delayacct_swapin_start();
if (frontswap_load(page) == 0) {
bio_put(bio);
out:
- if (workingset)
+ if (workingset) {
+ delayacct_thrashing_end(&in_thrashing);
psi_memstall_leave(&pflags);
+ }
delayacct_swapin_end();
return ret;
}
struct zone *zone = page_zone(page);
unsigned long pfn;
- VM_BUG_ON(ALIGN_DOWN(start_pfn, pageblock_nr_pages) !=
- ALIGN_DOWN(end_pfn - 1, pageblock_nr_pages));
+ VM_BUG_ON(pageblock_start_pfn(start_pfn) !=
+ pageblock_start_pfn(end_pfn - 1));
if (is_migrate_cma_page(page)) {
/*
* to avoid redundant checks.
*/
check_unmovable_start = max(page_to_pfn(page), start_pfn);
- check_unmovable_end = min(ALIGN(page_to_pfn(page) + 1, pageblock_nr_pages),
+ check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),
end_pfn);
unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end,
struct zone *zone;
int ret;
- VM_BUG_ON(!IS_ALIGNED(boundary_pfn, pageblock_nr_pages));
+ VM_BUG_ON(!pageblock_aligned(boundary_pfn));
if (isolate_before)
isolate_pageblock = boundary_pfn - pageblock_nr_pages;
unsigned long pfn;
struct page *page;
/* isolation is done at page block granularity */
- unsigned long isolate_start = ALIGN_DOWN(start_pfn, pageblock_nr_pages);
- unsigned long isolate_end = ALIGN(end_pfn, pageblock_nr_pages);
+ unsigned long isolate_start = pageblock_start_pfn(start_pfn);
+ unsigned long isolate_end = pageblock_align(end_pfn);
int ret;
bool skip_isolation = false;
{
unsigned long pfn;
struct page *page;
- unsigned long isolate_start = ALIGN_DOWN(start_pfn, pageblock_nr_pages);
- unsigned long isolate_end = ALIGN(end_pfn, pageblock_nr_pages);
-
+ unsigned long isolate_start = pageblock_start_pfn(start_pfn);
+ unsigned long isolate_end = pageblock_align(end_pfn);
for (pfn = isolate_start;
pfn < isolate_end;
struct page_owner *page_owner;
u64 free_ts_nsec = local_clock();
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_get(page);
if (unlikely(!page_ext))
return;
page_owner->free_ts_nsec = free_ts_nsec;
page_ext = page_ext_next(page_ext);
}
+ page_ext_put(page_ext);
}
static inline void __set_page_owner_handle(struct page_ext *page_ext,
noinline void __set_page_owner(struct page *page, unsigned short order,
gfp_t gfp_mask)
{
- struct page_ext *page_ext = lookup_page_ext(page);
+ struct page_ext *page_ext;
depot_stack_handle_t handle;
+ handle = save_stack(gfp_mask);
+
+ page_ext = page_ext_get(page);
if (unlikely(!page_ext))
return;
-
- handle = save_stack(gfp_mask);
__set_page_owner_handle(page_ext, handle, order, gfp_mask);
+ page_ext_put(page_ext);
}
void __set_page_owner_migrate_reason(struct page *page, int reason)
{
- struct page_ext *page_ext = lookup_page_ext(page);
+ struct page_ext *page_ext = page_ext_get(page);
struct page_owner *page_owner;
if (unlikely(!page_ext))
page_owner = get_page_owner(page_ext);
page_owner->last_migrate_reason = reason;
+ page_ext_put(page_ext);
}
void __split_page_owner(struct page *page, unsigned int nr)
{
int i;
- struct page_ext *page_ext = lookup_page_ext(page);
+ struct page_ext *page_ext = page_ext_get(page);
struct page_owner *page_owner;
if (unlikely(!page_ext))
page_owner->order = 0;
page_ext = page_ext_next(page_ext);
}
+ page_ext_put(page_ext);
}
void __folio_copy_owner(struct folio *newfolio, struct folio *old)
{
- struct page_ext *old_ext = lookup_page_ext(&old->page);
- struct page_ext *new_ext = lookup_page_ext(&newfolio->page);
+ struct page_ext *old_ext;
+ struct page_ext *new_ext;
struct page_owner *old_page_owner, *new_page_owner;
- if (unlikely(!old_ext || !new_ext))
+ old_ext = page_ext_get(&old->page);
+ if (unlikely(!old_ext))
return;
+ new_ext = page_ext_get(&newfolio->page);
+ if (unlikely(!new_ext)) {
+ page_ext_put(old_ext);
+ return;
+ }
+
old_page_owner = get_page_owner(old_ext);
new_page_owner = get_page_owner(new_ext);
new_page_owner->order = old_page_owner->order;
*/
__set_bit(PAGE_EXT_OWNER, &new_ext->flags);
__set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags);
+ page_ext_put(new_ext);
+ page_ext_put(old_ext);
}
void pagetypeinfo_showmixedcount_print(struct seq_file *m,
continue;
}
- block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
+ block_end_pfn = pageblock_end_pfn(pfn);
block_end_pfn = min(block_end_pfn, end_pfn);
pageblock_mt = get_pageblock_migratetype(page);
if (PageReserved(page))
continue;
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_get(page);
if (unlikely(!page_ext))
continue;
if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
- continue;
+ goto ext_put_continue;
page_owner = get_page_owner(page_ext);
page_mt = gfp_migratetype(page_owner->gfp_mask);
count[pageblock_mt]++;
pfn = block_end_pfn;
+ page_ext_put(page_ext);
break;
}
pfn += (1UL << page_owner->order) - 1;
+ext_put_continue:
+ page_ext_put(page_ext);
}
}
void __dump_page_owner(const struct page *page)
{
- struct page_ext *page_ext = lookup_page_ext(page);
+ struct page_ext *page_ext = page_ext_get((void *)page);
struct page_owner *page_owner;
depot_stack_handle_t handle;
gfp_t gfp_mask;
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
pr_alert("page_owner info is not present (never set?)\n");
+ page_ext_put(page_ext);
return;
}
if (page_owner->last_migrate_reason != -1)
pr_alert("page has been migrated, last migrate reason: %s\n",
migrate_reason_names[page_owner->last_migrate_reason]);
+ page_ext_put(page_ext);
}
static ssize_t
return -EINVAL;
page = NULL;
- pfn = min_low_pfn + *ppos;
-
+ if (*ppos == 0)
+ pfn = min_low_pfn;
+ else
+ pfn = *ppos;
/* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
pfn++;
- drain_all_pages(NULL);
-
/* Find an allocated page */
for (; pfn < max_pfn; pfn++) {
+ /*
+ * This temporary page_owner is required so
+ * that we can avoid the context switches while holding
+ * the rcu lock and copying the page owner information to
+ * user through copy_to_user() or GFP_KERNEL allocations.
+ */
+ struct page_owner page_owner_tmp;
+
/*
* If the new page is in a new MAX_ORDER_NR_PAGES area,
* validate the area as existing, skip it if not
continue;
}
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_get(page);
if (unlikely(!page_ext))
continue;
* because we don't hold the zone lock.
*/
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
- continue;
+ goto ext_put_continue;
/*
* Although we do have the info about past allocation of free
* pages, it's not relevant for current memory usage.
*/
if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
- continue;
+ goto ext_put_continue;
page_owner = get_page_owner(page_ext);
* would inflate the stats.
*/
if (!IS_ALIGNED(pfn, 1 << page_owner->order))
- continue;
+ goto ext_put_continue;
/*
* Access to page_ext->handle isn't synchronous so we should
*/
handle = READ_ONCE(page_owner->handle);
if (!handle)
- continue;
+ goto ext_put_continue;
/* Record the next PFN to read in the file offset */
- *ppos = (pfn - min_low_pfn) + 1;
+ *ppos = pfn + 1;
+ page_owner_tmp = *page_owner;
+ page_ext_put(page_ext);
return print_page_owner(buf, count, pfn, page,
- page_owner, handle);
+ &page_owner_tmp, handle);
+ext_put_continue:
+ page_ext_put(page_ext);
}
return 0;
}
+static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig)
+{
+ switch (orig) {
+ case SEEK_SET:
+ file->f_pos = offset;
+ break;
+ case SEEK_CUR:
+ file->f_pos += offset;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return file->f_pos;
+}
+
static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
{
unsigned long pfn = zone->zone_start_pfn;
continue;
}
- block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
+ block_end_pfn = pageblock_end_pfn(pfn);
block_end_pfn = min(block_end_pfn, end_pfn);
for (; pfn < block_end_pfn; pfn++) {
if (PageReserved(page))
continue;
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_get(page);
if (unlikely(!page_ext))
continue;
/* Maybe overlapping zone */
if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
- continue;
+ goto ext_put_continue;
/* Found early allocated page */
__set_page_owner_handle(page_ext, early_handle,
0, 0);
count++;
+ext_put_continue:
+ page_ext_put(page_ext);
}
cond_resched();
}
static const struct file_operations proc_page_owner_operations = {
.read = read_page_owner,
+ .llseek = lseek_page_owner,
};
static int __init pageowner_init(void)
}
/*
- * An enty is removed from the page table, decrement the counters for that page
+ * An entry is removed from the page table, decrement the counters for that page
* verify that it is of correct type and counters do not become negative.
*/
static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
return;
page = pfn_to_page(pfn);
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_get(page);
anon = PageAnon(page);
for (i = 0; i < pgcnt; i++) {
}
page_ext = page_ext_next(page_ext);
}
+ page_ext_put(page_ext);
}
/*
- * A new enty is added to the page table, increment the counters for that page
+ * A new entry is added to the page table, increment the counters for that page
* verify that it is of correct type and is not being mapped with a different
* type to a different process.
*/
return;
page = pfn_to_page(pfn);
- page_ext = lookup_page_ext(page);
+ page_ext = page_ext_get(page);
anon = PageAnon(page);
for (i = 0; i < pgcnt; i++) {
}
page_ext = page_ext_next(page_ext);
}
+ page_ext_put(page_ext);
}
/*
*/
void __page_table_check_zero(struct page *page, unsigned int order)
{
- struct page_ext *page_ext = lookup_page_ext(page);
+ struct page_ext *page_ext;
unsigned long i;
+ page_ext = page_ext_get(page);
BUG_ON(!page_ext);
for (i = 0; i < (1ul << order); i++) {
struct page_table_check *ptc = get_page_table_check(page_ext);
BUG_ON(atomic_read(&ptc->file_map_count));
page_ext = page_ext_next(page_ext);
}
+ page_ext_put(page_ext);
}
void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
!is_device_exclusive_entry(entry))
return false;
- pfn = swp_offset(entry);
+ pfn = swp_offset_pfn(entry);
} else if (is_swap_pte(*pvmw->pte)) {
swp_entry_t entry;
!is_device_exclusive_entry(entry))
return false;
- pfn = swp_offset(entry);
+ pfn = swp_offset_pfn(entry);
} else {
if (!pte_present(*pvmw->pte))
return false;
return not_found(pvmw);
entry = pmd_to_swp_entry(pmde);
if (!is_migration_entry(entry) ||
- !check_pmd(swp_offset(entry), pvmw))
+ !check_pmd(swp_offset_pfn(entry), pvmw))
return not_found(pvmw);
return true;
}
} else { /* inside vma */
walk.vma = vma;
next = min(end, vma->vm_end);
- vma = vma->vm_next;
+ vma = find_vma(mm, vma->vm_end);
err = walk_page_test(start, next, &walk);
if (err > 0) {
return err;
}
-/*
+/**
+ * walk_page_range_novma - walk a range of pagetables not backed by a vma
+ * @mm: mm_struct representing the target process of page table walk
+ * @start: start address of the virtual address range
+ * @end: end address of the virtual address range
+ * @ops: operation to call during the walk
+ * @pgd: pgd to walk if different from mm->pgd
+ * @private: private data for callbacks' usage
+ *
* Similar to walk_page_range() but can walk any page tables even if they are
* not backed by VMAs. Because 'unusual' entries may be walked this function
* will also not lock the PTEs for the pte_entry() callback. This is useful for
* inode->i_rwsem (while writing or truncating, not reading or faulting)
* mm->mmap_lock
* mapping->invalidate_lock (in filemap_fault)
- * page->flags PG_locked (lock_page) * (see hugetlbfs below)
- * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
+ * page->flags PG_locked (lock_page)
+ * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
* mapping->i_mmap_rwsem
- * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
* anon_vma->rwsem
* mm->page_table_lock or pte_lock
* swap_lock (in swap_duplicate, swap_info_get)
* ->tasklist_lock
* pte map lock
*
- * * hugetlbfs PageHuge() pages take locks in this order:
- * mapping->i_mmap_rwsem
- * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
- * page->flags PG_locked (lock_page)
+ * hugetlbfs PageHuge() take locks in this order:
+ * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
+ * vma_lock (hugetlb specific lock for pmd_sharing)
+ * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
+ * page->flags PG_locked (lock_page)
*/
#include <linux/mm.h>
* if there is a mapcount, we can dereference the anon_vma after observing
* those.
*/
-struct anon_vma *page_get_anon_vma(struct page *page)
+struct anon_vma *folio_get_anon_vma(struct folio *folio)
{
struct anon_vma *anon_vma = NULL;
unsigned long anon_mapping;
rcu_read_lock();
- anon_mapping = (unsigned long)READ_ONCE(page->mapping);
+ anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
goto out;
- if (!page_mapped(page))
+ if (!folio_mapped(folio))
goto out;
anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
}
/*
- * If this page is still mapped, then its anon_vma cannot have been
+ * If this folio is still mapped, then its anon_vma cannot have been
* freed. But if it has been unmapped, we have no security against the
* anon_vma structure being freed and reused (for another anon_vma:
* SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
* above cannot corrupt).
*/
- if (!page_mapped(page)) {
+ if (!folio_mapped(folio)) {
rcu_read_unlock();
put_anon_vma(anon_vma);
return NULL;
}
/*
- * Similar to page_get_anon_vma() except it locks the anon_vma.
+ * Similar to folio_get_anon_vma() except it locks the anon_vma.
*
* Its a little more complex as it tries to keep the fast path to a single
* atomic op -- the trylock. If we fail the trylock, we fall back to getting a
- * reference like with page_get_anon_vma() and then block on the mutex
+ * reference like with folio_get_anon_vma() and then block on the mutex
* on !rwc->try_lock case.
*/
struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,
return anon_vma;
}
-void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
-{
- anon_vma_unlock_read(anon_vma);
-}
-
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
/*
* Flush TLB entries for recently unmapped pages from remote CPUs. It is
return vma_address(page, vma);
}
+/*
+ * Returns the actual pmd_t* where we expect 'address' to be mapped from, or
+ * NULL if it doesn't exist. No guarantees / checks on what the pmd_t*
+ * represents.
+ */
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd = NULL;
- pmd_t pmde;
pgd = pgd_offset(mm, address);
if (!pgd_present(*pgd))
goto out;
pmd = pmd_offset(pud, address);
- /*
- * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
- * without holding anon_vma lock for write. So when looking for a
- * genuine pmde (in which to find pte), test present and !THP together.
- */
- pmde = *pmd;
- barrier();
- if (!pmd_present(pmde) || pmd_trans_huge(pmde))
- pmd = NULL;
out:
return pmd;
}
}
if (pvmw.pte) {
+ if (lru_gen_enabled() && pte_young(*pvmw.pte) &&
+ !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) {
+ lru_gen_look_around(&pvmw);
+ referenced++;
+ }
+
if (ptep_clear_flush_young_notify(vma, address,
pvmw.pte)) {
/*
*/
void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
{
- struct anon_vma *anon_vma = vma->anon_vma;
- struct page *subpage = page;
-
- page = compound_head(page);
+ void *anon_vma = vma->anon_vma;
+ struct folio *folio = page_folio(page);
- VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_VMA(!anon_vma, vma);
- anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
+ anon_vma += PAGE_MAPPING_ANON;
/*
* Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
* simultaneously, so a concurrent reader (eg folio_referenced()'s
* folio_test_anon()) will not see one without the other.
*/
- WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
- SetPageAnonExclusive(subpage);
+ WRITE_ONCE(folio->mapping, anon_vma);
+ SetPageAnonExclusive(page);
}
/**
* To call huge_pmd_unshare, i_mmap_rwsem must be
* held in write mode. Caller needs to explicitly
* do this outside rmap routines.
+ *
+ * We also must hold hugetlb vma_lock in write mode.
+ * Lock order dictates acquiring vma_lock BEFORE
+ * i_mmap_rwsem. We can only try lock here and fail
+ * if unsuccessful.
*/
- VM_BUG_ON(!anon && !(flags & TTU_RMAP_LOCKED));
- if (!anon && huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
- flush_tlb_range(vma, range.start, range.end);
- mmu_notifier_invalidate_range(mm, range.start,
- range.end);
-
- /*
- * The ref count of the PMD page was dropped
- * which is part of the way map counting
- * is done for shared PMDs. Return 'true'
- * here. When there is no other sharing,
- * huge_pmd_unshare returns false and we will
- * unmap the actual page and drop map count
- * to zero.
- */
- page_vma_mapped_walk_done(&pvmw);
- break;
+ if (!anon) {
+ VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
+ if (!hugetlb_vma_trylock_write(vma)) {
+ page_vma_mapped_walk_done(&pvmw);
+ ret = false;
+ break;
+ }
+ if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
+ hugetlb_vma_unlock_write(vma);
+ flush_tlb_range(vma,
+ range.start, range.end);
+ mmu_notifier_invalidate_range(mm,
+ range.start, range.end);
+ /*
+ * The ref count of the PMD page was
+ * dropped which is part of the way map
+ * counting is done for shared PMDs.
+ * Return 'true' here. When there is
+ * no other sharing, huge_pmd_unshare
+ * returns false and we will unmap the
+ * actual page and drop map count
+ * to zero.
+ */
+ page_vma_mapped_walk_done(&pvmw);
+ break;
+ }
+ hugetlb_vma_unlock_write(vma);
}
pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
} else {
flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
- /*
- * Nuke the page table entry. When having to clear
- * PageAnonExclusive(), we always have to flush.
- */
- if (should_defer_flush(mm, flags) && !anon_exclusive) {
+ /* Nuke the page table entry. */
+ if (should_defer_flush(mm, flags)) {
/*
* We clear the PTE but do not flush so potentially
* a remote CPU could still be writing to the folio.
page_vma_mapped_walk_done(&pvmw);
break;
}
+
+ /* See page_try_share_anon_rmap(): clear PTE first. */
if (anon_exclusive &&
page_try_share_anon_rmap(subpage)) {
swap_free(entry);
* To call huge_pmd_unshare, i_mmap_rwsem must be
* held in write mode. Caller needs to explicitly
* do this outside rmap routines.
+ *
+ * We also must hold hugetlb vma_lock in write mode.
+ * Lock order dictates acquiring vma_lock BEFORE
+ * i_mmap_rwsem. We can only try lock here and
+ * fail if unsuccessful.
*/
- VM_BUG_ON(!anon && !(flags & TTU_RMAP_LOCKED));
- if (!anon && huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
- flush_tlb_range(vma, range.start, range.end);
- mmu_notifier_invalidate_range(mm, range.start,
- range.end);
-
- /*
- * The ref count of the PMD page was dropped
- * which is part of the way map counting
- * is done for shared PMDs. Return 'true'
- * here. When there is no other sharing,
- * huge_pmd_unshare returns false and we will
- * unmap the actual page and drop map count
- * to zero.
- */
- page_vma_mapped_walk_done(&pvmw);
- break;
+ if (!anon) {
+ VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
+ if (!hugetlb_vma_trylock_write(vma)) {
+ page_vma_mapped_walk_done(&pvmw);
+ ret = false;
+ break;
+ }
+ if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
+ hugetlb_vma_unlock_write(vma);
+ flush_tlb_range(vma,
+ range.start, range.end);
+ mmu_notifier_invalidate_range(mm,
+ range.start, range.end);
+
+ /*
+ * The ref count of the PMD page was
+ * dropped which is part of the way map
+ * counting is done for shared PMDs.
+ * Return 'true' here. When there is
+ * no other sharing, huge_pmd_unshare
+ * returns false and we will unmap the
+ * actual page and drop map count
+ * to zero.
+ */
+ page_vma_mapped_walk_done(&pvmw);
+ break;
+ }
+ hugetlb_vma_unlock_write(vma);
}
-
/* Nuke the hugetlb page table entry */
pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
} else {
}
VM_BUG_ON_PAGE(pte_write(pteval) && folio_test_anon(folio) &&
!anon_exclusive, subpage);
+
+ /* See page_try_share_anon_rmap(): clear PTE first. */
if (anon_exclusive &&
page_try_share_anon_rmap(subpage)) {
if (folio_test_hugetlb(folio))
else
entry = make_readable_migration_entry(
page_to_pfn(subpage));
-
+ if (pte_young(pteval))
+ entry = make_migration_entry_young(entry);
+ if (pte_dirty(pteval))
+ entry = make_migration_entry_dirty(entry);
swp_pte = swp_entry_to_pte(entry);
if (pte_soft_dirty(pteval))
swp_pte = pte_swp_mksoft_dirty(swp_pte);
#include <linux/rodata_test.h>
#include <linux/uaccess.h>
+#include <linux/mm.h>
#include <asm/sections.h>
static const int rodata_test_data = 0xC3;
void rodata_test(void)
{
- unsigned long start, end;
int zero = 0;
/* test 1: read the value */
}
/* test 4: check if the rodata section is PAGE_SIZE aligned */
- start = (unsigned long)__start_rodata;
- end = (unsigned long)__end_rodata;
- if (start & (PAGE_SIZE - 1)) {
+ if (!PAGE_ALIGNED(__start_rodata)) {
pr_err("start of .rodata is not page size aligned\n");
return;
}
- if (end & (PAGE_SIZE - 1)) {
+ if (!PAGE_ALIGNED(__end_rodata)) {
pr_err("end of .rodata is not page size aligned\n");
return;
}
.kill_sb = kill_anon_super,
};
-static int secretmem_init(void)
+static int __init secretmem_init(void)
{
- int ret = 0;
-
if (!secretmem_enable)
- return ret;
+ return 0;
secretmem_mnt = kern_mount(&secretmem_fs);
if (IS_ERR(secretmem_mnt))
/* prevent secretmem mappings from ever getting PROT_EXEC */
secretmem_mnt->mnt_flags |= MNT_NOEXEC;
- return ret;
+ return 0;
}
fs_initcall(secretmem_init);
#include <linux/hugetlb.h>
#include <linux/fs_parser.h>
#include <linux/swapfile.h>
+#include <linux/iversion.h>
#include "swap.h"
static struct vfsmount *shm_mnt;
struct folio **foliop, enum sgp_type sgp,
gfp_t gfp, struct vm_area_struct *vma,
vm_fault_t *fault_type);
-static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
- struct page **pagep, enum sgp_type sgp,
- gfp_t gfp, struct vm_area_struct *vma,
- struct vm_fault *vmf, vm_fault_t *fault_type);
-
-int shmem_getpage(struct inode *inode, pgoff_t index,
- struct page **pagep, enum sgp_type sgp)
-{
- return shmem_getpage_gfp(inode, index, pagep, sgp,
- mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
-}
static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
{
/*
* ... whereas tmpfs objects are accounted incrementally as
* pages are allocated, in order to allow large sparse files.
- * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
+ * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
* so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
*/
static inline int shmem_acct_block(unsigned long flags, long pages)
static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
-bool shmem_is_huge(struct vm_area_struct *vma,
- struct inode *inode, pgoff_t index)
+bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
+ pgoff_t index, bool shmem_huge_force)
{
loff_t i_size;
if (!S_ISREG(inode->i_mode))
return false;
- if (shmem_huge == SHMEM_HUGE_DENY)
- return false;
if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) ||
test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)))
return false;
+ if (shmem_huge_force)
+ return true;
if (shmem_huge == SHMEM_HUGE_FORCE)
return true;
+ if (shmem_huge == SHMEM_HUGE_DENY)
+ return false;
switch (SHMEM_SB(inode->i_sb)->huge) {
case SHMEM_HUGE_ALWAYS:
goto move_back;
}
- ret = split_huge_page(&folio->page);
+ ret = split_folio(folio);
folio_unlock(folio);
folio_put(folio);
#define shmem_huge SHMEM_HUGE_DENY
-bool shmem_is_huge(struct vm_area_struct *vma,
- struct inode *inode, pgoff_t index)
+bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode,
+ pgoff_t index, bool shmem_huge_force)
{
return false;
}
}
/*
- * Like delete_from_page_cache, but substitutes swap for page.
+ * Like delete_from_page_cache, but substitutes swap for @folio.
*/
-static void shmem_delete_from_page_cache(struct page *page, void *radswap)
+static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
{
- struct address_space *mapping = page->mapping;
+ struct address_space *mapping = folio->mapping;
+ long nr = folio_nr_pages(folio);
int error;
- VM_BUG_ON_PAGE(PageCompound(page), page);
-
xa_lock_irq(&mapping->i_pages);
- error = shmem_replace_entry(mapping, page->index, page, radswap);
- page->mapping = NULL;
- mapping->nrpages--;
- __dec_lruvec_page_state(page, NR_FILE_PAGES);
- __dec_lruvec_page_state(page, NR_SHMEM);
+ error = shmem_replace_entry(mapping, folio->index, folio, radswap);
+ folio->mapping = NULL;
+ mapping->nrpages -= nr;
+ __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
+ __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
xa_unlock_irq(&mapping->i_pages);
- put_page(page);
+ folio_put(folio);
BUG_ON(error);
}
static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
{
struct folio *folio;
- struct page *page;
/*
- * At first avoid shmem_getpage(,,,SGP_READ): that fails
+ * At first avoid shmem_get_folio(,,,SGP_READ): that fails
* beyond i_size, and reports fallocated pages as holes.
*/
folio = __filemap_get_folio(inode->i_mapping, index,
* But read a page back from swap if any of it is within i_size
* (although in some cases this is just a waste of time).
*/
- page = NULL;
- shmem_getpage(inode, index, &page, SGP_READ);
- return page ? page_folio(page) : NULL;
+ folio = NULL;
+ shmem_get_folio(inode, index, &folio, SGP_READ);
+ return folio;
}
/*
{
shmem_undo_range(inode, lstart, lend, false);
inode->i_ctime = inode->i_mtime = current_time(inode);
+ inode_inc_iversion(inode);
}
EXPORT_SYMBOL_GPL(shmem_truncate_range);
STATX_ATTR_NODUMP);
generic_fillattr(&init_user_ns, inode, stat);
- if (shmem_is_huge(NULL, inode, 0))
+ if (shmem_is_huge(NULL, inode, 0, false))
stat->blksize = HPAGE_PMD_SIZE;
if (request_mask & STATX_BTIME) {
struct inode *inode = d_inode(dentry);
struct shmem_inode_info *info = SHMEM_I(inode);
int error;
+ bool update_mtime = false;
+ bool update_ctime = true;
error = setattr_prepare(&init_user_ns, dentry, attr);
if (error)
if (error)
return error;
i_size_write(inode, newsize);
- inode->i_ctime = inode->i_mtime = current_time(inode);
+ update_mtime = true;
+ } else {
+ update_ctime = false;
}
if (newsize <= oldsize) {
loff_t holebegin = round_up(newsize, PAGE_SIZE);
setattr_copy(&init_user_ns, inode, attr);
if (attr->ia_valid & ATTR_MODE)
error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode);
+ if (!error && update_ctime) {
+ inode->i_ctime = current_time(inode);
+ if (update_mtime)
+ inode->i_mtime = inode->i_ctime;
+ inode_inc_iversion(inode);
+ }
return error;
}
* "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
* and its shmem_writeback() needs them to be split when swapping.
*/
- if (PageTransCompound(page)) {
+ if (folio_test_large(folio)) {
/* Ensure the subpages are still dirty */
- SetPageDirty(page);
+ folio_test_set_dirty(folio);
if (split_huge_page(page) < 0)
goto redirty;
- ClearPageDirty(page);
+ folio = page_folio(page);
+ folio_clear_dirty(folio);
}
- BUG_ON(!PageLocked(page));
- mapping = page->mapping;
- index = page->index;
+ BUG_ON(!folio_test_locked(folio));
+ mapping = folio->mapping;
+ index = folio->index;
inode = mapping->host;
info = SHMEM_I(inode);
if (info->flags & VM_LOCKED)
/*
* This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
* value into swapfile.c, the only way we can correctly account for a
- * fallocated page arriving here is now to initialize it and write it.
+ * fallocated folio arriving here is now to initialize it and write it.
*
- * That's okay for a page already fallocated earlier, but if we have
+ * That's okay for a folio already fallocated earlier, but if we have
* not yet completed the fallocation, then (a) we want to keep track
- * of this page in case we have to undo it, and (b) it may not be a
+ * of this folio in case we have to undo it, and (b) it may not be a
* good idea to continue anyway, once we're pushing into swap. So
- * reactivate the page, and let shmem_fallocate() quit when too many.
+ * reactivate the folio, and let shmem_fallocate() quit when too many.
*/
- if (!PageUptodate(page)) {
+ if (!folio_test_uptodate(folio)) {
if (inode->i_private) {
struct shmem_falloc *shmem_falloc;
spin_lock(&inode->i_lock);
if (shmem_falloc)
goto redirty;
}
- clear_highpage(page);
- flush_dcache_page(page);
- SetPageUptodate(page);
+ folio_zero_range(folio, 0, folio_size(folio));
+ flush_dcache_folio(folio);
+ folio_mark_uptodate(folio);
}
swap = folio_alloc_swap(folio);
/*
* Add inode to shmem_unuse()'s list of swapped-out inodes,
- * if it's not already there. Do it now before the page is
+ * if it's not already there. Do it now before the folio is
* moved to swap cache, when its pagelock no longer protects
* the inode from eviction. But don't unlock the mutex until
* we've incremented swapped, because shmem_unuse_inode() will
if (list_empty(&info->swaplist))
list_add(&info->swaplist, &shmem_swaplist);
- if (add_to_swap_cache(page, swap,
+ if (add_to_swap_cache(folio, swap,
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
NULL) == 0) {
spin_lock_irq(&info->lock);
spin_unlock_irq(&info->lock);
swap_shmem_alloc(swap);
- shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
+ shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
mutex_unlock(&shmem_swaplist_mutex);
- BUG_ON(page_mapped(page));
- swap_writepage(page, wbc);
+ BUG_ON(folio_mapped(folio));
+ swap_writepage(&folio->page, wbc);
return 0;
}
mutex_unlock(&shmem_swaplist_mutex);
- put_swap_page(page, swap);
+ put_swap_folio(folio, swap);
redirty:
- set_page_dirty(page);
+ folio_mark_dirty(folio);
if (wbc->for_reclaim)
- return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
- unlock_page(page);
+ return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */
+ folio_unlock(folio);
return 0;
}
mpol_cond_put(vma->vm_policy);
}
-static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
+static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp,
struct shmem_inode_info *info, pgoff_t index)
{
struct vm_area_struct pvma;
page = swap_cluster_readahead(swap, gfp, &vmf);
shmem_pseudo_vma_destroy(&pvma);
- return page;
+ if (!page)
+ return NULL;
+ return page_folio(page);
}
/*
return folio;
}
-static struct page *shmem_alloc_page(gfp_t gfp,
- struct shmem_inode_info *info, pgoff_t index)
-{
- return &shmem_alloc_folio(gfp, info, index)->page;
-}
-
static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
pgoff_t index, bool huge)
{
/*
* When a page is moved from swapcache to shmem filecache (either by the
- * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
+ * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
* shmem_unuse_inode()), it may have been read in earlier from swap, in
* ignorance of the mapping it belongs to. If that mapping has special
* constraints (like the gma500 GEM driver, which requires RAM below 4GB),
return folio_zonenum(folio) > gfp_zone(gfp);
}
-static int shmem_replace_page(struct page **pagep, gfp_t gfp,
+static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
struct shmem_inode_info *info, pgoff_t index)
{
- struct page *oldpage, *newpage;
struct folio *old, *new;
struct address_space *swap_mapping;
swp_entry_t entry;
pgoff_t swap_index;
int error;
- oldpage = *pagep;
- entry.val = page_private(oldpage);
+ old = *foliop;
+ entry = folio_swap_entry(old);
swap_index = swp_offset(entry);
- swap_mapping = page_mapping(oldpage);
+ swap_mapping = swap_address_space(entry);
/*
* We have arrived here because our zones are constrained, so don't
* limit chance of success by further cpuset and node constraints.
*/
gfp &= ~GFP_CONSTRAINT_MASK;
- newpage = shmem_alloc_page(gfp, info, index);
- if (!newpage)
+ VM_BUG_ON_FOLIO(folio_test_large(old), old);
+ new = shmem_alloc_folio(gfp, info, index);
+ if (!new)
return -ENOMEM;
- get_page(newpage);
- copy_highpage(newpage, oldpage);
- flush_dcache_page(newpage);
+ folio_get(new);
+ folio_copy(new, old);
+ flush_dcache_folio(new);
- __SetPageLocked(newpage);
- __SetPageSwapBacked(newpage);
- SetPageUptodate(newpage);
- set_page_private(newpage, entry.val);
- SetPageSwapCache(newpage);
+ __folio_set_locked(new);
+ __folio_set_swapbacked(new);
+ folio_mark_uptodate(new);
+ folio_set_swap_entry(new, entry);
+ folio_set_swapcache(new);
/*
* Our caller will very soon move newpage out of swapcache, but it's
* a nice clean interface for us to replace oldpage by newpage there.
*/
xa_lock_irq(&swap_mapping->i_pages);
- error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
+ error = shmem_replace_entry(swap_mapping, swap_index, old, new);
if (!error) {
- old = page_folio(oldpage);
- new = page_folio(newpage);
mem_cgroup_migrate(old, new);
- __inc_lruvec_page_state(newpage, NR_FILE_PAGES);
- __inc_lruvec_page_state(newpage, NR_SHMEM);
- __dec_lruvec_page_state(oldpage, NR_FILE_PAGES);
- __dec_lruvec_page_state(oldpage, NR_SHMEM);
+ __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
+ __lruvec_stat_mod_folio(new, NR_SHMEM, 1);
+ __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
+ __lruvec_stat_mod_folio(old, NR_SHMEM, -1);
}
xa_unlock_irq(&swap_mapping->i_pages);
* both PageSwapCache and page_private after getting page lock;
* but be defensive. Reverse old to newpage for clear and free.
*/
- oldpage = newpage;
+ old = new;
} else {
- lru_cache_add(newpage);
- *pagep = newpage;
+ folio_add_lru(new);
+ *foliop = new;
}
- ClearPageSwapCache(oldpage);
- set_page_private(oldpage, 0);
+ folio_clear_swapcache(old);
+ old->private = NULL;
- unlock_page(oldpage);
- put_page(oldpage);
- put_page(oldpage);
+ folio_unlock(old);
+ folio_put_refs(old, 2);
return error;
}
struct address_space *mapping = inode->i_mapping;
struct shmem_inode_info *info = SHMEM_I(inode);
struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
- struct page *page;
struct folio *folio = NULL;
swp_entry_t swap;
int error;
return -EIO;
/* Look it up and read it in.. */
- page = lookup_swap_cache(swap, NULL, 0);
- if (!page) {
+ folio = swap_cache_get_folio(swap, NULL, 0);
+ if (!folio) {
/* Or update major stats only when swapin succeeds?? */
if (fault_type) {
*fault_type |= VM_FAULT_MAJOR;
count_memcg_event_mm(charge_mm, PGMAJFAULT);
}
/* Here we actually start the io */
- page = shmem_swapin(swap, gfp, info, index);
- if (!page) {
+ folio = shmem_swapin(swap, gfp, info, index);
+ if (!folio) {
error = -ENOMEM;
goto failed;
}
}
- folio = page_folio(page);
/* We have to do this with folio locked to prevent races */
folio_lock(folio);
arch_swap_restore(swap, folio);
if (shmem_should_replace_folio(folio, gfp)) {
- error = shmem_replace_page(&page, gfp, info, index);
- folio = page_folio(page);
+ error = shmem_replace_folio(&folio, gfp, info, index);
if (error)
goto failed;
}
}
/*
- * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
+ * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
*
* If we allocate a new one we do not mark it dirty. That's up to the
* vm. If we swap it in we mark it dirty since we also free the swap
* vma, vmf, and fault_type are only supplied by shmem_fault:
* otherwise they are NULL.
*/
-static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
- struct page **pagep, enum sgp_type sgp, gfp_t gfp,
- struct vm_area_struct *vma, struct vm_fault *vmf,
- vm_fault_t *fault_type)
+static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
+ struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
+ struct vm_area_struct *vma, struct vm_fault *vmf,
+ vm_fault_t *fault_type)
{
struct address_space *mapping = inode->i_mapping;
struct shmem_inode_info *info = SHMEM_I(inode);
if (error == -EEXIST)
goto repeat;
- *pagep = &folio->page;
+ *foliop = folio;
return error;
}
folio_mark_accessed(folio);
if (folio_test_uptodate(folio))
goto out;
- /* fallocated page */
+ /* fallocated folio */
if (sgp != SGP_READ)
goto clear;
folio_unlock(folio);
}
/*
- * SGP_READ: succeed on hole, with NULL page, letting caller zero.
- * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail.
+ * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
+ * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
*/
- *pagep = NULL;
+ *foliop = NULL;
if (sgp == SGP_READ)
return 0;
if (sgp == SGP_NOALLOC)
return 0;
}
- if (!shmem_is_huge(vma, inode, index))
+ if (!shmem_is_huge(vma, inode, index, false))
goto alloc_nohuge;
huge_gfp = vma_thp_gfp_mask(vma);
if (error != -ENOSPC)
goto unlock;
/*
- * Try to reclaim some space by splitting a huge page
+ * Try to reclaim some space by splitting a large folio
* beyond i_size on the filesystem.
*/
while (retry--) {
if (folio_test_pmd_mappable(folio) &&
DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
- hindex + HPAGE_PMD_NR - 1) {
+ folio_next_index(folio) - 1) {
/*
- * Part of the huge page is beyond i_size: subject
+ * Part of the large folio is beyond i_size: subject
* to shrink under memory pressure.
*/
spin_lock(&sbinfo->shrinklist_lock);
}
/*
- * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
+ * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
*/
if (sgp == SGP_FALLOC)
sgp = SGP_WRITE;
clear:
/*
- * Let SGP_WRITE caller clear ends if write does not fill page;
- * but SGP_FALLOC on a page fallocated earlier must initialize
+ * Let SGP_WRITE caller clear ends if write does not fill folio;
+ * but SGP_FALLOC on a folio fallocated earlier must initialize
* it now, lest undo on failure cancel our earlier guarantee.
*/
if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
goto unlock;
}
out:
- *pagep = folio_page(folio, index - hindex);
+ *foliop = folio;
return 0;
/*
return error;
}
+int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
+ enum sgp_type sgp)
+{
+ return shmem_get_folio_gfp(inode, index, foliop, sgp,
+ mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
+}
+
/*
* This is like autoremove_wake_function, but it removes the wait queue
* entry unconditionally - even if something else had already woken the
struct vm_area_struct *vma = vmf->vma;
struct inode *inode = file_inode(vma->vm_file);
gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
+ struct folio *folio = NULL;
int err;
vm_fault_t ret = VM_FAULT_LOCKED;
spin_unlock(&inode->i_lock);
}
- err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, SGP_CACHE,
+ err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
gfp, vma, vmf, &ret);
if (err)
return vmf_error(err);
+ if (folio)
+ vmf->page = folio_file_page(folio, vmf->pgoff);
return ret;
}
inode_init_owner(&init_user_ns, inode, dir, mode);
inode->i_blocks = 0;
inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
- inode->i_generation = prandom_u32();
+ inode->i_generation = get_random_u32();
info = SHMEM_I(inode);
memset(info, 0, (char *)inode - (char *)info);
spin_lock_init(&info->lock);
pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
void *page_kaddr;
struct folio *folio;
- struct page *page;
int ret;
pgoff_t max_off;
if (!*pagep) {
ret = -ENOMEM;
- page = shmem_alloc_page(gfp, info, pgoff);
- if (!page)
+ folio = shmem_alloc_folio(gfp, info, pgoff);
+ if (!folio)
goto out_unacct_blocks;
if (!zeropage) { /* COPY */
- page_kaddr = kmap_atomic(page);
+ page_kaddr = kmap_local_folio(folio, 0);
ret = copy_from_user(page_kaddr,
(const void __user *)src_addr,
PAGE_SIZE);
- kunmap_atomic(page_kaddr);
+ kunmap_local(page_kaddr);
/* fallback to copy_from_user outside mmap_lock */
if (unlikely(ret)) {
- *pagep = page;
+ *pagep = &folio->page;
ret = -ENOENT;
/* don't free the page */
goto out_unacct_blocks;
}
- flush_dcache_page(page);
+ flush_dcache_folio(folio);
} else { /* ZEROPAGE */
- clear_user_highpage(page, dst_addr);
+ clear_user_highpage(&folio->page, dst_addr);
}
} else {
- page = *pagep;
+ folio = page_folio(*pagep);
+ VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
*pagep = NULL;
}
- VM_BUG_ON(PageLocked(page));
- VM_BUG_ON(PageSwapBacked(page));
- __SetPageLocked(page);
- __SetPageSwapBacked(page);
- __SetPageUptodate(page);
+ VM_BUG_ON(folio_test_locked(folio));
+ VM_BUG_ON(folio_test_swapbacked(folio));
+ __folio_set_locked(folio);
+ __folio_set_swapbacked(folio);
+ __folio_mark_uptodate(folio);
ret = -EFAULT;
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
if (unlikely(pgoff >= max_off))
goto out_release;
- folio = page_folio(page);
ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
gfp & GFP_RECLAIM_MASK, dst_mm);
if (ret)
goto out_release;
ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
- page, true, wp_copy);
+ &folio->page, true, wp_copy);
if (ret)
goto out_delete_from_cache;
shmem_recalc_inode(inode);
spin_unlock_irq(&info->lock);
- unlock_page(page);
+ folio_unlock(folio);
return 0;
out_delete_from_cache:
- delete_from_page_cache(page);
+ filemap_remove_folio(folio);
out_release:
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
out_unacct_blocks:
shmem_inode_unacct_blocks(inode, 1);
return ret;
struct inode *inode = mapping->host;
struct shmem_inode_info *info = SHMEM_I(inode);
pgoff_t index = pos >> PAGE_SHIFT;
+ struct folio *folio;
int ret = 0;
/* i_rwsem is held by caller */
return -EPERM;
}
- ret = shmem_getpage(inode, index, pagep, SGP_WRITE);
+ ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
if (ret)
return ret;
+ *pagep = folio_file_page(folio, index);
if (PageHWPoison(*pagep)) {
- unlock_page(*pagep);
- put_page(*pagep);
+ folio_unlock(folio);
+ folio_put(folio);
*pagep = NULL;
return -EIO;
}
offset = *ppos & ~PAGE_MASK;
for (;;) {
+ struct folio *folio = NULL;
struct page *page = NULL;
pgoff_t end_index;
unsigned long nr, ret;
break;
}
- error = shmem_getpage(inode, index, &page, SGP_READ);
+ error = shmem_get_folio(inode, index, &folio, SGP_READ);
if (error) {
if (error == -EINVAL)
error = 0;
break;
}
- if (page) {
- unlock_page(page);
+ if (folio) {
+ folio_unlock(folio);
+ page = folio_file_page(folio, index);
if (PageHWPoison(page)) {
- put_page(page);
+ folio_put(folio);
error = -EIO;
break;
}
if (index == end_index) {
nr = i_size & ~PAGE_MASK;
if (nr <= offset) {
- if (page)
- put_page(page);
+ if (folio)
+ folio_put(folio);
break;
}
}
nr -= offset;
- if (page) {
+ if (folio) {
/*
* If users can be writing to this page using arbitrary
* virtual addresses, take care about potential aliasing
* Mark the page accessed if we read the beginning.
*/
if (!offset)
- mark_page_accessed(page);
+ folio_mark_accessed(folio);
/*
* Ok, we have the page, and it's up-to-date, so
* now we can copy it to user space...
*/
ret = copy_page_to_iter(page, offset, nr, to);
- put_page(page);
+ folio_put(folio);
} else if (user_backed_iter(to)) {
/*
info->fallocend = end;
for (index = start; index < end; ) {
- struct page *page;
+ struct folio *folio;
/*
* Good, the fallocate(2) manpage permits EINTR: we may have
else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
error = -ENOMEM;
else
- error = shmem_getpage(inode, index, &page, SGP_FALLOC);
+ error = shmem_get_folio(inode, index, &folio,
+ SGP_FALLOC);
if (error) {
info->fallocend = undo_fallocend;
- /* Remove the !PageUptodate pages we added */
+ /* Remove the !uptodate folios we added */
if (index > start) {
shmem_undo_range(inode,
(loff_t)start << PAGE_SHIFT,
goto undone;
}
- index++;
/*
* Here is a more important optimization than it appears:
- * a second SGP_FALLOC on the same huge page will clear it,
- * making it PageUptodate and un-undoable if we fail later.
+ * a second SGP_FALLOC on the same large folio will clear it,
+ * making it uptodate and un-undoable if we fail later.
*/
- if (PageTransCompound(page)) {
- index = round_up(index, HPAGE_PMD_NR);
- /* Beware 32-bit wraparound */
- if (!index)
- index--;
- }
+ index = folio_next_index(folio);
+ /* Beware 32-bit wraparound */
+ if (!index)
+ index--;
/*
* Inform shmem_writepage() how far we have reached.
* No need for lock or barrier: we have the page lock.
*/
- if (!PageUptodate(page))
+ if (!folio_test_uptodate(folio))
shmem_falloc.nr_falloced += index - shmem_falloc.next;
shmem_falloc.next = index;
/*
- * If !PageUptodate, leave it that way so that freeable pages
+ * If !uptodate, leave it that way so that freeable folios
* can be recognized if we need to rollback on error later.
- * But set_page_dirty so that memory pressure will swap rather
- * than free the pages we are allocating (and SGP_CACHE pages
+ * But mark it dirty so that memory pressure will swap rather
+ * than free the folios we are allocating (and SGP_CACHE folios
* might still be clean: we now need to mark those dirty too).
*/
- set_page_dirty(page);
- unlock_page(page);
- put_page(page);
+ folio_mark_dirty(folio);
+ folio_unlock(folio);
+ folio_put(folio);
cond_resched();
}
error = 0;
dir->i_size += BOGO_DIRENT_SIZE;
dir->i_ctime = dir->i_mtime = current_time(dir);
+ inode_inc_iversion(dir);
d_instantiate(dentry, inode);
dget(dentry); /* Extra count - pin the dentry in core */
}
static int
shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
- struct dentry *dentry, umode_t mode)
+ struct file *file, umode_t mode)
{
struct inode *inode;
int error = -ENOSPC;
error = simple_acl_create(dir, inode);
if (error)
goto out_iput;
- d_tmpfile(dentry, inode);
+ d_tmpfile(file, inode);
}
- return error;
+ return finish_open_simple(file, error);
out_iput:
iput(inode);
return error;
dir->i_size += BOGO_DIRENT_SIZE;
inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
+ inode_inc_iversion(dir);
inc_nlink(inode);
ihold(inode); /* New dentry reference */
dget(dentry); /* Extra pinning count for the created dentry */
dir->i_size -= BOGO_DIRENT_SIZE;
inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
+ inode_inc_iversion(dir);
drop_nlink(inode);
dput(dentry); /* Undo the count from "create" - this does all the work */
return 0;
old_dir->i_ctime = old_dir->i_mtime =
new_dir->i_ctime = new_dir->i_mtime =
inode->i_ctime = current_time(old_dir);
+ inode_inc_iversion(old_dir);
+ inode_inc_iversion(new_dir);
return 0;
}
int error;
int len;
struct inode *inode;
- struct page *page;
+ struct folio *folio;
len = strlen(symname) + 1;
if (len > PAGE_SIZE)
inode->i_op = &shmem_short_symlink_operations;
} else {
inode_nohighmem(inode);
- error = shmem_getpage(inode, 0, &page, SGP_WRITE);
+ error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
if (error) {
iput(inode);
return error;
}
inode->i_mapping->a_ops = &shmem_aops;
inode->i_op = &shmem_symlink_inode_operations;
- memcpy(page_address(page), symname, len);
- SetPageUptodate(page);
- set_page_dirty(page);
- unlock_page(page);
- put_page(page);
+ memcpy(folio_address(folio), symname, len);
+ folio_mark_uptodate(folio);
+ folio_mark_dirty(folio);
+ folio_unlock(folio);
+ folio_put(folio);
}
dir->i_size += BOGO_DIRENT_SIZE;
dir->i_ctime = dir->i_mtime = current_time(dir);
+ inode_inc_iversion(dir);
d_instantiate(dentry, inode);
dget(dentry);
return 0;
static void shmem_put_link(void *arg)
{
- mark_page_accessed(arg);
- put_page(arg);
+ folio_mark_accessed(arg);
+ folio_put(arg);
}
static const char *shmem_get_link(struct dentry *dentry,
struct inode *inode,
struct delayed_call *done)
{
- struct page *page = NULL;
+ struct folio *folio = NULL;
int error;
+
if (!dentry) {
- page = find_get_page(inode->i_mapping, 0);
- if (!page)
+ folio = filemap_get_folio(inode->i_mapping, 0);
+ if (!folio)
return ERR_PTR(-ECHILD);
- if (PageHWPoison(page) ||
- !PageUptodate(page)) {
- put_page(page);
+ if (PageHWPoison(folio_page(folio, 0)) ||
+ !folio_test_uptodate(folio)) {
+ folio_put(folio);
return ERR_PTR(-ECHILD);
}
} else {
- error = shmem_getpage(inode, 0, &page, SGP_READ);
+ error = shmem_get_folio(inode, 0, &folio, SGP_READ);
if (error)
return ERR_PTR(error);
- if (!page)
+ if (!folio)
return ERR_PTR(-ECHILD);
- if (PageHWPoison(page)) {
- unlock_page(page);
- put_page(page);
+ if (PageHWPoison(folio_page(folio, 0))) {
+ folio_unlock(folio);
+ folio_put(folio);
return ERR_PTR(-ECHILD);
}
- unlock_page(page);
+ folio_unlock(folio);
}
- set_delayed_call(done, shmem_put_link, page);
- return page_address(page);
+ set_delayed_call(done, shmem_put_link, folio);
+ return folio_address(folio);
}
#ifdef CONFIG_TMPFS_XATTR
shmem_set_inode_flags(inode, info->fsflags);
inode->i_ctime = current_time(inode);
+ inode_inc_iversion(inode);
return 0;
}
size_t size, int flags)
{
struct shmem_inode_info *info = SHMEM_I(inode);
+ int err;
name = xattr_full_name(handler, name);
- return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
+ err = simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
+ if (!err) {
+ inode->i_ctime = current_time(inode);
+ inode_inc_iversion(inode);
+ }
+ return err;
}
static const struct xattr_handler shmem_security_xattr_handler = {
sb->s_flags |= SB_NOUSER;
}
sb->s_export_op = &shmem_export_ops;
- sb->s_flags |= SB_NOSEC;
+ sb->s_flags |= SB_NOSEC | SB_I_VERSION;
#else
sb->s_flags |= SB_NOUSER;
#endif
{
#ifdef CONFIG_SHMEM
struct inode *inode = mapping->host;
+ struct folio *folio;
struct page *page;
int error;
BUG_ON(!shmem_mapping(mapping));
- error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
+ error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
gfp, NULL, NULL, NULL);
if (error)
return ERR_PTR(error);
- unlock_page(page);
+ folio_unlock(folio);
+ page = folio_file_page(folio, index);
if (PageHWPoison(page)) {
- put_page(page);
+ folio_put(folio);
return ERR_PTR(-EIO);
}
DEFINE_STATIC_KEY_FALSE(page_alloc_shuffle_key);
static bool shuffle_param;
-static int shuffle_show(char *buffer, const struct kernel_param *kp)
-{
- return sprintf(buffer, "%c\n", shuffle_param ? 'Y' : 'N');
-}
-static __meminit int shuffle_store(const char *val,
+static __meminit int shuffle_param_set(const char *val,
const struct kernel_param *kp)
{
- int rc = param_set_bool(val, kp);
-
- if (rc < 0)
- return rc;
- if (shuffle_param)
+ if (param_set_bool(val, kp))
+ return -EINVAL;
+ if (*(bool *)kp->arg)
static_branch_enable(&page_alloc_shuffle_key);
return 0;
}
-module_param_call(shuffle, shuffle_store, shuffle_show, &shuffle_param, 0400);
+
+static const struct kernel_param_ops shuffle_param_ops = {
+ .set = shuffle_param_set,
+ .get = param_get_bool,
+};
+module_param_cb(shuffle, &shuffle_param_ops, &shuffle_param, 0400);
/*
* For two pages to be swapped in the shuffle, they must be free (on a
* although actual page can be freed in rcu context
*/
if (OFF_SLAB(cachep))
- kmem_cache_free(cachep->freelist_cache, freelist);
+ kfree(freelist);
}
/*
if (flags & CFLGS_OFF_SLAB) {
struct kmem_cache *freelist_cache;
size_t freelist_size;
+ size_t freelist_cache_size;
freelist_size = num * sizeof(freelist_idx_t);
- freelist_cache = kmalloc_slab(freelist_size, 0u);
- if (!freelist_cache)
- continue;
-
- /*
- * Needed to avoid possible looping condition
- * in cache_grow_begin()
- */
- if (OFF_SLAB(freelist_cache))
- continue;
+ if (freelist_size > KMALLOC_MAX_CACHE_SIZE) {
+ freelist_cache_size = PAGE_SIZE << get_order(freelist_size);
+ } else {
+ freelist_cache = kmalloc_slab(freelist_size, 0u);
+ if (!freelist_cache)
+ continue;
+ freelist_cache_size = freelist_cache->size;
+
+ /*
+ * Needed to avoid possible looping condition
+ * in cache_grow_begin()
+ */
+ if (OFF_SLAB(freelist_cache))
+ continue;
+ }
/* check if off slab has enough benefit */
- if (freelist_cache->size > cachep->size / 2)
+ if (freelist_cache_size > cachep->size / 2)
continue;
}
cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
#endif
- if (OFF_SLAB(cachep)) {
- cachep->freelist_cache =
- kmalloc_slab(cachep->freelist_size, 0u);
- }
-
err = setup_cpu_cache(cachep, gfp);
if (err) {
__kmem_cache_release(cachep);
freelist = NULL;
else if (OFF_SLAB(cachep)) {
/* Slab management obj is off-slab. */
- freelist = kmem_cache_alloc_node(cachep->freelist_cache,
+ freelist = kmalloc_node(cachep->freelist_size,
local_flags, nodeid);
} else {
/* We will use last bytes at the slab for freelist */
unsigned int rand;
/* Use best entropy available to define a random shift */
- rand = get_random_int();
+ rand = get_random_u32();
/* Use a random state if the pre-computed list is not available */
if (!cachep->random_seq) {
memset(p[i], 0, s->object_size);
kmemleak_alloc_recursive(p[i], s->object_size, 1,
s->flags, flags);
+ kmsan_slab_alloc(s, p[i], flags);
}
memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
kmemleak_free(object);
kasan_kfree_large(object);
+ kmsan_kfree_large(object);
mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
-(PAGE_SIZE << order));
ptr = kasan_kmalloc_large(ptr, size, flags);
/* As ptr might get tagged, call kmemleak hook after KASAN. */
kmemleak_alloc(ptr, size, 1, flags);
+ kmsan_kmalloc_large(ptr, size, flags);
return ptr;
}
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/kasan.h>
+#include <linux/kmsan.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
prefetchw(object + s->offset);
}
+/*
+ * When running under KMSAN, get_freepointer_safe() may return an uninitialized
+ * pointer value in the case the current thread loses the race for the next
+ * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
+ * slab_alloc_node() will fail, so the uninitialized value won't be used, but
+ * KMSAN will still check all arguments of cmpxchg because of imperfect
+ * handling of inline assembly.
+ * To work around this problem, we apply __no_kmsan_checks to ensure that
+ * get_freepointer_safe() returns initialized memory.
+ */
+__no_kmsan_checks
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
unsigned long freepointer_addr;
void *x, bool init)
{
kmemleak_free_recursive(x, s->flags);
+ kmsan_slab_free(s, x);
debug_check_no_locks_freed(x, s->object_size);
return false;
freelist_count = oo_objects(s->oo);
- pos = get_random_int() % freelist_count;
+ pos = prandom_u32_max(freelist_count);
page_limit = slab->objects * s->size;
start = fixup_red_left(s, slab_address(slab));
STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
#endif /* CONFIG_SLUB_STATS */
+#ifdef CONFIG_KFENCE
+static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)
+{
+ return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE));
+}
+
+static ssize_t skip_kfence_store(struct kmem_cache *s,
+ const char *buf, size_t length)
+{
+ int ret = length;
+
+ if (buf[0] == '0')
+ s->flags &= ~SLAB_SKIP_KFENCE;
+ else if (buf[0] == '1')
+ s->flags |= SLAB_SKIP_KFENCE;
+ else
+ ret = -EINVAL;
+
+ return ret;
+}
+SLAB_ATTR(skip_kfence);
+#endif
+
static struct attribute *slab_attrs[] = {
&slab_size_attr.attr,
&object_size_attr.attr,
&failslab_attr.attr,
#endif
&usersize_attr.attr,
+#ifdef CONFIG_KFENCE
+ &skip_kfence_attr.attr,
+#endif
NULL
};
kfree(name);
return ERR_PTR(-EINVAL);
}
+ kmsan_unpoison_memory(name, p - name);
return name;
}
al->name = name;
al->next = alias_list;
alias_list = al;
+ kmsan_unpoison_memory(al, sizeof(*al));
return 0;
}
folio_batch_move_lru(fbatch, folio_activate_fn);
}
-static void folio_activate(struct folio *folio)
+void folio_activate(struct folio *folio)
{
if (folio_test_lru(folio) && !folio_test_active(folio) &&
!folio_test_unevictable(folio)) {
{
}
-static void folio_activate(struct folio *folio)
+void folio_activate(struct folio *folio)
{
struct lruvec *lruvec;
local_unlock(&cpu_fbatches.lock);
}
+#ifdef CONFIG_LRU_GEN
+static void folio_inc_refs(struct folio *folio)
+{
+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
+
+ if (folio_test_unevictable(folio))
+ return;
+
+ if (!folio_test_referenced(folio)) {
+ folio_set_referenced(folio);
+ return;
+ }
+
+ if (!folio_test_workingset(folio)) {
+ folio_set_workingset(folio);
+ return;
+ }
+
+ /* see the comment on MAX_NR_TIERS */
+ do {
+ new_flags = old_flags & LRU_REFS_MASK;
+ if (new_flags == LRU_REFS_MASK)
+ break;
+
+ new_flags += BIT(LRU_REFS_PGOFF);
+ new_flags |= old_flags & ~LRU_REFS_MASK;
+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
+}
+#else
+static void folio_inc_refs(struct folio *folio)
+{
+}
+#endif /* CONFIG_LRU_GEN */
+
/*
* Mark a page as having seen activity.
*
*/
void folio_mark_accessed(struct folio *folio)
{
+ if (lru_gen_enabled()) {
+ folio_inc_refs(folio);
+ return;
+ }
+
if (!folio_test_referenced(folio)) {
folio_set_referenced(folio);
} else if (folio_test_unevictable(folio)) {
folio_test_unevictable(folio), folio);
VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
+ /* see the comment in lru_gen_add_folio() */
+ if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
+ lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
+ folio_set_active(folio);
+
folio_get(folio);
local_lock(&cpu_fbatches.lock);
fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
EXPORT_SYMBOL(folio_add_lru);
/**
- * lru_cache_add_inactive_or_unevictable
- * @page: the page to be added to LRU
- * @vma: vma in which page is mapped for determining reclaimability
+ * folio_add_lru_vma() - Add a folio to the appropate LRU list for this VMA.
+ * @folio: The folio to be added to the LRU.
+ * @vma: VMA in which the folio is mapped.
*
- * Place @page on the inactive or unevictable LRU list, depending on its
- * evictability.
+ * If the VMA is mlocked, @folio is added to the unevictable list.
+ * Otherwise, it is treated the same way as folio_add_lru().
*/
-void lru_cache_add_inactive_or_unevictable(struct page *page,
- struct vm_area_struct *vma)
+void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)
{
- VM_BUG_ON_PAGE(PageLRU(page), page);
+ VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED))
- mlock_new_page(page);
+ mlock_new_page(&folio->page);
else
- lru_cache_add(page);
+ folio_add_lru(folio);
}
/*
static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio)
{
- if (folio_test_active(folio) && !folio_test_unevictable(folio)) {
+ if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) {
long nr_pages = folio_nr_pages(folio);
lruvec_del_folio(lruvec, folio);
{
struct folio *folio = page_folio(page);
- if (folio_test_lru(folio) && folio_test_active(folio) &&
- !folio_test_unevictable(folio)) {
+ if (folio_test_lru(folio) && !folio_test_unevictable(folio) &&
+ (folio_test_active(folio) || lru_gen_enabled())) {
struct folio_batch *fbatch;
folio_get(folio);
}
void swap_write_unplug(struct swap_iocb *sio);
int swap_writepage(struct page *page, struct writeback_control *wbc);
-void end_swap_bio_write(struct bio *bio);
-int __swap_writepage(struct page *page, struct writeback_control *wbc,
- bio_end_io_t end_write_func);
+int __swap_writepage(struct page *page, struct writeback_control *wbc);
/* linux/mm/swap_state.c */
/* One swap address space for each 64M swap space */
void show_swap_cache_info(void);
bool add_to_swap(struct folio *folio);
void *get_shadow_from_swap_cache(swp_entry_t entry);
-int add_to_swap_cache(struct page *page, swp_entry_t entry,
+int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
gfp_t gfp, void **shadowp);
void __delete_from_swap_cache(struct folio *folio,
swp_entry_t entry, void *shadow);
void delete_from_swap_cache(struct folio *folio);
void clear_shadow_from_swap_cache(int type, unsigned long begin,
unsigned long end);
-struct page *lookup_swap_cache(swp_entry_t entry,
- struct vm_area_struct *vma,
- unsigned long addr);
+struct folio *swap_cache_get_folio(swp_entry_t entry,
+ struct vm_area_struct *vma, unsigned long addr);
struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index);
struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
return 0;
}
-static inline struct page *lookup_swap_cache(swp_entry_t swp,
- struct vm_area_struct *vma,
- unsigned long addr)
+static inline struct folio *swap_cache_get_folio(swp_entry_t entry,
+ struct vm_area_struct *vma, unsigned long addr)
{
return NULL;
}
return NULL;
}
-static inline int add_to_swap_cache(struct page *page, swp_entry_t entry,
+static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
gfp_t gfp_mask, void **shadowp)
{
return -1;
unsigned long length;
struct swap_cgroup_ctrl *ctrl;
+ if (mem_cgroup_disabled())
+ return 0;
+
length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
array = vcalloc(length, sizeof(void *));
unsigned long i, length;
struct swap_cgroup_ctrl *ctrl;
+ if (mem_cgroup_disabled())
+ return;
+
mutex_lock(&swap_cgroup_mutex);
ctrl = &swap_cgroup_ctrl[type];
map = ctrl->map;
get_swap_pages(1, &entry, 1);
out:
if (mem_cgroup_try_charge_swap(folio, entry)) {
- put_swap_page(&folio->page, entry);
+ put_swap_folio(folio, entry);
entry.val = 0;
}
return entry;
* add_to_swap_cache resembles filemap_add_folio on swapper_space,
* but sets SwapCache flag and private instead of mapping and index.
*/
-int add_to_swap_cache(struct page *page, swp_entry_t entry,
+int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
gfp_t gfp, void **shadowp)
{
struct address_space *address_space = swap_address_space(entry);
pgoff_t idx = swp_offset(entry);
- XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page));
- unsigned long i, nr = thp_nr_pages(page);
+ XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio));
+ unsigned long i, nr = folio_nr_pages(folio);
void *old;
- VM_BUG_ON_PAGE(!PageLocked(page), page);
- VM_BUG_ON_PAGE(PageSwapCache(page), page);
- VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
+ VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
+ VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);
+ VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
- page_ref_add(page, nr);
- SetPageSwapCache(page);
+ folio_ref_add(folio, nr);
+ folio_set_swapcache(folio);
do {
xas_lock_irq(&xas);
if (xas_error(&xas))
goto unlock;
for (i = 0; i < nr; i++) {
- VM_BUG_ON_PAGE(xas.xa_index != idx + i, page);
+ VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio);
old = xas_load(&xas);
if (xa_is_value(old)) {
if (shadowp)
*shadowp = old;
}
- set_page_private(page + i, entry.val + i);
- xas_store(&xas, page);
+ set_page_private(folio_page(folio, i), entry.val + i);
+ xas_store(&xas, folio);
xas_next(&xas);
}
address_space->nrpages += nr;
- __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
- __mod_lruvec_page_state(page, NR_SWAPCACHE, nr);
+ __node_stat_mod_folio(folio, NR_FILE_PAGES, nr);
+ __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr);
unlock:
xas_unlock_irq(&xas);
} while (xas_nomem(&xas, gfp));
if (!xas_error(&xas))
return 0;
- ClearPageSwapCache(page);
- page_ref_sub(page, nr);
+ folio_clear_swapcache(folio);
+ folio_ref_sub(folio, nr);
return xas_error(&xas);
}
/*
* Add it to the swap cache.
*/
- err = add_to_swap_cache(&folio->page, entry,
+ err = add_to_swap_cache(folio, entry,
__GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL);
if (err)
/*
return true;
fail:
- put_swap_page(&folio->page, entry);
+ put_swap_folio(folio, entry);
return false;
}
__delete_from_swap_cache(folio, entry, NULL);
xa_unlock_irq(&address_space->i_pages);
- put_swap_page(&folio->page, entry);
+ put_swap_folio(folio, entry);
folio_ref_sub(folio, folio_nr_pages(folio));
}
/*
* If we are the only user, then try to free up the swap cache.
*
- * Its ok to check for PageSwapCache without the page lock
+ * Its ok to check the swapcache flag without the folio lock
* here because we are going to recheck again inside
- * try_to_free_swap() _with_ the lock.
+ * folio_free_swap() _with_ the lock.
* - Marcelo
*/
void free_swap_cache(struct page *page)
{
- if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) {
- try_to_free_swap(page);
- unlock_page(page);
+ struct folio *folio = page_folio(page);
+
+ if (folio_test_swapcache(folio) && !folio_mapped(folio) &&
+ folio_trylock(folio)) {
+ folio_free_swap(folio);
+ folio_unlock(folio);
}
}
}
/*
- * Lookup a swap entry in the swap cache. A found page will be returned
+ * Lookup a swap entry in the swap cache. A found folio will be returned
* unlocked and with its refcount incremented - we rely on the kernel
- * lock getting page table operations atomic even if we drop the page
+ * lock getting page table operations atomic even if we drop the folio
* lock before returning.
*/
-struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma,
- unsigned long addr)
+struct folio *swap_cache_get_folio(swp_entry_t entry,
+ struct vm_area_struct *vma, unsigned long addr)
{
- struct page *page;
+ struct folio *folio;
struct swap_info_struct *si;
si = get_swap_device(entry);
if (!si)
return NULL;
- page = find_get_page(swap_address_space(entry), swp_offset(entry));
+ folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry));
put_swap_device(si);
- if (page) {
+ if (folio) {
bool vma_ra = swap_use_vma_readahead();
bool readahead;
* At the moment, we don't support PG_readahead for anon THP
* so let's bail out rather than confusing the readahead stat.
*/
- if (unlikely(PageTransCompound(page)))
- return page;
+ if (unlikely(folio_test_large(folio)))
+ return folio;
- readahead = TestClearPageReadahead(page);
+ readahead = folio_test_clear_readahead(folio);
if (vma && vma_ra) {
unsigned long ra_val;
int win, hits;
}
}
- return page;
+ return folio;
}
/**
bool *new_page_allocated)
{
struct swap_info_struct *si;
- struct page *page;
+ struct folio *folio;
void *shadow = NULL;
*new_page_allocated = false;
int err;
/*
* First check the swap cache. Since this is normally
- * called after lookup_swap_cache() failed, re-calling
+ * called after swap_cache_get_folio() failed, re-calling
* that would confuse statistics.
*/
si = get_swap_device(entry);
if (!si)
return NULL;
- page = find_get_page(swap_address_space(entry),
- swp_offset(entry));
+ folio = filemap_get_folio(swap_address_space(entry),
+ swp_offset(entry));
put_swap_device(si);
- if (page)
- return page;
+ if (folio)
+ return folio_file_page(folio, swp_offset(entry));
/*
* Just skip read ahead for unused swap slot.
* before marking swap_map SWAP_HAS_CACHE, when -EEXIST will
* cause any racers to loop around until we add it to cache.
*/
- page = alloc_page_vma(gfp_mask, vma, addr);
- if (!page)
+ folio = vma_alloc_folio(gfp_mask, 0, vma, addr, false);
+ if (!folio)
return NULL;
/*
if (!err)
break;
- put_page(page);
+ folio_put(folio);
if (err != -EEXIST)
return NULL;
* The swap entry is ours to swap in. Prepare the new page.
*/
- __SetPageLocked(page);
- __SetPageSwapBacked(page);
+ __folio_set_locked(folio);
+ __folio_set_swapbacked(folio);
- if (mem_cgroup_swapin_charge_page(page, NULL, gfp_mask, entry))
+ if (mem_cgroup_swapin_charge_folio(folio, NULL, gfp_mask, entry))
goto fail_unlock;
/* May fail (-ENOMEM) if XArray node allocation failed. */
- if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow))
+ if (add_to_swap_cache(folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow))
goto fail_unlock;
mem_cgroup_swapin_uncharge_swap(entry);
if (shadow)
- workingset_refault(page_folio(page), shadow);
+ workingset_refault(folio, shadow);
- /* Caller will initiate read into locked page */
- lru_cache_add(page);
+ /* Caller will initiate read into locked folio */
+ folio_add_lru(folio);
*new_page_allocated = true;
- return page;
+ return &folio->page;
fail_unlock:
- put_swap_page(page, entry);
- unlock_page(page);
- put_page(page);
+ put_swap_folio(folio, entry);
+ folio_unlock(folio);
+ folio_put(folio);
return NULL;
}
/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
long total_swap_pages;
static int least_priority = -1;
+unsigned long swapfile_maximum_size;
+#ifdef CONFIG_MIGRATION
+bool swap_migration_ad_supported;
+#endif /* CONFIG_MIGRATION */
static const char Bad_file[] = "Bad swap file entry ";
static const char Unused_file[] = "Unused swap file entry ";
unsigned long offset, unsigned long flags)
{
swp_entry_t entry = swp_entry(si->type, offset);
- struct page *page;
+ struct folio *folio;
int ret = 0;
- page = find_get_page(swap_address_space(entry), offset);
- if (!page)
+ folio = filemap_get_folio(swap_address_space(entry), offset);
+ if (!folio)
return 0;
/*
* When this function is called from scan_swap_map_slots() and it's
- * called by vmscan.c at reclaiming pages. So, we hold a lock on a page,
+ * called by vmscan.c at reclaiming folios. So we hold a folio lock
* here. We have to use trylock for avoiding deadlock. This is a special
- * case and you should use try_to_free_swap() with explicit lock_page()
+ * case and you should use folio_free_swap() with explicit folio_lock()
* in usual operations.
*/
- if (trylock_page(page)) {
+ if (folio_trylock(folio)) {
if ((flags & TTRS_ANYWAY) ||
- ((flags & TTRS_UNMAPPED) && !page_mapped(page)) ||
- ((flags & TTRS_FULL) && mem_cgroup_swap_full(page)))
- ret = try_to_free_swap(page);
- unlock_page(page);
+ ((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) ||
+ ((flags & TTRS_FULL) && mem_cgroup_swap_full(folio)))
+ ret = folio_free_swap(folio);
+ folio_unlock(folio);
}
- put_page(page);
+ folio_put(folio);
return ret;
}
/*
* Called after dropping swapcache to decrease refcnt to swap entries.
*/
-void put_swap_page(struct page *page, swp_entry_t entry)
+void put_swap_folio(struct folio *folio, swp_entry_t entry)
{
unsigned long offset = swp_offset(entry);
unsigned long idx = offset / SWAPFILE_CLUSTER;
unsigned char *map;
unsigned int i, free_entries = 0;
unsigned char val;
- int size = swap_entry_size(thp_nr_pages(page));
+ int size = swap_entry_size(folio_nr_pages(folio));
si = _swap_info_get(entry);
if (!si)
spin_unlock(&p->lock);
}
-/*
- * How many references to page are currently swapped out?
- * This does not give an exact answer when swap count is continued,
- * but does include the high COUNT_CONTINUED flag to allow for that.
- */
-static int page_swapcount(struct page *page)
-{
- int count = 0;
- struct swap_info_struct *p;
- struct swap_cluster_info *ci;
- swp_entry_t entry;
- unsigned long offset;
-
- entry.val = page_private(page);
- p = _swap_info_get(entry);
- if (p) {
- offset = swp_offset(entry);
- ci = lock_cluster_or_swap_info(p, offset);
- count = swap_count(p->swap_map[offset]);
- unlock_cluster_or_swap_info(p, ci);
- }
- return count;
-}
-
int __swap_count(swp_entry_t entry)
{
struct swap_info_struct *si;
return count;
}
+/*
+ * How many references to @entry are currently swapped out?
+ * This does not give an exact answer when swap count is continued,
+ * but does include the high COUNT_CONTINUED flag to allow for that.
+ */
static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
{
- int count = 0;
pgoff_t offset = swp_offset(entry);
struct swap_cluster_info *ci;
+ int count;
ci = lock_cluster_or_swap_info(si, offset);
count = swap_count(si->swap_map[offset]);
static bool folio_swapped(struct folio *folio)
{
- swp_entry_t entry;
- struct swap_info_struct *si;
+ swp_entry_t entry = folio_swap_entry(folio);
+ struct swap_info_struct *si = _swap_info_get(entry);
+
+ if (!si)
+ return false;
if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio)))
- return page_swapcount(&folio->page) != 0;
+ return swap_swapcount(si, entry) != 0;
- entry = folio_swap_entry(folio);
- si = _swap_info_get(entry);
- if (si)
- return swap_page_trans_huge_swapped(si, entry);
- return false;
+ return swap_page_trans_huge_swapped(si, entry);
}
-/*
- * If swap is getting full, or if there are no more mappings of this page,
- * then try_to_free_swap is called to free its swap space.
+/**
+ * folio_free_swap() - Free the swap space used for this folio.
+ * @folio: The folio to remove.
+ *
+ * If swap is getting full, or if there are no more mappings of this folio,
+ * then call folio_free_swap to free its swap space.
+ *
+ * Return: true if we were able to release the swap space.
*/
-int try_to_free_swap(struct page *page)
+bool folio_free_swap(struct folio *folio)
{
- struct folio *folio = page_folio(page);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
if (!folio_test_swapcache(folio))
- return 0;
+ return false;
if (folio_test_writeback(folio))
- return 0;
+ return false;
if (folio_swapped(folio))
- return 0;
+ return false;
/*
* Once hibernation has begun to create its image of memory,
- * there's a danger that one of the calls to try_to_free_swap()
+ * there's a danger that one of the calls to folio_free_swap()
* - most probably a call from __try_to_reclaim_swap() while
* hibernation is allocating its own swap pages for the image,
* but conceivably even a call from memory reclaim - will free
- * the swap from a page which has already been recorded in the
- * image as a clean swapcache page, and then reuse its swap for
+ * the swap from a folio which has already been recorded in the
+ * image as a clean swapcache folio, and then reuse its swap for
* another page of the image. On waking from hibernation, the
- * original page might be freed under memory pressure, then
+ * original folio might be freed under memory pressure, then
* later read back in from swap, now with the wrong data.
*
* Hibernation suspends storage while it is writing the image
* to disk so check that here.
*/
if (pm_suspended_storage())
- return 0;
+ return false;
delete_from_swap_cache(folio);
folio_set_dirty(folio);
- return 1;
+ return true;
}
/*
* force COW, vm_page_prot omits write permission from any private vma.
*/
static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long addr, swp_entry_t entry, struct page *page)
+ unsigned long addr, swp_entry_t entry, struct folio *folio)
{
+ struct page *page = folio_file_page(folio, swp_offset(entry));
struct page *swapcache;
spinlock_t *ptl;
pte_t *pte, new_pte;
unsigned long addr, unsigned long end,
unsigned int type)
{
- struct page *page;
swp_entry_t entry;
pte_t *pte;
struct swap_info_struct *si;
- unsigned long offset;
int ret = 0;
volatile unsigned char *swap_map;
si = swap_info[type];
pte = pte_offset_map(pmd, addr);
do {
+ struct folio *folio;
+ unsigned long offset;
+
if (!is_swap_pte(*pte))
continue;
offset = swp_offset(entry);
pte_unmap(pte);
swap_map = &si->swap_map[offset];
- page = lookup_swap_cache(entry, vma, addr);
- if (!page) {
+ folio = swap_cache_get_folio(entry, vma, addr);
+ if (!folio) {
+ struct page *page;
struct vm_fault vmf = {
.vma = vma,
.address = addr,
page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
&vmf);
+ if (page)
+ folio = page_folio(page);
}
- if (!page) {
+ if (!folio) {
if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD)
goto try_next;
return -ENOMEM;
}
- lock_page(page);
- wait_on_page_writeback(page);
- ret = unuse_pte(vma, pmd, addr, entry, page);
+ folio_lock(folio);
+ folio_wait_writeback(folio);
+ ret = unuse_pte(vma, pmd, addr, entry, folio);
if (ret < 0) {
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
goto out;
}
- try_to_free_swap(page);
- unlock_page(page);
- put_page(page);
+ folio_free_swap(folio);
+ folio_unlock(folio);
+ folio_put(folio);
try_next:
pte = pte_offset_map(pmd, addr);
} while (pte++, addr += PAGE_SIZE, addr != end);
{
struct vm_area_struct *vma;
int ret = 0;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ for_each_vma(vmi, vma) {
if (vma->anon_vma) {
ret = unuse_vma(vma, type);
if (ret)
break;
}
+
cond_resched();
}
mmap_read_unlock(mm);
struct list_head *p;
int retval = 0;
struct swap_info_struct *si = swap_info[type];
- struct page *page;
+ struct folio *folio;
swp_entry_t entry;
unsigned int i;
(i = find_next_to_unuse(si, i)) != 0) {
entry = swp_entry(type, i);
- page = find_get_page(swap_address_space(entry), i);
- if (!page)
+ folio = filemap_get_folio(swap_address_space(entry), i);
+ if (!folio)
continue;
/*
- * It is conceivable that a racing task removed this page from
- * swap cache just before we acquired the page lock. The page
+ * It is conceivable that a racing task removed this folio from
+ * swap cache just before we acquired the page lock. The folio
* might even be back in swap cache on another swap area. But
- * that is okay, try_to_free_swap() only removes stale pages.
+ * that is okay, folio_free_swap() only removes stale folios.
*/
- lock_page(page);
- wait_on_page_writeback(page);
- try_to_free_swap(page);
- unlock_page(page);
- put_page(page);
+ folio_lock(folio);
+ folio_wait_writeback(folio);
+ folio_free_swap(folio);
+ folio_unlock(folio);
+ folio_put(folio);
}
/*
}
/* Can be overridden by an architecture for additional checks. */
-__weak unsigned long max_swapfile_size(void)
+__weak unsigned long arch_max_swapfile_size(void)
{
return generic_max_swapfile_size();
}
p->cluster_next = 1;
p->cluster_nr = 0;
- maxpages = max_swapfile_size();
+ maxpages = swapfile_maximum_size;
last_page = swap_header->info.last_page;
if (!last_page) {
pr_warn("Empty swap-file\n");
for_each_node(nid)
plist_head_init(&swap_avail_heads[nid]);
+ swapfile_maximum_size = arch_max_swapfile_size();
+
+#ifdef CONFIG_MIGRATION
+ if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
+ swap_migration_ad_supported = true;
+#endif /* CONFIG_MIGRATION */
+
return 0;
}
subsys_initcall(swapfile_init);
folio_invalidate(folio, offset, length);
if (!folio_test_large(folio))
return true;
- if (split_huge_page(&folio->page) == 0)
+ if (split_folio(folio) == 0)
return true;
if (folio_test_dirty(folio))
return false;
{
struct inode *inode = file_inode(dst_vma->vm_file);
pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
+ struct folio *folio;
struct page *page;
int ret;
- ret = shmem_getpage(inode, pgoff, &page, SGP_NOALLOC);
- /* Our caller expects us to return -EFAULT if we failed to find page. */
+ ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC);
+ /* Our caller expects us to return -EFAULT if we failed to find folio */
if (ret == -ENOENT)
ret = -EFAULT;
if (ret)
goto out;
- if (!page) {
+ if (!folio) {
ret = -EFAULT;
goto out;
}
+ page = folio_file_page(folio, pgoff);
if (PageHWPoison(page)) {
ret = -EIO;
goto out_release;
if (ret)
goto out_release;
- unlock_page(page);
+ folio_unlock(folio);
ret = 0;
out:
return ret;
out_release:
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
goto out;
}
BUG_ON(dst_addr >= dst_start + len);
/*
- * Serialize via i_mmap_rwsem and hugetlb_fault_mutex.
- * i_mmap_rwsem ensures the dst_pte remains valid even
+ * Serialize via vma_lock and hugetlb_fault_mutex.
+ * vma_lock ensures the dst_pte remains valid even
* in the case of shared pmds. fault mutex prevents
* races with other faulting threads.
*/
- mapping = dst_vma->vm_file->f_mapping;
- i_mmap_lock_read(mapping);
idx = linear_page_index(dst_vma, dst_addr);
+ mapping = dst_vma->vm_file->f_mapping;
hash = hugetlb_fault_mutex_hash(mapping, idx);
mutex_lock(&hugetlb_fault_mutex_table[hash]);
+ hugetlb_vma_lock_read(dst_vma);
err = -ENOMEM;
dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);
if (!dst_pte) {
+ hugetlb_vma_unlock_read(dst_vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
goto out_unlock;
}
if (mode != MCOPY_ATOMIC_CONTINUE &&
!huge_pte_none_mostly(huge_ptep_get(dst_pte))) {
err = -EEXIST;
+ hugetlb_vma_unlock_read(dst_vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
goto out_unlock;
}
dst_addr, src_addr, mode, &page,
wp_copy);
+ hugetlb_vma_unlock_read(dst_vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- i_mmap_unlock_read(mapping);
cond_resched();
}
EXPORT_SYMBOL(memdup_user_nul);
-void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
- struct vm_area_struct *prev)
-{
- struct vm_area_struct *next;
-
- vma->vm_prev = prev;
- if (prev) {
- next = prev->vm_next;
- prev->vm_next = vma;
- } else {
- next = mm->mmap;
- mm->mmap = vma;
- }
- vma->vm_next = next;
- if (next)
- next->vm_prev = vma;
-}
-
-void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma)
-{
- struct vm_area_struct *prev, *next;
-
- next = vma->vm_next;
- prev = vma->vm_prev;
- if (prev)
- prev->vm_next = next;
- else
- mm->mmap = next;
- if (next)
- next->vm_prev = prev;
-}
-
/* Check if the vma is being used as a stack by this task */
int vma_is_stack_for_current(struct vm_area_struct *vma)
{
return atomic_read(&folio->_mapcount) + 1;
compound = folio_entire_mapcount(folio);
- nr = folio_nr_pages(folio);
if (folio_test_hugetlb(folio))
return compound;
ret = compound;
+ nr = folio_nr_pages(folio);
for (i = 0; i < nr; i++)
ret += atomic_read(&folio_page(folio, i)->_mapcount) + 1;
/* File pages has compound_mapcount included in _mapcount */
if (percpu_counter_read_positive(&vm_committed_as) < allowed)
return 0;
error:
+ pr_warn_ratelimited("%s: pid: %d, comm: %s, no enough memory for the allocation\n",
+ __func__, current->pid, current->comm);
vm_unacct_memory(pages);
return -ENOMEM;
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2014 Davidlohr Bueso.
- */
-#include <linux/sched/signal.h>
-#include <linux/sched/task.h>
-#include <linux/mm.h>
-#include <linux/vmacache.h>
-
-/*
- * Hash based on the pmd of addr if configured with MMU, which provides a good
- * hit rate for workloads with spatial locality. Otherwise, use pages.
- */
-#ifdef CONFIG_MMU
-#define VMACACHE_SHIFT PMD_SHIFT
-#else
-#define VMACACHE_SHIFT PAGE_SHIFT
-#endif
-#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
-
-/*
- * This task may be accessing a foreign mm via (for example)
- * get_user_pages()->find_vma(). The vmacache is task-local and this
- * task's vmacache pertains to a different mm (ie, its own). There is
- * nothing we can do here.
- *
- * Also handle the case where a kernel thread has adopted this mm via
- * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm.
- */
-static inline bool vmacache_valid_mm(struct mm_struct *mm)
-{
- return current->mm == mm && !(current->flags & PF_KTHREAD);
-}
-
-void vmacache_update(unsigned long addr, struct vm_area_struct *newvma)
-{
- if (vmacache_valid_mm(newvma->vm_mm))
- current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma;
-}
-
-static bool vmacache_valid(struct mm_struct *mm)
-{
- struct task_struct *curr;
-
- if (!vmacache_valid_mm(mm))
- return false;
-
- curr = current;
- if (mm->vmacache_seqnum != curr->vmacache.seqnum) {
- /*
- * First attempt will always be invalid, initialize
- * the new cache for this task here.
- */
- curr->vmacache.seqnum = mm->vmacache_seqnum;
- vmacache_flush(curr);
- return false;
- }
- return true;
-}
-
-struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
-{
- int idx = VMACACHE_HASH(addr);
- int i;
-
- count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
- if (!vmacache_valid(mm))
- return NULL;
-
- for (i = 0; i < VMACACHE_SIZE; i++) {
- struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
- if (vma) {
-#ifdef CONFIG_DEBUG_VM_VMACACHE
- if (WARN_ON_ONCE(vma->vm_mm != mm))
- break;
-#endif
- if (vma->vm_start <= addr && vma->vm_end > addr) {
- count_vm_vmacache_event(VMACACHE_FIND_HITS);
- return vma;
- }
- }
- if (++idx == VMACACHE_SIZE)
- idx = 0;
- }
-
- return NULL;
-}
-
-#ifndef CONFIG_MMU
-struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
- unsigned long start,
- unsigned long end)
-{
- int idx = VMACACHE_HASH(start);
- int i;
-
- count_vm_vmacache_event(VMACACHE_FIND_CALLS);
-
- if (!vmacache_valid(mm))
- return NULL;
-
- for (i = 0; i < VMACACHE_SIZE; i++) {
- struct vm_area_struct *vma = current->vmacache.vmas[idx];
-
- if (vma && vma->vm_start == start && vma->vm_end == end) {
- count_vm_vmacache_event(VMACACHE_FIND_HITS);
- return vma;
- }
- if (++idx == VMACACHE_SIZE)
- idx = 0;
- }
-
- return NULL;
-}
-#endif
err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),
ioremap_max_page_shift);
flush_cache_vmap(addr, end);
+ if (!err)
+ kmsan_ioremap_page_range(addr, end, phys_addr, prot,
+ ioremap_max_page_shift);
return err;
}
*
* This is an internal function only. Do not use outside mm/.
*/
-void vunmap_range_noflush(unsigned long start, unsigned long end)
+void __vunmap_range_noflush(unsigned long start, unsigned long end)
{
unsigned long next;
pgd_t *pgd;
arch_sync_kernel_mappings(start, end);
}
+void vunmap_range_noflush(unsigned long start, unsigned long end)
+{
+ kmsan_vunmap_range_noflush(start, end);
+ __vunmap_range_noflush(start, end);
+}
+
/**
* vunmap_range - unmap kernel virtual addresses
* @addr: start of the VM area to unmap
*
* This is an internal function only. Do not use outside mm/.
*/
-int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
int err;
err = vmap_range_noflush(addr, addr + (1UL << page_shift),
- __pa(page_address(pages[i])), prot,
+ page_to_phys(pages[i]), prot,
page_shift);
if (err)
return err;
return 0;
}
+int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+ pgprot_t prot, struct page **pages, unsigned int page_shift)
+{
+ kmsan_vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+ return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+}
+
/**
* vmap_pages_range - map pages to a kernel virtual address
* @addr: start of the VM area to map
#include <linux/random.h>
static struct vmap_area *
-find_vmap_lowest_linear_match(unsigned long size,
+find_vmap_lowest_linear_match(struct list_head *head, unsigned long size,
unsigned long align, unsigned long vstart)
{
struct vmap_area *va;
- list_for_each_entry(va, &free_vmap_area_list, list) {
+ list_for_each_entry(va, head, list) {
if (!is_within_this_va(va, size, align, vstart))
continue;
}
static void
-find_vmap_lowest_match_check(unsigned long size, unsigned long align)
+find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head,
+ unsigned long size, unsigned long align)
{
struct vmap_area *va_1, *va_2;
unsigned long vstart;
get_random_bytes(&rnd, sizeof(rnd));
vstart = VMALLOC_START + rnd;
- va_1 = find_vmap_lowest_match(size, align, vstart, false);
- va_2 = find_vmap_lowest_linear_match(size, align, vstart);
+ va_1 = find_vmap_lowest_match(root, size, align, vstart, false);
+ va_2 = find_vmap_lowest_linear_match(head, size, align, vstart);
if (va_1 != va_2)
pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
return vend;
#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
- find_vmap_lowest_match_check(size, align);
+ find_vmap_lowest_match_check(root, head, size, align);
#endif
return nva_start_addr;
#include <linux/migrate.h>
#include <linux/delayacct.h>
#include <linux/sysctl.h>
+#include <linux/memory-tiers.h>
#include <linux/oom.h>
#include <linux/pagevec.h>
#include <linux/prefetch.h>
#include <linux/printk.h>
#include <linux/dax.h>
#include <linux/psi.h>
+#include <linux/pagewalk.h>
+#include <linux/shmem_fs.h>
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
#include <asm/tlbflush.h>
#include <asm/div64.h>
unsigned long anon_cost;
unsigned long file_cost;
- /* Can active pages be deactivated as part of reclaim? */
+ /* Can active folios be deactivated as part of reclaim? */
#define DEACTIVATE_ANON 1
#define DEACTIVATE_FILE 2
unsigned int may_deactivate:2;
/* Writepage batching in laptop mode; RECLAIM_WRITE */
unsigned int may_writepage:1;
- /* Can mapped pages be reclaimed? */
+ /* Can mapped folios be reclaimed? */
unsigned int may_unmap:1;
- /* Can pages be swapped as part of reclaim? */
+ /* Can folios be swapped as part of reclaim? */
unsigned int may_swap:1;
/* Proactive reclaim invoked by userspace through memory.reclaim */
/* There is easily reclaimable cold cache in the current node */
unsigned int cache_trim_mode:1;
- /* The file pages on the current node are dangerously low */
+ /* The file folios on the current node are dangerously low */
unsigned int file_is_tiny:1;
/* Always discard instead of demoting to lower tier memory */
unsigned int no_demotion:1;
+#ifdef CONFIG_LRU_GEN
+ /* help kswapd make better choices among multiple memcgs */
+ unsigned int memcgs_need_aging:1;
+ unsigned long last_reclaimed;
+#endif
+
/* Allocation order */
s8 order;
/* Scan (total_size >> priority) pages at once */
s8 priority;
- /* The highest zone to isolate pages for reclaim from */
+ /* The highest zone to isolate folios for reclaim from */
s8 reclaim_idx;
/* This context's GFP mask */
*
* The normal page dirty throttling mechanism in balance_dirty_pages() is
* completely broken with the legacy memcg and direct stalling in
- * shrink_page_list() is used for throttling instead, which lacks all the
+ * shrink_folio_list() is used for throttling instead, which lacks all the
* niceties such as fairness, adaptive pausing, bandwidth proportional
* allocation and configurability.
*
}
/*
- * This misses isolated pages which are not accounted for to save counters.
+ * This misses isolated folios which are not accounted for to save counters.
* As the data only determines if reclaim or compaction continues, it is
- * not expected that isolated pages will be a dominating factor.
+ * not expected that isolated folios will be a dominating factor.
*/
unsigned long zone_reclaimable_pages(struct zone *zone)
{
static inline int is_page_cache_freeable(struct folio *folio)
{
/*
- * A freeable page cache page is referenced only by the caller
- * that isolated the page, the page cache and optional buffer
- * heads at page->private.
+ * A freeable page cache folio is referenced only by the caller
+ * that isolated the folio, the page cache and optional filesystem
+ * private data at folio->private.
*/
return folio_ref_count(folio) - folio_test_private(folio) ==
1 + folio_nr_pages(folio);
return true;
/*
- * If there are a lot of dirty/writeback pages then do not
- * throttle as throttling will occur when the pages cycle
+ * If there are a lot of dirty/writeback folios then do not
+ * throttle as throttling will occur when the folios cycle
* towards the end of the LRU if still under writeback.
*/
for (i = 0; i < MAX_NR_ZONES; i++) {
* short. Failing to make progress or waiting on writeback are
* potentially long-lived events so use a longer timeout. This is shaky
* logic as a failure to make progress could be due to anything from
- * writeback to a slow device to excessive references pages at the tail
+ * writeback to a slow device to excessive referenced folios at the tail
* of the inactive LRU.
*/
switch(reason) {
}
/*
- * Account for pages written if tasks are throttled waiting on dirty
- * pages to clean. If enough pages have been cleaned since throttling
+ * Account for folios written if tasks are throttled waiting on dirty
+ * folios to clean. If enough folios have been cleaned since throttling
* started then wakeup the throttled tasks.
*/
void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
/* possible outcome of pageout() */
typedef enum {
- /* failed to write page out, page is locked */
+ /* failed to write folio out, folio is locked */
PAGE_KEEP,
- /* move page to the active list, page is locked */
+ /* move folio to the active list, folio is locked */
PAGE_ACTIVATE,
- /* page has been sent to the disk successfully, page is unlocked */
+ /* folio has been sent to the disk successfully, folio is unlocked */
PAGE_SUCCESS,
- /* page is clean and locked */
+ /* folio is clean and locked */
PAGE_CLEAN,
} pageout_t;
/*
- * pageout is called by shrink_page_list() for each dirty page.
+ * pageout is called by shrink_folio_list() for each dirty folio.
* Calls ->writepage().
*/
static pageout_t pageout(struct folio *folio, struct address_space *mapping,
}
/*
- * Same as remove_mapping, but if the page is removed from the mapping, it
+ * Same as remove_mapping, but if the folio is removed from the mapping, it
* gets returned with a refcount of 0.
*/
static int __remove_mapping(struct address_space *mapping, struct folio *folio,
spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages);
/*
- * The non racy check for a busy page.
+ * The non racy check for a busy folio.
*
* Must be careful with the order of the tests. When someone has
- * a ref to the page, it may be possible that they dirty it then
- * drop the reference. So if PageDirty is tested before page_count
- * here, then the following race may occur:
+ * a ref to the folio, it may be possible that they dirty it then
+ * drop the reference. So if the dirty flag is tested before the
+ * refcount here, then the following race may occur:
*
* get_user_pages(&page);
* [user mapping goes away]
* write_to(page);
- * !PageDirty(page) [good]
- * SetPageDirty(page);
- * put_page(page);
- * !page_count(page) [good, discard it]
+ * !folio_test_dirty(folio) [good]
+ * folio_set_dirty(folio);
+ * folio_put(folio);
+ * !refcount(folio) [good, discard it]
*
* [oops, our write_to data is lost]
*
* Reversing the order of the tests ensures such a situation cannot
- * escape unnoticed. The smp_rmb is needed to ensure the page->flags
- * load is not satisfied before that of page->_refcount.
+ * escape unnoticed. The smp_rmb is needed to ensure the folio->flags
+ * load is not satisfied before that of folio->_refcount.
*
- * Note that if SetPageDirty is always performed via set_page_dirty,
+ * Note that if the dirty flag is always set via folio_mark_dirty,
* and thus under the i_pages lock, then this ordering is not required.
*/
refcount = 1 + folio_nr_pages(folio);
if (!folio_ref_freeze(folio, refcount))
goto cannot_free;
- /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
+ /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */
if (unlikely(folio_test_dirty(folio))) {
folio_ref_unfreeze(folio, refcount);
goto cannot_free;
if (folio_test_swapcache(folio)) {
swp_entry_t swap = folio_swap_entry(folio);
- mem_cgroup_swapout(folio, swap);
+
+ /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */
if (reclaimed && !mapping_exiting(mapping))
shadow = workingset_eviction(folio, target_memcg);
+ mem_cgroup_swapout(folio, swap);
__delete_from_swap_cache(folio, swap, shadow);
xa_unlock_irq(&mapping->i_pages);
- put_swap_page(&folio->page, swap);
+ put_swap_folio(folio, swap);
} else {
void (*free_folio)(struct folio *);
* back.
*
* We also don't store shadows for DAX mappings because the
- * only page cache pages found in these are zero pages
+ * only page cache folios found in these are zero pages
* covering holes, and because we don't want to mix DAX
* exceptional entries and shadow exceptional entries in the
* same address_space.
folio_put(folio); /* drop ref from isolate */
}
-enum page_references {
- PAGEREF_RECLAIM,
- PAGEREF_RECLAIM_CLEAN,
- PAGEREF_KEEP,
- PAGEREF_ACTIVATE,
+enum folio_references {
+ FOLIOREF_RECLAIM,
+ FOLIOREF_RECLAIM_CLEAN,
+ FOLIOREF_KEEP,
+ FOLIOREF_ACTIVATE,
};
-static enum page_references folio_check_references(struct folio *folio,
+static enum folio_references folio_check_references(struct folio *folio,
struct scan_control *sc)
{
int referenced_ptes, referenced_folio;
* Let the folio, now marked Mlocked, be moved to the unevictable list.
*/
if (vm_flags & VM_LOCKED)
- return PAGEREF_ACTIVATE;
+ return FOLIOREF_ACTIVATE;
/* rmap lock contention: rotate */
if (referenced_ptes == -1)
- return PAGEREF_KEEP;
+ return FOLIOREF_KEEP;
if (referenced_ptes) {
/*
folio_set_referenced(folio);
if (referenced_folio || referenced_ptes > 1)
- return PAGEREF_ACTIVATE;
+ return FOLIOREF_ACTIVATE;
/*
* Activate file-backed executable folios after first usage.
*/
if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio))
- return PAGEREF_ACTIVATE;
+ return FOLIOREF_ACTIVATE;
- return PAGEREF_KEEP;
+ return FOLIOREF_KEEP;
}
/* Reclaim if clean, defer dirty folios to writeback */
if (referenced_folio && folio_is_file_lru(folio))
- return PAGEREF_RECLAIM_CLEAN;
+ return FOLIOREF_RECLAIM_CLEAN;
- return PAGEREF_RECLAIM;
+ return FOLIOREF_RECLAIM;
}
-/* Check if a page is dirty or under writeback */
+/* Check if a folio is dirty or under writeback */
static void folio_check_dirty_writeback(struct folio *folio,
bool *dirty, bool *writeback)
{
struct address_space *mapping;
/*
- * Anonymous pages are not handled by flushers and must be written
+ * Anonymous folios are not handled by flushers and must be written
* from reclaim context. Do not stall reclaim based on them.
- * MADV_FREE anonymous pages are put into inactive file list too.
+ * MADV_FREE anonymous folios are put into inactive file list too.
* They could be mistakenly treated as file lru. So further anon
* test is needed.
*/
mapping->a_ops->is_dirty_writeback(folio, dirty, writeback);
}
-static struct page *alloc_demote_page(struct page *page, unsigned long node)
+static struct page *alloc_demote_page(struct page *page, unsigned long private)
{
- struct migration_target_control mtc = {
- /*
- * Allocate from 'node', or fail quickly and quietly.
- * When this happens, 'page' will likely just be discarded
- * instead of migrated.
- */
- .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
- __GFP_THISNODE | __GFP_NOWARN |
- __GFP_NOMEMALLOC | GFP_NOWAIT,
- .nid = node
- };
+ struct page *target_page;
+ nodemask_t *allowed_mask;
+ struct migration_target_control *mtc;
+
+ mtc = (struct migration_target_control *)private;
+
+ allowed_mask = mtc->nmask;
+ /*
+ * make sure we allocate from the target node first also trying to
+ * demote or reclaim pages from the target node via kswapd if we are
+ * low on free memory on target node. If we don't do this and if
+ * we have free memory on the slower(lower) memtier, we would start
+ * allocating pages from slower(lower) memory tiers without even forcing
+ * a demotion of cold pages from the target memtier. This can result
+ * in the kernel placing hot pages in slower(lower) memory tiers.
+ */
+ mtc->nmask = NULL;
+ mtc->gfp_mask |= __GFP_THISNODE;
+ target_page = alloc_migration_target(page, (unsigned long)mtc);
+ if (target_page)
+ return target_page;
- return alloc_migration_target(page, (unsigned long)&mtc);
+ mtc->gfp_mask &= ~__GFP_THISNODE;
+ mtc->nmask = allowed_mask;
+
+ return alloc_migration_target(page, (unsigned long)mtc);
}
/*
- * Take pages on @demote_list and attempt to demote them to
- * another node. Pages which are not demoted are left on
- * @demote_pages.
+ * Take folios on @demote_folios and attempt to demote them to another node.
+ * Folios which are not demoted are left on @demote_folios.
*/
-static unsigned int demote_page_list(struct list_head *demote_pages,
+static unsigned int demote_folio_list(struct list_head *demote_folios,
struct pglist_data *pgdat)
{
int target_nid = next_demotion_node(pgdat->node_id);
unsigned int nr_succeeded;
+ nodemask_t allowed_mask;
+
+ struct migration_target_control mtc = {
+ /*
+ * Allocate from 'node', or fail quickly and quietly.
+ * When this happens, 'page' will likely just be discarded
+ * instead of migrated.
+ */
+ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN |
+ __GFP_NOMEMALLOC | GFP_NOWAIT,
+ .nid = target_nid,
+ .nmask = &allowed_mask
+ };
- if (list_empty(demote_pages))
+ if (list_empty(demote_folios))
return 0;
if (target_nid == NUMA_NO_NODE)
return 0;
+ node_get_allowed_targets(pgdat, &allowed_mask);
+
/* Demotion ignores all cpuset and mempolicy settings */
- migrate_pages(demote_pages, alloc_demote_page, NULL,
- target_nid, MIGRATE_ASYNC, MR_DEMOTION,
- &nr_succeeded);
+ migrate_pages(demote_folios, alloc_demote_page, NULL,
+ (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
+ &nr_succeeded);
if (current_is_kswapd())
__count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded);
}
/*
- * shrink_page_list() returns the number of reclaimed pages
+ * shrink_folio_list() returns the number of reclaimed pages
*/
-static unsigned int shrink_page_list(struct list_head *page_list,
- struct pglist_data *pgdat,
- struct scan_control *sc,
- struct reclaim_stat *stat,
- bool ignore_references)
-{
- LIST_HEAD(ret_pages);
- LIST_HEAD(free_pages);
- LIST_HEAD(demote_pages);
+static unsigned int shrink_folio_list(struct list_head *folio_list,
+ struct pglist_data *pgdat, struct scan_control *sc,
+ struct reclaim_stat *stat, bool ignore_references)
+{
+ LIST_HEAD(ret_folios);
+ LIST_HEAD(free_folios);
+ LIST_HEAD(demote_folios);
unsigned int nr_reclaimed = 0;
unsigned int pgactivate = 0;
bool do_demote_pass;
do_demote_pass = can_demote(pgdat->node_id, sc);
retry:
- while (!list_empty(page_list)) {
+ while (!list_empty(folio_list)) {
struct address_space *mapping;
struct folio *folio;
- enum page_references references = PAGEREF_RECLAIM;
+ enum folio_references references = FOLIOREF_RECLAIM;
bool dirty, writeback;
unsigned int nr_pages;
cond_resched();
- folio = lru_to_folio(page_list);
+ folio = lru_to_folio(folio_list);
list_del(&folio->lru);
if (!folio_trylock(folio))
if (!sc->may_unmap && folio_mapped(folio))
goto keep_locked;
+ /* folio_update_gen() tried to promote this page? */
+ if (lru_gen_enabled() && !ignore_references &&
+ folio_mapped(folio) && folio_test_referenced(folio))
+ goto keep_locked;
+
/*
* The number of dirty pages determines if a node is marked
* reclaim_congested. kswapd will stall and start writing
folio_unlock(folio);
folio_wait_writeback(folio);
/* then go back and try same folio again */
- list_add_tail(&folio->lru, page_list);
+ list_add_tail(&folio->lru, folio_list);
continue;
}
}
references = folio_check_references(folio, sc);
switch (references) {
- case PAGEREF_ACTIVATE:
+ case FOLIOREF_ACTIVATE:
goto activate_locked;
- case PAGEREF_KEEP:
+ case FOLIOREF_KEEP:
stat->nr_ref_keep += nr_pages;
goto keep_locked;
- case PAGEREF_RECLAIM:
- case PAGEREF_RECLAIM_CLEAN:
+ case FOLIOREF_RECLAIM:
+ case FOLIOREF_RECLAIM_CLEAN:
; /* try to reclaim the folio below */
}
*/
if (do_demote_pass &&
(thp_migration_supported() || !folio_test_large(folio))) {
- list_add(&folio->lru, &demote_pages);
+ list_add(&folio->lru, &demote_folios);
folio_unlock(folio);
continue;
}
*/
if (!folio_entire_mapcount(folio) &&
split_folio_to_list(folio,
- page_list))
+ folio_list))
goto activate_locked;
}
if (!add_to_swap(folio)) {
goto activate_locked_split;
/* Fallback to swap normal pages */
if (split_folio_to_list(folio,
- page_list))
+ folio_list))
goto activate_locked;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
count_vm_event(THP_SWPOUT_FALLBACK);
} else if (folio_test_swapbacked(folio) &&
folio_test_large(folio)) {
/* Split shmem folio */
- if (split_folio_to_list(folio, page_list))
+ if (split_folio_to_list(folio, folio_list))
goto keep_locked;
}
goto activate_locked;
}
- if (references == PAGEREF_RECLAIM_CLEAN)
+ if (references == FOLIOREF_RECLAIM_CLEAN)
goto keep_locked;
if (!may_enter_fs(folio, sc->gfp_mask))
goto keep_locked;
nr_reclaimed += nr_pages;
/*
- * Is there need to periodically free_page_list? It would
+ * Is there need to periodically free_folio_list? It would
* appear not as the counts should be low
*/
if (unlikely(folio_test_large(folio)))
destroy_large_folio(folio);
else
- list_add(&folio->lru, &free_pages);
+ list_add(&folio->lru, &free_folios);
continue;
activate_locked_split:
activate_locked:
/* Not a candidate for swapping, so reclaim swap space. */
if (folio_test_swapcache(folio) &&
- (mem_cgroup_swap_full(&folio->page) ||
- folio_test_mlocked(folio)))
- try_to_free_swap(&folio->page);
+ (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio)))
+ folio_free_swap(folio);
VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
if (!folio_test_mlocked(folio)) {
int type = folio_is_file_lru(folio);
keep_locked:
folio_unlock(folio);
keep:
- list_add(&folio->lru, &ret_pages);
+ list_add(&folio->lru, &ret_folios);
VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
folio_test_unevictable(folio), folio);
}
- /* 'page_list' is always empty here */
+ /* 'folio_list' is always empty here */
/* Migrate folios selected for demotion */
- nr_reclaimed += demote_page_list(&demote_pages, pgdat);
- /* Folios that could not be demoted are still in @demote_pages */
- if (!list_empty(&demote_pages)) {
- /* Folios which weren't demoted go back on @page_list for retry: */
- list_splice_init(&demote_pages, page_list);
+ nr_reclaimed += demote_folio_list(&demote_folios, pgdat);
+ /* Folios that could not be demoted are still in @demote_folios */
+ if (!list_empty(&demote_folios)) {
+ /* Folios which weren't demoted go back on @folio_list for retry: */
+ list_splice_init(&demote_folios, folio_list);
do_demote_pass = false;
goto retry;
}
pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
- mem_cgroup_uncharge_list(&free_pages);
+ mem_cgroup_uncharge_list(&free_folios);
try_to_unmap_flush();
- free_unref_page_list(&free_pages);
+ free_unref_page_list(&free_folios);
- list_splice(&ret_pages, page_list);
+ list_splice(&ret_folios, folio_list);
count_vm_events(PGACTIVATE, pgactivate);
if (plug)
}
unsigned int reclaim_clean_pages_from_list(struct zone *zone,
- struct list_head *folio_list)
+ struct list_head *folio_list)
{
struct scan_control sc = {
.gfp_mask = GFP_KERNEL,
* change in the future.
*/
noreclaim_flag = memalloc_noreclaim_save();
- nr_reclaimed = shrink_page_list(&clean_folios, zone->zone_pgdat, &sc,
+ nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc,
&stat, true);
memalloc_noreclaim_restore(noreclaim_flag);
*
* returns how many pages were moved onto *@dst.
*/
-static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
+static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
struct lruvec *lruvec, struct list_head *dst,
unsigned long *nr_scanned, struct scan_control *sc,
enum lru_list lru)
*
* Context:
*
- * (1) Must be called with an elevated refcount on the page. This is a
- * fundamental difference from isolate_lru_pages() (which is called
+ * (1) Must be called with an elevated refcount on the folio. This is a
+ * fundamental difference from isolate_lru_folios() (which is called
* without a stable reference).
* (2) The lru_lock must not be held.
* (3) Interrupts must be enabled.
}
/*
- * move_pages_to_lru() moves folios from private @list to appropriate LRU list.
+ * move_folios_to_lru() moves folios from private @list to appropriate LRU list.
* On return, @list is reused as a list of folios to be freed by the caller.
*
* Returns the number of pages moved to the given lruvec.
*/
-static unsigned int move_pages_to_lru(struct lruvec *lruvec,
- struct list_head *list)
+static unsigned int move_folios_to_lru(struct lruvec *lruvec,
+ struct list_head *list)
{
int nr_pages, nr_moved = 0;
LIST_HEAD(folios_to_free);
/*
* The folio_set_lru needs to be kept here for list integrity.
* Otherwise:
- * #0 move_pages_to_lru #1 release_pages
+ * #0 move_folios_to_lru #1 release_pages
* if (!folio_put_testzero())
* if (folio_put_testzero())
* !lru //skip lru_lock
* shrink_inactive_list() is a helper for shrink_node(). It returns the number
* of reclaimed pages
*/
-static unsigned long
-shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
- struct scan_control *sc, enum lru_list lru)
+static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
+ struct lruvec *lruvec, struct scan_control *sc,
+ enum lru_list lru)
{
- LIST_HEAD(page_list);
+ LIST_HEAD(folio_list);
unsigned long nr_scanned;
unsigned int nr_reclaimed = 0;
unsigned long nr_taken;
spin_lock_irq(&lruvec->lru_lock);
- nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
+ nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list,
&nr_scanned, sc, lru);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
if (nr_taken == 0)
return 0;
- nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
+ nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false);
spin_lock_irq(&lruvec->lru_lock);
- move_pages_to_lru(lruvec, &page_list);
+ move_folios_to_lru(lruvec, &folio_list);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
spin_unlock_irq(&lruvec->lru_lock);
lru_note_cost(lruvec, file, stat.nr_pageout);
- mem_cgroup_uncharge_list(&page_list);
- free_unref_page_list(&page_list);
+ mem_cgroup_uncharge_list(&folio_list);
+ free_unref_page_list(&folio_list);
/*
- * If dirty pages are scanned that are not queued for IO, it
+ * If dirty folios are scanned that are not queued for IO, it
* implies that flushers are not doing their job. This can
- * happen when memory pressure pushes dirty pages to the end of
+ * happen when memory pressure pushes dirty folios to the end of
* the LRU before the dirty limits are breached and the dirty
* data has expired. It can also happen when the proportion of
- * dirty pages grows not through writes but through memory
+ * dirty folios grows not through writes but through memory
* pressure reclaiming all the clean cache. And in some cases,
* the flushers simply cannot keep up with the allocation
* rate. Nudge the flusher threads in case they are asleep.
spin_lock_irq(&lruvec->lru_lock);
- nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
+ nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold,
&nr_scanned, sc, lru);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
*/
spin_lock_irq(&lruvec->lru_lock);
- nr_activate = move_pages_to_lru(lruvec, &l_active);
- nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
+ nr_activate = move_folios_to_lru(lruvec, &l_active);
+ nr_deactivate = move_folios_to_lru(lruvec, &l_inactive);
/* Keep all free folios in l_active list */
list_splice(&l_inactive, &l_active);
nr_deactivate, nr_rotated, sc->priority, file);
}
-static unsigned int reclaim_page_list(struct list_head *page_list,
+static unsigned int reclaim_folio_list(struct list_head *folio_list,
struct pglist_data *pgdat)
{
struct reclaim_stat dummy_stat;
.no_demotion = 1,
};
- nr_reclaimed = shrink_page_list(page_list, pgdat, &sc, &dummy_stat, false);
- while (!list_empty(page_list)) {
- folio = lru_to_folio(page_list);
+ nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false);
+ while (!list_empty(folio_list)) {
+ folio = lru_to_folio(folio_list);
list_del(&folio->lru);
folio_putback_lru(folio);
}
continue;
}
- nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid));
+ nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
nid = folio_nid(lru_to_folio(folio_list));
} while (!list_empty(folio_list));
- nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid));
+ nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
memalloc_noreclaim_restore(noreclaim_flag);
* but large enough to avoid thrashing the aggregate readahead window.
*
* Both inactive lists should also be large enough that each inactive
- * page has a chance to be referenced again before it is reclaimed.
+ * folio has a chance to be referenced again before it is reclaimed.
*
* If that fails and refaulting is observed, the inactive list grows.
*
- * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
+ * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios
* on this LRU, maintained by the pageout code. An inactive_ratio
- * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
+ * of 3 means 3:1 or 25% of the folios are kept on the inactive list.
*
* total target max
* memory ratio inactive
SCAN_FILE,
};
+static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc)
+{
+ unsigned long file;
+ struct lruvec *target_lruvec;
+
+ if (lru_gen_enabled())
+ return;
+
+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
+
+ /*
+ * Flush the memory cgroup stats, so that we read accurate per-memcg
+ * lruvec stats for heuristics.
+ */
+ mem_cgroup_flush_stats();
+
+ /*
+ * Determine the scan balance between anon and file LRUs.
+ */
+ spin_lock_irq(&target_lruvec->lru_lock);
+ sc->anon_cost = target_lruvec->anon_cost;
+ sc->file_cost = target_lruvec->file_cost;
+ spin_unlock_irq(&target_lruvec->lru_lock);
+
+ /*
+ * Target desirable inactive:active list ratios for the anon
+ * and file LRU lists.
+ */
+ if (!sc->force_deactivate) {
+ unsigned long refaults;
+
+ /*
+ * When refaults are being observed, it means a new
+ * workingset is being established. Deactivate to get
+ * rid of any stale active pages quickly.
+ */
+ refaults = lruvec_page_state(target_lruvec,
+ WORKINGSET_ACTIVATE_ANON);
+ if (refaults != target_lruvec->refaults[WORKINGSET_ANON] ||
+ inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
+ sc->may_deactivate |= DEACTIVATE_ANON;
+ else
+ sc->may_deactivate &= ~DEACTIVATE_ANON;
+
+ refaults = lruvec_page_state(target_lruvec,
+ WORKINGSET_ACTIVATE_FILE);
+ if (refaults != target_lruvec->refaults[WORKINGSET_FILE] ||
+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
+ sc->may_deactivate |= DEACTIVATE_FILE;
+ else
+ sc->may_deactivate &= ~DEACTIVATE_FILE;
+ } else
+ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
+
+ /*
+ * If we have plenty of inactive file pages that aren't
+ * thrashing, try to reclaim those first before touching
+ * anonymous pages.
+ */
+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
+ sc->cache_trim_mode = 1;
+ else
+ sc->cache_trim_mode = 0;
+
+ /*
+ * Prevent the reclaimer from falling into the cache trap: as
+ * cache pages start out inactive, every cache fault will tip
+ * the scan balance towards the file LRU. And as the file LRU
+ * shrinks, so does the window for rotation from references.
+ * This means we have a runaway feedback loop where a tiny
+ * thrashing file LRU becomes infinitely more attractive than
+ * anon pages. Try to detect this based on file LRU size.
+ */
+ if (!cgroup_reclaim(sc)) {
+ unsigned long total_high_wmark = 0;
+ unsigned long free, anon;
+ int z;
+
+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
+ file = node_page_state(pgdat, NR_ACTIVE_FILE) +
+ node_page_state(pgdat, NR_INACTIVE_FILE);
+
+ for (z = 0; z < MAX_NR_ZONES; z++) {
+ struct zone *zone = &pgdat->node_zones[z];
+
+ if (!managed_zone(zone))
+ continue;
+
+ total_high_wmark += high_wmark_pages(zone);
+ }
+
+ /*
+ * Consider anon: if that's low too, this isn't a
+ * runaway file reclaim problem, but rather just
+ * extreme pressure. Reclaim as per usual then.
+ */
+ anon = node_page_state(pgdat, NR_INACTIVE_ANON);
+
+ sc->file_is_tiny =
+ file + free <= total_high_wmark &&
+ !(sc->may_deactivate & DEACTIVATE_ANON) &&
+ anon >> sc->priority;
+ }
+}
+
/*
* Determine how aggressively the anon and file LRU lists should be
* scanned.
*
- * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
- * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
+ * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan
+ * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan
*/
static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
unsigned long *nr)
unsigned long ap, fp;
enum lru_list lru;
- /* If we have no swap space, do not bother scanning anon pages. */
+ /* If we have no swap space, do not bother scanning anon folios. */
if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
scan_balance = SCAN_FILE;
goto out;
return can_demote(pgdat->node_id, sc);
}
-static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
-{
- unsigned long nr[NR_LRU_LISTS];
- unsigned long targets[NR_LRU_LISTS];
- unsigned long nr_to_scan;
- enum lru_list lru;
- unsigned long nr_reclaimed = 0;
- unsigned long nr_to_reclaim = sc->nr_to_reclaim;
- struct blk_plug plug;
- bool scan_adjusted;
-
- get_scan_count(lruvec, sc, nr);
-
- /* Record the original scan target for proportional adjustments later */
- memcpy(targets, nr, sizeof(nr));
+#ifdef CONFIG_LRU_GEN
- /*
- * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
- * event that can occur when there is little memory pressure e.g.
- * multiple streaming readers/writers. Hence, we do not abort scanning
- * when the requested number of pages are reclaimed when scanning at
- * DEF_PRIORITY on the assumption that the fact we are direct
- * reclaiming implies that kswapd is not keeping up and it is best to
- * do a batch of work at once. For memcg reclaim one check is made to
- * abort proportional reclaim if either the file or anon lru has already
- * dropped to zero at the first pass.
- */
- scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
- sc->priority == DEF_PRIORITY);
+#ifdef CONFIG_LRU_GEN_ENABLED
+DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS);
+#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap])
+#else
+DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS);
+#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap])
+#endif
- blk_start_plug(&plug);
- while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
- nr[LRU_INACTIVE_FILE]) {
- unsigned long nr_anon, nr_file, percentage;
- unsigned long nr_scanned;
+/******************************************************************************
+ * shorthand helpers
+ ******************************************************************************/
- for_each_evictable_lru(lru) {
- if (nr[lru]) {
- nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
- nr[lru] -= nr_to_scan;
+#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset))
- nr_reclaimed += shrink_list(lru, nr_to_scan,
- lruvec, sc);
- }
- }
+#define DEFINE_MAX_SEQ(lruvec) \
+ unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
- cond_resched();
+#define DEFINE_MIN_SEQ(lruvec) \
+ unsigned long min_seq[ANON_AND_FILE] = { \
+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
+ READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
+ }
- if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
- continue;
+#define for_each_gen_type_zone(gen, type, zone) \
+ for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \
+ for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
+ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
- /*
- * For kswapd and memcg, reclaim at least the number of pages
- * requested. Ensure that the anon and file LRUs are scanned
- * proportionally what was requested by get_scan_count(). We
- * stop reclaiming one LRU and reduce the amount scanning
- * proportional to the original scan target.
- */
- nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
- nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
+static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
+{
+ struct pglist_data *pgdat = NODE_DATA(nid);
- /*
- * It's just vindictive to attack the larger once the smaller
- * has gone to zero. And given the way we stop scanning the
- * smaller below, this makes sure that we only make one nudge
- * towards proportionality once we've got nr_to_reclaim.
- */
- if (!nr_file || !nr_anon)
- break;
+#ifdef CONFIG_MEMCG
+ if (memcg) {
+ struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
- if (nr_file > nr_anon) {
- unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
- targets[LRU_ACTIVE_ANON] + 1;
- lru = LRU_BASE;
- percentage = nr_anon * 100 / scan_target;
- } else {
- unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
- targets[LRU_ACTIVE_FILE] + 1;
- lru = LRU_FILE;
- percentage = nr_file * 100 / scan_target;
- }
+ /* for hotadd_new_pgdat() */
+ if (!lruvec->pgdat)
+ lruvec->pgdat = pgdat;
- /* Stop scanning the smaller of the LRU */
- nr[lru] = 0;
- nr[lru + LRU_ACTIVE] = 0;
+ return lruvec;
+ }
+#endif
+ VM_WARN_ON_ONCE(!mem_cgroup_disabled());
- /*
- * Recalculate the other LRU scan count based on its original
- * scan target and the percentage scanning already complete
- */
- lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
- nr_scanned = targets[lru] - nr[lru];
- nr[lru] = targets[lru] * (100 - percentage) / 100;
- nr[lru] -= min(nr[lru], nr_scanned);
+ return pgdat ? &pgdat->__lruvec : NULL;
+}
- lru += LRU_ACTIVE;
- nr_scanned = targets[lru] - nr[lru];
- nr[lru] = targets[lru] * (100 - percentage) / 100;
- nr[lru] -= min(nr[lru], nr_scanned);
+static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
+{
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ struct pglist_data *pgdat = lruvec_pgdat(lruvec);
- scan_adjusted = true;
- }
- blk_finish_plug(&plug);
- sc->nr_reclaimed += nr_reclaimed;
+ if (!can_demote(pgdat->node_id, sc) &&
+ mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
+ return 0;
- /*
- * Even if we did not try to evict anon pages at all, we want to
- * rebalance the anon lru active/inactive ratio.
- */
- if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) &&
- inactive_is_low(lruvec, LRU_INACTIVE_ANON))
- shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
- sc, LRU_ACTIVE_ANON);
+ return mem_cgroup_swappiness(memcg);
}
-/* Use reclaim/compaction for costly allocs or under memory pressure */
-static bool in_reclaim_compaction(struct scan_control *sc)
+static int get_nr_gens(struct lruvec *lruvec, int type)
{
- if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
- (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
- sc->priority < DEF_PRIORITY - 2))
- return true;
+ return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
+}
- return false;
+static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
+{
+ /* see the comment on lru_gen_struct */
+ return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS &&
+ get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) &&
+ get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
}
-/*
- * Reclaim/compaction is used for high-order allocation requests. It reclaims
- * order-0 pages before compacting the zone. should_continue_reclaim() returns
- * true if more pages should be reclaimed such that when the page allocator
+/******************************************************************************
+ * mm_struct list
+ ******************************************************************************/
+
+static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg)
+{
+ static struct lru_gen_mm_list mm_list = {
+ .fifo = LIST_HEAD_INIT(mm_list.fifo),
+ .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock),
+ };
+
+#ifdef CONFIG_MEMCG
+ if (memcg)
+ return &memcg->mm_list;
+#endif
+ VM_WARN_ON_ONCE(!mem_cgroup_disabled());
+
+ return &mm_list;
+}
+
+void lru_gen_add_mm(struct mm_struct *mm)
+{
+ int nid;
+ struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
+
+ VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list));
+#ifdef CONFIG_MEMCG
+ VM_WARN_ON_ONCE(mm->lru_gen.memcg);
+ mm->lru_gen.memcg = memcg;
+#endif
+ spin_lock(&mm_list->lock);
+
+ for_each_node_state(nid, N_MEMORY) {
+ struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+ if (!lruvec)
+ continue;
+
+ /* the first addition since the last iteration */
+ if (lruvec->mm_state.tail == &mm_list->fifo)
+ lruvec->mm_state.tail = &mm->lru_gen.list;
+ }
+
+ list_add_tail(&mm->lru_gen.list, &mm_list->fifo);
+
+ spin_unlock(&mm_list->lock);
+}
+
+void lru_gen_del_mm(struct mm_struct *mm)
+{
+ int nid;
+ struct lru_gen_mm_list *mm_list;
+ struct mem_cgroup *memcg = NULL;
+
+ if (list_empty(&mm->lru_gen.list))
+ return;
+
+#ifdef CONFIG_MEMCG
+ memcg = mm->lru_gen.memcg;
+#endif
+ mm_list = get_mm_list(memcg);
+
+ spin_lock(&mm_list->lock);
+
+ for_each_node(nid) {
+ struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+ if (!lruvec)
+ continue;
+
+ /* where the last iteration ended (exclusive) */
+ if (lruvec->mm_state.tail == &mm->lru_gen.list)
+ lruvec->mm_state.tail = lruvec->mm_state.tail->next;
+
+ /* where the current iteration continues (inclusive) */
+ if (lruvec->mm_state.head != &mm->lru_gen.list)
+ continue;
+
+ lruvec->mm_state.head = lruvec->mm_state.head->next;
+ /* the deletion ends the current iteration */
+ if (lruvec->mm_state.head == &mm_list->fifo)
+ WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1);
+ }
+
+ list_del_init(&mm->lru_gen.list);
+
+ spin_unlock(&mm_list->lock);
+
+#ifdef CONFIG_MEMCG
+ mem_cgroup_put(mm->lru_gen.memcg);
+ mm->lru_gen.memcg = NULL;
+#endif
+}
+
+#ifdef CONFIG_MEMCG
+void lru_gen_migrate_mm(struct mm_struct *mm)
+{
+ struct mem_cgroup *memcg;
+ struct task_struct *task = rcu_dereference_protected(mm->owner, true);
+
+ VM_WARN_ON_ONCE(task->mm != mm);
+ lockdep_assert_held(&task->alloc_lock);
+
+ /* for mm_update_next_owner() */
+ if (mem_cgroup_disabled())
+ return;
+
+ rcu_read_lock();
+ memcg = mem_cgroup_from_task(task);
+ rcu_read_unlock();
+ if (memcg == mm->lru_gen.memcg)
+ return;
+
+ VM_WARN_ON_ONCE(!mm->lru_gen.memcg);
+ VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list));
+
+ lru_gen_del_mm(mm);
+ lru_gen_add_mm(mm);
+}
+#endif
+
+/*
+ * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when
+ * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of
+ * bits in a bitmap, k is the number of hash functions and n is the number of
+ * inserted items.
+ *
+ * Page table walkers use one of the two filters to reduce their search space.
+ * To get rid of non-leaf entries that no longer have enough leaf entries, the
+ * aging uses the double-buffering technique to flip to the other filter each
+ * time it produces a new generation. For non-leaf entries that have enough
+ * leaf entries, the aging carries them over to the next generation in
+ * walk_pmd_range(); the eviction also report them when walking the rmap
+ * in lru_gen_look_around().
+ *
+ * For future optimizations:
+ * 1. It's not necessary to keep both filters all the time. The spare one can be
+ * freed after the RCU grace period and reallocated if needed again.
+ * 2. And when reallocating, it's worth scaling its size according to the number
+ * of inserted entries in the other filter, to reduce the memory overhead on
+ * small systems and false positives on large systems.
+ * 3. Jenkins' hash function is an alternative to Knuth's.
+ */
+#define BLOOM_FILTER_SHIFT 15
+
+static inline int filter_gen_from_seq(unsigned long seq)
+{
+ return seq % NR_BLOOM_FILTERS;
+}
+
+static void get_item_key(void *item, int *key)
+{
+ u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2);
+
+ BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32));
+
+ key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1);
+ key[1] = hash >> BLOOM_FILTER_SHIFT;
+}
+
+static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq)
+{
+ unsigned long *filter;
+ int gen = filter_gen_from_seq(seq);
+
+ filter = lruvec->mm_state.filters[gen];
+ if (filter) {
+ bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
+ return;
+ }
+
+ filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
+ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
+ WRITE_ONCE(lruvec->mm_state.filters[gen], filter);
+}
+
+static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item)
+{
+ int key[2];
+ unsigned long *filter;
+ int gen = filter_gen_from_seq(seq);
+
+ filter = READ_ONCE(lruvec->mm_state.filters[gen]);
+ if (!filter)
+ return;
+
+ get_item_key(item, key);
+
+ if (!test_bit(key[0], filter))
+ set_bit(key[0], filter);
+ if (!test_bit(key[1], filter))
+ set_bit(key[1], filter);
+}
+
+static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item)
+{
+ int key[2];
+ unsigned long *filter;
+ int gen = filter_gen_from_seq(seq);
+
+ filter = READ_ONCE(lruvec->mm_state.filters[gen]);
+ if (!filter)
+ return true;
+
+ get_item_key(item, key);
+
+ return test_bit(key[0], filter) && test_bit(key[1], filter);
+}
+
+static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last)
+{
+ int i;
+ int hist;
+
+ lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);
+
+ if (walk) {
+ hist = lru_hist_from_seq(walk->max_seq);
+
+ for (i = 0; i < NR_MM_STATS; i++) {
+ WRITE_ONCE(lruvec->mm_state.stats[hist][i],
+ lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]);
+ walk->mm_stats[i] = 0;
+ }
+ }
+
+ if (NR_HIST_GENS > 1 && last) {
+ hist = lru_hist_from_seq(lruvec->mm_state.seq + 1);
+
+ for (i = 0; i < NR_MM_STATS; i++)
+ WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0);
+ }
+}
+
+static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
+{
+ int type;
+ unsigned long size = 0;
+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
+ int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
+
+ if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
+ return true;
+
+ clear_bit(key, &mm->lru_gen.bitmap);
+
+ for (type = !walk->can_swap; type < ANON_AND_FILE; type++) {
+ size += type ? get_mm_counter(mm, MM_FILEPAGES) :
+ get_mm_counter(mm, MM_ANONPAGES) +
+ get_mm_counter(mm, MM_SHMEMPAGES);
+ }
+
+ if (size < MIN_LRU_BATCH)
+ return true;
+
+ return !mmget_not_zero(mm);
+}
+
+static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk,
+ struct mm_struct **iter)
+{
+ bool first = false;
+ bool last = true;
+ struct mm_struct *mm = NULL;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
+
+ /*
+ * There are four interesting cases for this page table walker:
+ * 1. It tries to start a new iteration of mm_list with a stale max_seq;
+ * there is nothing left to do.
+ * 2. It's the first of the current generation, and it needs to reset
+ * the Bloom filter for the next generation.
+ * 3. It reaches the end of mm_list, and it needs to increment
+ * mm_state->seq; the iteration is done.
+ * 4. It's the last of the current generation, and it needs to reset the
+ * mm stats counters for the next generation.
+ */
+ spin_lock(&mm_list->lock);
+
+ VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq);
+ VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq);
+ VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers);
+
+ if (walk->max_seq <= mm_state->seq) {
+ if (!*iter)
+ last = false;
+ goto done;
+ }
+
+ if (!mm_state->nr_walkers) {
+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo);
+
+ mm_state->head = mm_list->fifo.next;
+ first = true;
+ }
+
+ while (!mm && mm_state->head != &mm_list->fifo) {
+ mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
+
+ mm_state->head = mm_state->head->next;
+
+ /* force scan for those added after the last iteration */
+ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) {
+ mm_state->tail = mm_state->head;
+ walk->force_scan = true;
+ }
+
+ if (should_skip_mm(mm, walk))
+ mm = NULL;
+ }
+
+ if (mm_state->head == &mm_list->fifo)
+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
+done:
+ if (*iter && !mm)
+ mm_state->nr_walkers--;
+ if (!*iter && mm)
+ mm_state->nr_walkers++;
+
+ if (mm_state->nr_walkers)
+ last = false;
+
+ if (*iter || last)
+ reset_mm_stats(lruvec, walk, last);
+
+ spin_unlock(&mm_list->lock);
+
+ if (mm && first)
+ reset_bloom_filter(lruvec, walk->max_seq + 1);
+
+ if (*iter)
+ mmput_async(*iter);
+
+ *iter = mm;
+
+ return last;
+}
+
+static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
+{
+ bool success = false;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ struct lru_gen_mm_list *mm_list = get_mm_list(memcg);
+ struct lru_gen_mm_state *mm_state = &lruvec->mm_state;
+
+ spin_lock(&mm_list->lock);
+
+ VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
+
+ if (max_seq > mm_state->seq && !mm_state->nr_walkers) {
+ VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo);
+
+ WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
+ reset_mm_stats(lruvec, NULL, true);
+ success = true;
+ }
+
+ spin_unlock(&mm_list->lock);
+
+ return success;
+}
+
+/******************************************************************************
+ * refault feedback loop
+ ******************************************************************************/
+
+/*
+ * A feedback loop based on Proportional-Integral-Derivative (PID) controller.
+ *
+ * The P term is refaulted/(evicted+protected) from a tier in the generation
+ * currently being evicted; the I term is the exponential moving average of the
+ * P term over the generations previously evicted, using the smoothing factor
+ * 1/2; the D term isn't supported.
+ *
+ * The setpoint (SP) is always the first tier of one type; the process variable
+ * (PV) is either any tier of the other type or any other tier of the same
+ * type.
+ *
+ * The error is the difference between the SP and the PV; the correction is to
+ * turn off protection when SP>PV or turn on protection when SP<PV.
+ *
+ * For future optimizations:
+ * 1. The D term may discount the other two terms over time so that long-lived
+ * generations can resist stale information.
+ */
+struct ctrl_pos {
+ unsigned long refaulted;
+ unsigned long total;
+ int gain;
+};
+
+static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
+ struct ctrl_pos *pos)
+{
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ int hist = lru_hist_from_seq(lrugen->min_seq[type]);
+
+ pos->refaulted = lrugen->avg_refaulted[type][tier] +
+ atomic_long_read(&lrugen->refaulted[hist][type][tier]);
+ pos->total = lrugen->avg_total[type][tier] +
+ atomic_long_read(&lrugen->evicted[hist][type][tier]);
+ if (tier)
+ pos->total += lrugen->protected[hist][type][tier - 1];
+ pos->gain = gain;
+}
+
+static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
+{
+ int hist, tier;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
+ unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
+
+ lockdep_assert_held(&lruvec->lru_lock);
+
+ if (!carryover && !clear)
+ return;
+
+ hist = lru_hist_from_seq(seq);
+
+ for (tier = 0; tier < MAX_NR_TIERS; tier++) {
+ if (carryover) {
+ unsigned long sum;
+
+ sum = lrugen->avg_refaulted[type][tier] +
+ atomic_long_read(&lrugen->refaulted[hist][type][tier]);
+ WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
+
+ sum = lrugen->avg_total[type][tier] +
+ atomic_long_read(&lrugen->evicted[hist][type][tier]);
+ if (tier)
+ sum += lrugen->protected[hist][type][tier - 1];
+ WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
+ }
+
+ if (clear) {
+ atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
+ atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
+ if (tier)
+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);
+ }
+ }
+}
+
+static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
+{
+ /*
+ * Return true if the PV has a limited number of refaults or a lower
+ * refaulted/total than the SP.
+ */
+ return pv->refaulted < MIN_LRU_BATCH ||
+ pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <=
+ (sp->refaulted + 1) * pv->total * pv->gain;
+}
+
+/******************************************************************************
+ * the aging
+ ******************************************************************************/
+
+/* promote pages accessed through page tables */
+static int folio_update_gen(struct folio *folio, int gen)
+{
+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
+
+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
+ VM_WARN_ON_ONCE(!rcu_read_lock_held());
+
+ do {
+ /* lru_gen_del_folio() has isolated this page? */
+ if (!(old_flags & LRU_GEN_MASK)) {
+ /* for shrink_folio_list() */
+ new_flags = old_flags | BIT(PG_referenced);
+ continue;
+ }
+
+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
+ new_flags |= (gen + 1UL) << LRU_GEN_PGOFF;
+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
+
+ return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+}
+
+/* protect pages accessed multiple times through file descriptors */
+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+{
+ int type = folio_is_file_lru(folio);
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+ unsigned long new_flags, old_flags = READ_ONCE(folio->flags);
+
+ VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
+
+ do {
+ new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ /* folio_update_gen() has promoted this page? */
+ if (new_gen >= 0 && new_gen != old_gen)
+ return new_gen;
+
+ new_gen = (old_gen + 1) % MAX_NR_GENS;
+
+ new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
+ new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
+ /* for folio_end_writeback() */
+ if (reclaiming)
+ new_flags |= BIT(PG_reclaim);
+ } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags));
+
+ lru_gen_update_size(lruvec, folio, old_gen, new_gen);
+
+ return new_gen;
+}
+
+static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
+ int old_gen, int new_gen)
+{
+ int type = folio_is_file_lru(folio);
+ int zone = folio_zonenum(folio);
+ int delta = folio_nr_pages(folio);
+
+ VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS);
+ VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS);
+
+ walk->batched++;
+
+ walk->nr_pages[old_gen][type][zone] -= delta;
+ walk->nr_pages[new_gen][type][zone] += delta;
+}
+
+static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk)
+{
+ int gen, type, zone;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ walk->batched = 0;
+
+ for_each_gen_type_zone(gen, type, zone) {
+ enum lru_list lru = type * LRU_INACTIVE_FILE;
+ int delta = walk->nr_pages[gen][type][zone];
+
+ if (!delta)
+ continue;
+
+ walk->nr_pages[gen][type][zone] = 0;
+ WRITE_ONCE(lrugen->nr_pages[gen][type][zone],
+ lrugen->nr_pages[gen][type][zone] + delta);
+
+ if (lru_gen_is_active(lruvec, gen))
+ lru += LRU_ACTIVE;
+ __update_lru_size(lruvec, lru, zone, delta);
+ }
+}
+
+static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args)
+{
+ struct address_space *mapping;
+ struct vm_area_struct *vma = args->vma;
+ struct lru_gen_mm_walk *walk = args->private;
+
+ if (!vma_is_accessible(vma))
+ return true;
+
+ if (is_vm_hugetlb_page(vma))
+ return true;
+
+ if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ))
+ return true;
+
+ if (vma == get_gate_vma(vma->vm_mm))
+ return true;
+
+ if (vma_is_anonymous(vma))
+ return !walk->can_swap;
+
+ if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping))
+ return true;
+
+ mapping = vma->vm_file->f_mapping;
+ if (mapping_unevictable(mapping))
+ return true;
+
+ if (shmem_mapping(mapping))
+ return !walk->can_swap;
+
+ /* to exclude special mappings like dax, etc. */
+ return !mapping->a_ops->read_folio;
+}
+
+/*
+ * Some userspace memory allocators map many single-page VMAs. Instead of
+ * returning back to the PGD table for each of such VMAs, finish an entire PMD
+ * table to reduce zigzags and improve cache performance.
+ */
+static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args,
+ unsigned long *vm_start, unsigned long *vm_end)
+{
+ unsigned long start = round_up(*vm_end, size);
+ unsigned long end = (start | ~mask) + 1;
+ VMA_ITERATOR(vmi, args->mm, start);
+
+ VM_WARN_ON_ONCE(mask & size);
+ VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask));
+
+ for_each_vma(vmi, args->vma) {
+ if (end && end <= args->vma->vm_start)
+ return false;
+
+ if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args))
+ continue;
+
+ *vm_start = max(start, args->vma->vm_start);
+ *vm_end = min(end - 1, args->vma->vm_end - 1) + 1;
+
+ return true;
+ }
+
+ return false;
+}
+
+static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr)
+{
+ unsigned long pfn = pte_pfn(pte);
+
+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
+
+ if (!pte_present(pte) || is_zero_pfn(pfn))
+ return -1;
+
+ if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte)))
+ return -1;
+
+ if (WARN_ON_ONCE(!pfn_valid(pfn)))
+ return -1;
+
+ return pfn;
+}
+
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
+static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr)
+{
+ unsigned long pfn = pmd_pfn(pmd);
+
+ VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end);
+
+ if (!pmd_present(pmd) || is_huge_zero_pmd(pmd))
+ return -1;
+
+ if (WARN_ON_ONCE(pmd_devmap(pmd)))
+ return -1;
+
+ if (WARN_ON_ONCE(!pfn_valid(pfn)))
+ return -1;
+
+ return pfn;
+}
+#endif
+
+static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg,
+ struct pglist_data *pgdat, bool can_swap)
+{
+ struct folio *folio;
+
+ /* try to avoid unnecessary memory loads */
+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
+ return NULL;
+
+ folio = pfn_folio(pfn);
+ if (folio_nid(folio) != pgdat->node_id)
+ return NULL;
+
+ if (folio_memcg_rcu(folio) != memcg)
+ return NULL;
+
+ /* file VMAs can contain anon pages from COW */
+ if (!folio_is_file_lru(folio) && !can_swap)
+ return NULL;
+
+ return folio;
+}
+
+static bool suitable_to_scan(int total, int young)
+{
+ int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8);
+
+ /* suitable if the average number of young PTEs per cacheline is >=1 */
+ return young * n >= total;
+}
+
+static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
+ struct mm_walk *args)
+{
+ int i;
+ pte_t *pte;
+ spinlock_t *ptl;
+ unsigned long addr;
+ int total = 0;
+ int young = 0;
+ struct lru_gen_mm_walk *walk = args->private;
+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
+
+ VM_WARN_ON_ONCE(pmd_leaf(*pmd));
+
+ ptl = pte_lockptr(args->mm, pmd);
+ if (!spin_trylock(ptl))
+ return false;
+
+ arch_enter_lazy_mmu_mode();
+
+ pte = pte_offset_map(pmd, start & PMD_MASK);
+restart:
+ for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) {
+ unsigned long pfn;
+ struct folio *folio;
+
+ total++;
+ walk->mm_stats[MM_LEAF_TOTAL]++;
+
+ pfn = get_pte_pfn(pte[i], args->vma, addr);
+ if (pfn == -1)
+ continue;
+
+ if (!pte_young(pte[i])) {
+ walk->mm_stats[MM_LEAF_OLD]++;
+ continue;
+ }
+
+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);
+ if (!folio)
+ continue;
+
+ if (!ptep_test_and_clear_young(args->vma, addr, pte + i))
+ VM_WARN_ON_ONCE(true);
+
+ young++;
+ walk->mm_stats[MM_LEAF_YOUNG]++;
+
+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) &&
+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
+ !folio_test_swapcache(folio)))
+ folio_mark_dirty(folio);
+
+ old_gen = folio_update_gen(folio, new_gen);
+ if (old_gen >= 0 && old_gen != new_gen)
+ update_batch_size(walk, folio, old_gen, new_gen);
+ }
+
+ if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
+ goto restart;
+
+ pte_unmap(pte);
+
+ arch_leave_lazy_mmu_mode();
+ spin_unlock(ptl);
+
+ return suitable_to_scan(total, young);
+}
+
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
+{
+ int i;
+ pmd_t *pmd;
+ spinlock_t *ptl;
+ struct lru_gen_mm_walk *walk = args->private;
+ struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
+ int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
+
+ VM_WARN_ON_ONCE(pud_leaf(*pud));
+
+ /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
+ if (*start == -1) {
+ *start = next;
+ return;
+ }
+
+ i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);
+ if (i && i <= MIN_LRU_BATCH) {
+ __set_bit(i - 1, bitmap);
+ return;
+ }
+
+ pmd = pmd_offset(pud, *start);
+
+ ptl = pmd_lockptr(args->mm, pmd);
+ if (!spin_trylock(ptl))
+ goto done;
+
+ arch_enter_lazy_mmu_mode();
+
+ do {
+ unsigned long pfn;
+ struct folio *folio;
+ unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;
+
+ pfn = get_pmd_pfn(pmd[i], vma, addr);
+ if (pfn == -1)
+ goto next;
+
+ if (!pmd_trans_huge(pmd[i])) {
+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) &&
+ get_cap(LRU_GEN_NONLEAF_YOUNG))
+ pmdp_test_and_clear_young(vma, addr, pmd + i);
+ goto next;
+ }
+
+ folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap);
+ if (!folio)
+ goto next;
+
+ if (!pmdp_test_and_clear_young(vma, addr, pmd + i))
+ goto next;
+
+ walk->mm_stats[MM_LEAF_YOUNG]++;
+
+ if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) &&
+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
+ !folio_test_swapcache(folio)))
+ folio_mark_dirty(folio);
+
+ old_gen = folio_update_gen(folio, new_gen);
+ if (old_gen >= 0 && old_gen != new_gen)
+ update_batch_size(walk, folio, old_gen, new_gen);
+next:
+ i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
+ } while (i <= MIN_LRU_BATCH);
+
+ arch_leave_lazy_mmu_mode();
+ spin_unlock(ptl);
+done:
+ *start = -1;
+ bitmap_zero(bitmap, MIN_LRU_BATCH);
+}
+#else
+static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
+ struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
+{
+}
+#endif
+
+static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
+ struct mm_walk *args)
+{
+ int i;
+ pmd_t *pmd;
+ unsigned long next;
+ unsigned long addr;
+ struct vm_area_struct *vma;
+ unsigned long pos = -1;
+ struct lru_gen_mm_walk *walk = args->private;
+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
+
+ VM_WARN_ON_ONCE(pud_leaf(*pud));
+
+ /*
+ * Finish an entire PMD in two passes: the first only reaches to PTE
+ * tables to avoid taking the PMD lock; the second, if necessary, takes
+ * the PMD lock to clear the accessed bit in PMD entries.
+ */
+ pmd = pmd_offset(pud, start & PUD_MASK);
+restart:
+ /* walk_pte_range() may call get_next_vma() */
+ vma = args->vma;
+ for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) {
+ pmd_t val = pmd_read_atomic(pmd + i);
+
+ /* for pmd_read_atomic() */
+ barrier();
+
+ next = pmd_addr_end(addr, end);
+
+ if (!pmd_present(val) || is_huge_zero_pmd(val)) {
+ walk->mm_stats[MM_LEAF_TOTAL]++;
+ continue;
+ }
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ if (pmd_trans_huge(val)) {
+ unsigned long pfn = pmd_pfn(val);
+ struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
+
+ walk->mm_stats[MM_LEAF_TOTAL]++;
+
+ if (!pmd_young(val)) {
+ walk->mm_stats[MM_LEAF_OLD]++;
+ continue;
+ }
+
+ /* try to avoid unnecessary memory loads */
+ if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
+ continue;
+
+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
+ continue;
+ }
+#endif
+ walk->mm_stats[MM_NONLEAF_TOTAL]++;
+
+#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG
+ if (get_cap(LRU_GEN_NONLEAF_YOUNG)) {
+ if (!pmd_young(val))
+ continue;
+
+ walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
+ }
+#endif
+ if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
+ continue;
+
+ walk->mm_stats[MM_NONLEAF_FOUND]++;
+
+ if (!walk_pte_range(&val, addr, next, args))
+ continue;
+
+ walk->mm_stats[MM_NONLEAF_ADDED]++;
+
+ /* carry over to the next generation */
+ update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
+ }
+
+ walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);
+
+ if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
+ goto restart;
+}
+
+static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
+ struct mm_walk *args)
+{
+ int i;
+ pud_t *pud;
+ unsigned long addr;
+ unsigned long next;
+ struct lru_gen_mm_walk *walk = args->private;
+
+ VM_WARN_ON_ONCE(p4d_leaf(*p4d));
+
+ pud = pud_offset(p4d, start & P4D_MASK);
+restart:
+ for (i = pud_index(start), addr = start; addr != end; i++, addr = next) {
+ pud_t val = READ_ONCE(pud[i]);
+
+ next = pud_addr_end(addr, end);
+
+ if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val)))
+ continue;
+
+ walk_pmd_range(&val, addr, next, args);
+
+ /* a racy check to curtail the waiting time */
+ if (wq_has_sleeper(&walk->lruvec->mm_state.wait))
+ return 1;
+
+ if (need_resched() || walk->batched >= MAX_LRU_BATCH) {
+ end = (addr | ~PUD_MASK) + 1;
+ goto done;
+ }
+ }
+
+ if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end))
+ goto restart;
+
+ end = round_up(end, P4D_SIZE);
+done:
+ if (!end || !args->vma)
+ return 1;
+
+ walk->next_addr = max(end, args->vma->vm_start);
+
+ return -EAGAIN;
+}
+
+static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk)
+{
+ static const struct mm_walk_ops mm_walk_ops = {
+ .test_walk = should_skip_vma,
+ .p4d_entry = walk_pud_range,
+ };
+
+ int err;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+
+ walk->next_addr = FIRST_USER_ADDRESS;
+
+ do {
+ err = -EBUSY;
+
+ /* folio_update_gen() requires stable folio_memcg() */
+ if (!mem_cgroup_trylock_pages(memcg))
+ break;
+
+ /* the caller might be holding the lock for write */
+ if (mmap_read_trylock(mm)) {
+ err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk);
+
+ mmap_read_unlock(mm);
+ }
+
+ mem_cgroup_unlock_pages();
+
+ if (walk->batched) {
+ spin_lock_irq(&lruvec->lru_lock);
+ reset_batch_size(lruvec, walk);
+ spin_unlock_irq(&lruvec->lru_lock);
+ }
+
+ cond_resched();
+ } while (err == -EAGAIN);
+}
+
+static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat)
+{
+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
+
+ if (pgdat && current_is_kswapd()) {
+ VM_WARN_ON_ONCE(walk);
+
+ walk = &pgdat->mm_walk;
+ } else if (!pgdat && !walk) {
+ VM_WARN_ON_ONCE(current_is_kswapd());
+
+ walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
+ }
+
+ current->reclaim_state->mm_walk = walk;
+
+ return walk;
+}
+
+static void clear_mm_walk(void)
+{
+ struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk;
+
+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages)));
+ VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats)));
+
+ current->reclaim_state->mm_walk = NULL;
+
+ if (!current_is_kswapd())
+ kfree(walk);
+}
+
+static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
+{
+ int zone;
+ int remaining = MAX_LRU_BATCH;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+
+ if (type == LRU_GEN_ANON && !can_swap)
+ goto done;
+
+ /* prevent cold/hot inversion if force_scan is true */
+ for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+ struct list_head *head = &lrugen->lists[old_gen][type][zone];
+
+ while (!list_empty(head)) {
+ struct folio *folio = lru_to_folio(head);
+
+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
+
+ new_gen = folio_inc_gen(lruvec, folio, false);
+ list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]);
+
+ if (!--remaining)
+ return false;
+ }
+ }
+done:
+ reset_ctrl_pos(lruvec, type, true);
+ WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
+
+ return true;
+}
+
+static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
+{
+ int gen, type, zone;
+ bool success = false;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ DEFINE_MIN_SEQ(lruvec);
+
+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
+
+ /* find the oldest populated generation */
+ for (type = !can_swap; type < ANON_AND_FILE; type++) {
+ while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
+ gen = lru_gen_from_seq(min_seq[type]);
+
+ for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+ if (!list_empty(&lrugen->lists[gen][type][zone]))
+ goto next;
+ }
+
+ min_seq[type]++;
+ }
+next:
+ ;
+ }
+
+ /* see the comment on lru_gen_struct */
+ if (can_swap) {
+ min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
+ min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
+ }
+
+ for (type = !can_swap; type < ANON_AND_FILE; type++) {
+ if (min_seq[type] == lrugen->min_seq[type])
+ continue;
+
+ reset_ctrl_pos(lruvec, type, true);
+ WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
+ success = true;
+ }
+
+ return success;
+}
+
+static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan)
+{
+ int prev, next;
+ int type, zone;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ spin_lock_irq(&lruvec->lru_lock);
+
+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
+
+ for (type = ANON_AND_FILE - 1; type >= 0; type--) {
+ if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
+ continue;
+
+ VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap));
+
+ while (!inc_min_seq(lruvec, type, can_swap)) {
+ spin_unlock_irq(&lruvec->lru_lock);
+ cond_resched();
+ spin_lock_irq(&lruvec->lru_lock);
+ }
+ }
+
+ /*
+ * Update the active/inactive LRU sizes for compatibility. Both sides of
+ * the current max_seq need to be covered, since max_seq+1 can overlap
+ * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do
+ * overlap, cold/hot inversion happens.
+ */
+ prev = lru_gen_from_seq(lrugen->max_seq - 1);
+ next = lru_gen_from_seq(lrugen->max_seq + 1);
+
+ for (type = 0; type < ANON_AND_FILE; type++) {
+ for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+ enum lru_list lru = type * LRU_INACTIVE_FILE;
+ long delta = lrugen->nr_pages[prev][type][zone] -
+ lrugen->nr_pages[next][type][zone];
+
+ if (!delta)
+ continue;
+
+ __update_lru_size(lruvec, lru, zone, delta);
+ __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
+ }
+ }
+
+ for (type = 0; type < ANON_AND_FILE; type++)
+ reset_ctrl_pos(lruvec, type, false);
+
+ WRITE_ONCE(lrugen->timestamps[next], jiffies);
+ /* make sure preceding modifications appear */
+ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
+
+ spin_unlock_irq(&lruvec->lru_lock);
+}
+
+static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+ struct scan_control *sc, bool can_swap, bool force_scan)
+{
+ bool success;
+ struct lru_gen_mm_walk *walk;
+ struct mm_struct *mm = NULL;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
+
+ /* see the comment in iterate_mm_list() */
+ if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) {
+ success = false;
+ goto done;
+ }
+
+ /*
+ * If the hardware doesn't automatically set the accessed bit, fallback
+ * to lru_gen_look_around(), which only clears the accessed bit in a
+ * handful of PTEs. Spreading the work out over a period of time usually
+ * is less efficient, but it avoids bursty page faults.
+ */
+ if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) {
+ success = iterate_mm_list_nowalk(lruvec, max_seq);
+ goto done;
+ }
+
+ walk = set_mm_walk(NULL);
+ if (!walk) {
+ success = iterate_mm_list_nowalk(lruvec, max_seq);
+ goto done;
+ }
+
+ walk->lruvec = lruvec;
+ walk->max_seq = max_seq;
+ walk->can_swap = can_swap;
+ walk->force_scan = force_scan;
+
+ do {
+ success = iterate_mm_list(lruvec, walk, &mm);
+ if (mm)
+ walk_mm(lruvec, mm, walk);
+
+ cond_resched();
+ } while (mm);
+done:
+ if (!success) {
+ if (sc->priority <= DEF_PRIORITY - 2)
+ wait_event_killable(lruvec->mm_state.wait,
+ max_seq < READ_ONCE(lrugen->max_seq));
+
+ return max_seq < READ_ONCE(lrugen->max_seq);
+ }
+
+ VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq));
+
+ inc_max_seq(lruvec, can_swap, force_scan);
+ /* either this sees any waiters or they will see updated max_seq */
+ if (wq_has_sleeper(&lruvec->mm_state.wait))
+ wake_up_all(&lruvec->mm_state.wait);
+
+ return true;
+}
+
+static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq,
+ struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
+{
+ int gen, type, zone;
+ unsigned long old = 0;
+ unsigned long young = 0;
+ unsigned long total = 0;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+
+ for (type = !can_swap; type < ANON_AND_FILE; type++) {
+ unsigned long seq;
+
+ for (seq = min_seq[type]; seq <= max_seq; seq++) {
+ unsigned long size = 0;
+
+ gen = lru_gen_from_seq(seq);
+
+ for (zone = 0; zone < MAX_NR_ZONES; zone++)
+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+
+ total += size;
+ if (seq == max_seq)
+ young += size;
+ else if (seq + MIN_NR_GENS == max_seq)
+ old += size;
+ }
+ }
+
+ /* try to scrape all its memory if this memcg was deleted */
+ *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
+
+ /*
+ * The aging tries to be lazy to reduce the overhead, while the eviction
+ * stalls when the number of generations reaches MIN_NR_GENS. Hence, the
+ * ideal number of generations is MIN_NR_GENS+1.
+ */
+ if (min_seq[!can_swap] + MIN_NR_GENS > max_seq)
+ return true;
+ if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
+ return false;
+
+ /*
+ * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
+ * of the total number of pages for each generation. A reasonable range
+ * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
+ * aging cares about the upper bound of hot pages, while the eviction
+ * cares about the lower bound of cold pages.
+ */
+ if (young * MIN_NR_GENS > total)
+ return true;
+ if (old * (MIN_NR_GENS + 2) < total)
+ return true;
+
+ return false;
+}
+
+static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl)
+{
+ bool need_aging;
+ unsigned long nr_to_scan;
+ int swappiness = get_swappiness(lruvec, sc);
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ DEFINE_MAX_SEQ(lruvec);
+ DEFINE_MIN_SEQ(lruvec);
+
+ VM_WARN_ON_ONCE(sc->memcg_low_reclaim);
+
+ mem_cgroup_calculate_protection(NULL, memcg);
+
+ if (mem_cgroup_below_min(memcg))
+ return false;
+
+ need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan);
+
+ if (min_ttl) {
+ int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]);
+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
+
+ if (time_is_after_jiffies(birth + min_ttl))
+ return false;
+
+ /* the size is likely too small to be helpful */
+ if (!nr_to_scan && sc->priority != DEF_PRIORITY)
+ return false;
+ }
+
+ if (need_aging)
+ try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false);
+
+ return true;
+}
+
+/* to protect the working set of the last N jiffies */
+static unsigned long lru_gen_min_ttl __read_mostly;
+
+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+{
+ struct mem_cgroup *memcg;
+ bool success = false;
+ unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl);
+
+ VM_WARN_ON_ONCE(!current_is_kswapd());
+
+ sc->last_reclaimed = sc->nr_reclaimed;
+
+ /*
+ * To reduce the chance of going into the aging path, which can be
+ * costly, optimistically skip it if the flag below was cleared in the
+ * eviction path. This improves the overall performance when multiple
+ * memcgs are available.
+ */
+ if (!sc->memcgs_need_aging) {
+ sc->memcgs_need_aging = true;
+ return;
+ }
+
+ set_mm_walk(pgdat);
+
+ memcg = mem_cgroup_iter(NULL, NULL, NULL);
+ do {
+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
+
+ if (age_lruvec(lruvec, sc, min_ttl))
+ success = true;
+
+ cond_resched();
+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
+
+ clear_mm_walk();
+
+ /* check the order to exclude compaction-induced reclaim */
+ if (success || !min_ttl || sc->order)
+ return;
+
+ /*
+ * The main goal is to OOM kill if every generation from all memcgs is
+ * younger than min_ttl. However, another possibility is all memcgs are
+ * either below min or empty.
+ */
+ if (mutex_trylock(&oom_lock)) {
+ struct oom_control oc = {
+ .gfp_mask = sc->gfp_mask,
+ };
+
+ out_of_memory(&oc);
+
+ mutex_unlock(&oom_lock);
+ }
+}
+
+/*
+ * This function exploits spatial locality when shrink_folio_list() walks the
+ * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
+ * the scan was done cacheline efficiently, it adds the PMD entry pointing to
+ * the PTE table to the Bloom filter. This forms a feedback loop between the
+ * eviction and the aging.
+ */
+void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
+{
+ int i;
+ pte_t *pte;
+ unsigned long start;
+ unsigned long end;
+ unsigned long addr;
+ struct lru_gen_mm_walk *walk;
+ int young = 0;
+ unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
+ struct folio *folio = pfn_folio(pvmw->pfn);
+ struct mem_cgroup *memcg = folio_memcg(folio);
+ struct pglist_data *pgdat = folio_pgdat(folio);
+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
+ DEFINE_MAX_SEQ(lruvec);
+ int old_gen, new_gen = lru_gen_from_seq(max_seq);
+
+ lockdep_assert_held(pvmw->ptl);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
+
+ if (spin_is_contended(pvmw->ptl))
+ return;
+
+ /* avoid taking the LRU lock under the PTL when possible */
+ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL;
+
+ start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start);
+ end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1;
+
+ if (end - start > MIN_LRU_BATCH * PAGE_SIZE) {
+ if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2)
+ end = start + MIN_LRU_BATCH * PAGE_SIZE;
+ else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2)
+ start = end - MIN_LRU_BATCH * PAGE_SIZE;
+ else {
+ start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2;
+ end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2;
+ }
+ }
+
+ pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE;
+
+ rcu_read_lock();
+ arch_enter_lazy_mmu_mode();
+
+ for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) {
+ unsigned long pfn;
+
+ pfn = get_pte_pfn(pte[i], pvmw->vma, addr);
+ if (pfn == -1)
+ continue;
+
+ if (!pte_young(pte[i]))
+ continue;
+
+ folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap);
+ if (!folio)
+ continue;
+
+ if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i))
+ VM_WARN_ON_ONCE(true);
+
+ young++;
+
+ if (pte_dirty(pte[i]) && !folio_test_dirty(folio) &&
+ !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
+ !folio_test_swapcache(folio)))
+ folio_mark_dirty(folio);
+
+ old_gen = folio_lru_gen(folio);
+ if (old_gen < 0)
+ folio_set_referenced(folio);
+ else if (old_gen != new_gen)
+ __set_bit(i, bitmap);
+ }
+
+ arch_leave_lazy_mmu_mode();
+ rcu_read_unlock();
+
+ /* feedback from rmap walkers to page table walkers */
+ if (suitable_to_scan(i, young))
+ update_bloom_filter(lruvec, max_seq, pvmw->pmd);
+
+ if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) {
+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) {
+ folio = pfn_folio(pte_pfn(pte[i]));
+ folio_activate(folio);
+ }
+ return;
+ }
+
+ /* folio_update_gen() requires stable folio_memcg() */
+ if (!mem_cgroup_trylock_pages(memcg))
+ return;
+
+ if (!walk) {
+ spin_lock_irq(&lruvec->lru_lock);
+ new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq);
+ }
+
+ for_each_set_bit(i, bitmap, MIN_LRU_BATCH) {
+ folio = pfn_folio(pte_pfn(pte[i]));
+ if (folio_memcg_rcu(folio) != memcg)
+ continue;
+
+ old_gen = folio_update_gen(folio, new_gen);
+ if (old_gen < 0 || old_gen == new_gen)
+ continue;
+
+ if (walk)
+ update_batch_size(walk, folio, old_gen, new_gen);
+ else
+ lru_gen_update_size(lruvec, folio, old_gen, new_gen);
+ }
+
+ if (!walk)
+ spin_unlock_irq(&lruvec->lru_lock);
+
+ mem_cgroup_unlock_pages();
+}
+
+/******************************************************************************
+ * the eviction
+ ******************************************************************************/
+
+static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
+{
+ bool success;
+ int gen = folio_lru_gen(folio);
+ int type = folio_is_file_lru(folio);
+ int zone = folio_zonenum(folio);
+ int delta = folio_nr_pages(folio);
+ int refs = folio_lru_refs(folio);
+ int tier = lru_tier_from_refs(refs);
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
+
+ /* unevictable */
+ if (!folio_evictable(folio)) {
+ success = lru_gen_del_folio(lruvec, folio, true);
+ VM_WARN_ON_ONCE_FOLIO(!success, folio);
+ folio_set_unevictable(folio);
+ lruvec_add_folio(lruvec, folio);
+ __count_vm_events(UNEVICTABLE_PGCULLED, delta);
+ return true;
+ }
+
+ /* dirty lazyfree */
+ if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) {
+ success = lru_gen_del_folio(lruvec, folio, true);
+ VM_WARN_ON_ONCE_FOLIO(!success, folio);
+ folio_set_swapbacked(folio);
+ lruvec_add_folio_tail(lruvec, folio);
+ return true;
+ }
+
+ /* promoted */
+ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]);
+ return true;
+ }
+
+ /* protected */
+ if (tier > tier_idx) {
+ int hist = lru_hist_from_seq(lrugen->min_seq[type]);
+
+ gen = folio_inc_gen(lruvec, folio, false);
+ list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]);
+
+ WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
+ lrugen->protected[hist][type][tier - 1] + delta);
+ __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
+ return true;
+ }
+
+ /* waiting for writeback */
+ if (folio_test_locked(folio) || folio_test_writeback(folio) ||
+ (type == LRU_GEN_FILE && folio_test_dirty(folio))) {
+ gen = folio_inc_gen(lruvec, folio, true);
+ list_move(&folio->lru, &lrugen->lists[gen][type][zone]);
+ return true;
+ }
+
+ return false;
+}
+
+static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc)
+{
+ bool success;
+
+ /* unmapping inhibited */
+ if (!sc->may_unmap && folio_mapped(folio))
+ return false;
+
+ /* swapping inhibited */
+ if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) &&
+ (folio_test_dirty(folio) ||
+ (folio_test_anon(folio) && !folio_test_swapcache(folio))))
+ return false;
+
+ /* raced with release_pages() */
+ if (!folio_try_get(folio))
+ return false;
+
+ /* raced with another isolation */
+ if (!folio_test_clear_lru(folio)) {
+ folio_put(folio);
+ return false;
+ }
+
+ /* see the comment on MAX_NR_TIERS */
+ if (!folio_test_referenced(folio))
+ set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);
+
+ /* for shrink_folio_list() */
+ folio_clear_reclaim(folio);
+ folio_clear_referenced(folio);
+
+ success = lru_gen_del_folio(lruvec, folio, true);
+ VM_WARN_ON_ONCE_FOLIO(!success, folio);
+
+ return true;
+}
+
+static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
+ int type, int tier, struct list_head *list)
+{
+ int gen, zone;
+ enum vm_event_item item;
+ int sorted = 0;
+ int scanned = 0;
+ int isolated = 0;
+ int remaining = MAX_LRU_BATCH;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+
+ VM_WARN_ON_ONCE(!list_empty(list));
+
+ if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
+ return 0;
+
+ gen = lru_gen_from_seq(lrugen->min_seq[type]);
+
+ for (zone = sc->reclaim_idx; zone >= 0; zone--) {
+ LIST_HEAD(moved);
+ int skipped = 0;
+ struct list_head *head = &lrugen->lists[gen][type][zone];
+
+ while (!list_empty(head)) {
+ struct folio *folio = lru_to_folio(head);
+ int delta = folio_nr_pages(folio);
+
+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
+
+ scanned += delta;
+
+ if (sort_folio(lruvec, folio, tier))
+ sorted += delta;
+ else if (isolate_folio(lruvec, folio, sc)) {
+ list_add(&folio->lru, list);
+ isolated += delta;
+ } else {
+ list_move(&folio->lru, &moved);
+ skipped += delta;
+ }
+
+ if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH)
+ break;
+ }
+
+ if (skipped) {
+ list_splice(&moved, head);
+ __count_zid_vm_events(PGSCAN_SKIP, zone, skipped);
+ }
+
+ if (!remaining || isolated >= MIN_LRU_BATCH)
+ break;
+ }
+
+ item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
+ if (!cgroup_reclaim(sc)) {
+ __count_vm_events(item, isolated);
+ __count_vm_events(PGREFILL, sorted);
+ }
+ __count_memcg_events(memcg, item, isolated);
+ __count_memcg_events(memcg, PGREFILL, sorted);
+ __count_vm_events(PGSCAN_ANON + type, isolated);
+
+ /*
+ * There might not be eligible pages due to reclaim_idx, may_unmap and
+ * may_writepage. Check the remaining to prevent livelock if it's not
+ * making progress.
+ */
+ return isolated || !remaining ? scanned : 0;
+}
+
+static int get_tier_idx(struct lruvec *lruvec, int type)
+{
+ int tier;
+ struct ctrl_pos sp, pv;
+
+ /*
+ * To leave a margin for fluctuations, use a larger gain factor (1:2).
+ * This value is chosen because any other tier would have at least twice
+ * as many refaults as the first tier.
+ */
+ read_ctrl_pos(lruvec, type, 0, 1, &sp);
+ for (tier = 1; tier < MAX_NR_TIERS; tier++) {
+ read_ctrl_pos(lruvec, type, tier, 2, &pv);
+ if (!positive_ctrl_err(&sp, &pv))
+ break;
+ }
+
+ return tier - 1;
+}
+
+static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx)
+{
+ int type, tier;
+ struct ctrl_pos sp, pv;
+ int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness };
+
+ /*
+ * Compare the first tier of anon with that of file to determine which
+ * type to scan. Also need to compare other tiers of the selected type
+ * with the first tier of the other type to determine the last tier (of
+ * the selected type) to evict.
+ */
+ read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp);
+ read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv);
+ type = positive_ctrl_err(&sp, &pv);
+
+ read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp);
+ for (tier = 1; tier < MAX_NR_TIERS; tier++) {
+ read_ctrl_pos(lruvec, type, tier, gain[type], &pv);
+ if (!positive_ctrl_err(&sp, &pv))
+ break;
+ }
+
+ *tier_idx = tier - 1;
+
+ return type;
+}
+
+static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness,
+ int *type_scanned, struct list_head *list)
+{
+ int i;
+ int type;
+ int scanned;
+ int tier = -1;
+ DEFINE_MIN_SEQ(lruvec);
+
+ /*
+ * Try to make the obvious choice first. When anon and file are both
+ * available from the same generation, interpret swappiness 1 as file
+ * first and 200 as anon first.
+ */
+ if (!swappiness)
+ type = LRU_GEN_FILE;
+ else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE])
+ type = LRU_GEN_ANON;
+ else if (swappiness == 1)
+ type = LRU_GEN_FILE;
+ else if (swappiness == 200)
+ type = LRU_GEN_ANON;
+ else
+ type = get_type_to_scan(lruvec, swappiness, &tier);
+
+ for (i = !swappiness; i < ANON_AND_FILE; i++) {
+ if (tier < 0)
+ tier = get_tier_idx(lruvec, type);
+
+ scanned = scan_folios(lruvec, sc, type, tier, list);
+ if (scanned)
+ break;
+
+ type = !type;
+ tier = -1;
+ }
+
+ *type_scanned = type;
+
+ return scanned;
+}
+
+static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness,
+ bool *need_swapping)
+{
+ int type;
+ int scanned;
+ int reclaimed;
+ LIST_HEAD(list);
+ struct folio *folio;
+ enum vm_event_item item;
+ struct reclaim_stat stat;
+ struct lru_gen_mm_walk *walk;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+
+ spin_lock_irq(&lruvec->lru_lock);
+
+ scanned = isolate_folios(lruvec, sc, swappiness, &type, &list);
+
+ scanned += try_to_inc_min_seq(lruvec, swappiness);
+
+ if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS)
+ scanned = 0;
+
+ spin_unlock_irq(&lruvec->lru_lock);
+
+ if (list_empty(&list))
+ return scanned;
+
+ reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false);
+
+ list_for_each_entry(folio, &list, lru) {
+ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */
+ if (folio_test_workingset(folio))
+ folio_set_referenced(folio);
+
+ /* don't add rejected pages to the oldest generation */
+ if (folio_test_reclaim(folio) &&
+ (folio_test_dirty(folio) || folio_test_writeback(folio)))
+ folio_clear_active(folio);
+ else
+ folio_set_active(folio);
+ }
+
+ spin_lock_irq(&lruvec->lru_lock);
+
+ move_folios_to_lru(lruvec, &list);
+
+ walk = current->reclaim_state->mm_walk;
+ if (walk && walk->batched)
+ reset_batch_size(lruvec, walk);
+
+ item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
+ if (!cgroup_reclaim(sc))
+ __count_vm_events(item, reclaimed);
+ __count_memcg_events(memcg, item, reclaimed);
+ __count_vm_events(PGSTEAL_ANON + type, reclaimed);
+
+ spin_unlock_irq(&lruvec->lru_lock);
+
+ mem_cgroup_uncharge_list(&list);
+ free_unref_page_list(&list);
+
+ sc->nr_reclaimed += reclaimed;
+
+ if (need_swapping && type == LRU_GEN_ANON)
+ *need_swapping = true;
+
+ return scanned;
+}
+
+/*
+ * For future optimizations:
+ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
+ * reclaim.
+ */
+static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
+ bool can_swap, bool *need_aging)
+{
+ unsigned long nr_to_scan;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ DEFINE_MAX_SEQ(lruvec);
+ DEFINE_MIN_SEQ(lruvec);
+
+ if (mem_cgroup_below_min(memcg) ||
+ (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim))
+ return 0;
+
+ *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan);
+ if (!*need_aging)
+ return nr_to_scan;
+
+ /* skip the aging path at the default priority */
+ if (sc->priority == DEF_PRIORITY)
+ goto done;
+
+ /* leave the work to lru_gen_age_node() */
+ if (current_is_kswapd())
+ return 0;
+
+ if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false))
+ return nr_to_scan;
+done:
+ return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0;
+}
+
+static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq,
+ struct scan_control *sc, bool need_swapping)
+{
+ int i;
+ DEFINE_MAX_SEQ(lruvec);
+
+ if (!current_is_kswapd()) {
+ /* age each memcg at most once to ensure fairness */
+ if (max_seq - seq > 1)
+ return true;
+
+ /* over-swapping can increase allocation latency */
+ if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping)
+ return true;
+
+ /* give this thread a chance to exit and free its memory */
+ if (fatal_signal_pending(current)) {
+ sc->nr_reclaimed += MIN_LRU_BATCH;
+ return true;
+ }
+
+ if (cgroup_reclaim(sc))
+ return false;
+ } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim)
+ return false;
+
+ /* keep scanning at low priorities to ensure fairness */
+ if (sc->priority > DEF_PRIORITY - 2)
+ return false;
+
+ /*
+ * A minimum amount of work was done under global memory pressure. For
+ * kswapd, it may be overshooting. For direct reclaim, the allocation
+ * may succeed if all suitable zones are somewhat safe. In either case,
+ * it's better to stop now, and restart later if necessary.
+ */
+ for (i = 0; i <= sc->reclaim_idx; i++) {
+ unsigned long wmark;
+ struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i;
+
+ if (!managed_zone(zone))
+ continue;
+
+ wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone);
+ if (wmark > zone_page_state(zone, NR_FREE_PAGES))
+ return false;
+ }
+
+ sc->nr_reclaimed += MIN_LRU_BATCH;
+
+ return true;
+}
+
+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+{
+ struct blk_plug plug;
+ bool need_aging = false;
+ bool need_swapping = false;
+ unsigned long scanned = 0;
+ unsigned long reclaimed = sc->nr_reclaimed;
+ DEFINE_MAX_SEQ(lruvec);
+
+ lru_add_drain();
+
+ blk_start_plug(&plug);
+
+ set_mm_walk(lruvec_pgdat(lruvec));
+
+ while (true) {
+ int delta;
+ int swappiness;
+ unsigned long nr_to_scan;
+
+ if (sc->may_swap)
+ swappiness = get_swappiness(lruvec, sc);
+ else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc))
+ swappiness = 1;
+ else
+ swappiness = 0;
+
+ nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging);
+ if (!nr_to_scan)
+ goto done;
+
+ delta = evict_folios(lruvec, sc, swappiness, &need_swapping);
+ if (!delta)
+ goto done;
+
+ scanned += delta;
+ if (scanned >= nr_to_scan)
+ break;
+
+ if (should_abort_scan(lruvec, max_seq, sc, need_swapping))
+ break;
+
+ cond_resched();
+ }
+
+ /* see the comment in lru_gen_age_node() */
+ if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging)
+ sc->memcgs_need_aging = false;
+done:
+ clear_mm_walk();
+
+ blk_finish_plug(&plug);
+}
+
+/******************************************************************************
+ * state change
+ ******************************************************************************/
+
+static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
+{
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ if (lrugen->enabled) {
+ enum lru_list lru;
+
+ for_each_evictable_lru(lru) {
+ if (!list_empty(&lruvec->lists[lru]))
+ return false;
+ }
+ } else {
+ int gen, type, zone;
+
+ for_each_gen_type_zone(gen, type, zone) {
+ if (!list_empty(&lrugen->lists[gen][type][zone]))
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool fill_evictable(struct lruvec *lruvec)
+{
+ enum lru_list lru;
+ int remaining = MAX_LRU_BATCH;
+
+ for_each_evictable_lru(lru) {
+ int type = is_file_lru(lru);
+ bool active = is_active_lru(lru);
+ struct list_head *head = &lruvec->lists[lru];
+
+ while (!list_empty(head)) {
+ bool success;
+ struct folio *folio = lru_to_folio(head);
+
+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio);
+
+ lruvec_del_folio(lruvec, folio);
+ success = lru_gen_add_folio(lruvec, folio, false);
+ VM_WARN_ON_ONCE(!success);
+
+ if (!--remaining)
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool drain_evictable(struct lruvec *lruvec)
+{
+ int gen, type, zone;
+ int remaining = MAX_LRU_BATCH;
+
+ for_each_gen_type_zone(gen, type, zone) {
+ struct list_head *head = &lruvec->lrugen.lists[gen][type][zone];
+
+ while (!list_empty(head)) {
+ bool success;
+ struct folio *folio = lru_to_folio(head);
+
+ VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
+ VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
+
+ success = lru_gen_del_folio(lruvec, folio, false);
+ VM_WARN_ON_ONCE(!success);
+ lruvec_add_folio(lruvec, folio);
+
+ if (!--remaining)
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static void lru_gen_change_state(bool enabled)
+{
+ static DEFINE_MUTEX(state_mutex);
+
+ struct mem_cgroup *memcg;
+
+ cgroup_lock();
+ cpus_read_lock();
+ get_online_mems();
+ mutex_lock(&state_mutex);
+
+ if (enabled == lru_gen_enabled())
+ goto unlock;
+
+ if (enabled)
+ static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
+ else
+ static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]);
+
+ memcg = mem_cgroup_iter(NULL, NULL, NULL);
+ do {
+ int nid;
+
+ for_each_node(nid) {
+ struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+ if (!lruvec)
+ continue;
+
+ spin_lock_irq(&lruvec->lru_lock);
+
+ VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
+ VM_WARN_ON_ONCE(!state_is_valid(lruvec));
+
+ lruvec->lrugen.enabled = enabled;
+
+ while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) {
+ spin_unlock_irq(&lruvec->lru_lock);
+ cond_resched();
+ spin_lock_irq(&lruvec->lru_lock);
+ }
+
+ spin_unlock_irq(&lruvec->lru_lock);
+ }
+
+ cond_resched();
+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
+unlock:
+ mutex_unlock(&state_mutex);
+ put_online_mems();
+ cpus_read_unlock();
+ cgroup_unlock();
+}
+
+/******************************************************************************
+ * sysfs interface
+ ******************************************************************************/
+
+static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+ return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl)));
+}
+
+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
+static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t len)
+{
+ unsigned int msecs;
+
+ if (kstrtouint(buf, 0, &msecs))
+ return -EINVAL;
+
+ WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs));
+
+ return len;
+}
+
+static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR(
+ min_ttl_ms, 0644, show_min_ttl, store_min_ttl
+);
+
+static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+ unsigned int caps = 0;
+
+ if (get_cap(LRU_GEN_CORE))
+ caps |= BIT(LRU_GEN_CORE);
+
+ if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))
+ caps |= BIT(LRU_GEN_MM_WALK);
+
+ if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG))
+ caps |= BIT(LRU_GEN_NONLEAF_YOUNG);
+
+ return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps);
+}
+
+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
+static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t len)
+{
+ int i;
+ unsigned int caps;
+
+ if (tolower(*buf) == 'n')
+ caps = 0;
+ else if (tolower(*buf) == 'y')
+ caps = -1;
+ else if (kstrtouint(buf, 0, &caps))
+ return -EINVAL;
+
+ for (i = 0; i < NR_LRU_GEN_CAPS; i++) {
+ bool enabled = caps & BIT(i);
+
+ if (i == LRU_GEN_CORE)
+ lru_gen_change_state(enabled);
+ else if (enabled)
+ static_branch_enable(&lru_gen_caps[i]);
+ else
+ static_branch_disable(&lru_gen_caps[i]);
+ }
+
+ return len;
+}
+
+static struct kobj_attribute lru_gen_enabled_attr = __ATTR(
+ enabled, 0644, show_enabled, store_enabled
+);
+
+static struct attribute *lru_gen_attrs[] = {
+ &lru_gen_min_ttl_attr.attr,
+ &lru_gen_enabled_attr.attr,
+ NULL
+};
+
+static struct attribute_group lru_gen_attr_group = {
+ .name = "lru_gen",
+ .attrs = lru_gen_attrs,
+};
+
+/******************************************************************************
+ * debugfs interface
+ ******************************************************************************/
+
+static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos)
+{
+ struct mem_cgroup *memcg;
+ loff_t nr_to_skip = *pos;
+
+ m->private = kvmalloc(PATH_MAX, GFP_KERNEL);
+ if (!m->private)
+ return ERR_PTR(-ENOMEM);
+
+ memcg = mem_cgroup_iter(NULL, NULL, NULL);
+ do {
+ int nid;
+
+ for_each_node_state(nid, N_MEMORY) {
+ if (!nr_to_skip--)
+ return get_lruvec(memcg, nid);
+ }
+ } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
+
+ return NULL;
+}
+
+static void lru_gen_seq_stop(struct seq_file *m, void *v)
+{
+ if (!IS_ERR_OR_NULL(v))
+ mem_cgroup_iter_break(NULL, lruvec_memcg(v));
+
+ kvfree(m->private);
+ m->private = NULL;
+}
+
+static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos)
+{
+ int nid = lruvec_pgdat(v)->node_id;
+ struct mem_cgroup *memcg = lruvec_memcg(v);
+
+ ++*pos;
+
+ nid = next_memory_node(nid);
+ if (nid == MAX_NUMNODES) {
+ memcg = mem_cgroup_iter(NULL, memcg, NULL);
+ if (!memcg)
+ return NULL;
+
+ nid = first_memory_node;
+ }
+
+ return get_lruvec(memcg, nid);
+}
+
+static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
+ unsigned long max_seq, unsigned long *min_seq,
+ unsigned long seq)
+{
+ int i;
+ int type, tier;
+ int hist = lru_hist_from_seq(seq);
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ for (tier = 0; tier < MAX_NR_TIERS; tier++) {
+ seq_printf(m, " %10d", tier);
+ for (type = 0; type < ANON_AND_FILE; type++) {
+ const char *s = " ";
+ unsigned long n[3] = {};
+
+ if (seq == max_seq) {
+ s = "RT ";
+ n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]);
+ n[1] = READ_ONCE(lrugen->avg_total[type][tier]);
+ } else if (seq == min_seq[type] || NR_HIST_GENS > 1) {
+ s = "rep";
+ n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]);
+ n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]);
+ if (tier)
+ n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]);
+ }
+
+ for (i = 0; i < 3; i++)
+ seq_printf(m, " %10lu%c", n[i], s[i]);
+ }
+ seq_putc(m, '\n');
+ }
+
+ seq_puts(m, " ");
+ for (i = 0; i < NR_MM_STATS; i++) {
+ const char *s = " ";
+ unsigned long n = 0;
+
+ if (seq == max_seq && NR_HIST_GENS == 1) {
+ s = "LOYNFA";
+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]);
+ } else if (seq != max_seq && NR_HIST_GENS > 1) {
+ s = "loynfa";
+ n = READ_ONCE(lruvec->mm_state.stats[hist][i]);
+ }
+
+ seq_printf(m, " %10lu%c", n, s[i]);
+ }
+ seq_putc(m, '\n');
+}
+
+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
+static int lru_gen_seq_show(struct seq_file *m, void *v)
+{
+ unsigned long seq;
+ bool full = !debugfs_real_fops(m->file)->write;
+ struct lruvec *lruvec = v;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+ int nid = lruvec_pgdat(lruvec)->node_id;
+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+ DEFINE_MAX_SEQ(lruvec);
+ DEFINE_MIN_SEQ(lruvec);
+
+ if (nid == first_memory_node) {
+ const char *path = memcg ? m->private : "";
+
+#ifdef CONFIG_MEMCG
+ if (memcg)
+ cgroup_path(memcg->css.cgroup, m->private, PATH_MAX);
+#endif
+ seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path);
+ }
+
+ seq_printf(m, " node %5d\n", nid);
+
+ if (!full)
+ seq = min_seq[LRU_GEN_ANON];
+ else if (max_seq >= MAX_NR_GENS)
+ seq = max_seq - MAX_NR_GENS + 1;
+ else
+ seq = 0;
+
+ for (; seq <= max_seq; seq++) {
+ int type, zone;
+ int gen = lru_gen_from_seq(seq);
+ unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
+
+ seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth));
+
+ for (type = 0; type < ANON_AND_FILE; type++) {
+ unsigned long size = 0;
+ char mark = full && seq < min_seq[type] ? 'x' : ' ';
+
+ for (zone = 0; zone < MAX_NR_ZONES; zone++)
+ size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
+
+ seq_printf(m, " %10lu%c", size, mark);
+ }
+
+ seq_putc(m, '\n');
+
+ if (full)
+ lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq);
+ }
+
+ return 0;
+}
+
+static const struct seq_operations lru_gen_seq_ops = {
+ .start = lru_gen_seq_start,
+ .stop = lru_gen_seq_stop,
+ .next = lru_gen_seq_next,
+ .show = lru_gen_seq_show,
+};
+
+static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
+ bool can_swap, bool force_scan)
+{
+ DEFINE_MAX_SEQ(lruvec);
+ DEFINE_MIN_SEQ(lruvec);
+
+ if (seq < max_seq)
+ return 0;
+
+ if (seq > max_seq)
+ return -EINVAL;
+
+ if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq)
+ return -ERANGE;
+
+ try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan);
+
+ return 0;
+}
+
+static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
+ int swappiness, unsigned long nr_to_reclaim)
+{
+ DEFINE_MAX_SEQ(lruvec);
+
+ if (seq + MIN_NR_GENS > max_seq)
+ return -EINVAL;
+
+ sc->nr_reclaimed = 0;
+
+ while (!signal_pending(current)) {
+ DEFINE_MIN_SEQ(lruvec);
+
+ if (seq < min_seq[!swappiness])
+ return 0;
+
+ if (sc->nr_reclaimed >= nr_to_reclaim)
+ return 0;
+
+ if (!evict_folios(lruvec, sc, swappiness, NULL))
+ return 0;
+
+ cond_resched();
+ }
+
+ return -EINTR;
+}
+
+static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq,
+ struct scan_control *sc, int swappiness, unsigned long opt)
+{
+ struct lruvec *lruvec;
+ int err = -EINVAL;
+ struct mem_cgroup *memcg = NULL;
+
+ if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY))
+ return -EINVAL;
+
+ if (!mem_cgroup_disabled()) {
+ rcu_read_lock();
+ memcg = mem_cgroup_from_id(memcg_id);
+#ifdef CONFIG_MEMCG
+ if (memcg && !css_tryget(&memcg->css))
+ memcg = NULL;
+#endif
+ rcu_read_unlock();
+
+ if (!memcg)
+ return -EINVAL;
+ }
+
+ if (memcg_id != mem_cgroup_id(memcg))
+ goto done;
+
+ lruvec = get_lruvec(memcg, nid);
+
+ if (swappiness < 0)
+ swappiness = get_swappiness(lruvec, sc);
+ else if (swappiness > 200)
+ goto done;
+
+ switch (cmd) {
+ case '+':
+ err = run_aging(lruvec, seq, sc, swappiness, opt);
+ break;
+ case '-':
+ err = run_eviction(lruvec, seq, sc, swappiness, opt);
+ break;
+ }
+done:
+ mem_cgroup_put(memcg);
+
+ return err;
+}
+
+/* see Documentation/admin-guide/mm/multigen_lru.rst for details */
+static ssize_t lru_gen_seq_write(struct file *file, const char __user *src,
+ size_t len, loff_t *pos)
+{
+ void *buf;
+ char *cur, *next;
+ unsigned int flags;
+ struct blk_plug plug;
+ int err = -EINVAL;
+ struct scan_control sc = {
+ .may_writepage = true,
+ .may_unmap = true,
+ .may_swap = true,
+ .reclaim_idx = MAX_NR_ZONES - 1,
+ .gfp_mask = GFP_KERNEL,
+ };
+
+ buf = kvmalloc(len + 1, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ if (copy_from_user(buf, src, len)) {
+ kvfree(buf);
+ return -EFAULT;
+ }
+
+ set_task_reclaim_state(current, &sc.reclaim_state);
+ flags = memalloc_noreclaim_save();
+ blk_start_plug(&plug);
+ if (!set_mm_walk(NULL)) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ next = buf;
+ next[len] = '\0';
+
+ while ((cur = strsep(&next, ",;\n"))) {
+ int n;
+ int end;
+ char cmd;
+ unsigned int memcg_id;
+ unsigned int nid;
+ unsigned long seq;
+ unsigned int swappiness = -1;
+ unsigned long opt = -1;
+
+ cur = skip_spaces(cur);
+ if (!*cur)
+ continue;
+
+ n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid,
+ &seq, &end, &swappiness, &end, &opt, &end);
+ if (n < 4 || cur[end]) {
+ err = -EINVAL;
+ break;
+ }
+
+ err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt);
+ if (err)
+ break;
+ }
+done:
+ clear_mm_walk();
+ blk_finish_plug(&plug);
+ memalloc_noreclaim_restore(flags);
+ set_task_reclaim_state(current, NULL);
+
+ kvfree(buf);
+
+ return err ? : len;
+}
+
+static int lru_gen_seq_open(struct inode *inode, struct file *file)
+{
+ return seq_open(file, &lru_gen_seq_ops);
+}
+
+static const struct file_operations lru_gen_rw_fops = {
+ .open = lru_gen_seq_open,
+ .read = seq_read,
+ .write = lru_gen_seq_write,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+static const struct file_operations lru_gen_ro_fops = {
+ .open = lru_gen_seq_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+/******************************************************************************
+ * initialization
+ ******************************************************************************/
+
+void lru_gen_init_lruvec(struct lruvec *lruvec)
+{
+ int i;
+ int gen, type, zone;
+ struct lru_gen_struct *lrugen = &lruvec->lrugen;
+
+ lrugen->max_seq = MIN_NR_GENS + 1;
+ lrugen->enabled = lru_gen_enabled();
+
+ for (i = 0; i <= MIN_NR_GENS + 1; i++)
+ lrugen->timestamps[i] = jiffies;
+
+ for_each_gen_type_zone(gen, type, zone)
+ INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]);
+
+ lruvec->mm_state.seq = MIN_NR_GENS;
+ init_waitqueue_head(&lruvec->mm_state.wait);
+}
+
+#ifdef CONFIG_MEMCG
+void lru_gen_init_memcg(struct mem_cgroup *memcg)
+{
+ INIT_LIST_HEAD(&memcg->mm_list.fifo);
+ spin_lock_init(&memcg->mm_list.lock);
+}
+
+void lru_gen_exit_memcg(struct mem_cgroup *memcg)
+{
+ int i;
+ int nid;
+
+ for_each_node(nid) {
+ struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+ VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
+ sizeof(lruvec->lrugen.nr_pages)));
+
+ for (i = 0; i < NR_BLOOM_FILTERS; i++) {
+ bitmap_free(lruvec->mm_state.filters[i]);
+ lruvec->mm_state.filters[i] = NULL;
+ }
+ }
+}
+#endif
+
+static int __init init_lru_gen(void)
+{
+ BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS);
+ BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS);
+
+ if (sysfs_create_group(mm_kobj, &lru_gen_attr_group))
+ pr_err("lru_gen: failed to create sysfs group\n");
+
+ debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops);
+ debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops);
+
+ return 0;
+};
+late_initcall(init_lru_gen);
+
+#else /* !CONFIG_LRU_GEN */
+
+static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
+{
+}
+
+static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+{
+}
+
+#endif /* CONFIG_LRU_GEN */
+
+static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
+{
+ unsigned long nr[NR_LRU_LISTS];
+ unsigned long targets[NR_LRU_LISTS];
+ unsigned long nr_to_scan;
+ enum lru_list lru;
+ unsigned long nr_reclaimed = 0;
+ unsigned long nr_to_reclaim = sc->nr_to_reclaim;
+ struct blk_plug plug;
+ bool scan_adjusted;
+
+ if (lru_gen_enabled()) {
+ lru_gen_shrink_lruvec(lruvec, sc);
+ return;
+ }
+
+ get_scan_count(lruvec, sc, nr);
+
+ /* Record the original scan target for proportional adjustments later */
+ memcpy(targets, nr, sizeof(nr));
+
+ /*
+ * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
+ * event that can occur when there is little memory pressure e.g.
+ * multiple streaming readers/writers. Hence, we do not abort scanning
+ * when the requested number of pages are reclaimed when scanning at
+ * DEF_PRIORITY on the assumption that the fact we are direct
+ * reclaiming implies that kswapd is not keeping up and it is best to
+ * do a batch of work at once. For memcg reclaim one check is made to
+ * abort proportional reclaim if either the file or anon lru has already
+ * dropped to zero at the first pass.
+ */
+ scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
+ sc->priority == DEF_PRIORITY);
+
+ blk_start_plug(&plug);
+ while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
+ nr[LRU_INACTIVE_FILE]) {
+ unsigned long nr_anon, nr_file, percentage;
+ unsigned long nr_scanned;
+
+ for_each_evictable_lru(lru) {
+ if (nr[lru]) {
+ nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
+ nr[lru] -= nr_to_scan;
+
+ nr_reclaimed += shrink_list(lru, nr_to_scan,
+ lruvec, sc);
+ }
+ }
+
+ cond_resched();
+
+ if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
+ continue;
+
+ /*
+ * For kswapd and memcg, reclaim at least the number of pages
+ * requested. Ensure that the anon and file LRUs are scanned
+ * proportionally what was requested by get_scan_count(). We
+ * stop reclaiming one LRU and reduce the amount scanning
+ * proportional to the original scan target.
+ */
+ nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
+ nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
+
+ /*
+ * It's just vindictive to attack the larger once the smaller
+ * has gone to zero. And given the way we stop scanning the
+ * smaller below, this makes sure that we only make one nudge
+ * towards proportionality once we've got nr_to_reclaim.
+ */
+ if (!nr_file || !nr_anon)
+ break;
+
+ if (nr_file > nr_anon) {
+ unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
+ targets[LRU_ACTIVE_ANON] + 1;
+ lru = LRU_BASE;
+ percentage = nr_anon * 100 / scan_target;
+ } else {
+ unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
+ targets[LRU_ACTIVE_FILE] + 1;
+ lru = LRU_FILE;
+ percentage = nr_file * 100 / scan_target;
+ }
+
+ /* Stop scanning the smaller of the LRU */
+ nr[lru] = 0;
+ nr[lru + LRU_ACTIVE] = 0;
+
+ /*
+ * Recalculate the other LRU scan count based on its original
+ * scan target and the percentage scanning already complete
+ */
+ lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
+ nr_scanned = targets[lru] - nr[lru];
+ nr[lru] = targets[lru] * (100 - percentage) / 100;
+ nr[lru] -= min(nr[lru], nr_scanned);
+
+ lru += LRU_ACTIVE;
+ nr_scanned = targets[lru] - nr[lru];
+ nr[lru] = targets[lru] * (100 - percentage) / 100;
+ nr[lru] -= min(nr[lru], nr_scanned);
+
+ scan_adjusted = true;
+ }
+ blk_finish_plug(&plug);
+ sc->nr_reclaimed += nr_reclaimed;
+
+ /*
+ * Even if we did not try to evict anon pages at all, we want to
+ * rebalance the anon lru active/inactive ratio.
+ */
+ if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) &&
+ inactive_is_low(lruvec, LRU_INACTIVE_ANON))
+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
+ sc, LRU_ACTIVE_ANON);
+}
+
+/* Use reclaim/compaction for costly allocs or under memory pressure */
+static bool in_reclaim_compaction(struct scan_control *sc)
+{
+ if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
+ (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
+ sc->priority < DEF_PRIORITY - 2))
+ return true;
+
+ return false;
+}
+
+/*
+ * Reclaim/compaction is used for high-order allocation requests. It reclaims
+ * order-0 pages before compacting the zone. should_continue_reclaim() returns
+ * true if more pages should be reclaimed such that when the page allocator
* calls try_to_compact_pages() that it will have enough free pages to succeed.
* It will give up earlier than that if there is difficulty reclaiming pages.
*/
unsigned long nr_reclaimed, nr_scanned;
struct lruvec *target_lruvec;
bool reclaimable = false;
- unsigned long file;
target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
again:
- /*
- * Flush the memory cgroup stats, so that we read accurate per-memcg
- * lruvec stats for heuristics.
- */
- mem_cgroup_flush_stats();
-
memset(&sc->nr, 0, sizeof(sc->nr));
nr_reclaimed = sc->nr_reclaimed;
nr_scanned = sc->nr_scanned;
- /*
- * Determine the scan balance between anon and file LRUs.
- */
- spin_lock_irq(&target_lruvec->lru_lock);
- sc->anon_cost = target_lruvec->anon_cost;
- sc->file_cost = target_lruvec->file_cost;
- spin_unlock_irq(&target_lruvec->lru_lock);
-
- /*
- * Target desirable inactive:active list ratios for the anon
- * and file LRU lists.
- */
- if (!sc->force_deactivate) {
- unsigned long refaults;
-
- refaults = lruvec_page_state(target_lruvec,
- WORKINGSET_ACTIVATE_ANON);
- if (refaults != target_lruvec->refaults[0] ||
- inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
- sc->may_deactivate |= DEACTIVATE_ANON;
- else
- sc->may_deactivate &= ~DEACTIVATE_ANON;
-
- /*
- * When refaults are being observed, it means a new
- * workingset is being established. Deactivate to get
- * rid of any stale active pages quickly.
- */
- refaults = lruvec_page_state(target_lruvec,
- WORKINGSET_ACTIVATE_FILE);
- if (refaults != target_lruvec->refaults[1] ||
- inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
- sc->may_deactivate |= DEACTIVATE_FILE;
- else
- sc->may_deactivate &= ~DEACTIVATE_FILE;
- } else
- sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
-
- /*
- * If we have plenty of inactive file pages that aren't
- * thrashing, try to reclaim those first before touching
- * anonymous pages.
- */
- file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
- if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
- sc->cache_trim_mode = 1;
- else
- sc->cache_trim_mode = 0;
-
- /*
- * Prevent the reclaimer from falling into the cache trap: as
- * cache pages start out inactive, every cache fault will tip
- * the scan balance towards the file LRU. And as the file LRU
- * shrinks, so does the window for rotation from references.
- * This means we have a runaway feedback loop where a tiny
- * thrashing file LRU becomes infinitely more attractive than
- * anon pages. Try to detect this based on file LRU size.
- */
- if (!cgroup_reclaim(sc)) {
- unsigned long total_high_wmark = 0;
- unsigned long free, anon;
- int z;
-
- free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
- file = node_page_state(pgdat, NR_ACTIVE_FILE) +
- node_page_state(pgdat, NR_INACTIVE_FILE);
-
- for (z = 0; z < MAX_NR_ZONES; z++) {
- struct zone *zone = &pgdat->node_zones[z];
- if (!managed_zone(zone))
- continue;
-
- total_high_wmark += high_wmark_pages(zone);
- }
-
- /*
- * Consider anon: if that's low too, this isn't a
- * runaway file reclaim problem, but rather just
- * extreme pressure. Reclaim as per usual then.
- */
- anon = node_page_state(pgdat, NR_INACTIVE_ANON);
-
- sc->file_is_tiny =
- file + free <= total_high_wmark &&
- !(sc->may_deactivate & DEACTIVATE_ANON) &&
- anon >> sc->priority;
- }
+ prepare_scan_count(pgdat, sc);
shrink_node_memcgs(pgdat, sc);
struct lruvec *target_lruvec;
unsigned long refaults;
+ if (lru_gen_enabled())
+ return;
+
target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
- target_lruvec->refaults[0] = refaults;
+ target_lruvec->refaults[WORKINGSET_ANON] = refaults;
refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
- target_lruvec->refaults[1] = refaults;
+ target_lruvec->refaults[WORKINGSET_FILE] = refaults;
}
/*
}
#endif
-static void age_active_anon(struct pglist_data *pgdat,
- struct scan_control *sc)
+static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)
{
struct mem_cgroup *memcg;
struct lruvec *lruvec;
+ if (lru_gen_enabled()) {
+ lru_gen_age_node(pgdat, sc);
+ return;
+ }
+
if (!can_age_anon_pages(pgdat, sc))
return;
sc.may_swap = !nr_boost_reclaim;
/*
- * Do some background aging of the anon list, to give
- * pages a chance to be referenced before reclaiming. All
- * pages are rotated regardless of classzone as this is
- * about consistent aging.
+ * Do some background aging, to give pages a chance to be
+ * referenced before reclaiming. All pages are rotated
+ * regardless of classzone as this is about consistent aging.
*/
- age_active_anon(pgdat, &sc);
+ kswapd_age_node(pgdat, &sc);
/*
* If we're getting trouble reclaiming, start doing writepage
{
pg_data_t *pgdat = NODE_DATA(nid);
- if (pgdat->kswapd)
- return;
-
- pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
- if (IS_ERR(pgdat->kswapd)) {
- /* failure at boot is fatal */
- BUG_ON(system_state < SYSTEM_RUNNING);
- pr_err("Failed to start kswapd on node %d\n", nid);
- pgdat->kswapd = NULL;
+ pgdat_kswapd_lock(pgdat);
+ if (!pgdat->kswapd) {
+ pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
+ if (IS_ERR(pgdat->kswapd)) {
+ /* failure at boot is fatal */
+ BUG_ON(system_state < SYSTEM_RUNNING);
+ pr_err("Failed to start kswapd on node %d\n", nid);
+ pgdat->kswapd = NULL;
+ }
}
+ pgdat_kswapd_unlock(pgdat);
}
/*
*/
void kswapd_stop(int nid)
{
- struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
+ pg_data_t *pgdat = NODE_DATA(nid);
+ struct task_struct *kswapd;
+ pgdat_kswapd_lock(pgdat);
+ kswapd = pgdat->kswapd;
if (kswapd) {
kthread_stop(kswapd);
- NODE_DATA(nid)->kswapd = NULL;
+ pgdat->kswapd = NULL;
}
+ pgdat_kswapd_unlock(pgdat);
}
static int __init kswapd_init(void)
#include <linux/mm_inline.h>
#include <linux/page_ext.h>
#include <linux/page_owner.h>
-#include <linux/migrate.h>
#include "internal.h"
#endif
#ifdef CONFIG_NUMA_BALANCING
"pgpromote_success",
+ "pgpromote_candidate",
#endif
/* enum writeback_stat_item counters */
"nr_tlb_local_flush_one",
#endif /* CONFIG_DEBUG_TLBFLUSH */
-#ifdef CONFIG_DEBUG_VM_VMACACHE
- "vmacache_find_calls",
- "vmacache_find_hits",
-#endif
#ifdef CONFIG_SWAP
"swap_ra",
"swap_ra_hit",
if (!node_state(cpu_to_node(cpu), N_CPU)) {
node_set_state(cpu_to_node(cpu), N_CPU);
- set_migration_target_nodes();
}
return 0;
return 0;
node_clear_state(node, N_CPU);
- set_migration_target_nodes();
return 0;
}
start_shepherd_timer();
#endif
- migrate_on_reclaim_init();
#ifdef CONFIG_PROC_FS
proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op);
proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op);
static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
bool workingset)
{
- eviction >>= bucket_order;
eviction &= EVICTION_MASK;
eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
*memcgidp = memcgid;
*pgdat = NODE_DATA(nid);
- *evictionp = entry << bucket_order;
+ *evictionp = entry;
*workingsetp = workingset;
}
+#ifdef CONFIG_LRU_GEN
+
+static void *lru_gen_eviction(struct folio *folio)
+{
+ int hist;
+ unsigned long token;
+ unsigned long min_seq;
+ struct lruvec *lruvec;
+ struct lru_gen_struct *lrugen;
+ int type = folio_is_file_lru(folio);
+ int delta = folio_nr_pages(folio);
+ int refs = folio_lru_refs(folio);
+ int tier = lru_tier_from_refs(refs);
+ struct mem_cgroup *memcg = folio_memcg(folio);
+ struct pglist_data *pgdat = folio_pgdat(folio);
+
+ BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);
+
+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
+ lrugen = &lruvec->lrugen;
+ min_seq = READ_ONCE(lrugen->min_seq[type]);
+ token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
+
+ hist = lru_hist_from_seq(min_seq);
+ atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
+
+ return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs);
+}
+
+static void lru_gen_refault(struct folio *folio, void *shadow)
+{
+ int hist, tier, refs;
+ int memcg_id;
+ bool workingset;
+ unsigned long token;
+ unsigned long min_seq;
+ struct lruvec *lruvec;
+ struct lru_gen_struct *lrugen;
+ struct mem_cgroup *memcg;
+ struct pglist_data *pgdat;
+ int type = folio_is_file_lru(folio);
+ int delta = folio_nr_pages(folio);
+
+ unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset);
+
+ if (pgdat != folio_pgdat(folio))
+ return;
+
+ rcu_read_lock();
+
+ memcg = folio_memcg_rcu(folio);
+ if (memcg_id != mem_cgroup_id(memcg))
+ goto unlock;
+
+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
+ lrugen = &lruvec->lrugen;
+
+ min_seq = READ_ONCE(lrugen->min_seq[type]);
+ if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH)))
+ goto unlock;
+
+ hist = lru_hist_from_seq(min_seq);
+ /* see the comment in folio_lru_refs() */
+ refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset;
+ tier = lru_tier_from_refs(refs);
+
+ atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
+ mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
+
+ /*
+ * Count the following two cases as stalls:
+ * 1. For pages accessed through page tables, hotter pages pushed out
+ * hot pages which refaulted immediately.
+ * 2. For pages accessed multiple times through file descriptors,
+ * numbers of accesses might have been out of the range.
+ */
+ if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) {
+ folio_set_workingset(folio);
+ mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
+ }
+unlock:
+ rcu_read_unlock();
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static void *lru_gen_eviction(struct folio *folio)
+{
+ return NULL;
+}
+
+static void lru_gen_refault(struct folio *folio, void *shadow)
+{
+}
+
+#endif /* CONFIG_LRU_GEN */
+
/**
* workingset_age_nonresident - age non-resident entries as LRU ages
* @lruvec: the lruvec that was aged
VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
+ if (lru_gen_enabled())
+ return lru_gen_eviction(folio);
+
lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
/* XXX: target_memcg can be NULL, go through lruvec */
memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
eviction = atomic_long_read(&lruvec->nonresident_age);
+ eviction >>= bucket_order;
workingset_age_nonresident(lruvec, folio_nr_pages(folio));
return pack_shadow(memcgid, pgdat, eviction,
folio_test_workingset(folio));
int memcgid;
long nr;
+ if (lru_gen_enabled()) {
+ lru_gen_refault(folio, shadow);
+ return;
+ }
+
unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset);
+ eviction <<= bucket_order;
rcu_read_lock();
/*
return first_page;
}
-static inline int get_first_obj_offset(struct page *page)
+static inline unsigned int get_first_obj_offset(struct page *page)
{
return page->page_type;
}
-static inline void set_first_obj_offset(struct page *page, int offset)
+static inline void set_first_obj_offset(struct page *page, unsigned int offset)
{
page->page_type = offset;
}
d_off += size;
d_size -= size;
+ /*
+ * Calling kunmap_atomic(d_addr) is necessary. kunmap_atomic()
+ * calls must occurs in reverse order of calls to kmap_atomic().
+ * So, to call kunmap_atomic(s_addr) we should first call
+ * kunmap_atomic(d_addr). For more details see
+ * Documentation/mm/highmem.rst.
+ */
if (s_off >= PAGE_SIZE) {
kunmap_atomic(d_addr);
kunmap_atomic(s_addr);
static unsigned long find_alloced_obj(struct size_class *class,
struct page *page, int *obj_idx)
{
- int offset = 0;
+ unsigned int offset;
int index = *obj_idx;
unsigned long handle = 0;
void *addr = kmap_atomic(page);
struct zspage *zspage;
struct page *dummy;
void *s_addr, *d_addr, *addr;
- int offset;
+ unsigned int offset;
unsigned long handle;
unsigned long old_obj, new_obj;
unsigned int obj_idx;
for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
class = pool->size_class[i];
- if (!class)
- continue;
if (class->index != i)
continue;
pages_freed += __zs_compact(pool, class);
for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
class = pool->size_class[i];
- if (!class)
- continue;
if (class->index != i)
continue;
int fg;
struct size_class *class = pool->size_class[i];
- if (!class)
- continue;
-
if (class->index != i)
continue;
SetPageReclaim(page);
/* start writeback */
- __swap_writepage(page, &wbc, end_swap_bio_write);
+ __swap_writepage(page, &wbc);
put_page(page);
zswap_written_back_pages++;
{
unsigned long delay;
- delay = (u64)msecs_to_jiffies(garp_join_time) * prandom_u32() >> 32;
+ delay = prandom_u32_max(msecs_to_jiffies(garp_join_time));
mod_timer(&app->join_timer, jiffies + delay);
}
{
unsigned long delay;
- delay = (u64)msecs_to_jiffies(mrp_join_time) * prandom_u32() >> 32;
+ delay = prandom_u32_max(msecs_to_jiffies(mrp_join_time));
mod_timer(&app->join_timer, jiffies + delay);
}
* p9_tag_alloc - Allocate a new request.
* @c: Client session.
* @type: Transaction type.
- * @max_size: Maximum packet size for this request.
+ * @t_size: Buffer size for holding this request
+ * (automatic calculation by format template if 0).
+ * @r_size: Buffer size for holding server's reply on this request
+ * (automatic calculation by format template if 0).
+ * @fmt: Format template for assembling 9p request message
+ * (see p9pdu_vwritef).
+ * @ap: Variable arguments to be fed to passed format template
+ * (see p9pdu_vwritef).
*
* Context: Process context.
* Return: Pointer to new request.
*/
static struct p9_req_t *
-p9_tag_alloc(struct p9_client *c, int8_t type, unsigned int max_size)
+p9_tag_alloc(struct p9_client *c, int8_t type, uint t_size, uint r_size,
+ const char *fmt, va_list ap)
{
struct p9_req_t *req = kmem_cache_alloc(p9_req_cache, GFP_NOFS);
- int alloc_msize = min(c->msize, max_size);
+ int alloc_tsize;
+ int alloc_rsize;
int tag;
+ va_list apc;
+
+ va_copy(apc, ap);
+ alloc_tsize = min_t(size_t, c->msize,
+ t_size ?: p9_msg_buf_size(c, type, fmt, apc));
+ va_end(apc);
+
+ alloc_rsize = min_t(size_t, c->msize,
+ r_size ?: p9_msg_buf_size(c, type + 1, fmt, ap));
if (!req)
return ERR_PTR(-ENOMEM);
- if (p9_fcall_init(c, &req->tc, alloc_msize))
+ if (p9_fcall_init(c, &req->tc, alloc_tsize))
goto free_req;
- if (p9_fcall_init(c, &req->rc, alloc_msize))
+ if (p9_fcall_init(c, &req->rc, alloc_rsize))
goto free;
p9pdu_reset(&req->tc);
}
static struct p9_req_t *p9_client_prepare_req(struct p9_client *c,
- int8_t type, int req_size,
+ int8_t type, uint t_size, uint r_size,
const char *fmt, va_list ap)
{
int err;
struct p9_req_t *req;
+ va_list apc;
p9_debug(P9_DEBUG_MUX, "client %p op %d\n", c, type);
if (c->status == BeginDisconnect && type != P9_TCLUNK)
return ERR_PTR(-EIO);
- req = p9_tag_alloc(c, type, req_size);
+ va_copy(apc, ap);
+ req = p9_tag_alloc(c, type, t_size, r_size, fmt, apc);
+ va_end(apc);
if (IS_ERR(req))
return req;
int sigpending, err;
unsigned long flags;
struct p9_req_t *req;
+ /* Passing zero for tsize/rsize to p9_client_prepare_req() tells it to
+ * auto determine an appropriate (small) request/response size
+ * according to actual message data being sent. Currently RDMA
+ * transport is excluded from this response message size optimization,
+ * as it would not cope with it, due to its pooled response buffers
+ * (using an optimized request size for RDMA as well though).
+ */
+ const uint tsize = 0;
+ const uint rsize = c->trans_mod->pooled_rbuffers ? c->msize : 0;
va_start(ap, fmt);
- req = p9_client_prepare_req(c, type, c->msize, fmt, ap);
+ req = p9_client_prepare_req(c, type, tsize, rsize, fmt, ap);
va_end(ap);
if (IS_ERR(req))
return req;
/* We allocate a inline protocol data of only 4k bytes.
* The actual content is passed in zero-copy fashion.
*/
- req = p9_client_prepare_req(c, type, P9_ZC_HDR_SZ, fmt, ap);
+ req = p9_client_prepare_req(c, type, P9_ZC_HDR_SZ, P9_ZC_HDR_SZ, fmt, ap);
va_end(ap);
if (IS_ERR(req))
return req;
#include <trace/events/9p.h>
+/* len[2] text[len] */
+#define P9_STRLEN(s) \
+ (2 + min_t(size_t, s ? strlen(s) : 0, USHRT_MAX))
+
+/**
+ * p9_msg_buf_size - Returns a buffer size sufficiently large to hold the
+ * intended 9p message.
+ * @c: client
+ * @type: message type
+ * @fmt: format template for assembling request message
+ * (see p9pdu_vwritef)
+ * @ap: variable arguments to be fed to passed format template
+ * (see p9pdu_vwritef)
+ *
+ * Note: Even for response types (P9_R*) the format template and variable
+ * arguments must always be for the originating request type (P9_T*).
+ */
+size_t p9_msg_buf_size(struct p9_client *c, enum p9_msg_t type,
+ const char *fmt, va_list ap)
+{
+ /* size[4] type[1] tag[2] */
+ const int hdr = 4 + 1 + 2;
+ /* ename[s] errno[4] */
+ const int rerror_size = hdr + P9_ERRMAX + 4;
+ /* ecode[4] */
+ const int rlerror_size = hdr + 4;
+ const int err_size =
+ c->proto_version == p9_proto_2000L ? rlerror_size : rerror_size;
+
+ static_assert(NAME_MAX <= 4*1024, "p9_msg_buf_size() currently assumes "
+ "a max. allowed directory entry name length of 4k");
+
+ switch (type) {
+
+ /* message types not used at all */
+ case P9_TERROR:
+ case P9_TLERROR:
+ case P9_TAUTH:
+ case P9_RAUTH:
+ BUG();
+
+ /* variable length & potentially large message types */
+ case P9_TATTACH:
+ BUG_ON(strcmp("ddss?u", fmt));
+ va_arg(ap, int32_t);
+ va_arg(ap, int32_t);
+ {
+ const char *uname = va_arg(ap, const char *);
+ const char *aname = va_arg(ap, const char *);
+ /* fid[4] afid[4] uname[s] aname[s] n_uname[4] */
+ return hdr + 4 + 4 + P9_STRLEN(uname) + P9_STRLEN(aname) + 4;
+ }
+ case P9_TWALK:
+ BUG_ON(strcmp("ddT", fmt));
+ va_arg(ap, int32_t);
+ va_arg(ap, int32_t);
+ {
+ uint i, nwname = va_arg(ap, int);
+ size_t wname_all;
+ const char **wnames = va_arg(ap, const char **);
+ for (i = 0, wname_all = 0; i < nwname; ++i) {
+ wname_all += P9_STRLEN(wnames[i]);
+ }
+ /* fid[4] newfid[4] nwname[2] nwname*(wname[s]) */
+ return hdr + 4 + 4 + 2 + wname_all;
+ }
+ case P9_RWALK:
+ BUG_ON(strcmp("ddT", fmt));
+ va_arg(ap, int32_t);
+ va_arg(ap, int32_t);
+ {
+ uint nwname = va_arg(ap, int);
+ /* nwqid[2] nwqid*(wqid[13]) */
+ return max_t(size_t, hdr + 2 + nwname * 13, err_size);
+ }
+ case P9_TCREATE:
+ BUG_ON(strcmp("dsdb?s", fmt));
+ va_arg(ap, int32_t);
+ {
+ const char *name = va_arg(ap, const char *);
+ if (c->proto_version == p9_proto_legacy) {
+ /* fid[4] name[s] perm[4] mode[1] */
+ return hdr + 4 + P9_STRLEN(name) + 4 + 1;
+ } else {
+ va_arg(ap, int32_t);
+ va_arg(ap, int);
+ {
+ const char *ext = va_arg(ap, const char *);
+ /* fid[4] name[s] perm[4] mode[1] extension[s] */
+ return hdr + 4 + P9_STRLEN(name) + 4 + 1 + P9_STRLEN(ext);
+ }
+ }
+ }
+ case P9_TLCREATE:
+ BUG_ON(strcmp("dsddg", fmt));
+ va_arg(ap, int32_t);
+ {
+ const char *name = va_arg(ap, const char *);
+ /* fid[4] name[s] flags[4] mode[4] gid[4] */
+ return hdr + 4 + P9_STRLEN(name) + 4 + 4 + 4;
+ }
+ case P9_RREAD:
+ case P9_RREADDIR:
+ BUG_ON(strcmp("dqd", fmt));
+ va_arg(ap, int32_t);
+ va_arg(ap, int64_t);
+ {
+ const int32_t count = va_arg(ap, int32_t);
+ /* count[4] data[count] */
+ return max_t(size_t, hdr + 4 + count, err_size);
+ }
+ case P9_TWRITE:
+ BUG_ON(strcmp("dqV", fmt));
+ va_arg(ap, int32_t);
+ va_arg(ap, int64_t);
+ {
+ const int32_t count = va_arg(ap, int32_t);
+ /* fid[4] offset[8] count[4] data[count] */
+ return hdr + 4 + 8 + 4 + count;
+ }
+ case P9_TRENAMEAT:
+ BUG_ON(strcmp("dsds", fmt));
+ va_arg(ap, int32_t);
+ {
+ const char *oldname, *newname;
+ oldname = va_arg(ap, const char *);
+ va_arg(ap, int32_t);
+ newname = va_arg(ap, const char *);
+ /* olddirfid[4] oldname[s] newdirfid[4] newname[s] */
+ return hdr + 4 + P9_STRLEN(oldname) + 4 + P9_STRLEN(newname);
+ }
+ case P9_TSYMLINK:
+ BUG_ON(strcmp("dssg", fmt));
+ va_arg(ap, int32_t);
+ {
+ const char *name = va_arg(ap, const char *);
+ const char *symtgt = va_arg(ap, const char *);
+ /* fid[4] name[s] symtgt[s] gid[4] */
+ return hdr + 4 + P9_STRLEN(name) + P9_STRLEN(symtgt) + 4;
+ }
+
+ case P9_RERROR:
+ return rerror_size;
+ case P9_RLERROR:
+ return rlerror_size;
+
+ /* small message types */
+ case P9_TWSTAT:
+ case P9_RSTAT:
+ case P9_RREADLINK:
+ case P9_TXATTRWALK:
+ case P9_TXATTRCREATE:
+ case P9_TLINK:
+ case P9_TMKDIR:
+ case P9_TMKNOD:
+ case P9_TRENAME:
+ case P9_TUNLINKAT:
+ case P9_TLOCK:
+ return 8 * 1024;
+
+ /* tiny message types */
+ default:
+ return 4 * 1024;
+
+ }
+}
+
static int
p9pdu_writef(struct p9_fcall *pdu, int proto_version, const char *fmt, ...);
* Copyright (C) 2008 by IBM, Corp.
*/
+size_t p9_msg_buf_size(struct p9_client *c, enum p9_msg_t type,
+ const char *fmt, va_list ap);
int p9pdu_vwritef(struct p9_fcall *pdu, int proto_version, const char *fmt,
va_list ap);
int p9pdu_readf(struct p9_fcall *pdu, int proto_version, const char *fmt, ...);
* @mux_list: list link for mux to manage multiple connections (?)
* @client: reference to client instance for this connection
* @err: error state
+ * @req_lock: lock protecting req_list and requests statuses
* @req_list: accounting for requests which have been sent
* @unsent_req_list: accounting for requests that haven't been sent
* @rreq: read request
struct list_head mux_list;
struct p9_client *client;
int err;
+ spinlock_t req_lock;
struct list_head req_list;
struct list_head unsent_req_list;
struct p9_req_t *rreq;
p9_debug(P9_DEBUG_ERROR, "mux %p err %d\n", m, err);
- spin_lock(&m->client->lock);
+ spin_lock(&m->req_lock);
if (m->err) {
- spin_unlock(&m->client->lock);
+ spin_unlock(&m->req_lock);
return;
}
list_move(&req->req_list, &cancel_list);
}
+ spin_unlock(&m->req_lock);
+
list_for_each_entry_safe(req, rtmp, &cancel_list, req_list) {
p9_debug(P9_DEBUG_ERROR, "call back req %p\n", req);
list_del(&req->req_list);
req->t_err = err;
p9_client_cb(m->client, req, REQ_STATUS_ERROR);
}
- spin_unlock(&m->client->lock);
}
static __poll_t
if ((m->rreq) && (m->rc.offset == m->rc.capacity)) {
p9_debug(P9_DEBUG_TRANS, "got new packet\n");
m->rreq->rc.size = m->rc.offset;
- spin_lock(&m->client->lock);
+ spin_lock(&m->req_lock);
if (m->rreq->status == REQ_STATUS_SENT) {
list_del(&m->rreq->req_list);
p9_client_cb(m->client, m->rreq, REQ_STATUS_RCVD);
p9_debug(P9_DEBUG_TRANS,
"Ignore replies associated with a cancelled request\n");
} else {
- spin_unlock(&m->client->lock);
+ spin_unlock(&m->req_lock);
p9_debug(P9_DEBUG_ERROR,
"Request tag %d errored out while we were reading the reply\n",
m->rc.tag);
err = -EIO;
goto error;
}
- spin_unlock(&m->client->lock);
+ spin_unlock(&m->req_lock);
m->rc.sdata = NULL;
m->rc.offset = 0;
m->rc.capacity = 0;
}
if (!m->wsize) {
- spin_lock(&m->client->lock);
+ spin_lock(&m->req_lock);
if (list_empty(&m->unsent_req_list)) {
clear_bit(Wworksched, &m->wsched);
- spin_unlock(&m->client->lock);
+ spin_unlock(&m->req_lock);
return;
}
m->wpos = 0;
p9_req_get(req);
m->wreq = req;
- spin_unlock(&m->client->lock);
+ spin_unlock(&m->req_lock);
}
p9_debug(P9_DEBUG_TRANS, "mux %p pos %d size %d\n",
INIT_LIST_HEAD(&m->mux_list);
m->client = client;
+ spin_lock_init(&m->req_lock);
INIT_LIST_HEAD(&m->req_list);
INIT_LIST_HEAD(&m->unsent_req_list);
INIT_WORK(&m->rq, p9_read_work);
if (m->err < 0)
return m->err;
- spin_lock(&client->lock);
+ spin_lock(&m->req_lock);
req->status = REQ_STATUS_UNSENT;
list_add_tail(&req->req_list, &m->unsent_req_list);
- spin_unlock(&client->lock);
+ spin_unlock(&m->req_lock);
if (test_and_clear_bit(Wpending, &m->wsched))
n = EPOLLOUT;
static int p9_fd_cancel(struct p9_client *client, struct p9_req_t *req)
{
+ struct p9_trans_fd *ts = client->trans;
+ struct p9_conn *m = &ts->conn;
int ret = 1;
p9_debug(P9_DEBUG_TRANS, "client %p req %p\n", client, req);
- spin_lock(&client->lock);
+ spin_lock(&m->req_lock);
if (req->status == REQ_STATUS_UNSENT) {
list_del(&req->req_list);
p9_req_put(client, req);
ret = 0;
}
- spin_unlock(&client->lock);
+ spin_unlock(&m->req_lock);
return ret;
}
static int p9_fd_cancelled(struct p9_client *client, struct p9_req_t *req)
{
+ struct p9_trans_fd *ts = client->trans;
+ struct p9_conn *m = &ts->conn;
+
p9_debug(P9_DEBUG_TRANS, "client %p req %p\n", client, req);
- spin_lock(&client->lock);
+ spin_lock(&m->req_lock);
/* Ignore cancelled request if message has been received
* before lock.
*/
if (req->status == REQ_STATUS_RCVD) {
- spin_unlock(&client->lock);
+ spin_unlock(&m->req_lock);
return 0;
}
*/
list_del(&req->req_list);
req->status = REQ_STATUS_FLSHD;
- spin_unlock(&client->lock);
+ spin_unlock(&m->req_lock);
+
p9_req_put(client, req);
return 0;
goto out_free_ts;
if (!(ts->rd->f_mode & FMODE_READ))
goto out_put_rd;
+ /* prevent workers from hanging on IO when fd is a pipe */
+ ts->rd->f_flags |= O_NONBLOCK;
ts->wr = fget(wfd);
if (!ts->wr)
goto out_put_rd;
if (!(ts->wr->f_mode & FMODE_WRITE))
goto out_put_wr;
+ ts->wr->f_flags |= O_NONBLOCK;
client->trans = ts;
client->status = Connected;
int err;
struct p9_fd_opts opts;
- parse_opts(args, &opts);
+ err = parse_opts(args, &opts);
+ if (err < 0)
+ return err;
client->trans_opts.fd.rfd = opts.rfd;
client->trans_opts.fd.wfd = opts.wfd;
static struct p9_trans_module p9_tcp_trans = {
.name = "tcp",
.maxsize = MAX_SOCK_BUF,
+ .pooled_rbuffers = false,
.def = 0,
.create = p9_fd_create_tcp,
.close = p9_fd_close,
static struct p9_trans_module p9_rdma_trans = {
.name = "rdma",
.maxsize = P9_RDMA_MAXSIZE,
+ .pooled_rbuffers = true,
.def = 0,
.owner = THIS_MODULE,
.create = rdma_create_trans,
* page in zero copy.
*/
.maxsize = PAGE_SIZE * (VIRTQUEUE_NUM - 3),
+ .pooled_rbuffers = false,
.def = 1,
.owner = THIS_MODULE,
};
static struct p9_trans_module p9_xen_trans = {
.name = "xen",
.maxsize = 1 << (XEN_9PFS_RING_ORDER + XEN_PAGE_SHIFT - 2),
+ .pooled_rbuffers = false,
.def = 1,
.create = p9_xen_create,
.close = p9_xen_close,
.otherend_changed = xen_9pfs_front_changed,
};
-static int p9_trans_xen_init(void)
+static int __init p9_trans_xen_init(void)
{
int rc;
module_init(p9_trans_xen_init);
MODULE_ALIAS_9P("xen");
-static void p9_trans_xen_exit(void)
+static void __exit p9_trans_xen_exit(void)
{
v9fs_unregister_trans(&p9_xen_trans);
return xenbus_unregister_driver(&xen_9pfs_front_driver);
it->iter.bi_size = cursor->resid;
BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));
- cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter);
}
static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,
cursor->resid -= bytes;
bio_advance_iter(it->bio, &it->iter, bytes);
- if (!cursor->resid) {
- BUG_ON(!cursor->last_piece);
+ if (!cursor->resid)
return false; /* no more data */
- }
if (!bytes || (it->iter.bi_size && it->iter.bi_bvec_done &&
page == bio_iter_page(it->bio, it->iter)))
it->iter.bi_size = cursor->resid;
}
- BUG_ON(cursor->last_piece);
BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));
- cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter);
return true;
}
#endif /* CONFIG_BLOCK */
cursor->bvec_iter.bi_size = cursor->resid;
BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));
- cursor->last_piece =
- cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter);
}
static struct page *ceph_msg_data_bvecs_next(struct ceph_msg_data_cursor *cursor,
cursor->resid -= bytes;
bvec_iter_advance(bvecs, &cursor->bvec_iter, bytes);
- if (!cursor->resid) {
- BUG_ON(!cursor->last_piece);
+ if (!cursor->resid)
return false; /* no more data */
- }
if (!bytes || (cursor->bvec_iter.bi_bvec_done &&
page == bvec_iter_page(bvecs, cursor->bvec_iter)))
return false; /* more bytes to process in this segment */
- BUG_ON(cursor->last_piece);
BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));
- cursor->last_piece =
- cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter);
return true;
}
BUG_ON(page_count > (int)USHRT_MAX);
cursor->page_count = (unsigned short)page_count;
BUG_ON(length > SIZE_MAX - cursor->page_offset);
- cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE;
}
static struct page *
BUG_ON(cursor->page_offset >= PAGE_SIZE);
*page_offset = cursor->page_offset;
- if (cursor->last_piece)
- *length = cursor->resid;
- else
- *length = PAGE_SIZE - *page_offset;
-
+ *length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset);
return data->pages[cursor->page_index];
}
BUG_ON(cursor->page_index >= cursor->page_count);
cursor->page_index++;
- cursor->last_piece = cursor->resid <= PAGE_SIZE;
-
return true;
}
cursor->resid = min(length, pagelist->length);
cursor->page = page;
cursor->offset = 0;
- cursor->last_piece = cursor->resid <= PAGE_SIZE;
}
static struct page *
/* offset of first page in pagelist is always 0 */
*page_offset = cursor->offset & ~PAGE_MASK;
- if (cursor->last_piece)
- *length = cursor->resid;
- else
- *length = PAGE_SIZE - *page_offset;
-
+ *length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset);
return cursor->page;
}
BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));
cursor->page = list_next_entry(cursor->page, lru);
- cursor->last_piece = cursor->resid <= PAGE_SIZE;
-
return true;
}
* Indicate whether this is the last piece in this data item.
*/
struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,
- size_t *page_offset, size_t *length,
- bool *last_piece)
+ size_t *page_offset, size_t *length)
{
struct page *page;
BUG_ON(*page_offset + *length > PAGE_SIZE);
BUG_ON(!*length);
BUG_ON(*length > cursor->resid);
- if (last_piece)
- *last_piece = cursor->last_piece;
return page;
}
cursor->total_resid -= bytes;
if (!cursor->resid && cursor->total_resid) {
- WARN_ON(!cursor->last_piece);
cursor->data++;
__ceph_msg_data_cursor_init(cursor);
new_piece = true;
continue;
}
- page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);
+ page = ceph_msg_data_next(cursor, &page_offset, &length);
if (length == cursor->total_resid)
more = MSG_MORE;
ret = ceph_tcp_sendpage(con->sock, page, page_offset, length,
continue;
}
- page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);
+ page = ceph_msg_data_next(cursor, &page_offset, &length);
ret = ceph_tcp_recvpage(con->sock, page, page_offset, length);
if (ret <= 0) {
if (do_datacrc)
continue;
}
- page = ceph_msg_data_next(cursor, &off, &len, NULL);
+ page = ceph_msg_data_next(cursor, &off, &len);
ret = ceph_tcp_recvpage(con->sock, con->bounce_page, 0, len);
if (ret <= 0) {
con->in_data_crc = crc;
ceph_msg_data_advance(cursor, 0);
/* get a piece of data, cursor isn't advanced */
- page = ceph_msg_data_next(cursor, &off, &len, NULL);
+ page = ceph_msg_data_next(cursor, &off, &len);
bv->bv_page = page;
bv->bv_offset = off;
max--;
}
- n = prandom_u32() % max;
+ n = prandom_u32_max(max);
if (o >= 0 && n >= o)
n++;
static int pick_random_replica(const struct ceph_osds *acting)
{
- int i = prandom_u32() % acting->size;
+ int i = prandom_u32_max(acting->size);
dout("%s picked osd%d, primary osd%d\n", __func__,
acting->osds[i], acting->primary);
unsigned long neigh_rand_reach_time(unsigned long base)
{
- return base ? (prandom_u32() % base) + (base >> 1) : 0;
+ return base ? prandom_u32_max(base) + (base >> 1) : 0;
}
EXPORT_SYMBOL(neigh_rand_reach_time);
pkt_dev->curfl = 0; /*reset */
}
} else {
- flow = prandom_u32() % pkt_dev->cflows;
+ flow = prandom_u32_max(pkt_dev->cflows);
pkt_dev->curfl = flow;
if (pkt_dev->flows[flow].count > pkt_dev->lflow) {
else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) {
__u16 t;
if (pkt_dev->flags & F_QUEUE_MAP_RND) {
- t = prandom_u32() %
- (pkt_dev->queue_map_max -
- pkt_dev->queue_map_min + 1)
- + pkt_dev->queue_map_min;
+ t = prandom_u32_max(pkt_dev->queue_map_max -
+ pkt_dev->queue_map_min + 1) +
+ pkt_dev->queue_map_min;
} else {
t = pkt_dev->cur_queue_map + 1;
if (t > pkt_dev->queue_map_max)
__u32 tmp;
if (pkt_dev->flags & F_MACSRC_RND)
- mc = prandom_u32() % pkt_dev->src_mac_count;
+ mc = prandom_u32_max(pkt_dev->src_mac_count);
else {
mc = pkt_dev->cur_src_mac_offset++;
if (pkt_dev->cur_src_mac_offset >=
__u32 tmp;
if (pkt_dev->flags & F_MACDST_RND)
- mc = prandom_u32() % pkt_dev->dst_mac_count;
+ mc = prandom_u32_max(pkt_dev->dst_mac_count);
else {
mc = pkt_dev->cur_dst_mac_offset++;
for (i = 0; i < pkt_dev->nr_labels; i++)
if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM)
pkt_dev->labels[i] = MPLS_STACK_BOTTOM |
- ((__force __be32)prandom_u32() &
+ ((__force __be32)get_random_u32() &
htonl(0x000fffff));
}
if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) {
- pkt_dev->vlan_id = prandom_u32() & (4096 - 1);
+ pkt_dev->vlan_id = prandom_u32_max(4096);
}
if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) {
- pkt_dev->svlan_id = prandom_u32() & (4096 - 1);
+ pkt_dev->svlan_id = prandom_u32_max(4096);
}
if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) {
if (pkt_dev->flags & F_UDPSRC_RND)
- pkt_dev->cur_udp_src = prandom_u32() %
- (pkt_dev->udp_src_max - pkt_dev->udp_src_min)
- + pkt_dev->udp_src_min;
+ pkt_dev->cur_udp_src = prandom_u32_max(
+ pkt_dev->udp_src_max - pkt_dev->udp_src_min) +
+ pkt_dev->udp_src_min;
else {
pkt_dev->cur_udp_src++;
if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) {
if (pkt_dev->flags & F_UDPDST_RND) {
- pkt_dev->cur_udp_dst = prandom_u32() %
- (pkt_dev->udp_dst_max - pkt_dev->udp_dst_min)
- + pkt_dev->udp_dst_min;
+ pkt_dev->cur_udp_dst = prandom_u32_max(
+ pkt_dev->udp_dst_max - pkt_dev->udp_dst_min) +
+ pkt_dev->udp_dst_min;
} else {
pkt_dev->cur_udp_dst++;
if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max)
if (imn < imx) {
__u32 t;
if (pkt_dev->flags & F_IPSRC_RND)
- t = prandom_u32() % (imx - imn) + imn;
+ t = prandom_u32_max(imx - imn) + imn;
else {
t = ntohl(pkt_dev->cur_saddr);
t++;
if (pkt_dev->flags & F_IPDST_RND) {
do {
- t = prandom_u32() %
- (imx - imn) + imn;
+ t = prandom_u32_max(imx - imn) +
+ imn;
s = htonl(t);
} while (ipv4_is_loopback(s) ||
ipv4_is_multicast(s) ||
for (i = 0; i < 4; i++) {
pkt_dev->cur_in6_daddr.s6_addr32[i] =
- (((__force __be32)prandom_u32() |
+ (((__force __be32)get_random_u32() |
pkt_dev->min_in6_daddr.s6_addr32[i]) &
pkt_dev->max_in6_daddr.s6_addr32[i]);
}
if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) {
__u32 t;
if (pkt_dev->flags & F_TXSIZE_RND) {
- t = prandom_u32() %
- (pkt_dev->max_pkt_size - pkt_dev->min_pkt_size)
- + pkt_dev->min_pkt_size;
+ t = prandom_u32_max(pkt_dev->max_pkt_size -
+ pkt_dev->min_pkt_size) +
+ pkt_dev->min_pkt_size;
} else {
t = pkt_dev->cur_pkt_size + 1;
if (t > pkt_dev->max_pkt_size)
pkt_dev->cur_pkt_size = t;
} else if (pkt_dev->n_imix_entries > 0) {
struct imix_pkt *entry;
- __u32 t = prandom_u32() % IMIX_PRECISION;
+ __u32 t = prandom_u32_max(IMIX_PRECISION);
__u8 entry_index = pkt_dev->imix_distribution[t];
entry = &pkt_dev->imix_entries[entry_index];
{
struct sock *sk = sock->sk;
- return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
+ /* IPV6_ADDRFORM can change sk->sk_prot under us. */
+ return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen);
}
EXPORT_SYMBOL(sock_common_getsockopt);
{
struct sock *sk = sock->sk;
- return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
+ /* IPV6_ADDRFORM can change sk->sk_prot under us. */
+ return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen);
}
EXPORT_SYMBOL(sock_common_setsockopt);
DEFINE_WAIT_FUNC(wait, woken_wake_function);
if (sk_stream_memory_free(sk))
- current_timeo = vm_wait = (prandom_u32() % (HZ / 5)) + 2;
+ current_timeo = vm_wait = prandom_u32_max(HZ / 5) + 2;
add_wait_queue(sk_sleep(sk), &wait);
inet->inet_daddr,
inet->inet_sport,
inet->inet_dport);
- inet->inet_id = prandom_u32();
+ inet->inet_id = get_random_u16();
err = dccp_connect(sk);
rt = NULL;
RCU_INIT_POINTER(newinet->inet_opt, rcu_dereference(ireq->ireq_opt));
newinet->mc_index = inet_iif(skb);
newinet->mc_ttl = ip_hdr(skb)->ttl;
- newinet->inet_id = prandom_u32();
+ newinet->inet_id = get_random_u16();
if (dst == NULL && (dst = inet_csk_route_child_sock(sk, newsk, req)) == NULL)
goto put_and_exit;
pl = phylink_create(&dp->pl_config, of_fwnode_handle(dp->dn),
mode, &dsa_port_phylink_mac_ops);
if (IS_ERR(pl)) {
- pr_err("error creating PHYLINK: %ld\n", PTR_ERR(dp->pl));
+ pr_err("error creating PHYLINK: %ld\n", PTR_ERR(pl));
return PTR_ERR(pl);
}
return -EOPNOTSUPP;
}
- if (!size)
- return -EINVAL;
-
lock_sock(sk);
if (!sk->sk_bound_dev_if)
dev = dev_getfirstbyhwtype(sock_net(sk), ARPHRD_IEEE802154);
err = -EMSGSIZE;
goto out_dev;
}
+ if (!size) {
+ err = 0;
+ goto out_dev;
+ }
hlen = LL_RESERVED_SPACE(dev);
tlen = dev->needed_tailroom;
int addr_len, int flags)
{
struct sock *sk = sock->sk;
+ const struct proto *prot;
int err;
if (addr_len < sizeof(uaddr->sa_family))
return -EINVAL;
+
+ /* IPV6_ADDRFORM can change sk->sk_prot under us. */
+ prot = READ_ONCE(sk->sk_prot);
+
if (uaddr->sa_family == AF_UNSPEC)
- return sk->sk_prot->disconnect(sk, flags);
+ return prot->disconnect(sk, flags);
if (BPF_CGROUP_PRE_CONNECT_ENABLED(sk)) {
- err = sk->sk_prot->pre_connect(sk, uaddr, addr_len);
+ err = prot->pre_connect(sk, uaddr, addr_len);
if (err)
return err;
}
if (data_race(!inet_sk(sk)->inet_num) && inet_autobind(sk))
return -EAGAIN;
- return sk->sk_prot->connect(sk, uaddr, addr_len);
+ return prot->connect(sk, uaddr, addr_len);
}
EXPORT_SYMBOL(inet_dgram_connect);
int inet_accept(struct socket *sock, struct socket *newsock, int flags,
bool kern)
{
- struct sock *sk1 = sock->sk;
+ struct sock *sk1 = sock->sk, *sk2;
int err = -EINVAL;
- struct sock *sk2 = sk1->sk_prot->accept(sk1, flags, &err, kern);
+ /* IPV6_ADDRFORM can change sk->sk_prot under us. */
+ sk2 = READ_ONCE(sk1->sk_prot)->accept(sk1, flags, &err, kern);
if (!sk2)
goto do_err;
size_t size, int flags)
{
struct sock *sk = sock->sk;
+ const struct proto *prot;
if (unlikely(inet_send_prepare(sk)))
return -EAGAIN;
- if (sk->sk_prot->sendpage)
- return sk->sk_prot->sendpage(sk, page, offset, size, flags);
+ /* IPV6_ADDRFORM can change sk->sk_prot under us. */
+ prot = READ_ONCE(sk->sk_prot);
+ if (prot->sendpage)
+ return prot->sendpage(sk, page, offset, size, flags);
return sock_no_sendpage(sock, page, offset, size, flags);
}
EXPORT_SYMBOL(inet_sendpage);
reuseport_has_conns(sk, true);
sk->sk_state = TCP_ESTABLISHED;
sk_set_txhash(sk);
- inet->inet_id = prandom_u32();
+ inet->inet_id = get_random_u16();
sk_dst_set(sk, &rt->dst);
err = 0;
return 1;
}
+ /* cannot match on nexthop object attributes */
+ if (fi->nh)
+ return 1;
+
if (cfg->fc_oif || cfg->fc_gw_family) {
struct fib_nh *nh;
- /* cannot match on nexthop object attributes */
- if (fi->nh)
- return 1;
-
nh = fib_info_nh(fi, 0);
if (cfg->fc_encap) {
if (fib_encap_match(net, cfg->fc_encap_type,
/* It must be called with locked im->lock */
static void igmp_start_timer(struct ip_mc_list *im, int max_delay)
{
- int tv = prandom_u32() % max_delay;
+ int tv = prandom_u32_max(max_delay);
im->tm_running = 1;
if (!mod_timer(&im->timer, jiffies+tv+2))
static void igmp_gq_start_timer(struct in_device *in_dev)
{
- int tv = prandom_u32() % in_dev->mr_maxdelay;
+ int tv = prandom_u32_max(in_dev->mr_maxdelay);
unsigned long exp = jiffies + tv + 2;
if (in_dev->mr_gq_running &&
static void igmp_ifc_start_timer(struct in_device *in_dev, int delay)
{
- int tv = prandom_u32() % delay;
+ int tv = prandom_u32_max(delay);
if (!mod_timer(&in_dev->mr_ifc_timer, jiffies+tv+2))
in_dev_hold(in_dev);
if (likely(remaining > 1))
remaining &= ~1U;
- offset = prandom_u32() % remaining;
+ offset = prandom_u32_max(remaining);
/* __inet_hash_connect() favors ports having @low parity
* We do the opposite to not pollute connect() users.
*/
* on low contention the randomness is maximal and on high contention
* it may be inexistent.
*/
- i = max_t(int, i, (prandom_u32() & 7) * 2);
+ i = max_t(int, i, prandom_u32_max(8) * 2);
WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
/* Head lock still held and bh's disabled */
rcu_read_lock();
restart:
sk_nulls_for_each_rcu(sk, node, &head->chain) {
- if (sk->sk_state != TCP_TIME_WAIT)
+ if (sk->sk_state != TCP_TIME_WAIT) {
+ /* A kernel listener socket might not hold refcnt for net,
+ * so reqsk_timer_handler() could be fired after net is
+ * freed. Userspace listener and reqsk never exist here.
+ */
+ if (unlikely(sk->sk_state == TCP_NEW_SYN_RECV &&
+ hashinfo->pernet)) {
+ struct request_sock *req = inet_reqsk(sk);
+
+ inet_csk_reqsk_queue_drop_and_put(req->rsk_listener, req);
+ }
+
continue;
+ }
+
tw = inet_twsk(sk);
if ((tw->tw_family != family) ||
refcount_read(&twsk_net(tw)->ns.count))
* Avoid using the hashed IP ident generator.
*/
if (sk->sk_protocol == IPPROTO_TCP)
- iph->id = (__force __be16)prandom_u32();
+ iph->id = (__force __be16)get_random_u16();
else
__ip_select_ident(net, iph, 1);
}
flow.flowi4_mark = info->flags & XT_RPFILTER_VALID_MARK ? skb->mark : 0;
flow.flowi4_tos = iph->tos & IPTOS_RT_MASK;
flow.flowi4_scope = RT_SCOPE_UNIVERSE;
- flow.flowi4_oif = l3mdev_master_ifindex_rcu(xt_in(par));
+ flow.flowi4_l3mdev = l3mdev_master_ifindex_rcu(xt_in(par));
return rpfilter_lookup_reverse(xt_net(par), &flow, xt_in(par), info->flags) ^ invert;
}
oif = NULL;
if (priv->flags & NFTA_FIB_F_IIF)
- fl4.flowi4_oif = l3mdev_master_ifindex_rcu(oif);
+ fl4.flowi4_l3mdev = l3mdev_master_ifindex_rcu(oif);
if (nft_hook(pkt) == NF_INET_PRE_ROUTING &&
nft_fib_is_loopback(pkt->skb, nft_in(pkt))) {
{
struct pingfakehdr *pfh = from;
- if (offset == 0) {
- fraglen -= sizeof(struct icmphdr);
- if (fraglen < 0)
- BUG();
- if (!csum_and_copy_from_iter_full(to + sizeof(struct icmphdr),
- fraglen, &pfh->wcheck,
- &pfh->msg->msg_iter))
- return -EFAULT;
- } else if (offset < sizeof(struct icmphdr)) {
- BUG();
- } else {
- if (!csum_and_copy_from_iter_full(to, fraglen, &pfh->wcheck,
- &pfh->msg->msg_iter))
- return -EFAULT;
- }
+ if (!csum_and_copy_from_iter_full(to, fraglen, &pfh->wcheck,
+ &pfh->msg->msg_iter))
+ return -EFAULT;
#if IS_ENABLED(CONFIG_IPV6)
/* For IPv6, checksum each skb as we go along, as expected by
* wcheck, it will be finalized in ping_v4_push_pending_frames.
*/
if (pfh->family == AF_INET6) {
- skb->csum = pfh->wcheck;
+ skb->csum = csum_block_add(skb->csum, pfh->wcheck, odd);
skb->ip_summed = CHECKSUM_NONE;
pfh->wcheck = 0;
}
pfh.family = AF_INET;
err = ip_append_data(sk, &fl4, ping_getfrag, &pfh, len,
- 0, &ipc, &rt, msg->msg_flags);
+ sizeof(struct icmphdr), &ipc, &rt,
+ msg->msg_flags);
if (err)
ip_flush_pending_frames(sk);
else
{
atomic_set(&net->ipv4.rt_genid, 0);
atomic_set(&net->fnhe_genid, 0);
- atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
+ atomic_set(&net->ipv4.dev_addr_genid, get_random_u32());
return 0;
}
ip_idents = idents_hash;
- prandom_bytes(ip_idents, (ip_idents_mask + 1) * sizeof(*ip_idents));
+ get_random_bytes(ip_idents, (ip_idents_mask + 1) * sizeof(*ip_idents));
ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents);
const struct inet_connection_sock *icsk = inet_csk(sk);
if (level != SOL_TCP)
- return icsk->icsk_af_ops->setsockopt(sk, level, optname,
- optval, optlen);
+ /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
+ return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname,
+ optval, optlen);
return do_tcp_setsockopt(sk, level, optname, optval, optlen);
}
EXPORT_SYMBOL(tcp_setsockopt);
struct inet_connection_sock *icsk = inet_csk(sk);
if (level != SOL_TCP)
- return icsk->icsk_af_ops->getsockopt(sk, level, optname,
- optval, optlen);
+ /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
+ return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname,
+ optval, optlen);
return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
USER_SOCKPTR(optlen));
}
struct cdg *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
- if (prandom_u32() <= nexp_u32(grad * backoff_factor))
+ if (get_random_u32() <= nexp_u32(grad * backoff_factor))
return false;
if (use_ineff) {
struct cdg *ca = inet_csk_ca(sk);
struct tcp_sock *tp = tcp_sk(sk);
+ ca->gradients = NULL;
/* We silently fall back to window = 1 if allocation fails. */
if (window > 1)
ca->gradients = kcalloc(window, sizeof(ca->gradients[0]),
struct cdg *ca = inet_csk_ca(sk);
kfree(ca->gradients);
+ ca->gradients = NULL;
}
static struct tcp_congestion_ops tcp_cdg __read_mostly = {
inet->inet_daddr);
}
- inet->inet_id = prandom_u32();
+ inet->inet_id = get_random_u16();
if (tcp_fastopen_defer_connect(sk, &err))
return err;
inet_csk(newsk)->icsk_ext_hdr_len = 0;
if (inet_opt)
inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
- newinet->inet_id = prandom_u32();
+ newinet->inet_id = get_random_u16();
/* Set ToS of the new socket based upon the value of incoming SYN.
* ECT bits are set later in tcp_init_transfer().
struct net *net;
list_for_each_entry(net, net_exit_list, exit_list) {
- /* The last refcount is decremented in tcp_sk_exit_batch() */
- if (refcount_read(&net->ipv4.tcp_death_row.tw_refcount) == 1)
- continue;
-
if (net->ipv4.tcp_death_row.hashinfo->pernet) {
+ /* Even if tw_refcount == 1, we must clean up kernel reqsk */
inet_twsk_purge(net->ipv4.tcp_death_row.hashinfo, family);
} else if (!purged_once) {
+ /* The last refcount is decremented in tcp_sk_exit_batch() */
+ if (refcount_read(&net->ipv4.tcp_death_row.tw_refcount) == 1)
+ continue;
+
inet_twsk_purge(&tcp_hashinfo, family);
purged_once = true;
}
inet_get_local_port_range(net, &low, &high);
remaining = (high - low) + 1;
- rand = prandom_u32();
+ rand = get_random_u32();
first = reciprocal_scale(rand, remaining) + low;
/*
* force rand to be an odd multiple of UDP_HTABLE_SIZE
}
EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
-void udp_destruct_sock(struct sock *sk)
+void udp_destruct_common(struct sock *sk)
{
/* reclaim completely the forward allocated memory */
struct udp_sock *up = udp_sk(sk);
kfree_skb(skb);
}
udp_rmem_release(sk, total, 0, true);
+}
+EXPORT_SYMBOL_GPL(udp_destruct_common);
+static void udp_destruct_sock(struct sock *sk)
+{
+ udp_destruct_common(sk);
inet_sock_destruct(sk);
}
-EXPORT_SYMBOL_GPL(udp_destruct_sock);
int udp_init_sock(struct sock *sk)
{
sk->sk_destruct = udp_destruct_sock;
return 0;
}
-EXPORT_SYMBOL_GPL(udp_init_sock);
void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
{
struct udp_table udplite_table __read_mostly;
EXPORT_SYMBOL(udplite_table);
+/* Designate sk as UDP-Lite socket */
+static int udplite_sk_init(struct sock *sk)
+{
+ udp_init_sock(sk);
+ udp_sk(sk)->pcflag = UDPLITE_BIT;
+ return 0;
+}
+
static int udplite_rcv(struct sk_buff *skb)
{
return __udp4_lib_rcv(skb, &udplite_table, IPPROTO_UDPLITE);
static inline s32 rfc3315_s14_backoff_init(s32 irt)
{
/* multiply 'initial retransmission time' by 0.9 .. 1.1 */
- u64 tmp = (900000 + prandom_u32() % 200001) * (u64)irt;
+ u64 tmp = (900000 + prandom_u32_max(200001)) * (u64)irt;
do_div(tmp, 1000000);
return (s32)tmp;
}
static inline s32 rfc3315_s14_backoff_update(s32 rt, s32 mrt)
{
/* multiply 'retransmission timeout' by 1.9 .. 2.1 */
- u64 tmp = (1900000 + prandom_u32() % 200001) * (u64)rt;
+ u64 tmp = (1900000 + prandom_u32_max(200001)) * (u64)rt;
do_div(tmp, 1000000);
if ((s32)tmp > mrt) {
/* multiply 'maximum retransmission time' by 0.9 .. 1.1 */
- tmp = (900000 + prandom_u32() % 200001) * (u64)mrt;
+ tmp = (900000 + prandom_u32_max(200001)) * (u64)mrt;
do_div(tmp, 1000000);
}
return (s32)tmp;
if (ifp->flags & IFA_F_OPTIMISTIC)
rand_num = 0;
else
- rand_num = prandom_u32() % (idev->cnf.rtr_solicit_delay ? : 1);
+ rand_num = prandom_u32_max(idev->cnf.rtr_solicit_delay ?: 1);
nonce = 0;
if (idev->cnf.enhanced_dad ||
return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
}
+void inet6_sock_destruct(struct sock *sk)
+{
+ inet6_cleanup_sock(sk);
+ inet_sock_destruct(sk);
+}
+
static int inet6_create(struct net *net, struct socket *sock, int protocol,
int kern)
{
inet->hdrincl = 1;
}
- sk->sk_destruct = inet_sock_destruct;
+ sk->sk_destruct = inet6_sock_destruct;
sk->sk_family = PF_INET6;
sk->sk_protocol = protocol;
}
EXPORT_SYMBOL_GPL(inet6_destroy_sock);
+void inet6_cleanup_sock(struct sock *sk)
+{
+ inet6_destroy_sock(sk);
+}
+EXPORT_SYMBOL_GPL(inet6_cleanup_sock);
+
/*
* This does both peername and sockname.
*/
spin_lock_bh(&ip6_fl_lock);
if (label == 0) {
for (;;) {
- fl->label = htonl(prandom_u32())&IPV6_FLOWLABEL_MASK;
+ fl->label = htonl(get_random_u32())&IPV6_FLOWLABEL_MASK;
if (fl->label) {
lfl = __fl_lookup(net, fl->label);
if (!lfl)
rtnl_lock();
sockopt_lock_sock(sk);
+ /* Another thread has converted the socket into IPv4 with
+ * IPV6_ADDRFORM concurrently.
+ */
+ if (unlikely(sk->sk_family != AF_INET6))
+ goto unlock;
+
switch (optname) {
case IPV6_ADDRFORM:
if (optlen < sizeof(int))
goto e_inval;
if (val == PF_INET) {
- struct ipv6_txoptions *opt;
- struct sk_buff *pktopt;
-
if (sk->sk_type == SOCK_RAW)
break;
break;
}
- fl6_free_socklist(sk);
__ipv6_sock_mc_close(sk);
__ipv6_sock_ac_close(sk);
sock_prot_inuse_add(net, sk->sk_prot, -1);
sock_prot_inuse_add(net, &tcp_prot, 1);
- /* Paired with READ_ONCE(sk->sk_prot) in net/ipv6/af_inet6.c */
+ /* Paired with READ_ONCE(sk->sk_prot) in inet6_stream_ops */
WRITE_ONCE(sk->sk_prot, &tcp_prot);
- icsk->icsk_af_ops = &ipv4_specific;
+ /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
+ WRITE_ONCE(icsk->icsk_af_ops, &ipv4_specific);
sk->sk_socket->ops = &inet_stream_ops;
sk->sk_family = PF_INET;
tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
sock_prot_inuse_add(net, sk->sk_prot, -1);
sock_prot_inuse_add(net, prot, 1);
- /* Paired with READ_ONCE(sk->sk_prot) in net/ipv6/af_inet6.c */
+ /* Paired with READ_ONCE(sk->sk_prot) in inet6_dgram_ops */
WRITE_ONCE(sk->sk_prot, prot);
sk->sk_socket->ops = &inet_dgram_ops;
sk->sk_family = PF_INET;
}
- opt = xchg((__force struct ipv6_txoptions **)&np->opt,
- NULL);
- if (opt) {
- atomic_sub(opt->tot_len, &sk->sk_omem_alloc);
- txopt_put(opt);
- }
- pktopt = xchg(&np->pktoptions, NULL);
- kfree_skb(pktopt);
+
+ /* Disable all options not to allocate memory anymore,
+ * but there is still a race. See the lockless path
+ * in udpv6_sendmsg() and ipv6_local_rxpmtu().
+ */
+ np->rxopt.all = 0;
+
+ inet6_cleanup_sock(sk);
/*
* ... and add it to the refcnt debug socks count
break;
}
+unlock:
sockopt_release_sock(sk);
if (needs_rtnl)
rtnl_unlock();
/* called with mc_lock */
static void mld_gq_start_work(struct inet6_dev *idev)
{
- unsigned long tv = prandom_u32() % idev->mc_maxdelay;
+ unsigned long tv = prandom_u32_max(idev->mc_maxdelay);
idev->mc_gq_running = 1;
if (!mod_delayed_work(mld_wq, &idev->mc_gq_work, tv + 2))
/* called with mc_lock */
static void mld_ifc_start_work(struct inet6_dev *idev, unsigned long delay)
{
- unsigned long tv = prandom_u32() % delay;
+ unsigned long tv = prandom_u32_max(delay);
if (!mod_delayed_work(mld_wq, &idev->mc_ifc_work, tv + 2))
in6_dev_hold(idev);
/* called with mc_lock */
static void mld_dad_start_work(struct inet6_dev *idev, unsigned long delay)
{
- unsigned long tv = prandom_u32() % delay;
+ unsigned long tv = prandom_u32_max(delay);
if (!mod_delayed_work(mld_wq, &idev->mc_dad_work, tv + 2))
in6_dev_hold(idev);
}
if (delay >= resptime)
- delay = prandom_u32() % resptime;
+ delay = prandom_u32_max(resptime);
if (!mod_delayed_work(mld_wq, &ma->mca_work, delay))
refcount_inc(&ma->mca_refcnt);
igmp6_send(&ma->mca_addr, ma->idev->dev, ICMPV6_MGM_REPORT);
- delay = prandom_u32() % unsolicited_report_interval(ma->idev);
+ delay = prandom_u32_max(unsolicited_report_interval(ma->idev));
if (cancel_delayed_work(&ma->mca_work)) {
refcount_dec(&ma->mca_refcnt);
bool ret = false;
struct flowi6 fl6 = {
.flowi6_iif = LOOPBACK_IFINDEX,
+ .flowi6_l3mdev = l3mdev_master_ifindex_rcu(dev),
.flowlabel = (* (__be32 *) iph) & IPV6_FLOWINFO_MASK,
.flowi6_proto = iph->nexthdr,
.daddr = iph->saddr,
if (rpfilter_addr_linklocal(&iph->saddr)) {
lookup_flags |= RT6_LOOKUP_F_IFACE;
fl6.flowi6_oif = dev->ifindex;
- /* Set flowi6_oif for vrf devices to lookup route in l3mdev domain. */
- } else if (netif_is_l3_master(dev) || netif_is_l3_slave(dev) ||
- (flags & XT_RPFILTER_LOOSE) == 0)
+ } else if ((flags & XT_RPFILTER_LOOSE) == 0)
fl6.flowi6_oif = dev->ifindex;
rt = (void *)ip6_route_lookup(net, &fl6, skb, lookup_flags);
goto out;
}
- if (rt->rt6i_idev->dev == dev ||
- l3mdev_master_ifindex_rcu(rt->rt6i_idev->dev) == dev->ifindex ||
- (flags & XT_RPFILTER_LOOSE))
+ if (rt->rt6i_idev->dev == dev || (flags & XT_RPFILTER_LOOSE))
ret = true;
out:
ip6_rt_put(rt);
if (ipv6_addr_type(&fl6->daddr) & IPV6_ADDR_LINKLOCAL) {
lookup_flags |= RT6_LOOKUP_F_IFACE;
fl6->flowi6_oif = get_ifindex(dev ? dev : pkt->skb->dev);
- } else if ((priv->flags & NFTA_FIB_F_IIF) &&
- (netif_is_l3_master(dev) || netif_is_l3_slave(dev))) {
- fl6->flowi6_oif = dev->ifindex;
+ } else if (priv->flags & NFTA_FIB_F_IIF) {
+ fl6->flowi6_l3mdev = l3mdev_master_ifindex_rcu(dev);
}
if (ipv6_addr_type(&fl6->saddr) & IPV6_ADDR_UNICAST)
u32 id;
do {
- id = prandom_u32();
+ id = get_random_u32();
} while (!id);
return id;
lock_sock(sk);
err = ip6_append_data(sk, ping_getfrag, &pfh, len,
- 0, &ipc6, &fl6, rt,
+ sizeof(struct icmp6hdr), &ipc6, &fl6, rt,
MSG_DONTWAIT);
if (err) {
sin.sin_port = usin->sin6_port;
sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
- icsk->icsk_af_ops = &ipv6_mapped;
+ /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
+ WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
if (sk_is_mptcp(sk))
mptcpv6_handle_mapped(sk, true);
sk->sk_backlog_rcv = tcp_v4_do_rcv;
if (err) {
icsk->icsk_ext_hdr_len = exthdrlen;
- icsk->icsk_af_ops = &ipv6_specific;
+ /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
+ WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
if (sk_is_mptcp(sk))
mptcpv6_handle_mapped(sk, false);
sk->sk_backlog_rcv = tcp_v6_do_rcv;
#include <trace/events/skb.h>
#include "udp_impl.h"
+static void udpv6_destruct_sock(struct sock *sk)
+{
+ udp_destruct_common(sk);
+ inet6_sock_destruct(sk);
+}
+
+int udpv6_init_sock(struct sock *sk)
+{
+ skb_queue_head_init(&udp_sk(sk)->reader_queue);
+ sk->sk_destruct = udpv6_destruct_sock;
+ return 0;
+}
+
static u32 udp6_ehashfn(const struct net *net,
const struct in6_addr *laddr,
const u16 lport,
.connect = ip6_datagram_connect,
.disconnect = udp_disconnect,
.ioctl = udp_ioctl,
- .init = udp_init_sock,
+ .init = udpv6_init_sock,
.destroy = udpv6_destroy_sock,
.setsockopt = udpv6_setsockopt,
.getsockopt = udpv6_getsockopt,
int __udp6_lib_err(struct sk_buff *, struct inet6_skb_parm *, u8, u8, int,
__be32, struct udp_table *);
+int udpv6_init_sock(struct sock *sk);
int udp_v6_get_port(struct sock *sk, unsigned short snum);
void udp_v6_rehash(struct sock *sk);
#include <linux/proc_fs.h>
#include "udp_impl.h"
+static int udplitev6_sk_init(struct sock *sk)
+{
+ udpv6_init_sock(sk);
+ udp_sk(sk)->pcflag = UDPLITE_BIT;
+ return 0;
+}
+
static int udplitev6_rcv(struct sk_buff *skb)
{
return __udp6_lib_rcv(skb, &udplite_table, IPPROTO_UDPLITE);
.connect = ip6_datagram_connect,
.disconnect = udp_disconnect,
.ioctl = udp_ioctl,
- .init = udplite_sk_init,
+ .init = udplitev6_sk_init,
.destroy = udpv6_destroy_sock,
.setsockopt = udpv6_setsockopt,
.getsockopt = udpv6_getsockopt,
kcm = kcm_sk(sk);
mux = kcm->mux;
+ lock_sock(sk);
sock_orphan(sk);
kfree_skb(kcm->seq_skb);
- lock_sock(sk);
/* Purge queue under lock to avoid race condition with tx_work trying
* to act when queue is nonempty. If tx_work runs after this point
* it will just return.
/* whether a parse error occurred while retrieving these elements */
bool parse_error;
+
+ /*
+ * scratch buffer that can be used for various element parsing related
+ * tasks, e.g., element de-fragmentation etc.
+ */
+ size_t scratch_len;
+ u8 *scratch_pos;
+ u8 scratch[];
};
static inline struct ieee80211_local *hw_to_local(
*/
break;
default:
- return -EOPNOTSUPP;
+ ret = -EOPNOTSUPP;
}
unlock:
/*
* Stop TX on this interface first.
*/
- if (sdata->dev)
+ if (!local->ops->wake_tx_queue && sdata->dev)
netif_tx_stop_all_queues(sdata->dev);
ieee80211_roc_purge(local, sdata);
sdata->vif.type != NL80211_IFTYPE_STATION);
}
- set_bit(SDATA_STATE_RUNNING, &sdata->state);
-
switch (sdata->vif.type) {
case NL80211_IFTYPE_P2P_DEVICE:
rcu_assign_pointer(local->p2p_sdata, sdata);
spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
}
+ set_bit(SDATA_STATE_RUNNING, &sdata->state);
+
return 0;
err_del_interface:
drv_remove_interface(local, sdata);
he_cap_elem = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_CAPABILITY,
ies->data, ies->len);
+ if (!he_cap_elem)
+ return false;
+
/* invalid HE IE */
- if (!he_cap_elem || he_cap_elem->datalen < 1 + sizeof(*he_cap)) {
+ if (he_cap_elem->datalen < 1 + sizeof(*he_cap)) {
sdata_info(sdata,
"Invalid HE elem, Disable HE\n");
return false;
}
if (!elems->vht_cap_elem) {
- sdata_info(sdata,
- "bad VHT capabilities, disabling VHT\n");
*conn_flags |= IEEE80211_CONN_DISABLE_VHT;
vht_oper = NULL;
}
memset(sample_table, 0xff, sizeof(sample_table));
for (col = 0; col < SAMPLE_COLUMNS; col++) {
- prandom_bytes(rnd, sizeof(rnd));
+ get_random_bytes(rnd, sizeof(rnd));
for (i = 0; i < MCS_GROUP_RATES; i++) {
new_idx = (i + rnd[i]) % MCS_GROUP_RATES;
while (sample_table[col][new_idx] != 0xff)
if (mmie_keyidx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS ||
mmie_keyidx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS +
- NUM_DEFAULT_BEACON_KEYS) {
- cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
- skb->data,
- skb->len);
+ NUM_DEFAULT_BEACON_KEYS) {
+ if (rx->sdata->dev)
+ cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
+ skb->data,
+ skb->len);
return RX_DROP_MONITOR; /* unexpected BIP keyidx */
}
/* either the frame has been decrypted or will be dropped */
status->flag |= RX_FLAG_DECRYPTED;
- if (unlikely(ieee80211_is_beacon(fc) && result == RX_DROP_UNUSABLE))
+ if (unlikely(ieee80211_is_beacon(fc) && result == RX_DROP_UNUSABLE &&
+ rx->sdata->dev))
cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
skb->data, skb->len);
.vif_type = sdata->vif.type,
.control_port_protocol = sdata->control_port_protocol,
}, *old, *new = NULL;
+ u32 offload_flags;
bool set_offload = false;
bool assign = false;
bool offload;
if (assign)
new = kmemdup(&fastrx, sizeof(fastrx), GFP_KERNEL);
- offload = assign &&
- (sdata->vif.offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED);
+ offload_flags = get_bss_sdata(sdata)->vif.offload_flags;
+ offload = offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED;
- if (offload)
+ if (assign && offload)
set_offload = !test_and_set_sta_flag(sta, WLAN_STA_DECAP_OFFLOAD);
else
set_offload = test_and_clear_sta_flag(sta, WLAN_STA_DECAP_OFFLOAD);
if (!(status->rx_flags & IEEE80211_RX_AMSDU)) {
if (!pskb_may_pull(skb, snap_offs + sizeof(*payload)))
- goto drop;
+ return false;
payload = (void *)(skb->data + snap_offs);
if (flags & IEEE80211_PROBE_FLAG_RANDOM_SN) {
struct ieee80211_hdr *hdr = (void *)skb->data;
struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
- u16 sn = get_random_u32();
+ u16 sn = get_random_u16();
info->control.flags |= IEEE80211_TX_CTRL_NO_SEQNO;
hdr->seq_ctrl =
u16 len_rthdr;
int hdrlen;
+ sdata = IEEE80211_DEV_TO_SUB_IF(dev);
+ if (unlikely(!ieee80211_sdata_running(sdata)))
+ goto fail;
+
memset(info, 0, sizeof(*info));
info->flags = IEEE80211_TX_CTL_REQ_TX_STATUS |
IEEE80211_TX_CTL_INJECTED;
* This is necessary, for example, for old hostapd versions that
* don't use nl80211-based management TX/RX.
*/
- sdata = IEEE80211_DEV_TO_SUB_IF(dev);
-
list_for_each_entry_rcu(tmp_sdata, &local->interfaces, list) {
if (!ieee80211_sdata_running(tmp_sdata))
continue;
struct sk_buff *next;
int len = skb->len;
- if (unlikely(skb->len < ETH_HLEN)) {
+ if (unlikely(!ieee80211_sdata_running(sdata) || skb->len < ETH_HLEN)) {
kfree_skb(skb);
return;
}
struct ieee80211_key *key;
struct sta_info *sta;
- if (unlikely(skb->len < ETH_HLEN)) {
+ if (unlikely(!ieee80211_sdata_running(sdata) || skb->len < ETH_HLEN)) {
kfree_skb(skb);
return NETDEV_TX_OK;
}
for_each_element_id(elem, WLAN_EID_MULTIPLE_BSSID, start, len) {
if (elem->datalen < 2)
continue;
+ if (elem->data[0] < 1 || elem->data[0] > 8)
+ continue;
for_each_element(sub, elem->data + 1, elem->datalen - 1) {
u8 new_bssid[ETH_ALEN];
const struct element *non_inherit = NULL;
u8 *nontransmitted_profile;
int nontransmitted_profile_len = 0;
+ size_t scratch_len = params->len;
- elems = kzalloc(sizeof(*elems), GFP_ATOMIC);
+ elems = kzalloc(sizeof(*elems) + scratch_len, GFP_ATOMIC);
if (!elems)
return NULL;
elems->ie_start = params->start;
elems->total_len = params->len;
-
- nontransmitted_profile = kmalloc(params->len, GFP_ATOMIC);
- if (nontransmitted_profile) {
- nontransmitted_profile_len =
- ieee802_11_find_bssid_profile(params->start, params->len,
- elems, params->bss,
- nontransmitted_profile);
- non_inherit =
- cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
- nontransmitted_profile,
- nontransmitted_profile_len);
- }
+ elems->scratch_len = scratch_len;
+ elems->scratch_pos = elems->scratch;
+
+ nontransmitted_profile = elems->scratch_pos;
+ nontransmitted_profile_len =
+ ieee802_11_find_bssid_profile(params->start, params->len,
+ elems, params->bss,
+ nontransmitted_profile);
+ elems->scratch_pos += nontransmitted_profile_len;
+ elems->scratch_len -= nontransmitted_profile_len;
+ non_inherit = cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
+ nontransmitted_profile,
+ nontransmitted_profile_len);
elems->crc = _ieee802_11_parse_elems_full(params, elems, non_inherit);
offsetofend(struct ieee80211_bssid_index, dtim_count))
elems->dtim_count = elems->bssid_index->dtim_count;
- kfree(nontransmitted_profile);
-
return elems;
}
if (he_cap) {
enum nl80211_iftype iftype =
ieee80211_vif_type_p2p(&sdata->vif);
- __le16 cap = ieee80211_get_he_6ghz_capa(sband, iftype);
+ __le16 cap = ieee80211_get_he_6ghz_capa(sband6, iftype);
pos = ieee80211_write_he_6ghz_cap(pos, cap, end);
}
mctp_dev_release_key(key->dev, key);
spin_unlock_irqrestore(&key->lock, flags);
- hlist_del(&key->hlist);
- hlist_del(&key->sklist);
-
- /* unref for the lists */
- mctp_key_unref(key);
+ if (!hlist_unhashed(&key->hlist)) {
+ hlist_del_init(&key->hlist);
+ hlist_del_init(&key->sklist);
+ /* unref for the lists */
+ mctp_key_unref(key);
+ }
kfree_skb(skb);
}
ctl.tag = tag | MCTP_TAG_OWNER | MCTP_TAG_PREALLOC;
if (copy_to_user((void __user *)arg, &ctl, sizeof(ctl))) {
- spin_lock_irqsave(&key->lock, flags);
- __mctp_key_remove(key, net, flags, MCTP_TRACE_KEY_DROPPED);
+ unsigned long fl2;
+ /* Unwind our key allocation: the keys list lock needs to be
+ * taken before the individual key locks, and we need a valid
+ * flags value (fl2) to pass to __mctp_key_remove, hence the
+ * second spin_lock_irqsave() rather than a plain spin_lock().
+ */
+ spin_lock_irqsave(&net->mctp.keys_lock, flags);
+ spin_lock_irqsave(&key->lock, fl2);
+ __mctp_key_remove(key, net, fl2, MCTP_TRACE_KEY_DROPPED);
mctp_key_unref(key);
+ spin_unlock_irqrestore(&net->mctp.keys_lock, flags);
return -EFAULT;
}
if (!key->manual_alloc) {
spin_lock_irqsave(&net->mctp.keys_lock, flags);
- hlist_del(&key->hlist);
- hlist_del(&key->sklist);
+ if (!hlist_unhashed(&key->hlist)) {
+ hlist_del_init(&key->hlist);
+ hlist_del_init(&key->sklist);
+ mctp_key_unref(key);
+ }
spin_unlock_irqrestore(&net->mctp.keys_lock, flags);
-
- /* unref for the lists */
- mctp_key_unref(key);
}
/* and one for the local reference */
* Randomly scan 1/32 of the whole table every second
*/
for (idx = 0; idx < (ip_vs_conn_tab_size>>5); idx++) {
- unsigned int hash = prandom_u32() & ip_vs_conn_tab_mask;
+ unsigned int hash = get_random_u32() & ip_vs_conn_tab_mask;
hlist_for_each_entry_rcu(cp, &ip_vs_conn_tab[hash], c_list) {
if (cp->ipvs != ipvs)
* from 0 to total_weight
*/
total_weight += 1;
- rweight1 = prandom_u32() % total_weight;
- rweight2 = prandom_u32() % total_weight;
+ rweight1 = prandom_u32_max(total_weight);
+ rweight2 = prandom_u32_max(total_weight);
/* Pick two weighted servers */
list_for_each_entry_rcu(dest, &svc->destinations, n_list) {
if (range->flags & NF_NAT_RANGE_PROTO_OFFSET)
off = (ntohs(*keyptr) - ntohs(range->base_proto.all));
else
- off = prandom_u32();
+ off = get_random_u16();
attempts = range_size;
if (attempts > max_attempts)
if (attempts >= range_size || attempts < 16)
return;
attempts /= 2;
- off = prandom_u32();
+ off = get_random_u16();
goto another_round;
}
switch (info->mode) {
case XT_STATISTIC_MODE_RANDOM:
- if ((prandom_u32() & 0x7FFFFFFF) < info->u.random.probability)
+ if ((get_random_u32() & 0x7FFFFFFF) < info->u.random.probability)
ret = !ret;
break;
case XT_STATISTIC_MODE_NTH:
actions = nla_next(sample_arg, &rem);
if ((arg->probability != U32_MAX) &&
- (!arg->probability || prandom_u32() > arg->probability)) {
+ (!arg->probability || get_random_u32() > arg->probability)) {
if (last)
consume_skb(skb);
return 0;
* connections which we will commit, we may need to attach
* the helper here.
*/
- if (info->commit && info->helper && !nfct_help(ct)) {
+ if (!nf_ct_is_confirmed(ct) && info->commit &&
+ info->helper && !nfct_help(ct)) {
int err = __nf_ct_try_assign_helper(ct, info->ct,
GFP_ATOMIC);
if (err)
if (READ_ONCE(history[i]) == rxhash)
count++;
- victim = prandom_u32() % ROLLOVER_HLEN;
+ victim = prandom_u32_max(ROLLOVER_HLEN);
/* Avoid dirtying the cache line if possible */
if (READ_ONCE(history[victim]) != rxhash)
return -EINVAL;
last = rover;
} else {
- rover = max_t(u16, prandom_u32(), 2);
+ rover = max_t(u16, get_random_u16(), 2);
last = rover - 1;
}
static int gact_net_rand(struct tcf_gact *gact)
{
smp_rmb(); /* coupled with smp_wmb() in tcf_gact_init() */
- if (prandom_u32() % gact->tcfg_pval)
+ if (prandom_u32_max(gact->tcfg_pval))
return gact->tcf_action;
return gact->tcfg_paction;
}
psample_group = rcu_dereference_bh(s->psample_group);
/* randomly sample packets according to rate */
- if (psample_group && (prandom_u32() % s->rate == 0)) {
+ if (psample_group && (prandom_u32_max(s->rate) == 0)) {
if (!skb_at_tc_ingress(skb)) {
md.in_ifindex = skb->skb_iif;
md.out_ifindex = skb->dev->ifindex;
/* Simple BLUE implementation. Lack of ECN is deliberate. */
if (vars->p_drop)
- drop |= (prandom_u32() < vars->p_drop);
+ drop |= (get_random_u32() < vars->p_drop);
/* Overload the drop_next field as an activity timeout */
if (!vars->count)
WARN_ON(host_load > CAKE_QUEUES);
- /* The shifted prandom_u32() is a way to apply dithering to
- * avoid accumulating roundoff errors
+ /* The get_random_u16() is a way to apply dithering to avoid
+ * accumulating roundoff errors
*/
flow->deficit += (b->flow_quantum * quantum_div[host_load] +
- (prandom_u32() >> 16)) >> 16;
+ get_random_u16()) >> 16;
list_move_tail(&flow->flowchain, &b->old_flows);
goto retry;
static void init_crandom(struct crndstate *state, unsigned long rho)
{
state->rho = rho;
- state->last = prandom_u32();
+ state->last = get_random_u32();
}
/* get_crandom - correlated random number generator
unsigned long answer;
if (!state || state->rho == 0) /* no correlation */
- return prandom_u32();
+ return get_random_u32();
- value = prandom_u32();
+ value = get_random_u32();
rho = (u64)state->rho + 1;
answer = (value * ((1ull<<32) - rho) + state->last * rho) >> 32;
state->last = answer;
static bool loss_4state(struct netem_sched_data *q)
{
struct clgstate *clg = &q->clg;
- u32 rnd = prandom_u32();
+ u32 rnd = get_random_u32();
/*
* Makes a comparison between rnd and the transition
switch (clg->state) {
case GOOD_STATE:
- if (prandom_u32() < clg->a1)
+ if (get_random_u32() < clg->a1)
clg->state = BAD_STATE;
- if (prandom_u32() < clg->a4)
+ if (get_random_u32() < clg->a4)
return true;
break;
case BAD_STATE:
- if (prandom_u32() < clg->a2)
+ if (get_random_u32() < clg->a2)
clg->state = GOOD_STATE;
- if (prandom_u32() > clg->a3)
+ if (get_random_u32() > clg->a3)
return true;
}
goto finish_segs;
}
- skb->data[prandom_u32() % skb_headlen(skb)] ^=
- 1<<(prandom_u32() % 8);
+ skb->data[prandom_u32_max(skb_headlen(skb))] ^=
+ 1<<prandom_u32_max(8);
}
if (unlikely(sch->q.qlen >= sch->limit)) {
if (!q->slot_dist)
next_delay = q->slot_config.min_delay +
- (prandom_u32() *
+ (get_random_u32() *
(q->slot_config.max_delay -
q->slot_config.min_delay) >> 32);
else
if (vars->accu_prob >= (MAX_PROB / 2) * 17)
return true;
- prandom_bytes(&rnd, 8);
+ get_random_bytes(&rnd, 8);
if ((rnd >> BITS_PER_BYTE) < local_prob) {
vars->accu_prob = 0;
return true;
goto enqueue;
}
- r = prandom_u32() & SFB_MAX_PROB;
+ r = get_random_u16() & SFB_MAX_PROB;
if (unlikely(r < p_min)) {
if (unlikely(p_min > SFB_MAX_PROB / 2)) {
static struct Qdisc *taprio_leaf(struct Qdisc *sch, unsigned long cl)
{
- struct taprio_sched *q = qdisc_priv(sch);
- struct net_device *dev = qdisc_dev(sch);
- unsigned int ntx = cl - 1;
+ struct netdev_queue *dev_queue = taprio_queue_get(sch, cl);
- if (ntx >= dev->num_tx_queues)
+ if (!dev_queue)
return NULL;
- return q->qdiscs[ntx];
+ return dev_queue->qdisc_sleeping;
}
static unsigned long taprio_find(struct Qdisc *sch, u32 classid)
inet_get_local_port_range(net, &low, &high);
remaining = (high - low) + 1;
- rover = prandom_u32() % remaining + low;
+ rover = prandom_u32_max(remaining) + low;
do {
rover++;
newinet->inet_rcv_saddr = inet->inet_rcv_saddr;
newinet->inet_dport = htons(asoc->peer.port);
newinet->pmtudisc = inet->pmtudisc;
- newinet->inet_id = prandom_u32();
+ newinet->inet_id = get_random_u16();
newinet->uc_ttl = inet->uc_ttl;
newinet->mc_loop = 1;
/* initialize to random value */
if (i == 0) {
- i = prandom_u32();
- i = (i << 32) | prandom_u32();
+ i = get_random_u32();
+ i = (i << 32) | get_random_u32();
}
switch (conflen) {
/* Consider removing either the first or the last */
if (cache_defer_cnt > DFR_MAX) {
- if (prandom_u32() & 1)
+ if (prandom_u32_max(2))
discard = list_entry(cache_defer_list.next,
struct cache_deferred_req, recent);
else
{
int clid;
- clid = ida_simple_get(&rpc_clids, 0, 0, GFP_KERNEL);
+ clid = ida_alloc(&rpc_clids, GFP_KERNEL);
if (clid < 0)
return clid;
clnt->cl_clid = clid;
static void rpc_free_clid(struct rpc_clnt *clnt)
{
- ida_simple_remove(&rpc_clids, clnt->cl_clid);
+ ida_free(&rpc_clids, clnt->cl_clid);
}
static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
}
EXPORT_SYMBOL_GPL(rpc_killall_tasks);
+/**
+ * rpc_cancel_tasks - try to cancel a set of RPC tasks
+ * @clnt: Pointer to RPC client
+ * @error: RPC task error value to set
+ * @fnmatch: Pointer to selector function
+ * @data: User data
+ *
+ * Uses @fnmatch to define a set of RPC tasks that are to be cancelled.
+ * The argument @error must be a negative error value.
+ */
+unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
+ bool (*fnmatch)(const struct rpc_task *,
+ const void *),
+ const void *data)
+{
+ struct rpc_task *task;
+ unsigned long count = 0;
+
+ if (list_empty(&clnt->cl_tasks))
+ return 0;
+ /*
+ * Spin lock all_tasks to prevent changes...
+ */
+ spin_lock(&clnt->cl_lock);
+ list_for_each_entry(task, &clnt->cl_tasks, tk_task) {
+ if (!RPC_IS_ACTIVATED(task))
+ continue;
+ if (!fnmatch(task, data))
+ continue;
+ rpc_task_try_cancel(task, error);
+ count++;
+ }
+ spin_unlock(&clnt->cl_lock);
+ return count;
+}
+EXPORT_SYMBOL_GPL(rpc_cancel_tasks);
+
+static int rpc_clnt_disconnect_xprt(struct rpc_clnt *clnt,
+ struct rpc_xprt *xprt, void *dummy)
+{
+ if (xprt_connected(xprt))
+ xprt_force_disconnect(xprt);
+ return 0;
+}
+
+void rpc_clnt_disconnect(struct rpc_clnt *clnt)
+{
+ rpc_clnt_iterate_for_each_xprt(clnt, rpc_clnt_disconnect_xprt, NULL);
+}
+EXPORT_SYMBOL_GPL(rpc_clnt_disconnect);
+
/*
* Properly shut down an RPC client, terminating all outstanding
* requests.
__rpc_call_rpcerror(struct rpc_task *task, int tk_status, int rpc_status)
{
trace_rpc_call_rpcerror(task, tk_status, rpc_status);
- task->tk_rpc_status = rpc_status;
+ rpc_task_set_rpc_status(task, rpc_status);
rpc_exit(task, tk_status);
}
{
struct rpc_clnt *clnt = task->tk_client;
- if (RPC_SIGNALLED(task)) {
- rpc_call_rpcerror(task, -ERESTARTSYS);
+ if (RPC_SIGNALLED(task))
return;
- }
if (xprt_adjust_timeout(task->tk_rqstp) == 0)
return;
}
EXPORT_SYMBOL_GPL(rpc_task_gfp_mask);
+bool rpc_task_set_rpc_status(struct rpc_task *task, int rpc_status)
+{
+ if (cmpxchg(&task->tk_rpc_status, 0, rpc_status) == 0)
+ return true;
+ return false;
+}
+
unsigned long
rpc_task_timeout(const struct rpc_task *task)
{
if (!RPC_IS_ACTIVATED(task))
return;
+ if (!rpc_task_set_rpc_status(task, -ERESTARTSYS))
+ return;
trace_rpc_task_signalled(task, task->tk_action);
set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate);
smp_mb__after_atomic();
queue = READ_ONCE(task->tk_waitqueue);
if (queue)
- rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS);
+ rpc_wake_up_queued_task(queue, task);
+}
+
+void rpc_task_try_cancel(struct rpc_task *task, int error)
+{
+ struct rpc_wait_queue *queue;
+
+ if (!rpc_task_set_rpc_status(task, error))
+ return;
+ queue = READ_ONCE(task->tk_waitqueue);
+ if (queue)
+ rpc_wake_up_queued_task(queue, task);
}
void rpc_exit(struct rpc_task *task, int status)
* Perform the next FSM step or a pending callback.
*
* tk_action may be NULL if the task has been killed.
- * In particular, note that rpc_killall_tasks may
- * do this at any time, so beware when dereferencing.
*/
do_action = task->tk_action;
+ /* Tasks with an RPC error status should exit */
+ if (do_action != rpc_exit_task &&
+ (status = READ_ONCE(task->tk_rpc_status)) != 0) {
+ task->tk_status = status;
+ if (do_action != NULL)
+ do_action = rpc_exit_task;
+ }
+ /* Callbacks override all actions */
if (task->tk_callback) {
do_action = task->tk_callback;
task->tk_callback = NULL;
continue;
}
- /*
- * Signalled tasks should exit rather than sleep.
- */
- if (RPC_SIGNALLED(task)) {
- task->tk_rpc_status = -ERESTARTSYS;
- rpc_exit(task, -ERESTARTSYS);
- }
-
/*
* The queue->lock protects against races with
* rpc_make_runnable().
spin_unlock(&queue->lock);
continue;
}
+ /* Wake up any task that has an exit status */
+ if (READ_ONCE(task->tk_rpc_status) != 0) {
+ rpc_wake_up_task_queue_locked(queue, task);
+ spin_unlock(&queue->lock);
+ continue;
+ }
rpc_clear_running(task);
spin_unlock(&queue->lock);
if (task_is_async)
* clean up after sleeping on some queue, we don't
* break the loop here, but go around once more.
*/
- trace_rpc_task_signalled(task, task->tk_action);
- set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate);
- task->tk_rpc_status = -ERESTARTSYS;
- rpc_exit(task, -ERESTARTSYS);
+ rpc_signal_task(task);
}
trace_rpc_task_sync_wake(task, task->tk_action);
}
{
int id;
- id = ida_simple_get(&rpc_xprt_ids, 0, 0, GFP_KERNEL);
+ id = ida_alloc(&rpc_xprt_ids, GFP_KERNEL);
if (id < 0)
return id;
static void xprt_free_id(struct rpc_xprt *xprt)
{
- ida_simple_remove(&rpc_xprt_ids, xprt->id);
+ ida_free(&rpc_xprt_ids, xprt->id);
}
struct rpc_xprt *xprt_alloc(struct net *net, size_t size,
goto out_free;
list_add(&req->rq_list, &xprt->free);
}
- if (max_alloc > num_prealloc)
- xprt->max_reqs = max_alloc;
- else
- xprt->max_reqs = num_prealloc;
+ xprt->max_reqs = max_t(unsigned int, max_alloc, num_prealloc);
xprt->min_reqs = num_prealloc;
xprt->num_reqs = num_prealloc;
static void
xprt_init_xid(struct rpc_xprt *xprt)
{
- xprt->xid = prandom_u32();
+ xprt->xid = get_random_u32();
}
static void
{
int id;
- id = ida_simple_get(&rpc_xprtswitch_ids, 0, 0, gfp_flags);
+ id = ida_alloc(&rpc_xprtswitch_ids, gfp_flags);
if (id < 0)
return id;
static void xprt_switch_free_id(struct rpc_xprt_switch *xps)
{
- ida_simple_remove(&rpc_xprtswitch_ids, xps->xps_id);
+ ida_free(&rpc_xprtswitch_ids, xps->xps_id);
}
/**
return NULL;
size = min_t(size_t, r_xprt->rx_ep->re_inline_recv, PAGE_SIZE);
- req = rpcrdma_req_create(r_xprt, size, GFP_KERNEL);
+ req = rpcrdma_req_create(r_xprt, size);
if (!req)
return NULL;
if (rpcrdma_req_setup(r_xprt, req)) {
unsigned int depth = ep->re_max_fr_depth;
struct scatterlist *sg;
struct ib_mr *frmr;
- int rc;
+
+ sg = kcalloc_node(depth, sizeof(*sg), XPRTRDMA_GFP_FLAGS,
+ ibdev_to_node(ep->re_id->device));
+ if (!sg)
+ return -ENOMEM;
frmr = ib_alloc_mr(ep->re_pd, ep->re_mrtype, depth);
if (IS_ERR(frmr))
goto out_mr_err;
- sg = kmalloc_array(depth, sizeof(*sg), GFP_KERNEL);
- if (!sg)
- goto out_list_err;
-
mr->mr_xprt = r_xprt;
mr->mr_ibmr = frmr;
mr->mr_device = NULL;
return 0;
out_mr_err:
- rc = PTR_ERR(frmr);
- trace_xprtrdma_frwr_alloc(mr, rc);
- return rc;
-
-out_list_err:
- ib_dereg_mr(frmr);
- return -ENOMEM;
+ kfree(sg);
+ trace_xprtrdma_frwr_alloc(mr, PTR_ERR(frmr));
+ return PTR_ERR(frmr);
}
/**
return -EINVAL;
}
- page = alloc_page(RPCRDMA_DEF_GFP);
+ page = alloc_page(GFP_NOIO | __GFP_NOWARN);
if (!page)
return -ENOMEM;
rqst->rq_buffer = page_address(page);
- rqst->rq_rbuffer = kmalloc(rqst->rq_rcvsize, RPCRDMA_DEF_GFP);
+ rqst->rq_rbuffer = kmalloc(rqst->rq_rcvsize, GFP_NOIO | __GFP_NOWARN);
if (!rqst->rq_rbuffer) {
put_page(page);
return -ENOMEM;
xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO);
}
trace_xprtrdma_op_connect(r_xprt, delay);
- queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker,
- delay);
+ queue_delayed_work(system_long_wq, &r_xprt->rx_connect_worker, delay);
}
/**
static void rpcrdma_ep_get(struct rpcrdma_ep *ep);
static int rpcrdma_ep_put(struct rpcrdma_ep *ep);
static struct rpcrdma_regbuf *
-rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
- gfp_t flags);
+rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction);
static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
struct rpcrdma_ep *ep;
int rc;
- ep = kzalloc(sizeof(*ep), GFP_KERNEL);
+ ep = kzalloc(sizeof(*ep), XPRTRDMA_GFP_FLAGS);
if (!ep)
return -ENOTCONN;
ep->re_xprt = &r_xprt->rx_xprt;
struct rpcrdma_sendctx *sc;
sc = kzalloc(struct_size(sc, sc_sges, ep->re_attr.cap.max_send_sge),
- GFP_KERNEL);
+ XPRTRDMA_GFP_FLAGS);
if (!sc)
return NULL;
* Sends are posted.
*/
i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS;
- buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), GFP_KERNEL);
+ buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), XPRTRDMA_GFP_FLAGS);
if (!buf->rb_sc_ctxs)
return -ENOMEM;
{
struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
struct rpcrdma_ep *ep = r_xprt->rx_ep;
+ struct ib_device *device = ep->re_id->device;
unsigned int count;
+ /* Try to allocate enough to perform one full-sized I/O */
for (count = 0; count < ep->re_max_rdma_segs; count++) {
struct rpcrdma_mr *mr;
int rc;
- mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ mr = kzalloc_node(sizeof(*mr), XPRTRDMA_GFP_FLAGS,
+ ibdev_to_node(device));
if (!mr)
break;
/* If there is no underlying connection, it's no use
* to wake the refresh worker.
*/
- if (ep->re_connect_status == 1) {
- /* The work is scheduled on a WQ_MEM_RECLAIM
- * workqueue in order to prevent MR allocation
- * from recursing into NFS during direct reclaim.
- */
- queue_work(xprtiod_workqueue, &buf->rb_refresh_worker);
- }
+ if (ep->re_connect_status != 1)
+ return;
+ queue_work(system_highpri_wq, &buf->rb_refresh_worker);
}
/**
* rpcrdma_req_create - Allocate an rpcrdma_req object
* @r_xprt: controlling r_xprt
* @size: initial size, in bytes, of send and receive buffers
- * @flags: GFP flags passed to memory allocators
*
* Returns an allocated and fully initialized rpcrdma_req or NULL.
*/
-struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,
- gfp_t flags)
+struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,
+ size_t size)
{
struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;
struct rpcrdma_req *req;
- req = kzalloc(sizeof(*req), flags);
+ req = kzalloc(sizeof(*req), XPRTRDMA_GFP_FLAGS);
if (req == NULL)
goto out1;
- req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE, flags);
+ req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE);
if (!req->rl_sendbuf)
goto out2;
- req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE, flags);
+ req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE);
if (!req->rl_recvbuf)
goto out3;
r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz;
maxhdrsize *= sizeof(__be32);
rb = rpcrdma_regbuf_alloc(__roundup_pow_of_two(maxhdrsize),
- DMA_TO_DEVICE, GFP_KERNEL);
+ DMA_TO_DEVICE);
if (!rb)
goto out;
struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
struct rpcrdma_rep *rep;
- rep = kzalloc(sizeof(*rep), GFP_KERNEL);
+ rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS);
if (rep == NULL)
goto out;
rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,
- DMA_FROM_DEVICE, GFP_KERNEL);
+ DMA_FROM_DEVICE);
if (!rep->rr_rdmabuf)
goto out_free;
for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) {
struct rpcrdma_req *req;
- req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2,
- GFP_KERNEL);
+ req = rpcrdma_req_create(r_xprt,
+ RPCRDMA_V1_DEF_INLINE_SIZE * 2);
if (!req)
goto out;
list_add(&req->rl_list, &buf->rb_send_bufs);
* or Replies they may be registered externally via frwr_map.
*/
static struct rpcrdma_regbuf *
-rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
- gfp_t flags)
+rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)
{
struct rpcrdma_regbuf *rb;
- rb = kmalloc(sizeof(*rb), flags);
+ rb = kmalloc(sizeof(*rb), XPRTRDMA_GFP_FLAGS);
if (!rb)
return NULL;
- rb->rg_data = kmalloc(size, flags);
+ rb->rg_data = kmalloc(size, XPRTRDMA_GFP_FLAGS);
if (!rb->rg_data) {
kfree(rb);
return NULL;
return rb->rg_data;
}
-#define RPCRDMA_DEF_GFP (GFP_NOIO | __GFP_NOWARN)
+/* Do not use emergency memory reserves, and fail quickly if memory
+ * cannot be allocated easily. These flags may be used wherever there
+ * is robust logic to handle a failure to allocate.
+ */
+#define XPRTRDMA_GFP_FLAGS (__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN)
/* To ensure a transport can always make forward progress,
* the number of RDMA segments allowed in header chunk lists
/*
* Buffer calls - xprtrdma/verbs.c
*/
-struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,
- gfp_t flags);
+struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,
+ size_t size);
int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
void rpcrdma_req_destroy(struct rpcrdma_req *req);
int rpcrdma_buffer_create(struct rpcrdma_xprt *);
switch (sap->sa_family) {
case AF_LOCAL:
sun = xs_addr_un(xprt);
- strlcpy(buf, sun->sun_path, sizeof(buf));
+ strscpy(buf, sun->sun_path, sizeof(buf));
xprt->address_strings[RPC_DISPLAY_ADDR] =
kstrdup(buf, GFP_KERNEL);
break;
if (max < min)
return -EADDRINUSE;
range = max - min + 1;
- rand = (unsigned short) prandom_u32() % range;
+ rand = prandom_u32_max(range);
return rand + min;
}
* we'll need to figure out how to pass a namespace to
* connect.
*/
- task->tk_rpc_status = -ENOTCONN;
- rpc_exit(task, -ENOTCONN);
+ rpc_task_set_rpc_status(task, -ENOTCONN);
goto out_wake;
}
ret = xs_local_setup_socket(transport);
struct net *net = sock_net(sk);
struct tipc_net *tn = net_generic(net, tipc_net_id);
u32 remaining = (TIPC_MAX_PORT - TIPC_MIN_PORT) + 1;
- u32 portid = prandom_u32() % remaining + TIPC_MIN_PORT;
+ u32 portid = prandom_u32_max(remaining) + TIPC_MIN_PORT;
while (remaining--) {
portid++;
addr->name->sun_family = AF_UNIX;
refcount_set(&addr->refcnt, 1);
- ordernum = prandom_u32();
+ ordernum = get_random_u32();
lastnum = ordernum & 0xFFFFF;
retry:
ordernum = (ordernum + 1) & 0xFFFFF;
/* The external entry point: unix_gc() */
void unix_gc(void)
{
+ struct sk_buff *next_skb, *skb;
struct unix_sock *u;
struct unix_sock *next;
struct sk_buff_head hitlist;
spin_unlock(&unix_gc_lock);
+ /* We need io_uring to clean its registered files, ignore all io_uring
+ * originated skbs. It's fine as io_uring doesn't keep references to
+ * other io_uring instances and so killing all other files in the cycle
+ * will put all io_uring references forcing it to go through normal
+ * release.path eventually putting registered files.
+ */
+ skb_queue_walk_safe(&hitlist, skb, next_skb) {
+ if (skb->scm_io_uring) {
+ __skb_unlink(skb, &hitlist);
+ skb_queue_tail(&skb->sk->sk_receive_queue, skb);
+ }
+ }
+
/* Here we are. Hitlist is filled. Die. */
__skb_queue_purge(&hitlist);
spin_lock(&unix_gc_lock);
+ /* There could be io_uring registered files, just push them back to
+ * the inflight list
+ */
+ list_for_each_entry_safe(u, next, &gc_candidates, link)
+ list_move_tail(&u->link, &gc_inflight_list);
+
/* All candidates should have been detached by now. */
BUG_ON(!list_empty(&gc_candidates));
wake_mask_size);
if (tok) {
cfg->tokens_size = tokens_size;
- memcpy(&cfg->payload_tok, tok, sizeof(*tok) + tokens_size);
+ cfg->payload_tok = *tok;
+ memcpy(cfg->payload_tok.token_stream, tok->token_stream,
+ tokens_size);
}
trig->tcp = cfg;
lockdep_assert_held(&rdev->bss_lock);
bss->refcount++;
- if (bss->pub.hidden_beacon_bss) {
- bss = container_of(bss->pub.hidden_beacon_bss,
- struct cfg80211_internal_bss,
- pub);
- bss->refcount++;
- }
- if (bss->pub.transmitted_bss) {
- bss = container_of(bss->pub.transmitted_bss,
- struct cfg80211_internal_bss,
- pub);
- bss->refcount++;
- }
+
+ if (bss->pub.hidden_beacon_bss)
+ bss_from_pub(bss->pub.hidden_beacon_bss)->refcount++;
+
+ if (bss->pub.transmitted_bss)
+ bss_from_pub(bss->pub.transmitted_bss)->refcount++;
}
static inline void bss_ref_put(struct cfg80211_registered_device *rdev,
tmp_old = cfg80211_find_ie(WLAN_EID_SSID, ie, ielen);
tmp_old = (tmp_old) ? tmp_old + tmp_old[1] + 2 : ie;
- while (tmp_old + tmp_old[1] + 2 - ie <= ielen) {
+ while (tmp_old + 2 - ie <= ielen &&
+ tmp_old + tmp_old[1] + 2 - ie <= ielen) {
if (tmp_old[0] == 0) {
tmp_old++;
continue;
* copied to new ie, skip ssid, capability, bssid-index ie
*/
tmp_new = sub_copy;
- while (tmp_new + tmp_new[1] + 2 - sub_copy <= subie_len) {
+ while (tmp_new + 2 - sub_copy <= subie_len &&
+ tmp_new + tmp_new[1] + 2 - sub_copy <= subie_len) {
if (!(tmp_new[0] == WLAN_EID_NON_TX_BSSID_CAP ||
tmp_new[0] == WLAN_EID_SSID)) {
memcpy(pos, tmp_new, tmp_new[1] + 2);
rcu_read_unlock();
+ /*
+ * This is a bit weird - it's not on the list, but already on another
+ * one! The only way that could happen is if there's some BSSID/SSID
+ * shared by multiple APs in their multi-BSSID profiles, potentially
+ * with hidden SSID mixed in ... ignore it.
+ */
+ if (!list_empty(&nontrans_bss->nontrans_list))
+ return -EINVAL;
+
/* add to the list */
list_add_tail(&nontrans_bss->nontrans_list, &trans_bss->nontrans_list);
return 0;
u8 bssid_index;
};
+static void cfg80211_update_hidden_bsses(struct cfg80211_internal_bss *known,
+ const struct cfg80211_bss_ies *new_ies,
+ const struct cfg80211_bss_ies *old_ies)
+{
+ struct cfg80211_internal_bss *bss;
+
+ /* Assign beacon IEs to all sub entries */
+ list_for_each_entry(bss, &known->hidden_list, hidden_list) {
+ const struct cfg80211_bss_ies *ies;
+
+ ies = rcu_access_pointer(bss->pub.beacon_ies);
+ WARN_ON(ies != old_ies);
+
+ rcu_assign_pointer(bss->pub.beacon_ies, new_ies);
+ }
+}
+
static bool
cfg80211_update_known_bss(struct cfg80211_registered_device *rdev,
struct cfg80211_internal_bss *known,
kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head);
} else if (rcu_access_pointer(new->pub.beacon_ies)) {
const struct cfg80211_bss_ies *old;
- struct cfg80211_internal_bss *bss;
if (known->pub.hidden_beacon_bss &&
!list_empty(&known->hidden_list)) {
if (old == rcu_access_pointer(known->pub.ies))
rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies);
- /* Assign beacon IEs to all sub entries */
- list_for_each_entry(bss, &known->hidden_list, hidden_list) {
- const struct cfg80211_bss_ies *ies;
-
- ies = rcu_access_pointer(bss->pub.beacon_ies);
- WARN_ON(ies != old);
-
- rcu_assign_pointer(bss->pub.beacon_ies,
- new->pub.beacon_ies);
- }
+ cfg80211_update_hidden_bsses(known, new->pub.beacon_ies, old);
if (old)
kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head);
new->refcount = 1;
INIT_LIST_HEAD(&new->hidden_list);
INIT_LIST_HEAD(&new->pub.nontrans_list);
+ /* we'll set this later if it was non-NULL */
+ new->pub.transmitted_bss = NULL;
if (rcu_access_pointer(tmp->pub.proberesp_ies)) {
hidden = rb_find_bss(rdev, tmp, BSS_CMP_HIDE_ZLEN);
spin_lock_bh(&rdev->bss_lock);
if (cfg80211_add_nontrans_list(non_tx_data->tx_bss,
&res->pub)) {
- if (__cfg80211_unlink_bss(rdev, res))
+ if (__cfg80211_unlink_bss(rdev, res)) {
rdev->bss_generation++;
+ res = NULL;
+ }
}
spin_unlock_bh(&rdev->bss_lock);
+
+ if (!res)
+ return NULL;
}
trace_cfg80211_return_bss(&res->pub);
for_each_element_id(elem, WLAN_EID_MULTIPLE_BSSID, ie, ielen) {
if (elem->datalen < 4)
continue;
+ if (elem->data[0] < 1 || (int)elem->data[0] > 8)
+ continue;
for_each_element(sub, elem->data + 1, elem->datalen - 1) {
u8 profile_len;
size_t new_ie_len;
struct cfg80211_bss_ies *new_ies;
const struct cfg80211_bss_ies *old;
- u8 cpy_len;
+ size_t cpy_len;
lockdep_assert_held(&wiphy_to_rdev(wiphy)->bss_lock);
} else {
old = rcu_access_pointer(nontrans_bss->beacon_ies);
rcu_assign_pointer(nontrans_bss->beacon_ies, new_ies);
+ cfg80211_update_hidden_bsses(bss_from_pub(nontrans_bss),
+ new_ies, old);
rcu_assign_pointer(nontrans_bss->ies, new_ies);
if (old)
kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head);
return -1;
hdrlen = ieee80211_hdrlen(hdr->frame_control) + data_offset;
- if (skb->len < hdrlen + 8)
+ if (skb->len < hdrlen)
return -1;
/* convert IEEE 802.11 header + possible LLC headers into Ethernet
memcpy(tmp.h_dest, ieee80211_get_DA(hdr), ETH_ALEN);
memcpy(tmp.h_source, ieee80211_get_SA(hdr), ETH_ALEN);
- if (iftype == NL80211_IFTYPE_MESH_POINT)
- skb_copy_bits(skb, hdrlen, &mesh_flags, 1);
+ if (iftype == NL80211_IFTYPE_MESH_POINT &&
+ skb_copy_bits(skb, hdrlen, &mesh_flags, 1) < 0)
+ return -1;
mesh_flags &= MESH_FLAGS_AE;
if (iftype == NL80211_IFTYPE_MESH_POINT) {
if (mesh_flags == MESH_FLAGS_AE_A4)
return -1;
- if (mesh_flags == MESH_FLAGS_AE_A5_A6) {
- skb_copy_bits(skb, hdrlen +
- offsetof(struct ieee80211s_hdr, eaddr1),
- tmp.h_dest, 2 * ETH_ALEN);
- }
+ if (mesh_flags == MESH_FLAGS_AE_A5_A6 &&
+ skb_copy_bits(skb, hdrlen +
+ offsetof(struct ieee80211s_hdr, eaddr1),
+ tmp.h_dest, 2 * ETH_ALEN) < 0)
+ return -1;
+
hdrlen += __ieee80211_get_mesh_hdrlen(mesh_flags);
}
break;
if (iftype == NL80211_IFTYPE_MESH_POINT) {
if (mesh_flags == MESH_FLAGS_AE_A5_A6)
return -1;
- if (mesh_flags == MESH_FLAGS_AE_A4)
- skb_copy_bits(skb, hdrlen +
- offsetof(struct ieee80211s_hdr, eaddr1),
- tmp.h_source, ETH_ALEN);
+ if (mesh_flags == MESH_FLAGS_AE_A4 &&
+ skb_copy_bits(skb, hdrlen +
+ offsetof(struct ieee80211s_hdr, eaddr1),
+ tmp.h_source, ETH_ALEN) < 0)
+ return -1;
hdrlen += __ieee80211_get_mesh_hdrlen(mesh_flags);
}
break;
break;
}
- skb_copy_bits(skb, hdrlen, &payload, sizeof(payload));
- tmp.h_proto = payload.proto;
-
- if (likely((!is_amsdu && ether_addr_equal(payload.hdr, rfc1042_header) &&
- tmp.h_proto != htons(ETH_P_AARP) &&
- tmp.h_proto != htons(ETH_P_IPX)) ||
- ether_addr_equal(payload.hdr, bridge_tunnel_header))) {
+ if (likely(skb_copy_bits(skb, hdrlen, &payload, sizeof(payload)) == 0 &&
+ ((!is_amsdu && ether_addr_equal(payload.hdr, rfc1042_header) &&
+ payload.proto != htons(ETH_P_AARP) &&
+ payload.proto != htons(ETH_P_IPX)) ||
+ ether_addr_equal(payload.hdr, bridge_tunnel_header)))) {
/* remove RFC1042 or Bridge-Tunnel encapsulation and
* replace EtherType */
hdrlen += ETH_ALEN + 2;
+ tmp.h_proto = payload.proto;
skb_postpull_rcsum(skb, &payload, ETH_ALEN + 2);
} else {
tmp.h_proto = htons(skb->len - hdrlen);
struct __compat_iw_event *compat_event;
struct compat_iw_point compat_wrqu;
struct sk_buff *compskb;
+ int ptr_len;
#endif
/*
nlmsg_end(skb, nlh);
#ifdef CONFIG_COMPAT
hdr_len = compat_event_type_size[descr->header_type];
+
+ /* ptr_len is remaining size in event header apart from LCP */
+ ptr_len = hdr_len - IW_EV_COMPAT_LCP_LEN;
event_len = hdr_len + extra_len;
compskb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
if (descr->header_type == IW_HEADER_TYPE_POINT) {
compat_wrqu.length = wrqu->data.length;
compat_wrqu.flags = wrqu->data.flags;
- memcpy(&compat_event->pointer,
- ((char *) &compat_wrqu) + IW_EV_COMPAT_POINT_OFF,
- hdr_len - IW_EV_COMPAT_LCP_LEN);
+ memcpy(compat_event->ptr_bytes,
+ ((char *)&compat_wrqu) + IW_EV_COMPAT_POINT_OFF,
+ ptr_len);
if (extra_len)
- memcpy(((char *) compat_event) + hdr_len,
- extra, extra_len);
+ memcpy(&compat_event->ptr_bytes[ptr_len],
+ extra, extra_len);
} else {
/* extra_len must be zero, so no if (extra) needed */
- memcpy(&compat_event->pointer, wrqu,
- hdr_len - IW_EV_COMPAT_LCP_LEN);
+ memcpy(compat_event->ptr_bytes, wrqu, ptr_len);
}
nlmsg_end(compskb, nlh);
} else {
u32 spi = 0;
for (h = 0; h < high-low+1; h++) {
- spi = low + prandom_u32()%(high-low+1);
+ spi = low + prandom_u32_max(high - low + 1);
x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
if (x0 == NULL) {
newspi = htonl(spi);
*/
#include <linux/init.h>
#include <linux/module.h>
-#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#define MBOCHS_TYPE_2 "medium"
#define MBOCHS_TYPE_3 "large"
-static const struct mbochs_type {
- const char *name;
+static struct mbochs_type {
+ struct mdev_type type;
u32 mbytes;
u32 max_x;
u32 max_y;
} mbochs_types[] = {
{
- .name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_1,
+ .type.sysfs_name = MBOCHS_TYPE_1,
+ .type.pretty_name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_1,
.mbytes = 4,
.max_x = 800,
.max_y = 600,
}, {
- .name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_2,
+ .type.sysfs_name = MBOCHS_TYPE_2,
+ .type.pretty_name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_2,
.mbytes = 16,
.max_x = 1920,
.max_y = 1440,
}, {
- .name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_3,
+ .type.sysfs_name = MBOCHS_TYPE_3,
+ .type.pretty_name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_3,
.mbytes = 64,
.max_x = 0,
.max_y = 0,
},
};
+static struct mdev_type *mbochs_mdev_types[] = {
+ &mbochs_types[0].type,
+ &mbochs_types[1].type,
+ &mbochs_types[2].type,
+};
static dev_t mbochs_devt;
static struct class *mbochs_class;
static struct cdev mbochs_cdev;
static struct device mbochs_dev;
+static struct mdev_parent mbochs_parent;
static atomic_t mbochs_avail_mbytes;
static const struct vfio_device_ops mbochs_dev_ops;
return 0;
}
-static int mbochs_probe(struct mdev_device *mdev)
+static int mbochs_init_dev(struct vfio_device *vdev)
{
+ struct mdev_state *mdev_state =
+ container_of(vdev, struct mdev_state, vdev);
+ struct mdev_device *mdev = to_mdev_device(vdev->dev);
+ struct mbochs_type *type =
+ container_of(mdev->type, struct mbochs_type, type);
int avail_mbytes = atomic_read(&mbochs_avail_mbytes);
- const struct mbochs_type *type =
- &mbochs_types[mdev_get_type_group_id(mdev)];
- struct device *dev = mdev_dev(mdev);
- struct mdev_state *mdev_state;
int ret = -ENOMEM;
do {
} while (!atomic_try_cmpxchg(&mbochs_avail_mbytes, &avail_mbytes,
avail_mbytes - type->mbytes));
- mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL);
- if (mdev_state == NULL)
- goto err_avail;
- vfio_init_group_dev(&mdev_state->vdev, &mdev->dev, &mbochs_dev_ops);
-
mdev_state->vconfig = kzalloc(MBOCHS_CONFIG_SPACE_SIZE, GFP_KERNEL);
- if (mdev_state->vconfig == NULL)
- goto err_mem;
+ if (!mdev_state->vconfig)
+ goto err_avail;
mdev_state->memsize = type->mbytes * 1024 * 1024;
mdev_state->pagecount = mdev_state->memsize >> PAGE_SHIFT;
sizeof(struct page *),
GFP_KERNEL);
if (!mdev_state->pages)
- goto err_mem;
-
- dev_info(dev, "%s: %s, %d MB, %ld pages\n", __func__,
- type->name, type->mbytes, mdev_state->pagecount);
+ goto err_vconfig;
mutex_init(&mdev_state->ops_lock);
mdev_state->mdev = mdev;
mbochs_create_config_space(mdev_state);
mbochs_reset(mdev_state);
+ dev_info(vdev->dev, "%s: %s, %d MB, %ld pages\n", __func__,
+ type->type.pretty_name, type->mbytes, mdev_state->pagecount);
+ return 0;
+
+err_vconfig:
+ kfree(mdev_state->vconfig);
+err_avail:
+ atomic_add(type->mbytes, &mbochs_avail_mbytes);
+ return ret;
+}
+
+static int mbochs_probe(struct mdev_device *mdev)
+{
+ struct mdev_state *mdev_state;
+ int ret = -ENOMEM;
+
+ mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev,
+ &mbochs_dev_ops);
+ if (IS_ERR(mdev_state))
+ return PTR_ERR(mdev_state);
+
ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev);
if (ret)
- goto err_mem;
+ goto err_put_vdev;
dev_set_drvdata(&mdev->dev, mdev_state);
return 0;
-err_mem:
- vfio_uninit_group_dev(&mdev_state->vdev);
+
+err_put_vdev:
+ vfio_put_device(&mdev_state->vdev);
+ return ret;
+}
+
+static void mbochs_release_dev(struct vfio_device *vdev)
+{
+ struct mdev_state *mdev_state =
+ container_of(vdev, struct mdev_state, vdev);
+
+ atomic_add(mdev_state->type->mbytes, &mbochs_avail_mbytes);
kfree(mdev_state->pages);
kfree(mdev_state->vconfig);
- kfree(mdev_state);
-err_avail:
- atomic_add(type->mbytes, &mbochs_avail_mbytes);
- return ret;
+ vfio_free_device(vdev);
}
static void mbochs_remove(struct mdev_device *mdev)
struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev);
vfio_unregister_group_dev(&mdev_state->vdev);
- vfio_uninit_group_dev(&mdev_state->vdev);
- atomic_add(mdev_state->type->mbytes, &mbochs_avail_mbytes);
- kfree(mdev_state->pages);
- kfree(mdev_state->vconfig);
- kfree(mdev_state);
+ vfio_put_device(&mdev_state->vdev);
}
static ssize_t mbochs_read(struct vfio_device *vdev, char __user *buf,
NULL,
};
-static ssize_t name_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
+static ssize_t mbochs_show_description(struct mdev_type *mtype, char *buf)
{
- const struct mbochs_type *type =
- &mbochs_types[mtype_get_type_group_id(mtype)];
-
- return sprintf(buf, "%s\n", type->name);
-}
-static MDEV_TYPE_ATTR_RO(name);
-
-static ssize_t description_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- const struct mbochs_type *type =
- &mbochs_types[mtype_get_type_group_id(mtype)];
+ struct mbochs_type *type =
+ container_of(mtype, struct mbochs_type, type);
return sprintf(buf, "virtual display, %d MB video memory\n",
type ? type->mbytes : 0);
}
-static MDEV_TYPE_ATTR_RO(description);
-static ssize_t available_instances_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr,
- char *buf)
+static unsigned int mbochs_get_available(struct mdev_type *mtype)
{
- const struct mbochs_type *type =
- &mbochs_types[mtype_get_type_group_id(mtype)];
- int count = atomic_read(&mbochs_avail_mbytes) / type->mbytes;
-
- return sprintf(buf, "%d\n", count);
-}
-static MDEV_TYPE_ATTR_RO(available_instances);
+ struct mbochs_type *type =
+ container_of(mtype, struct mbochs_type, type);
-static ssize_t device_api_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
+ return atomic_read(&mbochs_avail_mbytes) / type->mbytes;
}
-static MDEV_TYPE_ATTR_RO(device_api);
-
-static struct attribute *mdev_types_attrs[] = {
- &mdev_type_attr_name.attr,
- &mdev_type_attr_description.attr,
- &mdev_type_attr_device_api.attr,
- &mdev_type_attr_available_instances.attr,
- NULL,
-};
-
-static struct attribute_group mdev_type_group1 = {
- .name = MBOCHS_TYPE_1,
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group mdev_type_group2 = {
- .name = MBOCHS_TYPE_2,
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group mdev_type_group3 = {
- .name = MBOCHS_TYPE_3,
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group *mdev_type_groups[] = {
- &mdev_type_group1,
- &mdev_type_group2,
- &mdev_type_group3,
- NULL,
-};
static const struct vfio_device_ops mbochs_dev_ops = {
.close_device = mbochs_close_device,
+ .init = mbochs_init_dev,
+ .release = mbochs_release_dev,
.read = mbochs_read,
.write = mbochs_write,
.ioctl = mbochs_ioctl,
};
static struct mdev_driver mbochs_driver = {
+ .device_api = VFIO_DEVICE_API_PCI_STRING,
.driver = {
.name = "mbochs",
.owner = THIS_MODULE,
},
.probe = mbochs_probe,
.remove = mbochs_remove,
- .supported_type_groups = mdev_type_groups,
+ .get_available = mbochs_get_available,
+ .show_description = mbochs_show_description,
};
static const struct file_operations vd_fops = {
if (ret)
goto err_class;
- ret = mdev_register_device(&mbochs_dev, &mbochs_driver);
+ ret = mdev_register_parent(&mbochs_parent, &mbochs_dev, &mbochs_driver,
+ mbochs_mdev_types,
+ ARRAY_SIZE(mbochs_mdev_types));
if (ret)
goto err_device;
static void __exit mbochs_dev_exit(void)
{
mbochs_dev.bus = NULL;
- mdev_unregister_device(&mbochs_dev);
+ mdev_unregister_parent(&mbochs_parent);
device_unregister(&mbochs_dev);
mdev_unregister_driver(&mbochs_driver);
*/
#include <linux/init.h>
#include <linux/module.h>
-#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
MODULE_LICENSE("GPL v2");
-static int max_devices = 4;
-module_param_named(count, max_devices, int, 0444);
-MODULE_PARM_DESC(count, "number of " MDPY_NAME " devices");
-
-
#define MDPY_TYPE_1 "vga"
#define MDPY_TYPE_2 "xga"
#define MDPY_TYPE_3 "hd"
-static const struct mdpy_type {
- const char *name;
+static struct mdpy_type {
+ struct mdev_type type;
u32 format;
u32 bytepp;
u32 width;
u32 height;
} mdpy_types[] = {
{
- .name = MDPY_CLASS_NAME "-" MDPY_TYPE_1,
+ .type.sysfs_name = MDPY_TYPE_1,
+ .type.pretty_name = MDPY_CLASS_NAME "-" MDPY_TYPE_1,
.format = DRM_FORMAT_XRGB8888,
.bytepp = 4,
.width = 640,
.height = 480,
}, {
- .name = MDPY_CLASS_NAME "-" MDPY_TYPE_2,
+ .type.sysfs_name = MDPY_TYPE_2,
+ .type.pretty_name = MDPY_CLASS_NAME "-" MDPY_TYPE_2,
.format = DRM_FORMAT_XRGB8888,
.bytepp = 4,
.width = 1024,
.height = 768,
}, {
- .name = MDPY_CLASS_NAME "-" MDPY_TYPE_3,
+ .type.sysfs_name = MDPY_TYPE_3,
+ .type.pretty_name = MDPY_CLASS_NAME "-" MDPY_TYPE_3,
.format = DRM_FORMAT_XRGB8888,
.bytepp = 4,
.width = 1920,
},
};
+static struct mdev_type *mdpy_mdev_types[] = {
+ &mdpy_types[0].type,
+ &mdpy_types[1].type,
+ &mdpy_types[2].type,
+};
+
static dev_t mdpy_devt;
static struct class *mdpy_class;
static struct cdev mdpy_cdev;
static struct device mdpy_dev;
-static u32 mdpy_count;
+static struct mdev_parent mdpy_parent;
static const struct vfio_device_ops mdpy_dev_ops;
/* State of each mdev device */
return 0;
}
-static int mdpy_probe(struct mdev_device *mdev)
+static int mdpy_init_dev(struct vfio_device *vdev)
{
+ struct mdev_state *mdev_state =
+ container_of(vdev, struct mdev_state, vdev);
+ struct mdev_device *mdev = to_mdev_device(vdev->dev);
const struct mdpy_type *type =
- &mdpy_types[mdev_get_type_group_id(mdev)];
- struct device *dev = mdev_dev(mdev);
- struct mdev_state *mdev_state;
+ container_of(mdev->type, struct mdpy_type, type);
u32 fbsize;
- int ret;
-
- if (mdpy_count >= max_devices)
- return -ENOMEM;
-
- mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL);
- if (mdev_state == NULL)
- return -ENOMEM;
- vfio_init_group_dev(&mdev_state->vdev, &mdev->dev, &mdpy_dev_ops);
+ int ret = -ENOMEM;
mdev_state->vconfig = kzalloc(MDPY_CONFIG_SPACE_SIZE, GFP_KERNEL);
- if (mdev_state->vconfig == NULL) {
- ret = -ENOMEM;
- goto err_state;
- }
+ if (!mdev_state->vconfig)
+ return ret;
fbsize = roundup_pow_of_two(type->width * type->height * type->bytepp);
mdev_state->memblk = vmalloc_user(fbsize);
- if (!mdev_state->memblk) {
- ret = -ENOMEM;
- goto err_vconfig;
- }
- dev_info(dev, "%s: %s (%dx%d)\n", __func__, type->name, type->width,
- type->height);
+ if (!mdev_state->memblk)
+ goto out_vconfig;
mutex_init(&mdev_state->ops_lock);
mdev_state->mdev = mdev;
- mdev_state->type = type;
+ mdev_state->type = type;
mdev_state->memsize = fbsize;
mdpy_create_config_space(mdev_state);
mdpy_reset(mdev_state);
- mdpy_count++;
+ dev_info(vdev->dev, "%s: %s (%dx%d)\n", __func__, type->type.pretty_name,
+ type->width, type->height);
+ return 0;
+
+out_vconfig:
+ kfree(mdev_state->vconfig);
+ return ret;
+}
+
+static int mdpy_probe(struct mdev_device *mdev)
+{
+ struct mdev_state *mdev_state;
+ int ret;
+
+ mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev,
+ &mdpy_dev_ops);
+ if (IS_ERR(mdev_state))
+ return PTR_ERR(mdev_state);
ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev);
if (ret)
- goto err_mem;
+ goto err_put_vdev;
dev_set_drvdata(&mdev->dev, mdev_state);
return 0;
-err_mem:
+
+err_put_vdev:
+ vfio_put_device(&mdev_state->vdev);
+ return ret;
+}
+
+static void mdpy_release_dev(struct vfio_device *vdev)
+{
+ struct mdev_state *mdev_state =
+ container_of(vdev, struct mdev_state, vdev);
+
vfree(mdev_state->memblk);
-err_vconfig:
kfree(mdev_state->vconfig);
-err_state:
- vfio_uninit_group_dev(&mdev_state->vdev);
- kfree(mdev_state);
- return ret;
+ vfio_free_device(vdev);
}
static void mdpy_remove(struct mdev_device *mdev)
dev_info(&mdev->dev, "%s\n", __func__);
vfio_unregister_group_dev(&mdev_state->vdev);
- vfree(mdev_state->memblk);
- kfree(mdev_state->vconfig);
- vfio_uninit_group_dev(&mdev_state->vdev);
- kfree(mdev_state);
-
- mdpy_count--;
+ vfio_put_device(&mdev_state->vdev);
}
static ssize_t mdpy_read(struct vfio_device *vdev, char __user *buf,
NULL,
};
-static ssize_t name_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- const struct mdpy_type *type =
- &mdpy_types[mtype_get_type_group_id(mtype)];
-
- return sprintf(buf, "%s\n", type->name);
-}
-static MDEV_TYPE_ATTR_RO(name);
-
-static ssize_t description_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
+static ssize_t mdpy_show_description(struct mdev_type *mtype, char *buf)
{
- const struct mdpy_type *type =
- &mdpy_types[mtype_get_type_group_id(mtype)];
+ struct mdpy_type *type = container_of(mtype, struct mdpy_type, type);
return sprintf(buf, "virtual display, %dx%d framebuffer\n",
type->width, type->height);
}
-static MDEV_TYPE_ATTR_RO(description);
-
-static ssize_t available_instances_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "%d\n", max_devices - mdpy_count);
-}
-static MDEV_TYPE_ATTR_RO(available_instances);
-
-static ssize_t device_api_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
-}
-static MDEV_TYPE_ATTR_RO(device_api);
-
-static struct attribute *mdev_types_attrs[] = {
- &mdev_type_attr_name.attr,
- &mdev_type_attr_description.attr,
- &mdev_type_attr_device_api.attr,
- &mdev_type_attr_available_instances.attr,
- NULL,
-};
-
-static struct attribute_group mdev_type_group1 = {
- .name = MDPY_TYPE_1,
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group mdev_type_group2 = {
- .name = MDPY_TYPE_2,
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group mdev_type_group3 = {
- .name = MDPY_TYPE_3,
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group *mdev_type_groups[] = {
- &mdev_type_group1,
- &mdev_type_group2,
- &mdev_type_group3,
- NULL,
-};
static const struct vfio_device_ops mdpy_dev_ops = {
+ .init = mdpy_init_dev,
+ .release = mdpy_release_dev,
.read = mdpy_read,
.write = mdpy_write,
.ioctl = mdpy_ioctl,
};
static struct mdev_driver mdpy_driver = {
+ .device_api = VFIO_DEVICE_API_PCI_STRING,
+ .max_instances = 4,
.driver = {
.name = "mdpy",
.owner = THIS_MODULE,
},
.probe = mdpy_probe,
.remove = mdpy_remove,
- .supported_type_groups = mdev_type_groups,
+ .show_description = mdpy_show_description,
};
static const struct file_operations vd_fops = {
if (ret)
goto err_class;
- ret = mdev_register_device(&mdpy_dev, &mdpy_driver);
+ ret = mdev_register_parent(&mdpy_parent, &mdpy_dev, &mdpy_driver,
+ mdpy_mdev_types,
+ ARRAY_SIZE(mdpy_mdev_types));
if (ret)
goto err_device;
static void __exit mdpy_dev_exit(void)
{
mdpy_dev.bus = NULL;
- mdev_unregister_device(&mdpy_dev);
+ mdev_unregister_parent(&mdpy_parent);
device_unregister(&mdpy_dev);
mdev_unregister_driver(&mdpy_driver);
mdpy_class = NULL;
}
+module_param_named(count, mdpy_driver.max_instances, int, 0444);
+MODULE_PARM_DESC(count, "number of " MDPY_NAME " devices");
+
module_init(mdpy_dev_init)
module_exit(mdpy_dev_exit)
#include <linux/init.h>
#include <linux/module.h>
-#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/poll.h>
#include <linux/cdev.h>
#include <linux/sched.h>
#include <linux/wait.h>
-#include <linux/uuid.h>
#include <linux/vfio.h>
#include <linux/iommu.h>
#include <linux/sysfs.h>
struct cdev vd_cdev;
struct idr vd_idr;
struct device dev;
+ struct mdev_parent parent;
} mtty_dev;
struct mdev_region_info {
int nr_ports;
};
+static struct mtty_type {
+ struct mdev_type type;
+ int nr_ports;
+} mtty_types[2] = {
+ { .nr_ports = 1, .type.sysfs_name = "1",
+ .type.pretty_name = "Single port serial" },
+ { .nr_ports = 2, .type.sysfs_name = "2",
+ .type.pretty_name = "Dual port serial" },
+};
+
+static struct mdev_type *mtty_mdev_types[] = {
+ &mtty_types[0].type,
+ &mtty_types[1].type,
+};
+
static atomic_t mdev_avail_ports = ATOMIC_INIT(MAX_MTTYS);
static const struct file_operations vd_fops = {
return ret;
}
-static int mtty_probe(struct mdev_device *mdev)
+static int mtty_init_dev(struct vfio_device *vdev)
{
- struct mdev_state *mdev_state;
- int nr_ports = mdev_get_type_group_id(mdev) + 1;
+ struct mdev_state *mdev_state =
+ container_of(vdev, struct mdev_state, vdev);
+ struct mdev_device *mdev = to_mdev_device(vdev->dev);
+ struct mtty_type *type =
+ container_of(mdev->type, struct mtty_type, type);
int avail_ports = atomic_read(&mdev_avail_ports);
int ret;
do {
- if (avail_ports < nr_ports)
+ if (avail_ports < type->nr_ports)
return -ENOSPC;
} while (!atomic_try_cmpxchg(&mdev_avail_ports,
- &avail_ports, avail_ports - nr_ports));
+ &avail_ports,
+ avail_ports - type->nr_ports));
- mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL);
- if (mdev_state == NULL) {
- ret = -ENOMEM;
- goto err_nr_ports;
- }
-
- vfio_init_group_dev(&mdev_state->vdev, &mdev->dev, &mtty_dev_ops);
-
- mdev_state->nr_ports = nr_ports;
+ mdev_state->nr_ports = type->nr_ports;
mdev_state->irq_index = -1;
mdev_state->s[0].max_fifo_size = MAX_FIFO_SIZE;
mdev_state->s[1].max_fifo_size = MAX_FIFO_SIZE;
mutex_init(&mdev_state->rxtx_lock);
- mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL);
- if (mdev_state->vconfig == NULL) {
+ mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL);
+ if (!mdev_state->vconfig) {
ret = -ENOMEM;
- goto err_state;
+ goto err_nr_ports;
}
mutex_init(&mdev_state->ops_lock);
mdev_state->mdev = mdev;
-
mtty_create_config_space(mdev_state);
+ return 0;
+
+err_nr_ports:
+ atomic_add(type->nr_ports, &mdev_avail_ports);
+ return ret;
+}
+
+static int mtty_probe(struct mdev_device *mdev)
+{
+ struct mdev_state *mdev_state;
+ int ret;
+
+ mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev,
+ &mtty_dev_ops);
+ if (IS_ERR(mdev_state))
+ return PTR_ERR(mdev_state);
ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev);
if (ret)
- goto err_vconfig;
+ goto err_put_vdev;
dev_set_drvdata(&mdev->dev, mdev_state);
return 0;
-err_vconfig:
- kfree(mdev_state->vconfig);
-err_state:
- vfio_uninit_group_dev(&mdev_state->vdev);
- kfree(mdev_state);
-err_nr_ports:
- atomic_add(nr_ports, &mdev_avail_ports);
+err_put_vdev:
+ vfio_put_device(&mdev_state->vdev);
return ret;
}
+static void mtty_release_dev(struct vfio_device *vdev)
+{
+ struct mdev_state *mdev_state =
+ container_of(vdev, struct mdev_state, vdev);
+
+ atomic_add(mdev_state->nr_ports, &mdev_avail_ports);
+ kfree(mdev_state->vconfig);
+ vfio_free_device(vdev);
+}
+
static void mtty_remove(struct mdev_device *mdev)
{
struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev);
- int nr_ports = mdev_state->nr_ports;
vfio_unregister_group_dev(&mdev_state->vdev);
-
- kfree(mdev_state->vconfig);
- vfio_uninit_group_dev(&mdev_state->vdev);
- kfree(mdev_state);
- atomic_add(nr_ports, &mdev_avail_ports);
+ vfio_put_device(&mdev_state->vdev);
}
static int mtty_reset(struct mdev_state *mdev_state)
NULL,
};
-static ssize_t name_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- static const char *name_str[2] = { "Single port serial",
- "Dual port serial" };
-
- return sysfs_emit(buf, "%s\n",
- name_str[mtype_get_type_group_id(mtype)]);
-}
-
-static MDEV_TYPE_ATTR_RO(name);
-
-static ssize_t available_instances_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr,
- char *buf)
+static unsigned int mtty_get_available(struct mdev_type *mtype)
{
- unsigned int ports = mtype_get_type_group_id(mtype) + 1;
+ struct mtty_type *type = container_of(mtype, struct mtty_type, type);
- return sprintf(buf, "%d\n", atomic_read(&mdev_avail_ports) / ports);
+ return atomic_read(&mdev_avail_ports) / type->nr_ports;
}
-static MDEV_TYPE_ATTR_RO(available_instances);
-
-static ssize_t device_api_show(struct mdev_type *mtype,
- struct mdev_type_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
-}
-
-static MDEV_TYPE_ATTR_RO(device_api);
-
-static struct attribute *mdev_types_attrs[] = {
- &mdev_type_attr_name.attr,
- &mdev_type_attr_device_api.attr,
- &mdev_type_attr_available_instances.attr,
- NULL,
-};
-
-static struct attribute_group mdev_type_group1 = {
- .name = "1",
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group mdev_type_group2 = {
- .name = "2",
- .attrs = mdev_types_attrs,
-};
-
-static struct attribute_group *mdev_type_groups[] = {
- &mdev_type_group1,
- &mdev_type_group2,
- NULL,
-};
-
static const struct vfio_device_ops mtty_dev_ops = {
.name = "vfio-mtty",
+ .init = mtty_init_dev,
+ .release = mtty_release_dev,
.read = mtty_read,
.write = mtty_write,
.ioctl = mtty_ioctl,
};
static struct mdev_driver mtty_driver = {
+ .device_api = VFIO_DEVICE_API_PCI_STRING,
.driver = {
.name = "mtty",
.owner = THIS_MODULE,
},
.probe = mtty_probe,
.remove = mtty_remove,
- .supported_type_groups = mdev_type_groups,
+ .get_available = mtty_get_available,
};
static void mtty_device_release(struct device *dev)
if (ret)
goto err_class;
- ret = mdev_register_device(&mtty_dev.dev, &mtty_driver);
+ ret = mdev_register_parent(&mtty_dev.parent, &mtty_dev.dev,
+ &mtty_driver, mtty_mdev_types,
+ ARRAY_SIZE(mtty_mdev_types));
if (ret)
goto err_device;
return 0;
static void __exit mtty_dev_exit(void)
{
mtty_dev.dev.bus = NULL;
- mdev_unregister_device(&mtty_dev.dev);
+ mdev_unregister_parent(&mtty_dev.parent);
device_unregister(&mtty_dev.dev);
idr_destroy(&mtty_dev.vd_idr);
# LLVM assembly
# Generate .ll files from .c
quiet_cmd_cc_ll_c = CC $(quiet_modtag) $@
- cmd_cc_ll_c = $(CC) $(c_flags) -emit-llvm -S -o $@ $<
+ cmd_cc_ll_c = $(CC) $(c_flags) -emit-llvm -S -fno-discard-value-names -o $@ $<
$(obj)/%.ll: $(src)/%.c FORCE
$(call if_changed_dep,cc_ll_c)
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+kmsan-cflags := -fsanitize=kernel-memory
+
+ifdef CONFIG_KMSAN_CHECK_PARAM_RETVAL
+kmsan-cflags += -fsanitize-memory-param-retval
+endif
+
+export CFLAGS_KMSAN := $(kmsan-cflags)
endif
endif
+ifeq ($(CONFIG_KMSAN),y)
+_c_flags += $(if $(patsubst n%,, \
+ $(KMSAN_SANITIZE_$(basetarget).o)$(KMSAN_SANITIZE)y), \
+ $(CFLAGS_KMSAN))
+_c_flags += $(if $(patsubst n%,, \
+ $(KMSAN_ENABLE_CHECKS_$(basetarget).o)$(KMSAN_ENABLE_CHECKS)y), \
+ , -mllvm -msan-disable-checks=1)
+endif
+
ifeq ($(CONFIG_UBSAN),y)
_c_flags += $(if $(patsubst n%,, \
$(UBSAN_SANITIZE_$(basetarget).o)$(UBSAN_SANITIZE)$(CONFIG_UBSAN_SANITIZE_ALL)), \
echo >&2 "WARNING: $(missing-input) is missing."; \
echo >&2 " Modules may not have dependencies or modversions."; \
echo >&2 " You may get many unresolved symbol warnings.";) \
- sed 's/ko$$/o/' $(or $(modorder-if-needed), /dev/null) | $(MODPOST) $(modpost-args) $(vmlinux.o-if-present) -T -
+ sed 's/ko$$/o/' $(or $(modorder-if-needed), /dev/null) | $(MODPOST) $(modpost-args) -T - $(vmlinux.o-if-present)
targets += $(output-symdump)
$(output-symdump): $(modorder-if-needed) $(vmlinux.o-if-present) $(moudle.symvers-if-present) $(MODPOST) FORCE
(?:__)?(?:u|s|be|le)(?:8|16|32|64)|
atomic_t
)};
+our $typeStdioTypedefs = qr{(?x:
+ FILE
+)};
our $typeTypedefs = qr{(?x:
$typeC99Typedefs\b|
$typeOtherOSTypedefs\b|
- $typeKernelTypedefs\b
+ $typeKernelTypedefs\b|
+ $typeStdioTypedefs\b
)};
our $zero_initializer = qr{(?:(?:0[xX])?0+$Int_type?|NULL|false)\b};
"rcu_barrier_sched" => "rcu_barrier",
"get_state_synchronize_sched" => "get_state_synchronize_rcu",
"cond_synchronize_sched" => "cond_synchronize_rcu",
+ "kmap" => "kmap_local_page",
+ "kmap_atomic" => "kmap_local_page",
);
#Create a search pattern for all these strings to speed up a loop below
}
}
+# Check Fixes: styles is correct
+ if (!$in_header_lines &&
+ $line =~ /^\s*fixes:?\s*(?:commit\s*)?[0-9a-f]{5,}\b/i) {
+ my $orig_commit = "";
+ my $id = "0123456789ab";
+ my $title = "commit title";
+ my $tag_case = 1;
+ my $tag_space = 1;
+ my $id_length = 1;
+ my $id_case = 1;
+ my $title_has_quotes = 0;
+
+ if ($line =~ /(\s*fixes:?)\s+([0-9a-f]{5,})\s+($balanced_parens)/i) {
+ my $tag = $1;
+ $orig_commit = $2;
+ $title = $3;
+
+ $tag_case = 0 if $tag eq "Fixes:";
+ $tag_space = 0 if ($line =~ /^fixes:? [0-9a-f]{5,} ($balanced_parens)/i);
+
+ $id_length = 0 if ($orig_commit =~ /^[0-9a-f]{12}$/i);
+ $id_case = 0 if ($orig_commit !~ /[A-F]/);
+
+ # Always strip leading/trailing parens then double quotes if existing
+ $title = substr($title, 1, -1);
+ if ($title =~ /^".*"$/) {
+ $title = substr($title, 1, -1);
+ $title_has_quotes = 1;
+ }
+ }
+
+ my ($cid, $ctitle) = git_commit_info($orig_commit, $id,
+ $title);
+
+ if ($ctitle ne $title || $tag_case || $tag_space ||
+ $id_length || $id_case || !$title_has_quotes) {
+ if (WARN("BAD_FIXES_TAG",
+ "Please use correct Fixes: style 'Fixes: <12 chars of sha1> (\"<title line>\")' - ie: 'Fixes: $cid (\"$ctitle\")'\n" . $herecurr) &&
+ $fix) {
+ $fixed[$fixlinenr] = "Fixes: $cid (\"$ctitle\")";
+ }
+ }
+ }
+
# Check email subject for common tools that don't need to be mentioned
if ($in_header_lines &&
$line =~ /^Subject:.*\b(?:checkpatch|sparse|smatch)\b[^:]/i) {
def run_analysis(entry):
# Disable all checks, then re-enable the ones we want
- checks = "-checks=-*,"
+ checks = []
+ checks.append("-checks=-*")
if args.type == "clang-tidy":
- checks += "linuxkernel-*"
+ checks.append("linuxkernel-*")
else:
- checks += "clang-analyzer-*"
- checks += ",-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling"
- p = subprocess.run(["clang-tidy", "-p", args.path, checks, entry["file"]],
+ checks.append("clang-analyzer-*")
+ checks.append("-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling")
+ p = subprocess.run(["clang-tidy", "-p", args.path, ",".join(checks), entry["file"]],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=entry["directory"])
-#!/bin/sh
+#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
# Disassemble the Code: line in Linux oopses
# usage: decodecode < oops.file
# AFLAGS=--32 decodecode < 386.oops
# PC=hex - the PC (program counter) the oops points to
+faultlinenum=1
+
cleanup() {
rm -f $T $T.s $T.o $T.oo $T.aa $T.dis
exit 1
grep -v "/tmp\|Disassembly\|\.text\|^$" > $t.dis 2>&1
}
+# Match the maximum number of opcode bytes from @op_bytes contained within
+# @opline
+#
+# Params:
+# @op_bytes: The string of bytes from the Code: line
+# @opline: The disassembled line coming from objdump
+#
+# Returns:
+# The max number of opcode bytes from the beginning of @op_bytes which match
+# the opcode bytes in the objdump line.
+get_substr_opcode_bytes_num()
+{
+ local op_bytes=$1
+ local opline=$2
+
+ local retval=0
+ substr=""
+
+ for opc in $op_bytes;
+ do
+ substr+="$opc"
+
+ # return if opcode bytes do not match @opline anymore
+ if ! echo $opline | grep -q "$substr";
+ then
+ break
+ fi
+
+ # add trailing space
+ substr+=" "
+ retval=$((retval+1))
+ done
+
+ return $retval
+}
+
+# Return the line number in objdump output to where the IP marker in the Code:
+# line points to
+#
+# Params:
+# @all_code: code in bytes without the marker
+# @dis_file: disassembled file
+# @ip_byte: The byte to which the IP points to
+get_faultlinenum()
+{
+ local all_code="$1"
+ local dis_file="$2"
+
+ # num bytes including IP byte
+ local num_bytes_ip=$(( $3 + 1 * $width ))
+
+ # Add the two header lines (we're counting from 1).
+ local retval=3
+
+ # remove marker
+ all_code=$(echo $all_code | sed -e 's/[<>()]//g')
+
+ while read line
+ do
+ get_substr_opcode_bytes_num "$all_code" "$line"
+ ate_opcodes=$?
+
+ if ! (( $ate_opcodes )); then
+ continue
+ fi
+
+ num_bytes_ip=$((num_bytes_ip - ($ate_opcodes * $width) ))
+ if (( $num_bytes_ip <= 0 )); then
+ break
+ fi
+
+ # Delete matched opcode bytes from all_code. For that, compute
+ # how many chars those opcodes are represented by and include
+ # trailing space.
+ #
+ # a byte is 2 chars, ate_opcodes is also the number of trailing
+ # spaces
+ del_chars=$(( ($ate_opcodes * $width * 2) + $ate_opcodes ))
+
+ all_code=$(echo $all_code | sed -e "s!^.\{$del_chars\}!!")
+
+ let "retval+=1"
+
+ done < $dis_file
+
+ return $retval
+}
+
marker=`expr index "$code" "\<"`
if [ $marker -eq 0 ]; then
marker=`expr index "$code" "\("`
fi
-
touch $T.oo
if [ $marker -ne 0 ]; then
- # 2 opcode bytes and a single space
- pc_sub=$(( $marker / 3 ))
+ # How many bytes to subtract from the program counter
+ # in order to get to the beginning virtual address of the
+ # Code:
+ pc_sub=$(( (($marker - 1) / (2 * $width + 1)) * $width ))
echo All code >> $T.oo
echo ======== >> $T.oo
beforemark=`echo "$code"`
echo -n " .$type 0x" > $T.s
+
echo $beforemark | sed -e 's/ /,0x/g; s/[<>()]//g' >> $T.s
+
disas $T $pc_sub
+
cat $T.dis >> $T.oo
- rm -f $T.o $T.s $T.dis
-# and fix code at-and-after marker
+ get_faultlinenum "$code" "$T.dis" $pc_sub
+ faultlinenum=$?
+
+ # and fix code at-and-after marker
code=`echo "$code" | cut -c$((${marker} + 1))-`
+
+ rm -f $T.o $T.s $T.dis
fi
+
echo Code starting with the faulting instruction > $T.aa
echo =========================================== >> $T.aa
code=`echo $code | sed -e 's/\r//;s/ [<(]/ /;s/[>)] / /;s/ /,0x/g; s/[>)]$//'`
disas $T 0
cat $T.dis >> $T.aa
-# (lines of whole $T.oo) - (lines of $T.aa, i.e. "Code starting") + 3,
-# i.e. the title + the "===..=" line (sed is counting from 1, 0 address is
-# special)
-faultlinenum=$(( $(wc -l $T.oo | cut -d" " -f1) - \
- $(wc -l $T.aa | cut -d" " -f1) + 3))
-
-faultline=`cat $T.dis | head -1 | cut -d":" -f2-`
-faultline=`echo "$faultline" | sed -e 's/\[/\\\[/g; s/\]/\\\]/g'`
-
cat $T.oo | sed -e "${faultlinenum}s/^\([^:]*:\)\(.*\)/\1\*\2\t\t<-- trapping instruction/"
echo
cat $T.aa
$MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install
cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE
cp .config %{buildroot}/boot/config-$KERNELRELEASE
- bzip2 -9 --keep vmlinux
- mv vmlinux.bz2 %{buildroot}/boot/vmlinux-$KERNELRELEASE.bz2
$S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/build
$S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/source
$S$M mkdir -p %{buildroot}/usr/src/kernels/$KERNELRELEASE
config INIT_STACK_ALL_PATTERN
bool "pattern-init everything (strongest)"
depends on CC_HAS_AUTO_VAR_INIT_PATTERN
+ depends on !KMSAN
help
Initializes everything on the stack (including padding)
with a specific debug value. This is intended to eliminate
config INIT_STACK_ALL_ZERO
bool "zero-init everything (strongest and safest)"
depends on CC_HAS_AUTO_VAR_INIT_ZERO
+ depends on !KMSAN
help
Initializes everything on the stack (including padding)
with a zero value. This is intended to eliminate all
config INIT_ON_ALLOC_DEFAULT_ON
bool "Enable heap memory zeroing on allocation by default"
+ depends on !KMSAN
help
This has the effect of setting "init_on_alloc=1" on the kernel
command line. This can be disabled with "init_on_alloc=0".
config INIT_ON_FREE_DEFAULT_ON
bool "Enable heap memory zeroing on free by default"
+ depends on !KMSAN
help
This has the effect of setting "init_on_free=1" on the kernel
command line. This can be disabled with "init_on_free=0".
snd_info_free_entry(rmidi->proc_entry);
rmidi->proc_entry = NULL;
- mutex_lock(®ister_mutex);
if (rmidi->ops && rmidi->ops->dev_unregister)
rmidi->ops->dev_unregister(rmidi);
- mutex_unlock(®ister_mutex);
snd_rawmidi_free_substreams(&rmidi->streams[SNDRV_RAWMIDI_STREAM_INPUT]);
snd_rawmidi_free_substreams(&rmidi->streams[SNDRV_RAWMIDI_STREAM_OUTPUT]);
mutex_unlock(&sound_oss_mutex);
return -ENOENT;
}
- unregister_sound_special(minor);
switch (SNDRV_MINOR_OSS_DEVICE(minor)) {
case SNDRV_MINOR_OSS_PCM:
track2 = SNDRV_MINOR_OSS(cidx, SNDRV_MINOR_OSS_AUDIO);
track2 = SNDRV_MINOR_OSS(cidx, SNDRV_MINOR_OSS_DMMIDI1);
break;
}
- if (track2 >= 0) {
- unregister_sound_special(track2);
+ if (track2 >= 0)
snd_oss_minors[track2] = NULL;
- }
snd_oss_minors[minor] = NULL;
mutex_unlock(&sound_oss_mutex);
+
+ /* call unregister_sound_special() outside sound_oss_mutex;
+ * otherwise may deadlock, as it can trigger the release of a card
+ */
+ unregister_sound_special(minor);
+ if (track2 >= 0)
+ unregister_sound_special(track2);
+
kfree(mptr);
return 0;
}
{ CS35L41_AMP_DIG_VOL_CTRL, 0x0000A678 }, // AMP_VOL_PCM Mute
};
-static int cs35l41_control_add(struct cs_dsp_coeff_ctl *cs_ctl)
+static void cs35l41_add_controls(struct cs35l41_hda *cs35l41)
{
- struct cs35l41_hda *cs35l41 = container_of(cs_ctl->dsp, struct cs35l41_hda, cs_dsp);
struct hda_cs_dsp_ctl_info info;
info.device_name = cs35l41->amp_name;
info.fw_type = cs35l41->firmware_type;
info.card = cs35l41->codec->card;
- return hda_cs_dsp_control_add(cs_ctl, &info);
+ hda_cs_dsp_add_controls(&cs35l41->cs_dsp, &info);
}
static const struct cs_dsp_client_ops client_ops = {
- .control_add = cs35l41_control_add,
.control_remove = hda_cs_dsp_control_remove,
};
if (ret)
goto err_release;
+ cs35l41_add_controls(cs35l41);
+
ret = cs35l41_save_calibration(cs35l41);
err_release:
struct cs_dsp *dsp = &cs35l41->cs_dsp;
cancel_work_sync(&cs35l41->fw_load_work);
+
+ mutex_lock(&cs35l41->fw_mutex);
cs35l41_shutdown_dsp(cs35l41);
cs_dsp_remove(dsp);
cs35l41->halo_initialized = false;
+ mutex_unlock(&cs35l41->fw_mutex);
}
/* Protection release cycle to get the speaker out of Safe-Mode */
struct regmap *reg = cs35l41->regmap;
int ret = 0;
- mutex_lock(&cs35l41->fw_mutex);
-
switch (action) {
case HDA_GEN_PCM_ACT_OPEN:
+ pm_runtime_get_sync(dev);
+ mutex_lock(&cs35l41->fw_mutex);
cs35l41->playback_started = true;
if (cs35l41->firmware_running) {
regmap_multi_reg_write(reg, cs35l41_hda_config_dsp,
CS35L41_AMP_EN_MASK, 1 << CS35L41_AMP_EN_SHIFT);
if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
regmap_write(reg, CS35L41_GPIO1_CTRL1, 0x00008001);
+ mutex_unlock(&cs35l41->fw_mutex);
break;
case HDA_GEN_PCM_ACT_PREPARE:
+ mutex_lock(&cs35l41->fw_mutex);
ret = cs35l41_global_enable(reg, cs35l41->hw_cfg.bst_type, 1);
+ mutex_unlock(&cs35l41->fw_mutex);
break;
case HDA_GEN_PCM_ACT_CLEANUP:
+ mutex_lock(&cs35l41->fw_mutex);
regmap_multi_reg_write(reg, cs35l41_hda_mute, ARRAY_SIZE(cs35l41_hda_mute));
ret = cs35l41_global_enable(reg, cs35l41->hw_cfg.bst_type, 0);
+ mutex_unlock(&cs35l41->fw_mutex);
break;
case HDA_GEN_PCM_ACT_CLOSE:
+ mutex_lock(&cs35l41->fw_mutex);
ret = regmap_update_bits(reg, CS35L41_PWR_CTRL2,
CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT);
if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
}
cs35l41_irq_release(cs35l41);
cs35l41->playback_started = false;
+ mutex_unlock(&cs35l41->fw_mutex);
+
+ pm_runtime_mark_last_busy(dev);
+ pm_runtime_put_autosuspend(dev);
break;
default:
dev_warn(cs35l41->dev, "Playback action not supported: %d\n", action);
break;
}
- mutex_unlock(&cs35l41->fw_mutex);
-
if (ret)
dev_err(cs35l41->dev, "Regmap access fail: %d\n", ret);
}
rx_slot);
}
+static void cs35l41_ready_for_reset(struct cs35l41_hda *cs35l41)
+{
+ mutex_lock(&cs35l41->fw_mutex);
+ if (cs35l41->firmware_running) {
+
+ regcache_cache_only(cs35l41->regmap, false);
+
+ cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);
+ cs35l41_shutdown_dsp(cs35l41);
+ cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type);
+
+ regcache_cache_only(cs35l41->regmap, true);
+ regcache_mark_dirty(cs35l41->regmap);
+ }
+ mutex_unlock(&cs35l41->fw_mutex);
+}
+
+static int cs35l41_system_suspend(struct device *dev)
+{
+ struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
+ int ret;
+
+ dev_dbg(cs35l41->dev, "System Suspend\n");
+
+ if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) {
+ dev_err(cs35l41->dev, "System Suspend not supported\n");
+ return -EINVAL;
+ }
+
+ ret = pm_runtime_force_suspend(dev);
+ if (ret)
+ return ret;
+
+ /* Shutdown DSP before system suspend */
+ cs35l41_ready_for_reset(cs35l41);
+
+ /*
+ * Reset GPIO may be shared, so cannot reset here.
+ * However beyond this point, amps may be powered down.
+ */
+ return 0;
+}
+
+static int cs35l41_system_resume(struct device *dev)
+{
+ struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
+ int ret;
+
+ dev_dbg(cs35l41->dev, "System Resume\n");
+
+ if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) {
+ dev_err(cs35l41->dev, "System Resume not supported\n");
+ return -EINVAL;
+ }
+
+ if (cs35l41->reset_gpio) {
+ usleep_range(2000, 2100);
+ gpiod_set_value_cansleep(cs35l41->reset_gpio, 1);
+ }
+
+ usleep_range(2000, 2100);
+
+ ret = pm_runtime_force_resume(dev);
+
+ mutex_lock(&cs35l41->fw_mutex);
+ if (!ret && cs35l41->request_fw_load && !cs35l41->fw_request_ongoing) {
+ cs35l41->fw_request_ongoing = true;
+ schedule_work(&cs35l41->fw_load_work);
+ }
+ mutex_unlock(&cs35l41->fw_mutex);
+
+ return ret;
+}
+
static int cs35l41_runtime_suspend(struct device *dev)
{
struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
+ int ret = 0;
- dev_dbg(cs35l41->dev, "Suspend\n");
+ dev_dbg(cs35l41->dev, "Runtime Suspend\n");
- if (!cs35l41->firmware_running)
+ if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) {
+ dev_dbg(cs35l41->dev, "Runtime Suspend not supported\n");
return 0;
+ }
- if (cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap, cs35l41->hw_cfg.bst_type) < 0)
- return 0;
+ mutex_lock(&cs35l41->fw_mutex);
+
+ if (cs35l41->playback_started) {
+ regmap_multi_reg_write(cs35l41->regmap, cs35l41_hda_mute,
+ ARRAY_SIZE(cs35l41_hda_mute));
+ cs35l41_global_enable(cs35l41->regmap, cs35l41->hw_cfg.bst_type, 0);
+ regmap_update_bits(cs35l41->regmap, CS35L41_PWR_CTRL2,
+ CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT);
+ if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
+ regmap_write(cs35l41->regmap, CS35L41_GPIO1_CTRL1, 0x00000001);
+ regmap_update_bits(cs35l41->regmap, CS35L41_PWR_CTRL2,
+ CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK,
+ 0 << CS35L41_VMON_EN_SHIFT | 0 << CS35L41_IMON_EN_SHIFT);
+ cs35l41->playback_started = false;
+ }
+
+ if (cs35l41->firmware_running) {
+ ret = cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap,
+ cs35l41->hw_cfg.bst_type);
+ if (ret)
+ goto err;
+ } else {
+ cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type);
+ }
regcache_cache_only(cs35l41->regmap, true);
regcache_mark_dirty(cs35l41->regmap);
- return 0;
+err:
+ mutex_unlock(&cs35l41->fw_mutex);
+
+ return ret;
}
static int cs35l41_runtime_resume(struct device *dev)
{
struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
- int ret;
+ int ret = 0;
- dev_dbg(cs35l41->dev, "Resume.\n");
+ dev_dbg(cs35l41->dev, "Runtime Resume\n");
if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) {
- dev_dbg(cs35l41->dev, "System does not support Resume\n");
+ dev_dbg(cs35l41->dev, "Runtime Resume not supported\n");
return 0;
}
- if (!cs35l41->firmware_running)
- return 0;
+ mutex_lock(&cs35l41->fw_mutex);
regcache_cache_only(cs35l41->regmap, false);
- ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);
- if (ret) {
- regcache_cache_only(cs35l41->regmap, true);
- return ret;
+ if (cs35l41->firmware_running) {
+ ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);
+ if (ret) {
+ dev_warn(cs35l41->dev, "Unable to exit Hibernate.");
+ goto err;
+ }
}
/* Test key needs to be unlocked to allow the OTP settings to re-apply */
cs35l41_test_key_lock(cs35l41->dev, cs35l41->regmap);
if (ret) {
dev_err(cs35l41->dev, "Failed to restore register cache: %d\n", ret);
- return ret;
+ goto err;
}
if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
cs35l41_init_boost(cs35l41->dev, cs35l41->regmap, &cs35l41->hw_cfg);
- return 0;
-}
-
-static int cs35l41_hda_suspend_hook(struct device *dev)
-{
- dev_dbg(dev, "Request Suspend\n");
- pm_runtime_mark_last_busy(dev);
- return pm_runtime_put_autosuspend(dev);
-}
+err:
+ mutex_unlock(&cs35l41->fw_mutex);
-static int cs35l41_hda_resume_hook(struct device *dev)
-{
- dev_dbg(dev, "Request Resume\n");
- return pm_runtime_get_sync(dev);
+ return ret;
}
static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
static void cs35l41_load_firmware(struct cs35l41_hda *cs35l41, bool load)
{
- pm_runtime_get_sync(cs35l41->dev);
-
if (cs35l41->firmware_running && !load) {
dev_dbg(cs35l41->dev, "Unloading Firmware\n");
cs35l41_shutdown_dsp(cs35l41);
} else {
dev_dbg(cs35l41->dev, "Unable to Load firmware.\n");
}
-
- pm_runtime_mark_last_busy(cs35l41->dev);
- pm_runtime_put_autosuspend(cs35l41->dev);
}
static int cs35l41_fw_load_ctl_get(struct snd_kcontrol *kcontrol,
{
struct cs35l41_hda *cs35l41 = container_of(work, struct cs35l41_hda, fw_load_work);
+ pm_runtime_get_sync(cs35l41->dev);
+
mutex_lock(&cs35l41->fw_mutex);
/* Recheck if playback is ongoing, mutex will block playback during firmware loading */
if (cs35l41->playback_started)
- dev_err(cs35l41->dev, "Cannot Load/Unload firmware during Playback\n");
+ dev_err(cs35l41->dev, "Cannot Load/Unload firmware during Playback. Retrying...\n");
else
cs35l41_load_firmware(cs35l41, cs35l41->request_fw_load);
cs35l41->fw_request_ongoing = false;
mutex_unlock(&cs35l41->fw_mutex);
+
+ pm_runtime_mark_last_busy(cs35l41->dev);
+ pm_runtime_put_autosuspend(cs35l41->dev);
}
static int cs35l41_fw_load_ctl_put(struct snd_kcontrol *kcontrol,
pm_runtime_get_sync(dev);
+ mutex_lock(&cs35l41->fw_mutex);
+
comps->dev = dev;
if (!cs35l41->acpi_subsystem_id)
cs35l41->acpi_subsystem_id = kasprintf(GFP_KERNEL, "%.8x",
if (firmware_autostart) {
dev_dbg(cs35l41->dev, "Firmware Autostart.\n");
cs35l41->request_fw_load = true;
- mutex_lock(&cs35l41->fw_mutex);
if (cs35l41_smart_amp(cs35l41) < 0)
dev_warn(cs35l41->dev, "Cannot Run Firmware, reverting to dsp bypass...\n");
- mutex_unlock(&cs35l41->fw_mutex);
} else {
dev_dbg(cs35l41->dev, "Firmware Autostart is disabled.\n");
}
ret = cs35l41_create_controls(cs35l41);
comps->playback_hook = cs35l41_hda_playback_hook;
- comps->suspend_hook = cs35l41_hda_suspend_hook;
- comps->resume_hook = cs35l41_hda_resume_hook;
+
+ mutex_unlock(&cs35l41->fw_mutex);
pm_runtime_mark_last_busy(dev);
pm_runtime_put_autosuspend(dev);
const struct dev_pm_ops cs35l41_hda_pm_ops = {
RUNTIME_PM_OPS(cs35l41_runtime_suspend, cs35l41_runtime_resume, NULL)
+ SYSTEM_SLEEP_PM_OPS(cs35l41_system_suspend, cs35l41_system_resume)
};
EXPORT_SYMBOL_NS_GPL(cs35l41_hda_pm_ops, SND_HDA_SCODEC_CS35L41);
char name[HDA_MAX_NAME_SIZE];
struct hda_codec *codec;
void (*playback_hook)(struct device *dev, int action);
- int (*suspend_hook)(struct device *dev);
- int (*resume_hook)(struct device *dev);
};
return out;
}
-static int hda_cs_dsp_add_kcontrol(struct hda_cs_dsp_coeff_ctl *ctl, const char *name)
+static void hda_cs_dsp_add_kcontrol(struct hda_cs_dsp_coeff_ctl *ctl, const char *name)
{
struct cs_dsp_coeff_ctl *cs_ctl = ctl->cs_ctl;
struct snd_kcontrol_new kcontrol = {0};
if (cs_ctl->len > ADSP_MAX_STD_CTRL_SIZE) {
dev_err(cs_ctl->dsp->dev, "KControl %s: length %zu exceeds maximum %d\n", name,
cs_ctl->len, ADSP_MAX_STD_CTRL_SIZE);
- return -EINVAL;
+ return;
}
kcontrol.name = name;
/* Save ctl inside private_data, ctl is owned by cs_dsp,
* and will be freed when cs_dsp removes the control */
kctl = snd_ctl_new1(&kcontrol, (void *)ctl);
- if (!kctl) {
- ret = -ENOMEM;
- return ret;
- }
+ if (!kctl)
+ return;
ret = snd_ctl_add(ctl->card, kctl);
if (ret) {
dev_err(cs_ctl->dsp->dev, "Failed to add KControl %s = %d\n", kcontrol.name, ret);
- return ret;
+ return;
}
dev_dbg(cs_ctl->dsp->dev, "Added KControl: %s\n", kcontrol.name);
ctl->kctl = kctl;
-
- return 0;
}
-int hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, struct hda_cs_dsp_ctl_info *info)
+static void hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl,
+ const struct hda_cs_dsp_ctl_info *info)
{
struct cs_dsp *cs_dsp = cs_ctl->dsp;
char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
const char *region_name;
int ret;
- if (cs_ctl->flags & WMFW_CTL_FLAG_SYS)
- return 0;
-
region_name = cs_dsp_mem_region_name(cs_ctl->alg_region.type);
if (!region_name) {
- dev_err(cs_dsp->dev, "Unknown region type: %d\n", cs_ctl->alg_region.type);
- return -EINVAL;
+ dev_warn(cs_dsp->dev, "Unknown region type: %d\n", cs_ctl->alg_region.type);
+ return;
}
ret = scnprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s %s %.12s %x", info->device_name,
ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
if (!ctl)
- return -ENOMEM;
+ return;
ctl->cs_ctl = cs_ctl;
ctl->card = info->card;
cs_ctl->priv = ctl;
- ret = hda_cs_dsp_add_kcontrol(ctl, name);
- if (ret) {
- dev_err(cs_dsp->dev, "Error (%d) adding control %s\n", ret, name);
- kfree(ctl);
- return ret;
- }
+ hda_cs_dsp_add_kcontrol(ctl, name);
+}
- return 0;
+void hda_cs_dsp_add_controls(struct cs_dsp *dsp, const struct hda_cs_dsp_ctl_info *info)
+{
+ struct cs_dsp_coeff_ctl *cs_ctl;
+
+ /*
+ * pwr_lock would cause mutex inversion with ALSA control lock compared
+ * to the get/put functions.
+ * It is safe to walk the list without holding a mutex because entries
+ * are persistent and only cs_dsp_power_up() or cs_dsp_remove() can
+ * change the list.
+ */
+ lockdep_assert_not_held(&dsp->pwr_lock);
+
+ list_for_each_entry(cs_ctl, &dsp->ctl_list, list) {
+ if (cs_ctl->flags & WMFW_CTL_FLAG_SYS)
+ continue;
+
+ if (cs_ctl->priv)
+ continue;
+
+ hda_cs_dsp_control_add(cs_ctl, info);
+ }
}
-EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_control_add, SND_HDA_CS_DSP_CONTROLS);
+EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_add_controls, SND_HDA_CS_DSP_CONTROLS);
void hda_cs_dsp_control_remove(struct cs_dsp_coeff_ctl *cs_ctl)
{
struct hda_cs_dsp_coeff_ctl *ctl;
int ret;
+ mutex_lock(&dsp->pwr_lock);
cs_ctl = cs_dsp_get_ctl(dsp, name, type, alg);
- if (!cs_ctl)
- return -EINVAL;
-
- ctl = cs_ctl->priv;
-
ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, buf, len);
+ mutex_unlock(&dsp->pwr_lock);
if (ret)
return ret;
if (cs_ctl->flags & WMFW_CTL_FLAG_SYS)
return 0;
+ ctl = cs_ctl->priv;
+
snd_ctl_notify(ctl->card, SNDRV_CTL_EVENT_MASK_VALUE, &ctl->kctl->id);
return 0;
int hda_cs_dsp_read_ctl(struct cs_dsp *dsp, const char *name, int type,
unsigned int alg, void *buf, size_t len)
{
- struct cs_dsp_coeff_ctl *cs_ctl;
+ int ret;
- cs_ctl = cs_dsp_get_ctl(dsp, name, type, alg);
- if (!cs_ctl)
- return -EINVAL;
+ mutex_lock(&dsp->pwr_lock);
+ ret = cs_dsp_coeff_read_ctrl(cs_dsp_get_ctl(dsp, name, type, alg), 0, buf, len);
+ mutex_unlock(&dsp->pwr_lock);
+
+ return ret;
- return cs_dsp_coeff_read_ctrl(cs_ctl, 0, buf, len);
}
EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_read_ctl, SND_HDA_CS_DSP_CONTROLS);
extern const char * const hda_cs_dsp_fw_ids[HDA_CS_DSP_NUM_FW];
-int hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, struct hda_cs_dsp_ctl_info *info);
+void hda_cs_dsp_add_controls(struct cs_dsp *dsp, const struct hda_cs_dsp_ctl_info *info);
void hda_cs_dsp_control_remove(struct cs_dsp_coeff_ctl *cs_ctl);
int hda_cs_dsp_write_ctl(struct cs_dsp *dsp, const char *name, int type,
unsigned int alg, const void *buf, size_t len);
static int alc269_suspend(struct hda_codec *codec)
{
struct alc_spec *spec = codec->spec;
- int i;
if (spec->has_alc5505_dsp)
alc5505_dsp_suspend(codec);
- for (i = 0; i < HDA_MAX_COMPONENTS; i++)
- if (spec->comps[i].suspend_hook)
- spec->comps[i].suspend_hook(spec->comps[i].dev);
-
return alc_suspend(codec);
}
static int alc269_resume(struct hda_codec *codec)
{
struct alc_spec *spec = codec->spec;
- int i;
if (spec->codec_variant == ALC269_TYPE_ALC269VB)
alc269vb_toggle_power_output(codec, 0);
if (spec->has_alc5505_dsp)
alc5505_dsp_resume(codec);
- for (i = 0; i < HDA_MAX_COMPONENTS; i++)
- if (spec->comps[i].resume_hook)
- spec->comps[i].resume_hook(spec->comps[i].dev);
-
return 0;
}
#endif /* CONFIG_PM */
{
struct hda_codec *cdc = dev_to_hda_codec(dev);
struct alc_spec *spec = cdc->spec;
- int ret, i;
+ int ret;
ret = component_bind_all(dev, spec->comps);
if (ret)
return ret;
- if (snd_hdac_is_power_on(&cdc->core)) {
- codec_dbg(cdc, "Resuming after bind.\n");
- for (i = 0; i < HDA_MAX_COMPONENTS; i++)
- if (spec->comps[i].resume_hook)
- spec->comps[i].resume_hook(spec->comps[i].dev);
- }
-
return 0;
}
[ALC285_FIXUP_ASUS_G533Z_PINS] = {
.type = HDA_FIXUP_PINS,
.v.pins = (const struct hda_pintbl[]) {
- { 0x14, 0x90170120 },
+ { 0x14, 0x90170152 }, /* Speaker Surround Playback Switch */
+ { 0x19, 0x03a19020 }, /* Mic Boost Volume */
+ { 0x1a, 0x03a11c30 }, /* Mic Boost Volume */
+ { 0x1e, 0x90170151 }, /* Rear jack, IN OUT EAPD Detect */
+ { 0x21, 0x03211420 },
{ }
},
- .chained = true,
- .chain_id = ALC294_FIXUP_ASUS_G513_PINS,
},
[ALC294_FIXUP_ASUS_COEF_1B] = {
.type = HDA_FIXUP_VERBS,
SND_PCI_QUIRK(0x1028, 0x0871, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC),
SND_PCI_QUIRK(0x1028, 0x0872, "Dell Precision 3630", ALC255_FIXUP_DELL_HEADSET_MIC),
SND_PCI_QUIRK(0x1028, 0x0873, "Dell Precision 3930", ALC255_FIXUP_DUMMY_LINEOUT_VERB),
- SND_PCI_QUIRK(0x1028, 0x087d, "Dell Precision 5530", ALC289_FIXUP_DUAL_SPK),
SND_PCI_QUIRK(0x1028, 0x08ad, "Dell WYSE AIO", ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE),
SND_PCI_QUIRK(0x1028, 0x08ae, "Dell WYSE NB", ALC225_FIXUP_DELL1_MIC_NO_PRESENCE),
SND_PCI_QUIRK(0x1028, 0x0935, "Dell", ALC274_FIXUP_DELL_AIO_LINEOUT_VERB),
SND_PCI_QUIRK(0x1043, 0x1e8e, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA401),
SND_PCI_QUIRK(0x1043, 0x1c52, "ASUS Zephyrus G15 2022", ALC289_FIXUP_ASUS_GA401),
SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401),
+ SND_PCI_QUIRK(0x1043, 0x1f92, "ASUS ROG Flow X16", ALC289_FIXUP_ASUS_GA401),
SND_PCI_QUIRK(0x1043, 0x3030, "ASUS ZN270IE", ALC256_FIXUP_ASUS_AIO_GPIO2),
SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC),
SND_PCI_QUIRK(0x1043, 0x834a, "ASUS S101", ALC269_FIXUP_STEREO_DMIC),
SND_PCI_QUIRK(0x10ec, 0x10f2, "Intel Reference board", ALC700_FIXUP_INTEL_REFERENCE),
SND_PCI_QUIRK(0x10ec, 0x118c, "Medion EE4254 MD62100", ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE),
SND_PCI_QUIRK(0x10ec, 0x1230, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK),
+ SND_PCI_QUIRK(0x10ec, 0x124c, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK),
SND_PCI_QUIRK(0x10ec, 0x1252, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK),
SND_PCI_QUIRK(0x10ec, 0x1254, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK),
SND_PCI_QUIRK(0x10f7, 0x8338, "Panasonic CF-SZ6", ALC269_FIXUP_HEADSET_MODE),
in a stream */
bool implicit_fb_sync; /* syncs with implicit feedback */
bool lowlatency_playback; /* low-latency playback mode */
- bool need_setup; /* (re-)need for configure? */
+ bool need_setup; /* (re-)need for hw_params? */
+ bool need_prepare; /* (re-)need for prepare? */
/* for hw constraints */
const struct audioformat *cur_audiofmt;
unsigned char iface;
bool need_setup;
int opened;
+ int altset;
struct list_head list;
};
ep->implicit_fb_sync = fp->implicit_fb;
ep->need_setup = true;
+ ep->need_prepare = true;
usb_audio_dbg(chip, " channels=%d, rate=%d, format=%s, period_bytes=%d, periods=%d, implicit_fb=%d\n",
ep->cur_channels, ep->cur_rate,
int altset = set ? ep->altsetting : 0;
int err;
+ if (ep->iface_ref->altset == altset)
+ return 0;
+
usb_audio_dbg(chip, "Setting usb interface %d:%d for EP 0x%x\n",
ep->iface, altset, ep->ep_num);
err = usb_set_interface(chip->dev, ep->iface, altset);
if (chip->quirk_flags & QUIRK_FLAG_IFACE_DELAY)
msleep(50);
+ ep->iface_ref->altset = altset;
return 0;
}
/* Prepare for suspening EP, called from the main suspend handler */
void snd_usb_endpoint_suspend(struct snd_usb_endpoint *ep)
{
- ep->need_setup = true;
+ ep->need_prepare = true;
if (ep->iface_ref)
ep->iface_ref->need_setup = true;
if (ep->clock_ref)
struct snd_usb_endpoint *ep)
{
const struct audioformat *fmt = ep->cur_audiofmt;
- int err;
+ int err = 0;
+
+ mutex_lock(&chip->mutex);
+ if (!ep->need_setup)
+ goto unlock;
/* release old buffers, if any */
err = release_urbs(ep, false);
if (err < 0)
- return err;
+ goto unlock;
ep->datainterval = fmt->datainterval;
ep->maxpacksize = fmt->maxpacksize;
usb_audio_dbg(chip, "Set up %d URBS, ret=%d\n", ep->nurbs, err);
if (err < 0)
- return err;
+ goto unlock;
/* some unit conversions in runtime */
ep->maxframesize = ep->maxpacksize / ep->cur_frame_bytes;
ep->curframesize = ep->curpacksize / ep->cur_frame_bytes;
- return update_clock_ref_rate(chip, ep);
+ err = update_clock_ref_rate(chip, ep);
+ if (err >= 0) {
+ ep->need_setup = false;
+ err = 0;
+ }
+
+ unlock:
+ mutex_unlock(&chip->mutex);
+ return err;
}
static int init_sample_rate(struct snd_usb_audio *chip,
mutex_lock(&chip->mutex);
if (WARN_ON(!ep->iface_ref))
goto unlock;
- if (!ep->need_setup)
+ if (!ep->need_prepare)
goto unlock;
/* If the interface has been already set up, just set EP parameters */
ep->iface_ref->need_setup = false;
done:
- ep->need_setup = false;
+ ep->need_prepare = false;
err = 1;
unlock:
#include "msr-index.h"
+/* IBS_OP_DATA2 DataSrc */
+#define IBS_DATA_SRC_LOC_CACHE 2
+#define IBS_DATA_SRC_DRAM 3
+#define IBS_DATA_SRC_REM_CACHE 4
+#define IBS_DATA_SRC_IO 7
+
+/* IBS_OP_DATA2 DataSrc Extension */
+#define IBS_DATA_SRC_EXT_LOC_CACHE 1
+#define IBS_DATA_SRC_EXT_NEAR_CCX_CACHE 2
+#define IBS_DATA_SRC_EXT_DRAM 3
+#define IBS_DATA_SRC_EXT_FAR_CCX_CACHE 5
+#define IBS_DATA_SRC_EXT_PMEM 6
+#define IBS_DATA_SRC_EXT_IO 7
+#define IBS_DATA_SRC_EXT_EXT_MEM 8
+#define IBS_DATA_SRC_EXT_PEER_AGENT_MEM 12
+
/*
* IBS Hardware MSRs
*/
* Return Stack Buffer Predictions.
*/
+#define ARCH_CAP_XAPIC_DISABLE BIT(21) /*
+ * IA32_XAPIC_DISABLE_STATUS MSR
+ * supported
+ */
+
#define MSR_IA32_FLUSH_CMD 0x0000010b
#define L1D_FLUSH BIT(0) /*
* Writeback and invalidate the
#define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301
#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302
+/* AMD Last Branch Record MSRs */
+#define MSR_AMD64_LBR_SELECT 0xc000010e
+
/* Fam 17h MSRs */
#define MSR_F17H_IRPERF 0xc00000e9
#define MSR_AMD_DBG_EXTN_CFG 0xc000010f
#define MSR_AMD_SAMP_BR_FROM 0xc0010300
+#define DBG_EXTN_CFG_LBRV2EN BIT_ULL(6)
+
#define MSR_IA32_MPERF 0x000000e7
#define MSR_IA32_APERF 0x000000e8
#define MSR_IA32_HW_FEEDBACK_PTR 0x17d0
#define MSR_IA32_HW_FEEDBACK_CONFIG 0x17d1
+/* x2APIC locked status */
+#define MSR_IA32_XAPIC_DISABLE_STATUS 0xBD
+#define LEGACY_XAPIC_DISABLED BIT(0) /*
+ * x2APIC mode is locked and
+ * disabling x2APIC will cause
+ * a #GP
+ */
+
#endif /* _ASM_X86_MSR_INDEX_H */
libaio \
libzstd
+#
+# Declare group members of a feature to display the logical OR of the detection
+# result instead of each member result.
+#
+FEATURE_GROUP_MEMBERS-libbfd = libbfd-liberty libbfd-liberty-z
+
# Set FEATURE_CHECK_(C|LD)FLAGS-all for all FEATURE_TESTS features.
# If in the future we need per-feature checks/flags for features not
# mentioned in this list we need to refactor this ;-).
#
# Print the result of the feature test:
#
-feature_print_status = $(eval $(feature_print_status_code)) $(info $(MSG))
+feature_print_status = $(eval $(feature_print_status_code))
+
+feature_group = $(eval $(feature_gen_group)) $(GROUP)
+
+define feature_gen_group
+ GROUP := $(1)
+ ifneq ($(feature_verbose),1)
+ GROUP += $(FEATURE_GROUP_MEMBERS-$(1))
+ endif
+endef
define feature_print_status_code
- ifeq ($(feature-$(1)), 1)
- MSG = $(shell printf '...%30s: [ \033[32mon\033[m ]' $(1))
+ ifneq (,$(filter 1,$(foreach feat,$(call feature_group,$(feat)),$(feature-$(feat)))))
+ MSG = $(shell printf '...%40s: [ \033[32mon\033[m ]' $(1))
else
- MSG = $(shell printf '...%30s: [ \033[31mOFF\033[m ]' $(1))
+ MSG = $(shell printf '...%40s: [ \033[31mOFF\033[m ]' $(1))
endif
endef
-feature_print_text = $(eval $(feature_print_text_code)) $(info $(MSG))
+feature_print_text = $(eval $(feature_print_text_code))
define feature_print_text_code
- MSG = $(shell printf '...%30s: %s' $(1) $(2))
+ MSG = $(shell printf '...%40s: %s' $(1) $(2))
endef
#
feature_verbose := 1
endif
+ifneq ($(feature_verbose),1)
+ #
+ # Determine the features to omit from the displayed message, as only the
+ # logical OR of the detection result will be shown.
+ #
+ FEATURE_OMIT := $(foreach feat,$(FEATURE_DISPLAY),$(FEATURE_GROUP_MEMBERS-$(feat)))
+endif
+
feature_display_entries = $(eval $(feature_display_entries_code))
define feature_display_entries_code
ifeq ($(feature_display),1)
- $(info )
- $(info Auto-detecting system features:)
- $(foreach feat,$(FEATURE_DISPLAY),$(call feature_print_status,$(feat),))
- ifneq ($(feature_verbose),1)
- $(info )
- endif
+ $$(info )
+ $$(info Auto-detecting system features:)
+ $(foreach feat,$(filter-out $(FEATURE_OMIT),$(FEATURE_DISPLAY)),$(call feature_print_status,$(feat),) $$(info $(MSG)))
endif
ifeq ($(feature_verbose),1)
- TMP := $(filter-out $(FEATURE_DISPLAY),$(FEATURE_TESTS))
- $(foreach feat,$(TMP),$(call feature_print_status,$(feat),))
- $(info )
+ $(eval TMP := $(filter-out $(FEATURE_DISPLAY),$(FEATURE_TESTS)))
+ $(foreach feat,$(TMP),$(call feature_print_status,$(feat),) $$(info $(MSG)))
endif
endef
ifeq ($(FEATURE_DISPLAY_DEFERRED),)
$(call feature_display_entries)
+ $(info )
endif
#define HUGETLB_FLAG_ENCODE_SHIFT 26
#define HUGETLB_FLAG_ENCODE_MASK 0x3f
-#define HUGETLB_FLAG_ENCODE_16KB (14 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_64KB (16 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_512KB (19 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_1MB (20 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_2MB (21 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_8MB (23 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_16MB (24 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_32MB (25 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_256MB (28 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_512MB (29 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_1GB (30 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_2GB (31 << HUGETLB_FLAG_ENCODE_SHIFT)
-#define HUGETLB_FLAG_ENCODE_16GB (34 << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_16KB (14U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_64KB (16U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_512KB (19U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_1MB (20U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_2MB (21U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_8MB (23U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_16MB (24U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_32MB (25U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_256MB (28U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_512MB (29U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_1GB (30U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_2GB (31U << HUGETLB_FLAG_ENCODE_SHIFT)
+#define HUGETLB_FLAG_ENCODE_16GB (34U << HUGETLB_FLAG_ENCODE_SHIFT)
#endif /* _ASM_GENERIC_HUGETLB_ENCODE_H_ */
unsigned int align, unsigned int flags,
void (*ctor)(void *));
+void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list);
+int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
+ void **list);
+
#endif /* _TOOLS_SLAB_H */
#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */
+#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */
+
/* compatibility flags */
#define MAP_FILE 0
PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
+ PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */
+
PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
};
PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
+ PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
+
PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
};
PERF_BR_COND_RET = 10, /* conditional function return */
PERF_BR_ERET = 11, /* exception return */
PERF_BR_IRQ = 12, /* irq */
+ PERF_BR_SERROR = 13, /* system error */
+ PERF_BR_NO_TX = 14, /* not in transaction */
+ PERF_BR_EXTEND_ABI = 15, /* extend ABI */
PERF_BR_MAX,
};
+enum {
+ PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */
+ PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */
+ PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */
+ PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */
+ PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */
+ PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */
+ PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */
+ PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */
+ PERF_BR_NEW_MAX,
+};
+
+enum {
+ PERF_BR_PRIV_UNKNOWN = 0,
+ PERF_BR_PRIV_USER = 1,
+ PERF_BR_PRIV_KERNEL = 2,
+ PERF_BR_PRIV_HV = 3,
+};
+
+#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1
+#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2
+#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3
+#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4
+#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5
+
#define PERF_SAMPLE_BRANCH_PLM_ALL \
(PERF_SAMPLE_BRANCH_USER|\
PERF_SAMPLE_BRANCH_KERNEL|\
#define PERF_MEM_LVLNUM_L2 0x02 /* L2 */
#define PERF_MEM_LVLNUM_L3 0x03 /* L3 */
#define PERF_MEM_LVLNUM_L4 0x04 /* L4 */
-/* 5-0xa available */
+/* 5-0x8 available */
+#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */
+#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */
#define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */
#define PERF_MEM_LVLNUM_LFB 0x0c /* LFB */
#define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */
abort:1, /* transaction abort */
cycles:16, /* cycle count to last branch */
type:4, /* branch type */
- reserved:40;
+ new_type:4, /* additional branch type */
+ priv:3, /* privilege level */
+ reserved:33;
};
union perf_sample_weight {
};
enum fdarray_flags {
- fdarray_flag__default = 0x00000000,
- fdarray_flag__nonfilterable = 0x00000001
+ fdarray_flag__default = 0x00000000,
+ fdarray_flag__nonfilterable = 0x00000001,
+ fdarray_flag__non_perf_event = 0x00000002,
};
void fdarray__init(struct fdarray *fda, int nr_autogrow);
* We already have cpus for evsel (via PMU sysfs) so
* keep it, if there's no target cpu list defined.
*/
- if (!evsel->own_cpus ||
- (!evsel->system_wide && evlist->has_user_cpus) ||
- (!evsel->system_wide &&
- !evsel->requires_cpu &&
- perf_cpu_map__empty(evlist->user_requested_cpus))) {
+ if (evsel->system_wide) {
+ perf_cpu_map__put(evsel->cpus);
+ evsel->cpus = perf_cpu_map__new(NULL);
+ } else if (!evsel->own_cpus || evlist->has_user_cpus ||
+ (!evsel->requires_cpu && perf_cpu_map__empty(evlist->user_requested_cpus))) {
perf_cpu_map__put(evsel->cpus);
evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus);
} else if (evsel->cpus != evsel->own_cpus) {
evsel->cpus = perf_cpu_map__get(evsel->own_cpus);
}
- if (!evsel->system_wide) {
+ if (evsel->system_wide) {
+ perf_thread_map__put(evsel->threads);
+ evsel->threads = perf_thread_map__new_dummy();
+ } else {
perf_thread_map__put(evsel->threads);
evsel->threads = perf_thread_map__get(evlist->threads);
}
{
struct perf_evsel *evsel;
- /* Recomputing all_cpus, so start with a blank slate. */
- perf_cpu_map__put(evlist->all_cpus);
- evlist->all_cpus = NULL;
+ evlist->needs_map_propagation = true;
perf_evlist__for_each_evsel(evlist, evsel)
__perf_evlist__propagate_maps(evlist, evsel);
evsel->idx = evlist->nr_entries;
list_add_tail(&evsel->node, &evlist->entries);
evlist->nr_entries += 1;
- __perf_evlist__propagate_maps(evlist, evsel);
+
+ if (evlist->needs_map_propagation)
+ __perf_evlist__propagate_maps(evlist, evsel);
}
void perf_evlist__remove(struct perf_evlist *evlist,
evlist->threads = perf_thread_map__get(threads);
}
- if (!evlist->all_cpus && cpus)
- evlist->all_cpus = perf_cpu_map__get(cpus);
-
perf_evlist__propagate_maps(evlist);
}
if (ops->idx)
ops->idx(evlist, evsel, mp, idx);
+ /* Debug message used by test scripts */
pr_debug("idx %d: mmapping fd %d\n", idx, *output);
if (ops->mmap(map, mp, *output, evlist_cpu) < 0)
return -1;
if (!idx)
perf_evlist__set_mmap_first(evlist, map, overwrite);
} else {
+ /* Debug message used by test scripts */
pr_debug("idx %d: set output fd %d -> %d\n", idx, fd, *output);
if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
return -1;
if (ncpus == 0 || nthreads == 0)
return 0;
- if (evsel->system_wide)
- nthreads = 1;
-
evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
if (evsel->sample_id == NULL)
return -ENOMEM;
int nr_entries;
int nr_groups;
bool has_user_cpus;
+ bool needs_map_propagation;
/**
* The cpus passed from the command line or all online CPUs by
* default.
#include <linux/types.h>
#include <linux/limits.h>
#include <linux/bpf.h>
-#include <linux/compiler.h>
#include <sys/types.h> /* pid_t */
#define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem))
enum {
PERF_CPU_MAP__CPUS = 0,
PERF_CPU_MAP__MASK = 1,
+ PERF_CPU_MAP__RANGE_CPUS = 2,
};
/*
#pragma GCC diagnostic ignored "-Wpacked"
#pragma GCC diagnostic ignored "-Wattributes"
-struct __packed perf_record_cpu_map_data {
+/*
+ * An encoding of a CPU map for a range starting at start_cpu through to
+ * end_cpu. If any_cpu is 1, an any CPU (-1) value (aka dummy value) is present.
+ */
+struct perf_record_range_cpu_map {
+ __u8 any_cpu;
+ __u8 __pad;
+ __u16 start_cpu;
+ __u16 end_cpu;
+};
+
+struct perf_record_cpu_map_data {
__u16 type;
union {
/* Used when type == PERF_CPU_MAP__CPUS. */
struct perf_record_mask_cpu_map32 mask32_data;
/* Used when type == PERF_CPU_MAP__MASK and long_size == 8. */
struct perf_record_mask_cpu_map64 mask64_data;
+ /* Used when type == PERF_CPU_MAP__RANGE_CPUS. */
+ struct perf_record_range_cpu_map range_cpu_data;
};
-};
+} __attribute__((packed));
#pragma GCC diagnostic pop
struct perf_event_header header;
__u64 type;
__u64 id;
- char data[];
+ union {
+ /* Used when type == PERF_EVENT_UPDATE__SCALE. */
+ struct perf_record_event_update_scale scale;
+ /* Used when type == PERF_EVENT_UPDATE__UNIT. */
+ char unit[0];
+ /* Used when type == PERF_EVENT_UPDATE__NAME. */
+ char name[0];
+ /* Used when type == PERF_EVENT_UPDATE__CPUS. */
+ struct perf_record_event_update_cpus cpus;
+ };
};
#define MAX_EVENT_NAME 64
subcmd_config.prefix = prefix;
subcmd_config.exec_path = exec_path;
subcmd_config.exec_path_env = exec_path_env;
+
+ /* Setup environment variable for invoked shell script. */
+ setenv("PREFIX", prefix, 1);
}
#define is_dir_sep(c) ((c) == '/')
"__sanitizer_cov_trace_cmp4",
"__sanitizer_cov_trace_cmp8",
"__sanitizer_cov_trace_switch",
+ /* KMSAN */
+ "kmsan_copy_to_user",
+ "kmsan_report",
+ "kmsan_unpoison_entry_regs",
+ "kmsan_unpoison_memory",
+ "__msan_chain_origin",
+ "__msan_get_context_state",
+ "__msan_instrument_asm_store",
+ "__msan_metadata_ptr_for_load_1",
+ "__msan_metadata_ptr_for_load_2",
+ "__msan_metadata_ptr_for_load_4",
+ "__msan_metadata_ptr_for_load_8",
+ "__msan_metadata_ptr_for_load_n",
+ "__msan_metadata_ptr_for_store_1",
+ "__msan_metadata_ptr_for_store_2",
+ "__msan_metadata_ptr_for_store_4",
+ "__msan_metadata_ptr_for_store_8",
+ "__msan_metadata_ptr_for_store_n",
+ "__msan_poison_alloca",
+ "__msan_warning",
/* UBSAN */
"ubsan_type_mismatch_common",
"__ubsan_handle_type_mismatch",
perf*.xml
perf*.html
common-cmds.h
-perf.data
-perf.data.old
+perf*.data
+perf*.data.old
output.svg
perf-archive
perf-iostat
tags
TAGS
+stats-*.csv
cscope*
config.mak
config.mak.autogen
*-flex.*
*.pyc
*.pyo
+*.stdout
.config-detected
util/intel-pt-decoder/inat-tables.c
arch/*/include/generated/
debug messages will or will not be logged. Each flag must be preceded
by either '+' or '-'. The flags are:
a all perf events
+ e output only on errors (size configurable - see linkperf:perf-config[1])
o output to stdout
If supported, the 'q' option may be repeated to increase the effect.
--- /dev/null
+Arm CoreSight Support
+=====================
+
+For full documentation, see Documentation/trace/coresight/coresight-perf.rst
+in the kernel tree.
The perf c2c tool provides means for Shared Data C2C/HITM analysis. It allows
you to track down the cacheline contentions.
-On x86, the tool is based on load latency and precise store facility events
+On Intel, the tool is based on load latency and precise store facility events
provided by Intel CPUs. On PowerPC, the tool uses random instruction sampling
-with thresholding feature.
+with thresholding feature. On AMD, the tool uses IBS op pmu (due to hardware
+limitations, perf c2c is not supported on Zen3 cpus).
These events provide:
- memory address of the access
-l::
--ldlat::
- Configure mem-loads latency. (x86 only)
+ Configure mem-loads latency. Supported on Intel and Arm64 processors
+ only. Ignored on other archs.
-k::
--all-kernel::
-W,-d,--phys-data,--sample-cpu
Unless specified otherwise with '-e' option, following events are monitored by
-default on x86:
+default on Intel:
cpu/mem-loads,ldlat=30/P
cpu/mem-stores/P
+following on AMD:
+
+ ibs_op//
+
and following on PowerPC:
cpu/mem-loads/
If the directory does not exist or has the wrong file type,
the current directory is used.
+itrace.*::
+
+ debug-log-buffer-size::
+ Log size in bytes to output when using the option --itrace=d+e
+ Refer 'itrace' option of linkperf:perf-script[1] or
+ linkperf:perf-report[1]. The default is 16384.
+
daemon.*::
daemon.base::
-------
-b::
--build-ids::
- Inject build-ids into the output stream
+ Inject build-ids of DSOs hit by samples into the output stream.
+ This means it needs to process all SAMPLE records to find the DSOs.
---buildid-all:
- Inject build-ids of all DSOs into the output stream
+--buildid-all::
+ Inject build-ids of all DSOs into the output stream regardless of hits
+ and skip SAMPLE processing.
+
+--known-build-ids=::
+ Override build-ids to inject using these comma-separated pairs of
+ build-id and path. Understands file://filename to read these pairs
+ from a file, which can be generated with perf buildid-list.
-v::
--verbose::
the config terms section above. The power events record information about
C-state changes, whereas CBR is indicative of CPU frequency. perf script
"event,synth" fields display information like this:
+
cbr: cbr: 22 freq: 2189 MHz (200%)
mwait: hints: 0x60 extensions: 0x1
pwre: hw: 0 cstate: 2 sub-cstate: 0
exstop: ip: 1
pwrx: deepest cstate: 2 last cstate: 2 wake reason: 0x4
+
Where:
+
"cbr" includes the frequency and the percentage of maximum non-turbo
"mwait" shows mwait hints and extensions
"pwre" shows C-state transitions (to a C-state deeper than C0) and
"exstop" indicates execution stopped and whether the IP was recorded
exactly,
"pwrx" indicates return to C0
+
For more details refer to the Intel 64 and IA-32 Architectures Software
Developer Manuals.
picture or not. The "e" option may be followed by flags which affect what errors
will or will not be reported. Each flag must be preceded by either '+' or '-'.
The flags supported by Intel PT are:
+
-o Suppress overflow errors
-l Suppress trace data lost errors
+
For example, for errors but not overflow or data lost errors:
--itrace=e-o-l
and that the resulting file may be very large. The "d" option may be followed
by flags which affect what debug messages will or will not be logged. Each flag
must be preceded by either '+' or '-'. The flags support by Intel PT are:
+
-a Suppress logging of perf events
+a Log all perf events
+ +e Output only on decoding errors (size configurable)
+o Output to stdout instead of "intel_pt.log"
+
By default, logged perf events are filtered by any specified time ranges, but
-flag +a overrides that.
+flag +a overrides that. The +e flag can be useful for analyzing errors. By
+default, the log size in that case is 16384 bytes, but can be altered by
+linkperf:perf-config[1] e.g. perf config itrace.debug-log-buffer-size=30000
In addition, the period of the "instructions" event can be specified. e.g.
--verbose::
Be more verbose (show symbol address, etc).
+-q::
+--quiet::
+ Do not show any message. (Suppress -v)
+
-D::
--dump-raw-trace::
Dump raw trace in ASCII.
EventManager_De 1845 1 636
futex-default-S 1609 0 0
+-E::
+--entries=<value>::
+ Display this many entries.
+
+
INFO OPTIONS
------------
--map::
dump map of lock instances (address:name table)
+
CONTENTION OPTIONS
--------------
--map-nr-entries::
Maximum number of BPF map entries (default: 10240).
+--max-stack::
+ Maximum stack depth when collecting lock contention (default: 8).
+
+--stack-skip
+ Number of stack depth to skip when finding a lock caller (default: 3).
+
+-E::
+--entries=<value>::
+ Display this many entries.
+
SEE ALSO
--------
Be more verbose (show counter open errors, etc)
--ldlat <n>::
- Specify desired latency for loads event. (x86 only)
+ Specify desired latency for loads event. Supported on Intel and Arm64
+ processors only. Ignored on other archs.
In addition, for report all perf report options are valid, and for record
all perf record options.
For the platforms with Intel Arch LBR support (12th-Gen+ client or
4th-Gen Xeon+ server), the save branch type is unconditionally enabled
when the taken branch stack sampling is enabled.
+ - priv: save privilege state during sampling in case binary is not available later
+
The option requires at least one branch type among any, any_call, any_ret, ind_call, cond.
The various filters must be specified as a comma separated list: --branch-filter any_ret,u,k
Note that this feature may not be available on all processors.
+-W::
--weight::
Enable weightened sampling. An additional weight is recorded per sample and can be
displayed with the weight and local_weight sort keys. This currently works for TSX
-D::
--delay=::
After starting the program, wait msecs before measuring (-1: start with events
-disabled). This is useful to filter out the startup phase of the program, which
-is often very different.
+disabled), or enable events only for specified ranges of msecs (e.g.
+-D 10-20,30-40 means wait 10 msecs, enable for 10 msecs, wait 10 msecs, enable
+for 10 msecs, then stop). Note, delaying enabling of events is useful to filter
+out the startup phase of the program, which is often very different.
-I::
--intr-regs::
Sort histogram entries by given key(s) - multiple keys can be specified
in CSV format. Following sort keys are available:
pid, comm, dso, symbol, parent, cpu, socket, srcline, weight,
- local_weight, cgroup_id.
+ local_weight, cgroup_id, addr.
Each key has following meaning:
- local_ins_lat: Local instruction latency version
- p_stage_cyc: On powerpc, this presents the number of cycles spent in a
pipeline stage. And currently supported only on powerpc.
+ - addr: (Full) virtual address of the sampled instruction
By default, comm, dso and symbol keys are used.
(i.e. --sort comm,dso,symbol)
CFLAGS := $(EXTRA_CFLAGS) $(filter-out -Wnested-externs,$(EXTRA_WARNINGS))
HOSTCFLAGS := $(filter-out -Wnested-externs,$(EXTRA_WARNINGS))
+# Enabled Wthread-safety analysis for clang builds.
+ifeq ($(CC_NO_CLANG), 0)
+ CFLAGS += -Wthread-safety
+endif
+
include $(srctree)/tools/scripts/Makefile.arch
$(call detected_var,SRCARCH)
STRACE_GROUPS_INSTDIR_SQ = $(subst ','\'',$(STRACE_GROUPS_INSTDIR))
tip_instdir_SQ = $(subst ','\'',$(tip_instdir))
+export perfexec_instdir_SQ
+
# If we install to $(HOME) we keep the traceevent default:
# $(HOME)/.traceevent/plugins
# Otherwise we install plugins into the global $(libdir).
print_var = $(eval $(print_var_code)) $(info $(MSG))
define print_var_code
- MSG = $(shell printf '...%30s: %s' $(1) $($(1)))
+ MSG = $(shell printf '...%40s: %s' $(1) $($(1)))
endef
+ifeq ($(feature_display),1)
+ $(call feature_display_entries)
+endif
+
ifeq ($(VF),1)
# Display EXTRA features which are detected manualy
# from here with feature_check call and thus cannot
# be partof global state output.
- $(foreach feat,$(FEATURE_TESTS_EXTRA),$(call feature_print_status,$(feat),))
+ $(foreach feat,$(FEATURE_TESTS_EXTRA),$(call feature_print_status,$(feat),) $(info $(MSG)))
$(call print_var,prefix)
$(call print_var,bindir)
$(call print_var,libdir)
$(call print_var,JDIR)
ifeq ($(dwarf-post-unwind),1)
- $(call feature_print_text,"DWARF post unwind library", $(dwarf-post-unwind-text))
+ $(call feature_print_text,"DWARF post unwind library", $(dwarf-post-unwind-text)) $(info $(MSG))
endif
- $(info )
endif
+$(info )
+
$(call detected_var,bindir_SQ)
$(call detected_var,PYTHON_WORD)
ifneq ($(OUTPUT),)
# tests.
$(shell rm -f $(FEATURE_DUMP_FILENAME))
$(foreach feat,$(FEATURE_TESTS),$(shell echo "$(call feature_assign,$(feat))" >> $(FEATURE_DUMP_FILENAME)))
-
-ifeq ($(feature_display),1)
- $(call feature_display_entries)
-endif
$(sync_file_range_arrays): $(linux_uapi_dir)/fs.h $(sync_file_range_tbls)
$(Q)$(SHELL) '$(sync_file_range_tbls)' $(linux_uapi_dir) > $@
-all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS)
+TESTS_CORESIGHT_DIR := $(srctree)/tools/perf/tests/shell/coresight
+
+tests-coresight-targets: FORCE
+ $(Q)$(MAKE) -C $(TESTS_CORESIGHT_DIR)
+
+tests-coresight-targets-clean:
+ $(call QUIET_CLEAN, coresight)
+ $(Q)$(MAKE) -C $(TESTS_CORESIGHT_DIR) O=$(OUTPUT) clean >/dev/null
+
+all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS) tests-coresight-targets
# Create python binding output directory if not already present
_dummy := $(shell [ -d '$(OUTPUT)python' ] || mkdir -p '$(OUTPUT)python')
$(INSTALL) tests/shell/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \
$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \
$(INSTALL) tests/shell/lib/*.sh -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \
- $(INSTALL) tests/shell/lib/*.py -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'
+ $(INSTALL) tests/shell/lib/*.py -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \
+ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/coresight' ; \
+ $(INSTALL) tests/shell/coresight/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/coresight'
+ $(Q)$(MAKE) -C tests/shell/coresight install-tests
install-bin: install-tools install-tests install-traceevent-plugins
bpf-skel-clean:
$(call QUIET_CLEAN, bpf-skel) $(RM) -r $(SKEL_TMP_OUT) $(SKELETONS)
-clean:: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBPERF)-clean fixdep-clean python-clean bpf-skel-clean
+clean:: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBPERF)-clean fixdep-clean python-clean bpf-skel-clean tests-coresight-targets-clean
$(call QUIET_CLEAN, core-objs) $(RM) $(LIBPERF_A) $(OUTPUT)perf-archive $(OUTPUT)perf-iostat $(LANG_BINDINGS)
$(Q)find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete
$(Q)$(RM) $(OUTPUT).config-detected
* Author: Mathieu Poirier <mathieu.poirier@linaro.org>
*/
+#include <dirent.h>
#include <stdbool.h>
#include <linux/coresight-pmu.h>
#include <linux/zalloc.h>
+#include <api/fs/fs.h>
#include "../../../util/auxtrace.h"
#include "../../../util/debug.h"
#include "../../../util/pmu.h"
#include "cs-etm.h"
#include "arm-spe.h"
+#include "hisi-ptt.h"
static struct perf_pmu **find_all_arm_spe_pmus(int *nr_spes, int *err)
{
return arm_spe_pmus;
}
+static struct perf_pmu **find_all_hisi_ptt_pmus(int *nr_ptts, int *err)
+{
+ const char *sysfs = sysfs__mountpoint();
+ struct perf_pmu **hisi_ptt_pmus = NULL;
+ struct dirent *dent;
+ char path[PATH_MAX];
+ DIR *dir = NULL;
+ int idx = 0;
+
+ snprintf(path, PATH_MAX, "%s" EVENT_SOURCE_DEVICE_PATH, sysfs);
+ dir = opendir(path);
+ if (!dir) {
+ pr_err("can't read directory '%s'\n", EVENT_SOURCE_DEVICE_PATH);
+ *err = -EINVAL;
+ return NULL;
+ }
+
+ while ((dent = readdir(dir))) {
+ if (strstr(dent->d_name, HISI_PTT_PMU_NAME))
+ (*nr_ptts)++;
+ }
+
+ if (!(*nr_ptts))
+ goto out;
+
+ hisi_ptt_pmus = zalloc(sizeof(struct perf_pmu *) * (*nr_ptts));
+ if (!hisi_ptt_pmus) {
+ pr_err("hisi_ptt alloc failed\n");
+ *err = -ENOMEM;
+ goto out;
+ }
+
+ rewinddir(dir);
+ while ((dent = readdir(dir))) {
+ if (strstr(dent->d_name, HISI_PTT_PMU_NAME) && idx < *nr_ptts) {
+ hisi_ptt_pmus[idx] = perf_pmu__find(dent->d_name);
+ if (hisi_ptt_pmus[idx])
+ idx++;
+ }
+ }
+
+out:
+ closedir(dir);
+ return hisi_ptt_pmus;
+}
+
+static struct perf_pmu *find_pmu_for_event(struct perf_pmu **pmus,
+ int pmu_nr, struct evsel *evsel)
+{
+ int i;
+
+ if (!pmus)
+ return NULL;
+
+ for (i = 0; i < pmu_nr; i++) {
+ if (evsel->core.attr.type == pmus[i]->type)
+ return pmus[i];
+ }
+
+ return NULL;
+}
+
struct auxtrace_record
*auxtrace_record__init(struct evlist *evlist, int *err)
{
- struct perf_pmu *cs_etm_pmu;
+ struct perf_pmu *cs_etm_pmu = NULL;
+ struct perf_pmu **arm_spe_pmus = NULL;
+ struct perf_pmu **hisi_ptt_pmus = NULL;
struct evsel *evsel;
- bool found_etm = false;
+ struct perf_pmu *found_etm = NULL;
struct perf_pmu *found_spe = NULL;
- struct perf_pmu **arm_spe_pmus = NULL;
+ struct perf_pmu *found_ptt = NULL;
+ int auxtrace_event_cnt = 0;
int nr_spes = 0;
- int i = 0;
+ int nr_ptts = 0;
if (!evlist)
return NULL;
cs_etm_pmu = perf_pmu__find(CORESIGHT_ETM_PMU_NAME);
arm_spe_pmus = find_all_arm_spe_pmus(&nr_spes, err);
+ hisi_ptt_pmus = find_all_hisi_ptt_pmus(&nr_ptts, err);
evlist__for_each_entry(evlist, evsel) {
- if (cs_etm_pmu &&
- evsel->core.attr.type == cs_etm_pmu->type)
- found_etm = true;
-
- if (!nr_spes || found_spe)
- continue;
-
- for (i = 0; i < nr_spes; i++) {
- if (evsel->core.attr.type == arm_spe_pmus[i]->type) {
- found_spe = arm_spe_pmus[i];
- break;
- }
- }
+ if (cs_etm_pmu && !found_etm)
+ found_etm = find_pmu_for_event(&cs_etm_pmu, 1, evsel);
+
+ if (arm_spe_pmus && !found_spe)
+ found_spe = find_pmu_for_event(arm_spe_pmus, nr_spes, evsel);
+
+ if (hisi_ptt_pmus && !found_ptt)
+ found_ptt = find_pmu_for_event(hisi_ptt_pmus, nr_ptts, evsel);
}
+
free(arm_spe_pmus);
+ free(hisi_ptt_pmus);
+
+ if (found_etm)
+ auxtrace_event_cnt++;
- if (found_etm && found_spe) {
- pr_err("Concurrent ARM Coresight ETM and SPE operation not currently supported\n");
+ if (found_spe)
+ auxtrace_event_cnt++;
+
+ if (found_ptt)
+ auxtrace_event_cnt++;
+
+ if (auxtrace_event_cnt > 1) {
+ pr_err("Concurrent AUX trace operation not currently supported\n");
*err = -EOPNOTSUPP;
return NULL;
}
#if defined(__aarch64__)
if (found_spe)
return arm_spe_recording_init(err, found_spe);
+
+ if (found_ptt)
+ return hisi_ptt_recording_init(err, found_ptt);
#endif
/*
#include <linux/string.h>
#include "arm-spe.h"
+#include "hisi-ptt.h"
#include "../../../util/pmu.h"
struct perf_event_attr
#if defined(__aarch64__)
} else if (strstarts(pmu->name, ARM_SPE_PMU_NAME)) {
return arm_spe_pmu_default_config(pmu);
+ } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) {
+ pmu->selectable = true;
#endif
}
if (err)
goto out_free_arm;
/* b, b.cond, br, cbz/cbnz, tbz/tbnz */
- err = regcomp(&arm->jump_insn, "^[ct]?br?\\.?(cc|cs|eq|ge|gt|hi|le|ls|lt|mi|ne|pl)?n?z?$",
+ err = regcomp(&arm->jump_insn, "^[ct]?br?\\.?(cc|cs|eq|ge|gt|hi|hs|le|lo|ls|lt|mi|ne|pl|vc|vs)?n?z?$",
REG_EXTENDED);
if (err)
goto out_free_call;
perf-$(CONFIG_AUXTRACE) += ../../arm/util/pmu.o \
../../arm/util/auxtrace.o \
../../arm/util/cs-etm.o \
- arm-spe.o mem-events.o
+ arm-spe.o mem-events.o hisi-ptt.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * HiSilicon PCIe Trace and Tuning (PTT) support
+ * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/bitops.h>
+#include <linux/log2.h>
+#include <linux/zalloc.h>
+#include <time.h>
+
+#include <internal/lib.h> // page_size
+#include "../../../util/auxtrace.h"
+#include "../../../util/cpumap.h"
+#include "../../../util/debug.h"
+#include "../../../util/event.h"
+#include "../../../util/evlist.h"
+#include "../../../util/evsel.h"
+#include "../../../util/hisi-ptt.h"
+#include "../../../util/pmu.h"
+#include "../../../util/record.h"
+#include "../../../util/session.h"
+#include "../../../util/tsc.h"
+
+#define KiB(x) ((x) * 1024)
+#define MiB(x) ((x) * 1024 * 1024)
+
+struct hisi_ptt_recording {
+ struct auxtrace_record itr;
+ struct perf_pmu *hisi_ptt_pmu;
+ struct evlist *evlist;
+};
+
+static size_t
+hisi_ptt_info_priv_size(struct auxtrace_record *itr __maybe_unused,
+ struct evlist *evlist __maybe_unused)
+{
+ return HISI_PTT_AUXTRACE_PRIV_SIZE;
+}
+
+static int hisi_ptt_info_fill(struct auxtrace_record *itr,
+ struct perf_session *session,
+ struct perf_record_auxtrace_info *auxtrace_info,
+ size_t priv_size)
+{
+ struct hisi_ptt_recording *pttr =
+ container_of(itr, struct hisi_ptt_recording, itr);
+ struct perf_pmu *hisi_ptt_pmu = pttr->hisi_ptt_pmu;
+
+ if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE)
+ return -EINVAL;
+
+ if (!session->evlist->core.nr_mmaps)
+ return -EINVAL;
+
+ auxtrace_info->type = PERF_AUXTRACE_HISI_PTT;
+ auxtrace_info->priv[0] = hisi_ptt_pmu->type;
+
+ return 0;
+}
+
+static int hisi_ptt_set_auxtrace_mmap_page(struct record_opts *opts)
+{
+ bool privileged = perf_event_paranoid_check(-1);
+
+ if (!opts->full_auxtrace)
+ return 0;
+
+ if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
+ if (privileged) {
+ opts->auxtrace_mmap_pages = MiB(16) / page_size;
+ } else {
+ opts->auxtrace_mmap_pages = KiB(128) / page_size;
+ if (opts->mmap_pages == UINT_MAX)
+ opts->mmap_pages = KiB(256) / page_size;
+ }
+ }
+
+ /* Validate auxtrace_mmap_pages */
+ if (opts->auxtrace_mmap_pages) {
+ size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
+ size_t min_sz = KiB(8);
+
+ if (sz < min_sz || !is_power_of_2(sz)) {
+ pr_err("Invalid mmap size for HISI PTT: must be at least %zuKiB and a power of 2\n",
+ min_sz / 1024);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int hisi_ptt_recording_options(struct auxtrace_record *itr,
+ struct evlist *evlist,
+ struct record_opts *opts)
+{
+ struct hisi_ptt_recording *pttr =
+ container_of(itr, struct hisi_ptt_recording, itr);
+ struct perf_pmu *hisi_ptt_pmu = pttr->hisi_ptt_pmu;
+ struct evsel *evsel, *hisi_ptt_evsel = NULL;
+ struct evsel *tracking_evsel;
+ int err;
+
+ pttr->evlist = evlist;
+ evlist__for_each_entry(evlist, evsel) {
+ if (evsel->core.attr.type == hisi_ptt_pmu->type) {
+ if (hisi_ptt_evsel) {
+ pr_err("There may be only one " HISI_PTT_PMU_NAME "x event\n");
+ return -EINVAL;
+ }
+ evsel->core.attr.freq = 0;
+ evsel->core.attr.sample_period = 1;
+ evsel->needs_auxtrace_mmap = true;
+ hisi_ptt_evsel = evsel;
+ opts->full_auxtrace = true;
+ }
+ }
+
+ err = hisi_ptt_set_auxtrace_mmap_page(opts);
+ if (err)
+ return err;
+ /*
+ * To obtain the auxtrace buffer file descriptor, the auxtrace event
+ * must come first.
+ */
+ evlist__to_front(evlist, hisi_ptt_evsel);
+ evsel__set_sample_bit(hisi_ptt_evsel, TIME);
+
+ /* Add dummy event to keep tracking */
+ err = parse_event(evlist, "dummy:u");
+ if (err)
+ return err;
+
+ tracking_evsel = evlist__last(evlist);
+ evlist__set_tracking_event(evlist, tracking_evsel);
+
+ tracking_evsel->core.attr.freq = 0;
+ tracking_evsel->core.attr.sample_period = 1;
+ evsel__set_sample_bit(tracking_evsel, TIME);
+
+ return 0;
+}
+
+static u64 hisi_ptt_reference(struct auxtrace_record *itr __maybe_unused)
+{
+ return rdtsc();
+}
+
+static void hisi_ptt_recording_free(struct auxtrace_record *itr)
+{
+ struct hisi_ptt_recording *pttr =
+ container_of(itr, struct hisi_ptt_recording, itr);
+
+ free(pttr);
+}
+
+struct auxtrace_record *hisi_ptt_recording_init(int *err,
+ struct perf_pmu *hisi_ptt_pmu)
+{
+ struct hisi_ptt_recording *pttr;
+
+ if (!hisi_ptt_pmu) {
+ *err = -ENODEV;
+ return NULL;
+ }
+
+ pttr = zalloc(sizeof(*pttr));
+ if (!pttr) {
+ *err = -ENOMEM;
+ return NULL;
+ }
+
+ pttr->hisi_ptt_pmu = hisi_ptt_pmu;
+ pttr->itr.pmu = hisi_ptt_pmu;
+ pttr->itr.recording_options = hisi_ptt_recording_options;
+ pttr->itr.info_priv_size = hisi_ptt_info_priv_size;
+ pttr->itr.info_fill = hisi_ptt_info_fill;
+ pttr->itr.free = hisi_ptt_recording_free;
+ pttr->itr.reference = hisi_ptt_reference;
+ pttr->itr.read_finish = auxtrace_record__read_finish;
+ pttr->itr.alignment = 0;
+
+ *err = 0;
+ return &pttr->itr;
+}
#include <linux/bitops.h>
#include <linux/log2.h>
#include <linux/zalloc.h>
+#include <linux/err.h>
#include <cpuid.h>
#include "../../../util/session.h"
if (!evlist__can_select_event(evlist, sched_switch))
return -EPERM;
- err = parse_event(evlist, sched_switch);
- if (err) {
- pr_debug2("%s: failed to parse %s, error %d\n",
+ evsel = evlist__add_sched_switch(evlist, true);
+ if (IS_ERR(evsel)) {
+ err = PTR_ERR(evsel);
+ pr_debug2("%s: failed to create %s, error = %d\n",
__func__, sched_switch, err);
return err;
}
- evsel = evlist__last(evlist);
-
- evsel__set_sample_bit(evsel, CPU);
- evsel__set_sample_bit(evsel, TIME);
-
- evsel->core.system_wide = true;
- evsel->no_aux_samples = true;
evsel->immediate = true;
return 0;
* User space tasks can migrate between CPUs, so when tracing
* selected CPUs, sideband for all CPUs is still needed.
*/
- need_system_wide_tracking = evlist->core.has_user_cpus &&
+ need_system_wide_tracking = opts->target.cpu_list &&
!intel_pt_evsel->core.attr.exclude_user;
tracking_evsel = evlist__add_aux_dummy(evlist, need_system_wide_tracking);
// SPDX-License-Identifier: GPL-2.0
#include "util/pmu.h"
+#include "util/env.h"
#include "map_symbol.h"
#include "mem-events.h"
+#include "linux/string.h"
static char mem_loads_name[100];
static bool mem_loads_name__init;
#define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s }
-static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = {
+static struct perf_mem_event perf_mem_events_intel[PERF_MEM_EVENTS__MAX] = {
E("ldlat-loads", "%s/mem-loads,ldlat=%u/P", "%s/events/mem-loads"),
E("ldlat-stores", "%s/mem-stores/P", "%s/events/mem-stores"),
E(NULL, NULL, NULL),
};
+static struct perf_mem_event perf_mem_events_amd[PERF_MEM_EVENTS__MAX] = {
+ E(NULL, NULL, NULL),
+ E(NULL, NULL, NULL),
+ E("mem-ldst", "ibs_op//", "ibs_op"),
+};
+
+static int perf_mem_is_amd_cpu(void)
+{
+ struct perf_env env = { .total_mem = 0, };
+
+ perf_env__cpuid(&env);
+ if (env.cpuid && strstarts(env.cpuid, "AuthenticAMD"))
+ return 1;
+ return -1;
+}
+
struct perf_mem_event *perf_mem_events__ptr(int i)
{
+ /* 0: Uninitialized, 1: Yes, -1: No */
+ static int is_amd;
+
if (i >= PERF_MEM_EVENTS__MAX)
return NULL;
- return &perf_mem_events[i];
+ if (!is_amd)
+ is_amd = perf_mem_is_amd_cpu();
+
+ if (is_amd == 1)
+ return &perf_mem_events_amd[i];
+
+ return &perf_mem_events_intel[i];
}
bool is_mem_loads_aux_event(struct evsel *leader)
#include <sys/eventfd.h>
#include <perf/cpumap.h>
+#include "../util/mutex.h"
#include "../util/stat.h"
#include <subcmd/parse-options.h>
#include "bench.h"
/* amount of fds to monitor, per thread */
static unsigned int nfds = 64;
-static pthread_mutex_t thread_lock;
+static struct mutex thread_lock;
static unsigned int threads_starting;
static struct stats all_stats[EPOLL_NR_OPS];
-static pthread_cond_t thread_parent, thread_worker;
+static struct cond thread_parent, thread_worker;
struct worker {
int tid;
struct timespec ts = { .tv_sec = 0,
.tv_nsec = 250 };
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
/* Let 'em loose */
do {
for (i = 0; i < EPOLL_NR_OPS; i++)
init_stats(&all_stats[i]);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
threads_starting = nthreads;
do_threads(worker, cpu);
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
sleep(nsecs);
toggle_done(0, NULL, NULL);
}
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
for (i = 0; i < nthreads; i++) {
unsigned long t[EPOLL_NR_OPS];
#include <perf/cpumap.h>
#include "../util/stat.h"
+#include "../util/mutex.h"
#include <subcmd/parse-options.h>
#include "bench.h"
/* amount of fds to monitor, per thread */
static unsigned int nfds = 64;
-static pthread_mutex_t thread_lock;
+static struct mutex thread_lock;
static unsigned int threads_starting;
static struct stats throughput_stats;
-static pthread_cond_t thread_parent, thread_worker;
+static struct cond thread_parent, thread_worker;
struct worker {
int tid;
int to = nonblocking? 0 : -1;
int efd = multiq ? w->epollfd : epollfd;
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
do {
/*
getpid(), nthreads, oneshot ? " (EPOLLONESHOT semantics)": "", nfds, nsecs);
init_stats(&throughput_stats);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
threads_starting = nthreads;
do_threads(worker, cpu);
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
/*
* At this point the workers should be blocked waiting for read events
err(EXIT_FAILURE, "pthread_join");
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
/* sort the array back before reporting */
if (randomize)
#include <sys/mman.h>
#include <perf/cpumap.h>
+#include "../util/mutex.h"
#include "../util/stat.h"
#include <subcmd/parse-options.h>
#include "bench.h"
static int futex_flag = 0;
struct timeval bench__start, bench__end, bench__runtime;
-static pthread_mutex_t thread_lock;
+static struct mutex thread_lock;
static unsigned int threads_starting;
static struct stats throughput_stats;
-static pthread_cond_t thread_parent, thread_worker;
+static struct cond thread_parent, thread_worker;
struct worker {
int tid;
unsigned int i;
unsigned long ops = w->ops; /* avoid cacheline bouncing */
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
do {
for (i = 0; i < params.nfutexes; i++, ops++) {
getpid(), params.nthreads, params.nfutexes, params.fshared ? "shared":"private", params.runtime);
init_stats(&throughput_stats);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
threads_starting = params.nthreads;
pthread_attr_init(&thread_attr);
CPU_FREE(cpuset);
pthread_attr_destroy(&thread_attr);
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
sleep(params.runtime);
toggle_done(0, NULL, NULL);
}
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
for (i = 0; i < params.nthreads; i++) {
unsigned long t = bench__runtime.tv_sec > 0 ?
#include <pthread.h>
#include <signal.h>
+#include "../util/mutex.h"
#include "../util/stat.h"
#include <subcmd/parse-options.h>
#include <linux/compiler.h>
static struct worker *worker;
static bool done = false;
static int futex_flag = 0;
-static pthread_mutex_t thread_lock;
+static struct mutex thread_lock;
static unsigned int threads_starting;
static struct stats throughput_stats;
-static pthread_cond_t thread_parent, thread_worker;
+static struct cond thread_parent, thread_worker;
static struct bench_futex_parameters params = {
.runtime = 10,
struct worker *w = (struct worker *) arg;
unsigned long ops = w->ops;
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
do {
int ret;
getpid(), params.nthreads, params.runtime);
init_stats(&throughput_stats);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
threads_starting = params.nthreads;
pthread_attr_init(&thread_attr);
create_threads(worker, thread_attr, cpu);
pthread_attr_destroy(&thread_attr);
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
sleep(params.runtime);
toggle_done(0, NULL, NULL);
}
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
for (i = 0; i < params.nthreads; i++) {
unsigned long t = bench__runtime.tv_sec > 0 ?
#include <pthread.h>
#include <signal.h>
+#include "../util/mutex.h"
#include "../util/stat.h"
#include <subcmd/parse-options.h>
#include <linux/compiler.h>
static pthread_t *worker;
static bool done = false;
-static pthread_mutex_t thread_lock;
-static pthread_cond_t thread_parent, thread_worker;
+static struct mutex thread_lock;
+static struct cond thread_parent, thread_worker;
static struct stats requeuetime_stats, requeued_stats;
static unsigned int threads_starting;
static int futex_flag = 0;
{
int ret;
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
while (1) {
if (!params.pi) {
init_stats(&requeued_stats);
init_stats(&requeuetime_stats);
pthread_attr_init(&thread_attr);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
for (j = 0; j < bench_repeat && !done; j++) {
unsigned int nrequeued = 0, wakeups = 0;
block_threads(worker, thread_attr, cpu);
/* make sure all threads are already blocked */
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
usleep(100000);
}
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
pthread_attr_destroy(&thread_attr);
print_summary();
#include "bench.h"
#include <linux/compiler.h>
#include "../util/debug.h"
+#include "../util/mutex.h"
#ifndef HAVE_PTHREAD_BARRIER
int bench_futex_wake_parallel(int argc __maybe_unused, const char **argv __maybe_unused)
static pthread_t *blocked_worker;
static bool done = false;
-static pthread_mutex_t thread_lock;
-static pthread_cond_t thread_parent, thread_worker;
+static struct mutex thread_lock;
+static struct cond thread_parent, thread_worker;
static pthread_barrier_t barrier;
static struct stats waketime_stats, wakeup_stats;
static unsigned int threads_starting;
static void *blocked_workerfn(void *arg __maybe_unused)
{
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
while (1) { /* handle spurious wakeups */
if (futex_wait(&futex, 0, NULL, futex_flag) != EINTR)
init_stats(&waketime_stats);
pthread_attr_init(&thread_attr);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
for (j = 0; j < bench_repeat && !done; j++) {
waking_worker = calloc(params.nwakes, sizeof(*waking_worker));
block_threads(blocked_worker, thread_attr, cpu);
/* make sure all threads are already blocked */
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
usleep(100000);
}
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
pthread_attr_destroy(&thread_attr);
print_summary();
#include <pthread.h>
#include <signal.h>
+#include "../util/mutex.h"
#include "../util/stat.h"
#include <subcmd/parse-options.h>
#include <linux/compiler.h>
static pthread_t *worker;
static bool done = false;
-static pthread_mutex_t thread_lock;
-static pthread_cond_t thread_parent, thread_worker;
+static struct mutex thread_lock;
+static struct cond thread_parent, thread_worker;
static struct stats waketime_stats, wakeup_stats;
static unsigned int threads_starting;
static int futex_flag = 0;
static void *workerfn(void *arg __maybe_unused)
{
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
threads_starting--;
if (!threads_starting)
- pthread_cond_signal(&thread_parent);
- pthread_cond_wait(&thread_worker, &thread_lock);
- pthread_mutex_unlock(&thread_lock);
+ cond_signal(&thread_parent);
+ cond_wait(&thread_worker, &thread_lock);
+ mutex_unlock(&thread_lock);
while (1) {
if (futex_wait(&futex1, 0, NULL, futex_flag) != EINTR)
init_stats(&wakeup_stats);
init_stats(&waketime_stats);
pthread_attr_init(&thread_attr);
- pthread_mutex_init(&thread_lock, NULL);
- pthread_cond_init(&thread_parent, NULL);
- pthread_cond_init(&thread_worker, NULL);
+ mutex_init(&thread_lock);
+ cond_init(&thread_parent);
+ cond_init(&thread_worker);
for (j = 0; j < bench_repeat && !done; j++) {
unsigned int nwoken = 0;
block_threads(worker, thread_attr, cpu);
/* make sure all threads are already blocked */
- pthread_mutex_lock(&thread_lock);
+ mutex_lock(&thread_lock);
while (threads_starting)
- pthread_cond_wait(&thread_parent, &thread_lock);
- pthread_cond_broadcast(&thread_worker);
- pthread_mutex_unlock(&thread_lock);
+ cond_wait(&thread_parent, &thread_lock);
+ cond_broadcast(&thread_worker);
+ mutex_unlock(&thread_lock);
usleep(100000);
}
/* cleanup & report results */
- pthread_cond_destroy(&thread_parent);
- pthread_cond_destroy(&thread_worker);
- pthread_mutex_destroy(&thread_lock);
+ cond_destroy(&thread_parent);
+ cond_destroy(&thread_worker);
+ mutex_destroy(&thread_lock);
pthread_attr_destroy(&thread_attr);
print_summary();
*/
#include <inttypes.h>
-/* For the CLR_() macros */
-#include <pthread.h>
#include <subcmd/parse-options.h>
#include "../util/cloexec.h"
#include <linux/zalloc.h>
#include "../util/header.h"
+#include "../util/mutex.h"
#include <numa.h>
#include <numaif.h>
u64 system_time_ns;
u64 user_time_ns;
double speed_gbs;
- pthread_mutex_t *process_lock;
+ struct mutex *process_lock;
};
/* Parameters set by options: */
struct global_info {
u8 *data;
- pthread_mutex_t startup_mutex;
- pthread_cond_t startup_cond;
+ struct mutex startup_mutex;
+ struct cond startup_cond;
int nr_tasks_started;
- pthread_mutex_t start_work_mutex;
- pthread_cond_t start_work_cond;
+ struct mutex start_work_mutex;
+ struct cond start_work_cond;
int nr_tasks_working;
bool start_work;
- pthread_mutex_t stop_work_mutex;
+ struct mutex stop_work_mutex;
u64 bytes_done;
struct thread_data *threads;
return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0, g->p.thp, g->p.init_random);
}
-/*
- * Return a process-shared (global) mutex:
- */
-static void init_global_mutex(pthread_mutex_t *mutex)
-{
- pthread_mutexattr_t attr;
-
- pthread_mutexattr_init(&attr);
- pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
- pthread_mutex_init(mutex, &attr);
-}
-
-/*
- * Return a process-shared (global) condition variable:
- */
-static void init_global_cond(pthread_cond_t *cond)
-{
- pthread_condattr_t attr;
-
- pthread_condattr_init(&attr);
- pthread_condattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
- pthread_cond_init(cond, &attr);
-}
-
static int parse_cpu_list(const char *arg)
{
p0.cpu_list_str = strdup(arg);
}
if (g->p.serialize_startup) {
- pthread_mutex_lock(&g->startup_mutex);
+ mutex_lock(&g->startup_mutex);
g->nr_tasks_started++;
/* The last thread wakes the main process. */
if (g->nr_tasks_started == g->p.nr_tasks)
- pthread_cond_signal(&g->startup_cond);
+ cond_signal(&g->startup_cond);
- pthread_mutex_unlock(&g->startup_mutex);
+ mutex_unlock(&g->startup_mutex);
/* Here we will wait for the main process to start us all at once: */
- pthread_mutex_lock(&g->start_work_mutex);
+ mutex_lock(&g->start_work_mutex);
g->start_work = false;
g->nr_tasks_working++;
while (!g->start_work)
- pthread_cond_wait(&g->start_work_cond, &g->start_work_mutex);
+ cond_wait(&g->start_work_cond, &g->start_work_mutex);
- pthread_mutex_unlock(&g->start_work_mutex);
+ mutex_unlock(&g->start_work_mutex);
}
gettimeofday(&start0, NULL);
val += do_work(thread_data, g->p.bytes_thread, 0, 1, l, val);
if (g->p.sleep_usecs) {
- pthread_mutex_lock(td->process_lock);
+ mutex_lock(td->process_lock);
usleep(g->p.sleep_usecs);
- pthread_mutex_unlock(td->process_lock);
+ mutex_unlock(td->process_lock);
}
/*
* Amount of work to be done under a process-global lock:
*/
if (g->p.bytes_process_locked) {
- pthread_mutex_lock(td->process_lock);
+ mutex_lock(td->process_lock);
val += do_work(process_data, g->p.bytes_process_locked, thread_nr, g->p.nr_threads, l, val);
- pthread_mutex_unlock(td->process_lock);
+ mutex_unlock(td->process_lock);
}
work_done = g->p.bytes_global + g->p.bytes_process +
free_data(thread_data, g->p.bytes_thread);
- pthread_mutex_lock(&g->stop_work_mutex);
+ mutex_lock(&g->stop_work_mutex);
g->bytes_done += bytes_done;
- pthread_mutex_unlock(&g->stop_work_mutex);
+ mutex_unlock(&g->stop_work_mutex);
return NULL;
}
*/
static void worker_process(int process_nr)
{
- pthread_mutex_t process_lock;
+ struct mutex process_lock;
struct thread_data *td;
pthread_t *pthreads;
u8 *process_data;
int ret;
int t;
- pthread_mutex_init(&process_lock, NULL);
+ mutex_init(&process_lock);
set_taskname("process %d", process_nr);
/*
g->data = setup_shared_data(g->p.bytes_global);
/* Startup serialization: */
- init_global_mutex(&g->start_work_mutex);
- init_global_cond(&g->start_work_cond);
- init_global_mutex(&g->startup_mutex);
- init_global_cond(&g->startup_cond);
- init_global_mutex(&g->stop_work_mutex);
+ mutex_init_pshared(&g->start_work_mutex);
+ cond_init_pshared(&g->start_work_cond);
+ mutex_init_pshared(&g->startup_mutex);
+ cond_init_pshared(&g->startup_cond);
+ mutex_init_pshared(&g->stop_work_mutex);
init_thread_data();
* Wait for all the threads to start up. The last thread will
* signal this process.
*/
- pthread_mutex_lock(&g->startup_mutex);
+ mutex_lock(&g->startup_mutex);
while (g->nr_tasks_started != g->p.nr_tasks)
- pthread_cond_wait(&g->startup_cond, &g->startup_mutex);
+ cond_wait(&g->startup_cond, &g->startup_mutex);
- pthread_mutex_unlock(&g->startup_mutex);
+ mutex_unlock(&g->startup_mutex);
/* Wait for all threads to be at the start_work_cond. */
while (!threads_ready) {
- pthread_mutex_lock(&g->start_work_mutex);
+ mutex_lock(&g->start_work_mutex);
threads_ready = (g->nr_tasks_working == g->p.nr_tasks);
- pthread_mutex_unlock(&g->start_work_mutex);
+ mutex_unlock(&g->start_work_mutex);
if (!threads_ready)
usleep(1);
}
start = stop;
/* Start all threads running. */
- pthread_mutex_lock(&g->start_work_mutex);
+ mutex_lock(&g->start_work_mutex);
g->start_work = true;
- pthread_mutex_unlock(&g->start_work_mutex);
- pthread_cond_broadcast(&g->start_work_cond);
+ mutex_unlock(&g->start_work_mutex);
+ cond_broadcast(&g->start_work_cond);
} else {
gettimeofday(&start, NULL);
}
STAT_FN(ld_llchit)
STAT_FN(rmt_hit)
-static uint64_t total_records(struct c2c_stats *stats)
+static uint64_t get_load_llc_misses(struct c2c_stats *stats)
{
- uint64_t lclmiss, ldcnt, total;
-
- lclmiss = stats->lcl_dram +
- stats->rmt_dram +
- stats->rmt_hitm +
- stats->rmt_hit;
+ return stats->lcl_dram +
+ stats->rmt_dram +
+ stats->rmt_hitm +
+ stats->rmt_hit;
+}
- ldcnt = lclmiss +
- stats->ld_fbhit +
- stats->ld_l1hit +
- stats->ld_l2hit +
- stats->ld_llchit +
- stats->lcl_hitm;
+static uint64_t get_load_cache_hits(struct c2c_stats *stats)
+{
+ return stats->ld_fbhit +
+ stats->ld_l1hit +
+ stats->ld_l2hit +
+ stats->ld_llchit +
+ stats->lcl_hitm;
+}
- total = ldcnt +
- stats->st_l1hit +
- stats->st_l1miss +
- stats->st_na;
+static uint64_t get_stores(struct c2c_stats *stats)
+{
+ return stats->st_l1hit +
+ stats->st_l1miss +
+ stats->st_na;
+}
- return total;
+static uint64_t total_records(struct c2c_stats *stats)
+{
+ return get_load_llc_misses(stats) +
+ get_load_cache_hits(stats) +
+ get_stores(stats);
}
static int
static uint64_t total_loads(struct c2c_stats *stats)
{
- uint64_t lclmiss, ldcnt;
-
- lclmiss = stats->lcl_dram +
- stats->rmt_dram +
- stats->rmt_hitm +
- stats->rmt_hit;
-
- ldcnt = lclmiss +
- stats->ld_fbhit +
- stats->ld_l1hit +
- stats->ld_l2hit +
- stats->ld_llchit +
- stats->lcl_hitm;
-
- return ldcnt;
+ return get_load_llc_misses(stats) +
+ get_load_cache_hits(stats);
}
static int
int llc_misses;
struct c2c_stats *stats = &c2c.hists.stats;
- llc_misses = stats->lcl_dram +
- stats->rmt_dram +
- stats->rmt_hit +
- stats->rmt_hitm;
+ llc_misses = get_load_llc_misses(stats);
fprintf(out, "=================================================\n");
fprintf(out, " Trace Event Information \n");
*/
if (e->tag) {
e->record = true;
+ rec_argv[i++] = "-W";
} else {
e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD);
e->record = true;
#include "util/data.h"
#include "util/auxtrace.h"
#include "util/jit.h"
+#include "util/string2.h"
#include "util/symbol.h"
#include "util/synthetic-events.h"
#include "util/thread.h"
#include <linux/string.h>
#include <linux/zalloc.h>
#include <linux/hash.h>
+#include <ctype.h>
#include <errno.h>
#include <signal.h>
#include <inttypes.h>
char event_copy[PERF_SAMPLE_MAX_SIZE];
struct perf_file_section secs[HEADER_FEAT_BITS];
struct guest_session guest_session;
+ struct strlist *known_build_ids;
};
struct event_entry {
}
if (dso) {
+ mutex_lock(&dso->lock);
nsinfo__put(dso->nsinfo);
dso->nsinfo = nsi;
+ mutex_unlock(&dso->lock);
} else
nsinfo__put(nsi);
if (dso->has_build_id)
return 0;
+ mutex_lock(&dso->lock);
nsinfo__mountns_enter(dso->nsinfo, &nsc);
if (filename__read_build_id(dso->long_name, &dso->bid) > 0)
dso->has_build_id = true;
free(new_name);
}
nsinfo__mountns_exit(&nsc);
+ mutex_unlock(&dso->lock);
return dso->has_build_id ? 0 : -1;
}
+static struct strlist *perf_inject__parse_known_build_ids(
+ const char *known_build_ids_string)
+{
+ struct str_node *pos, *tmp;
+ struct strlist *known_build_ids;
+ int bid_len;
+
+ known_build_ids = strlist__new(known_build_ids_string, NULL);
+ if (known_build_ids == NULL)
+ return NULL;
+ strlist__for_each_entry_safe(pos, tmp, known_build_ids) {
+ const char *build_id, *dso_name;
+
+ build_id = skip_spaces(pos->s);
+ dso_name = strchr(build_id, ' ');
+ if (dso_name == NULL) {
+ strlist__remove(known_build_ids, pos);
+ continue;
+ }
+ bid_len = dso_name - pos->s;
+ dso_name = skip_spaces(dso_name);
+ if (bid_len % 2 != 0 || bid_len >= SBUILD_ID_SIZE) {
+ strlist__remove(known_build_ids, pos);
+ continue;
+ }
+ for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
+ if (!isxdigit(build_id[2 * ix]) ||
+ !isxdigit(build_id[2 * ix + 1])) {
+ strlist__remove(known_build_ids, pos);
+ break;
+ }
+ }
+ }
+ return known_build_ids;
+}
+
+static bool perf_inject__lookup_known_build_id(struct perf_inject *inject,
+ struct dso *dso)
+{
+ struct str_node *pos;
+ int bid_len;
+
+ strlist__for_each_entry(pos, inject->known_build_ids) {
+ const char *build_id, *dso_name;
+
+ build_id = skip_spaces(pos->s);
+ dso_name = strchr(build_id, ' ');
+ bid_len = dso_name - pos->s;
+ dso_name = skip_spaces(dso_name);
+ if (strcmp(dso->long_name, dso_name))
+ continue;
+ for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
+ dso->bid.data[ix] = (hex(build_id[2 * ix]) << 4 |
+ hex(build_id[2 * ix + 1]));
+ }
+ dso->bid.size = bid_len / 2;
+ dso->has_build_id = 1;
+ return true;
+ }
+ return false;
+}
+
static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
struct machine *machine, u8 cpumode, u32 flags)
{
+ struct perf_inject *inject = container_of(tool, struct perf_inject,
+ tool);
int err;
if (is_anon_memory(dso->long_name) || flags & MAP_HUGETLB)
if (is_no_dso_memory(dso->long_name))
return 0;
+ if (inject->known_build_ids != NULL &&
+ perf_inject__lookup_known_build_id(inject, dso))
+ return 1;
+
if (dso__read_build_id(dso) < 0) {
pr_debug("no build_id found for %s\n", dso->long_name);
return -1;
};
int ret;
bool repipe = true;
+ const char *known_build_ids = NULL;
struct option options[] = {
OPT_BOOLEAN('b', "build-ids", &inject.build_ids,
"Inject build-ids into the output stream"),
OPT_BOOLEAN(0, "buildid-all", &inject.build_id_all,
"Inject build-ids of all DSOs into the output stream"),
+ OPT_STRING(0, "known-build-ids", &known_build_ids,
+ "buildid path [,buildid path...]",
+ "build-ids to use for given paths"),
OPT_STRING('i', "input", &inject.input_name, "file",
"input file name"),
OPT_STRING('o', "output", &inject.output.path, "file",
*/
inject.tool.ordered_events = true;
inject.tool.ordering_requires_timestamps = true;
+ if (known_build_ids != NULL) {
+ inject.known_build_ids =
+ perf_inject__parse_known_build_ids(known_build_ids);
+
+ if (inject.known_build_ids == NULL) {
+ pr_err("Couldn't parse known build ids.\n");
+ goto out_delete;
+ }
+ }
}
if (inject.sched_stat) {
guest_session__exit(&inject.guest_session);
out_delete:
+ strlist__delete(inject.known_build_ids);
zstd_fini(&(inject.session->zstd_data));
perf_session__delete(inject.session);
out_close_output:
setup_pager();
if (!raw_dump && pager_in_use())
- printf("\nList of pre-defined events (to be used in -e):\n\n");
+ printf("\nList of pre-defined events (to be used in -e or -M):\n\n");
if (hybrid_type) {
pmu_name = perf_pmu__hybrid_type_to_pmu(hybrid_type);
#include <sys/types.h>
#include <sys/prctl.h>
#include <semaphore.h>
-#include <pthread.h>
#include <math.h>
#include <limits.h>
static bool show_thread_stats;
static bool use_bpf;
static unsigned long bpf_map_entries = 10240;
+static int max_stack_depth = CONTENTION_STACK_DEPTH;
+static int stack_skip = CONTENTION_STACK_SKIP;
+static int print_nr_entries = INT_MAX / 2;
static enum {
LOCK_AGGR_ADDR,
READ_LOCK = 2,
};
-static int report_lock_acquire_event(struct evsel *evsel,
- struct perf_sample *sample)
+static int get_key_by_aggr_mode_simple(u64 *key, u64 addr, u32 tid)
{
- struct lock_stat *ls;
- struct thread_stat *ts;
- struct lock_seq_stat *seq;
- const char *name = evsel__strval(evsel, sample, "name");
- u64 addr = evsel__intval(evsel, sample, "lockdep_addr");
- int flag = evsel__intval(evsel, sample, "flags");
- u64 key;
-
switch (aggr_mode) {
case LOCK_AGGR_ADDR:
- key = addr;
+ *key = addr;
break;
case LOCK_AGGR_TASK:
- key = sample->tid;
+ *key = tid;
break;
case LOCK_AGGR_CALLER:
default:
pr_err("Invalid aggregation mode: %d\n", aggr_mode);
return -EINVAL;
}
+ return 0;
+}
+
+static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample);
+
+static int get_key_by_aggr_mode(u64 *key, u64 addr, struct evsel *evsel,
+ struct perf_sample *sample)
+{
+ if (aggr_mode == LOCK_AGGR_CALLER) {
+ *key = callchain_id(evsel, sample);
+ return 0;
+ }
+ return get_key_by_aggr_mode_simple(key, addr, sample->tid);
+}
+
+static int report_lock_acquire_event(struct evsel *evsel,
+ struct perf_sample *sample)
+{
+ struct lock_stat *ls;
+ struct thread_stat *ts;
+ struct lock_seq_stat *seq;
+ const char *name = evsel__strval(evsel, sample, "name");
+ u64 addr = evsel__intval(evsel, sample, "lockdep_addr");
+ int flag = evsel__intval(evsel, sample, "flags");
+ u64 key;
+ int ret;
+
+ ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid);
+ if (ret < 0)
+ return ret;
ls = lock_stat_findnew(key, name, 0);
if (!ls)
const char *name = evsel__strval(evsel, sample, "name");
u64 addr = evsel__intval(evsel, sample, "lockdep_addr");
u64 key;
+ int ret;
- switch (aggr_mode) {
- case LOCK_AGGR_ADDR:
- key = addr;
- break;
- case LOCK_AGGR_TASK:
- key = sample->tid;
- break;
- case LOCK_AGGR_CALLER:
- default:
- pr_err("Invalid aggregation mode: %d\n", aggr_mode);
- return -EINVAL;
- }
+ ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid);
+ if (ret < 0)
+ return ret;
ls = lock_stat_findnew(key, name, 0);
if (!ls)
const char *name = evsel__strval(evsel, sample, "name");
u64 addr = evsel__intval(evsel, sample, "lockdep_addr");
u64 key;
+ int ret;
- switch (aggr_mode) {
- case LOCK_AGGR_ADDR:
- key = addr;
- break;
- case LOCK_AGGR_TASK:
- key = sample->tid;
- break;
- case LOCK_AGGR_CALLER:
- default:
- pr_err("Invalid aggregation mode: %d\n", aggr_mode);
- return -EINVAL;
- }
+ ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid);
+ if (ret < 0)
+ return ret;
ls = lock_stat_findnew(key, name, 0);
if (!ls)
const char *name = evsel__strval(evsel, sample, "name");
u64 addr = evsel__intval(evsel, sample, "lockdep_addr");
u64 key;
+ int ret;
- switch (aggr_mode) {
- case LOCK_AGGR_ADDR:
- key = addr;
- break;
- case LOCK_AGGR_TASK:
- key = sample->tid;
- break;
- case LOCK_AGGR_CALLER:
- default:
- pr_err("Invalid aggregation mode: %d\n", aggr_mode);
- return -EINVAL;
- }
+ ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid);
+ if (ret < 0)
+ return ret;
ls = lock_stat_findnew(key, name, 0);
if (!ls)
return false;
}
+static int get_symbol_name_offset(struct map *map, struct symbol *sym, u64 ip,
+ char *buf, int size)
+{
+ u64 offset;
+
+ if (map == NULL || sym == NULL) {
+ buf[0] = '\0';
+ return 0;
+ }
+
+ offset = map->map_ip(map, ip) - sym->start;
+
+ if (offset)
+ return scnprintf(buf, size, "%s+%#lx", sym->name, offset);
+ else
+ return strlcpy(buf, sym->name, size);
+}
static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sample,
char *buf, int size)
{
/* use caller function name from the callchain */
ret = thread__resolve_callchain(thread, cursor, evsel, sample,
- NULL, NULL, CONTENTION_STACK_DEPTH);
+ NULL, NULL, max_stack_depth);
if (ret != 0) {
thread__put(thread);
return -1;
break;
/* skip first few entries - for lock functions */
- if (++skip <= CONTENTION_STACK_SKIP)
+ if (++skip <= stack_skip)
goto next;
sym = node->ms.sym;
if (sym && !is_lock_function(machine, node->ip)) {
- struct map *map = node->ms.map;
- u64 offset;
-
- offset = map->map_ip(map, node->ip) - sym->start;
-
- if (offset)
- scnprintf(buf, size, "%s+%#lx", sym->name, offset);
- else
- strlcpy(buf, sym->name, size);
+ get_symbol_name_offset(node->ms.map, sym, node->ip,
+ buf, size);
return 0;
}
/* use caller function name from the callchain */
ret = thread__resolve_callchain(thread, cursor, evsel, sample,
- NULL, NULL, CONTENTION_STACK_DEPTH);
+ NULL, NULL, max_stack_depth);
thread__put(thread);
if (ret != 0)
break;
/* skip first few entries - for lock functions */
- if (++skip <= CONTENTION_STACK_SKIP)
+ if (++skip <= stack_skip)
goto next;
if (node->ms.sym && is_lock_function(machine, node->ip))
return hash;
}
+static u64 *get_callstack(struct perf_sample *sample, int max_stack)
+{
+ u64 *callstack;
+ u64 i;
+ int c;
+
+ callstack = calloc(max_stack, sizeof(*callstack));
+ if (callstack == NULL)
+ return NULL;
+
+ for (i = 0, c = 0; i < sample->callchain->nr && c < max_stack; i++) {
+ u64 ip = sample->callchain->ips[i];
+
+ if (ip >= PERF_CONTEXT_MAX)
+ continue;
+
+ callstack[c++] = ip;
+ }
+ return callstack;
+}
+
static int report_lock_contention_begin_event(struct evsel *evsel,
struct perf_sample *sample)
{
struct lock_seq_stat *seq;
u64 addr = evsel__intval(evsel, sample, "lock_addr");
u64 key;
+ int ret;
- switch (aggr_mode) {
- case LOCK_AGGR_ADDR:
- key = addr;
- break;
- case LOCK_AGGR_TASK:
- key = sample->tid;
- break;
- case LOCK_AGGR_CALLER:
- key = callchain_id(evsel, sample);
- break;
- default:
- pr_err("Invalid aggregation mode: %d\n", aggr_mode);
- return -EINVAL;
- }
+ ret = get_key_by_aggr_mode(&key, addr, evsel, sample);
+ if (ret < 0)
+ return ret;
ls = lock_stat_find(key);
if (!ls) {
ls = lock_stat_findnew(key, caller, flags);
if (!ls)
return -ENOMEM;
+
+ if (aggr_mode == LOCK_AGGR_CALLER && verbose) {
+ ls->callstack = get_callstack(sample, max_stack_depth);
+ if (ls->callstack == NULL)
+ return -ENOMEM;
+ }
}
ts = thread_stat_findnew(sample->tid);
u64 contended_term;
u64 addr = evsel__intval(evsel, sample, "lock_addr");
u64 key;
+ int ret;
- switch (aggr_mode) {
- case LOCK_AGGR_ADDR:
- key = addr;
- break;
- case LOCK_AGGR_TASK:
- key = sample->tid;
- break;
- case LOCK_AGGR_CALLER:
- key = callchain_id(evsel, sample);
- break;
- default:
- pr_err("Invalid aggregation mode: %d\n", aggr_mode);
- return -EINVAL;
- }
+ ret = get_key_by_aggr_mode(&key, addr, evsel, sample);
+ if (ret < 0)
+ return ret;
ls = lock_stat_find(key);
if (!ls)
for (i = 0; i < BROKEN_MAX; i++)
broken += bad_hist[i];
- if (broken == 0 && !verbose)
+ if (quiet || (broken == 0 && !verbose))
return;
pr_info("\n=== output for debug===\n\n");
struct lock_stat *st;
struct lock_key *key;
char cut_name[20];
- int bad, total;
+ int bad, total, printed;
- pr_info("%20s ", "Name");
- list_for_each_entry(key, &lock_keys, list)
- pr_info("%*s ", key->len, key->header);
- pr_info("\n\n");
+ if (!quiet) {
+ pr_info("%20s ", "Name");
+ list_for_each_entry(key, &lock_keys, list)
+ pr_info("%*s ", key->len, key->header);
+ pr_info("\n\n");
+ }
- bad = total = 0;
+ bad = total = printed = 0;
while ((st = pop_from_result())) {
total++;
if (st->broken)
pr_info(" ");
}
pr_info("\n");
+
+ if (++printed >= print_nr_entries)
+ break;
}
print_bad_events(bad, total);
sort_result();
}
-static void print_contention_result(void)
+static void print_contention_result(struct lock_contention *con)
{
struct lock_stat *st;
struct lock_key *key;
- int bad, total;
+ int bad, total, printed;
- list_for_each_entry(key, &lock_keys, list)
- pr_info("%*s ", key->len, key->header);
+ if (!quiet) {
+ list_for_each_entry(key, &lock_keys, list)
+ pr_info("%*s ", key->len, key->header);
- if (show_thread_stats)
- pr_info(" %10s %s\n\n", "pid", "comm");
- else
- pr_info(" %10s %s\n\n", "type", "caller");
+ if (show_thread_stats)
+ pr_info(" %10s %s\n\n", "pid", "comm");
+ else
+ pr_info(" %10s %s\n\n", "type", "caller");
+ }
- bad = total = 0;
+ bad = total = printed = 0;
if (use_bpf)
bad = bad_hist[BROKEN_CONTENDED];
/* st->addr contains tid of thread */
t = perf_session__findnew(session, pid);
pr_info(" %10d %s\n", pid, thread__comm_str(t));
- continue;
+ goto next;
}
pr_info(" %10s %s\n", get_type_str(st), st->name);
+ if (verbose) {
+ struct map *kmap;
+ struct symbol *sym;
+ char buf[128];
+ u64 ip;
+
+ for (int i = 0; i < max_stack_depth; i++) {
+ if (!st->callstack || !st->callstack[i])
+ break;
+
+ ip = st->callstack[i];
+ sym = machine__find_kernel_symbol(con->machine, ip, &kmap);
+ get_symbol_name_offset(kmap, sym, ip, buf, sizeof(buf));
+ pr_info("\t\t\t%#lx %s\n", (unsigned long)ip, buf);
+ }
+ }
+
+next:
+ if (++printed >= print_nr_entries)
+ break;
}
print_bad_events(bad, total);
.target = &target,
.result = &lockhash_table[0],
.map_nr_entries = bpf_map_entries,
+ .max_stack = max_stack_depth,
+ .stack_skip = stack_skip,
};
session = perf_session__new(use_bpf ? NULL : &data, &eops);
return PTR_ERR(session);
}
+ con.machine = &session->machines.host;
+
/* for lock function check */
symbol_conf.sort_by_name = true;
symbol__init(&session->header.env);
signal(SIGCHLD, sighandler);
signal(SIGTERM, sighandler);
- con.machine = &session->machines.host;
-
con.evlist = evlist__new();
if (con.evlist == NULL) {
err = -ENOMEM;
setup_pager();
sort_contention_result();
- print_contention_result();
+ print_contention_result(&con);
out_delete:
evlist__delete(con.evlist);
"file", "vmlinux pathname"),
OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name,
"file", "kallsyms pathname"),
+ OPT_BOOLEAN('q', "quiet", &quiet, "Do not show any message"),
OPT_END()
};
"combine locks in the same class"),
OPT_BOOLEAN('t', "threads", &show_thread_stats,
"show per-thread lock stats"),
+ OPT_INTEGER('E', "entries", &print_nr_entries, "display this many functions"),
OPT_PARENT(lock_options)
};
"Trace on existing thread id (exclusive to --pid)"),
OPT_CALLBACK(0, "map-nr-entries", &bpf_map_entries, "num",
"Max number of BPF map entries", parse_map_entry),
+ OPT_INTEGER(0, "max-stack", &max_stack_depth,
+ "Set the maximum stack depth when collecting lock contention, "
+ "Default: " __stringify(CONTENTION_STACK_DEPTH)),
+ OPT_INTEGER(0, "stack-skip", &stack_skip,
+ "Set the number of stack depth to skip when finding a lock caller, "
+ "Default: " __stringify(CONTENTION_STACK_SKIP)),
+ OPT_INTEGER('E', "entries", &print_nr_entries, "display this many functions"),
OPT_PARENT(lock_options)
};
else
rec_argc = argc + 9 * perf_pmu__hybrid_pmu_num();
+ if (mem->cpu_list)
+ rec_argc += 2;
+
rec_argv = calloc(rec_argc + 1, sizeof(char *));
if (!rec_argv)
return -1;
(mem->operation & MEM_OPERATION_LOAD) &&
(mem->operation & MEM_OPERATION_STORE)) {
e->record = true;
+ rec_argv[i++] = "-W";
} else {
if (mem->operation & MEM_OPERATION_LOAD) {
e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD);
if (all_kernel)
rec_argv[i++] = "--all-kernel";
+ if (mem->cpu_list) {
+ rec_argv[i++] = "-C";
+ rec_argv[i++] = mem->cpu_list;
+ }
+
for (j = 0; j < argc; j++, i++)
rec_argv[i] = argv[j];
#include "util/build-id.h"
#include <subcmd/parse-options.h>
+#include <internal/xyarray.h>
#include "util/parse-events.h"
#include "util/config.h"
#include "util/evsel.h"
#include "util/debug.h"
#include "util/mmap.h"
+#include "util/mutex.h"
#include "util/target.h"
#include "util/session.h"
#include "util/tool.h"
"undefined", "cpu", "core", "package", "numa", "user"
};
+struct pollfd_index_map {
+ int evlist_pollfd_index;
+ int thread_pollfd_index;
+};
+
struct record {
struct perf_tool tool;
struct record_opts opts;
int nr_threads;
struct thread_mask *thread_masks;
struct record_thread *thread_data;
+ struct pollfd_index_map *index_map;
+ size_t index_map_sz;
+ size_t index_map_cnt;
};
static volatile int done;
return record__write(rec, NULL, event, event->header.size);
}
+static struct mutex synth_lock;
+
static int process_locked_synthesized_event(struct perf_tool *tool,
union perf_event *event,
struct perf_sample *sample __maybe_unused,
struct machine *machine __maybe_unused)
{
- static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
int ret;
- pthread_mutex_lock(&synth_lock);
+ mutex_lock(&synth_lock);
ret = process_synthesized_event(tool, event, sample, machine);
- pthread_mutex_unlock(&synth_lock);
+ mutex_unlock(&synth_lock);
return ret;
}
zfree(&rec->thread_data);
}
+static int record__map_thread_evlist_pollfd_indexes(struct record *rec,
+ int evlist_pollfd_index,
+ int thread_pollfd_index)
+{
+ size_t x = rec->index_map_cnt;
+
+ if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL))
+ return -ENOMEM;
+ rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index;
+ rec->index_map[x].thread_pollfd_index = thread_pollfd_index;
+ rec->index_map_cnt += 1;
+ return 0;
+}
+
+static int record__update_evlist_pollfd_from_thread(struct record *rec,
+ struct evlist *evlist,
+ struct record_thread *thread_data)
+{
+ struct pollfd *e_entries = evlist->core.pollfd.entries;
+ struct pollfd *t_entries = thread_data->pollfd.entries;
+ int err = 0;
+ size_t i;
+
+ for (i = 0; i < rec->index_map_cnt; i++) {
+ int e_pos = rec->index_map[i].evlist_pollfd_index;
+ int t_pos = rec->index_map[i].thread_pollfd_index;
+
+ if (e_entries[e_pos].fd != t_entries[t_pos].fd ||
+ e_entries[e_pos].events != t_entries[t_pos].events) {
+ pr_err("Thread and evlist pollfd index mismatch\n");
+ err = -EINVAL;
+ continue;
+ }
+ e_entries[e_pos].revents = t_entries[t_pos].revents;
+ }
+ return err;
+}
+
+static int record__dup_non_perf_events(struct record *rec,
+ struct evlist *evlist,
+ struct record_thread *thread_data)
+{
+ struct fdarray *fda = &evlist->core.pollfd;
+ int i, ret;
+
+ for (i = 0; i < fda->nr; i++) {
+ if (!(fda->priv[i].flags & fdarray_flag__non_perf_event))
+ continue;
+ ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda);
+ if (ret < 0) {
+ pr_err("Failed to duplicate descriptor in main thread pollfd\n");
+ return ret;
+ }
+ pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n",
+ thread_data, ret, fda->entries[i].fd);
+ ret = record__map_thread_evlist_pollfd_indexes(rec, i, ret);
+ if (ret < 0) {
+ pr_err("Failed to map thread and evlist pollfd indexes\n");
+ return ret;
+ }
+ }
+ return 0;
+}
+
static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
{
int t, ret;
thread_data[t].pipes.msg[0]);
} else {
thread_data[t].tid = gettid();
- if (evlist->ctl_fd.pos == -1)
- continue;
- ret = fdarray__dup_entry_from(&thread_data[t].pollfd, evlist->ctl_fd.pos,
- &evlist->core.pollfd);
- if (ret < 0) {
- pr_err("Failed to duplicate descriptor in main thread pollfd\n");
+
+ ret = record__dup_non_perf_events(rec, evlist, &thread_data[t]);
+ if (ret < 0)
goto out_free;
- }
- thread_data[t].ctlfd_pos = ret;
- pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
- thread_data, thread_data[t].ctlfd_pos,
- evlist->core.pollfd.entries[evlist->ctl_fd.pos].fd);
+
+ thread_data[t].ctlfd_pos = -1; /* Not used */
}
}
return fd;
}
+static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
+ struct perf_record_lost_samples *lost,
+ int cpu_idx, int thread_idx)
+{
+ struct perf_counts_values count;
+ struct perf_sample_id *sid;
+ struct perf_sample sample = {};
+ int id_hdr_size;
+
+ if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
+ pr_err("read LOST count failed\n");
+ return;
+ }
+
+ if (count.lost == 0)
+ return;
+
+ lost->lost = count.lost;
+ if (evsel->core.ids) {
+ sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
+ sample.id = sid->id;
+ }
+
+ id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
+ evsel->core.attr.sample_type, &sample);
+ lost->header.size = sizeof(*lost) + id_hdr_size;
+ record__write(rec, NULL, lost, lost->header.size);
+}
+
+static void record__read_lost_samples(struct record *rec)
+{
+ struct perf_session *session = rec->session;
+ struct perf_record_lost_samples *lost;
+ struct evsel *evsel;
+
+ /* there was an error during record__open */
+ if (session->evlist == NULL)
+ return;
+
+ lost = zalloc(PERF_SAMPLE_MAX_SIZE);
+ if (lost == NULL) {
+ pr_debug("Memory allocation failed\n");
+ return;
+ }
+
+ lost->header.type = PERF_RECORD_LOST_SAMPLES;
+
+ evlist__for_each_entry(session->evlist, evsel) {
+ struct xyarray *xy = evsel->core.sample_id;
+
+ if (xy == NULL || evsel->core.fd == NULL)
+ continue;
+ if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
+ xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
+ pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
+ continue;
+ }
+
+ for (int x = 0; x < xyarray__max_x(xy); x++) {
+ for (int y = 0; y < xyarray__max_y(xy); y++) {
+ __record__read_lost_samples(rec, evsel, lost, x, y);
+ }
+ }
+ }
+ free(lost);
+
+}
+
static volatile int workload_exec_errno;
/*
}
if (rec->opts.nr_threads_synthesize > 1) {
+ mutex_init(&synth_lock);
perf_set_multithreaded();
f = process_locked_synthesized_event;
}
rec->opts.nr_threads_synthesize);
}
- if (rec->opts.nr_threads_synthesize > 1)
+ if (rec->opts.nr_threads_synthesize > 1) {
perf_set_singlethreaded();
+ mutex_destroy(&synth_lock);
+ }
out:
return err;
record__uniquify_name(rec);
+ /* Debug message used by test scripts */
+ pr_debug3("perf record opening and mmapping events\n");
if (record__open(rec) != 0) {
err = -1;
goto out_free_threads;
}
+ /* Debug message used by test scripts */
+ pr_debug3("perf record done opening and mmapping events\n");
session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
if (rec->opts.kcore) {
}
}
+ err = event_enable_timer__start(rec->evlist->eet);
+ if (err)
+ goto out_child;
+
+ /* Debug message used by test scripts */
+ pr_debug3("perf record has started\n");
+ fflush(stderr);
+
trigger_ready(&auxtrace_snapshot_trigger);
trigger_ready(&switch_output_trigger);
perf_hooks__invoke_record_start();
record__thread_munmap_filtered, NULL) == 0)
draining = true;
- evlist__ctlfd_update(rec->evlist,
- &thread->pollfd.entries[thread->ctlfd_pos]);
+ err = record__update_evlist_pollfd_from_thread(rec, rec->evlist, thread);
+ if (err)
+ goto out_child;
}
if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
}
}
+ err = event_enable_timer__process(rec->evlist->eet);
+ if (err < 0)
+ goto out_child;
+ if (err) {
+ err = 0;
+ done = 1;
+ }
+
/*
* When perf is starting the traced process, at the end events
* die with the process and we wait for that. Thus no need to
if (rec->off_cpu)
rec->bytes_written += off_cpu_write(rec->session);
+ record__read_lost_samples(rec);
record__synthesize(rec, true);
/* this will be recalculated during process_buildids() */
rec->samples = 0;
return 0;
}
+static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset)
+{
+ struct record *rec = (struct record *)opt->value;
+
+ return evlist__parse_event_enable_time(rec->evlist, &rec->opts, str, unset);
+}
static int record__parse_affinity(const struct option *opt, const char *str, int unset)
{
OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
"monitor event in cgroup name only",
parse_cgroups),
- OPT_INTEGER('D', "delay", &record.opts.initial_delay,
- "ms to wait before starting measurement after program start (-1: start with events disabled)"),
+ OPT_CALLBACK('D', "delay", &record, "ms",
+ "ms to wait before starting measurement after program start (-1: start with events disabled), "
+ "or ranges of time to enable events e.g. '-D 10-20,30-40'",
+ record__parse_event_enable_time),
OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
"user to profile"),
return 0;
}
+static int count_lost_samples_event(struct perf_tool *tool,
+ union perf_event *event,
+ struct perf_sample *sample,
+ struct machine *machine __maybe_unused)
+{
+ struct report *rep = container_of(tool, struct report, tool);
+ struct evsel *evsel;
+
+ evsel = evlist__id2evsel(rep->session->evlist, sample->id);
+ if (evsel) {
+ hists__inc_nr_lost_samples(evsel__hists(evsel),
+ event->lost_samples.lost);
+ }
+ return 0;
+}
+
static int process_attr(struct perf_tool *tool __maybe_unused,
union perf_event *event,
struct evlist **pevlist);
memset(&rep->tool, 0, sizeof(rep->tool));
rep->tool.attr = process_attr;
rep->tool.sample = count_sample_event;
+ rep->tool.lost_samples = count_lost_samples_event;
rep->tool.no_warn = true;
}
#include "util/evlist.h"
#include "util/evsel.h"
#include "util/evsel_fprintf.h"
+#include "util/mutex.h"
#include "util/symbol.h"
#include "util/thread.h"
#include "util/header.h"
struct task_desc **pid_to_task;
struct task_desc **tasks;
const struct trace_sched_handler *tp_handler;
- pthread_mutex_t start_work_mutex;
- pthread_mutex_t work_done_wait_mutex;
+ struct mutex start_work_mutex;
+ struct mutex work_done_wait_mutex;
int profile_cpu;
/*
* Track the current task - that way we can know whether there's any
const char *time_str;
struct perf_time_interval ptime;
struct perf_time_interval hist_time;
+ volatile bool thread_funcs_exit;
};
/* per thread run time data */
prctl(PR_SET_NAME, comm2);
if (fd < 0)
return NULL;
-again:
- ret = sem_post(&this_task->ready_for_work);
- BUG_ON(ret);
- ret = pthread_mutex_lock(&sched->start_work_mutex);
- BUG_ON(ret);
- ret = pthread_mutex_unlock(&sched->start_work_mutex);
- BUG_ON(ret);
- cpu_usage_0 = get_cpu_usage_nsec_self(fd);
+ while (!sched->thread_funcs_exit) {
+ ret = sem_post(&this_task->ready_for_work);
+ BUG_ON(ret);
+ mutex_lock(&sched->start_work_mutex);
+ mutex_unlock(&sched->start_work_mutex);
- for (i = 0; i < this_task->nr_events; i++) {
- this_task->curr_event = i;
- perf_sched__process_event(sched, this_task->atoms[i]);
- }
+ cpu_usage_0 = get_cpu_usage_nsec_self(fd);
- cpu_usage_1 = get_cpu_usage_nsec_self(fd);
- this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
- ret = sem_post(&this_task->work_done_sem);
- BUG_ON(ret);
+ for (i = 0; i < this_task->nr_events; i++) {
+ this_task->curr_event = i;
+ perf_sched__process_event(sched, this_task->atoms[i]);
+ }
- ret = pthread_mutex_lock(&sched->work_done_wait_mutex);
- BUG_ON(ret);
- ret = pthread_mutex_unlock(&sched->work_done_wait_mutex);
- BUG_ON(ret);
+ cpu_usage_1 = get_cpu_usage_nsec_self(fd);
+ this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
+ ret = sem_post(&this_task->work_done_sem);
+ BUG_ON(ret);
- goto again;
+ mutex_lock(&sched->work_done_wait_mutex);
+ mutex_unlock(&sched->work_done_wait_mutex);
+ }
+ return NULL;
}
static void create_tasks(struct perf_sched *sched)
+ EXCLUSIVE_LOCK_FUNCTION(sched->start_work_mutex)
+ EXCLUSIVE_LOCK_FUNCTION(sched->work_done_wait_mutex)
{
struct task_desc *task;
pthread_attr_t attr;
err = pthread_attr_setstacksize(&attr,
(size_t) max(16 * 1024, (int)PTHREAD_STACK_MIN));
BUG_ON(err);
- err = pthread_mutex_lock(&sched->start_work_mutex);
- BUG_ON(err);
- err = pthread_mutex_lock(&sched->work_done_wait_mutex);
- BUG_ON(err);
+ mutex_lock(&sched->start_work_mutex);
+ mutex_lock(&sched->work_done_wait_mutex);
for (i = 0; i < sched->nr_tasks; i++) {
struct sched_thread_parms *parms = malloc(sizeof(*parms));
BUG_ON(parms == NULL);
}
}
+static void destroy_tasks(struct perf_sched *sched)
+ UNLOCK_FUNCTION(sched->start_work_mutex)
+ UNLOCK_FUNCTION(sched->work_done_wait_mutex)
+{
+ struct task_desc *task;
+ unsigned long i;
+ int err;
+
+ mutex_unlock(&sched->start_work_mutex);
+ mutex_unlock(&sched->work_done_wait_mutex);
+ /* Get rid of threads so they won't be upset by mutex destrunction */
+ for (i = 0; i < sched->nr_tasks; i++) {
+ task = sched->tasks[i];
+ err = pthread_join(task->thread, NULL);
+ BUG_ON(err);
+ sem_destroy(&task->sleep_sem);
+ sem_destroy(&task->ready_for_work);
+ sem_destroy(&task->work_done_sem);
+ }
+}
+
static void wait_for_tasks(struct perf_sched *sched)
+ EXCLUSIVE_LOCKS_REQUIRED(sched->work_done_wait_mutex)
+ EXCLUSIVE_LOCKS_REQUIRED(sched->start_work_mutex)
{
u64 cpu_usage_0, cpu_usage_1;
struct task_desc *task;
sched->start_time = get_nsecs();
sched->cpu_usage = 0;
- pthread_mutex_unlock(&sched->work_done_wait_mutex);
+ mutex_unlock(&sched->work_done_wait_mutex);
for (i = 0; i < sched->nr_tasks; i++) {
task = sched->tasks[i];
BUG_ON(ret);
sem_init(&task->ready_for_work, 0, 0);
}
- ret = pthread_mutex_lock(&sched->work_done_wait_mutex);
- BUG_ON(ret);
+ mutex_lock(&sched->work_done_wait_mutex);
cpu_usage_0 = get_cpu_usage_nsec_parent();
- pthread_mutex_unlock(&sched->start_work_mutex);
+ mutex_unlock(&sched->start_work_mutex);
for (i = 0; i < sched->nr_tasks; i++) {
task = sched->tasks[i];
sched->runavg_parent_cpu_usage = (sched->runavg_parent_cpu_usage * (sched->replay_repeat - 1) +
sched->parent_cpu_usage)/sched->replay_repeat;
- ret = pthread_mutex_lock(&sched->start_work_mutex);
- BUG_ON(ret);
+ mutex_lock(&sched->start_work_mutex);
for (i = 0; i < sched->nr_tasks; i++) {
task = sched->tasks[i];
}
static void run_one_test(struct perf_sched *sched)
+ EXCLUSIVE_LOCKS_REQUIRED(sched->work_done_wait_mutex)
+ EXCLUSIVE_LOCKS_REQUIRED(sched->start_work_mutex)
{
u64 T0, T1, delta, avg_delta, fluct;
print_task_traces(sched);
add_cross_task_wakeups(sched);
+ sched->thread_funcs_exit = false;
create_tasks(sched);
printf("------------------------------------------------------------\n");
for (i = 0; i < sched->replay_repeat; i++)
run_one_test(sched);
+ sched->thread_funcs_exit = true;
+ destroy_tasks(sched);
return 0;
}
},
.cmp_pid = LIST_HEAD_INIT(sched.cmp_pid),
.sort_list = LIST_HEAD_INIT(sched.sort_list),
- .start_work_mutex = PTHREAD_MUTEX_INITIALIZER,
- .work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER,
.sort_order = default_sort_order,
.replay_repeat = 10,
.profile_cpu = -1,
.fork_event = replay_fork_event,
};
unsigned int i;
- int ret;
+ int ret = 0;
+ mutex_init(&sched.start_work_mutex);
+ mutex_init(&sched.work_done_wait_mutex);
for (i = 0; i < ARRAY_SIZE(sched.curr_pid); i++)
sched.curr_pid[i] = -1;
/*
* Aliased to 'perf script' for now:
*/
- if (!strcmp(argv[0], "script"))
- return cmd_script(argc, argv);
-
- if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
- return __cmd_record(argc, argv);
+ if (!strcmp(argv[0], "script")) {
+ ret = cmd_script(argc, argv);
+ } else if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
+ ret = __cmd_record(argc, argv);
} else if (strlen(argv[0]) > 2 && strstarts("latency", argv[0])) {
sched.tp_handler = &lat_ops;
if (argc > 1) {
usage_with_options(latency_usage, latency_options);
}
setup_sorting(&sched, latency_options, latency_usage);
- return perf_sched__lat(&sched);
+ ret = perf_sched__lat(&sched);
} else if (!strcmp(argv[0], "map")) {
if (argc) {
argc = parse_options(argc, argv, map_options, map_usage, 0);
}
sched.tp_handler = &map_ops;
setup_sorting(&sched, latency_options, latency_usage);
- return perf_sched__map(&sched);
+ ret = perf_sched__map(&sched);
} else if (strlen(argv[0]) > 2 && strstarts("replay", argv[0])) {
sched.tp_handler = &replay_ops;
if (argc) {
if (argc)
usage_with_options(replay_usage, replay_options);
}
- return perf_sched__replay(&sched);
+ ret = perf_sched__replay(&sched);
} else if (!strcmp(argv[0], "timehist")) {
if (argc) {
argc = parse_options(argc, argv, timehist_options,
parse_options_usage(NULL, timehist_options, "w", true);
if (sched.show_next)
parse_options_usage(NULL, timehist_options, "n", true);
- return -EINVAL;
+ ret = -EINVAL;
+ goto out;
}
ret = symbol__validate_sym_arguments();
if (ret)
- return ret;
+ goto out;
- return perf_sched__timehist(&sched);
+ ret = perf_sched__timehist(&sched);
} else {
usage_with_options(sched_usage, sched_options);
}
- return 0;
+out:
+ mutex_destroy(&sched.start_work_mutex);
+ mutex_destroy(&sched.work_done_wait_mutex);
+
+ return ret;
}
br->flags.in_tx ? 'X' : '-',
br->flags.abort ? 'A' : '-',
br->flags.cycles,
- br->flags.type ? branch_type_name(br->flags.type) : "-");
+ get_branch_type(br));
}
static int perf_sample__fprintf_brstack(struct perf_sample *sample,
struct perf_cpu cpu;
static int header_printed;
- if (counter->core.system_wide)
- nthreads = 1;
-
if (!header_printed) {
printf("%3s %8s %15s %15s %15s %15s %s\n",
"CPU", "THREAD", "VAL", "ENA", "RUN", "TIME", "EVENT");
"Valid types: hw,sw,trace,raw,synth. "
"Fields: comm,tid,pid,time,cpu,event,trace,ip,sym,dso,"
"addr,symoff,srcline,period,iregs,uregs,brstack,"
- "brstacksym,flags,bpf-output,brstackinsn,brstackinsnlen,brstackoff,"
- "callindent,insn,insnlen,synth,phys_addr,metric,misc,ipc,tod,"
- "data_page_size,code_page_size,ins_lat",
+ "brstacksym,flags,data_src,weight,bpf-output,brstackinsn,"
+ "brstackinsnlen,brstackoff,callindent,insn,insnlen,synth,"
+ "phys_addr,metric,misc,srccode,ipc,tod,data_page_size,"
+ "code_page_size,ins_lat",
parse_output_fields),
OPT_BOOLEAN('a', "all-cpus", &system_wide,
"system-wide collection from all CPUs"),
static bool interval_count;
static const char *output_name;
static int output_fd;
+static char *metrics;
struct perf_stat {
bool record;
static void perf_stat__reset_stats(void)
{
- int i;
-
evlist__reset_stats(evsel_list);
perf_stat__reset_shadow_stats();
-
- for (i = 0; i < stat_config.stats_num; i++)
- perf_stat__reset_shadow_per_stat(&stat_config.stats[i]);
}
static int process_synthesized_event(struct perf_tool *tool __maybe_unused,
}
}
-static int runtime_stat_new(struct perf_stat_config *config, int nthreads)
-{
- int i;
-
- config->stats = calloc(nthreads, sizeof(struct runtime_stat));
- if (!config->stats)
- return -1;
-
- config->stats_num = nthreads;
-
- for (i = 0; i < nthreads; i++)
- runtime_stat__init(&config->stats[i]);
-
- return 0;
-}
-
-static void runtime_stat_delete(struct perf_stat_config *config)
-{
- int i;
-
- if (!config->stats)
- return;
-
- for (i = 0; i < config->stats_num; i++)
- runtime_stat__exit(&config->stats[i]);
-
- zfree(&config->stats);
-}
-
-static void runtime_stat_reset(struct perf_stat_config *config)
-{
- int i;
-
- if (!config->stats)
- return;
-
- for (i = 0; i < config->stats_num; i++)
- perf_stat__reset_shadow_per_stat(&config->stats[i]);
-}
-
static void process_interval(void)
{
struct timespec ts, rs;
diff_timespec(&rs, &ts, &ref_time);
perf_stat__reset_shadow_per_stat(&rt_stat);
- runtime_stat_reset(&stat_config);
read_counters(&rs);
if (STAT_RECORD) {
if (evlist__ctlfd_process(evlist, &cmd) > 0) {
switch (cmd) {
case EVLIST_CTL_CMD_ENABLE:
- if (interval)
- process_interval();
- break;
+ __fallthrough;
case EVLIST_CTL_CMD_DISABLE:
if (interval)
process_interval();
evlist__for_each_cpu(evlist_cpu_itr, evsel_list, affinity) {
counter = evlist_cpu_itr.evsel;
- if (!counter->reset_group && !counter->errored)
- continue;
if (!counter->reset_group)
continue;
try_again_reset:
evlist__copy_prev_raw_counts(evsel_list);
evlist__reset_prev_raw_counts(evsel_list);
- runtime_stat_reset(&stat_config);
perf_stat__reset_shadow_per_stat(&rt_stat);
} else {
update_stats(&walltime_nsecs_stats, t1 - t0);
return 0;
}
-static int parse_metric_groups(const struct option *opt,
+static int append_metric_groups(const struct option *opt __maybe_unused,
const char *str,
int unset __maybe_unused)
{
- return metricgroup__parse_groups(opt, str,
- stat_config.metric_no_group,
- stat_config.metric_no_merge,
- &stat_config.metric_events);
+ if (metrics) {
+ char *tmp;
+
+ if (asprintf(&tmp, "%s,%s", metrics, str) < 0)
+ return -ENOMEM;
+ free(metrics);
+ metrics = tmp;
+ } else {
+ metrics = strdup(str);
+ if (!metrics)
+ return -ENOMEM;
+ }
+ return 0;
}
static int parse_control_option(const struct option *opt,
"measure SMI cost"),
OPT_CALLBACK('M', "metrics", &evsel_list, "metric/metric group list",
"monitor specified metrics or metric groups (separated by ,)",
- parse_metric_groups),
+ append_metric_groups),
OPT_BOOLEAN_FLAG(0, "all-kernel", &stat_config.all_kernel,
"Configure all used events to run in kernel space.",
PARSE_OPT_EXCLUSIVE),
* on an architecture test for such a metric name.
*/
if (metricgroup__has_metric("transaction")) {
- struct option opt = { .value = &evsel_list };
-
- return metricgroup__parse_groups(&opt, "transaction",
+ return metricgroup__parse_groups(evsel_list, "transaction",
stat_config.metric_no_group,
- stat_config.metric_no_merge,
+ stat_config.metric_no_merge,
+ stat_config.user_requested_cpu_list,
+ stat_config.system_wide,
&stat_config.metric_events);
}
input_name = "perf.data";
}
+ perf_stat__init_shadow_stats();
+
perf_stat.data.path = input_name;
perf_stat.data.mode = PERF_DATA_MODE_READ;
argc = parse_options_subcommand(argc, argv, stat_options, stat_subcommands,
(const char **) stat_usage,
PARSE_OPT_STOP_AT_NON_OPTION);
- perf_stat__collect_metric_expr(evsel_list);
- perf_stat__init_shadow_stats();
if (stat_config.csv_sep) {
stat_config.csv_output = true;
target.system_wide = true;
}
+ if ((stat_config.aggr_mode == AGGR_THREAD) && (target.system_wide))
+ target.per_thread = true;
+
+ stat_config.system_wide = target.system_wide;
+ if (target.cpu_list) {
+ stat_config.user_requested_cpu_list = strdup(target.cpu_list);
+ if (!stat_config.user_requested_cpu_list) {
+ status = -ENOMEM;
+ goto out;
+ }
+ }
+
+ /*
+ * Metric parsing needs to be delayed as metrics may optimize events
+ * knowing the target is system-wide.
+ */
+ if (metrics) {
+ metricgroup__parse_groups(evsel_list, metrics,
+ stat_config.metric_no_group,
+ stat_config.metric_no_merge,
+ stat_config.user_requested_cpu_list,
+ stat_config.system_wide,
+ &stat_config.metric_events);
+ zfree(&metrics);
+ }
+ perf_stat__collect_metric_expr(evsel_list);
+ perf_stat__init_shadow_stats();
+
if (add_default_attributes())
goto out;
}
}
- if ((stat_config.aggr_mode == AGGR_THREAD) && (target.system_wide))
- target.per_thread = true;
-
if (evlist__fix_hybrid_cpus(evsel_list, target.cpu_list)) {
pr_err("failed to use cpu list %s\n", target.cpu_list);
goto out;
*/
if (stat_config.aggr_mode == AGGR_THREAD) {
thread_map__read_comms(evsel_list->core.threads);
- if (target.system_wide) {
- if (runtime_stat_new(&stat_config,
- perf_thread_map__nr(evsel_list->core.threads))) {
- goto out;
- }
- }
}
if (stat_config.aggr_mode == AGGR_NODE)
iostat_release(evsel_list);
zfree(&stat_config.walltime_run);
+ zfree(&stat_config.user_requested_cpu_list);
if (smi_cost && smi_reset)
sysfs__write_int(FREEZE_ON_SMI_PATH, 0);
evlist__delete(evsel_list);
metricgroup__rblist_exit(&stat_config.metric_events);
- runtime_stat_delete(&stat_config);
evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close);
return status;
return cursor;
}
+static struct per_pidcomm *create_pidcomm(struct per_pid *p)
+{
+ struct per_pidcomm *c;
+
+ c = zalloc(sizeof(*c));
+ if (!c)
+ return NULL;
+ p->current = c;
+ c->next = p->all;
+ p->all = c;
+ return c;
+}
+
static void pid_set_comm(struct timechart *tchart, int pid, char *comm)
{
struct per_pid *p;
}
c = c->next;
}
- c = zalloc(sizeof(*c));
+ c = create_pidcomm(p);
assert(c != NULL);
c->comm = strdup(comm);
- p->current = c;
- c->next = p->all;
- p->all = c;
}
static void pid_fork(struct timechart *tchart, int pid, int ppid, u64 timestamp)
p = find_create_pid(tchart, pid);
c = p->current;
if (!c) {
- c = zalloc(sizeof(*c));
+ c = create_pidcomm(p);
assert(c != NULL);
- p->current = c;
- c->next = p->all;
- p->all = c;
}
sample = zalloc(sizeof(*sample));
tchart->power_events = pwr;
}
-static void p_state_change(struct timechart *tchart, int cpu, u64 timestamp, u64 new_freq)
+static struct power_event *p_state_end(struct timechart *tchart, int cpu,
+ u64 timestamp)
{
- struct power_event *pwr;
-
- if (new_freq > 8000000) /* detect invalid data */
- return;
+ struct power_event *pwr = zalloc(sizeof(*pwr));
- pwr = zalloc(sizeof(*pwr));
if (!pwr)
- return;
+ return NULL;
pwr->state = cpus_pstate_state[cpu];
pwr->start_time = cpus_pstate_start_times[cpu];
pwr->cpu = cpu;
pwr->type = PSTATE;
pwr->next = tchart->power_events;
-
if (!pwr->start_time)
pwr->start_time = tchart->first_time;
tchart->power_events = pwr;
+ return pwr;
+}
+
+static void p_state_change(struct timechart *tchart, int cpu, u64 timestamp, u64 new_freq)
+{
+ struct power_event *pwr;
+
+ if (new_freq > 8000000) /* detect invalid data */
+ return;
+
+ pwr = p_state_end(tchart, cpu, timestamp);
+ if (!pwr)
+ return;
cpus_pstate_state[cpu] = new_freq;
cpus_pstate_start_times[cpu] = timestamp;
#endif
/* P state */
- pwr = zalloc(sizeof(*pwr));
+ pwr = p_state_end(tchart, cpu, tchart->last_time);
if (!pwr)
return;
- pwr->state = cpus_pstate_state[cpu];
- pwr->start_time = cpus_pstate_start_times[cpu];
- pwr->end_time = tchart->last_time;
- pwr->cpu = cpu;
- pwr->type = PSTATE;
- pwr->next = tchart->power_events;
-
- if (!pwr->start_time)
- pwr->start_time = tchart->first_time;
if (!pwr->state)
pwr->state = tchart->min_freq;
- tchart->power_events = pwr;
}
}
struct io_sample *prev;
if (!c) {
- c = zalloc(sizeof(*c));
+ c = create_pidcomm(p);
if (!c)
return -ENOMEM;
- p->current = c;
- c->next = p->all;
- p->all = c;
}
prev = c->io_samples;
}
notes = symbol__annotation(sym);
- pthread_mutex_lock(¬es->lock);
+ mutex_lock(¬es->lock);
if (!symbol__hists(sym, top->evlist->core.nr_entries)) {
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
pr_err("Not enough memory for annotating '%s' symbol!\n",
sym->name);
sleep(1);
pr_err("Couldn't annotate %s: %s\n", sym->name, msg);
}
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
return err;
}
struct hist_entry *he,
struct perf_sample *sample,
struct evsel *evsel, u64 ip)
+ EXCLUSIVE_LOCKS_REQUIRED(he->hists->lock)
{
struct annotation *notes;
struct symbol *sym = he->ms.sym;
notes = symbol__annotation(sym);
- if (pthread_mutex_trylock(¬es->lock))
+ if (!mutex_trylock(¬es->lock))
return;
err = hist_entry__inc_addr_samples(he, sample, evsel, ip);
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
if (unlikely(err)) {
/*
* This function is now called with he->hists->lock held.
* Release it before going to sleep.
*/
- pthread_mutex_unlock(&he->hists->lock);
+ mutex_unlock(&he->hists->lock);
if (err == -ERANGE && !he->ms.map->erange_warned)
ui__warn_map_erange(he->ms.map, sym, ip);
sleep(1);
}
- pthread_mutex_lock(&he->hists->lock);
+ mutex_lock(&he->hists->lock);
}
}
symbol = he->ms.sym;
notes = symbol__annotation(symbol);
- pthread_mutex_lock(¬es->lock);
+ mutex_lock(¬es->lock);
symbol__calc_percent(symbol, evsel);
if (more != 0)
printf("%d lines not displayed, maybe increase display entries [e]\n", more);
out_unlock:
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
}
static void perf_top__resort_hists(struct perf_top *t)
static int hist_iter__top_callback(struct hist_entry_iter *iter,
struct addr_location *al, bool single,
void *arg)
+ EXCLUSIVE_LOCKS_REQUIRED(iter->he->hists->lock)
{
struct perf_top *top = arg;
- struct hist_entry *he = iter->he;
struct evsel *evsel = iter->evsel;
if (perf_hpp_list.sym && single)
- perf_top__record_precise_ip(top, he, iter->sample, evsel, al->addr);
+ perf_top__record_precise_ip(top, iter->he, iter->sample, evsel, al->addr);
hist__account_cycles(iter->sample->branch_stack, al, iter->sample,
!(top->record_opts.branch_stack & PERF_SAMPLE_BRANCH_ANY),
else
iter.ops = &hist_iter_normal;
- pthread_mutex_lock(&hists->lock);
+ mutex_lock(&hists->lock);
if (hist_entry_iter__add(&iter, &al, top->max_stack, top) < 0)
pr_err("Problem incrementing symbol period, skipping event\n");
- pthread_mutex_unlock(&hists->lock);
+ mutex_unlock(&hists->lock);
}
addr_location__put(&al);
perf_mmap__consume(&md->core);
if (top->qe.rotate) {
- pthread_mutex_lock(&top->qe.mutex);
+ mutex_lock(&top->qe.mutex);
top->qe.rotate = false;
- pthread_cond_signal(&top->qe.cond);
- pthread_mutex_unlock(&top->qe.mutex);
+ cond_signal(&top->qe.cond);
+ mutex_unlock(&top->qe.mutex);
}
}
out = rotate_queues(top);
- pthread_mutex_lock(&top->qe.mutex);
+ mutex_lock(&top->qe.mutex);
top->qe.rotate = true;
- pthread_cond_wait(&top->qe.cond, &top->qe.mutex);
- pthread_mutex_unlock(&top->qe.mutex);
+ cond_wait(&top->qe.cond, &top->qe.mutex);
+ mutex_unlock(&top->qe.mutex);
if (ordered_events__flush(out, OE_FLUSH__TOP))
pr_err("failed to process events\n");
ordered_events__set_copy_on_queue(&top->qe.data[0], true);
ordered_events__set_copy_on_queue(&top->qe.data[1], true);
top->qe.in = &top->qe.data[0];
- pthread_mutex_init(&top->qe.mutex, NULL);
- pthread_cond_init(&top->qe.cond, NULL);
+ mutex_init(&top->qe.mutex);
+ cond_init(&top->qe.cond);
}
static int __cmd_top(struct perf_top *top)
out_join:
pthread_join(thread, NULL);
out_join_thread:
- pthread_cond_signal(&top->qe.cond);
+ cond_signal(&top->qe.cond);
pthread_join(thread_process, NULL);
return ret;
}
if (evlist__create_maps(top.evlist, target) < 0) {
ui__error("Couldn't create thread/CPU maps: %s\n",
errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf)));
+ status = -errno;
goto out_delete_evlist;
}
if (top.sb_evlist == NULL) {
pr_err("Couldn't create side band evlist.\n.");
+ status = -EINVAL;
goto out_delete_evlist;
}
if (evlist__add_bpf_sb_event(top.sb_evlist, &perf_env)) {
pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
+ status = -EINVAL;
goto out_delete_evlist;
}
}
if (isalpha(*tok) || *tok == '_') {
if (!strarray__strtoul(sa, tok, toklen, &val))
return false;
- } else {
- bool is_hexa = tok[0] == 0 && (tok[1] = 'x' || tok[1] == 'X');
-
- val = strtoul(tok, NULL, is_hexa ? 16 : 0);
- }
+ } else
+ val = strtoul(tok, NULL, 0);
*ret |= (1 << (val - 1));
stats = inode->priv;
if (stats == NULL) {
- stats = malloc(sizeof(*stats));
+ stats = zalloc(sizeof(*stats));
if (stats == NULL)
return;
- stats->nr_failures = 0;
- stats->max_errno = 0;
- stats->errnos = NULL;
init_stats(&stats->stats);
inode->priv = stats;
}
printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : "");
- /*
- * XXX Perhaps we should have a show_tp_arg_names,
- * leaving show_arg_names just for syscalls?
- */
- if (1 || trace->show_arg_names)
+ if (trace->show_arg_names)
printed += scnprintf(bf + printed, size - printed, "%s: ", field->name);
printed += syscall_arg_fmt__scnprintf_val(arg, bf + printed, size - printed, &syscall_arg, val);
int val;
};
+static bool same_cmd_with_prefix(const char *var, struct pager_config *c,
+ const char *header)
+{
+ return (strstarts(var, header) && !strcmp(var + strlen(header), c->cmd));
+}
+
static int pager_command_config(const char *var, const char *value, void *data)
{
struct pager_config *c = data;
- if (strstarts(var, "pager.") && !strcmp(var + 6, c->cmd))
+ if (same_cmd_with_prefix(var, c, "pager."))
c->val = perf_config_bool(var, value);
return 0;
}
static int browser_command_config(const char *var, const char *value, void *data)
{
struct pager_config *c = data;
- if (strstarts(var, "tui.") && !strcmp(var + 4, c->cmd))
+ if (same_cmd_with_prefix(var, c, "tui."))
c->val = perf_config_bool(var, value);
- if (strstarts(var, "gtk.") && !strcmp(var + 4, c->cmd))
+ if (same_cmd_with_prefix(var, c, "gtk."))
c->val = perf_config_bool(var, value) ? 2 : 0;
return 0;
}
"PublicDescription": "This event counts memory accesses due to load or store instructions. This event counts the sum of MEM_ACCESS_RD and MEM_ACCESS_WR.",
"ArchStdEvent": "MEM_ACCESS"
},
+ {
+ "ArchStdEvent": "REMOTE_ACCESS"
+ },
{
"ArchStdEvent": "MEM_ACCESS_RD"
},
+++ /dev/null
-[
- {
- "ArchStdEvent": "REMOTE_ACCESS"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "BR_MIS_PRED"
- },
- {
- "ArchStdEvent": "BR_PRED"
- },
- {
- "ArchStdEvent": "BR_IMMED_SPEC"
- },
- {
- "ArchStdEvent": "BR_RETURN_SPEC"
- },
- {
- "ArchStdEvent": "BR_INDIRECT_SPEC"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "CPU_CYCLES"
- },
- {
- "ArchStdEvent": "BUS_ACCESS"
- },
- {
- "ArchStdEvent": "BUS_CYCLES"
- },
- {
- "ArchStdEvent": "BUS_ACCESS_RD"
- },
- {
- "ArchStdEvent": "BUS_ACCESS_WR"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "L1I_CACHE_REFILL"
- },
- {
- "ArchStdEvent": "L1I_TLB_REFILL"
- },
- {
- "ArchStdEvent": "L1D_CACHE_REFILL"
- },
- {
- "ArchStdEvent": "L1D_CACHE"
- },
- {
- "ArchStdEvent": "L1D_TLB_REFILL"
- },
- {
- "ArchStdEvent": "L1I_CACHE"
- },
- {
- "ArchStdEvent": "L1D_CACHE_WB"
- },
- {
- "ArchStdEvent": "L2D_CACHE"
- },
- {
- "ArchStdEvent": "L2D_CACHE_REFILL"
- },
- {
- "ArchStdEvent": "L2D_CACHE_WB"
- },
- {
- "ArchStdEvent": "L1D_CACHE_ALLOCATE"
- },
- {
- "ArchStdEvent": "L2D_CACHE_ALLOCATE"
- },
- {
- "ArchStdEvent": "L1D_TLB"
- },
- {
- "ArchStdEvent": "L1I_TLB"
- },
- {
- "ArchStdEvent": "L3D_CACHE_ALLOCATE"
- },
- {
- "ArchStdEvent": "L3D_CACHE_REFILL"
- },
- {
- "ArchStdEvent": "L3D_CACHE"
- },
- {
- "ArchStdEvent": "L2D_TLB_REFILL"
- },
- {
- "ArchStdEvent": "L2D_TLB"
- },
- {
- "ArchStdEvent": "DTLB_WALK"
- },
- {
- "ArchStdEvent": "ITLB_WALK"
- },
- {
- "ArchStdEvent": "LL_CACHE_RD"
- },
- {
- "ArchStdEvent": "LL_CACHE_MISS_RD"
- },
- {
- "ArchStdEvent": "L1D_CACHE_RD"
- },
- {
- "ArchStdEvent": "L1D_CACHE_WR"
- },
- {
- "ArchStdEvent": "L1D_CACHE_REFILL_RD"
- },
- {
- "ArchStdEvent": "L1D_CACHE_REFILL_WR"
- },
- {
- "ArchStdEvent": "L1D_CACHE_REFILL_INNER"
- },
- {
- "ArchStdEvent": "L1D_CACHE_REFILL_OUTER"
- },
- {
- "ArchStdEvent": "L2D_CACHE_RD"
- },
- {
- "ArchStdEvent": "L2D_CACHE_WR"
- },
- {
- "ArchStdEvent": "L2D_CACHE_REFILL_RD"
- },
- {
- "ArchStdEvent": "L2D_CACHE_REFILL_WR"
- },
- {
- "ArchStdEvent": "L3D_CACHE_RD"
- },
- {
- "ArchStdEvent": "L3D_CACHE_REFILL_RD"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "EXC_TAKEN"
- },
- {
- "ArchStdEvent": "MEMORY_ERROR"
- },
- {
- "ArchStdEvent": "EXC_IRQ"
- },
- {
- "ArchStdEvent": "EXC_FIQ"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "SW_INCR"
- },
- {
- "ArchStdEvent": "LD_RETIRED"
- },
- {
- "ArchStdEvent": "ST_RETIRED"
- },
- {
- "ArchStdEvent": "INST_RETIRED"
- },
- {
- "ArchStdEvent": "EXC_RETURN"
- },
- {
- "ArchStdEvent": "CID_WRITE_RETIRED"
- },
- {
- "ArchStdEvent": "PC_WRITE_RETIRED"
- },
- {
- "ArchStdEvent": "BR_IMMED_RETIRED"
- },
- {
- "ArchStdEvent": "BR_RETURN_RETIRED"
- },
- {
- "ArchStdEvent": "INST_SPEC"
- },
- {
- "ArchStdEvent": "TTBR_WRITE_RETIRED"
- },
- {
- "ArchStdEvent": "BR_RETIRED"
- },
- {
- "ArchStdEvent": "BR_MIS_PRED_RETIRED"
- },
- {
- "ArchStdEvent": "LD_SPEC"
- },
- {
- "ArchStdEvent": "ST_SPEC"
- },
- {
- "ArchStdEvent": "LDST_SPEC"
- },
- {
- "ArchStdEvent": "DP_SPEC"
- },
- {
- "ArchStdEvent": "ASE_SPEC"
- },
- {
- "ArchStdEvent": "VFP_SPEC"
- },
- {
- "ArchStdEvent": "CRYPTO_SPEC"
- },
- {
- "ArchStdEvent": "ISB_SPEC"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "MEM_ACCESS"
- },
- {
- "ArchStdEvent": "REMOTE_ACCESS_RD"
- },
- {
- "ArchStdEvent": "MEM_ACCESS_RD"
- },
- {
- "ArchStdEvent": "MEM_ACCESS_WR"
- },
- {
- "ArchStdEvent": "UNALIGNED_LD_SPEC"
- },
- {
- "ArchStdEvent": "UNALIGNED_ST_SPEC"
- },
- {
- "ArchStdEvent": "UNALIGNED_LDST_SPEC"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "STALL_FRONTEND"
- },
- {
- "ArchStdEvent": "STALL_BACKEND"
- }
-]
+++ /dev/null
-[
- {
- "ArchStdEvent": "SAMPLE_POP"
- },
- {
- "ArchStdEvent": "SAMPLE_FEED"
- },
- {
- "ArchStdEvent": "SAMPLE_FILTRATE"
- },
- {
- "ArchStdEvent": "SAMPLE_COLLISION"
- }
-]
{
"ArchStdEvent": "MEM_ACCESS"
},
+ {
+ "ArchStdEvent": "REMOTE_ACCESS"
+ },
{
"ArchStdEvent": "MEM_ACCESS_RD"
},
+++ /dev/null
-[
- {
- "ArchStdEvent": "REMOTE_ACCESS"
- }
-]
},
{
"ArchStdEvent": "RC_ST_SPEC"
+ },
+ {
+ "ArchStdEvent": "ASE_INST_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_INST_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_PRED_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_PRED_EMPTY_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_PRED_FULL_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_PRED_PARTIAL_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_LDFF_SPEC"
+ },
+ {
+ "ArchStdEvent": "SVE_LDFF_FAULT_SPEC"
+ },
+ {
+ "ArchStdEvent": "FP_SCALE_OPS_SPEC"
+ },
+ {
+ "ArchStdEvent": "FP_FIXED_OPS_SPEC"
}
]
{
"ArchStdEvent": "MEM_ACCESS"
},
+ {
+ "ArchStdEvent": "REMOTE_ACCESS"
+ },
{
"ArchStdEvent": "MEM_ACCESS_RD"
},
+++ /dev/null
-[
- {
- "ArchStdEvent": "REMOTE_ACCESS"
- }
-]
0x00000000420f1000,v1,arm/cortex-a53,core
0x00000000410fd040,v1,arm/cortex-a35,core
0x00000000410fd050,v1,arm/cortex-a55,core
-0x00000000410fd060,v1,arm/cortex-a65,core
+0x00000000410fd060,v1,arm/cortex-a65-e1,core
+0x00000000410fd4a0,v1,arm/cortex-a65-e1,core
0x00000000410fd070,v1,arm/cortex-a57-a72,core
0x00000000410fd080,v1,arm/cortex-a57-a72,core
0x00000000410fd090,v1,arm/cortex-a73,core
0x00000000410fd470,v1,arm/cortex-a710,core
0x00000000410fd480,v1,arm/cortex-x2,core
0x00000000410fd490,v1,arm/neoverse-n2,core
-0x00000000410fd4a0,v1,arm/neoverse-e1,core
0x00000000420f5160,v1,cavium/thunderx2,core
0x00000000430f0af0,v1,cavium/thunderx2,core
0x00000000460f0010,v1,fujitsu/a64fx,core
"MetricName": "DCache_L2_All_Miss"
},
{
- "MetricExpr": "dcache_l2_all_hits + dcache_l2_all_miss",
+ "MetricExpr": "DCache_L2_All_Hits + DCache_L2_All_Miss",
"MetricName": "DCache_L2_All"
},
{
- "MetricExpr": "d_ratio(dcache_l2_all_hits, dcache_l2_all)",
+ "MetricExpr": "d_ratio(DCache_L2_All_Hits, DCache_L2_All)",
"MetricName": "DCache_L2_Hits"
},
{
- "MetricExpr": "d_ratio(dcache_l2_all_miss, dcache_l2_all)",
+ "MetricExpr": "d_ratio(DCache_L2_All_Miss, DCache_L2_All)",
"MetricName": "DCache_L2_Misses"
},
{
[
+ {
+ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
+ "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "(topdown\\-fetch\\-lat / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS)",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "ICACHE_DATA.STALLS / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_TAG.STALLS / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(tma_branch_mispredicts / tma_bad_speculation) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (tma_branch_mispredicts / tma_bad_speculation)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "INT_MISC.UNKNOWN_BRANCH_CYCLES / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: FRONTEND_RETIRED.UNKNOWN_BRANCH",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "DECODE.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu_core@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu_core@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit",
+ "MetricExpr": "(LSD.CYCLES_ACTIVE - LSD.CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "FetchBW;LSD;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_lsd",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
+ "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "topdown\\-br\\-mispredict / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: TOPDOWN.BR_MISPREDICT_SLOTS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
+ "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "topdown\\-mem\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((EXE_ACTIVITY.BOUND_ON_LOADS - MEMORY_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(7 * cpu_core@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - MEMORY_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L1D_MISS - MEMORY_ACTIVITY.STALLS_L2_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L2_MISS - MEMORY_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((25 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (24 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(24 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(9 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "(XQ.FULL_CYCLES + L1D_PEND_MISS.L2_STALLS) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L3_MISS / CLKS)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((MEM_STORE_RETIRED.L2_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(28 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores",
+ "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_streaming_stores",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * cpu_core@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(cpu_core@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu_core@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu_core@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "cpu_core@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions",
+ "MetricExpr": "CPU_CLK_UNHALTED.PAUSE / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_slow_pause",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: CPU_CLK_UNHALTED.PAUSE_INST",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to LFENCE Instructions.",
+ "MetricExpr": "13 * MISC2_RETIRED.LFENCE / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_memory_fence",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "160 * ASSISTS.SSE_AVX_MIX / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5_11 + UOPS_DISPATCHED.PORT_6) / (5 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3_10",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_2_3_10 / (3 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
+ "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_int_vector_128b + tma_int_vector_256b + tma_shuffles",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_int_operations",
+ "PublicDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired). Vector/Matrix Int operations and shuffles are counted. Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents 128-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.",
+ "MetricExpr": "(INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_int_vector_128b",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents 256-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.",
+ "MetricExpr": "(INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_int_vector_256b",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents Shuffle (cross \"vector lane\" data transfers) uops fraction the CPU has retired.",
+ "MetricExpr": "INT_VEC_RETIRED.SHUFFLES / (tma_retiring * SLOTS)",
+ "MetricGroup": "HPC;Pipeline;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_shuffles",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_UOP_RETIRED.ANY / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.MACRO_FUSED / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fused_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused",
+ "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_non_fused_branches",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_int_operations + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "topdown\\-heavy\\-ops / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences. Sample with: UOPS_RETIRED.HEAVY",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "UOPS_RETIRED.MS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: UOPS_RETIRED.MS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * cpu_core@ASSISTS.ANY\\,umask\\=0x1B@ / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults",
+ "MetricExpr": "99 * ASSISTS.PAGE_FAULT / SLOTS",
+ "MetricGroup": "TopdownL5;tma_assists_group",
+ "MetricName": "tma_page_faults",
+ "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults. A Page Fault may apply on first application access to a memory page. Note operating system handling of page faults accounts for the majority of its cost.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists",
+ "MetricExpr": "30 * ASSISTS.FP / SLOTS",
+ "MetricGroup": "HPC;TopdownL5;tma_assists_group",
+ "MetricName": "tma_fp_assists",
+ "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists. FP Assist may apply when working with very small floating point values (so-called denormals).",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops as a result of handing SSE to AVX* or AVX* to SSE transition Assists. ",
+ "MetricExpr": "63 * ASSISTS.SSE_AVX_MIX / SLOTS",
+ "MetricGroup": "HPC;TopdownL5;tma_assists_group",
+ "MetricName": "tma_avx_assists",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources. Sample with: FRONTEND_RETIRED.MS_FLOWS",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
+ "MetricGroup": "Mem;MemoryBW;Offcore",
+ "MetricName": "Memory_Bandwidth",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))",
+ "MetricGroup": "Mem;MemoryLat;Offcore",
+ "MetricName": "Memory_Latency",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ",
+ "MetricGroup": "Mem;MemoryTLB;Offcore",
+ "MetricName": "Memory_Data_TLBs",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
+ "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
+ "MetricName": "Big_Code",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
+ "MetricGroup": "Fed;FetchBW;Frontend",
+ "MetricName": "Instruction_Fetch_BW",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Uops Per Instruction",
+ "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;Ret;Retire",
+ "MetricName": "UPI",
+ "Unit": "cpu_core"
+ },
+ {
+ "BriefDescription": "Instruction per taken branch",
+ "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN",
+ "MetricGroup": "Branches;Fed;FetchBW",
+ "MetricName": "UpTB",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI",
"Unit": "cpu_core"
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
"MetricExpr": "TOPDOWN.SLOTS",
- "MetricGroup": "TmaL1",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS",
"Unit": "cpu_core"
},
{
"BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor",
- "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1",
- "MetricGroup": "SMT;TmaL1",
+ "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1",
+ "MetricGroup": "SMT;tma_L1_group",
"MetricName": "Slots_Utilization",
"Unit": "cpu_core"
},
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC",
"Unit": "cpu_core"
},
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc",
"Unit": "cpu_core"
},
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5 ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common).",
},
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
+ "MetricGroup": "Cor;SMT",
+ "MetricName": "Core_Bound_Likely",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
"MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED",
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP",
"Unit": "cpu_core"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW.",
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting.",
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting.",
"Unit": "cpu_core"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.",
+ "MetricExpr": "(tma_retiring * SLOTS) / cpu_core@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
+ "MetricGroup": "Pipeline;Ret",
+ "MetricName": "Retire",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Estimated fraction of retirement-cycles dealing with repeat instructions",
"MetricExpr": "INST_RETIRED.REP_ITERATION / cpu_core@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
"MetricName": "DSB_Switch_Cost",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_lsd + tma_mite))",
+ "MetricGroup": "DSBmiss;Fed",
+ "MetricName": "DSB_Misses",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
"MetricName": "IpMispredict",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricGroup": "Bad;BrMispredicts",
+ "MetricName": "Branch_Misprediction_Cost",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet",
"Unit": "cpu_core"
},
{
"BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)",
- "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )",
+ "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)",
"MetricGroup": "Bad;Branches",
"MetricName": "Other_Branches",
"Unit": "cpu_core"
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP",
"Unit": "cpu_core"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI",
"Unit": "cpu_core"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load",
"Unit": "cpu_core"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI",
"Unit": "cpu_core"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All",
"Unit": "cpu_core"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load",
"Unit": "cpu_core"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All",
"Unit": "cpu_core"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load",
"Unit": "cpu_core"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI",
"Unit": "cpu_core"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI",
"Unit": "cpu_core"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 4 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (4 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization",
"Unit": "cpu_core"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T",
"Unit": "cpu_core"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T",
"Unit": "cpu_core"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T",
"Unit": "cpu_core"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T",
"Unit": "cpu_core"
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency",
"Unit": "cpu_core"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine.",
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization",
"Unit": "cpu_core"
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use",
"Unit": "cpu_core"
},
{
"BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to frontend stalls.",
- "MetricExpr": "TOPDOWN_FE_BOUND.ALL / (5 * CPU_CLK_UNHALTED.CORE)",
+ "MetricExpr": "TOPDOWN_FE_BOUND.ALL / SLOTS",
"MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
+ "MetricName": "tma_frontend_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to frontend bandwidth restrictions due to decode, predecode, cisc, and other limitations.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.FRONTEND_LATENCY / SLOTS",
+ "MetricGroup": "TopdownL2;tma_frontend_bound_group",
+ "MetricName": "tma_frontend_latency",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to instruction cache misses.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.ICACHE / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_latency_group",
+ "MetricName": "tma_icache",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to Instruction Table Lookaside Buffer (ITLB) misses.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.ITLB / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_latency_group",
+ "MetricName": "tma_itlb",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to BACLEARS, which occurs when the Branch Target Buffer (BTB) prediction or lack thereof, was corrected by a later branch predictor in the frontend",
+ "MetricExpr": "TOPDOWN_FE_BOUND.BRANCH_DETECT / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_latency_group",
+ "MetricName": "tma_branch_detect",
+ "PublicDescription": "Counts the number of issue slots that were not delivered by the frontend due to BACLEARS, which occurs when the Branch Target Buffer (BTB) prediction or lack thereof, was corrected by a later branch predictor in the frontend. Includes BACLEARS due to all branch types including conditional and unconditional jumps, returns, and indirect branches.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to BTCLEARS, which occurs when the Branch Target Buffer (BTB) predicts a taken branch.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.BRANCH_RESTEER / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_latency_group",
+ "MetricName": "tma_branch_resteer",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to frontend bandwidth restrictions due to decode, predecode, cisc, and other limitations.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.FRONTEND_BANDWIDTH / SLOTS",
+ "MetricGroup": "TopdownL2;tma_frontend_bound_group",
+ "MetricName": "tma_frontend_bandwidth",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to the microcode sequencer (MS).",
+ "MetricExpr": "TOPDOWN_FE_BOUND.CISC / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group",
+ "MetricName": "tma_cisc",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to decode stalls.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.DECODE / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group",
+ "MetricName": "tma_decode",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to wrong predecodes.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.PREDECODE / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group",
+ "MetricName": "tma_predecode",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to other common frontend stalls not categorized.",
+ "MetricExpr": "TOPDOWN_FE_BOUND.OTHER / SLOTS",
+ "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group",
+ "MetricName": "tma_other_fb",
+ "ScaleUnit": "100%",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear",
- "MetricExpr": "TOPDOWN_BAD_SPECULATION.ALL / (5 * CPU_CLK_UNHALTED.CORE)",
+ "MetricExpr": "(SLOTS - (TOPDOWN_FE_BOUND.ALL + TOPDOWN_BE_BOUND.ALL + TOPDOWN_RETIRING.ALL)) / SLOTS",
"MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
+ "MetricName": "tma_bad_speculation",
"PublicDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. Only issue slots wasted due to fast nukes such as memory ordering nukes are counted. Other nukes are not accounted for. Counts all issue slots blocked during this recovery window including relevant microcode flows and while uops are not yet available in the instruction queue (IQ). Also includes the issue slots that were consumed by the backend but were thrown away because they were younger than the mispredict or machine clear.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to branch mispredicts.",
+ "MetricExpr": "TOPDOWN_BAD_SPECULATION.MISPREDICT / SLOTS",
+ "MetricGroup": "TopdownL2;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a machine clear (nuke) of any kind including memory ordering and memory disambiguation.",
+ "MetricExpr": "TOPDOWN_BAD_SPECULATION.MACHINE_CLEARS / SLOTS",
+ "MetricGroup": "TopdownL2;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to a machine clear (slow nuke).",
+ "MetricExpr": "TOPDOWN_BAD_SPECULATION.NUKE / SLOTS",
+ "MetricGroup": "TopdownL3;tma_machine_clears_group",
+ "MetricName": "tma_nuke",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to SMC. ",
+ "MetricExpr": "tma_nuke * (MACHINE_CLEARS.SMC / MACHINE_CLEARS.SLOW)",
+ "MetricGroup": "TopdownL4;tma_nuke_group",
+ "MetricName": "tma_smc",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to memory ordering. ",
+ "MetricExpr": "tma_nuke * (MACHINE_CLEARS.MEMORY_ORDERING / MACHINE_CLEARS.SLOW)",
+ "MetricGroup": "TopdownL4;tma_nuke_group",
+ "MetricName": "tma_memory_ordering",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to FP assists. ",
+ "MetricExpr": "tma_nuke * (MACHINE_CLEARS.FP_ASSIST / MACHINE_CLEARS.SLOW)",
+ "MetricGroup": "TopdownL4;tma_nuke_group",
+ "MetricName": "tma_fp_assist",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to memory disambiguation. ",
+ "MetricExpr": "tma_nuke * (MACHINE_CLEARS.DISAMBIGUATION / MACHINE_CLEARS.SLOW)",
+ "MetricGroup": "TopdownL4;tma_nuke_group",
+ "MetricName": "tma_disambiguation",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to page faults. ",
+ "MetricExpr": "tma_nuke * (MACHINE_CLEARS.PAGE_FAULT / MACHINE_CLEARS.SLOW)",
+ "MetricGroup": "TopdownL4;tma_nuke_group",
+ "MetricName": "tma_page_fault",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to a machine clear classified as a fast nuke due to memory ordering, memory disambiguation and memory renaming.",
+ "MetricExpr": "TOPDOWN_BAD_SPECULATION.FASTNUKE / SLOTS",
+ "MetricGroup": "TopdownL3;tma_machine_clears_group",
+ "MetricName": "tma_fast_nuke",
+ "ScaleUnit": "100%",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "TOPDOWN_BE_BOUND.ALL / (5 * CPU_CLK_UNHALTED.CORE)",
+ "MetricExpr": "TOPDOWN_BE_BOUND.ALL / SLOTS",
"MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
+ "MetricName": "tma_backend_bound",
"PublicDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls. Note that uops must be available for consumption in order for this event to count. If a uop is not available (IQ is empty), this event will not count. The rest of these subevents count backend stalls, in cycles, due to an outstanding request which is memory bound vs core bound. The subevents are not slot based events and therefore can not be precisely added or subtracted from the Backend_Bound_Aux subevents which are slot based.",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles due to backend bound stalls that are core execution bound and not attributed to outstanding demand load or store stalls. ",
+ "MetricExpr": "max(0, tma_backend_bound - tma_load_store_bound)",
+ "MetricGroup": "TopdownL2;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles the core is stalled due to stores or loads. ",
+ "MetricExpr": "min((TOPDOWN_BE_BOUND.ALL / SLOTS), (LD_HEAD.ANY_AT_RET / CLKS) + tma_store_bound)",
+ "MetricGroup": "TopdownL2;tma_backend_bound_group",
+ "MetricName": "tma_load_store_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles the core is stalled due to store buffer full.",
+ "MetricExpr": "tma_mem_scheduler * (MEM_SCHEDULER_BLOCK.ST_BUF / MEM_SCHEDULER_BLOCK.ALL)",
+ "MetricGroup": "TopdownL3;tma_load_store_bound_group",
+ "MetricName": "tma_store_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a load block.",
+ "MetricExpr": "LD_HEAD.L1_BOUND_AT_RET / CLKS",
+ "MetricGroup": "TopdownL3;tma_load_store_bound_group",
+ "MetricName": "tma_l1_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a store forward block.",
+ "MetricExpr": "LD_HEAD.ST_ADDR_AT_RET / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a first level TLB miss.",
+ "MetricExpr": "LD_HEAD.DTLB_MISS_AT_RET / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_stlb_hit",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a second level TLB miss requiring a page walk.",
+ "MetricExpr": "LD_HEAD.PGWALK_AT_RET / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_stlb_miss",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a number of other load blocks.",
+ "MetricExpr": "LD_HEAD.OTHER_AT_RET / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_other_l1",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles a core is stalled due to a demand load which hit in the L2 Cache.",
+ "MetricExpr": "(MEM_BOUND_STALLS.LOAD_L2_HIT / CLKS) - (MEM_BOUND_STALLS_AT_RET_CORRECTION * MEM_BOUND_STALLS.LOAD_L2_HIT / MEM_BOUND_STALLS.LOAD)",
+ "MetricGroup": "TopdownL3;tma_load_store_bound_group",
+ "MetricName": "tma_l2_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles a core is stalled due to a demand load which hit in the Last Level Cache (LLC) or other core with HITE/F/M.",
+ "MetricExpr": "(MEM_BOUND_STALLS.LOAD_LLC_HIT / CLKS) - (MEM_BOUND_STALLS_AT_RET_CORRECTION * MEM_BOUND_STALLS.LOAD_LLC_HIT / MEM_BOUND_STALLS.LOAD)",
+ "MetricGroup": "TopdownL3;tma_load_store_bound_group",
+ "MetricName": "tma_l3_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load miss which hit in DRAM or MMIO (Non-DRAM).",
+ "MetricExpr": "(MEM_BOUND_STALLS.LOAD_DRAM_HIT / CLKS) - (MEM_BOUND_STALLS_AT_RET_CORRECTION * MEM_BOUND_STALLS.LOAD_DRAM_HIT / MEM_BOUND_STALLS.LOAD)",
+ "MetricGroup": "TopdownL3;tma_load_store_bound_group",
+ "MetricName": "tma_dram_bound",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load miss which hits in the L2, LLC, DRAM or MMIO (Non-DRAM) but could not be correctly attributed or cycles in which the load miss is waiting on a request buffer.",
+ "MetricExpr": "max(0, tma_load_store_bound - (tma_store_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_dram_bound))",
+ "MetricGroup": "TopdownL3;tma_load_store_bound_group",
+ "MetricName": "tma_other_load_store",
+ "ScaleUnit": "100%",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls",
- "MetricExpr": "(TOPDOWN_BE_BOUND.ALL / (5 * CPU_CLK_UNHALTED.CORE))",
+ "MetricExpr": "tma_backend_bound",
"MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound_Aux",
+ "MetricName": "tma_backend_bound_aux",
"PublicDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls. Note that UOPS must be available for consumption in order for this event to count. If a uop is not available (IQ is empty), this event will not count. All of these subevents count backend stalls, in slots, due to a resource limitation. These are not cycle based events and therefore can not be precisely added or subtracted from the Backend_Bound subevents which are cycle based. These subevents are supplementary to Backend_Bound and can be used to analyze results from a resource perspective at allocation. ",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls",
+ "MetricExpr": "tma_backend_bound",
+ "MetricGroup": "TopdownL2;tma_backend_bound_aux_group",
+ "MetricName": "tma_resource_bound",
+ "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls. Note that uops must be available for consumption in order for this event to count. If a uop is not available (IQ is empty), this event will not count. ",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to memory reservation stalls in which a scheduler is not able to accept uops.",
+ "MetricExpr": "TOPDOWN_BE_BOUND.MEM_SCHEDULER / SLOTS",
+ "MetricGroup": "TopdownL3;tma_resource_bound_group",
+ "MetricName": "tma_mem_scheduler",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles, relative to the number of mem_scheduler slots, in which uops are blocked due to store buffer full",
+ "MetricExpr": "tma_mem_scheduler * (MEM_SCHEDULER_BLOCK.ST_BUF / MEM_SCHEDULER_BLOCK.ALL)",
+ "MetricGroup": "TopdownL4;tma_mem_scheduler_group",
+ "MetricName": "tma_st_buffer",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles, relative to the number of mem_scheduler slots, in which uops are blocked due to load buffer full",
+ "MetricExpr": "tma_mem_scheduler * MEM_SCHEDULER_BLOCK.LD_BUF / MEM_SCHEDULER_BLOCK.ALL",
+ "MetricGroup": "TopdownL4;tma_mem_scheduler_group",
+ "MetricName": "tma_ld_buffer",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cycles, relative to the number of mem_scheduler slots, in which uops are blocked due to RSV full relative ",
+ "MetricExpr": "tma_mem_scheduler * MEM_SCHEDULER_BLOCK.RSV / MEM_SCHEDULER_BLOCK.ALL",
+ "MetricGroup": "TopdownL4;tma_mem_scheduler_group",
+ "MetricName": "tma_rsv",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to IEC or FPC RAT stalls, which can be due to FIQ or IEC reservation stalls in which the integer, floating point or SIMD scheduler is not able to accept uops.",
+ "MetricExpr": "TOPDOWN_BE_BOUND.NON_MEM_SCHEDULER / SLOTS",
+ "MetricGroup": "TopdownL3;tma_resource_bound_group",
+ "MetricName": "tma_non_mem_scheduler",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to the physical register file unable to accept an entry (marble stalls).",
+ "MetricExpr": "TOPDOWN_BE_BOUND.REGISTER / SLOTS",
+ "MetricGroup": "TopdownL3;tma_resource_bound_group",
+ "MetricName": "tma_register",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to the reorder buffer being full (ROB stalls).",
+ "MetricExpr": "TOPDOWN_BE_BOUND.REORDER_BUFFER / SLOTS",
+ "MetricGroup": "TopdownL3;tma_resource_bound_group",
+ "MetricName": "tma_reorder_buffer",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to certain allocation restrictions.",
+ "MetricExpr": "TOPDOWN_BE_BOUND.ALLOC_RESTRICTIONS / SLOTS",
+ "MetricGroup": "TopdownL3;tma_resource_bound_group",
+ "MetricName": "tma_alloc_restriction",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to scoreboards from the instruction queue (IQ), jump execution unit (JEU), or microcode sequencer (MS).",
+ "MetricExpr": "TOPDOWN_BE_BOUND.SERIALIZATION / SLOTS",
+ "MetricGroup": "TopdownL3;tma_resource_bound_group",
+ "MetricName": "tma_serialization",
+ "ScaleUnit": "100%",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Counts the numer of issue slots that result in retirement slots. ",
- "MetricExpr": "TOPDOWN_RETIRING.ALL / (5 * CPU_CLK_UNHALTED.CORE)",
+ "MetricExpr": "TOPDOWN_RETIRING.ALL / SLOTS",
"MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
+ "MetricName": "tma_retiring",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of uops that are not from the microsequencer. ",
+ "MetricExpr": "(TOPDOWN_RETIRING.ALL - UOPS_RETIRED.MS) / SLOTS",
+ "MetricGroup": "TopdownL2;tma_retiring_group",
+ "MetricName": "tma_base",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of floating point operations per uop with all default weighting.",
+ "MetricExpr": "UOPS_RETIRED.FPDIV / SLOTS",
+ "MetricGroup": "TopdownL3;tma_base_group",
+ "MetricName": "tma_fp_uops",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of uops retired excluding ms and fp div uops.",
+ "MetricExpr": "(TOPDOWN_RETIRING.ALL - UOPS_RETIRED.MS - UOPS_RETIRED.FPDIV) / SLOTS",
+ "MetricGroup": "TopdownL3;tma_base_group",
+ "MetricName": "tma_other_ret",
+ "ScaleUnit": "100%",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of uops that are from the complex flows issued by the micro-sequencer (MS)",
+ "MetricExpr": "UOPS_RETIRED.MS / SLOTS",
+ "MetricGroup": "TopdownL2;tma_retiring_group",
+ "MetricName": "tma_ms_uops",
+ "PublicDescription": "Counts the number of uops that are from the complex flows issued by the micro-sequencer (MS). This includes uops from flows due to complex instructions, faults, assists, and inserted flows.",
+ "ScaleUnit": "100%",
"Unit": "cpu_atom"
},
{
},
{
"BriefDescription": "",
- "MetricExpr": "5 * CPU_CLK_UNHALTED.CORE",
+ "MetricExpr": "5 * CLKS",
"MetricName": "SLOTS",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Instructions Per Cycle",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.CORE",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricName": "IPC",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Cycles Per Instruction",
- "MetricExpr": "CPU_CLK_UNHALTED.CORE / INST_RETIRED.ANY",
+ "MetricExpr": "CLKS / INST_RETIRED.ANY",
"MetricName": "CPI",
"Unit": "cpu_atom"
},
},
{
"BriefDescription": "Instructions per Far Branch",
- "MetricExpr": "INST_RETIRED.ANY / ( BR_INST_RETIRED.FAR_BRANCH / 2 )",
+ "MetricExpr": "INST_RETIRED.ANY / (BR_INST_RETIRED.FAR_BRANCH / 2)",
"MetricName": "IpFarBranch",
"Unit": "cpu_atom"
},
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.CORE / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricName": "Turbo_Utilization",
"Unit": "cpu_atom"
},
"MetricName": "CPU_Utilization",
"Unit": "cpu_atom"
},
- {
- "BriefDescription": "Estimated Pause cost. In percent",
- "MetricExpr": "100 * SERIALIZATION.NON_C01_MS_SCB / (5 * CPU_CLK_UNHALTED.CORE)",
- "MetricName": "Estimated_Pause_Cost",
- "Unit": "cpu_atom"
- },
{
"BriefDescription": "Cycle cost per L2 hit",
"MetricExpr": "MEM_BOUND_STALLS.LOAD_L2_HIT / MEM_LOAD_UOPS_RETIRED.L2_HIT",
},
{
"BriefDescription": "Percent of instruction miss cost that hit in the L2",
- "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_L2_HIT / ( MEM_BOUND_STALLS.IFETCH )",
+ "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_L2_HIT / (MEM_BOUND_STALLS.IFETCH)",
"MetricName": "Inst_Miss_Cost_L2Hit_Percent",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Percent of instruction miss cost that hit in the L3",
- "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_LLC_HIT / ( MEM_BOUND_STALLS.IFETCH )",
+ "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_LLC_HIT / (MEM_BOUND_STALLS.IFETCH)",
"MetricName": "Inst_Miss_Cost_L3Hit_Percent",
"Unit": "cpu_atom"
},
{
"BriefDescription": "Percent of instruction miss cost that hit in DRAM",
- "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_DRAM_HIT / ( MEM_BOUND_STALLS.IFETCH )",
+ "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_DRAM_HIT / (MEM_BOUND_STALLS.IFETCH)",
"MetricName": "Inst_Miss_Cost_DRAMHit_Percent",
"Unit": "cpu_atom"
},
[
+ {
+ "BriefDescription": "Counts the number of cacheable memory requests that miss in the LLC. Counts on a per core basis.",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0x2e",
+ "EventName": "LONGEST_LAT_CACHE.MISS",
+ "PEBScounters": "0,1,2,3,4,5",
+ "SampleAfterValue": "200003",
+ "Speculative": "1",
+ "UMask": "0x41",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts the number of cacheable memory requests that access the LLC. Counts on a per core basis.",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0x2e",
+ "EventName": "LONGEST_LAT_CACHE.REFERENCE",
+ "PEBScounters": "0,1,2,3,4,5",
+ "SampleAfterValue": "200003",
+ "Speculative": "1",
+ "UMask": "0x4f",
+ "Unit": "cpu_atom"
+ },
{
"BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which hit in the L2, LLC, DRAM or MMIO (Non-DRAM).",
"CollectPEBSRecord": "2",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 128 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_128",
"MSRIndex": "0x3F6",
"MSRValue": "0x80",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 16 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_16",
"MSRIndex": "0x3F6",
"MSRValue": "0x10",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 256 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_256",
"MSRIndex": "0x3F6",
"MSRValue": "0x100",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 32 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_32",
"MSRIndex": "0x3F6",
"MSRValue": "0x20",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 4 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_4",
"MSRIndex": "0x3F6",
"MSRValue": "0x4",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 512 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_512",
"MSRIndex": "0x3F6",
"MSRValue": "0x200",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 64 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_64",
"MSRIndex": "0x3F6",
"MSRValue": "0x40",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 8 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.",
- "CollectPEBSRecord": "3",
- "Counter": "0,1,2,3,4,5",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1",
"Data_LA": "1",
"EventCode": "0xd0",
"EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_8",
"MSRIndex": "0x3F6",
"MSRValue": "0x8",
"PEBS": "2",
- "PEBScounters": "0,1,2,3,4,5",
+ "PEBScounters": "0,1",
"SampleAfterValue": "1000003",
"TakenAlone": "1",
"UMask": "0x5",
},
{
"BriefDescription": "Counts the number of stores uops retired. Counts with or without PEBS enabled.",
- "CollectPEBSRecord": "3",
+ "CollectPEBSRecord": "2",
"Counter": "0,1,2,3,4,5",
"Data_LA": "1",
"EventCode": "0xd0",
"UMask": "0x6",
"Unit": "cpu_atom"
},
+ {
+ "BriefDescription": "Counts demand data reads that were supplied by the L3 cache.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_DATA_RD.L3_HIT",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x3F803C0001",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, and modified data was forwarded.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x10003C0001",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, but no data was forwarded.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_NO_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x4003C0001",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, and non-modified data was forwarded.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x8003C0001",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
+ {
+ "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were supplied by the L3 cache.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_RFO.L3_HIT",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x3F803C0002",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
{
"BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were supplied by the L3 cache where a snoop was sent, the snoop hit, and modified data was forwarded.",
"Counter": "0,1,2,3,4,5",
"UMask": "0x1",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Cycles the Microcode Sequencer is busy.",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1,2,3",
+ "EventCode": "0x87",
+ "EventName": "DECODE.MS_BUSY",
+ "PEBScounters": "0,1,2,3",
+ "SampleAfterValue": "500009",
+ "Speculative": "1",
+ "UMask": "0x2",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "DSB-to-MITE switch true penalty cycles.",
"CollectPEBSRecord": "2",
"UMask": "0x1",
"Unit": "cpu_atom"
},
+ {
+ "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_DATA_RD.L3_MISS_LOCAL",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x3F84400001",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
{
"BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.",
"Counter": "0,1,2,3,4,5",
"UMask": "0x1",
"Unit": "cpu_atom"
},
+ {
+ "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.DEMAND_RFO.L3_MISS_LOCAL",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x3F84400002",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
{
"BriefDescription": "Execution stalls while L3 cache miss demand load is outstanding.",
"CollectPEBSRecord": "2",
[
+ {
+ "BriefDescription": "Counts modified writebacks from L1 cache and L2 cache that have any type of response.",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0xB7",
+ "EventName": "OCR.COREWB_M.ANY_RESPONSE",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x10008",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
{
"BriefDescription": "Counts demand data reads that have any type of response.",
"Counter": "0,1,2,3,4,5",
"UMask": "0x1",
"Unit": "cpu_core"
},
+ {
+ "BriefDescription": "Counts demand data reads that were supplied by DRAM.",
+ "Counter": "0,1,2,3,4,5,6,7",
+ "EventCode": "0x2A,0x2B",
+ "EventName": "OCR.DEMAND_DATA_RD.DRAM",
+ "MSRIndex": "0x1a6,0x1a7",
+ "MSRValue": "0x184000001",
+ "SampleAfterValue": "100003",
+ "UMask": "0x1",
+ "Unit": "cpu_core"
+ },
{
"BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that have any type of response.",
"Counter": "0,1,2,3,4,5,6,7",
"UMask": "0x3",
"Unit": "cpu_atom"
},
+ {
+ "BriefDescription": "Counts the number of unhalted reference clock cycles at TSC frequency.",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1,2,3,4,5",
+ "EventCode": "0x3c",
+ "EventName": "CPU_CLK_UNHALTED.REF_TSC_P",
+ "PEBScounters": "0,1,2,3,4,5",
+ "SampleAfterValue": "2000003",
+ "Speculative": "1",
+ "UMask": "0x1",
+ "Unit": "cpu_atom"
+ },
{
"BriefDescription": "Counts the number of unhalted core clock cycles. (Fixed event)",
"CollectPEBSRecord": "2",
"PEBScounters": "0,1,2,3,4,5,6,7",
"SampleAfterValue": "100003",
"Speculative": "1",
- "UMask": "0x1f",
+ "UMask": "0x1b",
"Unit": "cpu_core"
},
{
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
+ "MetricExpr": "ICACHE.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ITLB_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. ",
+ "MetricExpr": "BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. ",
+ "MetricExpr": "MACHINE_CLEARS.COUNT * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "tma_branch_resteers - tma_mispredicts_resteers - tma_clears_resteers",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB) / (CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_LOAD_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_STORE_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (CYCLE_ACTIVITY.STALLS_TOTAL - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
- {
- "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Cor;Flops;HPC_SMT",
- "MetricName": "FP_Arith_Utilization_SMT",
- "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;BrMispredicts",
"MetricName": "Branch_Misprediction_Cost"
},
- {
- "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
- "MetricGroup": "Bad;BrMispredicts_SMT",
- "MetricName": "Branch_Misprediction_Cost_SMT"
- },
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "(cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * (DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED)) / CORE_CLKS",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "ICACHE.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ITLB_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "MACHINE_CLEARS.COUNT * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "tma_branch_resteers - tma_mispredicts_resteers - tma_clears_resteers",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: FRONTEND_RETIRED.UNKNOWN_BRANCH",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: TOPDOWN.BR_MISPREDICT_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB) / (CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_LOAD_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_STORE_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (CYCLE_ACTIVITY.STALLS_TOTAL - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU)",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3_10",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads)",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads)",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data)",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address)",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: UOPS_RETIRED.MS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
- {
- "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Cor;Flops;HPC_SMT",
- "MetricName": "FP_Arith_Utilization_SMT",
- "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;BrMispredicts",
"MetricName": "Branch_Misprediction_Cost"
},
- {
- "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
- "MetricGroup": "Bad;BrMispredicts_SMT",
- "MetricName": "Branch_Misprediction_Cost_SMT"
- },
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( 2 * (( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) if #core_wide < 1 else ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD) )",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
- "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@ ) / ( cbox_0@event\\=0x0@ / duration_time )",
- "MetricGroup": "Mem;MemoryLat;SoC",
- "MetricName": "MEM_Read_Latency"
- },
- {
- "BriefDescription": "Average number of parallel data read requests to external memory. Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182\\,thresh\\=1@",
- "MetricGroup": "Mem;MemoryBW;SoC",
- "MetricName": "MEM_Parallel_Reads"
- },
- {
- "BriefDescription": "Socket actual clocks when any core is active on that socket",
- "MetricExpr": "cbox_0@event\\=0x0@",
- "MetricGroup": "SoC",
- "MetricName": "Socket_CLKS"
+ "BriefDescription": "Average latency of all requests to external memory (in Uncore cycles)",
+ "MetricExpr": "UNC_ARB_TRK_OCCUPANCY.ALL / arb@event\\=0x81\\,umask\\=0x1@",
+ "MetricGroup": "Mem;SoC",
+ "MetricName": "MEM_Request_Latency"
},
{
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
+ "BriefDescription": "Average number of parallel requests to external memory. Accounts for all requests",
+ "MetricExpr": "UNC_ARB_TRK_OCCUPANCY.ALL / arb@event\\=0x81\\,umask\\=0x1@",
+ "MetricGroup": "Mem;SoC",
+ "MetricName": "MEM_Parallel_Requests"
},
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
+ "MetricExpr": "ICACHE.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ITLB_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. ",
+ "MetricExpr": "BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. ",
+ "MetricExpr": "MACHINE_CLEARS.COUNT * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "tma_branch_resteers - tma_mispredicts_resteers - tma_clears_resteers",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB) / (CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_LOAD_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "41 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "310 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "(200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 180 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(200 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_MISS.REMOTE_HITM + 60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_STORE_MISSES.WALK_COMPLETED) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (CYCLE_ACTIVITY.STALLS_TOTAL - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
"MetricGroup": "Branches;Fed;FetchBW",
"MetricName": "UpTB"
},
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
+ "MetricName": "CPI"
+ },
{
"BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
"MetricExpr": "CPU_CLK_UNHALTED.THREAD",
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
- {
- "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Cor;Flops;HPC_SMT",
- "MetricName": "FP_Arith_Utilization_SMT",
- "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;BrMispredicts",
"MetricName": "Branch_Misprediction_Cost"
},
- {
- "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
- "MetricGroup": "Bad;BrMispredicts_SMT",
- "MetricName": "Branch_Misprediction_Cost_SMT"
- },
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * (DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED)) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
"BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@ ) / ( cbox_0@event\\=0x0@ / duration_time )",
+ "MetricExpr": "1000000000 * (cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@) / (Socket_CLKS / duration_time)",
"MetricGroup": "Mem;MemoryLat;SoC",
"MetricName": "MEM_Read_Latency"
},
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricGroup": "Power",
"MetricName": "C7_Pkg_Residency"
},
+ {
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
+ },
{
"BriefDescription": "CPU operating frequency (in GHz)",
- "MetricExpr": "( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000",
+ "MetricExpr": "(( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000) / duration_time",
"MetricGroup": "",
"MetricName": "cpu_operating_frequency",
"ScaleUnit": "1GHz"
},
- {
- "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY",
- "MetricGroup": "",
- "MetricName": "cpi",
- "ScaleUnit": "1per_instr"
- },
{
"BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
"MetricExpr": "MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
"BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches",
+ "MetricName": "l1d_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches",
+ "MetricName": "l2_mpi",
"ScaleUnit": "1per_instr"
},
{
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds",
- "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) ) ) * duration_time",
+ "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_data_read_demand_plus_prefetch_miss_latency",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds",
- "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) ) ) * duration_time",
+ "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds",
- "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) ) ) * duration_time",
+ "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
- "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )",
+ "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_local_dram",
+ "MetricName": "numa_reads_addressed_to_local_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
- "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )",
+ "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_remote_dram",
+ "MetricName": "numa_reads_addressed_to_remote_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uncore operating frequency in GHz",
- "MetricExpr": "UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) / 1000000000",
+ "MetricExpr": "( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) / 1000000000) / duration_time",
"MetricGroup": "",
"MetricName": "uncore_frequency",
"ScaleUnit": "1GHz"
"BriefDescription": "Intel(R) Quick Path Interconnect (QPI) data transmit bandwidth (MB/sec)",
"MetricExpr": "( UNC_Q_TxL_FLITS_G0.DATA * 8 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "qpi_data_transmit_bw_only_data",
+ "MetricName": "qpi_data_transmit_bw",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.",
"MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x19e@ * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_read",
+ "MetricName": "io_bandwidth_disk_or_network_writes",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.",
"MetricExpr": "(( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x1c8\\,filter_tid\\=0x3e@ + cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x180\\,filter_tid\\=0x3e@ ) * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_write",
+ "MetricName": "io_bandwidth_disk_or_network_reads",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_decoded_icache_dsb",
+ "MetricName": "percent_uops_delivered_from_decoded_icache",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite",
+ "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms",
+ "MetricName": "percent_uops_delivered_from_microcode_sequencer",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from loop stream detector(LSD) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( LSD.UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_loop_stream_detector_lsd",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
- "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1;PGO",
- "MetricName": "tma_frontend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.",
- "MetricExpr": "100 * ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_latency_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
- "MetricExpr": "100 * ( ICACHE.IFDATA_STALL / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_icache_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.",
- "MetricExpr": "100 * ( ( 14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=0x1@ + 7 * ITLB_MISSES.WALK_COMPLETED ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_itlb_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.",
- "MetricExpr": "100 * ( ( 12 ) * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_branch_resteers_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
- "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_dsb_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
- "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_lcp_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.",
- "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_ms_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
- "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_bandwidth_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
- "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_mite_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
- "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_dsb_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
- "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_bad_speculation_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.",
- "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_branch_mispredicts_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.",
- "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_machine_clears_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
- "MetricExpr": "100 * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_backend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
- "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB ) / ( ( CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - ( UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent",
- "MetricName": "tma_memory_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.",
- "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l1_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l2_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) * CYCLE_ACTIVITY.STALLS_L2_MISS / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l3_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.",
- "MetricExpr": "100 * ( min( ( ( 1 - ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) ) * CYCLE_ACTIVITY.STALLS_L2_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_dram_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.",
- "MetricExpr": "100 * ( RESOURCE_STALLS.SB / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_store_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
- "MetricExpr": "100 * ( ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB ) / ( ( CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - ( UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent",
- "MetricName": "tma_core_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.",
- "MetricExpr": "100 * ( ARITH.FPU_DIV_ACTIVE / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_divider_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
- "MetricExpr": "100 * ( ( ( ( CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - ( UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_ports_utilization_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ",
- "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_retiring_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_light_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
- "MetricExpr": "100 * ( ( INST_RETIRED.X87 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / INST_RETIRED.ANY ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) )",
- "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fp_arith_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_heavy_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_microcode_sequencer_percent",
+ "MetricName": "percent_uops_delivered_from_loop_stream_detector",
"ScaleUnit": "1%"
}
]
"Unit": "CBO"
},
{
- "BriefDescription": "LLC misses - demand and prefetch data reads - excludes LLC prefetches. Derived from unc_c_tor_inserts.miss_opcode",
+ "BriefDescription": "TOR Inserts; Miss Opcode Match",
"Counter": "0,1,2,3",
"EventCode": "0x35",
- "EventName": "LLC_MISSES.DATA_READ",
- "Filter": "filter_opc=0x182",
+ "EventName": "UNC_C_TOR_INSERTS.MISS_OPCODE",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "CBO"
},
{
- "BriefDescription": "LLC misses - demand and prefetch data reads - excludes LLC prefetches",
+ "BriefDescription": "LLC misses - demand and prefetch data reads - excludes LLC prefetches. Derived from unc_c_tor_inserts.miss_opcode",
"Counter": "0,1,2,3",
"EventCode": "0x35",
- "EventName": "UNC_C_TOR_INSERTS.MISS_OPCODE",
+ "EventName": "LLC_MISSES.DATA_READ",
"Filter": "filter_opc=0x182",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data",
+ "BriefDescription": "Flits Transferred - Group 0; Data Tx Flits",
"Counter": "0,1,2,3",
- "EventName": "QPI_DATA_BANDWIDTH_TX",
+ "EventName": "UNC_Q_TxL_FLITS_G0.DATA",
"PerPkg": "1",
- "ScaleUnit": "8Bytes",
"UMask": "0x2",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of data flits transmitted ",
+ "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data",
"Counter": "0,1,2,3",
- "EventName": "UNC_Q_TxL_FLITS_G0.DATA",
+ "EventName": "QPI_DATA_BANDWIDTH_TX",
"PerPkg": "1",
"ScaleUnit": "8Bytes",
"UMask": "0x2",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data",
+ "BriefDescription": "Flits Transferred - Group 0; Non-Data protocol Tx Flits",
"Counter": "0,1,2,3",
- "EventName": "QPI_CTL_BANDWIDTH_TX",
+ "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA",
"PerPkg": "1",
- "ScaleUnit": "8Bytes",
"UMask": "0x4",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of non data (control) flits transmitted ",
+ "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data",
"Counter": "0,1,2,3",
- "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA",
+ "EventName": "QPI_CTL_BANDWIDTH_TX",
"PerPkg": "1",
"ScaleUnit": "8Bytes",
"UMask": "0x4",
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
+ "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM Reads (RD_CAS + Underfills)",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_READ",
+ "EventName": "UNC_M_CAS_COUNT.RD",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller",
+ "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.RD",
+ "EventName": "LLC_MISSES.MEM_READ",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
+ "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM WR_CAS (both Modes)",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_WRITE",
+ "EventName": "UNC_M_CAS_COUNT.WR",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0xC",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller",
+ "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.WR",
+ "EventName": "LLC_MISSES.MEM_WRITE",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0xC",
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "(ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "9 * BACLEARS.ANY / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - tma_frontend_bound - (UOPS_ISSUED.ANY + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(12 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (11 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OCR.DEMAND_DATA_RD.L3_HIT.HIT_OTHER_CORE_FWD))) + (44 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OCR.DEMAND_DATA_RD.L3_HIT.HIT_OTHER_CORE_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(17 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound) - tma_pmm_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "(59.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "(127 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "((89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a",
+ "MetricExpr": "(((1 - ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) / ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) + (25 * (MEM_LOAD_RETIRED.LOCAL_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 33 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))))) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)) if (1000000 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM) > MEM_LOAD_RETIRED.L1_MISS) else 0)",
+ "MetricGroup": "MemoryBound;Server;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_pmm_bound",
+ "PublicDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 11 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "((110 * Average_Frequency) * (OCR.DEMAND_RFO.L3_MISS.REMOTE_HITM + OCR.PF_L2_RFO.L3_MISS.REMOTE_HITM) + (47.5 * Average_Frequency) * (OCR.DEMAND_RFO.L3_HIT.HITM_OTHER_CORE + OCR.PF_L2_RFO.L3_HIT.HITM_OTHER_CORE)) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_NONE / 2 if #SMT_on else CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "PARTIAL_RAT_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: PARTIAL_RAT_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions",
+ "MetricExpr": "40 * ROB_MISC_EVENTS.PAUSE_INST / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_slow_pause",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_1 - UOPS_EXECUTED.CORE_CYCLES_GE_2) / 2 if #SMT_on else EXE_ACTIVITY.1_PORTS_UTIL) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_2 - UOPS_EXECUTED.CORE_CYCLES_GE_3) / 2 if #SMT_on else EXE_ACTIVITY.2_PORTS_UTIL) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_GE_3 / 2 if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_3) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
- "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "Bad;BadSpec;BrMispredicts",
- "MetricName": "Mispredictions"
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_512b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions",
+ "MetricExpr": "tma_light_operations * UOPS_RETIRED.MACRO_FUSED / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fused_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused",
+ "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_non_fused_branches",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY) / SLOTS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * (FP_ASSIST.ANY + OTHER_ASSISTS.ANY) / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
- "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT",
- "MetricName": "Mispredictions_SMT"
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
},
{
"BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "Memory_Bandwidth"
},
- {
- "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
- "MetricGroup": "Mem;MemoryBW;Offcore_SMT",
- "MetricName": "Memory_Bandwidth_SMT"
- },
{
"BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)))",
"MetricGroup": "Mem;MemoryLat;Offcore",
"MetricName": "Memory_Latency"
},
- {
- "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )",
- "MetricGroup": "Mem;MemoryLat;Offcore_SMT",
- "MetricName": "Memory_Latency_SMT"
- },
{
"BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency))) ",
"MetricGroup": "Mem;MemoryTLB;Offcore",
"MetricName": "Memory_Data_TLBs"
},
- {
- "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
- "MetricGroup": "Mem;MemoryTLB;Offcore_SMT",
- "MetricName": "Memory_Data_TLBs_SMT"
- },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
- {
- "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
- "MetricGroup": "Ret_SMT",
- "MetricName": "Branching_Overhead_SMT"
- },
{
"BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
"MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
"MetricName": "Big_Code"
},
- {
- "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
- "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT",
- "MetricName": "Big_Code_SMT"
- },
{
"BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
- "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
"MetricGroup": "Fed;FetchBW;Frontend",
"MetricName": "Instruction_Fetch_BW"
},
- {
- "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
- "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))",
- "MetricGroup": "Fed;FetchBW;Frontend_SMT",
- "MetricName": "Instruction_Fetch_BW_SMT"
- },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
"MetricGroup": "Branches;Fed;FetchBW",
"MetricName": "UpTB"
},
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
+ "MetricName": "CPI"
+ },
{
"BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
"MetricExpr": "CPU_CLK_UNHALTED.THREAD",
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
- {
- "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Cor;Flops;HPC_SMT",
- "MetricName": "FP_Arith_Utilization_SMT",
- "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
- "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if 0 > 0.5 else 0",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
"MetricGroup": "Cor;SMT",
"MetricName": "Core_Bound_Likely"
},
- {
- "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
- "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if (1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 )) > 0.5 else 0",
- "MetricGroup": "Cor;SMT_SMT",
- "MetricName": "Core_Bound_Likely_SMT"
- },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX512",
"PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
- "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) )",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))",
"MetricGroup": "DSBmiss;Fed",
"MetricName": "DSB_Misses"
},
- {
- "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
- "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) )",
- "MetricGroup": "DSBmiss;Fed_SMT",
- "MetricName": "DSB_Misses_SMT"
- },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
},
{
"BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;BrMispredicts",
"MetricName": "Branch_Misprediction_Cost"
},
- {
- "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
- "MetricGroup": "Bad;BrMispredicts_SMT",
- "MetricName": "Branch_Misprediction_Cost_SMT"
- },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are taken conditionals",
- "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches;CodeGen;PGO",
"MetricName": "Cond_TK"
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
{
"BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps",
- "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "Jump"
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)",
- "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_Silent_PKI"
},
{
"BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction",
- "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_NonSilent_PKI"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License0_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes."
},
- {
- "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Power_SMT",
- "MetricName": "Power_License0_Utilization_SMT",
- "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes. SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License1_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions."
},
- {
- "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Power_SMT",
- "MetricName": "Power_License1_Utilization_SMT",
- "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License2_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions."
},
- {
- "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Power_SMT",
- "MetricName": "Power_License2_Utilization_SMT",
- "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
"BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@ ) / ( cha_0@event\\=0x0@ / duration_time )",
+ "MetricExpr": "1000000000 * (cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@) / (Socket_CLKS / duration_time)",
"MetricGroup": "Mem;MemoryLat;SoC",
"MetricName": "MEM_Read_Latency"
},
},
{
"BriefDescription": "Average latency of data read request to external 3D X-Point memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": "( 1000000000 * ( imc@event\\=0xe0\\,umask\\=0x1@ / imc@event\\=0xe3@ ) / imc_0@event\\=0x0@ )",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": "(1000000000 * (imc@event\\=0xe0\\,umask\\=0x1@ / imc@event\\=0xe3@) / imc_0@event\\=0x0@)",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_PMM_Read_Latency"
},
{
"BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": "1000000000 * ( UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS ) / imc_0@event\\=0x0@",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": "1000000000 * (UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS) / imc_0@event\\=0x0@",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_DRAM_Read_Latency"
},
{
"BriefDescription": "Average 3DXP Memory Bandwidth Use for reads [GB / sec]",
- "MetricExpr": "( ( 64 * imc@event\\=0xe3@ / 1000000000 ) / duration_time )",
- "MetricGroup": "Mem;MemoryBW;SoC;Server",
+ "MetricExpr": "((64 * imc@event\\=0xe3@ / 1000000000) / duration_time)",
+ "MetricGroup": "Mem;MemoryBW;Server;SoC",
"MetricName": "PMM_Read_BW"
},
{
"BriefDescription": "Average 3DXP Memory Bandwidth Use for Writes [GB / sec]",
- "MetricExpr": "( ( 64 * imc@event\\=0xe7@ / 1000000000 ) / duration_time )",
- "MetricGroup": "Mem;MemoryBW;SoC;Server",
+ "MetricExpr": "((64 * imc@event\\=0xe7@ / 1000000000) / duration_time)",
+ "MetricGroup": "Mem;MemoryBW;Server;SoC",
"MetricName": "PMM_Write_BW"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]",
- "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3) * 4 / 1000000000 / duration_time",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Write_BW"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]",
- "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3 ) * 4 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3) * 4 / 1000000000 / duration_time",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Read_BW"
},
{
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cha_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricName": "C7_Pkg_Residency"
},
{
- "BriefDescription": "Percentage of time spent in the active CPU power state C0",
- "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC",
- "MetricGroup": "",
- "MetricName": "cpu_utilization_percent",
- "ScaleUnit": "1%"
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
},
{
"BriefDescription": "CPU operating frequency (in GHz)",
- "MetricExpr": "( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000",
+ "MetricExpr": "(( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000) / duration_time",
"MetricGroup": "",
"MetricName": "cpu_operating_frequency",
"ScaleUnit": "1GHz"
},
- {
- "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY",
- "MetricGroup": "",
- "MetricName": "cpi",
- "ScaleUnit": "1per_instr"
- },
{
"BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
"MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
"BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches",
+ "MetricName": "l1d_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches",
+ "MetricName": "l2_mpi",
"ScaleUnit": "1per_instr"
},
{
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043300000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043300000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043300000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043300000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_data_read_demand_plus_prefetch_miss_latency",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043200000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043200000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043100000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043100000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests",
"ScaleUnit": "1ns"
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB.",
"MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "itlb_2nd_level_mpi",
+ "MetricName": "itlb_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for 2 megabyte and 4 megabyte page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the Instruction Translation Lookaside Buffer (ITLB) and further levels of TLB.",
"MetricExpr": "ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "itlb_2nd_level_large_page_mpi",
+ "MetricName": "itlb_large_page_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.",
"MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "dtlb_2nd_level_load_mpi",
+ "MetricName": "dtlb_load_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for 2 megabyte page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the Data Translation Lookaside Buffer (DTLB) and further levels of TLB.",
"MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "dtlb_2nd_level_2mb_large_page_load_mpi",
+ "MetricName": "dtlb_2mb_large_page_load_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.",
"MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "dtlb_2nd_level_store_mpi",
+ "MetricName": "dtlb_store_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_local_dram",
+ "MetricName": "numa_reads_addressed_to_local_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_remote_dram",
+ "MetricName": "numa_reads_addressed_to_remote_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uncore operating frequency in GHz",
- "MetricExpr": "UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) / 1000000000",
+ "MetricExpr": "( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) / 1000000000) / duration_time",
"MetricGroup": "",
"MetricName": "uncore_frequency",
"ScaleUnit": "1GHz"
"BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)",
"MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "upi_data_transmit_bw_only_data",
+ "MetricName": "upi_data_transmit_bw",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.",
"MetricExpr": "(( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_read",
+ "MetricName": "io_bandwidth_disk_or_network_writes",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.",
"MetricExpr": "(( UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART0 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART1 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART2 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART3 ) * 4 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_write",
+ "MetricName": "io_bandwidth_disk_or_network_reads",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.DSB_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_decoded_icache_dsb",
+ "MetricName": "percent_uops_delivered_from_decoded_icache",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MITE_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite",
+ "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MS_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms",
+ "MetricName": "percent_uops_delivered_from_microcode_sequencer",
"ScaleUnit": "1%"
},
{
"ScaleUnit": "1MB/s"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
- "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1;PGO",
- "MetricName": "tma_frontend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.",
- "MetricExpr": "100 * ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_latency_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
- "MetricExpr": "100 * ( ( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_icache_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.",
- "MetricExpr": "100 * ( ICACHE_64B.IFTAG_STALL / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_itlb_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.",
- "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( ( 9 ) * BACLEARS.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_branch_resteers_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
- "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_dsb_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
- "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_lcp_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.",
- "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_ms_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
- "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_bandwidth_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
- "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_mite_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
- "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_dsb_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
- "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_bad_speculation_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.",
- "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_branch_mispredicts_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.",
- "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_machine_clears_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
- "MetricExpr": "100 * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_backend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
- "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent",
- "MetricName": "tma_memory_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.",
- "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l1_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l2_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l3_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.",
- "MetricExpr": "100 * ( min( ( ( ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) ) ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_dram_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ",
- "MetricExpr": "100 * ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;Server;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_pmm_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.",
- "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_store_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
- "MetricExpr": "100 * ( ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent",
- "MetricName": "tma_core_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.",
- "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_divider_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
- "MetricExpr": "100 * ( ( EXE_ACTIVITY.EXE_BOUND_0_PORTS + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) ) / ( CPU_CLK_UNHALTED.THREAD ) if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_ports_utilization_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ",
- "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_retiring_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_light_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) )",
- "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fp_arith_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_memory_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fused_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_non_fused_branches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_nop_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
- "MetricExpr": "100 * ( max( 0 , ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) - ( ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) ) ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_other_light_ops_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_heavy_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_few_uops_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_microcode_sequencer_percent",
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.",
+ "MetricExpr": "100 * ( ( LSD.CYCLES_ACTIVE - LSD.CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
+ "MetricGroup": "FetchBW;LSD;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
+ "MetricName": "tma_lsd",
"ScaleUnit": "1%"
}
]
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
+ "BriefDescription": "All DRAM Read CAS Commands issued (including underfills)",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_READ",
+ "EventName": "UNC_M_CAS_COUNT.RD",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller",
+ "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.RD",
+ "EventName": "LLC_MISSES.MEM_READ",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
+ "BriefDescription": "All DRAM Write CAS commands issued",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_WRITE",
+ "EventName": "UNC_M_CAS_COUNT.WR",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0xC",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller",
+ "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.WR",
+ "EventName": "LLC_MISSES.MEM_WRITE",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0xC",
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x01",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x02",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x04",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x08",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x01",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x02",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x04",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x08",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"Unit": "UPI LL"
},
{
- "BriefDescription": "FLITs received which bypassed the Slot0 Receive Buffer",
+ "BriefDescription": "FLITs received which bypassed the Slot0 Recieve Buffer",
"Counter": "0,1,2,3",
"EventCode": "0x31",
"EventName": "UNC_UPI_RxL_BYPASSED.SLOT2",
"UMask": "0x2"
},
{
- "BriefDescription": "L1D miss oustandings duration in cycles",
+ "BriefDescription": "L1D miss outstanding duration in cycles",
"Counter": "2",
"CounterHTOff": "2",
"EventCode": "0x48",
"UMask": "0x8"
},
{
- "BriefDescription": "Cacheable and noncachaeble code read requests",
+ "BriefDescription": "Cacheable and noncacheable code read requests",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0xB0",
"UMask": "0x4"
},
{
- "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"UMask": "0x30"
},
{
- "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.",
+ "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"UMask": "0x10"
},
{
- "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.",
+ "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"UMask": "0x10"
},
{
- "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
"UMask": "0x10"
},
{
- "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
"UMask": "0x30"
},
{
- "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
+ "MetricExpr": "ICACHE.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.REQUEST_FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.HITM_OTHER_CORE / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "10 * ARITH.DIVIDER_UOPS / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "( UOPS_EXECUTED.CORE / 2 / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@) ) if #SMT_on else UOPS_EXECUTED.CORE / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)",
+ "MetricExpr": "(UOPS_EXECUTED.CORE / 2 / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)) if #SMT_on else UOPS_EXECUTED.CORE / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
"MetricName": "BpTkBranch"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
"UMask": "0x8"
},
{
- "BriefDescription": "Cacheable and noncachaeble code read requests",
+ "BriefDescription": "Cacheable and noncacheable code read requests",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0xB0",
"UMask": "0x4"
},
{
- "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"UMask": "0x30"
},
{
- "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.",
+ "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"UMask": "0x10"
},
{
- "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.",
+ "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"UMask": "0x10"
},
{
- "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
"UMask": "0x10"
},
{
- "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
"UMask": "0x30"
},
{
- "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
+ "MetricExpr": "ICACHE.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.REQUEST_FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "41 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "310 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "(200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 180 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(200 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_MISS.REMOTE_HITM + 60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "10 * ARITH.DIVIDER_UOPS / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
"MetricGroup": "Branches;Fed;FetchBW",
"MetricName": "UpTB"
},
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
+ "MetricName": "CPI"
+ },
{
"BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
"MetricExpr": "CPU_CLK_UNHALTED.THREAD",
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "( UOPS_EXECUTED.CORE / 2 / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@) ) if #SMT_on else UOPS_EXECUTED.CORE / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)",
+ "MetricExpr": "(UOPS_EXECUTED.CORE / 2 / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)) if #SMT_on else UOPS_EXECUTED.CORE / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
"MetricName": "BpTkBranch"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
"BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@ ) / ( cbox_0@event\\=0x0@ / duration_time )",
+ "MetricExpr": "1000000000 * (cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@) / (Socket_CLKS / duration_time)",
"MetricGroup": "Mem;MemoryLat;SoC",
"MetricName": "MEM_Read_Latency"
},
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricGroup": "Power",
"MetricName": "C7_Pkg_Residency"
},
+ {
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
+ },
{
"BriefDescription": "CPU operating frequency (in GHz)",
- "MetricExpr": "( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000",
+ "MetricExpr": "(( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000) / duration_time",
"MetricGroup": "",
"MetricName": "cpu_operating_frequency",
"ScaleUnit": "1GHz"
},
- {
- "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.",
- "MetricExpr": " CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY ",
- "MetricGroup": "",
- "MetricName": "cpi",
- "ScaleUnit": "1per_instr"
- },
{
"BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
- "MetricExpr": " MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY ",
+ "MetricExpr": "MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "loads_per_instr",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "The ratio of number of completed memory store instructions to the total number completed instructions",
- "MetricExpr": " MEM_UOPS_RETIRED.ALL_STORES / INST_RETIRED.ANY ",
+ "MetricExpr": "MEM_UOPS_RETIRED.ALL_STORES / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "stores_per_instr",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions",
- "MetricExpr": " L1D.REPLACEMENT / INST_RETIRED.ANY ",
+ "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches",
+ "MetricName": "l1d_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of demand load requests hitting in L1 data cache to the total number of completed instructions",
- "MetricExpr": " MEM_LOAD_UOPS_RETIRED.L1_HIT / INST_RETIRED.ANY ",
+ "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L1_HIT / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "l1d_demand_data_read_hits_per_instr",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of code read requests missing in L1 instruction cache (includes prefetches) to the total number of completed instructions",
- "MetricExpr": " L2_RQSTS.ALL_CODE_RD / INST_RETIRED.ANY ",
+ "MetricExpr": "L2_RQSTS.ALL_CODE_RD / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "l1_i_code_read_misses_with_prefetches_per_instr",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed demand load requests hitting in L2 cache to the total number of completed instructions",
- "MetricExpr": " MEM_LOAD_UOPS_RETIRED.L2_HIT / INST_RETIRED.ANY ",
+ "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L2_HIT / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "l2_demand_data_read_hits_per_instr",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions",
- "MetricExpr": " L2_LINES_IN.ALL / INST_RETIRED.ANY ",
+ "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches",
+ "MetricName": "l2_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed data read request missing L2 cache to the total number of completed instructions",
- "MetricExpr": " MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY ",
+ "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "l2_demand_data_read_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of code read request missing L2 cache to the total number of completed instructions",
- "MetricExpr": " L2_RQSTS.CODE_RD_MISS / INST_RETIRED.ANY ",
+ "MetricExpr": "L2_RQSTS.CODE_RD_MISS / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "l2_demand_code_mpi",
"ScaleUnit": "1per_instr"
},
+ {
+ "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds",
+ "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
+ "MetricGroup": "",
+ "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency",
+ "ScaleUnit": "1ns"
+ },
+ {
+ "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds",
+ "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
+ "MetricGroup": "",
+ "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests",
+ "ScaleUnit": "1ns"
+ },
+ {
+ "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds",
+ "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
+ "MetricGroup": "",
+ "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests",
+ "ScaleUnit": "1ns"
+ },
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB.",
- "MetricExpr": " ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY ",
+ "MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "itlb_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for 2 megabyte and 4 megabyte page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the Instruction Translation Lookaside Buffer (ITLB) and further levels of TLB.",
- "MetricExpr": " ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY ",
+ "MetricExpr": "ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "itlb_large_page_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.",
- "MetricExpr": " DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY ",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "dtlb_load_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.",
- "MetricExpr": " DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY ",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "dtlb_store_mpi",
"ScaleUnit": "1per_instr"
},
+ {
+ "BriefDescription": "Uncore operating frequency in GHz",
+ "MetricExpr": "( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) / 1000000000) / duration_time",
+ "MetricGroup": "",
+ "MetricName": "uncore_frequency",
+ "ScaleUnit": "1GHz"
+ },
{
"BriefDescription": "Intel(R) Quick Path Interconnect (QPI) data transmit bandwidth (MB/sec)",
- "MetricExpr": "( UNC_Q_TxL_FLITS_G0.DATA * 8 / 1000000) / duration_time",
+ "MetricExpr": "( UNC_Q_TxL_FLITS_G0.DATA * 8 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "qpi_data_transmit_bw_only_data",
+ "MetricName": "qpi_data_transmit_bw",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "DDR memory read bandwidth (MB/sec)",
- "MetricExpr": "( UNC_M_CAS_COUNT.RD * 64 / 1000000) / duration_time",
+ "MetricExpr": "( UNC_M_CAS_COUNT.RD * 64 / 1000000) / duration_time",
"MetricGroup": "",
"MetricName": "memory_bandwidth_read",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "DDR memory write bandwidth (MB/sec)",
- "MetricExpr": "( UNC_M_CAS_COUNT.WR * 64 / 1000000) / duration_time",
+ "MetricExpr": "( UNC_M_CAS_COUNT.WR * 64 / 1000000) / duration_time",
"MetricGroup": "",
"MetricName": "memory_bandwidth_write",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "DDR memory bandwidth (MB/sec)",
- "MetricExpr": "(( UNC_M_CAS_COUNT.RD + UNC_M_CAS_COUNT.WR ) * 64 / 1000000) / duration_time",
+ "MetricExpr": "(( UNC_M_CAS_COUNT.RD + UNC_M_CAS_COUNT.WR ) * 64 / 1000000) / duration_time",
"MetricGroup": "",
"MetricName": "memory_bandwidth_total",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.",
- "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x19e@ * 64 / 1000000) / duration_time",
+ "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x19e@ * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_read",
+ "MetricName": "io_bandwidth_disk_or_network_writes",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.",
- "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x1c8\\,filter_tid\\=0x3e@ * 64 / 1000000) / duration_time",
+ "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x1c8\\,filter_tid\\=0x3e@ * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_write",
+ "MetricName": "io_bandwidth_disk_or_network_reads",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue",
- "MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )",
+ "MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_frodecoded_icache_dsb",
+ "MetricName": "percent_uops_delivered_from_decoded_icache",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue",
- "MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )",
+ "MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_frolegacy_decode_pipeline_mite",
+ "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue",
- "MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )",
+ "MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_fromicrocode_sequencer_ms",
+ "MetricName": "percent_uops_delivered_from_microcode_sequencer",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from loop stream detector(LSD) as a percent of total uops delivered to Instruction Decode Queue",
- "MetricExpr": "100 * ( UOPS_ISSUED.ANY - IDQ.MITE_UOPS - IDQ.MS_UOPS - IDQ.DSB_UOPS ) / UOPS_ISSUED.ANY ",
+ "MetricExpr": "100 * ( UOPS_ISSUED.ANY - IDQ.MITE_UOPS - IDQ.MS_UOPS - IDQ.DSB_UOPS ) / UOPS_ISSUED.ANY",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_froloop_streadetector_lsd",
+ "MetricName": "percent_uops_delivered_from_loop_stream_detector",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Ratio of number of data read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions",
- "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x192@ ) / INST_RETIRED.ANY ",
+ "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x192@ ) / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "llc_data_read_mpi_demand_plus_prefetch",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions",
- "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x181@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x191@ ) / INST_RETIRED.ANY ",
+ "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x181@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x191@ ) / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "llc_code_read_mpi_demand_plus_prefetch",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
- "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )",
+ "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_local_dram",
+ "MetricName": "numa_reads_addressed_to_local_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
- "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )",
- "MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_remote_dram",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
- "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1, PGO",
- "MetricName": "tma_frontend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.",
- "MetricExpr": "100 * ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "Frontend, TmaL2",
- "MetricName": "tma_fetch_latency_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
- "MetricExpr": "100 * ( ICACHE.IFDATA_STALL / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot, FetchLat, IcMiss",
- "MetricName": "tma_icache_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.",
- "MetricExpr": "100 * ( ( 14 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot, FetchLat, MemoryTLB",
- "MetricName": "tma_itlb_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.",
- "MetricExpr": "100 * ( ( 12 ) * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat",
- "MetricName": "tma_branch_resteers_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
- "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "DSBmiss, FetchLat",
- "MetricName": "tma_dsb_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
- "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat",
- "MetricName": "tma_lcp_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.",
- "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat, MicroSeq",
- "MetricName": "tma_ms_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
- "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "FetchBW, Frontend, TmaL2",
- "MetricName": "tma_fetch_bandwidth_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
- "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSBmiss, FetchBW",
- "MetricName": "tma_mite_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
- "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSB, FetchBW",
- "MetricName": "tma_dsb_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
- "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_bad_speculation_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.",
- "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "BadSpec, BrMispredicts, TmaL2",
- "MetricName": "tma_branch_mispredicts_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.",
- "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "BadSpec, MachineClears, TmaL2",
- "MetricName": "tma_machine_clears_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
- "MetricExpr": "100 * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_backend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
- "MetricExpr": "100 * ( ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) + RESOURCE_STALLS.SB ) / ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) ) / 2 - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) if #SMT_on else ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) )",
- "MetricGroup": "Backend, TmaL2",
- "MetricName": "tma_memory_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.",
- "MetricExpr": "100 * ( max( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) - CYCLE_ACTIVITY.STALLS_L1D_PENDING ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )",
- "MetricGroup": "CacheMisses, MemoryBound, TmaL3mem",
- "MetricName": "tma_l1_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses, MemoryBound, TmaL3mem",
- "MetricName": "tma_l2_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) * CYCLE_ACTIVITY.STALLS_L2_PENDING / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses, MemoryBound, TmaL3mem",
- "MetricName": "tma_l3_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.",
- "MetricExpr": "100 * ( min( ( ( 1 - ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) ) * CYCLE_ACTIVITY.STALLS_L2_PENDING / ( CPU_CLK_UNHALTED.THREAD ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound, TmaL3mem",
- "MetricName": "tma_drabound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.",
- "MetricExpr": "100 * ( RESOURCE_STALLS.SB / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "MemoryBound, TmaL3mem",
- "MetricName": "tma_store_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
- "MetricExpr": "100 * ( ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) - ( ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) + RESOURCE_STALLS.SB ) / ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) ) / 2 - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) if #SMT_on else ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) ) )",
- "MetricGroup": "Backend, TmaL2, Compute",
- "MetricName": "tma_core_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.",
- "MetricExpr": "100 * ( 10 * ARITH.DIVIDER_UOPS / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) )",
+ "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )",
"MetricGroup": "",
- "MetricName": "tma_divider_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
- "MetricExpr": "100 * ( ( ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) ) / 2 - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) if #SMT_on else ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) - RESOURCE_STALLS.SB - ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "PortsUtil",
- "MetricName": "tma_ports_utilization_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ",
- "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_retiring_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "Retire, TmaL2",
- "MetricName": "tma_light_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "Retire, TmaL2",
- "MetricName": "tma_heavy_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "MicroSeq",
- "MetricName": "tma_microcode_sequencer_percent",
+ "MetricName": "numa_reads_addressed_to_remote_dram",
"ScaleUnit": "1%"
}
]
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data",
+ "BriefDescription": "Flits Transferred - Group 0; Data Tx Flits",
"Counter": "0,1,2,3",
- "EventName": "QPI_DATA_BANDWIDTH_TX",
+ "EventName": "UNC_Q_TxL_FLITS_G0.DATA",
"PerPkg": "1",
- "ScaleUnit": "8Bytes",
"UMask": "0x2",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of data flits transmitted ",
+ "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data",
"Counter": "0,1,2,3",
- "EventName": "UNC_Q_TxL_FLITS_G0.DATA",
+ "EventName": "QPI_DATA_BANDWIDTH_TX",
"PerPkg": "1",
"ScaleUnit": "8Bytes",
"UMask": "0x2",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data",
+ "BriefDescription": "Flits Transferred - Group 0; Non-Data protocol Tx Flits",
"Counter": "0,1,2,3",
- "EventName": "QPI_CTL_BANDWIDTH_TX",
+ "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA",
"PerPkg": "1",
- "ScaleUnit": "8Bytes",
"UMask": "0x4",
"Unit": "QPI LL"
},
{
- "BriefDescription": "Number of non data (control) flits transmitted ",
+ "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data",
"Counter": "0,1,2,3",
- "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA",
+ "EventName": "QPI_CTL_BANDWIDTH_TX",
"PerPkg": "1",
"ScaleUnit": "8Bytes",
"UMask": "0x4",
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
+ "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM Reads (RD_CAS + Underfills)",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_READ",
+ "EventName": "UNC_M_CAS_COUNT.RD",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller",
+ "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.RD",
+ "EventName": "LLC_MISSES.MEM_READ",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
+ "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM WR_CAS (both Modes)",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_WRITE",
+ "EventName": "UNC_M_CAS_COUNT.WR",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0xC",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller",
+ "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.WR",
+ "EventName": "LLC_MISSES.MEM_WRITE",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0xC",
"EventCode": "0x48",
"EventName": "L1D_PEND_MISS.FB_FULL",
"PEBScounters": "0,1,2,3",
- "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+ "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
"SampleAfterValue": "1000003",
"Speculative": "1",
"UMask": "0x2"
},
{
- "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability.",
+ "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.",
"CollectPEBSRecord": "2",
"Counter": "0,1,2,3",
"CounterMask": "1",
"EventCode": "0x48",
"EventName": "L1D_PEND_MISS.FB_FULL_PERIODS",
"PEBScounters": "0,1,2,3",
- "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+ "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
"SampleAfterValue": "1000003",
"Speculative": "1",
"UMask": "0x2"
[
+ {
+ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
+ "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "(5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "ICACHE_16B.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "10 * BACLEARS.ANY / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where (only) 4 uops were delivered by the MITE pipeline",
+ "MetricExpr": "(cpu@IDQ.MITE_UOPS\\,cmask\\=4@ - cpu@IDQ.MITE_UOPS\\,cmask\\=5@) / CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_mite_4wide",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit",
+ "MetricExpr": "(LSD.CYCLES_ACTIVE - LSD.CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "FetchBW;LSD;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_lsd",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
+ "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
+ "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + (5 * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=1\\,edge@) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + L1D_PEND_MISS.FB_FULL_PERIODS)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((29 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM + (23.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(23.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(9 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "L1D_PEND_MISS.L2_STALL / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(32.5 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores",
+ "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_streaming_stores",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions",
+ "MetricExpr": "140 * MISC_RETIRED.PAUSE_INST / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_slow_pause",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5 + UOPS_DISPATCHED.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_2_3 / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
+ "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_512b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.",
+ "MetricExpr": "tma_light_operations * BR_INST_RETIRED.ALL_BRANCHES / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_branch_instructions",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_branch_instructions + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer + tma_retiring * (UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=1@) / IDQ.MITE_UOPS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "((tma_retiring * SLOTS) / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * ASSISTS.ANY / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
+ "MetricGroup": "Mem;MemoryBW;Offcore",
+ "MetricName": "Memory_Bandwidth"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))",
+ "MetricGroup": "Mem;MemoryLat;Offcore",
+ "MetricName": "Memory_Latency"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ",
+ "MetricGroup": "Mem;MemoryTLB;Offcore",
+ "MetricName": "Memory_Data_TLBs"
+ },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
{
"BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (ICACHE_16B.IFDATA_STALL / CPU_CLK_UNHALTED.THREAD) + (10 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
"MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
"MetricName": "Big_Code"
},
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
+ "MetricGroup": "Fed;FetchBW;Frontend",
+ "MetricName": "Instruction_Fetch_BW"
+ },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
+ {
+ "BriefDescription": "Uops Per Instruction",
+ "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;Ret;Retire",
+ "MetricName": "UPI"
+ },
+ {
+ "BriefDescription": "Instruction per taken branch",
+ "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN",
+ "MetricGroup": "Branches;Fed;FetchBW",
+ "MetricName": "UpTB"
+ },
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
"MetricExpr": "TOPDOWN.SLOTS",
- "MetricGroup": "TmaL1",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
{
"BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor",
- "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1",
- "MetricGroup": "SMT;TmaL1",
+ "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1",
+ "MetricGroup": "SMT;tma_L1_group",
"MetricName": "Slots_Utilization"
},
{
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
+ {
+ "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
+ "MetricGroup": "Cor;SMT",
+ "MetricName": "Core_Bound_Likely"
+ },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
"MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED",
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX512",
"PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
+ {
+ "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.",
+ "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
+ "MetricGroup": "Pipeline;Ret",
+ "MetricName": "Retire"
+ },
{
"BriefDescription": "",
"MetricExpr": "UOPS_EXECUTED.THREAD / cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@",
"MetricGroup": "DSBmiss",
"MetricName": "DSB_Switch_Cost"
},
+ {
+ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_lsd + tma_mite))",
+ "MetricGroup": "DSBmiss;Fed",
+ "MetricName": "DSB_Misses"
+ },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
"MetricGroup": "Bad;BadSpec;BrMispredicts",
"MetricName": "IpMispredict"
},
+ {
+ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricGroup": "Bad;BrMispredicts",
+ "MetricName": "Branch_Misprediction_Cost"
+ },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
},
{
"BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)",
- "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )",
+ "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)",
"MetricGroup": "Bad;Branches",
"MetricName": "Other_Branches"
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( ( OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD ) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricExpr": "1000 * ((OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS) / Instructions",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License0_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes."
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License1_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions."
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License2_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions."
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
"UMask": "0x10"
},
{
- "BriefDescription": "number of branch instructions retired that were mispredicted and taken. Non PEBS",
+ "BriefDescription": "number of branch instructions retired that were mispredicted and taken.",
"CollectPEBSRecord": "2",
"Counter": "0,1,2,3,4,5,6,7",
"EventCode": "0xc5",
"EventCode": "0x48",
"EventName": "L1D_PEND_MISS.FB_FULL",
"PEBScounters": "0,1,2,3",
- "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+ "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
"SampleAfterValue": "1000003",
"Speculative": "1",
"UMask": "0x2"
},
{
- "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability.",
+ "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.",
"CollectPEBSRecord": "2",
"Counter": "0,1,2,3",
"CounterMask": "1",
"EventCode": "0x48",
"EventName": "L1D_PEND_MISS.FB_FULL_PERIODS",
"PEBScounters": "0,1,2,3",
- "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+ "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
"SampleAfterValue": "1000003",
"Speculative": "1",
"UMask": "0x2"
[
+ {
+ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
+ "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "(5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "ICACHE_16B.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "10 * BACLEARS.ANY / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where (only) 4 uops were delivered by the MITE pipeline",
+ "MetricExpr": "(cpu@IDQ.MITE_UOPS\\,cmask\\=4@ - cpu@IDQ.MITE_UOPS\\,cmask\\=5@) / CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_mite_4wide",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
+ "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
+ "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + (5 * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=1\\,edge@) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + L1D_PEND_MISS.FB_FULL_PERIODS)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (43.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(43.5 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(19 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "L1D_PEND_MISS.L2_STALL / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound) - tma_pmm_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "(43.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "(108 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "((97 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (97 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a",
+ "MetricExpr": "(((1 - ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) / ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) + (25 * (MEM_LOAD_RETIRED.LOCAL_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 33 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))))) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)) if (1000000 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM) > MEM_LOAD_RETIRED.L1_MISS) else 0)",
+ "MetricGroup": "MemoryBound;Server;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_pmm_bound",
+ "PublicDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(48 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores",
+ "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_streaming_stores",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions",
+ "MetricExpr": "37 * MISC_RETIRED.PAUSE_INST / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_slow_pause",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5 + UOPS_DISPATCHED.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_2_3 / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
+ "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_512b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.",
+ "MetricExpr": "tma_light_operations * BR_INST_RETIRED.ALL_BRANCHES / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_branch_instructions",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_branch_instructions + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer + tma_retiring * (UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=1@) / IDQ.MITE_UOPS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "((tma_retiring * SLOTS) / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * ASSISTS.ANY / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
+ "MetricGroup": "Mem;MemoryBW;Offcore",
+ "MetricName": "Memory_Bandwidth"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)))",
+ "MetricGroup": "Mem;MemoryLat;Offcore",
+ "MetricName": "Memory_Latency"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ",
+ "MetricGroup": "Mem;MemoryTLB;Offcore",
+ "MetricName": "Memory_Data_TLBs"
+ },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
{
"BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (ICACHE_16B.IFDATA_STALL / CPU_CLK_UNHALTED.THREAD) + (10 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
"MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
"MetricName": "Big_Code"
},
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
+ "MetricGroup": "Fed;FetchBW;Frontend",
+ "MetricName": "Instruction_Fetch_BW"
+ },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
+ {
+ "BriefDescription": "Uops Per Instruction",
+ "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;Ret;Retire",
+ "MetricName": "UPI"
+ },
+ {
+ "BriefDescription": "Instruction per taken branch",
+ "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN",
+ "MetricGroup": "Branches;Fed;FetchBW",
+ "MetricName": "UpTB"
+ },
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
+ "MetricName": "CPI"
+ },
{
"BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
"MetricExpr": "CPU_CLK_UNHALTED.THREAD",
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
"MetricExpr": "TOPDOWN.SLOTS",
- "MetricGroup": "TmaL1",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
{
"BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor",
- "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1",
- "MetricGroup": "SMT;TmaL1",
+ "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1",
+ "MetricGroup": "SMT;tma_L1_group",
"MetricName": "Slots_Utilization"
},
{
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
+ {
+ "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
+ "MetricGroup": "Cor;SMT",
+ "MetricName": "Core_Bound_Likely"
+ },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
"MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED",
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX512",
"PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
+ {
+ "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.",
+ "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
+ "MetricGroup": "Pipeline;Ret",
+ "MetricName": "Retire"
+ },
{
"BriefDescription": "",
"MetricExpr": "UOPS_EXECUTED.THREAD / cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@",
"MetricGroup": "DSBmiss",
"MetricName": "DSB_Switch_Cost"
},
+ {
+ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))",
+ "MetricGroup": "DSBmiss;Fed",
+ "MetricName": "DSB_Misses"
+ },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
"MetricGroup": "Bad;BadSpec;BrMispredicts",
"MetricName": "IpMispredict"
},
+ {
+ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricGroup": "Bad;BrMispredicts",
+ "MetricName": "Branch_Misprediction_Cost"
+ },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
},
{
"BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)",
- "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )",
+ "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)",
"MetricGroup": "Bad;Branches",
"MetricName": "Other_Branches"
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( ( OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD ) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricExpr": "1000 * ((OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS) / Instructions",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
},
{
"BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)",
- "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_Silent_PKI"
},
{
"BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction",
- "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_NonSilent_PKI"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License0_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes."
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License1_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions."
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License2_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions."
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
"BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( cha_0@event\\=0x0@ / duration_time )",
+ "MetricExpr": "1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD) / (Socket_CLKS / duration_time)",
"MetricGroup": "Mem;MemoryLat;SoC",
"MetricName": "MEM_Read_Latency"
},
},
{
"BriefDescription": "Average latency of data read request to external 3D X-Point memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / cha_0@event\\=0x0@ )",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": "(1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM) / cha_0@event\\=0x0@)",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_PMM_Read_Latency"
},
{
"BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": " 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / cha_0@event\\=0x0@",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": " 1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR) / cha_0@event\\=0x0@",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_DRAM_Read_Latency"
},
{
"BriefDescription": "Average 3DXP Memory Bandwidth Use for reads [GB / sec]",
- "MetricExpr": "( ( 64 * imc@event\\=0xe3@ / 1000000000 ) / duration_time )",
- "MetricGroup": "Mem;MemoryBW;SoC;Server",
+ "MetricExpr": "((64 * imc@event\\=0xe3@ / 1000000000) / duration_time)",
+ "MetricGroup": "Mem;MemoryBW;Server;SoC",
"MetricName": "PMM_Read_BW"
},
{
"BriefDescription": "Average 3DXP Memory Bandwidth Use for Writes [GB / sec]",
- "MetricExpr": "( ( 64 * imc@event\\=0xe7@ / 1000000000 ) / duration_time )",
- "MetricGroup": "Mem;MemoryBW;SoC;Server",
+ "MetricExpr": "((64 * imc@event\\=0xe7@ / 1000000000) / duration_time)",
+ "MetricGroup": "Mem;MemoryBW;Server;SoC",
"MetricName": "PMM_Write_BW"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]",
"MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Write_BW"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]",
- "MetricExpr": "( UNC_CHA_TOR_INSERTS.IO_HIT_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR ) * 64 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_HIT_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR) * 64 / 1000000000 / duration_time",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Read_BW"
},
{
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cha_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricName": "C6_Pkg_Residency"
},
{
- "BriefDescription": "Percentage of time spent in the active CPU power state C0",
- "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC",
- "MetricGroup": "",
- "MetricName": "cpu_utilization_percent",
- "ScaleUnit": "1%"
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
},
{
"BriefDescription": "CPU operating frequency (in GHz)",
"MetricName": "cpu_operating_frequency",
"ScaleUnit": "1GHz"
},
- {
- "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY",
- "MetricGroup": "",
- "MetricName": "cpi",
- "ScaleUnit": "1per_instr"
- },
{
"BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
"MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
"BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches",
+ "MetricName": "l1d_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches",
+ "MetricName": "l2_mpi",
"ScaleUnit": "1per_instr"
},
{
},
{
"BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions",
- "MetricExpr": "( UNC_CHA_TOR_INSERTS.IA_MISS_CRD ) / INST_RETIRED.ANY",
+ "MetricExpr": "( UNC_CHA_TOR_INSERTS.IA_MISS_CRD + UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF ) / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "llc_code_read_mpi_demand_plus_prefetch",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_latency",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to local memory in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_latency_for_local_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to remote memory in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_latency_for_remote_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to Intel(R) Optane(TM) Persistent Memory(PMEM) in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_to_pmem_latency",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to DRAM in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_to_dram_latency",
"ScaleUnit": "1ns"
"BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_local_dram",
+ "MetricName": "numa_reads_addressed_to_local_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_remote_dram",
+ "MetricName": "numa_reads_addressed_to_remote_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)",
"MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "upi_data_transmit_bw_only_data",
+ "MetricName": "upi_data_transmit_bw",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.",
"MetricExpr": "(( UNC_CHA_TOR_INSERTS.IO_HIT_PCIRDCUR + UNC_CHA_TOR_INSERTS.IO_MISS_PCIRDCUR ) * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_read",
+ "MetricName": "io_bandwidth_disk_or_network_writes",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.",
"MetricExpr": "(( UNC_CHA_TOR_INSERTS.IO_HIT_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR ) * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_write",
+ "MetricName": "io_bandwidth_disk_or_network_reads",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.DSB_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_decoded_icache_dsb",
+ "MetricName": "percent_uops_delivered_from_decoded_icache",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MITE_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite",
+ "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MS_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms",
+ "MetricName": "percent_uops_delivered_from_microcode_sequencer",
"ScaleUnit": "1%"
},
{
"ScaleUnit": "1MB/s"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
- "MetricExpr": "100 * ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) )",
- "MetricGroup": "TmaL1;PGO",
- "MetricName": "tma_frontend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.",
- "MetricExpr": "100 * ( ( ( 5 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / ( slots ) )",
- "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_latency_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
- "MetricExpr": "100 * ( ICACHE_16B.IFDATA_STALL / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_icache_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.",
- "MetricExpr": "100 * ( ICACHE_64B.IFTAG_STALL / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_itlb_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.",
- "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( ( 10 ) * BACLEARS.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_branch_resteers_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
- "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_dsb_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
- "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_lcp_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.",
- "MetricExpr": "100 * ( ( 3 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_ms_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
- "MetricExpr": "100 * ( max( 0 , ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) - ( ( ( 5 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / ( slots ) ) ) )",
- "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_bandwidth_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
- "MetricExpr": "100 * ( ( IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )",
- "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_mite_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
- "MetricExpr": "100 * ( ( IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )",
- "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_dsb_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
- "MetricExpr": "100 * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_bad_speculation_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.",
- "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) )",
- "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_branch_mispredicts_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.",
- "MetricExpr": "100 * ( max( 0 , ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) ) ) )",
- "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_machine_clears_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
- "MetricExpr": "100 * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_backend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
- "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) )",
- "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent",
- "MetricName": "tma_memory_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.",
- "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l1_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l2_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l3_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.",
- "MetricExpr": "100 * ( min( ( ( ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) ) ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_dram_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ",
- "MetricExpr": "100 * ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;Server;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_pmm_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.",
- "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_store_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
- "MetricExpr": "100 * ( max( 0 , ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent",
- "MetricName": "tma_core_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.",
- "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_divider_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ",
- "MetricExpr": "( 100 * ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_retiring_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.",
- "MetricExpr": "100 * ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_light_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
- "MetricExpr": "100 * ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) )",
- "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fp_arith_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_memory_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * BR_INST_RETIRED.ALL_BRANCHES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_branch_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_nop_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
- "MetricExpr": "100 * ( max( 0 , ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) - ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * BR_INST_RETIRED.ALL_BRANCHES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_other_light_ops_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
- "MetricExpr": "100 * ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_heavy_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
- "MetricExpr": "100 * ( ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) - ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) )",
- "MetricGroup": "TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_few_uops_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.",
- "MetricExpr": "100 * ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) )",
- "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_microcode_sequencer_percent",
+ "BriefDescription": "%",
+ "MetricExpr": "100 * ( ( LSD.CYCLES_ACTIVE - LSD.CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )",
+ "MetricGroup": "FetchBW;LSD;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
+ "MetricName": "tma_lsd",
"ScaleUnit": "1%"
}
]
"UMask": "0x10"
},
{
- "BriefDescription": "number of branch instructions retired that were mispredicted and taken. Non PEBS",
+ "BriefDescription": "number of branch instructions retired that were mispredicted and taken.",
"CollectPEBSRecord": "2",
"Counter": "0,1,2,3,4,5,6,7",
"EventCode": "0xc5",
"Unit": "M3UPI"
},
{
- "BriefDescription": "Flit Gen - Header 1 : Acumullate",
+ "BriefDescription": "Flit Gen - Header 1 : Accumulate",
"Counter": "0,1,2,3",
"CounterType": "PGMABLE",
"EventCode": "0x51",
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
+ "MetricExpr": "ICACHE.IFETCH_STALL / CLKS - tma_itlb_misses",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "13 * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS))) + 43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.LLC_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5) / (3 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "1 / ( ((FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD) + ((FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD) )",
+ "MetricExpr": "1 / (tma_fp_scalar + tma_fp_vector)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.LLC_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
"UMask": "0x2"
},
{
- "BriefDescription": "L1D miss oustandings duration in cycles",
+ "BriefDescription": "L1D miss outstanding duration in cycles",
"Counter": "2",
"CounterHTOff": "2",
"EventCode": "0x48",
"UMask": "0x8"
},
{
- "BriefDescription": "Cacheable and noncachaeble code read requests",
+ "BriefDescription": "Cacheable and noncacheable code read requests",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0xB0",
"UMask": "0x20"
},
{
- "BriefDescription": "Number of FP Computational Uops Executed this cycle. The number of FADD, FSUB, FCOM, FMULs, integer MULsand IMULs, FDIVs, FPREMs, FSQRTS, integer DIVs, and IDIVs. This event does not distinguish an FADD used in the middle of a transcendental flow from a s",
+ "BriefDescription": "Number of FP Computational Uops Executed this cycle. The number of FADD, FSUB, FCOM, FMULs, integer MULs and IMULs, FDIVs, FPREMs, FSQRTS, integer DIVs, and IDIVs. This event does not distinguish an FADD used in the middle of a transcendental flow from a s",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x10",
"UMask": "0x4"
},
{
- "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"EventCode": "0x79",
"EventName": "IDQ.MS_CYCLES",
- "PublicDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.",
+ "PublicDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.",
"SampleAfterValue": "2000003",
"UMask": "0x30"
},
{
- "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"EventCode": "0x79",
"EventName": "IDQ.MS_DSB_CYCLES",
- "PublicDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.",
+ "PublicDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.",
"SampleAfterValue": "2000003",
"UMask": "0x10"
},
{
- "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"CounterMask": "1",
"EdgeDetect": "1",
"EventCode": "0x79",
"EventName": "IDQ.MS_DSB_OCCUR",
- "PublicDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.",
+ "PublicDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.",
"SampleAfterValue": "2000003",
"UMask": "0x10"
},
{
- "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
"UMask": "0x10"
},
{
- "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
"UMask": "0x30"
},
{
- "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy",
+ "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
"Counter": "0,1,2,3",
"CounterHTOff": "0,1,2,3,4,5,6,7",
"EventCode": "0x79",
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
+ "MetricExpr": "ICACHE.IFETCH_STALL / CLKS - tma_itlb_misses",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "13 * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "(60 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) + 43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD)))) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "41 * (MEM_LOAD_UOPS_RETIRED.LLC_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "200 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "310 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "(200 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) + 180 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD)))) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(200 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_MISS.REMOTE_HITM + 60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5) / (3 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "1 / ( ((FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD) + ((FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD) )",
+ "MetricExpr": "1 / (tma_fp_scalar + tma_fp_vector)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.LLC_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricExpr": "(cstate_pkg@c7\\-residency@ / msr@tsc@) * 100",
"MetricGroup": "Power",
"MetricName": "C7_Pkg_Residency"
+ },
+ {
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
}
]
"EventCode": "0x34",
"EventName": "UNC_C_LLC_LOOKUP.WRITE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set filter mask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CBoGlCtrl[22:17] bits correspond to [M'FMESI] state.; Writeback transactions from L2 to the LLC This includes all write transactions -- both Cachable and UC.",
+ "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set filter mask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CBoGlCtrl[22:17] bits correspond to [M'FMESI] state.; Writeback transactions from L2 to the LLC This includes all write transactions -- both Cacheable and UC.",
"UMask": "0x5",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.ALL",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR. This includes requests that reside in the TOR for a short time, such as LLC Hits that do not need to snoop cores or requests that get rejected and have to be retried through one of the ingress queues. The TOR is more commonly a bottleneck in skews with smaller core counts, where the ratio of RTIDs to TOR entries is larger. Note that there are reserved TOR entries for various request types, so it is possible that a given request type be blocked with an occupancy that is less than 20. Also note that generally requests will not be able to arbitrate into the TOR pipeline if there are no available TOR slots.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR. This includes requests that reside in the TOR for a short time, such as LLC Hits that do not need to snoop cores or requests that get rejected and have to be retried through one of the ingress queues. The TOR is more commonly a bottleneck in skews with smaller core counts, where the ratio of RTIDs to TOR entries is larger. Note that there are reserved TOR entries for various request types, so it is possible that a given request type be blocked with an occupancy that is less than 20. Also note that generally requests will not be able to arbitrate into the TOR pipeline if there are no available TOR slots.",
"UMask": "0x8",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.EVICTION",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Eviction transactions inserted into the TOR. Evictions can be quick, such as when the line is in the F, S, or E states and no core valid bits are set. They can also be longer if either CV bits are set (so the cores need to be snooped) and/or if there is a HitM (in which case it is necessary to write the request out to memory).",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Eviction transactions inserted into the TOR. Evictions can be quick, such as when the line is in the F, S, or E states and no core valid bits are set. They can also be longer if either CV bits are set (so the cores need to be snooped) and/or if there is a HitM (in which case it is necessary to write the request out to memory).",
"UMask": "0x4",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.LOCAL",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisifed by locally HOMed memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisfied by locally HOMed memory.",
"UMask": "0x28",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.LOCAL_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisifed by an opcode, inserted into the TOR that are satisifed by locally HOMed memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisfied by an opcode, inserted into the TOR that are satisfied by locally HOMed memory.",
"UMask": "0x21",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.MISS_LOCAL",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisifed by locally HOMed memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisfied by locally HOMed memory.",
"UMask": "0x2A",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisifed by an opcode, inserted into the TOR that are satisifed by locally HOMed memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisfied by an opcode, inserted into the TOR that are satisfied by locally HOMed memory.",
"UMask": "0x23",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.MISS_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match an opcode.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match an opcode.",
"UMask": "0x3",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.MISS_REMOTE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisifed by remote caches or remote memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisfied by remote caches or remote memory.",
"UMask": "0x8A",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisifed by an opcode, inserted into the TOR that are satisifed by remote caches or remote memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisfied by an opcode, inserted into the TOR that are satisfied by remote caches or remote memory.",
"UMask": "0x83",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.NID_ALL",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched (matches an RTID destination) transactions inserted into the TOR. The NID is programmed in Cn_MSR_PMON_BOX_FILTER.nid. In conjunction with STATE = I, it is possible to monitor misses to specific NIDs in the system.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched (matches an RTID destination) transactions inserted into the TOR. The NID is programmed in Cn_MSR_PMON_BOX_FILTER.nid. In conjunction with STATE = I, it is possible to monitor misses to specific NIDs in the system.",
"UMask": "0x48",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.NID_EVICTION",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched eviction transactions inserted into the TOR.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched eviction transactions inserted into the TOR.",
"UMask": "0x44",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.NID_MISS_ALL",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched miss requests that were inserted into the TOR.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched miss requests that were inserted into the TOR.",
"UMask": "0x4A",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.NID_MISS_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match a NID and an opcode.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match a NID and an opcode.",
"UMask": "0x43",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.NID_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match a NID and an opcode.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match a NID and an opcode.",
"UMask": "0x41",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.NID_WB",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched write transactions inserted into the TOR.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched write transactions inserted into the TOR.",
"UMask": "0x50",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match an opcode (matched by Cn_MSR_PMON_BOX_FILTER.opc)",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match an opcode (matched by Cn_MSR_PMON_BOX_FILTER.opc)",
"UMask": "0x1",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.REMOTE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisifed by remote caches or remote memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisfied by remote caches or remote memory.",
"UMask": "0x88",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.REMOTE_OPCODE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisifed by an opcode, inserted into the TOR that are satisifed by remote caches or remote memory.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisfied by an opcode, inserted into the TOR that are satisfied by remote caches or remote memory.",
"UMask": "0x81",
"Unit": "CBO"
},
"EventCode": "0x35",
"EventName": "UNC_C_TOR_INSERTS.WB",
"PerPkg": "1",
- "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Write transactions inserted into the TOR. This does not include RFO, but actual operations that contain data being sent from the core.",
+ "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Write transactions inserted into the TOR. This does not include RFO, but actual operations that contain data being sent from the core.",
"UMask": "0x10",
"Unit": "CBO"
},
"EventCode": "0x36",
"EventName": "UNC_C_TOR_OCCUPANCY.LOCAL_OPCODE",
"PerPkg": "1",
- "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisifed by an opcode, in the TOR that are satisifed by locally HOMed memory.",
+ "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisfied by an opcode, in the TOR that are satisfied by locally HOMed memory.",
"UMask": "0x21",
"Unit": "CBO"
},
"EventCode": "0x36",
"EventName": "UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE",
"PerPkg": "1",
- "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisifed by an opcode, in the TOR that are satisifed by locally HOMed memory.",
+ "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisfied by an opcode, in the TOR that are satisfied by locally HOMed memory.",
"UMask": "0x23",
"Unit": "CBO"
},
"EventCode": "0x36",
"EventName": "UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE",
"PerPkg": "1",
- "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisifed by an opcode, in the TOR that are satisifed by remote caches or remote memory.",
+ "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisfied by an opcode, in the TOR that are satisfied by remote caches or remote memory.",
"UMask": "0x83",
"Unit": "CBO"
},
"EventCode": "0x36",
"EventName": "UNC_C_TOR_OCCUPANCY.REMOTE_OPCODE",
"PerPkg": "1",
- "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisifed by an opcode, in the TOR that are satisifed by remote caches or remote memory.",
+ "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisfied by an opcode, in the TOR that are satisfied by remote caches or remote memory.",
"UMask": "0x81",
"Unit": "CBO"
},
"EventCode": "0x2",
"EventName": "UNC_C_TxR_INSERTS.BL_CORE",
"PerPkg": "1",
- "PublicDescription": "Number of allocations into the Cbo Egress. The Egress is used to queue up requests destined for the ring.; Ring transactions from the Corebo destined for the BL ring. This is commonly used for transfering writeback data to the cache.",
+ "PublicDescription": "Number of allocations into the Cbo Egress. The Egress is used to queue up requests destined for the ring.; Ring transactions from the Corebo destined for the BL ring. This is commonly used for transferring writeback data to the cache.",
"UMask": "0x40",
"Unit": "CBO"
},
"EventCode": "0xb",
"EventName": "UNC_H_CONFLICT_CYCLES.LAST",
"PerPkg": "1",
- "PublicDescription": "Count every last conflictor in conflict chain. Can be used to compute the average conflict chain length as (#Ackcnflts/#LastConflictor)+1. This can be used to give a feel for the conflict chain lenghts while analyzing lock kernels.",
+ "PublicDescription": "Count every last conflictor in conflict chain. Can be used to compute the average conflict chain length as (#Ackcnflts/#LastConflictor)+1. This can be used to give a feel for the conflict chain lengths while analyzing lock kernels.",
"UMask": "0x4",
"Unit": "HA"
},
"EventCode": "0x41",
"EventName": "UNC_H_DIRECTORY_LAT_OPT",
"PerPkg": "1",
- "PublicDescription": "Directory Latency Optimization Data Return Path Taken. When directory mode is enabled and the directory retuned for a read is Dir=I, then data can be returned using a faster path if certain conditions are met (credits, free pipeline, etc).",
+ "PublicDescription": "Directory Latency Optimization Data Return Path Taken. When directory mode is enabled and the directory returned for a read is Dir=I, then data can be returned using a faster path if certain conditions are met (credits, free pipeline, etc).",
"Unit": "HA"
},
{
"EventCode": "0x21",
"EventName": "UNC_H_SNOOP_RESP.RSPSFWD",
"PerPkg": "1",
- "PublicDescription": "Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1.; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currentl copy. This is common for data and code reads that hit in a remote socket in E or F state.",
+ "PublicDescription": "Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1.; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currently copy. This is common for data and code reads that hit in a remote socket in E or F state.",
"UMask": "0x8",
"Unit": "HA"
},
"EventCode": "0x60",
"EventName": "UNC_H_SNP_RESP_RECV_LOCAL.RSPSFWD",
"PerPkg": "1",
- "PublicDescription": "Number of snoop responses received for a Local request; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currentl copy. This is common for data and code reads that hit in a remote socket in E or F state.",
+ "PublicDescription": "Number of snoop responses received for a Local request; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currently copy. This is common for data and code reads that hit in a remote socket in E or F state.",
"UMask": "0x8",
"Unit": "HA"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits. Had there been enough credits, the spawn would have worked as the RBT bit was set and the RBT tag matched.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits. Had there been enough credits, the spawn would have worked as the RBT bit was set and the RBT tag matched.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS_MISS",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and there weren't enough Egress credits. The valid bit was set.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and there weren't enough Egress credits. The valid bit was set.",
"UMask": "0x20",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS_RBT",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits AND the RBT bit was not set, but the RBT tag matched.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits AND the RBT bit was not set, but the RBT tag matched.",
"UMask": "0x8",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS_RBT_MISS",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match, the valid bit was not set and there weren't enough Egress credits.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match, the valid bit was not set and there weren't enough Egress credits.",
"UMask": "0x80",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_MISS",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match although the valid bit was set and there were enough Egress credits.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match although the valid bit was set and there were enough Egress credits.",
"UMask": "0x10",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_RBT_HIT",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the route-back table (RBT) specified that the transaction should not trigger a direct2core tranaction. This is common for IO transactions. There were enough Egress credits and the RBT tag matched but the valid bit was not set.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the route-back table (RBT) specified that the transaction should not trigger a direct2core transaction. This is common for IO transactions. There were enough Egress credits and the RBT tag matched but the valid bit was not set.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.FAILURE_RBT_MISS",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and the valid bit was not set although there were enough Egress credits.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and the valid bit was not set although there were enough Egress credits.",
"UMask": "0x40",
"Unit": "QPI LL"
},
"EventCode": "0x13",
"EventName": "UNC_Q_DIRECT2CORE.SUCCESS_RBT_HIT",
"PerPkg": "1",
- "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn was successful. There were sufficient credits, the RBT valid bit was set and there was an RBT tag match. The message was marked to spawn direct2core.",
+ "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn was successful. There were sufficient credits, the RBT valid bit was set and there was an RBT tag match. The message was marked to spawn direct2core.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventCode": "0x9",
"EventName": "UNC_Q_RxL_BYPASSED",
"PerPkg": "1",
- "PublicDescription": "Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transfered, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.",
+ "PublicDescription": "Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transferred, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.",
"Unit": "QPI LL"
},
{
"EventCode": "0x1",
"EventName": "UNC_Q_RxL_FLITS_G0.DATA",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flitsreceived over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flits received over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"EventCode": "0x1",
"EventName": "UNC_Q_RxL_FLITS_G0.IDLE",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of flits received over QPI that do not hold protocol payload. When QPI is not in a power saving state, it continuously transmits flits across the link. When there are no protocol flits to send, it will send IDLE and NULL flits across. These flits sometimes do carry a payload, such as credit returns, but are generall not considered part of the QPI bandwidth.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of flits received over QPI that do not hold protocol payload. When QPI is not in a power saving state, it continuously transmits flits across the link. When there are no protocol flits to send, it will send IDLE and NULL flits across. These flits sometimes do carry a payload, such as credit returns, but are generally not considered part of the QPI bandwidth.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventCode": "0x1",
"EventName": "UNC_Q_RxL_FLITS_G0.NON_DATA",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits received across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits received across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.DRS",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data.",
"UMask": "0x18",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.DRS_DATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).",
"UMask": "0x8",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.DRS_NONDATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.",
"UMask": "0x10",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.HOM",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits received over QPI on the home channel.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits received over QPI on the home channel.",
"UMask": "0x6",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.HOM_NONREQ",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits received over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits received over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.HOM_REQ",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request received over QPI on the home channel. This basically counts the number of remote memory requests received over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request received over QPI on the home channel. This basically counts the number of remote memory requests received over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G1.SNP",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits received over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are received on the home channel.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits received over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are received on the home channel.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G2.NCB",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.",
"UMask": "0xC",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G2.NCB_DATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not the NCB headers.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not the NCB headers.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G2.NCB_NONDATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.",
"UMask": "0x8",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G2.NCS",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits received over QPI. This includes extended headers.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits received over QPI. This includes extended headers.",
"UMask": "0x10",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G2.NDR_AD",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_RxL_FLITS_G2.NDR_AK",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.",
+ "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"Counter": "0,1,2,3",
"EventName": "UNC_Q_TxL_FLITS_G0.DATA",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flits transmitted over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flits transmitted over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"Counter": "0,1,2,3",
"EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits transmitted across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits transmitted across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.DRS",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency.",
"UMask": "0x18",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.DRS_DATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).",
"UMask": "0x8",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.DRS_NONDATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.",
"UMask": "0x10",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.HOM",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits transmitted over QPI on the home channel.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits transmitted over QPI on the home channel.",
"UMask": "0x6",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.HOM_NONREQ",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits transmitted over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits transmitted over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.HOM_REQ",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request transmitted over QPI on the home channel. This basically counts the number of remote memory requests transmitted over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request transmitted over QPI on the home channel. This basically counts the number of remote memory requests transmitted over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G1.SNP",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits transmitted over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are transmitted on the home channel.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits transmitted over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are transmitted on the home channel.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G2.NCB",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.",
"UMask": "0xC",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G2.NCB_DATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not te NCB headers.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not the NCB headers.",
"UMask": "0x4",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G2.NCB_NONDATA",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.",
"UMask": "0x8",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G2.NCS",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits transmitted over QPI. This includes extended headers.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits transmitted over QPI. This includes extended headers.",
"UMask": "0x10",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G2.NDR_AD",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxL_FLITS_G2.NDR_AK",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.",
+ "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxR_AD_SNP_CREDIT_OCCUPANCY.VN0",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO fro Snoop messages on AD.",
+ "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO for Snoop messages on AD.",
"UMask": "0x1",
"Unit": "QPI LL"
},
"EventName": "UNC_Q_TxR_AD_SNP_CREDIT_OCCUPANCY.VN1",
"ExtSel": "1",
"PerPkg": "1",
- "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO fro Snoop messages on AD.",
+ "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO for Snoop messages on AD.",
"UMask": "0x2",
"Unit": "QPI LL"
},
"EventCode": "0x9",
"EventName": "UNC_M_ECC_CORRECTABLE_ERRORS",
"PerPkg": "1",
- "PublicDescription": "Counts the number of ECC errors detected and corrected by the iMC on this channel. This counter is only useful with ECC DRAM devices. This count will increment one time for each correction regardless of the number of bits corrected. The iMC can correct up to 4 bit errors in independent channel mode and 8 bit erros in lockstep mode.",
+ "PublicDescription": "Counts the number of ECC errors detected and corrected by the iMC on this channel. This counter is only useful with ECC DRAM devices. This count will increment one time for each correction regardless of the number of bits corrected. The iMC can correct up to 4 bit errors in independent channel mode and 8 bit errors in lockstep mode.",
"Unit": "iMC"
},
{
"EventCode": "0x33",
"EventName": "UNC_R3_VNA_CREDITS_ACQUIRED",
"PerPkg": "1",
- "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credts from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transfered). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transfered in a given message class using an qfclk event.",
+ "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credits from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transferred). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transferred in a given message class using an qfclk event.",
"Unit": "R3QPI"
},
{
"EventCode": "0x33",
"EventName": "UNC_R3_VNA_CREDITS_ACQUIRED.AD",
"PerPkg": "1",
- "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credts from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transfered). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transfered in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.",
+ "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credits from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transferred). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transferred in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.",
"UMask": "0x1",
"Unit": "R3QPI"
},
"EventCode": "0x33",
"EventName": "UNC_R3_VNA_CREDITS_ACQUIRED.BL",
"PerPkg": "1",
- "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credts from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transfered). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transfered in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.",
+ "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credits from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transferred). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transferred in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.",
"UMask": "0x4",
"Unit": "R3QPI"
},
"EventCode": "0x80",
"EventName": "UNC_P_POWER_STATE_OCCUPANCY.CORES_C0",
"PerPkg": "1",
- "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with threshholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.",
+ "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.",
"Unit": "PCU"
},
{
"EventCode": "0x80",
"EventName": "UNC_P_POWER_STATE_OCCUPANCY.CORES_C3",
"PerPkg": "1",
- "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with threshholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.",
+ "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.",
"Unit": "PCU"
},
{
"EventCode": "0x80",
"EventName": "UNC_P_POWER_STATE_OCCUPANCY.CORES_C6",
"PerPkg": "1",
- "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with threshholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.",
+ "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.",
"Unit": "PCU"
},
{
"EventCode": "0x9",
"EventName": "UNC_P_PROCHOT_INTERNAL_CYCLES",
"PerPkg": "1",
- "PublicDescription": "Counts the number of cycles that we are in Interal PROCHOT mode. This mode is triggered when a sensor on the die determines that we are too hot and must throttle to avoid damaging the chip.",
+ "PublicDescription": "Counts the number of cycles that we are in Internal PROCHOT mode. This mode is triggered when a sensor on the die determines that we are too hot and must throttle to avoid damaging the chip.",
"Unit": "PCU"
},
{
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING)) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_DISPATCHED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_DISPATCHED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_DISPATCHED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_DISPATCHED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_DISPATCHED.THREAD / (( cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)",
+ "MetricExpr": "UOPS_DISPATCHED.THREAD / ((cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricExpr": "(cstate_pkg@c7\\-residency@ / msr@tsc@) * 100",
"MetricGroup": "Power",
"MetricName": "C7_Pkg_Residency"
+ },
+ {
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
}
]
Family-model,Version,Filename,EventType
-GenuineIntel-6-9[7A],v1.13,alderlake,core
+GenuineIntel-6-(97|9A|B7|BA|BE|BF),v1.15,alderlake,core
GenuineIntel-6-(1C|26|27|35|36),v4,bonnell,core
GenuineIntel-6-(3D|47),v26,broadwell,core
GenuineIntel-6-56,v23,broadwellde,core
GenuineIntel-6-4F,v19,broadwellx,core
GenuineIntel-6-55-[56789ABCDEF],v1.16,cascadelakex,core
-GenuineIntel-6-96,v1.03,elkhartlake,core
+GenuineIntel-6-9[6C],v1.03,elkhartlake,core
GenuineIntel-6-5[CF],v13,goldmont,core
GenuineIntel-6-7A,v1.01,goldmontplus,core
-GenuineIntel-6-(3C|45|46),v31,haswell,core
-GenuineIntel-6-3F,v25,haswellx,core
-GenuineIntel-6-(7D|7E|A7),v1.14,icelake,core
-GenuineIntel-6-6[AC],v1.15,icelakex,core
+GenuineIntel-6-(3C|45|46),v32,haswell,core
+GenuineIntel-6-3F,v26,haswellx,core
+GenuineIntel-6-(7D|7E|A7),v1.15,icelake,core
+GenuineIntel-6-6[AC],v1.16,icelakex,core
GenuineIntel-6-3A,v22,ivybridge,core
-GenuineIntel-6-3E,v21,ivytown,core
+GenuineIntel-6-3E,v22,ivytown,core
GenuineIntel-6-2D,v21,jaketown,core
GenuineIntel-6-(57|85),v9,knightslanding,core
GenuineIntel-6-AA,v1.00,meteorlake,core
GenuineIntel-6-1[AEF],v3,nehalemep,core
GenuineIntel-6-2E,v3,nehalemex,core
GenuineIntel-6-2A,v17,sandybridge,core
-GenuineIntel-6-8F,v1.04,sapphirerapids,core
-GenuineIntel-6-(37|4C|4D),v14,silvermont,core
+GenuineIntel-6-8F,v1.06,sapphirerapids,core
+GenuineIntel-6-(37|4A|4C|4D|5A),v14,silvermont,core
GenuineIntel-6-(4E|5E|8E|9E|A5|A6),v53,skylake,core
GenuineIntel-6-55-[01234],v1.28,skylakex,core
GenuineIntel-6-86,v1.20,snowridgex,core
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "RESOURCE_STALLS.SB / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING)) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_DISPATCHED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_DISPATCHED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_DISPATCHED.THREAD",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_DISPATCHED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_DISPATCHED.THREAD / (( cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)",
+ "MetricExpr": "UOPS_DISPATCHED.THREAD / ((cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
- "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )",
+ "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))",
"MetricGroup": "DSB;Fed;FetchBW",
"MetricName": "DSB_Coverage"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
"UMask": "0x2"
},
{
- "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability.",
+ "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.",
"CollectPEBSRecord": "2",
"Counter": "0,1,2,3",
"CounterMask": "1",
"EventCode": "0x48",
"EventName": "L1D_PEND_MISS.FB_FULL_PERIODS",
"PEBScounters": "0,1,2,3",
- "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+ "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
"SampleAfterValue": "1000003",
"Speculative": "1",
"UMask": "0x2"
"Speculative": "1",
"UMask": "0x1"
},
+ {
+ "BriefDescription": "Cycles the Microcode Sequencer is busy.",
+ "CollectPEBSRecord": "2",
+ "Counter": "0,1,2,3",
+ "EventCode": "0x87",
+ "EventName": "DECODE.MS_BUSY",
+ "PEBScounters": "0,1,2,3",
+ "SampleAfterValue": "500009",
+ "Speculative": "1",
+ "UMask": "0x2"
+ },
{
"BriefDescription": "DSB-to-MITE switch true penalty cycles.",
"CollectPEBSRecord": "2",
"EventCode": "0xc1",
"EventName": "ASSISTS.ANY",
"PEBScounters": "0,1,2,3,4,5,6,7",
- "PublicDescription": "Counts the number of occurrences where a microcode assist is invoked by hardware Examples include AD (page Access Dirty), FP and AVX related assists.",
+ "PublicDescription": "Counts the number of occurrences where a microcode assist is invoked by hardware. Examples include AD (page Access Dirty), FP and AVX related assists.",
"SampleAfterValue": "100003",
"Speculative": "1",
- "UMask": "0x1f"
+ "UMask": "0x1b"
},
{
"BriefDescription": "All branch instructions retired.",
[
+ {
+ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
+ "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "(topdown\\-fetch\\-lat / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS)",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "ICACHE_DATA.STALLS / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_TAG.STALLS / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(tma_branch_mispredicts / tma_bad_speculation) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (tma_branch_mispredicts / tma_bad_speculation)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "INT_MISC.UNKNOWN_BRANCH_CYCLES / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: FRONTEND_RETIRED.UNKNOWN_BRANCH",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "DECODE.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
+ "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "topdown\\-br\\-mispredict / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: TOPDOWN.BR_MISPREDICT_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
+ "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "topdown\\-mem\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((EXE_ACTIVITY.BOUND_ON_LOADS - MEMORY_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - MEMORY_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L1D_MISS - MEMORY_ACTIVITY.STALLS_L2_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L2_MISS - MEMORY_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((25 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (24 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(24 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(9 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "(XQ.FULL_CYCLES + L1D_PEND_MISS.L2_STALLS) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "((MEMORY_ACTIVITY.STALLS_L3_MISS / CLKS) - tma_pmm_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to memory bandwidth Allocation feature (RDT's memory bandwidth throttling).",
+ "MetricExpr": "INT_MISC.MBA_STALLS / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;Server;TopdownL5;tma_mem_bandwidth_group",
+ "MetricName": "tma_mba_stalls",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "(54.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "(119 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "((108 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (108 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a",
+ "MetricExpr": "(((1 - ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) / ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) + (25 * (MEM_LOAD_RETIRED.LOCAL_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 33 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))))) * (MEMORY_ACTIVITY.STALLS_L3_MISS / CLKS)) if (1000000 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM) > MEM_LOAD_RETIRED.L1_MISS) else 0)",
+ "MetricGroup": "MemoryBound;Server;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_pmm_bound",
+ "PublicDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((MEM_STORE_RETIRED.L2_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(28 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores",
+ "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_streaming_stores",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions",
+ "MetricExpr": "CPU_CLK_UNHALTED.PAUSE / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_slow_pause",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: CPU_CLK_UNHALTED.PAUSE_INST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to LFENCE Instructions.",
+ "MetricExpr": "13 * MISC2_RETIRED.LFENCE / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_memory_fence",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "160 * ASSISTS.SSE_AVX_MIX / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the Advanced Matrix Extensions (AMX) execution engine was busy with tile (arithmetic) operations",
+ "MetricExpr": "EXE.AMX_BUSY / CORE_CLKS",
+ "MetricGroup": "Compute;HPC;Server;TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_amx_busy",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5_11 + UOPS_DISPATCHED.PORT_6) / (5 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3_10",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_2_3_10 / (3 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
+ "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector + tma_fp_amx",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.128B_PACKED_HALF) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.256B_PACKED_HALF) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.512B_PACKED_HALF) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_512b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) matrix uops fraction the CPU has retired (aggregated across all supported FP datatypes in AMX engine)",
+ "MetricExpr": "cpu@AMX_OPS_RETIRED.BF16\\,cmask\\=1@ / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;HPC;Pipeline;Server;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_amx",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) matrix uops fraction the CPU has retired (aggregated across all supported FP datatypes in AMX engine). Refer to AMX_Busy and GFLOPs metrics for actual AMX utilization and FP performance, resp.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_int_vector_128b + tma_int_vector_256b + tma_shuffles",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_int_operations",
+ "PublicDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired). Vector/Matrix Int operations and shuffles are counted. Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents 128-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.",
+ "MetricExpr": "(INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_int_vector_128b",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents 256-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.",
+ "MetricExpr": "(INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_int_vector_256b",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic Integer (Int) matrix uops fraction the CPU has retired (aggregated across all supported Int datatypes in AMX engine)",
+ "MetricExpr": "cpu@AMX_OPS_RETIRED.INT8\\,cmask\\=1@ / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;HPC;IntVector;Pipeline;Server;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_int_amx",
+ "PublicDescription": "This metric approximates arithmetic Integer (Int) matrix uops fraction the CPU has retired (aggregated across all supported Int datatypes in AMX engine). Refer to AMX_Busy and TIOPs metrics for actual AMX utilization and Int performance, resp.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Shuffle (cross \"vector lane\" data transfers) uops fraction the CPU has retired.",
+ "MetricExpr": "INT_VEC_RETIRED.SHUFFLES / (tma_retiring * SLOTS)",
+ "MetricGroup": "HPC;Pipeline;TopdownL4;tma_int_operations_group",
+ "MetricName": "tma_shuffles",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_UOP_RETIRED.ANY / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.MACRO_FUSED / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fused_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused",
+ "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_non_fused_branches",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_int_operations + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "topdown\\-heavy\\-ops / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences. Sample with: UOPS_RETIRED.HEAVY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "UOPS_RETIRED.MS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: UOPS_RETIRED.MS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * cpu@ASSISTS.ANY\\,umask\\=0x1B@ / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults",
+ "MetricExpr": "99 * ASSISTS.PAGE_FAULT / SLOTS",
+ "MetricGroup": "TopdownL5;tma_assists_group",
+ "MetricName": "tma_page_faults",
+ "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults. A Page Fault may apply on first application access to a memory page. Note operating system handling of page faults accounts for the majority of its cost.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists",
+ "MetricExpr": "30 * ASSISTS.FP / SLOTS",
+ "MetricGroup": "HPC;TopdownL5;tma_assists_group",
+ "MetricName": "tma_fp_assists",
+ "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists. FP Assist may apply when working with very small floating point values (so-called denormals).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops as a result of handing SSE to AVX* or AVX* to SSE transition Assists. ",
+ "MetricExpr": "63 * ASSISTS.SSE_AVX_MIX / SLOTS",
+ "MetricGroup": "HPC;TopdownL5;tma_assists_group",
+ "MetricName": "tma_avx_assists",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources. Sample with: FRONTEND_RETIRED.MS_FLOWS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
+ "MetricGroup": "Mem;MemoryBW;Offcore",
+ "MetricName": "Memory_Bandwidth"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)))",
+ "MetricGroup": "Mem;MemoryLat;Offcore",
+ "MetricName": "Memory_Latency"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ",
+ "MetricGroup": "Mem;MemoryTLB;Offcore",
+ "MetricName": "Memory_Data_TLBs"
+ },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
+ "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
+ "MetricName": "Big_Code"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
+ "MetricGroup": "Fed;FetchBW;Frontend",
+ "MetricName": "Instruction_Fetch_BW"
+ },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
+ {
+ "BriefDescription": "Uops Per Instruction",
+ "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;Ret;Retire",
+ "MetricName": "UPI"
+ },
+ {
+ "BriefDescription": "Instruction per taken branch",
+ "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN",
+ "MetricGroup": "Branches;Fed;FetchBW",
+ "MetricName": "UpTB"
+ },
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
+ "MetricName": "CPI"
+ },
{
"BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
"MetricExpr": "CPU_CLK_UNHALTED.THREAD",
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
"MetricExpr": "TOPDOWN.SLOTS",
- "MetricGroup": "TmaL1",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
{
"BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor",
- "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1",
- "MetricGroup": "SMT;TmaL1",
+ "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1",
+ "MetricGroup": "SMT;tma_L1_group",
"MetricName": "Slots_Utilization"
},
{
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF ) + 2 * ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF ) + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * ( FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16 ) / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF) + 2 * (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF) + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * (FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5 ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
+ {
+ "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
+ "MetricGroup": "Cor;SMT",
+ "MetricName": "Core_Bound_Likely"
+ },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
"MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED",
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF ) + 2 * ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF ) + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * ( FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16 )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF) + 2 * (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF) + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * (FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.128B_PACKED_HALF )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.128B_PACKED_HALF)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.256B_PACKED_HALF )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.256B_PACKED_HALF)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.512B_PACKED_HALF )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.512B_PACKED_HALF)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX512",
"PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
{
"BriefDescription": "Instructions per Integer Arithmetic AMX operation (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / AMX_OPS_RETIRED.INT8",
- "MetricGroup": "IntVector;InsType;Server",
+ "MetricGroup": "InsType;IntVector;Server",
"MetricName": "IpArith_AMX_Int8",
"PublicDescription": "Instructions per Integer Arithmetic AMX operation (lower number means higher occurrence rate). Operations factored per matrices' sizes of the AMX instructions."
},
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
+ {
+ "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.",
+ "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
+ "MetricGroup": "Pipeline;Ret",
+ "MetricName": "Retire"
+ },
{
"BriefDescription": "Estimated fraction of retirement-cycles dealing with repeat instructions",
"MetricExpr": "INST_RETIRED.REP_ITERATION / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
"MetricGroup": "DSBmiss",
"MetricName": "DSB_Switch_Cost"
},
+ {
+ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))",
+ "MetricGroup": "DSBmiss;Fed",
+ "MetricName": "DSB_Misses"
+ },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
"MetricGroup": "Bad;BadSpec;BrMispredicts",
"MetricName": "IpMispredict"
},
+ {
+ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricGroup": "Bad;BrMispredicts",
+ "MetricName": "Branch_Misprediction_Cost"
+ },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
},
{
"BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)",
- "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )",
+ "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)",
"MetricGroup": "Bad;Branches",
"MetricName": "Other_Branches"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 4 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (4 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
},
{
"BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)",
- "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_Silent_PKI"
},
{
"BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction",
- "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_NonSilent_PKI"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF ) + 2 * ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF ) + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * ( FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16 ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF) + 2 * (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF) + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * (FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Tera Integer (matrix) Operations Per Second",
- "MetricExpr": "( 8 * AMX_OPS_RETIRED.INT8 / 1000000000000 ) / duration_time",
+ "MetricExpr": "(8 * AMX_OPS_RETIRED.INT8 / 1e12) / duration_time",
"MetricGroup": "Cor;HPC;IntVector;Server",
"MetricName": "TIOPS"
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
"BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( uncore_cha_0@event\\=0x1@ / duration_time )",
+ "MetricExpr": "1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD) / (Socket_CLKS / duration_time)",
"MetricGroup": "Mem;MemoryLat;SoC",
"MetricName": "MEM_Read_Latency"
},
},
{
"BriefDescription": "Average latency of data read request to external 3D X-Point memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / uncore_cha_0@event\\=0x1@ )",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": "(1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM) / uncore_cha_0@event\\=0x1@)",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_PMM_Read_Latency"
},
{
"BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": " 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / uncore_cha_0@event\\=0x1@",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": " 1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR) / uncore_cha_0@event\\=0x1@",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_DRAM_Read_Latency"
},
{
"BriefDescription": "Average 3DXP Memory Bandwidth Use for reads [GB / sec]",
- "MetricExpr": "( ( 64 * UNC_M_PMM_RPQ_INSERTS / 1000000000 ) / duration_time )",
- "MetricGroup": "Mem;MemoryBW;SoC;Server",
+ "MetricExpr": "((64 * UNC_M_PMM_RPQ_INSERTS / 1000000000) / duration_time)",
+ "MetricGroup": "Mem;MemoryBW;Server;SoC",
"MetricName": "PMM_Read_BW"
},
{
"BriefDescription": "Average 3DXP Memory Bandwidth Use for Writes [GB / sec]",
- "MetricExpr": "( ( 64 * UNC_M_PMM_WPQ_INSERTS / 1000000000 ) / duration_time )",
- "MetricGroup": "Mem;MemoryBW;SoC;Server",
+ "MetricExpr": "((64 * UNC_M_PMM_WPQ_INSERTS / 1000000000) / duration_time)",
+ "MetricGroup": "Mem;MemoryBW;Server;SoC",
"MetricName": "PMM_Write_BW"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]",
"MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Write_BW"
},
{
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "uncore_cha_0@event\\=0x1@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricName": "C6_Pkg_Residency"
},
{
- "BriefDescription": "Percentage of time spent in the active CPU power state C0",
- "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC",
- "MetricGroup": "",
- "MetricName": "cpu_utilization_percent",
- "ScaleUnit": "1%"
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
},
{
"BriefDescription": "CPU operating frequency (in GHz)",
"MetricName": "cpu_operating_frequency",
"ScaleUnit": "1GHz"
},
- {
- "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY",
- "MetricGroup": "",
- "MetricName": "cpi",
- "ScaleUnit": "1per_instr"
- },
{
"BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
"MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
"BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches",
+ "MetricName": "l1d_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches",
+ "MetricName": "l2_mpi",
"ScaleUnit": "1per_instr"
},
{
},
{
"BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions",
- "MetricExpr": "( UNC_CHA_TOR_INSERTS.IA_MISS_CRD ) / INST_RETIRED.ANY",
+ "MetricExpr": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD / INST_RETIRED.ANY",
"MetricGroup": "",
"MetricName": "llc_code_read_mpi_demand_plus_prefetch",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_latency",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to local memory in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_latency_for_local_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to remote memory in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_latency_for_remote_requests",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to Intel(R) Optane(TM) Persistent Memory(PMEM) in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_to_pmem_latency",
"ScaleUnit": "1ns"
},
{
"BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to DRAM in nano seconds",
- "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time )",
+ "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time",
"MetricGroup": "",
"MetricName": "llc_demand_data_read_miss_to_dram_latency",
"ScaleUnit": "1ns"
"BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_local_dram",
+ "MetricName": "numa_reads_addressed_to_local_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_remote_dram",
+ "MetricName": "numa_reads_addressed_to_remote_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)",
"MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "upi_data_transmit_bw_only_data",
+ "MetricName": "upi_data_transmit_bw",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.",
"MetricExpr": "( UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_read",
+ "MetricName": "io_bandwidth_disk_or_network_writes",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.",
"MetricExpr": "(( UNC_CHA_TOR_INSERTS.IO_ITOM + UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR ) * 64 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_write",
+ "MetricName": "io_bandwidth_disk_or_network_reads",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.DSB_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_decoded_icache_dsb",
+ "MetricName": "percent_uops_delivered_from_decoded_icache",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MITE_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite",
+ "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MS_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms",
+ "MetricName": "percent_uops_delivered_from_microcode_sequencer",
"ScaleUnit": "1%"
},
{
"MetricGroup": "",
"MetricName": "llc_miss_remote_memory_bandwidth_write",
"ScaleUnit": "1MB/s"
- },
- {
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
- "MetricExpr": "100 * ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) )",
- "MetricGroup": "TmaL1;PGO",
- "MetricName": "tma_frontend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.",
- "MetricExpr": "100 * ( ( topdown\\-fetch\\-lat / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) )",
- "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_latency_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
- "MetricExpr": "100 * ( ICACHE_DATA.STALLS / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_icache_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.",
- "MetricExpr": "100 * ( ICACHE_TAG.STALLS / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_itlb_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.",
- "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( INT_MISC.UNKNOWN_BRANCH_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_branch_resteers_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
- "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_dsb_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
- "MetricExpr": "100 * ( DECODE.LCP / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_lcp_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.",
- "MetricExpr": "100 * ( ( 3 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_ms_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
- "MetricExpr": "100 * ( max( 0 , ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) - ( ( topdown\\-fetch\\-lat / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) ) ) )",
- "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_bandwidth_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
- "MetricExpr": "100 * ( ( IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )",
- "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_mite_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
- "MetricExpr": "100 * ( ( IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )",
- "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_dsb_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
- "MetricExpr": "100 * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_bad_speculation_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.",
- "MetricExpr": "( 100 * ( topdown\\-br\\-mispredict / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )",
- "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_branch_mispredicts_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.",
- "MetricExpr": "100 * ( max( 0 , ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) - ( topdown\\-br\\-mispredict / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) )",
- "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_machine_clears_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
- "MetricExpr": "( 100 * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_backend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
- "MetricExpr": "( 100 * ( topdown\\-mem\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )",
- "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent",
- "MetricName": "tma_memory_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.",
- "MetricExpr": "100 * ( max( ( EXE_ACTIVITY.BOUND_ON_LOADS - MEMORY_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l1_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( MEMORY_ACTIVITY.STALLS_L1D_MISS - MEMORY_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l2_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( MEMORY_ACTIVITY.STALLS_L2_MISS - MEMORY_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l3_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.",
- "MetricExpr": "100 * ( min( ( ( ( MEMORY_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) - ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( MEMORY_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) ) ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_dram_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ",
- "MetricExpr": "100 * ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( MEMORY_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;Server;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_pmm_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.",
- "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_store_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
- "MetricExpr": "( 100 * ( max( 0 , ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-mem\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) ) + ( 0 * slots )",
- "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent",
- "MetricName": "tma_core_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.",
- "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_divider_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
- "MetricExpr": "( 100 * ( ( EXE_ACTIVITY.EXE_BOUND_0_PORTS + ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@ ) ) / ( CPU_CLK_UNHALTED.THREAD ) if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS ) ) else ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@ ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) + ( 0 * slots )",
- "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_ports_utilization_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ",
- "MetricExpr": "( 100 * ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_retiring_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.",
- "MetricExpr": "( 100 * ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) ) + ( 0 * slots )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_light_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
- "MetricExpr": "100 * ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) + ( cpu@AMX_OPS_RETIRED.BF16\\,cmask\\=0x1@ / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) )",
- "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fp_arith_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired). Vector/Matrix Int operations and shuffles are counted. Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain.",
- "MetricExpr": "100 * ( ( ( INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( INT_VEC_RETIRED.SHUFFLES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_int_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * MEM_UOP_RETIRED.ANY / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_memory_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.MACRO_FUSED / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fused_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_non_fused_branches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.",
- "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_nop_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
- "MetricExpr": "100 * ( max( 0 , ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) - ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) + ( cpu@AMX_OPS_RETIRED.BF16\\,cmask\\=0x1@ / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) + ( ( ( INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( INT_VEC_RETIRED.SHUFFLES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * MEM_UOP_RETIRED.ANY / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.MACRO_FUSED / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_other_light_ops_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
- "MetricExpr": "( 100 * ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_heavy_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
- "MetricExpr": "100 * ( ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( UOPS_RETIRED.MS / ( slots ) ) )",
- "MetricGroup": "TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_few_uops_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.",
- "MetricExpr": "100 * ( UOPS_RETIRED.MS / ( slots ) )",
- "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_microcode_sequencer_percent",
- "ScaleUnit": "1%"
}
]
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "(ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "9 * BACLEARS.ANY / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - tma_frontend_bound - (UOPS_ISSUED.ANY + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(12 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (9 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((18.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM + (16.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(16.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(6.5 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(22 * Average_Frequency) * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_NONE / 2 if #SMT_on else CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "PARTIAL_RAT_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: PARTIAL_RAT_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_1 - UOPS_EXECUTED.CORE_CYCLES_GE_2) / 2 if #SMT_on else EXE_ACTIVITY.1_PORTS_UTIL) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_2 - UOPS_EXECUTED.CORE_CYCLES_GE_3) / 2 if #SMT_on else EXE_ACTIVITY.2_PORTS_UTIL) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_GE_3 / 2 if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_3) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
- "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "Bad;BadSpec;BrMispredicts",
- "MetricName": "Mispredictions"
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions",
+ "MetricExpr": "tma_light_operations * UOPS_RETIRED.MACRO_FUSED / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fused_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused",
+ "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_non_fused_branches",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY) / SLOTS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * (FP_ASSIST.ANY + OTHER_ASSISTS.ANY) / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
- "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT",
- "MetricName": "Mispredictions_SMT"
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
},
{
"BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "Memory_Bandwidth"
},
- {
- "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
- "MetricGroup": "Mem;MemoryBW;Offcore_SMT",
- "MetricName": "Memory_Bandwidth_SMT"
- },
{
"BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (10 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))",
"MetricGroup": "Mem;MemoryLat;Offcore",
"MetricName": "Memory_Latency"
},
- {
- "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (10 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )",
- "MetricGroup": "Mem;MemoryLat;Offcore_SMT",
- "MetricName": "Memory_Latency_SMT"
- },
{
"BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency))) ",
"MetricGroup": "Mem;MemoryTLB;Offcore",
"MetricName": "Memory_Data_TLBs"
},
- {
- "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
- "MetricGroup": "Mem;MemoryTLB;Offcore_SMT",
- "MetricName": "Memory_Data_TLBs_SMT"
- },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
- {
- "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
- "MetricGroup": "Ret_SMT",
- "MetricName": "Branching_Overhead_SMT"
- },
{
"BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
"MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
"MetricName": "Big_Code"
},
- {
- "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
- "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT",
- "MetricName": "Big_Code_SMT"
- },
{
"BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
- "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
"MetricGroup": "Fed;FetchBW;Frontend",
"MetricName": "Instruction_Fetch_BW"
},
- {
- "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
- "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))",
- "MetricGroup": "Fed;FetchBW;Frontend_SMT",
- "MetricName": "Instruction_Fetch_BW_SMT"
- },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
},
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
- {
- "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Cor;Flops;HPC_SMT",
- "MetricName": "FP_Arith_Utilization_SMT",
- "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
- "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if 0 > 0.5 else 0",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
"MetricGroup": "Cor;SMT",
"MetricName": "Core_Bound_Likely"
},
- {
- "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
- "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if (1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 )) > 0.5 else 0",
- "MetricGroup": "Cor;SMT_SMT",
- "MetricName": "Core_Bound_Likely_SMT"
- },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
- "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) )",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))",
"MetricGroup": "DSBmiss;Fed",
"MetricName": "DSB_Misses"
},
- {
- "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
- "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) )",
- "MetricGroup": "DSBmiss;Fed_SMT",
- "MetricName": "DSB_Misses_SMT"
- },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
},
{
"BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;BrMispredicts",
"MetricName": "Branch_Misprediction_Cost"
},
- {
- "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
- "MetricGroup": "Bad;BrMispredicts_SMT",
- "MetricName": "Branch_Misprediction_Cost_SMT"
- },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are taken conditionals",
- "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches;CodeGen;PGO",
"MetricName": "Cond_TK"
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
{
"BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps",
- "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "Jump"
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
[
{
"BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Frontend_Bound",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound."
+ "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Frontend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "(ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@) / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "9 * BACLEARS.ANY / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "tma_frontend_bound - tma_fetch_latency",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Bad_Speculation",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example."
+ "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Bad_Speculation_SMT",
- "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
- "MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Backend_Bound",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound."
+ "MetricExpr": "1 - tma_frontend_bound - (UOPS_ISSUED.ANY + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(12 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (11 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (44 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (1 - (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(17 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory",
+ "MetricExpr": "(59.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_local_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
+ "MetricExpr": "(127 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_dram",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
+ "MetricExpr": "((89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group",
+ "MetricName": "tma_remote_cache",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 11 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "((110 * Average_Frequency) * (OFFCORE_RESPONSE.DEMAND_RFO.L3_MISS.REMOTE_HITM + OFFCORE_RESPONSE.PF_L2_RFO.L3_MISS.REMOTE_HITM) + (47.5 * Average_Frequency) * (OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.HITM_OTHER_CORE + OFFCORE_RESPONSE.PF_L2_RFO.L3_HIT.HITM_OTHER_CORE)) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "tma_backend_bound - tma_memory_bound",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_NONE / 2 if #SMT_on else CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "PARTIAL_RAT_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: PARTIAL_RAT_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Backend_Bound_SMT",
- "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_1 - UOPS_EXECUTED.CORE_CYCLES_GE_2) / 2 if #SMT_on else EXE_ACTIVITY.1_PORTS_UTIL) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_2 - UOPS_EXECUTED.CORE_CYCLES_GE_3) / 2 if #SMT_on else EXE_ACTIVITY.2_PORTS_UTIL) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).",
+ "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_GE_3 / 2 if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_3) / CORE_CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_2",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_load_op_utilization_group",
+ "MetricName": "tma_port_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_4",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7",
+ "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_store_op_utilization_group",
+ "MetricName": "tma_port_7",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "TopdownL1",
- "MetricName": "Retiring",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. "
+ "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))",
- "MetricGroup": "TopdownL1_SMT",
- "MetricName": "Retiring_SMT",
- "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU."
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "tma_retiring - tma_heavy_operations",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
},
{
- "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
- "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )",
- "MetricGroup": "Bad;BadSpec;BrMispredicts",
- "MetricName": "Mispredictions"
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_512b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions",
+ "MetricExpr": "tma_light_operations * UOPS_RETIRED.MACRO_FUSED / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fused_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused",
+ "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED) / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_non_fused_branches",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / UOPS_RETIRED.RETIRE_SLOTS",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY) / SLOTS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * (FP_ASSIST.ANY + OTHER_ASSISTS.ANY) / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
},
{
"BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
- "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )",
- "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT",
- "MetricName": "Mispredictions_SMT"
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
},
{
"BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "Memory_Bandwidth"
},
- {
- "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ",
- "MetricGroup": "Mem;MemoryBW;Offcore_SMT",
- "MetricName": "Memory_Bandwidth_SMT"
- },
{
"BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))",
"MetricGroup": "Mem;MemoryLat;Offcore",
"MetricName": "Memory_Latency"
},
- {
- "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )",
- "MetricGroup": "Mem;MemoryLat;Offcore_SMT",
- "MetricName": "Memory_Latency_SMT"
- },
{
"BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency))) ",
"MetricGroup": "Mem;MemoryTLB;Offcore",
"MetricName": "Memory_Data_TLBs"
},
- {
- "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
- "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ",
- "MetricGroup": "Mem;MemoryTLB;Offcore_SMT",
- "MetricName": "Memory_Data_TLBs_SMT"
- },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
- {
- "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
- "MetricGroup": "Ret_SMT",
- "MetricName": "Branching_Overhead_SMT"
- },
{
"BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
"MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
"MetricName": "Big_Code"
},
- {
- "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))",
- "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT",
- "MetricName": "Big_Code_SMT"
- },
{
"BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
- "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
"MetricGroup": "Fed;FetchBW;Frontend",
"MetricName": "Instruction_Fetch_BW"
},
- {
- "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
- "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))",
- "MetricGroup": "Fed;FetchBW;Frontend_SMT",
- "MetricName": "Instruction_Fetch_BW_SMT"
- },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
"MetricGroup": "Branches;Fed;FetchBW",
"MetricName": "UpTB"
},
+ {
+ "BriefDescription": "Cycles Per Instruction (per Logical Processor)",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
+ "MetricName": "CPI"
+ },
{
"BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
"MetricExpr": "CPU_CLK_UNHALTED.THREAD",
},
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "TmaL1",
+ "MetricExpr": "4 * CORE_CLKS",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
- {
- "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
- "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "TmaL1_SMT",
- "MetricName": "SLOTS_SMT"
- },
{
"BriefDescription": "The ratio of Executed- by Issued-Uops",
"MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY",
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
- {
- "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;SMT;TmaL1_SMT",
- "MetricName": "CoreIPC_SMT"
- },
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
- {
- "BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Ret;Flops_SMT",
- "MetricName": "FLOPc_SMT"
- },
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
- {
- "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Cor;Flops;HPC_SMT",
- "MetricName": "FP_Arith_Utilization_SMT",
- "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
{
"BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
- "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if 0 > 0.5 else 0",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
"MetricGroup": "Cor;SMT",
"MetricName": "Core_Bound_Likely"
},
- {
- "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
- "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if (1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 )) > 0.5 else 0",
- "MetricGroup": "Cor;SMT_SMT",
- "MetricName": "Core_Bound_Likely_SMT"
- },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
- "MetricExpr": "( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS",
"MetricGroup": "SMT",
"MetricName": "CORE_CLKS"
},
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX512",
"PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
{
},
{
"BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
- "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) )",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))",
"MetricGroup": "DSBmiss;Fed",
"MetricName": "DSB_Misses"
},
- {
- "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
- "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) )",
- "MetricGroup": "DSBmiss;Fed_SMT",
- "MetricName": "DSB_Misses_SMT"
- },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
},
{
"BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;BrMispredicts",
"MetricName": "Branch_Misprediction_Cost"
},
- {
- "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
- "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES",
- "MetricGroup": "Bad;BrMispredicts_SMT",
- "MetricName": "Branch_Misprediction_Cost_SMT"
- },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are taken conditionals",
- "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches;CodeGen;PGO",
"MetricName": "Cond_TK"
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
{
"BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps",
- "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "Jump"
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
- {
- "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )",
- "MetricGroup": "Mem;MemoryTLB_SMT",
- "MetricName": "Page_Walks_Utilization_SMT"
- },
{
"BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
"MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time",
},
{
"BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)",
- "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_Silent_PKI"
},
{
"BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction",
- "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY",
+ "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions",
"MetricGroup": "L2Evicts;Mem;Server",
"MetricName": "L2_Evictions_NonSilent_PKI"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License0_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes."
},
- {
- "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Power_SMT",
- "MetricName": "Power_License0_Utilization_SMT",
- "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes. SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License1_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions."
},
- {
- "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Power_SMT",
- "MetricName": "Power_License1_Utilization_SMT",
- "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License2_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions."
},
- {
- "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). SMT version; use when SMT is enabled and measuring per logical CPU.",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )",
- "MetricGroup": "Power_SMT",
- "MetricName": "Power_License2_Utilization_SMT",
- "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU."
- },
{
"BriefDescription": "Fraction of cycles where both hardware Logical Processors were active",
- "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0",
+ "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0",
"MetricGroup": "SMT",
"MetricName": "SMT_2T_Utilization"
},
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time",
+ "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
{
"BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches",
- "MetricExpr": "1000000000 * ( cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@ ) / ( cha_0@event\\=0x0@ / duration_time )",
+ "MetricExpr": "1000000000 * (cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@) / (Socket_CLKS / duration_time)",
"MetricGroup": "Mem;MemoryLat;SoC",
"MetricName": "MEM_Read_Latency"
},
},
{
"BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches",
- "MetricExpr": "1000000000 * ( UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS ) / imc_0@event\\=0x0@",
- "MetricGroup": "Mem;MemoryLat;SoC;Server",
+ "MetricExpr": "1000000000 * (UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS) / imc_0@event\\=0x0@",
+ "MetricGroup": "Mem;MemoryLat;Server;SoC",
"MetricName": "MEM_DRAM_Read_Latency"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]",
- "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3) * 4 / 1000000000 / duration_time",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Write_BW"
},
{
"BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]",
- "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3 ) * 4 / 1000000000 / duration_time",
- "MetricGroup": "IoBW;Mem;SoC;Server",
+ "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3) * 4 / 1000000000 / duration_time",
+ "MetricGroup": "IoBW;Mem;Server;SoC",
"MetricName": "IO_Read_BW"
},
{
"MetricGroup": "SoC",
"MetricName": "Socket_CLKS"
},
- {
- "BriefDescription": "Uncore frequency per die [GHZ]",
- "MetricExpr": "cha_0@event\\=0x0@ / #num_dies / duration_time / 1000000000",
- "MetricGroup": "SoC",
- "MetricName": "UNCORE_FREQ"
- },
{
"BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]",
"MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u",
"MetricName": "C7_Pkg_Residency"
},
{
- "BriefDescription": "Percentage of time spent in the active CPU power state C0",
- "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC",
- "MetricGroup": "",
- "MetricName": "cpu_utilization_percent",
- "ScaleUnit": "1%"
+ "BriefDescription": "Uncore frequency per die [GHZ]",
+ "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000",
+ "MetricGroup": "SoC",
+ "MetricName": "UNCORE_FREQ"
},
{
"BriefDescription": "CPU operating frequency (in GHz)",
"MetricName": "cpu_operating_frequency",
"ScaleUnit": "1GHz"
},
- {
- "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY",
- "MetricGroup": "",
- "MetricName": "cpi",
- "ScaleUnit": "1per_instr"
- },
{
"BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
"MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
"BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches",
+ "MetricName": "l1d_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions",
"MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches",
+ "MetricName": "l2_mpi",
"ScaleUnit": "1per_instr"
},
{
"MetricName": "llc_code_read_mpi_demand_plus_prefetch",
"ScaleUnit": "1per_instr"
},
+ {
+ "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds",
+ "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043300000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043300000000@ ) / ( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
+ "MetricGroup": "",
+ "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency",
+ "ScaleUnit": "1ns"
+ },
+ {
+ "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds",
+ "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043200000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ ) / ( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
+ "MetricGroup": "",
+ "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests",
+ "ScaleUnit": "1ns"
+ },
+ {
+ "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds",
+ "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043100000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ ) / ( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time",
+ "MetricGroup": "",
+ "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests",
+ "ScaleUnit": "1ns"
+ },
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB.",
"MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "itlb_2nd_level_mpi",
+ "MetricName": "itlb_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for 2 megabyte and 4 megabyte page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the Instruction Translation Lookaside Buffer (ITLB) and further levels of TLB.",
"MetricExpr": "ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "itlb_2nd_level_large_page_mpi",
+ "MetricName": "itlb_large_page_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.",
"MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "dtlb_2nd_level_load_mpi",
+ "MetricName": "dtlb_load_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for 2 megabyte page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the Data Translation Lookaside Buffer (DTLB) and further levels of TLB.",
"MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "dtlb_2nd_level_2mb_large_page_load_mpi",
+ "MetricName": "dtlb_2mb_large_page_load_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.",
"MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY",
"MetricGroup": "",
- "MetricName": "dtlb_2nd_level_store_mpi",
+ "MetricName": "dtlb_store_mpi",
"ScaleUnit": "1per_instr"
},
{
"BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_local_dram",
+ "MetricName": "numa_reads_addressed_to_local_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.",
"MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )",
"MetricGroup": "",
- "MetricName": "numa_percent_reads_addressed_to_remote_dram",
+ "MetricName": "numa_reads_addressed_to_remote_dram",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uncore operating frequency in GHz",
- "MetricExpr": "( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) / 1000000000) / duration_time",
+ "MetricExpr": "( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) / 1000000000) / duration_time",
"MetricGroup": "",
"MetricName": "uncore_frequency",
"ScaleUnit": "1GHz"
"BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)",
"MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "upi_data_transmit_bw_only_data",
+ "MetricName": "upi_data_transmit_bw",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.",
"MetricExpr": "(( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_read",
+ "MetricName": "io_bandwidth_disk_or_network_writes",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.",
"MetricExpr": "(( UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART0 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART1 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART2 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART3 ) * 4 / 1000000) / duration_time",
"MetricGroup": "",
- "MetricName": "io_bandwidth_write",
+ "MetricName": "io_bandwidth_disk_or_network_reads",
"ScaleUnit": "1MB/s"
},
{
"BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_decoded_icache_dsb",
+ "MetricName": "percent_uops_delivered_from_decoded_icache",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite",
+ "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline",
"ScaleUnit": "1%"
},
{
"BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue",
"MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )",
"MetricGroup": "",
- "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms",
+ "MetricName": "percent_uops_delivered_from_microcode_sequencer",
"ScaleUnit": "1%"
},
{
"MetricGroup": "",
"MetricName": "llc_miss_remote_memory_bandwidth_read",
"ScaleUnit": "1MB/s"
- },
- {
- "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.",
- "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1;PGO",
- "MetricName": "tma_frontend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.",
- "MetricExpr": "100 * ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_latency_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.",
- "MetricExpr": "100 * ( ( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_icache_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.",
- "MetricExpr": "100 * ( ICACHE_64B.IFTAG_STALL / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_itlb_misses_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.",
- "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( ( 9 ) * BACLEARS.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_branch_resteers_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.",
- "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_dsb_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
- "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_lcp_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.",
- "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent",
- "MetricName": "tma_ms_switches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.",
- "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent",
- "MetricName": "tma_fetch_bandwidth_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.",
- "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_mite_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
- "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )",
- "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent",
- "MetricName": "tma_dsb_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
- "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_bad_speculation_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.",
- "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_branch_mispredicts_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.",
- "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent",
- "MetricName": "tma_machine_clears_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.",
- "MetricExpr": "100 * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_backend_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
- "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent",
- "MetricName": "tma_memory_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.",
- "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l1_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l2_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.",
- "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_l3_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.",
- "MetricExpr": "100 * ( min( ( ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) , ( 1 ) ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_dram_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.",
- "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent",
- "MetricName": "tma_store_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
- "MetricExpr": "100 * ( ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )",
- "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent",
- "MetricName": "tma_core_bound_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.",
- "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_divider_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
- "MetricExpr": "100 * ( ( EXE_ACTIVITY.EXE_BOUND_0_PORTS + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) ) / ( CPU_CLK_UNHALTED.THREAD ) if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) / ( CPU_CLK_UNHALTED.THREAD ) )",
- "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent",
- "MetricName": "tma_ports_utilization_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ",
- "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "TmaL1",
- "MetricName": "tma_retiring_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_light_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) )",
- "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fp_arith_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_memory_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_fused_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_non_fused_branches_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_nop_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
- "MetricExpr": "100 * ( max( 0 , ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) - ( ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) ) ) ) )",
- "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent",
- "MetricName": "tma_other_light_ops_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent",
- "MetricName": "tma_heavy_operations_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
- "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )",
- "MetricGroup": "TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_few_uops_instructions_percent",
- "ScaleUnit": "1%"
- },
- {
- "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.",
- "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )",
- "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent",
- "MetricName": "tma_microcode_sequencer_percent",
- "ScaleUnit": "1%"
}
]
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
+ "BriefDescription": "All DRAM Read CAS Commands issued (including underfills)",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_READ",
+ "EventName": "UNC_M_CAS_COUNT.RD",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "read requests to memory controller",
+ "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.RD",
+ "EventName": "LLC_MISSES.MEM_READ",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0x3",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
+ "BriefDescription": "All DRAM Write CAS commands issued",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "LLC_MISSES.MEM_WRITE",
+ "EventName": "UNC_M_CAS_COUNT.WR",
"PerPkg": "1",
- "ScaleUnit": "64Bytes",
"UMask": "0xC",
"Unit": "iMC"
},
{
- "BriefDescription": "write requests to memory controller",
+ "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr",
"Counter": "0,1,2,3",
"EventCode": "0x4",
- "EventName": "UNC_M_CAS_COUNT.WR",
+ "EventName": "LLC_MISSES.MEM_WRITE",
"PerPkg": "1",
"ScaleUnit": "64Bytes",
"UMask": "0xC",
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x01",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x02",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x04",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x08",
- "ScaleUnit": "4Bytes",
"UMask": "0x01",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x01",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x02",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x04",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"FCMask": "0x07",
"PerPkg": "1",
"PortMask": "0x08",
- "ScaleUnit": "4Bytes",
"UMask": "0x04",
"Unit": "IIO"
},
"Unit": "UPI LL"
},
{
- "BriefDescription": "UPI interconnect send bandwidth for payload. Derived from unc_upi_txl_flits.all_data",
+ "BriefDescription": "Valid data FLITs transmitted via any slot",
"Counter": "0,1,2,3",
"EventCode": "0x2",
- "EventName": "UPI_DATA_BANDWIDTH_TX",
+ "EventName": "UNC_UPI_TxL_FLITS.ALL_DATA",
"PerPkg": "1",
- "ScaleUnit": "7.11E-06Bytes",
- "UMask": "0xf",
+ "UMask": "0x0F",
"Unit": "UPI LL"
},
{
- "BriefDescription": "UPI interconnect send bandwidth for payload",
+ "BriefDescription": "UPI interconnect send bandwidth for payload. Derived from unc_upi_txl_flits.all_data",
"Counter": "0,1,2,3",
"EventCode": "0x2",
- "EventName": "UNC_UPI_TxL_FLITS.ALL_DATA",
+ "EventName": "UPI_DATA_BANDWIDTH_TX",
"PerPkg": "1",
"ScaleUnit": "7.11E-06Bytes",
"UMask": "0xf",
[
+ {
+ "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend",
+ "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS",
+ "MetricGroup": "PGO;TopdownL1;tma_L1_group",
+ "MetricName": "tma_frontend_bound",
+ "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues",
+ "MetricExpr": "(5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING) / SLOTS",
+ "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_latency",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
+ "MetricExpr": "ICACHE_16B.IFDATA_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_icache_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
+ "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_itlb_misses",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers",
+ "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_branch_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_mispredicts_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears",
+ "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_clears_resteers",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
+ "MetricExpr": "10 * BACLEARS.ANY / CLKS",
+ "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group",
+ "MetricName": "tma_unknown_branches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines",
+ "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS",
+ "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_dsb_switches",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
+ "MetricExpr": "ILD_STALL.LCP / CLKS",
+ "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_lcp",
+ "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)",
+ "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS",
+ "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group",
+ "MetricName": "tma_ms_switches",
+ "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues",
+ "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)",
+ "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group",
+ "MetricName": "tma_fetch_bandwidth",
+ "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
+ "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_mite",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder",
+ "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_decoder0_alone",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where (only) 4 uops were delivered by the MITE pipeline",
+ "MetricExpr": "(cpu@IDQ.MITE_UOPS\\,cmask\\=4@ - cpu@IDQ.MITE_UOPS\\,cmask\\=5@) / CLKS",
+ "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group",
+ "MetricName": "tma_mite_4wide",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
+ "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_dsb",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit",
+ "MetricExpr": "(LSD.CYCLES_ACTIVE - LSD.CYCLES_OK) / CORE_CLKS / 2",
+ "MetricGroup": "FetchBW;LSD;TopdownL3;tma_fetch_bandwidth_group",
+ "MetricName": "tma_lsd",
+ "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations",
+ "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_bad_speculation",
+ "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction",
+ "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation",
+ "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_branch_mispredicts",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears",
+ "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)",
+ "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group",
+ "MetricName": "tma_machine_clears",
+ "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend",
+ "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + (5 * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=1\\,edge@) / SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_backend_bound",
+ "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck",
+ "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound",
+ "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_memory_bound",
+ "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache",
+ "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l1_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses",
+ "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_dtlb_load",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group",
+ "MetricName": "tma_load_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores",
+ "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_store_fwd_blk",
+ "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
+ "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS",
+ "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_lock_latency",
+ "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
+ "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_split_loads",
+ "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset",
+ "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS",
+ "MetricGroup": "TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_4k_aliasing",
+ "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed",
+ "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS",
+ "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group",
+ "MetricName": "tma_fb_full",
+ "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
+ "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + L1D_PEND_MISS.FB_FULL_PERIODS)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l2_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS",
+ "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_l3_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses",
+ "MetricExpr": "((49 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (48 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_contested_accesses",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+ "MetricExpr": "(48 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_data_sharing",
+ "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)",
+ "MetricExpr": "(17.5 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS",
+ "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_l3_hit_latency",
+ "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)",
+ "MetricExpr": "L1D_PEND_MISS.L2_STALL / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group",
+ "MetricName": "tma_sq_full",
+ "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+ "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_dram_bound",
+ "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_bandwidth",
+ "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)",
+ "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group",
+ "MetricName": "tma_mem_latency",
+ "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write",
+ "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS",
+ "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group",
+ "MetricName": "tma_store_bound",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses",
+ "MetricExpr": "((L2_RQSTS.RFO_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS",
+ "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_store_latency",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing",
+ "MetricExpr": "(54 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS",
+ "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_false_sharing",
+ "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents rate of split store accesses",
+ "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS",
+ "MetricGroup": "TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_split_stores",
+ "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores",
+ "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS",
+ "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_streaming_stores",
+ "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses",
+ "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group",
+ "MetricName": "tma_dtlb_store",
+ "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)",
+ "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_hit",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk",
+ "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS",
+ "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group",
+ "MetricName": "tma_store_stlb_miss",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck",
+ "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)",
+ "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group",
+ "MetricName": "tma_core_bound",
+ "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active",
+ "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS",
+ "MetricGroup": "TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_divider",
+ "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)",
+ "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group",
+ "MetricName": "tma_ports_utilization",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_0",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations",
+ "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_serializing_operation",
+ "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions",
+ "MetricExpr": "140 * MISC_RETIRED.PAUSE_INST / CLKS",
+ "MetricGroup": "TopdownL6;tma_serializing_operation_group",
+ "MetricName": "tma_slow_pause",
+ "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued",
+ "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_0_group",
+ "MetricName": "tma_mixing_vectors",
+ "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_1",
+ "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_2",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)",
+ "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS",
+ "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group",
+ "MetricName": "tma_ports_utilized_3m",
+ "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5 + UOPS_DISPATCHED.PORT_6) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_alu_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS",
+ "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_0",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_1",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_5 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_5",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS",
+ "MetricGroup": "TopdownL6;tma_alu_op_utilization_group",
+ "MetricName": "tma_port_6",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3",
+ "MetricExpr": "UOPS_DISPATCHED.PORT_2_3 / (2 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_load_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8",
+ "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)",
+ "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group",
+ "MetricName": "tma_store_op_utilization",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired",
+ "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS",
+ "MetricGroup": "TopdownL1;tma_L1_group",
+ "MetricName": "tma_retiring",
+ "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)",
+ "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_light_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)",
+ "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector",
+ "MetricGroup": "HPC;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_fp_arith",
+ "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric serves as an approximation of legacy x87 usage",
+ "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD",
+ "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_x87_use",
+ "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_scalar",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group",
+ "MetricName": "tma_fp_vector",
+ "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_128b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_256b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors",
+ "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)",
+ "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group",
+ "MetricName": "tma_fp_vector_512b",
+ "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.",
+ "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_memory_operations",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.",
+ "MetricExpr": "tma_light_operations * BR_INST_RETIRED.ALL_BRANCHES / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_branch_instructions",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions",
+ "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_nop_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting",
+ "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_branch_instructions + tma_nop_instructions))",
+ "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group",
+ "MetricName": "tma_other_light_ops",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences",
+ "MetricExpr": "tma_microcode_sequencer + tma_retiring * (UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=1@) / IDQ.MITE_UOPS",
+ "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group",
+ "MetricName": "tma_heavy_operations",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops",
+ "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer",
+ "MetricGroup": "TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_few_uops_instructions",
+ "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit",
+ "MetricExpr": "((tma_retiring * SLOTS) / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS",
+ "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group",
+ "MetricName": "tma_microcode_sequencer",
+ "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists",
+ "MetricExpr": "100 * ASSISTS.ANY / SLOTS",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_assists",
+ "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction",
+ "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)",
+ "MetricGroup": "TopdownL4;tma_microcode_sequencer_group",
+ "MetricName": "tma_cisc",
+ "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.",
+ "ScaleUnit": "100%"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks",
+ "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))",
+ "MetricGroup": "Bad;BadSpec;BrMispredicts",
+ "MetricName": "Mispredictions"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ",
+ "MetricGroup": "Mem;MemoryBW;Offcore",
+ "MetricName": "Memory_Bandwidth"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))",
+ "MetricGroup": "Mem;MemoryLat;Offcore",
+ "MetricName": "Memory_Latency"
+ },
+ {
+ "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
+ "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ",
+ "MetricGroup": "Mem;MemoryTLB;Offcore",
+ "MetricName": "Memory_Data_TLBs"
+ },
{
"BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)",
- "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)",
"MetricGroup": "Ret",
"MetricName": "Branching_Overhead"
},
{
"BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)",
- "MetricExpr": "100 * (( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (ICACHE_16B.IFDATA_STALL / CPU_CLK_UNHALTED.THREAD) + (10 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS)",
+ "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)",
"MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB",
"MetricName": "Big_Code"
},
+ {
+ "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks",
+ "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code",
+ "MetricGroup": "Fed;FetchBW;Frontend",
+ "MetricName": "Instruction_Fetch_BW"
+ },
{
"BriefDescription": "Instructions Per Cycle (per Logical Processor)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD",
+ "MetricExpr": "INST_RETIRED.ANY / CLKS",
"MetricGroup": "Ret;Summary",
"MetricName": "IPC"
},
+ {
+ "BriefDescription": "Uops Per Instruction",
+ "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY",
+ "MetricGroup": "Pipeline;Ret;Retire",
+ "MetricName": "UPI"
+ },
+ {
+ "BriefDescription": "Instruction per taken branch",
+ "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN",
+ "MetricGroup": "Branches;Fed;FetchBW",
+ "MetricName": "UpTB"
+ },
{
"BriefDescription": "Cycles Per Instruction (per Logical Processor)",
- "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)",
- "MetricGroup": "Pipeline;Mem",
+ "MetricExpr": "1 / IPC",
+ "MetricGroup": "Mem;Pipeline",
"MetricName": "CPI"
},
{
{
"BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)",
"MetricExpr": "TOPDOWN.SLOTS",
- "MetricGroup": "TmaL1",
+ "MetricGroup": "tma_L1_group",
"MetricName": "SLOTS"
},
{
"BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor",
- "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1",
- "MetricGroup": "SMT;TmaL1",
+ "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1",
+ "MetricGroup": "SMT;tma_L1_group",
"MetricName": "Slots_Utilization"
},
{
},
{
"BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)",
- "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;SMT;TmaL1",
+ "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS",
+ "MetricGroup": "Ret;SMT;tma_L1_group",
"MetricName": "CoreIPC"
},
{
"BriefDescription": "Floating Point Operations Per Cycle",
- "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED",
- "MetricGroup": "Ret;Flops",
+ "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS",
+ "MetricGroup": "Flops;Ret",
"MetricName": "FLOPc"
},
{
"BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)",
- "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "FP_Arith_Utilization",
"PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)."
},
{
"BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core",
- "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
+ "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)",
"MetricGroup": "Backend;Cor;Pipeline;PortsUtil",
"MetricName": "ILP"
},
+ {
+ "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts",
+ "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0",
+ "MetricGroup": "Cor;SMT",
+ "MetricName": "Core_Bound_Likely"
+ },
{
"BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core",
"MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED",
},
{
"BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;InsType",
"MetricName": "IpFLOP"
},
{
"BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )",
+ "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))",
"MetricGroup": "Flops;InsType",
"MetricName": "IpArith",
"PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX128",
"PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX256",
"PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
},
{
"BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)",
- "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )",
+ "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
"MetricGroup": "Flops;FpVector;InsType",
"MetricName": "IpArith_AVX512",
"PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting."
"MetricName": "IpSWPF"
},
{
- "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST",
+ "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST",
"MetricExpr": "INST_RETIRED.ANY",
- "MetricGroup": "Summary;TmaL1",
+ "MetricGroup": "Summary;tma_L1_group",
"MetricName": "Instructions"
},
+ {
+ "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.",
+ "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
+ "MetricGroup": "Pipeline;Ret",
+ "MetricName": "Retire"
+ },
{
"BriefDescription": "",
"MetricExpr": "UOPS_EXECUTED.THREAD / cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@",
"MetricGroup": "DSBmiss",
"MetricName": "DSB_Switch_Cost"
},
+ {
+ "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.",
+ "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_lsd + tma_mite))",
+ "MetricGroup": "DSBmiss;Fed",
+ "MetricName": "DSB_Misses"
+ },
{
"BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)",
"MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS",
"MetricGroup": "Bad;BadSpec;BrMispredicts",
"MetricName": "IpMispredict"
},
+ {
+ "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)",
+ "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES",
+ "MetricGroup": "Bad;BrMispredicts",
+ "MetricName": "Branch_Misprediction_Cost"
+ },
{
"BriefDescription": "Fraction of branches that are non-taken conditionals",
"MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES",
},
{
"BriefDescription": "Fraction of branches that are CALL or RET",
- "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES",
+ "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES",
"MetricGroup": "Bad;Branches",
"MetricName": "CallRet"
},
},
{
"BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)",
- "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )",
+ "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)",
"MetricGroup": "Bad;Branches",
"MetricName": "Other_Branches"
},
{
"BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)",
- "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )",
+ "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)",
"MetricGroup": "Mem;MemoryBound;MemoryLat",
"MetricName": "Load_Miss_Real_Latency"
},
{
"BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)",
"MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES",
- "MetricGroup": "Mem;MemoryBound;MemoryBW",
+ "MetricGroup": "Mem;MemoryBW;MemoryBound",
"MetricName": "MLP"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI"
},
{
"BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L1MPKI_Load"
},
{
"BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;Backend;CacheMisses",
+ "MetricGroup": "Backend;CacheMisses;Mem",
"MetricName": "L2MPKI"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses;Offcore",
+ "MetricGroup": "CacheMisses;Mem;Offcore",
"MetricName": "L2MPKI_All"
},
{
"BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2MPKI_Load"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)",
- "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_All"
},
{
"BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)",
"MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L2HPKI_Load"
},
{
"BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "L3MPKI"
},
{
"BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
"MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY",
- "MetricGroup": "Mem;CacheMisses",
+ "MetricGroup": "CacheMisses;Mem",
"MetricName": "FB_HPKI"
},
{
"BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses",
"MetricConstraint": "NO_NMI_WATCHDOG",
- "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )",
+ "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (2 * CORE_CLKS)",
"MetricGroup": "Mem;MemoryTLB",
"MetricName": "Page_Walks_Utilization"
},
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]",
- "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)",
+ "MetricExpr": "L1D_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L1D_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]",
- "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)",
+ "MetricExpr": "L2_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L2_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Fill_BW",
"MetricGroup": "Mem;MemoryBW",
"MetricName": "L3_Cache_Fill_BW_1T"
},
{
"BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]",
- "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)",
+ "MetricExpr": "L3_Cache_Access_BW",
"MetricGroup": "Mem;MemoryBW;Offcore",
"MetricName": "L3_Cache_Access_BW_1T"
},
},
{
"BriefDescription": "Measured Average Frequency for unhalted processors [GHz]",
- "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time",
- "MetricGroup": "Summary;Power",
+ "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time",
+ "MetricGroup": "Power;Summary",
"MetricName": "Average_Frequency"
},
{
"BriefDescription": "Giga Floating Point Operations Per Second",
- "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time",
+ "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time",
"MetricGroup": "Cor;Flops;HPC",
"MetricName": "GFLOPs",
"PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine."
},
{
"BriefDescription": "Average Frequency Utilization relative nominal frequency",
- "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC",
+ "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC",
"MetricGroup": "Power",
"MetricName": "Turbo_Utilization"
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0",
- "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License0_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes."
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1",
- "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License1_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions."
},
{
"BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)",
- "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED",
+ "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS",
"MetricGroup": "Power",
"MetricName": "Power_License2_Utilization",
"PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions."
},
{
"BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
- "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000",
+ "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000",
"MetricGroup": "HPC;Mem;MemoryBW;SoC",
"MetricName": "DRAM_BW_Use"
},
.metric_name = "DCache_L2_All_Miss",
},
{
- .metric_expr = "dcache_l2_all_hits + dcache_l2_all_miss",
+ .metric_expr = "DCache_L2_All_Hits + DCache_L2_All_Miss",
.metric_name = "DCache_L2_All",
},
{
- .metric_expr = "d_ratio(dcache_l2_all_hits, dcache_l2_all)",
+ .metric_expr = "d_ratio(DCache_L2_All_Hits, DCache_L2_All)",
.metric_name = "DCache_L2_Hits",
},
{
- .metric_expr = "d_ratio(dcache_l2_all_miss, dcache_l2_all)",
+ .metric_expr = "d_ratio(DCache_L2_All_Miss, DCache_L2_All)",
.metric_name = "DCache_L2_Misses",
},
{
config=0
sample_period=*
sample_type=263
-read_format=0|4
+read_format=0|4|20
disabled=1
inherit=1
pinned=0
config=9
sample_period=4000
sample_type=455
-read_format=4
+read_format=4|20
# Event will be enabled right away.
disabled=0
inherit=1
fd=1
group_fd=-1
sample_type=327
-read_format=4
+read_format=4|20
[event-2:base-record]
fd=2
group_fd=1
config=1
sample_type=327
-read_format=4
+read_format=4|20
mmap=0
comm=0
task=0
fd=1
group_fd=-1
sample_type=343
-read_format=12
+read_format=12|28
inherit=0
[event-2:base-record]
# default | PERF_SAMPLE_READ
sample_type=343
-# PERF_FORMAT_ID | PERF_FORMAT_GROUP
-read_format=12
+# PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST
+read_format=12|28
task=0
mmap=0
comm=0
fd=1
group_fd=-1
sample_type=327
-read_format=4
+read_format=4|20
[event-2:base-record]
fd=2
type=0
config=1
sample_type=327
-read_format=4
+read_format=4|20
mmap=0
comm=0
task=0
config=0|1
sample_period=1234000
sample_type=87
-read_format=12
+read_format=12|28
inherit=0
freq=0
config=0|1
sample_period=6789000
sample_type=87
-read_format=12
+read_format=12|28
disabled=0
inherit=0
mmap=0
struct perf_record_cpu_map *map_event = &event->cpu_map;
struct perf_record_cpu_map_data *data;
struct perf_cpu_map *map;
- int i;
unsigned int long_size;
data = &map_event->data;
TEST_ASSERT_VAL("wrong nr", data->mask32_data.nr == 1);
- for (i = 0; i < 20; i++) {
+ TEST_ASSERT_VAL("wrong cpu", perf_record_cpu_map_data__test_bit(0, data));
+ TEST_ASSERT_VAL("wrong cpu", !perf_record_cpu_map_data__test_bit(1, data));
+ for (int i = 2; i <= 20; i++)
TEST_ASSERT_VAL("wrong cpu", perf_record_cpu_map_data__test_bit(i, data));
- }
map = cpu_map__new_data(data);
TEST_ASSERT_VAL("wrong nr", perf_cpu_map__nr(map) == 20);
- for (i = 0; i < 20; i++) {
- TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, i).cpu == i);
- }
+ TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, 0).cpu == 0);
+ for (int i = 2; i <= 20; i++)
+ TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, i - 1).cpu == i);
perf_cpu_map__put(map);
return 0;
return 0;
}
+static int process_event_range_cpus(struct perf_tool *tool __maybe_unused,
+ union perf_event *event,
+ struct perf_sample *sample __maybe_unused,
+ struct machine *machine __maybe_unused)
+{
+ struct perf_record_cpu_map *map_event = &event->cpu_map;
+ struct perf_record_cpu_map_data *data;
+ struct perf_cpu_map *map;
+
+ data = &map_event->data;
+
+ TEST_ASSERT_VAL("wrong type", data->type == PERF_CPU_MAP__RANGE_CPUS);
+
+ TEST_ASSERT_VAL("wrong any_cpu", data->range_cpu_data.any_cpu == 0);
+ TEST_ASSERT_VAL("wrong start_cpu", data->range_cpu_data.start_cpu == 1);
+ TEST_ASSERT_VAL("wrong end_cpu", data->range_cpu_data.end_cpu == 256);
+
+ map = cpu_map__new_data(data);
+ TEST_ASSERT_VAL("wrong nr", perf_cpu_map__nr(map) == 256);
+ TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, 0).cpu == 1);
+ TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__max(map).cpu == 256);
+ TEST_ASSERT_VAL("wrong refcnt", refcount_read(&map->refcnt) == 1);
+ perf_cpu_map__put(map);
+ return 0;
+}
+
static int test__cpu_map_synthesize(struct test_suite *test __maybe_unused, int subtest __maybe_unused)
{
struct perf_cpu_map *cpus;
- /* This one is better stores in mask. */
- cpus = perf_cpu_map__new("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19");
+ /* This one is better stored in a mask. */
+ cpus = perf_cpu_map__new("0,2-20");
TEST_ASSERT_VAL("failed to synthesize map",
!perf_event__synthesize_cpu_map(NULL, cpus, process_event_mask, NULL));
perf_cpu_map__put(cpus);
- /* This one is better stores in cpu values. */
+ /* This one is better stored in cpu values. */
cpus = perf_cpu_map__new("1,256");
TEST_ASSERT_VAL("failed to synthesize map",
!perf_event__synthesize_cpu_map(NULL, cpus, process_event_cpus, NULL));
+ perf_cpu_map__put(cpus);
+
+ /* This one is better stored as a range. */
+ cpus = perf_cpu_map__new("1-256");
+
+ TEST_ASSERT_VAL("failed to synthesize map",
+ !perf_event__synthesize_cpu_map(NULL, cpus, process_event_range_cpus, NULL));
+
perf_cpu_map__put(cpus);
return 0;
}
TEST_ASSERT_VAL("wrong id", ev->id == 123);
TEST_ASSERT_VAL("wrong id", ev->type == PERF_EVENT_UPDATE__UNIT);
- TEST_ASSERT_VAL("wrong unit", !strcmp(ev->data, "KRAVA"));
+ TEST_ASSERT_VAL("wrong unit", !strcmp(ev->unit, "KRAVA"));
return 0;
}
struct machine *machine __maybe_unused)
{
struct perf_record_event_update *ev = (struct perf_record_event_update *)event;
- struct perf_record_event_update_scale *ev_data;
-
- ev_data = (struct perf_record_event_update_scale *)ev->data;
TEST_ASSERT_VAL("wrong id", ev->id == 123);
TEST_ASSERT_VAL("wrong id", ev->type == PERF_EVENT_UPDATE__SCALE);
- TEST_ASSERT_VAL("wrong scale", ev_data->scale == 0.123);
+ TEST_ASSERT_VAL("wrong scale", ev->scale.scale == 0.123);
return 0;
}
TEST_ASSERT_VAL("wrong id", ev->id == 123);
TEST_ASSERT_VAL("wrong id", ev->type == PERF_EVENT_UPDATE__NAME);
- TEST_ASSERT_VAL("wrong name", !strcmp(ev->data, tmp->name));
+ TEST_ASSERT_VAL("wrong name", !strcmp(ev->name, tmp->name));
return 0;
}
struct machine *machine __maybe_unused)
{
struct perf_record_event_update *ev = (struct perf_record_event_update *)event;
- struct perf_record_event_update_cpus *ev_data;
struct perf_cpu_map *map;
- ev_data = (struct perf_record_event_update_cpus *) ev->data;
-
- map = cpu_map__new_data(&ev_data->cpus);
+ map = cpu_map__new_data(&ev->cpus.cpus);
TEST_ASSERT_VAL("wrong id", ev->id == 123);
TEST_ASSERT_VAL("wrong type", ev->type == PERF_EVENT_UPDATE__CPUS);
// SPDX-License-Identifier: GPL-2.0
+#include "util/cputopo.h"
#include "util/debug.h"
#include "util/expr.h"
#include "util/header.h"
ret |= test(ctx, "min(1,2) + 1", 2);
ret |= test(ctx, "max(1,2) + 1", 3);
ret |= test(ctx, "1+1 if 3*4 else 0", 2);
+ ret |= test(ctx, "100 if 1 else 200 if 1 else 300", 100);
+ ret |= test(ctx, "100 if 0 else 200 if 1 else 300", 200);
+ ret |= test(ctx, "100 if 1 else 200 if 0 else 300", 100);
+ ret |= test(ctx, "100 if 0 else 200 if 0 else 300", 300);
ret |= test(ctx, "1.1 + 2.1", 3.2);
ret |= test(ctx, ".1 + 2.", 2.1);
ret |= test(ctx, "d_ratio(1, 2)", 0.5);
(void **)&val_ptr));
expr__ctx_clear(ctx);
- ctx->runtime = 3;
+ ctx->sctx.runtime = 3;
TEST_ASSERT_VAL("find ids",
expr__find_ids("EVENT1\\,param\\=?@ + EVENT2\\,param\\=?@",
NULL, ctx) == 0);
(void **)&val_ptr));
/* Only EVENT1 or EVENT2 need be measured depending on the value of smt_on. */
- expr__ctx_clear(ctx);
- TEST_ASSERT_VAL("find ids",
- expr__find_ids("EVENT1 if #smt_on else EVENT2",
- NULL, ctx) == 0);
- TEST_ASSERT_VAL("find ids", hashmap__size(ctx->ids) == 1);
- TEST_ASSERT_VAL("find ids", hashmap__find(ctx->ids,
- smt_on() ? "EVENT1" : "EVENT2",
- (void **)&val_ptr));
+ {
+ struct cpu_topology *topology = cpu_topology__new();
+ bool smton = smt_on(topology);
+ bool corewide = core_wide(/*system_wide=*/false,
+ /*user_requested_cpus=*/false,
+ topology);
+
+ cpu_topology__delete(topology);
+ expr__ctx_clear(ctx);
+ TEST_ASSERT_VAL("find ids",
+ expr__find_ids("EVENT1 if #smt_on else EVENT2",
+ NULL, ctx) == 0);
+ TEST_ASSERT_VAL("find ids", hashmap__size(ctx->ids) == 1);
+ TEST_ASSERT_VAL("find ids", hashmap__find(ctx->ids,
+ smton ? "EVENT1" : "EVENT2",
+ (void **)&val_ptr));
+
+ expr__ctx_clear(ctx);
+ TEST_ASSERT_VAL("find ids",
+ expr__find_ids("EVENT1 if #core_wide else EVENT2",
+ NULL, ctx) == 0);
+ TEST_ASSERT_VAL("find ids", hashmap__size(ctx->ids) == 1);
+ TEST_ASSERT_VAL("find ids", hashmap__find(ctx->ids,
+ corewide ? "EVENT1" : "EVENT2",
+ (void **)&val_ptr));
+ }
/* The expression is a constant 1.0 without needing to evaluate EVENT1. */
expr__ctx_clear(ctx);
TEST_ASSERT_VAL("find ids",
// SPDX-License-Identifier: GPL-2.0
#include <errno.h>
#include <inttypes.h>
-/* For the CLR_() macros */
-#include <pthread.h>
#include <stdlib.h>
#include <perf/cpumap.h>
#include <errno.h>
#include <inttypes.h>
/* For the CPU_* macros */
-#include <pthread.h>
+#include <sched.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <inttypes.h>
#include <linux/string.h>
-/* For the CLR_() macros */
-#include <pthread.h>
#include <sched.h>
#include <perf/mmap.h>
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+include ../../../../../tools/scripts/Makefile.include
+include ../../../../../tools/scripts/Makefile.arch
+include ../../../../../tools/scripts/utilities.mak
+
+SUBDIRS = \
+ asm_pure_loop \
+ memcpy_thread \
+ thread_loop \
+ unroll_loop_thread
+
+all: $(SUBDIRS)
+$(SUBDIRS):
+ @$(MAKE) -C $@ >/dev/null
+
+INSTALLDIRS = $(SUBDIRS:%=install-%)
+
+install-tests: $(INSTALLDIRS)
+$(INSTALLDIRS):
+ @$(MAKE) -C $(@:install-%=%) install-tests >/dev/null
+
+CLEANDIRS = $(SUBDIRS:%=clean-%)
+
+clean: $(CLEANDIRS)
+$(CLEANDIRS):
+ $(call QUIET_CLEAN, test-$(@:clean-%=%)) $(Q)$(MAKE) -C $(@:clean-%=%) clean >/dev/null
+
+.PHONY: all clean $(SUBDIRS) $(CLEANDIRS) $(INSTALLDIRS)
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+ifndef DESTDIR
+prefix ?= $(HOME)
+endif
+
+DESTDIR_SQ = $(subst ','\'',$(DESTDIR))
+INSTALL = install
+INSTDIR_SUB = tests/shell/coresight
+
+include ../../../../../scripts/Makefile.include
+include ../../../../../scripts/Makefile.arch
+include ../../../../../scripts/utilities.mak
--- /dev/null
+#!/bin/sh -e
+# CoreSight / ASM Pure Loop
+
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+TEST="asm_pure_loop"
+. $(dirname $0)/../lib/coresight.sh
+ARGS=""
+DATV="out"
+DATA="$DATD/perf-$TEST-$DATV.data"
+
+perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS
+
+perf_dump_aux_verify "$DATA" 10 10 10
+
+err=$?
+exit $err
--- /dev/null
+asm_pure_loop
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+include ../Makefile.miniconfig
+
+# Binary to produce
+BIN=asm_pure_loop
+# Any linking/libraries needed for the binary - empty if none needed
+LIB=
+
+all: $(BIN)
+
+$(BIN): $(BIN).S
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Build line - this is raw asm with no libc to have an always exact binary
+ $(Q)$(CC) $(BIN).S -nostdlib -static -o $(BIN) $(LIB)
+endif
+endif
+
+install-tests: all
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Install the test tool in the right place
+ $(call QUIET_INSTALL, tests) \
+ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \
+ $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)'
+endif
+endif
+
+clean:
+ $(Q)$(RM) -f $(BIN)
+
+.PHONY: all clean install-tests
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Tamas Zsoldos <tamas.zsoldos@arm.com>, 2021 */
+
+.globl _start
+_start:
+ mov x0, 0x0000ffff
+ mov x1, xzr
+loop:
+ nop
+ nop
+ cbnz x1, noskip
+ nop
+ nop
+ adrp x2, skip
+ add x2, x2, :lo12:skip
+ br x2
+ nop
+ nop
+noskip:
+ nop
+ nop
+skip:
+ sub x0, x0, 1
+ cbnz x0, loop
+
+ mov x0, #0
+ mov x8, #93 // __NR_exit syscall
+ svc #0
--- /dev/null
+memcpy_thread
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+include ../Makefile.miniconfig
+
+# Binary to produce
+BIN=memcpy_thread
+# Any linking/libraries needed for the binary - empty if none needed
+LIB=-pthread
+
+all: $(BIN)
+
+$(BIN): $(BIN).c
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Build line
+ $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB)
+endif
+endif
+
+install-tests: all
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Install the test tool in the right place
+ $(call QUIET_INSTALL, tests) \
+ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \
+ $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)'
+endif
+endif
+
+clean:
+ $(Q)$(RM) -f $(BIN)
+
+.PHONY: all clean install-tests
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+
+struct args {
+ unsigned long loops;
+ unsigned long size;
+ pthread_t th;
+ void *ret;
+};
+
+static void *thrfn(void *arg)
+{
+ struct args *a = arg;
+ unsigned long i, len = a->loops;
+ unsigned char *src, *dst;
+
+ src = malloc(a->size * 1024);
+ dst = malloc(a->size * 1024);
+ if ((!src) || (!dst)) {
+ printf("ERR: Can't allocate memory\n");
+ exit(1);
+ }
+ for (i = 0; i < len; i++)
+ memcpy(dst, src, a->size * 1024);
+}
+
+static pthread_t new_thr(void *(*fn) (void *arg), void *arg)
+{
+ pthread_t t;
+ pthread_attr_t attr;
+
+ pthread_attr_init(&attr);
+ pthread_create(&t, &attr, fn, arg);
+ return t;
+}
+
+int main(int argc, char **argv)
+{
+ unsigned long i, len, size, thr;
+ pthread_t threads[256];
+ struct args args[256];
+ long long v;
+
+ if (argc < 4) {
+ printf("ERR: %s [copysize Kb] [numthreads] [numloops (hundreds)]\n", argv[0]);
+ exit(1);
+ }
+
+ v = atoll(argv[1]);
+ if ((v < 1) || (v > (1024 * 1024))) {
+ printf("ERR: max memory 1GB (1048576 KB)\n");
+ exit(1);
+ }
+ size = v;
+ thr = atol(argv[2]);
+ if ((thr < 1) || (thr > 256)) {
+ printf("ERR: threads 1-256\n");
+ exit(1);
+ }
+ v = atoll(argv[3]);
+ if ((v < 1) || (v > 40000000000ll)) {
+ printf("ERR: loops 1-40000000000 (hundreds)\n");
+ exit(1);
+ }
+ len = v * 100;
+ for (i = 0; i < thr; i++) {
+ args[i].loops = len;
+ args[i].size = size;
+ args[i].th = new_thr(thrfn, &(args[i]));
+ }
+ for (i = 0; i < thr; i++)
+ pthread_join(args[i].th, &(args[i].ret));
+ return 0;
+}
--- /dev/null
+#!/bin/sh -e
+# CoreSight / Memcpy 16k 10 Threads
+
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+TEST="memcpy_thread"
+. $(dirname $0)/../lib/coresight.sh
+ARGS="16 10 1"
+DATV="16k_10"
+DATA="$DATD/perf-$TEST-$DATV.data"
+
+perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS
+
+perf_dump_aux_verify "$DATA" 10 10 10
+
+err=$?
+exit $err
--- /dev/null
+thread_loop
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+include ../Makefile.miniconfig
+
+# Binary to produce
+BIN=thread_loop
+# Any linking/libraries needed for the binary - empty if none needed
+LIB=-pthread
+
+all: $(BIN)
+
+$(BIN): $(BIN).c
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Build line
+ $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB)
+endif
+endif
+
+install-tests: all
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Install the test tool in the right place
+ $(call QUIET_INSTALL, tests) \
+ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \
+ $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)'
+endif
+endif
+
+clean:
+ $(Q)$(RM) -f $(BIN)
+
+.PHONY: all clean install-tests
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+// define this for gettid()
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <sys/syscall.h>
+#ifndef SYS_gettid
+// gettid is 178 on arm64
+# define SYS_gettid 178
+#endif
+#define gettid() syscall(SYS_gettid)
+
+struct args {
+ unsigned int loops;
+ pthread_t th;
+ void *ret;
+};
+
+static void *thrfn(void *arg)
+{
+ struct args *a = arg;
+ int i = 0, len = a->loops;
+
+ if (getenv("SHOW_TID")) {
+ unsigned long long tid = gettid();
+
+ printf("%llu\n", tid);
+ }
+ asm volatile(
+ "loop:\n"
+ "add %[i], %[i], #1\n"
+ "cmp %[i], %[len]\n"
+ "blt loop\n"
+ : /* out */
+ : /* in */ [i] "r" (i), [len] "r" (len)
+ : /* clobber */
+ );
+ return (void *)(long)i;
+}
+
+static pthread_t new_thr(void *(*fn) (void *arg), void *arg)
+{
+ pthread_t t;
+ pthread_attr_t attr;
+
+ pthread_attr_init(&attr);
+ pthread_create(&t, &attr, fn, arg);
+ return t;
+}
+
+int main(int argc, char **argv)
+{
+ unsigned int i, len, thr;
+ pthread_t threads[256];
+ struct args args[256];
+
+ if (argc < 3) {
+ printf("ERR: %s [numthreads] [numloops (millions)]\n", argv[0]);
+ exit(1);
+ }
+
+ thr = atoi(argv[1]);
+ if ((thr < 1) || (thr > 256)) {
+ printf("ERR: threads 1-256\n");
+ exit(1);
+ }
+ len = atoi(argv[2]);
+ if ((len < 1) || (len > 4000)) {
+ printf("ERR: max loops 4000 (millions)\n");
+ exit(1);
+ }
+ len *= 1000000;
+ for (i = 0; i < thr; i++) {
+ args[i].loops = len;
+ args[i].th = new_thr(thrfn, &(args[i]));
+ }
+ for (i = 0; i < thr; i++)
+ pthread_join(args[i].th, &(args[i].ret));
+ return 0;
+}
--- /dev/null
+#!/bin/sh -e
+# CoreSight / Thread Loop 10 Threads - Check TID
+
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+TEST="thread_loop"
+. $(dirname $0)/../lib/coresight.sh
+ARGS="10 1"
+DATV="check-tid-10th"
+DATA="$DATD/perf-$TEST-$DATV.data"
+STDO="$DATD/perf-$TEST-$DATV.stdout"
+
+SHOW_TID=1 perf record -s $PERFRECOPT -o "$DATA" "$BIN" $ARGS > $STDO
+
+perf_dump_aux_tid_verify "$DATA" "$STDO"
+
+err=$?
+exit $err
--- /dev/null
+#!/bin/sh -e
+# CoreSight / Thread Loop 2 Threads - Check TID
+
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+TEST="thread_loop"
+. $(dirname $0)/../lib/coresight.sh
+ARGS="2 20"
+DATV="check-tid-2th"
+DATA="$DATD/perf-$TEST-$DATV.data"
+STDO="$DATD/perf-$TEST-$DATV.stdout"
+
+SHOW_TID=1 perf record -s $PERFRECOPT -o "$DATA" "$BIN" $ARGS > $STDO
+
+perf_dump_aux_tid_verify "$DATA" "$STDO"
+
+err=$?
+exit $err
--- /dev/null
+unroll_loop_thread
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+include ../Makefile.miniconfig
+
+# Binary to produce
+BIN=unroll_loop_thread
+# Any linking/libraries needed for the binary - empty if none needed
+LIB=-pthread
+
+all: $(BIN)
+
+$(BIN): $(BIN).c
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Build line
+ $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB)
+endif
+endif
+
+install-tests: all
+ifdef CORESIGHT
+ifeq ($(ARCH),arm64)
+# Install the test tool in the right place
+ $(call QUIET_INSTALL, tests) \
+ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \
+ $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)'
+endif
+endif
+
+clean:
+ $(Q)$(RM) -f $(BIN)
+
+.PHONY: all clean install-tests
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+
+struct args {
+ pthread_t th;
+ unsigned int in;
+ void *ret;
+};
+
+static void *thrfn(void *arg)
+{
+ struct args *a = arg;
+ unsigned int i, in = a->in;
+
+ for (i = 0; i < 10000; i++) {
+ asm volatile (
+// force an unroll of thia add instruction so we can test long runs of code
+#define SNIP1 "add %[in], %[in], #1\n"
+// 10
+#define SNIP2 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1
+// 100
+#define SNIP3 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2
+// 1000
+#define SNIP4 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3
+// 10000
+#define SNIP5 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4
+// 100000
+ SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5
+ : /* out */
+ : /* in */ [in] "r" (in)
+ : /* clobber */
+ );
+ }
+}
+
+static pthread_t new_thr(void *(*fn) (void *arg), void *arg)
+{
+ pthread_t t;
+ pthread_attr_t attr;
+
+ pthread_attr_init(&attr);
+ pthread_create(&t, &attr, fn, arg);
+ return t;
+}
+
+int main(int argc, char **argv)
+{
+ unsigned int i, thr;
+ pthread_t threads[256];
+ struct args args[256];
+
+ if (argc < 2) {
+ printf("ERR: %s [numthreads]\n", argv[0]);
+ exit(1);
+ }
+
+ thr = atoi(argv[1]);
+ if ((thr > 256) || (thr < 1)) {
+ printf("ERR: threads 1-256\n");
+ exit(1);
+ }
+ for (i = 0; i < thr; i++) {
+ args[i].in = rand();
+ args[i].th = new_thr(thrfn, &(args[i]));
+ }
+ for (i = 0; i < thr; i++)
+ pthread_join(args[i].th, &(args[i].ret));
+ return 0;
+}
--- /dev/null
+#!/bin/sh -e
+# CoreSight / Unroll Loop Thread 10
+
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+TEST="unroll_loop_thread"
+. $(dirname $0)/../lib/coresight.sh
+ARGS="10"
+DATV="10"
+DATA="$DATD/perf-$TEST-$DATV.data"
+
+perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS
+
+perf_dump_aux_verify "$DATA" 10 10 10
+
+err=$?
+exit $err
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Carsten Haitzler <carsten.haitzler@arm.com>, 2021
+
+# This is sourced from a driver script so no need for #!/bin... etc. at the
+# top - the assumption below is that it runs as part of sourcing after the
+# test sets up some basic env vars to say what it is.
+
+# This currently works with ETMv4 / ETF not any other packet types at thi
+# point. This will need changes if that changes.
+
+# perf record options for the perf tests to use
+PERFRECMEM="-m ,16M"
+PERFRECOPT="$PERFRECMEM -e cs_etm//u"
+
+TOOLS=$(dirname $0)
+DIR="$TOOLS/$TEST"
+BIN="$DIR/$TEST"
+# If the test tool/binary does not exist and is executable then skip the test
+if ! test -x "$BIN"; then exit 2; fi
+DATD="."
+# If the data dir env is set then make the data dir use that instead of ./
+if test -n "$PERF_TEST_CORESIGHT_DATADIR"; then
+ DATD="$PERF_TEST_CORESIGHT_DATADIR";
+fi
+# If the stat dir env is set then make the data dir use that instead of ./
+STATD="."
+if test -n "$PERF_TEST_CORESIGHT_STATDIR"; then
+ STATD="$PERF_TEST_CORESIGHT_STATDIR";
+fi
+
+# Called if the test fails - error code 1
+err() {
+ echo "$1"
+ exit 1
+}
+
+# Check that some statistics from our perf
+check_val_min() {
+ STATF="$4"
+ if test "$2" -lt "$3"; then
+ echo ", FAILED" >> "$STATF"
+ err "Sanity check number of $1 is too low ($2 < $3)"
+ fi
+}
+
+perf_dump_aux_verify() {
+ # Some basic checking that the AUX chunk contains some sensible data
+ # to see that we are recording something and at least a minimum
+ # amount of it. We should almost always see Fn packets in just about
+ # anything but certainly we will see some trace info and async
+ # packets
+ DUMP="$DATD/perf-tmp-aux-dump.txt"
+ perf report --stdio --dump -i "$1" | \
+ grep -o -e I_ATOM_F -e I_ASYNC -e I_TRACE_INFO > "$DUMP"
+ # Simply count how many of these packets we find to see that we are
+ # producing a reasonable amount of data - exact checks are not sane
+ # as this is a lossy process where we may lose some blocks and the
+ # compiler may produce different code depending on the compiler and
+ # optimization options, so this is rough just to see if we're
+ # either missing almost all the data or all of it
+ ATOM_FX_NUM=`grep I_ATOM_F "$DUMP" | wc -l`
+ ASYNC_NUM=`grep I_ASYNC "$DUMP" | wc -l`
+ TRACE_INFO_NUM=`grep I_TRACE_INFO "$DUMP" | wc -l`
+ rm -f "$DUMP"
+
+ # Arguments provide minimums for a pass
+ CHECK_FX_MIN="$2"
+ CHECK_ASYNC_MIN="$3"
+ CHECK_TRACE_INFO_MIN="$4"
+
+ # Write out statistics, so over time you can track results to see if
+ # there is a pattern - for example we have less "noisy" results that
+ # produce more consistent amounts of data each run, to see if over
+ # time any techinques to minimize data loss are having an effect or
+ # not
+ STATF="$STATD/stats-$TEST-$DATV.csv"
+ if ! test -f "$STATF"; then
+ echo "ATOM Fx Count, Minimum, ASYNC Count, Minimum, TRACE INFO Count, Minimum" > "$STATF"
+ fi
+ echo -n "$ATOM_FX_NUM, $CHECK_FX_MIN, $ASYNC_NUM, $CHECK_ASYNC_MIN, $TRACE_INFO_NUM, $CHECK_TRACE_INFO_MIN" >> "$STATF"
+
+ # Actually check to see if we passed or failed.
+ check_val_min "ATOM_FX" "$ATOM_FX_NUM" "$CHECK_FX_MIN" "$STATF"
+ check_val_min "ASYNC" "$ASYNC_NUM" "$CHECK_ASYNC_MIN" "$STATF"
+ check_val_min "TRACE_INFO" "$TRACE_INFO_NUM" "$CHECK_TRACE_INFO_MIN" "$STATF"
+ echo ", Ok" >> "$STATF"
+}
+
+perf_dump_aux_tid_verify() {
+ # Specifically crafted test will produce a list of Tread ID's to
+ # stdout that need to be checked to see that they have had trace
+ # info collected in AUX blocks in the perf data. This will go
+ # through all the TID's that are listed as CID=0xabcdef and see
+ # that all the Thread IDs the test tool reports are in the perf
+ # data AUX chunks
+
+ # The TID test tools will print a TID per stdout line that are being
+ # tested
+ TIDS=`cat "$2"`
+ # Scan the perf report to find the TIDs that are actually CID in hex
+ # and build a list of the ones found
+ FOUND_TIDS=`perf report --stdio --dump -i "$1" | \
+ grep -o "CID=0x[0-9a-z]\+" | sed 's/CID=//g' | \
+ uniq | sort | uniq`
+ # No CID=xxx found - maybe your kernel is reporting these as
+ # VMID=xxx so look there
+ if test -z "$FOUND_TIDS"; then
+ FOUND_TIDS=`perf report --stdio --dump -i "$1" | \
+ grep -o "VMID=0x[0-9a-z]\+" | sed 's/VMID=//g' | \
+ uniq | sort | uniq`
+ fi
+
+ # Iterate over the list of TIDs that the test says it has and find
+ # them in the TIDs found in the perf report
+ MISSING=""
+ for TID2 in $TIDS; do
+ FOUND=""
+ for TIDHEX in $FOUND_TIDS; do
+ TID=`printf "%i" $TIDHEX`
+ if test "$TID" -eq "$TID2"; then
+ FOUND="y"
+ break
+ fi
+ done
+ if test -z "$FOUND"; then
+ MISSING="$MISSING $TID"
+ fi
+ done
+ if test -n "$MISSING"; then
+ err "Thread IDs $MISSING not found in perf AUX data"
+ fi
+}
}
skip_if_no_debuginfo() {
- add_probe_vfs_getname -v 2>&1 | egrep -q "^(Failed to find the path for the kernel|Debuginfo-analysis is not supported)" && return 2
+ add_probe_vfs_getname -v 2>&1 | egrep -q "^(Failed to find the path for the kernel|Debuginfo-analysis is not supported)|(file has no debug information)" && return 2
return 1
}
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+
+tenths=date\ +%s%1N
+
+# Wait for PID $1 to have $2 number of threads started
+# Time out after $3 tenths of a second or 5 seconds if $3 is ""
+wait_for_threads()
+{
+ tm_out=$3 ; [ -n "${tm_out}" ] || tm_out=50
+ start_time=$($tenths)
+ while [ -e "/proc/$1/task" ] ; do
+ th_cnt=$(find "/proc/$1/task" -mindepth 1 -maxdepth 1 -printf x | wc -c)
+ if [ "${th_cnt}" -ge "$2" ] ; then
+ return 0
+ fi
+ # Wait at most tm_out tenths of a second
+ if [ $(($($tenths) - start_time)) -ge $tm_out ] ; then
+ echo "PID $1 does not have $2 threads"
+ return 1
+ fi
+ done
+ return 1
+}
+
+# Wait for perf record -vvv 2>$2 with PID $1 to start by looking at file $2
+# It depends on capturing perf record debug message "perf record has started"
+# Time out after $3 tenths of a second or 5 seconds if $3 is ""
+wait_for_perf_to_start()
+{
+ tm_out=$3 ; [ -n "${tm_out}" ] || tm_out=50
+ echo "Waiting for \"perf record has started\" message"
+ start_time=$($tenths)
+ while [ -e "/proc/$1" ] ; do
+ if grep -q "perf record has started" "$2" ; then
+ echo OK
+ break
+ fi
+ # Wait at most tm_out tenths of a second
+ if [ $(($($tenths) - start_time)) -ge $tm_out ] ; then
+ echo "perf recording did not start"
+ return 1
+ fi
+ done
+ return 0
+}
+
+# Wait for process PID %1 to exit
+# Time out after $2 tenths of a second or 5 seconds if $2 is ""
+wait_for_process_to_exit()
+{
+ tm_out=$2 ; [ -n "${tm_out}" ] || tm_out=50
+ start_time=$($tenths)
+ while [ -e "/proc/$1" ] ; do
+ # Wait at most tm_out tenths of a second
+ if [ $(($($tenths) - start_time)) -ge $tm_out ] ; then
+ echo "PID $1 did not exit as expected"
+ return 1
+ fi
+ done
+ return 0
+}
+
+# Check if PID $1 is still running after $2 tenths of a second
+# or 0.3 seconds if $2 is ""
+is_running()
+{
+ tm_out=$2 ; [ -n "${tm_out}" ] || tm_out=3
+ start_time=$($tenths)
+ while [ -e "/proc/$1" ] ; do
+ # Check for at least tm_out tenths of a second
+ if [ $(($($tenths) - start_time)) -gt $tm_out ] ; then
+ return 0
+ fi
+ done
+ echo "PID $1 exited prematurely"
+ return 1
+}
--- /dev/null
+#!/bin/sh
+# kernel lock contention analysis test
+# SPDX-License-Identifier: GPL-2.0
+
+set -e
+
+err=0
+perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
+result=$(mktemp /tmp/__perf_test.result.XXXXX)
+
+cleanup() {
+ rm -f ${perfdata}
+ rm -f ${result}
+ trap - exit term int
+}
+
+trap_cleanup() {
+ cleanup
+ exit ${err}
+}
+trap trap_cleanup exit term int
+
+check() {
+ if [ `id -u` != 0 ]; then
+ echo "[Skip] No root permission"
+ err=2
+ exit
+ fi
+
+ if ! perf list | grep -q lock:contention_begin; then
+ echo "[Skip] No lock contention tracepoints"
+ err=2
+ exit
+ fi
+}
+
+test_record()
+{
+ echo "Testing perf lock record and perf lock contention"
+ perf lock record -o ${perfdata} -- perf bench sched messaging > /dev/null 2>&1
+ # the output goes to the stderr and we expect only 1 output (-E 1)
+ perf lock contention -i ${perfdata} -E 1 -q 2> ${result}
+ if [ $(cat "${result}" | wc -l) != "1" ]; then
+ echo "[Fail] Recorded result count is not 1:" $(cat "${result}" | wc -l)
+ err=1
+ exit
+ fi
+}
+
+test_bpf()
+{
+ echo "Testing perf lock contention --use-bpf"
+
+ if ! perf lock con -b true > /dev/null 2>&1 ; then
+ echo "[Skip] No BPF support"
+ exit
+ fi
+
+ # the perf lock contention output goes to the stderr
+ perf lock con -a -b -E 1 -q -- perf bench sched messaging > /dev/null 2> ${result}
+ if [ $(cat "${result}" | wc -l) != "1" ]; then
+ echo "[Fail] BPF result count is not 1:" $(cat "${result}" | wc -l)
+ err=1
+ exit
+ fi
+}
+
+check
+
+test_record
+test_bpf
+
+exit ${err}
set -e
+skip_test=0
+
function commachecker()
{
local -i cnt=0
echo "[Success]"
}
+# The perf stat options for per-socket, per-core, per-die
+# and -A ( no_aggr mode ) uses the info fetched from this
+# directory: "/sys/devices/system/cpu/cpu*/topology". For
+# example, socket value is fetched from "physical_package_id"
+# file in topology directory.
+# Reference: cpu__get_topology_int in util/cpumap.c
+# If the platform doesn't expose topology information, values
+# will be set to -1. For example, incase of pSeries platform
+# of powerpc, value for "physical_package_id" is restricted
+# and set to -1. Check here validates the socket-id read from
+# topology file before proceeding further
+
+FILE_LOC="/sys/devices/system/cpu/cpu*/topology/"
+FILE_NAME="physical_package_id"
+
+check_for_topology()
+{
+ if ! ParanoidAndNotRoot 0
+ then
+ socket_file=`ls $FILE_LOC/$FILE_NAME | head -n 1`
+ [ -z $socket_file ] && return 0
+ socket_id=`cat $socket_file`
+ [ $socket_id == -1 ] && skip_test=1
+ return 0
+ fi
+}
+
+check_for_topology
check_no_args
check_system_wide
-check_system_wide_no_aggr
check_interval
check_event
-check_per_core
check_per_thread
-check_per_die
check_per_node
-check_per_socket
+if [ $skip_test -ne 1 ]
+then
+ check_system_wide_no_aggr
+ check_per_core
+ check_per_die
+ check_per_socket
+else
+ echo "[Skip] Skipping tests for system_wide_no_aggr, per_core, per_die and per_socket since socket id exposed via topology is invalid"
+fi
exit 0
set -e
+skip_test=0
+
pythonchecker=$(dirname $0)/lib/perf_json_output_lint.py
if [ "x$PYTHON" == "x" ]
then
echo "[Success]"
}
+# The perf stat options for per-socket, per-core, per-die
+# and -A ( no_aggr mode ) uses the info fetched from this
+# directory: "/sys/devices/system/cpu/cpu*/topology". For
+# example, socket value is fetched from "physical_package_id"
+# file in topology directory.
+# Reference: cpu__get_topology_int in util/cpumap.c
+# If the platform doesn't expose topology information, values
+# will be set to -1. For example, incase of pSeries platform
+# of powerpc, value for "physical_package_id" is restricted
+# and set to -1. Check here validates the socket-id read from
+# topology file before proceeding further
+
+FILE_LOC="/sys/devices/system/cpu/cpu*/topology/"
+FILE_NAME="physical_package_id"
+
+check_for_topology()
+{
+ if ! ParanoidAndNotRoot 0
+ then
+ socket_file=`ls $FILE_LOC/$FILE_NAME | head -n 1`
+ [ -z $socket_file ] && return 0
+ socket_id=`cat $socket_file`
+ [ $socket_id == -1 ] && skip_test=1
+ return 0
+ fi
+}
+
+check_for_topology
check_no_args
check_system_wide
-check_system_wide_no_aggr
check_interval
check_event
-check_per_core
check_per_thread
-check_per_die
check_per_node
-check_per_socket
+if [ $skip_test -ne 1 ]
+then
+ check_system_wide_no_aggr
+ check_per_core
+ check_per_die
+ check_per_socket
+else
+ echo "[Skip] Skipping tests for system_wide_no_aggr, per_core, per_die and per_socket since socket id exposed via topology is invalid"
+fi
exit 0
# 68.12% touch libc-2.27.so [.] _dl_addr
# 5.80% touch libc-2.27.so [.] getenv
# 4.35% touch ld-2.27.so [.] _dl_fixup
- perf report --itrace=i1000i --stdio -i ${perfdata} 2>&1 | \
+ perf report --itrace=i20i --stdio -i ${perfdata} 2>&1 | \
egrep " +[0-9]+\.[0-9]+% +$1" > /dev/null 2>&1
}
--- /dev/null
+#!/bin/bash
+# Test data symbol
+
+# SPDX-License-Identifier: GPL-2.0
+# Leo Yan <leo.yan@linaro.org>, 2022
+
+skip_if_no_mem_event() {
+ perf mem record -e list 2>&1 | egrep -q 'available' && return 0
+ return 2
+}
+
+skip_if_no_mem_event || exit 2
+
+# skip if there's no compiler
+if ! [ -x "$(command -v cc)" ]; then
+ echo "skip: no compiler, install gcc"
+ exit 2
+fi
+
+TEST_PROGRAM=$(mktemp /tmp/__perf_test.program.XXXXX)
+PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
+
+check_result() {
+ # The memory report format is as below:
+ # 99.92% ... [.] buf1+0x38
+ result=$(perf mem report -i ${PERF_DATA} -s symbol_daddr -q 2>&1 |
+ awk '/buf1/ { print $4 }')
+
+ # Testing is failed if has no any sample for "buf1"
+ [ -z "$result" ] && return 1
+
+ while IFS= read -r line; do
+ # The "data1" and "data2" fields in structure "buf1" have
+ # offset "0x0" and "0x38", returns failure if detect any
+ # other offset value.
+ if [ "$line" != "buf1+0x0" ] && [ "$line" != "buf1+0x38" ]; then
+ return 1
+ fi
+ done <<< "$result"
+
+ return 0
+}
+
+cleanup_files()
+{
+ echo "Cleaning up files..."
+ rm -f ${PERF_DATA}
+ rm -f ${TEST_PROGRAM}
+}
+
+trap cleanup_files exit term int
+
+# compile test program
+echo "Compiling test program..."
+cat << EOF | cc -o ${TEST_PROGRAM} -x c -
+typedef struct _buf {
+ char data1;
+ char reserved[55];
+ char data2;
+} buf __attribute__((aligned(64)));
+
+static buf buf1;
+
+int main(void) {
+ for (;;) {
+ buf1.data1++;
+ buf1.data2 += buf1.data1;
+ }
+ return 0;
+}
+EOF
+
+echo "Recording workload..."
+
+# perf mem/c2c internally uses IBS PMU on AMD CPU which doesn't support
+# user/kernel filtering and per-process monitoring, spin program on
+# specific CPU and test in per-CPU mode.
+is_amd=$(egrep -c 'vendor_id.*AuthenticAMD' /proc/cpuinfo)
+if (($is_amd >= 1)); then
+ perf mem record -o ${PERF_DATA} -C 0 -- taskset -c 0 $TEST_PROGRAM &
+else
+ perf mem record --all-user -o ${PERF_DATA} -- $TEST_PROGRAM &
+fi
+
+PERFPID=$!
+
+sleep 1
+
+kill $PERFPID
+wait $PERFPID
+
+check_result
+exit $?
# Skip if no Intel PT
perf list | grep -q 'intel_pt//' || exit 2
+shelldir=$(dirname "$0")
+. "${shelldir}"/lib/waiting.sh
+
skip_cnt=0
ok_cnt=0
err_cnt=0
-tmpfile=`mktemp`
-perfdatafile=`mktemp`
+temp_dir=$(mktemp -d /tmp/perf-test-intel-pt-sh.XXXXXXXXXX)
+
+tmpfile="${temp_dir}/tmp-perf.data"
+perfdatafile="${temp_dir}/test-perf.data"
+outfile="${temp_dir}/test-out.txt"
+errfile="${temp_dir}/test-err.txt"
+workload="${temp_dir}/workload"
+awkscript="${temp_dir}/awkscript"
+jitdump_workload="${temp_dir}/jitdump_workload"
+maxbrstack="${temp_dir}/maxbrstack.py"
+
+cleanup()
+{
+ trap - EXIT TERM INT
+ sane=$(echo "${temp_dir}" | cut -b 1-26)
+ if [ "${sane}" = "/tmp/perf-test-intel-pt-sh" ] ; then
+ echo "--- Cleaning up ---"
+ rm -f "${temp_dir}/"*
+ rmdir "${temp_dir}"
+ fi
+}
+
+trap_cleanup()
+{
+ cleanup
+ exit 1
+}
+
+trap trap_cleanup EXIT TERM INT
+
+# perf record for testing without decoding
+perf_record_no_decode()
+{
+ # Options to speed up recording: no post-processing, no build-id cache update,
+ # and no BPF events.
+ perf record -B -N --no-bpf-event "$@"
+}
+
+# perf record for testing should not need BPF events
+perf_record_no_bpf()
+{
+ # Options for no BPF events
+ perf record --no-bpf-event "$@"
+}
+
+have_workload=false
+cat << _end_of_file_ | /usr/bin/cc -o "${workload}" -xc - -pthread && have_workload=true
+#include <time.h>
+#include <pthread.h>
+
+void work(void) {
+ struct timespec tm = {
+ .tv_nsec = 1000000,
+ };
+ int i;
+
+ /* Run for about 30 seconds */
+ for (i = 0; i < 30000; i++)
+ nanosleep(&tm, NULL);
+}
+
+void *threadfunc(void *arg) {
+ work();
+ return NULL;
+}
+
+int main(void) {
+ pthread_t th;
+
+ pthread_create(&th, NULL, threadfunc, NULL);
+ work();
+ pthread_join(th, NULL);
+ return 0;
+}
+_end_of_file_
can_cpu_wide()
{
- perf record -o ${tmpfile} -B -N --no-bpf-event -e dummy:u -C $1 true 2>&1 >/dev/null || return 2
+ echo "Checking for CPU-wide recording on CPU $1"
+ if ! perf_record_no_decode -o "${tmpfile}" -e dummy:u -C "$1" true >/dev/null 2>&1 ; then
+ echo "No so skipping"
+ return 2
+ fi
+ echo OK
return 0
}
test_system_wide_side_band()
{
+ echo "--- Test system-wide sideband ---"
+
# Need CPU 0 and CPU 1
can_cpu_wide 0 || return $?
can_cpu_wide 1 || return $?
# Record on CPU 0 a task running on CPU 1
- perf record -B -N --no-bpf-event -o ${perfdatafile} -e intel_pt//u -C 0 -- taskset --cpu-list 1 uname
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u -C 0 -- taskset --cpu-list 1 uname
# Should get MMAP events from CPU 1 because they can be needed to decode
- mmap_cnt=`perf script -i ${perfdatafile} --no-itrace --show-mmap-events -C 1 2>/dev/null | grep MMAP | wc -l`
+ mmap_cnt=$(perf script -i "${perfdatafile}" --no-itrace --show-mmap-events -C 1 2>/dev/null | grep -c MMAP)
- if [ ${mmap_cnt} -gt 0 ] ; then
+ if [ "${mmap_cnt}" -gt 0 ] ; then
+ echo OK
return 0
fi
return 1
}
+can_kernel()
+{
+ if [ -z "${can_kernel_trace}" ] ; then
+ can_kernel_trace=0
+ perf_record_no_decode -o "${tmpfile}" -e dummy:k true >/dev/null 2>&1 && can_kernel_trace=1
+ fi
+ if [ ${can_kernel_trace} -eq 0 ] ; then
+ echo "SKIP: no kernel tracing"
+ return 2
+ fi
+ return 0
+}
+
+test_per_thread()
+{
+ k="$1"
+ desc="$2"
+
+ echo "--- Test per-thread ${desc}recording ---"
+
+ if ! $have_workload ; then
+ echo "No workload, so skipping"
+ return 2
+ fi
+
+ if [ "${k}" = "k" ] ; then
+ can_kernel || return 2
+ fi
+
+ cat <<- "_end_of_file_" > "${awkscript}"
+ BEGIN {
+ s = "[ ]*"
+ u = s"[0-9]+"s
+ d = s"[0-9-]+"s
+ x = s"[0-9a-fA-FxX]+"s
+ mmapping = "idx"u": mmapping fd"u
+ set_output = "idx"u": set output fd"u"->"u
+ perf_event_open = "sys_perf_event_open: pid"d"cpu"d"group_fd"d"flags"x"="u
+ }
+
+ /perf record opening and mmapping events/ {
+ if (!done)
+ active = 1
+ }
+
+ /perf record done opening and mmapping events/ {
+ active = 0
+ done = 1
+ }
+
+ $0 ~ perf_event_open && active {
+ match($0, perf_event_open)
+ $0 = substr($0, RSTART, RLENGTH)
+ pid = $3
+ cpu = $5
+ fd = $11
+ print "pid " pid " cpu " cpu " fd " fd " : " $0
+ fd_array[fd] = fd
+ pid_array[fd] = pid
+ cpu_array[fd] = cpu
+ }
+
+ $0 ~ mmapping && active {
+ match($0, mmapping)
+ $0 = substr($0, RSTART, RLENGTH)
+ fd = $5
+ print "fd " fd " : " $0
+ if (fd in fd_array) {
+ mmap_array[fd] = 1
+ } else {
+ print "Unknown fd " fd
+ exit 1
+ }
+ }
+
+ $0 ~ set_output && active {
+ match($0, set_output)
+ $0 = substr($0, RSTART, RLENGTH)
+ fd = $6
+ fd_to = $8
+ print "fd " fd " fd_to " fd_to " : " $0
+ if (fd in fd_array) {
+ if (fd_to in fd_array) {
+ set_output_array[fd] = fd_to
+ } else {
+ print "Unknown fd " fd_to
+ exit 1
+ }
+ } else {
+ print "Unknown fd " fd
+ exit 1
+ }
+ }
+
+ END {
+ print "Checking " length(fd_array) " fds"
+ for (fd in fd_array) {
+ if (fd in mmap_array) {
+ pid = pid_array[fd]
+ if (pid != -1) {
+ if (pid in pids) {
+ print "More than 1 mmap for PID " pid
+ exit 1
+ }
+ pids[pid] = 1
+ }
+ cpu = cpu_array[fd]
+ if (cpu != -1) {
+ if (cpu in cpus) {
+ print "More than 1 mmap for CPU " cpu
+ exit 1
+ }
+ cpus[cpu] = 1
+ }
+ } else if (!(fd in set_output_array)) {
+ print "No mmap for fd " fd
+ exit 1
+ }
+ }
+ n = length(pids)
+ if (n != thread_cnt) {
+ print "Expected " thread_cnt " per-thread mmaps - found " n
+ exit 1
+ }
+ }
+ _end_of_file_
+
+ $workload &
+ w1=$!
+ $workload &
+ w2=$!
+ echo "Workload PIDs are $w1 and $w2"
+ wait_for_threads ${w1} 2
+ wait_for_threads ${w2} 2
+
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u"${k}" -vvv --per-thread -p "${w1},${w2}" 2>"${errfile}" >"${outfile}" &
+ ppid=$!
+ echo "perf PID is $ppid"
+ wait_for_perf_to_start ${ppid} "${errfile}" || return 1
+
+ kill ${w1}
+ wait_for_process_to_exit ${w1} || return 1
+ is_running ${ppid} || return 1
+
+ kill ${w2}
+ wait_for_process_to_exit ${w2} || return 1
+ wait_for_process_to_exit ${ppid} || return 1
+
+ awk -v thread_cnt=4 -f "${awkscript}" "${errfile}" || return 1
+
+ echo OK
+ return 0
+}
+
+test_jitdump()
+{
+ echo "--- Test tracing self-modifying code that uses jitdump ---"
+
+ script_path=$(realpath "$0")
+ script_dir=$(dirname "$script_path")
+ jitdump_incl_dir="${script_dir}/../../util"
+ jitdump_h="${jitdump_incl_dir}/jitdump.h"
+
+ if [ ! -e "${jitdump_h}" ] ; then
+ echo "SKIP: Include file jitdump.h not found"
+ return 2
+ fi
+
+ if [ -z "${have_jitdump_workload}" ] ; then
+ have_jitdump_workload=false
+ # Create a workload that uses self-modifying code and generates its own jitdump file
+ cat <<- "_end_of_file_" | /usr/bin/cc -o "${jitdump_workload}" -I "${jitdump_incl_dir}" -xc - -pthread && have_jitdump_workload=true
+ #define _GNU_SOURCE
+ #include <sys/mman.h>
+ #include <sys/types.h>
+ #include <stddef.h>
+ #include <stdio.h>
+ #include <stdint.h>
+ #include <unistd.h>
+ #include <string.h>
+
+ #include "jitdump.h"
+
+ #define CHK_BYTE 0x5a
+
+ static inline uint64_t rdtsc(void)
+ {
+ unsigned int low, high;
+
+ asm volatile("rdtsc" : "=a" (low), "=d" (high));
+
+ return low | ((uint64_t)high) << 32;
+ }
+
+ static FILE *open_jitdump(void)
+ {
+ struct jitheader header = {
+ .magic = JITHEADER_MAGIC,
+ .version = JITHEADER_VERSION,
+ .total_size = sizeof(header),
+ .pid = getpid(),
+ .timestamp = rdtsc(),
+ .flags = JITDUMP_FLAGS_ARCH_TIMESTAMP,
+ };
+ char filename[256];
+ FILE *f;
+ void *m;
+
+ snprintf(filename, sizeof(filename), "jit-%d.dump", getpid());
+ f = fopen(filename, "w+");
+ if (!f)
+ goto err;
+ /* Create an MMAP event for the jitdump file. That is how perf tool finds it. */
+ m = mmap(0, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, fileno(f), 0);
+ if (m == MAP_FAILED)
+ goto err_close;
+ munmap(m, 4096);
+ if (fwrite(&header,sizeof(header),1,f) != 1)
+ goto err_close;
+ return f;
+
+ err_close:
+ fclose(f);
+ err:
+ return NULL;
+ }
+
+ static int write_jitdump(FILE *f, void *addr, const uint8_t *dat, size_t sz, uint64_t *idx)
+ {
+ struct jr_code_load rec = {
+ .p.id = JIT_CODE_LOAD,
+ .p.total_size = sizeof(rec) + sz,
+ .p.timestamp = rdtsc(),
+ .pid = getpid(),
+ .tid = gettid(),
+ .vma = (unsigned long)addr,
+ .code_addr = (unsigned long)addr,
+ .code_size = sz,
+ .code_index = ++*idx,
+ };
+
+ if (fwrite(&rec,sizeof(rec),1,f) != 1 ||
+ fwrite(dat, sz, 1, f) != 1)
+ return -1;
+ return 0;
+ }
+
+ static void close_jitdump(FILE *f)
+ {
+ fclose(f);
+ }
+
+ int main()
+ {
+ /* Get a memory page to store executable code */
+ void *addr = mmap(0, 4096, PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ /* Code to execute: mov CHK_BYTE, %eax ; ret */
+ uint8_t dat[] = {0xb8, CHK_BYTE, 0x00, 0x00, 0x00, 0xc3};
+ FILE *f = open_jitdump();
+ uint64_t idx = 0;
+ int ret = 1;
+
+ if (!f)
+ return 1;
+ /* Copy executable code to executable memory page */
+ memcpy(addr, dat, sizeof(dat));
+ /* Record it in the jitdump file */
+ if (write_jitdump(f, addr, dat, sizeof(dat), &idx))
+ goto out_close;
+ /* Call it */
+ ret = ((int (*)(void))addr)() - CHK_BYTE;
+ out_close:
+ close_jitdump(f);
+ return ret;
+ }
+ _end_of_file_
+ fi
+
+ if ! $have_jitdump_workload ; then
+ echo "SKIP: No jitdump workload"
+ return 2
+ fi
+
+ # Change to temp_dir so jitdump collateral files go there
+ cd "${temp_dir}"
+ perf_record_no_bpf -o "${tmpfile}" -e intel_pt//u "${jitdump_workload}"
+ perf inject -i "${tmpfile}" -o "${perfdatafile}" --jit
+ decode_br_cnt=$(perf script -i "${perfdatafile}" --itrace=b | wc -l)
+ # Note that overflow and lost errors are suppressed for the error count
+ decode_err_cnt=$(perf script -i "${perfdatafile}" --itrace=e-o-l | grep -ci error)
+ cd -
+ # Should be thousands of branches
+ if [ "${decode_br_cnt}" -lt 1000 ] ; then
+ echo "Decode failed, only ${decode_br_cnt} branches"
+ return 1
+ fi
+ # Should be no errors
+ if [ "${decode_err_cnt}" -ne 0 ] ; then
+ echo "Decode failed, ${decode_err_cnt} errors"
+ perf script -i "${perfdatafile}" --itrace=e-o-l --show-mmap-events | cat
+ return 1
+ fi
+
+ echo OK
+ return 0
+}
+
+test_packet_filter()
+{
+ echo "--- Test with MTC and TSC disabled ---"
+ # Disable MTC and TSC
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt/mtc=0,tsc=0/u uname
+ # Should not get MTC packet
+ mtc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "MTC 0x")
+ if [ "${mtc_cnt}" -ne 0 ] ; then
+ echo "Failed to filter with mtc=0"
+ return 1
+ fi
+ # Should not get TSC package
+ tsc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TSC 0x")
+ if [ "${tsc_cnt}" -ne 0 ] ; then
+ echo "Failed to filter with tsc=0"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_disable_branch()
+{
+ echo "--- Test with branches disabled ---"
+ # Disable branch
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt/branch=0/u uname
+ # Should not get branch related packets
+ tnt_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TNT 0x")
+ tip_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TIP 0x")
+ fup_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "FUP 0x")
+ if [ "${tnt_cnt}" -ne 0 ] || [ "${tip_cnt}" -ne 0 ] || [ "${fup_cnt}" -ne 0 ] ; then
+ echo "Failed to disable branches"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_time_cyc()
+{
+ echo "--- Test with/without CYC ---"
+ # Check if CYC is supported
+ cyc=$(cat /sys/bus/event_source/devices/intel_pt/caps/psb_cyc)
+ if [ "${cyc}" != "1" ] ; then
+ echo "SKIP: CYC is not supported"
+ return 2
+ fi
+ # Enable CYC
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt/cyc/u uname
+ # should get CYC packets
+ cyc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "CYC 0x")
+ if [ "${cyc_cnt}" = "0" ] ; then
+ echo "Failed to get CYC packet"
+ return 1
+ fi
+ # Without CYC
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u uname
+ # Should not get CYC packets
+ cyc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "CYC 0x")
+ if [ "${cyc_cnt}" -gt 0 ] ; then
+ echo "Still get CYC packet without cyc"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_sample()
+{
+ echo "--- Test recording with sample mode ---"
+ # Check if recording with sample mode is working
+ if ! perf_record_no_decode -o "${perfdatafile}" --aux-sample=8192 -e '{intel_pt//u,branch-misses:u}' uname ; then
+ echo "perf record failed with --aux-sample"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_kernel_trace()
+{
+ echo "--- Test with kernel trace ---"
+ # Check if recording with kernel trace is working
+ can_kernel || return 2
+ if ! perf_record_no_decode -o "${perfdatafile}" -e intel_pt//k -m1,128 uname ; then
+ echo "perf record failed with intel_pt//k"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_virtual_lbr()
+{
+ echo "--- Test virtual LBR ---"
+
+ # Python script to determine the maximum size of branch stacks
+ cat << "_end_of_file_" > "${maxbrstack}"
+from __future__ import print_function
+
+bmax = 0
+
+def process_event(param_dict):
+ if "brstack" in param_dict:
+ brstack = param_dict["brstack"]
+ n = len(brstack)
+ global bmax
+ if n > bmax:
+ bmax = n
+
+def trace_end():
+ print("max brstack", bmax)
+_end_of_file_
+
+ # Check if virtual lbr is working
+ perf_record_no_bpf -o "${perfdatafile}" --aux-sample -e '{intel_pt//,cycles}:u' uname
+ times_val=$(perf script -i "${perfdatafile}" --itrace=L -s "${maxbrstack}" 2>/dev/null | grep "max brstack " | cut -d " " -f 3)
+ case "${times_val}" in
+ [0-9]*) ;;
+ *) times_val=0;;
+ esac
+ if [ "${times_val}" -lt 2 ] ; then
+ echo "Failed with virtual lbr"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_power_event()
+{
+ echo "--- Test power events ---"
+ # Check if power events are supported
+ power_event=$(cat /sys/bus/event_source/devices/intel_pt/caps/power_event_trace)
+ if [ "${power_event}" != "1" ] ; then
+ echo "SKIP: power_event_trace is not supported"
+ return 2
+ fi
+ if ! perf_record_no_decode -o "${perfdatafile}" -a -e intel_pt/pwr_evt/u uname ; then
+ echo "perf record failed with pwr_evt"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_no_tnt()
+{
+ echo "--- Test with TNT packets disabled ---"
+ # Check if TNT disable is supported
+ notnt=$(cat /sys/bus/event_source/devices/intel_pt/caps/tnt_disable)
+ if [ "${notnt}" != "1" ] ; then
+ echo "SKIP: tnt_disable is not supported"
+ return 2
+ fi
+ perf_record_no_decode -o "${perfdatafile}" -e intel_pt/notnt/u uname
+ # Should be no TNT packets
+ tnt_cnt=$(perf script -i "${perfdatafile}" -D | grep -c TNT)
+ if [ "${tnt_cnt}" -ne 0 ] ; then
+ echo "TNT packets still there after notnt"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
+test_event_trace()
+{
+ echo "--- Test with event_trace ---"
+ # Check if event_trace is supported
+ event_trace=$(cat /sys/bus/event_source/devices/intel_pt/caps/event_trace)
+ if [ "${event_trace}" != 1 ] ; then
+ echo "SKIP: event_trace is not supported"
+ return 2
+ fi
+ if ! perf_record_no_decode -o "${perfdatafile}" -e intel_pt/event/u uname ; then
+ echo "perf record failed with event trace"
+ return 1
+ fi
+ echo OK
+ return 0
+}
+
count_result()
{
- if [ $1 -eq 2 ] ; then
- skip_cnt=`expr ${skip_cnt} \+ 1`
+ if [ "$1" -eq 2 ] ; then
+ skip_cnt=$((skip_cnt + 1))
return
fi
- if [ $1 -eq 0 ] ; then
- ok_cnt=`expr ${ok_cnt} \+ 1`
+ if [ "$1" -eq 0 ] ; then
+ ok_cnt=$((ok_cnt + 1))
return
fi
- err_cnt=`expr ${err_cnt} \+ 1`
+ err_cnt=$((err_cnt + 1))
}
-test_system_wide_side_band
+ret=0
+test_system_wide_side_band || ret=$? ; count_result $ret ; ret=0
+test_per_thread "" "" || ret=$? ; count_result $ret ; ret=0
+test_per_thread "k" "(incl. kernel) " || ret=$? ; count_result $ret ; ret=0
+test_jitdump || ret=$? ; count_result $ret ; ret=0
+test_packet_filter || ret=$? ; count_result $ret ; ret=0
+test_disable_branch || ret=$? ; count_result $ret ; ret=0
+test_time_cyc || ret=$? ; count_result $ret ; ret=0
+test_sample || ret=$? ; count_result $ret ; ret=0
+test_kernel_trace || ret=$? ; count_result $ret ; ret=0
+test_virtual_lbr || ret=$? ; count_result $ret ; ret=0
+test_power_event || ret=$? ; count_result $ret ; ret=0
+test_no_tnt || ret=$? ; count_result $ret ; ret=0
+test_event_trace || ret=$? ; count_result $ret ; ret=0
-count_result $?
+cleanup
-rm -f ${tmpfile}
-rm -f ${perfdatafile}
+echo "--- Done ---"
if [ ${err_cnt} -gt 0 ] ; then
exit 1
--- /dev/null
+#!/bin/bash
+# Test java symbol
+
+# SPDX-License-Identifier: GPL-2.0
+# Leo Yan <leo.yan@linaro.org>, 2022
+
+# skip if there's no jshell
+if ! [ -x "$(command -v jshell)" ]; then
+ echo "skip: no jshell, install JDK"
+ exit 2
+fi
+
+PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
+PERF_INJ_DATA=$(mktemp /tmp/__perf_test.perf.data.inj.XXXXX)
+
+cleanup_files()
+{
+ echo "Cleaning up files..."
+ rm -f ${PERF_DATA}
+ rm -f ${PERF_INJ_DATA}
+}
+
+trap cleanup_files exit term int
+
+if [ -e "$PWD/tools/perf/libperf-jvmti.so" ]; then
+ LIBJVMTI=$PWD/tools/perf/libperf-jvmti.so
+elif [ -e "$PWD/libperf-jvmti.so" ]; then
+ LIBJVMTI=$PWD/libperf-jvmti.so
+elif [ -e "$PREFIX/lib64/libperf-jvmti.so" ]; then
+ LIBJVMTI=$PREFIX/lib64/libperf-jvmti.so
+elif [ -e "$PREFIX/lib/libperf-jvmti.so" ]; then
+ LIBJVMTI=$PREFIX/lib/libperf-jvmti.so
+elif [ -e "/usr/lib/linux-tools-$(uname -a | awk '{ print $3 }' | sed -r 's/-generic//')/libperf-jvmti.so" ]; then
+ LIBJVMTI=/usr/lib/linux-tools-$(uname -a | awk '{ print $3 }' | sed -r 's/-generic//')/libperf-jvmti.so
+else
+ echo "Fail to find libperf-jvmti.so"
+ # JVMTI is a build option, skip the test if fail to find lib
+ exit 2
+fi
+
+cat <<EOF | perf record -k 1 -o $PERF_DATA jshell -s -J-agentpath:$LIBJVMTI
+int fib(int x) {
+ return x > 1 ? fib(x - 2) + fib(x - 1) : 1;
+}
+
+int q = 0;
+
+for (int i = 0; i < 10; i++)
+ q += fib(i);
+
+System.out.println(q);
+EOF
+
+if [ $? -ne 0 ]; then
+ echo "Fail to record for java program"
+ exit 1
+fi
+
+if ! perf inject -i $PERF_DATA -o $PERF_INJ_DATA -j; then
+ echo "Fail to inject samples"
+ exit 1
+fi
+
+# Below is an example of the instruction samples reporting:
+# 8.18% jshell jitted-50116-29.so [.] Interpreter
+# 0.75% Thread-1 jitted-83602-1670.so [.] jdk.internal.jimage.BasicImageReader.getString(int)
+perf report --stdio -i ${PERF_INJ_DATA} 2>&1 | \
+ egrep " +[0-9]+\.[0-9]+% .* (Interpreter|jdk\.internal).*" > /dev/null 2>&1
+
+if [ $? -ne 0 ]; then
+ echo "Fail to find java symbols"
+ exit 1
+fi
+
+exit 0
return attr;
}
+#ifdef HAVE_BPF_SKEL
+#include <bpf/btf.h>
+
+static bool attr_has_sigtrap(void)
+{
+ bool ret = false;
+ struct btf *btf;
+ const struct btf_type *t;
+ const struct btf_member *m;
+ const char *name;
+ int i, id;
+
+ btf = btf__load_vmlinux_btf();
+ if (btf == NULL) {
+ /* should be an old kernel */
+ return false;
+ }
+
+ id = btf__find_by_name_kind(btf, "perf_event_attr", BTF_KIND_STRUCT);
+ if (id < 0)
+ goto out;
+
+ t = btf__type_by_id(btf, id);
+ for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
+ name = btf__name_by_offset(btf, m->name_off);
+ if (!strcmp(name, "sigtrap")) {
+ ret = true;
+ break;
+ }
+ }
+out:
+ btf__free(btf);
+ return ret;
+}
+#else /* !HAVE_BPF_SKEL */
+static bool attr_has_sigtrap(void)
+{
+ struct perf_event_attr attr = {
+ .type = PERF_TYPE_SOFTWARE,
+ .config = PERF_COUNT_SW_DUMMY,
+ .size = sizeof(attr),
+ .remove_on_exec = 1, /* Required by sigtrap. */
+ .sigtrap = 1, /* Request synchronous SIGTRAP on event. */
+ };
+ int fd;
+ bool ret = false;
+
+ fd = sys_perf_event_open(&attr, 0, -1, -1, perf_event_open_cloexec_flag());
+ if (fd >= 0) {
+ ret = true;
+ close(fd);
+ }
+
+ return ret;
+}
+#endif /* HAVE_BPF_SKEL */
+
static void
sigtrap_handler(int signum __maybe_unused, siginfo_t *info, void *ucontext __maybe_unused)
{
fd = sys_perf_event_open(&attr, 0, -1, -1, perf_event_open_cloexec_flag());
if (fd < 0) {
- pr_debug("FAILED sys_perf_event_open(): %s\n", str_error_r(errno, sbuf, sizeof(sbuf)));
+ if (attr_has_sigtrap()) {
+ pr_debug("FAILED sys_perf_event_open(): %s\n",
+ str_error_r(errno, sbuf, sizeof(sbuf)));
+ } else {
+ pr_debug("perf_event_attr doesn't have sigtrap\n");
+ ret = TEST_SKIP;
+ }
goto out_restore_sigaction;
}
#include <time.h>
#include <stdlib.h>
#include <linux/zalloc.h>
+#include <linux/err.h>
#include <perf/cpumap.h>
#include <perf/evlist.h>
#include <perf/mmap.h>
goto out;
}
- err = parse_event(evlist, sched_switch);
- if (err) {
- pr_debug("Failed to parse event %s\n", sched_switch);
+ switch_evsel = evlist__add_sched_switch(evlist, true);
+ if (IS_ERR(switch_evsel)) {
+ err = PTR_ERR(switch_evsel);
+ pr_debug("Failed to create event %s\n", sched_switch);
goto out_err;
}
- switch_evsel = evlist__last(evlist);
-
- evsel__set_sample_bit(switch_evsel, CPU);
- evsel__set_sample_bit(switch_evsel, TIME);
-
- switch_evsel->core.system_wide = true;
- switch_evsel->no_aux_samples = true;
switch_evsel->immediate = true;
/* Test moving an event to the front */
TEST_ASSERT_VAL("Cpu map - Die ID doesn't match",
session->header.env.cpu[perf_cpu_map__cpu(map, i).cpu].die_id == id.die);
TEST_ASSERT_VAL("Cpu map - Node ID is set", id.node == -1);
- TEST_ASSERT_VAL("Cpu map - Thread is set", id.thread == -1);
+ TEST_ASSERT_VAL("Cpu map - Thread IDX is set", id.thread_idx == -1);
}
// Test that core ID contains socket, die and core
TEST_ASSERT_VAL("Core map - Die ID doesn't match",
session->header.env.cpu[perf_cpu_map__cpu(map, i).cpu].die_id == id.die);
TEST_ASSERT_VAL("Core map - Node ID is set", id.node == -1);
- TEST_ASSERT_VAL("Core map - Thread is set", id.thread == -1);
+ TEST_ASSERT_VAL("Core map - Thread IDX is set", id.thread_idx == -1);
}
// Test that die ID contains socket and die
TEST_ASSERT_VAL("Die map - Node ID is set", id.node == -1);
TEST_ASSERT_VAL("Die map - Core is set", id.core == -1);
TEST_ASSERT_VAL("Die map - CPU is set", id.cpu.cpu == -1);
- TEST_ASSERT_VAL("Die map - Thread is set", id.thread == -1);
+ TEST_ASSERT_VAL("Die map - Thread IDX is set", id.thread_idx == -1);
}
// Test that socket ID contains only socket
TEST_ASSERT_VAL("Socket map - Die ID is set", id.die == -1);
TEST_ASSERT_VAL("Socket map - Core is set", id.core == -1);
TEST_ASSERT_VAL("Socket map - CPU is set", id.cpu.cpu == -1);
- TEST_ASSERT_VAL("Socket map - Thread is set", id.thread == -1);
+ TEST_ASSERT_VAL("Socket map - Thread IDX is set", id.thread_idx == -1);
}
// Test that node ID contains only node
TEST_ASSERT_VAL("Node map - Die ID is set", id.die == -1);
TEST_ASSERT_VAL("Node map - Core is set", id.core == -1);
TEST_ASSERT_VAL("Node map - CPU is set", id.cpu.cpu == -1);
- TEST_ASSERT_VAL("Node map - Thread is set", id.thread == -1);
+ TEST_ASSERT_VAL("Node map - Thread IDX is set", id.thread_idx == -1);
}
perf_session__delete(session);
void ui_browser__show_title(struct ui_browser *browser, const char *title)
{
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
__ui_browser__show_title(browser, title);
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
int ui_browser__show(struct ui_browser *browser, const char *title,
browser->refresh_dimensions(browser);
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
__ui_browser__show_title(browser, title);
browser->title = title;
va_end(ap);
if (err > 0)
ui_helpline__push(browser->helpline);
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
return err ? 0 : -1;
}
void ui_browser__hide(struct ui_browser *browser)
{
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
ui_helpline__pop();
zfree(&browser->helpline);
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
static void ui_browser__scrollbar_set(struct ui_browser *browser)
int ui_browser__refresh(struct ui_browser *browser)
{
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
__ui_browser__refresh(browser);
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
return 0;
}
while (1) {
off_t offset;
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
err = __ui_browser__refresh(browser);
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
if (err < 0)
break;
#include "../../util/hist.h"
#include "../../util/sort.h"
#include "../../util/map.h"
+#include "../../util/mutex.h"
#include "../../util/symbol.h"
#include "../../util/evsel.h"
#include "../../util/evlist.h"
#include <inttypes.h>
-#include <pthread.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/zalloc.h>
#include <sys/ttydefaults.h>
#include <asm/bug.h>
-struct disasm_line_samples {
- double percent;
- struct sym_hist_entry he;
-};
-
struct arch;
struct annotate_browser {
browser->entries = RB_ROOT;
- pthread_mutex_lock(¬es->lock);
+ mutex_lock(¬es->lock);
symbol__calc_percent(sym, evsel);
}
disasm_rb_tree__insert(browser, &pos->al);
}
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
browser->curr_hot = rb_last(&browser->entries);
}
}
notes = symbol__annotation(dl->ops.target.sym);
- pthread_mutex_lock(¬es->lock);
+ mutex_lock(¬es->lock);
if (!symbol__hists(dl->ops.target.sym, evsel->evlist->core.nr_entries)) {
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
ui__warning("Not enough memory for annotating '%s' symbol!\n",
dl->ops.target.sym->name);
return true;
target_ms.maps = ms->maps;
target_ms.map = ms->map;
target_ms.sym = dl->ops.target.sym;
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
symbol__tui_annotate(&target_ms, evsel, hbt, browser->opts);
sym_title(ms->sym, ms->map, title, sizeof(title), browser->opts->percent_type);
ui_browser__show_title(&browser->b, title);
"r Run available scripts\n"
"p Toggle percent type [local/global]\n"
"b Toggle percent base [period/hits]\n"
- "? Search string backwards\n");
+ "? Search string backwards\n"
+ "f Toggle showing offsets to full address\n");
continue;
case 'r':
script_browse(NULL, NULL);
hists__scnprintf_title(hists, title, sizeof(title));
annotate_browser__show(&browser->b, title, help);
continue;
+ case 'f':
+ annotation__toggle_full_addr(notes, ms);
+ continue;
case K_LEFT:
case K_ESC:
case 'q':
// SPDX-License-Identifier: GPL-2.0
-#include <pthread.h>
#include <dlfcn.h>
#include <unistd.h>
#include "../util/hist.h"
#include "ui.h"
-pthread_mutex_t ui__lock = PTHREAD_MUTEX_INITIALIZER;
+struct mutex ui__lock;
void *perf_gtk_handle;
int use_browser = -1;
void setup_browser(bool fallback_to_pager)
{
+ mutex_init(&ui__lock);
if (use_browser < 2 && (!isatty(1) || dump_trace))
use_browser = 0;
default:
break;
}
+ mutex_destroy(&ui__lock);
}
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <pthread.h>
#include <linux/kernel.h>
#include <linux/string.h>
int ret;
static int backlog;
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
ret = vscnprintf(ui_helpline__last_msg + backlog,
sizeof(ui_helpline__last_msg) - backlog, format, ap);
backlog += ret;
SLsmg_refresh();
backlog = 0;
}
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
return ret;
}
}
ui__refresh_dimensions(false);
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
y = SLtt_Screen_Rows / 2 - 2;
SLsmg_set_color(0);
SLsmg_draw_box(y, 0, 3, SLtt_Screen_Cols);
bar = ((SLtt_Screen_Cols - 2) * p->curr) / p->total;
SLsmg_fill_region(y, 1, 1, bar, ' ');
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
static void tui_progress__finish(void)
return;
ui__refresh_dimensions(false);
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
y = SLtt_Screen_Rows / 2 - 2;
SLsmg_set_color(0);
SLsmg_fill_region(y, 0, 3, SLtt_Screen_Cols, ' ');
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
static struct ui_progress_ops tui_progress__ops = {
{
if (force || ui__need_resize) {
ui__need_resize = 0;
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
SLtt_get_screen_size();
SLsmg_reinit_smg();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
}
"Press any key...", 0);
SLtt_set_cursor_visibility(1);
- if (!pthread_mutex_trylock(&ui__lock)) {
+ if (mutex_trylock(&ui__lock)) {
SLsmg_refresh();
SLsmg_reset_smg();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
SLang_reset_tty();
perf_error__unregister(&perf_tui_eops);
t = sep + 1;
}
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
max_len += 2;
nr_lines += 8;
SLsmg_write_nstring((char *)exit_msg, max_len);
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
x += 2;
len = 0;
key = ui__getch(delay_secs);
while (key != K_TIMER && key != K_ENTER && key != K_ESC) {
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
if (key == K_BKSPC) {
if (len == 0) {
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
goto next_key;
}
SLsmg_gotorc(y, x + --len);
}
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
/* XXX more graceful overflow handling needed */
if (len == sizeof(buf) - 1) {
void ui__info_window(const char *title, const char *text)
{
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
__ui__info_window(title, text, NULL);
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
}
int ui__question_window(const char *title, const char *text,
const char *exit_msg, int delay_secs)
{
- pthread_mutex_lock(&ui__lock);
+ mutex_lock(&ui__lock);
__ui__info_window(title, text, exit_msg);
SLsmg_refresh();
- pthread_mutex_unlock(&ui__lock);
+ mutex_unlock(&ui__lock);
return ui__getch(delay_secs);
}
#ifndef _PERF_UI_H_
#define _PERF_UI_H_ 1
-#include <pthread.h>
+#include "../util/mutex.h"
#include <stdbool.h>
#include <linux/compiler.h>
-extern pthread_mutex_t ui__lock;
+extern struct mutex ui__lock;
extern void *perf_gtk_handle;
extern int use_browser;
perf-$(CONFIG_AUXTRACE) += intel-bts.o
perf-$(CONFIG_AUXTRACE) += arm-spe.o
perf-$(CONFIG_AUXTRACE) += arm-spe-decoder/
+perf-$(CONFIG_AUXTRACE) += hisi-ptt.o
+perf-$(CONFIG_AUXTRACE) += hisi-ptt-decoder/
perf-$(CONFIG_AUXTRACE) += s390-cpumsf.o
ifdef CONFIG_LIBOPENCSD
perf-y += mem2node.o
perf-y += clockid.o
perf-y += list_sort.o
+perf-y += mutex.o
perf-$(CONFIG_LIBBPF) += bpf-loader.o
perf-$(CONFIG_LIBBPF) += bpf_map.o
'
#
-# Always try first to get the version from the kernel Makefile
+# Use version from kernel Makefile unless not in a git repository and
+# PERF-VERSION-FILE exists
#
CID=
TAG=
then
TAG=$(MAKEFLAGS= make -sC ../.. kernelversion)
CID=$(git log -1 --abbrev=12 --pretty=format:"%h" 2>/dev/null) && CID="-g$CID"
-else
+elif test -f ../../PERF-VERSION-FILE
+then
TAG=$(cut -d' ' -f3 ../../PERF-VERSION-FILE | sed -e 's/\"//g')
fi
+if test -z "$TAG"
+then
+ TAG=$(MAKEFLAGS= make -sC ../.. kernelversion)
+fi
VN="$TAG$CID"
if test -n "$CID"
#include "arch/common.h"
#include "namespaces.h"
#include <regex.h>
-#include <pthread.h>
#include <linux/bitops.h>
#include <linux/kernel.h>
#include <linux/string.h>
{
struct annotation *notes = symbol__annotation(sym);
- pthread_mutex_lock(¬es->lock);
+ mutex_lock(¬es->lock);
if (notes->src != NULL) {
memset(notes->src->histograms, 0,
notes->src->nr_histograms * notes->src->sizeof_sym_hist);
memset(notes->src->cycles_hist, 0,
symbol__size(sym) * sizeof(struct cyc_hist));
}
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
}
static int __symbol__account_cycles(struct cyc_hist *ch,
notes->hit_insn = 0;
notes->cover_insn = 0;
- pthread_mutex_lock(¬es->lock);
+ mutex_lock(¬es->lock);
for (offset = size - 1; offset >= 0; --offset) {
struct cyc_hist *ch;
notes->have_cycles = true;
}
}
- pthread_mutex_unlock(¬es->lock);
+ mutex_unlock(¬es->lock);
}
int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample,
void annotation__init(struct annotation *notes)
{
- pthread_mutex_init(¬es->lock, NULL);
+ mutex_init(¬es->lock);
}
void annotation__exit(struct annotation *notes)
{
annotated_source__delete(notes->src);
- pthread_mutex_destroy(¬es->lock);
+ mutex_destroy(¬es->lock);
}
static void annotation_line__add(struct annotation_line *al, struct list_head *head)
*/
__symbol__join_symfs(filename, filename_size, dso->long_name);
+ mutex_lock(&dso->lock);
if (access(filename, R_OK) && errno == ENOENT && dso->nsinfo) {
char *new_name = filename_with_chroot(dso->nsinfo->pid,
filename);
free(new_name);
}
}
+ mutex_unlock(&dso->lock);
}
free(build_id_path);
}
args.ms = *ms;
- notes->start = map__rip_2objdump(ms->map, sym->start);
+ if (notes->options && notes->options->full_addr)
+ notes->start = map__objdump_2mem(ms->map, ms->sym->start);
+ else
+ notes->start = map__rip_2objdump(ms->map, ms->sym->start);
return symbol__disassemble(sym, &args);
}
{
if (notes->options->use_offset)
notes->widths.target = notes->widths.min_addr;
+ else if (notes->options->full_addr)
+ notes->widths.target = BITS_PER_LONG / 4;
else
notes->widths.target = notes->widths.max_addr;
notes->widths.addr += notes->widths.jumps + 1;
}
+void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms)
+{
+ notes->options->full_addr = !notes->options->full_addr;
+
+ if (notes->options->full_addr)
+ notes->start = map__objdump_2mem(ms->map, ms->sym->start);
+ else
+ notes->start = map__rip_2objdump(ms->map, ms->sym->start);
+
+ annotation__update_column_widths(notes);
+}
+
static void annotation__calc_lines(struct annotation *notes, struct map *map,
struct rb_root *root,
struct annotation_options *opts)
#include <linux/types.h>
#include <linux/list.h>
#include <linux/rbtree.h>
-#include <pthread.h>
#include <asm/bug.h>
#include "symbol_conf.h"
+#include "mutex.h"
#include "spark.h"
struct hist_browser_timer;
show_nr_jumps,
show_minmax_cycle,
show_asm_raw,
- annotate_src;
+ annotate_src,
+ full_addr;
u8 offset_level;
int min_pcnt;
int max_lines;
};
struct annotation {
- pthread_mutex_t lock;
+ struct mutex lock;
u64 max_coverage;
u64 start;
u64 hit_cycles;
void annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym);
void annotation__update_column_widths(struct annotation *notes);
void annotation__init_column_widths(struct annotation *notes, struct symbol *sym);
+void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms);
static inline struct sym_hist *annotated_source__histogram(struct annotated_source *src, int idx)
{
#include <linux/list.h>
#include <linux/zalloc.h>
+#include "config.h"
#include "evlist.h"
#include "dso.h"
#include "map.h"
#include "intel-pt.h"
#include "intel-bts.h"
#include "arm-spe.h"
+#include "hisi-ptt.h"
#include "s390-cpumsf.h"
#include "util/mmap.h"
case PERF_AUXTRACE_S390_CPUMSF:
err = s390_cpumsf_process_auxtrace_info(event, session);
break;
+ case PERF_AUXTRACE_HISI_PTT:
+ err = hisi_ptt_process_auxtrace_info(event, session);
+ break;
case PERF_AUXTRACE_UNKNOWN:
default:
return -EINVAL;
}
}
+#define ITRACE_DFLT_LOG_ON_ERROR_SZ 16384
+
+static unsigned int itrace_log_on_error_size(void)
+{
+ unsigned int sz = 0;
+
+ perf_config_scan("itrace.debug-log-buffer-size", "%u", &sz);
+ return sz ?: ITRACE_DFLT_LOG_ON_ERROR_SZ;
+}
+
/*
* Please check tools/perf/Documentation/perf-script.txt for information
* about the options parsed here, which is introduced after this cset,
if (get_flags(&p, &synth_opts->log_plus_flags,
&synth_opts->log_minus_flags))
goto out_err;
+ if (synth_opts->log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR)
+ synth_opts->log_on_error_size = itrace_log_on_error_size();
break;
case 'c':
synth_opts->branches = true;
PERF_AUXTRACE_CS_ETM,
PERF_AUXTRACE_ARM_SPE,
PERF_AUXTRACE_S390_CPUMSF,
+ PERF_AUXTRACE_HISI_PTT,
};
enum itrace_period_type {
#define AUXTRACE_ERR_FLG_DATA_LOST (1 << ('l' - 'a'))
#define AUXTRACE_LOG_FLG_ALL_PERF_EVTS (1 << ('a' - 'a'))
+#define AUXTRACE_LOG_FLG_ON_ERROR (1 << ('e' - 'a'))
#define AUXTRACE_LOG_FLG_USE_STDOUT (1 << ('o' - 'a'))
/**
* @log_plus_flags: flags to affect what is logged
* @log_minus_flags: flags to affect what is logged
* @quick: quicker (less detailed) decoding
+ * @log_on_error_size: size of log to keep for outputting log only on errors
*/
struct itrace_synth_opts {
bool set;
unsigned int log_plus_flags;
unsigned int log_minus_flags;
unsigned int quick;
+ unsigned int log_on_error_size;
};
/**
#include <linux/compiler.h>
#include <linux/rbtree.h>
-#include <pthread.h>
#include <api/fd/array.h>
#include <stdio.h>
#include "util.h"
#include "llvm-utils.h"
#include "c++/clang-c.h"
-#include "hashmap.h"
+#ifdef HAVE_LIBBPF_SUPPORT
+#include <bpf/hashmap.h>
+#else
+#include "util/hashmap.h"
+#endif
#include "asm/bug.h"
#include <internal/xyarray.h>
#include "util/thread_map.h"
#include "util/lock-contention.h"
#include <linux/zalloc.h>
+#include <linux/string.h>
#include <bpf/bpf.h>
#include "bpf_skel/lock_contention.skel.h"
static struct lock_contention_bpf *skel;
-/* should be same as bpf_skel/lock_contention.bpf.c */
-struct lock_contention_key {
- s32 stack_id;
-};
-
struct lock_contention_data {
u64 total_time;
u64 min_time;
return -1;
}
+ bpf_map__set_value_size(skel->maps.stacks, con->max_stack * sizeof(u64));
bpf_map__set_max_entries(skel->maps.stacks, con->map_nr_entries);
bpf_map__set_max_entries(skel->maps.lock_stat, con->map_nr_entries);
bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
}
+ skel->bss->stack_skip = con->stack_skip;
+
lock_contention_bpf__attach(skel);
return 0;
}
struct lock_contention_data data;
struct lock_stat *st;
struct machine *machine = con->machine;
- u64 stack_trace[CONTENTION_STACK_DEPTH];
+ u64 stack_trace[con->max_stack];
fd = bpf_map__fd(skel->maps.lock_stat);
stack = bpf_map__fd(skel->maps.stacks);
while (!bpf_map_get_next_key(fd, &prev_key, &key)) {
struct map *kmap;
struct symbol *sym;
- int idx;
+ int idx = 0;
bpf_map_lookup_elem(fd, &key, &data);
st = zalloc(sizeof(*st));
bpf_map_lookup_elem(stack, &key, stack_trace);
- /* skip BPF + lock internal functions */
- idx = CONTENTION_STACK_SKIP;
+ /* skip lock internal functions */
while (is_lock_function(machine, stack_trace[idx]) &&
- idx < CONTENTION_STACK_DEPTH - 1)
+ idx < con->max_stack - 1)
idx++;
st->addr = stack_trace[idx];
return -1;
}
+ if (verbose) {
+ st->callstack = memdup(stack_trace, sizeof(stack_trace));
+ if (st->callstack == NULL) {
+ free(st);
+ return -1;
+ }
+ }
+
hlist_add_head(&st->hash_entry, con->result);
prev_key = key;
}
__uint(value_size, sizeof(struct bpf_perf_event_value));
} cgrp_readings SEC(".maps");
+/* new kernel cgroup definition */
+struct cgroup___new {
+ int level;
+ struct cgroup *ancestors[];
+} __attribute__((preserve_access_index));
+
+/* old kernel cgroup definition */
+struct cgroup___old {
+ int level;
+ u64 ancestor_ids[];
+} __attribute__((preserve_access_index));
+
const volatile __u32 num_events = 1;
const volatile __u32 num_cpus = 1;
int use_cgroup_v2 = 0;
int perf_subsys_id = -1;
+static inline __u64 get_cgroup_v1_ancestor_id(struct cgroup *cgrp, int level)
+{
+ /* recast pointer to capture new type for compiler */
+ struct cgroup___new *cgrp_new = (void *)cgrp;
+
+ if (bpf_core_field_exists(cgrp_new->ancestors)) {
+ return BPF_CORE_READ(cgrp_new, ancestors[level], kn, id);
+ } else {
+ /* recast pointer to capture old type for compiler */
+ struct cgroup___old *cgrp_old = (void *)cgrp;
+
+ return BPF_CORE_READ(cgrp_old, ancestor_ids[level]);
+ }
+}
+
static inline int get_cgroup_v1_idx(__u32 *cgrps, int size)
{
struct task_struct *p = (void *)bpf_get_current_task();
break;
// convert cgroup-id to a map index
- cgrp_id = BPF_CORE_READ(cgrp, ancestors[i], kn, id);
+ cgrp_id = get_cgroup_v1_ancestor_id(cgrp, i);
elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id);
if (!elem)
continue;
int enabled;
int has_cpu;
int has_task;
+int stack_skip;
/* error stat */
-unsigned long lost;
+int lost;
static inline int can_record(void)
{
pelem->timestamp = bpf_ktime_get_ns();
pelem->lock = (__u64)ctx[0];
pelem->flags = (__u32)ctx[1];
- pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP);
+ pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP | stack_skip);
if (pelem->stack_id < 0)
lost++;
if (flags->type == PERF_BR_UNKNOWN || from == 0)
return;
- st->counts[flags->type]++;
+ if (flags->type == PERF_BR_EXTEND_ABI)
+ st->new_counts[flags->new_type]++;
+ else
+ st->counts[flags->type]++;
if (flags->type == PERF_BR_COND) {
if (to > from)
st->cross_4k++;
}
+const char *branch_new_type_name(int new_type)
+{
+ const char *branch_new_names[PERF_BR_NEW_MAX] = {
+ "FAULT_ALGN",
+ "FAULT_DATA",
+ "FAULT_INST",
+/*
+ * TODO: This switch should happen on 'session->header.env.arch'
+ * instead, because an arm64 platform perf recording could be
+ * opened for analysis on other platforms as well.
+ */
+#ifdef __aarch64__
+ "ARM64_FIQ",
+ "ARM64_DEBUG_HALT",
+ "ARM64_DEBUG_EXIT",
+ "ARM64_DEBUG_INST",
+ "ARM64_DEBUG_DATA"
+#else
+ "ARCH_1",
+ "ARCH_2",
+ "ARCH_3",
+ "ARCH_4",
+ "ARCH_5"
+#endif
+ };
+
+ if (new_type >= 0 && new_type < PERF_BR_NEW_MAX)
+ return branch_new_names[new_type];
+
+ return NULL;
+}
+
const char *branch_type_name(int type)
{
const char *branch_names[PERF_BR_MAX] = {
"COND_CALL",
"COND_RET",
"ERET",
- "IRQ"
+ "IRQ",
+ "SERROR",
+ "NO_TX",
+ "", // Needed for PERF_BR_EXTEND_ABI that ends up triggering some compiler warnings about NULL deref
};
if (type >= 0 && type < PERF_BR_MAX)
return NULL;
}
+const char *get_branch_type(struct branch_entry *e)
+{
+ if (e->flags.type == PERF_BR_UNKNOWN)
+ return "";
+
+ if (e->flags.type == PERF_BR_EXTEND_ABI)
+ return branch_new_type_name(e->flags.new_type);
+
+ return branch_type_name(e->flags.type);
+}
+
void branch_type_stat_display(FILE *fp, struct branch_type_stat *st)
{
u64 total = 0;
100.0 *
(double)st->counts[i] / (double)total);
}
+
+ for (i = 0; i < PERF_BR_NEW_MAX; i++) {
+ if (st->new_counts[i] > 0)
+ fprintf(fp, "\n%8s: %5.1f%%",
+ branch_new_type_name(i),
+ 100.0 *
+ (double)st->new_counts[i] / (double)total);
+ }
+
}
static int count_str_scnprintf(int idx, const char *str, char *bf, int size)
for (i = 0; i < PERF_BR_MAX; i++)
total += st->counts[i];
+ for (i = 0; i < PERF_BR_NEW_MAX; i++)
+ total += st->new_counts[i];
+
if (total == 0)
return 0;
printed += count_str_scnprintf(j++, branch_type_name(i), bf + printed, size - printed);
}
+ for (i = 0; i < PERF_BR_NEW_MAX; i++) {
+ if (st->new_counts[i] > 0)
+ printed += count_str_scnprintf(j++, branch_new_type_name(i), bf + printed, size - printed);
+ }
+
if (st->cross_4k > 0)
printed += count_str_scnprintf(j++, "CROSS_4K", bf + printed, size - printed);
u64 abort:1;
u64 cycles:16;
u64 type:4;
- u64 reserved:40;
+ u64 new_type:4;
+ u64 priv:3;
+ u64 reserved:33;
};
};
};
struct branch_type_stat {
bool branch_to;
u64 counts[PERF_BR_MAX];
+ u64 new_counts[PERF_BR_NEW_MAX];
u64 cond_fwd;
u64 cond_bwd;
u64 cross_4k;
u64 from, u64 to);
const char *branch_type_name(int type);
+const char *branch_new_type_name(int new_type);
+const char *get_branch_type(struct branch_entry *e);
void branch_type_stat_display(FILE *fp, struct branch_type_stat *st);
int branch_type_str(struct branch_type_stat *st, char *bf, int bfsize);
static bool dso__build_id_mismatch(struct dso *dso, const char *name)
{
struct build_id bid;
+ bool ret = false;
- if (filename__read_build_id_ns(name, &bid, dso->nsinfo) < 0)
- return false;
+ mutex_lock(&dso->lock);
+ if (filename__read_build_id_ns(name, &bid, dso->nsinfo) >= 0)
+ ret = !dso__build_id_equal(dso, &bid);
- return !dso__build_id_equal(dso, &bid);
+ mutex_unlock(&dso->lock);
+
+ return ret;
}
static int dso__cache_build_id(struct dso *dso, struct machine *machine,
if (!is_kallsyms && dso__build_id_mismatch(dso, name))
goto out_free;
+ mutex_lock(&dso->lock);
ret = build_id_cache__add_b(&dso->bid, name, dso->nsinfo,
is_kallsyms, is_vdso, proper_name, root_dir);
+ mutex_unlock(&dso->lock);
out_free:
free(allocated_name);
return ret;
static int count_pri64_printf(int idx, const char *str, u64 value, char *bf, int bfsize)
{
- int printed;
-
- printed = scnprintf(bf, bfsize, "%s%s:%" PRId64 "", (idx) ? " " : " (", str, value);
-
- return printed;
+ return scnprintf(bf, bfsize, "%s%s:%" PRId64 "", (idx) ? " " : " (", str, value);
}
static int count_float_printf(int idx, const char *str, float value,
char *bf, int bfsize, float threshold)
{
- int printed;
-
if (threshold != 0.0 && value < threshold)
return 0;
- printed = scnprintf(bf, bfsize, "%s%s:%.1f%%", (idx) ? " " : " (", str, value);
-
- return printed;
+ return scnprintf(bf, bfsize, "%s%s:%.1f%%", (idx) ? " " : " (", str, value);
}
static int branch_to_str(char *bf, int bfsize,
/* for communicating with external commands */
setenv("PERF_BUILDID_DIR", buildid_dir, 1);
}
+
+struct perf_config_scan_data {
+ const char *name;
+ const char *fmt;
+ va_list args;
+ int ret;
+};
+
+static int perf_config_scan_cb(const char *var, const char *value, void *data)
+{
+ struct perf_config_scan_data *d = data;
+
+ if (!strcmp(var, d->name))
+ d->ret = vsscanf(value, d->fmt, d->args);
+
+ return 0;
+}
+
+int perf_config_scan(const char *name, const char *fmt, ...)
+{
+ struct perf_config_scan_data d = {
+ .name = name,
+ .fmt = fmt,
+ };
+
+ va_start(d.args, fmt);
+ perf_config(perf_config_scan_cb, &d);
+ va_end(d.args);
+
+ return d.ret;
+}
int perf_default_config(const char *, const char *, void *);
int perf_config(config_fn_t fn, void *);
+int perf_config_scan(const char *name, const char *fmt, ...) __scanf(2, 3);
int perf_config_set(struct perf_config_set *set,
config_fn_t fn, void *data);
int perf_config_int(int *dest, const char *, const char *);
}
+static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data)
+{
+ struct perf_cpu_map *map;
+ unsigned int i = 0;
+
+ map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu -
+ data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu);
+ if (!map)
+ return NULL;
+
+ if (data->range_cpu_data.any_cpu)
+ map->map[i++].cpu = -1;
+
+ for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu;
+ i++, cpu++)
+ map->map[i].cpu = cpu;
+
+ return map;
+}
+
struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data)
{
- if (data->type == PERF_CPU_MAP__CPUS)
+ switch (data->type) {
+ case PERF_CPU_MAP__CPUS:
return cpu_map__from_entries(data);
- else
+ case PERF_CPU_MAP__MASK:
return cpu_map__from_mask(data);
+ case PERF_CPU_MAP__RANGE_CPUS:
+ return cpu_map__from_range(data);
+ default:
+ pr_err("cpu_map__new_data unknown type %d\n", data->type);
+ return NULL;
+ }
}
size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp)
else if (a->core != b->core)
return a->core - b->core;
else
- return a->thread - b->thread;
+ return a->thread_idx - b->thread_idx;
}
struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus,
bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b)
{
- return a->thread == b->thread &&
+ return a->thread_idx == b->thread_idx &&
a->node == b->node &&
a->socket == b->socket &&
a->die == b->die &&
bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a)
{
- return a->thread == -1 &&
+ return a->thread_idx == -1 &&
a->node == -1 &&
a->socket == -1 &&
a->die == -1 &&
struct aggr_cpu_id aggr_cpu_id__empty(void)
{
struct aggr_cpu_id ret = {
- .thread = -1,
+ .thread_idx = -1,
.node = -1,
.socket = -1,
.die = -1,
/** Identify where counts are aggregated, -1 implies not to aggregate. */
struct aggr_cpu_id {
/** A value in the range 0 to number of threads. */
- int thread;
+ int thread_idx;
/** The numa node X as read from /sys/devices/system/node/nodeX. */
int node;
/**
free(tp);
}
+bool cpu_topology__smt_on(const struct cpu_topology *topology)
+{
+ for (u32 i = 0; i < topology->core_cpus_lists; i++) {
+ const char *cpu_list = topology->core_cpus_list[i];
+
+ /*
+ * If there is a need to separate siblings in a core then SMT is
+ * enabled.
+ */
+ if (strchr(cpu_list, ',') || strchr(cpu_list, '-'))
+ return true;
+ }
+ return false;
+}
+
+bool cpu_topology__core_wide(const struct cpu_topology *topology,
+ const char *user_requested_cpu_list)
+{
+ struct perf_cpu_map *user_requested_cpus;
+
+ /*
+ * If user_requested_cpu_list is empty then all CPUs are recorded and so
+ * core_wide is true.
+ */
+ if (!user_requested_cpu_list)
+ return true;
+
+ user_requested_cpus = perf_cpu_map__new(user_requested_cpu_list);
+ /* Check that every user requested CPU is the complete set of SMT threads on a core. */
+ for (u32 i = 0; i < topology->core_cpus_lists; i++) {
+ const char *core_cpu_list = topology->core_cpus_list[i];
+ struct perf_cpu_map *core_cpus = perf_cpu_map__new(core_cpu_list);
+ struct perf_cpu cpu;
+ int idx;
+ bool has_first, first = true;
+
+ perf_cpu_map__for_each_cpu(cpu, idx, core_cpus) {
+ if (first) {
+ has_first = perf_cpu_map__has(user_requested_cpus, cpu);
+ first = false;
+ } else {
+ /*
+ * If the first core CPU is user requested then
+ * all subsequent CPUs in the core must be user
+ * requested too. If the first CPU isn't user
+ * requested then none of the others must be
+ * too.
+ */
+ if (perf_cpu_map__has(user_requested_cpus, cpu) != has_first) {
+ perf_cpu_map__put(core_cpus);
+ perf_cpu_map__put(user_requested_cpus);
+ return false;
+ }
+ }
+ }
+ perf_cpu_map__put(core_cpus);
+ }
+ perf_cpu_map__put(user_requested_cpus);
+ return true;
+}
+
static bool has_die_topology(void)
{
char filename[MAXPATHLEN];
struct cpu_topology *cpu_topology__new(void);
void cpu_topology__delete(struct cpu_topology *tp);
+/* Determine from the core list whether SMT was enabled. */
+bool cpu_topology__smt_on(const struct cpu_topology *topology);
+/* Are the sets of SMT siblings all enabled or all disabled in user_requested_cpus. */
+bool cpu_topology__core_wide(const struct cpu_topology *topology,
+ const char *user_requested_cpu_list);
struct numa_topology *numa_topology__new(void);
void numa_topology__delete(struct numa_topology *tp);
if (!name)
return -ENOMEM;
+ mutex_lock(&dso->lock);
if (machine)
root_dir = machine->root_dir;
unlink(name);
out:
+ mutex_unlock(&dso->lock);
free(name);
return fd;
}
int fd;
struct nscookie nsc;
- if (dso->binary_type != DSO_BINARY_TYPE__BUILD_ID_CACHE)
+ if (dso->binary_type != DSO_BINARY_TYPE__BUILD_ID_CACHE) {
+ mutex_lock(&dso->lock);
nsinfo__mountns_enter(dso->nsinfo, &nsc);
+ mutex_unlock(&dso->lock);
+ }
fd = __open_dso(dso, machine);
if (dso->binary_type != DSO_BINARY_TYPE__BUILD_ID_CACHE)
nsinfo__mountns_exit(&nsc);
struct rb_root *root = &dso->data.cache;
struct rb_node *next = rb_first(root);
- pthread_mutex_lock(&dso->lock);
+ mutex_lock(&dso->lock);
while (next) {
struct dso_cache *cache;
rb_erase(&cache->rb_node, root);
free(cache);
}
- pthread_mutex_unlock(&dso->lock);
+ mutex_unlock(&dso->lock);
}
static struct dso_cache *__dso_cache__find(struct dso *dso, u64 offset)
struct dso_cache *cache;
u64 offset = new->offset;
- pthread_mutex_lock(&dso->lock);
+ mutex_lock(&dso->lock);
while (*p != NULL) {
u64 end;
cache = NULL;
out:
- pthread_mutex_unlock(&dso->lock);
+ mutex_unlock(&dso->lock);
return cache;
}
dso->root = NULL;
INIT_LIST_HEAD(&dso->node);
INIT_LIST_HEAD(&dso->data.open_entry);
- pthread_mutex_init(&dso->lock, NULL);
+ mutex_init(&dso->lock);
refcount_set(&dso->refcnt, 1);
}
dso__free_a2l(dso);
zfree(&dso->symsrc_filename);
nsinfo__zput(dso->nsinfo);
- pthread_mutex_destroy(&dso->lock);
+ mutex_destroy(&dso->lock);
free(dso);
}
#ifndef __PERF_DSO
#define __PERF_DSO
-#include <pthread.h>
#include <linux/refcount.h>
#include <linux/types.h>
#include <linux/rbtree.h>
#include <stdio.h>
#include <linux/bitops.h>
#include "build-id.h"
+#include "mutex.h"
struct machine;
struct map;
struct auxtrace_cache;
struct dso {
- pthread_mutex_t lock;
+ struct mutex lock;
struct list_head node;
struct rb_node rb_node; /* rbtree node sorted by long name */
struct rb_root *root; /* root of rbtree that rb_node is in */
u64 total_non_filtered_period;
u32 nr_samples;
u32 nr_non_filtered_samples;
+ u32 nr_lost_samples;
};
void events_stats__inc(struct events_stats *stats, u32 type);
#include "target.h"
#include "evlist.h"
#include "evsel.h"
+#include "record.h"
#include "debug.h"
#include "units.h"
#include "bpf_counter.h"
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/prctl.h>
+#include <sys/timerfd.h>
#include <linux/bitops.h>
#include <linux/hash.h>
#include <linux/log2.h>
#include <linux/err.h>
#include <linux/string.h>
+#include <linux/time64.h>
#include <linux/zalloc.h>
#include <perf/evlist.h>
#include <perf/evsel.h>
void evlist__exit(struct evlist *evlist)
{
+ event_enable_timer__exit(&evlist->eet);
zfree(&evlist->mmap);
zfree(&evlist->overwrite_mmap);
perf_evlist__exit(&evlist->core);
return 0;
}
-static void evlist__add_on_all_cpus(struct evlist *evlist, struct evsel *evsel)
-{
- evsel->core.system_wide = true;
-
- /*
- * All CPUs.
- *
- * Note perf_event_open() does not accept CPUs that are not online, so
- * in fact this CPU list will include only all online CPUs.
- */
- perf_cpu_map__put(evsel->core.own_cpus);
- evsel->core.own_cpus = perf_cpu_map__new(NULL);
- perf_cpu_map__put(evsel->core.cpus);
- evsel->core.cpus = perf_cpu_map__get(evsel->core.own_cpus);
-
- /* No threads */
- perf_thread_map__put(evsel->core.threads);
- evsel->core.threads = perf_thread_map__new_dummy();
-
- evlist__add(evlist, evsel);
-}
-
struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide)
{
struct evsel *evsel = evlist__dummy_event(evlist);
evsel->core.attr.exclude_hv = 1;
evsel->core.attr.freq = 0;
evsel->core.attr.sample_period = 1;
+ evsel->core.system_wide = system_wide;
evsel->no_aux_samples = true;
evsel->name = strdup("dummy:u");
- if (system_wide)
- evlist__add_on_all_cpus(evlist, evsel);
- else
- evlist__add(evlist, evsel);
-
+ evlist__add(evlist, evsel);
return evsel;
}
+struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide)
+{
+ struct evsel *evsel = evsel__newtp_idx("sched", "sched_switch", 0);
+
+ if (IS_ERR(evsel))
+ return evsel;
+
+ evsel__set_sample_bit(evsel, CPU);
+ evsel__set_sample_bit(evsel, TIME);
+
+ evsel->core.system_wide = system_wide;
+ evsel->no_aux_samples = true;
+
+ evlist__add(evlist, evsel);
+ return evsel;
+};
+
int evlist__add_attrs(struct evlist *evlist, struct perf_event_attr *attrs, size_t nr_attrs)
{
struct evsel *evsel, *n;
return false;
}
-static void __evlist__disable(struct evlist *evlist, char *evsel_name)
+static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl_dummy)
{
struct evsel *pos;
struct evlist_cpu_iterator evlist_cpu_itr;
continue;
if (pos->disabled || !evsel__is_group_leader(pos) || !pos->core.fd)
continue;
+ if (excl_dummy && evsel__is_dummy_event(pos))
+ continue;
if (pos->immediate)
has_imm = true;
if (pos->immediate != imm)
continue;
if (!evsel__is_group_leader(pos) || !pos->core.fd)
continue;
+ if (excl_dummy && evsel__is_dummy_event(pos))
+ continue;
pos->disabled = true;
}
void evlist__disable(struct evlist *evlist)
{
- __evlist__disable(evlist, NULL);
+ __evlist__disable(evlist, NULL, false);
+}
+
+void evlist__disable_non_dummy(struct evlist *evlist)
+{
+ __evlist__disable(evlist, NULL, true);
}
void evlist__disable_evsel(struct evlist *evlist, char *evsel_name)
{
- __evlist__disable(evlist, evsel_name);
+ __evlist__disable(evlist, evsel_name, false);
}
-static void __evlist__enable(struct evlist *evlist, char *evsel_name)
+static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_dummy)
{
struct evsel *pos;
struct evlist_cpu_iterator evlist_cpu_itr;
continue;
if (!evsel__is_group_leader(pos) || !pos->core.fd)
continue;
+ if (excl_dummy && evsel__is_dummy_event(pos))
+ continue;
evsel__enable_cpu(pos, evlist_cpu_itr.cpu_map_idx);
}
affinity__cleanup(affinity);
continue;
if (!evsel__is_group_leader(pos) || !pos->core.fd)
continue;
+ if (excl_dummy && evsel__is_dummy_event(pos))
+ continue;
pos->disabled = false;
}
void evlist__enable(struct evlist *evlist)
{
- __evlist__enable(evlist, NULL);
+ __evlist__enable(evlist, NULL, false);
+}
+
+void evlist__enable_non_dummy(struct evlist *evlist)
+{
+ __evlist__enable(evlist, NULL, true);
}
void evlist__enable_evsel(struct evlist *evlist, char *evsel_name)
{
- __evlist__enable(evlist, evsel_name);
+ __evlist__enable(evlist, evsel_name, false);
}
void evlist__toggle_enable(struct evlist *evlist)
int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd)
{
return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN,
- fdarray_flag__nonfilterable);
+ fdarray_flag__nonfilterable |
+ fdarray_flag__non_perf_event);
}
#endif
}
evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN,
- fdarray_flag__nonfilterable);
+ fdarray_flag__nonfilterable |
+ fdarray_flag__non_perf_event);
if (evlist->ctl_fd.pos < 0) {
evlist->ctl_fd.pos = -1;
pr_err("Failed to add ctl fd entry: %m\n");
return err;
}
-int evlist__ctlfd_update(struct evlist *evlist, struct pollfd *update)
+/**
+ * struct event_enable_time - perf record -D/--delay single time range.
+ * @start: start of time range to enable events in milliseconds
+ * @end: end of time range to enable events in milliseconds
+ *
+ * N.B. this structure is also accessed as an array of int.
+ */
+struct event_enable_time {
+ int start;
+ int end;
+};
+
+static int parse_event_enable_time(const char *str, struct event_enable_time *range, bool first)
{
- int ctlfd_pos = evlist->ctl_fd.pos;
- struct pollfd *entries = evlist->core.pollfd.entries;
+ const char *fmt = first ? "%u - %u %n" : " , %u - %u %n";
+ int ret, start, end, n;
- if (!evlist__ctlfd_initialized(evlist))
+ ret = sscanf(str, fmt, &start, &end, &n);
+ if (ret != 2 || end <= start)
+ return -EINVAL;
+ if (range) {
+ range->start = start;
+ range->end = end;
+ }
+ return n;
+}
+
+static ssize_t parse_event_enable_times(const char *str, struct event_enable_time *range)
+{
+ int incr = !!range;
+ bool first = true;
+ ssize_t ret, cnt;
+
+ for (cnt = 0; *str; cnt++) {
+ ret = parse_event_enable_time(str, range, first);
+ if (ret < 0)
+ return ret;
+ /* Check no overlap */
+ if (!first && range && range->start <= range[-1].end)
+ return -EINVAL;
+ str += ret;
+ range += incr;
+ first = false;
+ }
+ return cnt;
+}
+
+/**
+ * struct event_enable_timer - control structure for perf record -D/--delay.
+ * @evlist: event list
+ * @times: time ranges that events are enabled (N.B. this is also accessed as an
+ * array of int)
+ * @times_cnt: number of time ranges
+ * @timerfd: timer file descriptor
+ * @pollfd_pos: position in @evlist array of file descriptors to poll (fdarray)
+ * @times_step: current position in (int *)@times)[],
+ * refer event_enable_timer__process()
+ *
+ * Note, this structure is only used when there are time ranges, not when there
+ * is only an initial delay.
+ */
+struct event_enable_timer {
+ struct evlist *evlist;
+ struct event_enable_time *times;
+ size_t times_cnt;
+ int timerfd;
+ int pollfd_pos;
+ size_t times_step;
+};
+
+static int str_to_delay(const char *str)
+{
+ char *endptr;
+ long d;
+
+ d = strtol(str, &endptr, 10);
+ if (*endptr || d > INT_MAX || d < -1)
return 0;
+ return d;
+}
- if (entries[ctlfd_pos].fd != update->fd ||
- entries[ctlfd_pos].events != update->events)
- return -1;
+int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts,
+ const char *str, int unset)
+{
+ enum fdarray_flags flags = fdarray_flag__nonfilterable | fdarray_flag__non_perf_event;
+ struct event_enable_timer *eet;
+ ssize_t times_cnt;
+ ssize_t ret;
+ int err;
+
+ if (unset)
+ return 0;
+
+ opts->initial_delay = str_to_delay(str);
+ if (opts->initial_delay)
+ return 0;
+
+ ret = parse_event_enable_times(str, NULL);
+ if (ret < 0)
+ return ret;
+
+ times_cnt = ret;
+ if (times_cnt == 0)
+ return -EINVAL;
+
+ eet = zalloc(sizeof(*eet));
+ if (!eet)
+ return -ENOMEM;
+
+ eet->times = calloc(times_cnt, sizeof(*eet->times));
+ if (!eet->times) {
+ err = -ENOMEM;
+ goto free_eet;
+ }
+
+ if (parse_event_enable_times(str, eet->times) != times_cnt) {
+ err = -EINVAL;
+ goto free_eet_times;
+ }
+
+ eet->times_cnt = times_cnt;
+
+ eet->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC);
+ if (eet->timerfd == -1) {
+ err = -errno;
+ pr_err("timerfd_create failed: %s\n", strerror(errno));
+ goto free_eet_times;
+ }
+
+ eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags);
+ if (eet->pollfd_pos < 0) {
+ err = eet->pollfd_pos;
+ goto close_timerfd;
+ }
+
+ eet->evlist = evlist;
+ evlist->eet = eet;
+ opts->initial_delay = eet->times[0].start;
- entries[ctlfd_pos].revents = update->revents;
return 0;
+
+close_timerfd:
+ close(eet->timerfd);
+free_eet_times:
+ free(eet->times);
+free_eet:
+ free(eet);
+ return err;
+}
+
+static int event_enable_timer__set_timer(struct event_enable_timer *eet, int ms)
+{
+ struct itimerspec its = {
+ .it_value.tv_sec = ms / MSEC_PER_SEC,
+ .it_value.tv_nsec = (ms % MSEC_PER_SEC) * NSEC_PER_MSEC,
+ };
+ int err = 0;
+
+ if (timerfd_settime(eet->timerfd, 0, &its, NULL) < 0) {
+ err = -errno;
+ pr_err("timerfd_settime failed: %s\n", strerror(errno));
+ }
+ return err;
+}
+
+int event_enable_timer__start(struct event_enable_timer *eet)
+{
+ int ms;
+
+ if (!eet)
+ return 0;
+
+ ms = eet->times[0].end - eet->times[0].start;
+ eet->times_step = 1;
+
+ return event_enable_timer__set_timer(eet, ms);
+}
+
+int event_enable_timer__process(struct event_enable_timer *eet)
+{
+ struct pollfd *entries;
+ short revents;
+
+ if (!eet)
+ return 0;
+
+ entries = eet->evlist->core.pollfd.entries;
+ revents = entries[eet->pollfd_pos].revents;
+ entries[eet->pollfd_pos].revents = 0;
+
+ if (revents & POLLIN) {
+ size_t step = eet->times_step;
+ size_t pos = step / 2;
+
+ if (step & 1) {
+ evlist__disable_non_dummy(eet->evlist);
+ pr_info(EVLIST_DISABLED_MSG);
+ if (pos >= eet->times_cnt - 1) {
+ /* Disarm timer */
+ event_enable_timer__set_timer(eet, 0);
+ return 1; /* Stop */
+ }
+ } else {
+ evlist__enable_non_dummy(eet->evlist);
+ pr_info(EVLIST_ENABLED_MSG);
+ }
+
+ step += 1;
+ pos = step / 2;
+
+ if (pos < eet->times_cnt) {
+ int *times = (int *)eet->times; /* Accessing 'times' as array of int */
+ int ms = times[step] - times[step - 1];
+
+ eet->times_step = step;
+ return event_enable_timer__set_timer(eet, ms);
+ }
+ }
+
+ return 0;
+}
+
+void event_enable_timer__exit(struct event_enable_timer **ep)
+{
+ if (!ep || !*ep)
+ return;
+ free((*ep)->times);
+ zfree(ep);
}
struct evsel *evlist__find_evsel(struct evlist *evlist, int idx)
BKW_MMAP_EMPTY,
};
+struct event_enable_timer;
+
struct evlist {
struct perf_evlist core;
bool enabled;
int ack; /* ack file descriptor for control commands */
int pos; /* index at evlist core object to check signals */
} ctl_fd;
+ struct event_enable_timer *eet;
};
struct evsel_str_handler {
{
return evlist__add_aux_dummy(evlist, true);
}
+struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide);
int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr,
evsel__sb_cb_t cb, void *data);
void evlist__toggle_enable(struct evlist *evlist);
void evlist__disable_evsel(struct evlist *evlist, char *evsel_name);
void evlist__enable_evsel(struct evlist *evlist, char *evsel_name);
+void evlist__disable_non_dummy(struct evlist *evlist);
+void evlist__enable_non_dummy(struct evlist *evlist);
void evlist__set_selected(struct evlist *evlist, struct evsel *evsel);
int evlist__initialize_ctlfd(struct evlist *evlist, int ctl_fd, int ctl_fd_ack);
int evlist__finalize_ctlfd(struct evlist *evlist);
bool evlist__ctlfd_initialized(struct evlist *evlist);
-int evlist__ctlfd_update(struct evlist *evlist, struct pollfd *update);
int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd);
int evlist__ctlfd_ack(struct evlist *evlist);
#define EVLIST_ENABLED_MSG "Events enabled\n"
#define EVLIST_DISABLED_MSG "Events disabled\n"
+int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts,
+ const char *str, int unset);
+int event_enable_timer__start(struct event_enable_timer *eet);
+void event_enable_timer__exit(struct event_enable_timer **ep);
+int event_enable_timer__process(struct event_enable_timer *eet);
+
struct evsel *evlist__find_evsel(struct evlist *evlist, int idx);
int evlist__scnprintf_evsels(struct evlist *evlist, size_t size, char *bf);
#include "string2.h"
#include "memswap.h"
#include "util.h"
-#include "hashmap.h"
+#ifdef HAVE_LIBBPF_SUPPORT
+#include <bpf/hashmap.h>
+#else
+#include "util/hashmap.h"
+#endif
#include "pmu-hybrid.h"
#include "off_cpu.h"
#include "../perf-sys.h"
attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
attr->inherit = !opts->no_inherit;
attr->write_backward = opts->overwrite ? 1 : 0;
+ attr->read_format = PERF_FORMAT_LOST;
evsel__set_sample_bit(evsel, IP);
evsel__set_sample_bit(evsel, TID);
static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
struct perf_thread_map *threads)
{
- int nthreads;
+ int nthreads = perf_thread_map__nr(threads);
if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
(perf_missing_features.aux_output && evsel->core.attr.aux_output))
threads = empty_thread_map;
}
- if (evsel->core.system_wide)
- nthreads = 1;
- else
- nthreads = threads->nr;
-
if (evsel->core.fd == NULL &&
perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0)
return -ENOMEM;
static void evsel__disable_missing_features(struct evsel *evsel)
{
+ if (perf_missing_features.read_lost)
+ evsel->core.attr.read_format &= ~PERF_FORMAT_LOST;
if (perf_missing_features.weight_struct) {
evsel__set_sample_bit(evsel, WEIGHT);
evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
* Must probe features in the order they were added to the
* perf_event_attr interface.
*/
- if (!perf_missing_features.weight_struct &&
+ if (!perf_missing_features.read_lost &&
+ (evsel->core.attr.read_format & PERF_FORMAT_LOST)) {
+ perf_missing_features.read_lost = true;
+ pr_debug2("switching off PERF_FORMAT_LOST support\n");
+ return true;
+ } else if (!perf_missing_features.weight_struct &&
(evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
perf_missing_features.weight_struct = true;
pr_debug2("switching off weight struct support\n");
if (threads == NULL)
threads = empty_thread_map;
- if (evsel->core.system_wide)
- nthreads = 1;
- else
- nthreads = threads->nr;
+ nthreads = perf_thread_map__nr(threads);
if (evsel->cgrp)
pid = evsel->cgrp->fd;
test_attr__ready();
+ /* Debug message used by test scripts */
pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx",
pid, perf_cpu_map__cpu(cpus, idx).cpu, group_fd, evsel->open_flags);
fd, group_fd, evsel->open_flags);
}
+ /* Debug message used by test scripts */
pr_debug2_peo(" = %d\n", fd);
if (evsel->bpf_fd >= 0) {
bool data_page_size;
bool code_page_size;
bool weight_struct;
+ bool read_lost;
};
extern struct perf_missing_features perf_missing_features;
{
struct expr_id_data *data_ptr = NULL, *old_data = NULL;
char *old_key = NULL;
- char *name, *p;
+ char *name;
int ret;
data_ptr = zalloc(sizeof(*data_ptr));
return -ENOMEM;
}
- /*
- * The jevents tool converts all metric expressions
- * to lowercase, including metric references, hence
- * we need to add lowercase name for metric, so it's
- * properly found.
- */
- for (p = name; *p; p++)
- *p = tolower(*p);
-
/*
* Intentionally passing just const char pointers,
* originally from 'struct pmu_event' object.
free(ctx);
return NULL;
}
- ctx->runtime = 0;
+ ctx->sctx.user_requested_cpu_list = NULL;
+ ctx->sctx.runtime = 0;
+ ctx->sctx.system_wide = false;
return ctx;
}
struct hashmap_entry *cur;
size_t bkt;
+ if (!ctx)
+ return;
+
+ free(ctx->sctx.user_requested_cpu_list);
hashmap__for_each_entry(ctx->ids, cur, bkt) {
free((char *)cur->key);
free(cur->value);
__expr__parse(double *val, struct expr_parse_ctx *ctx, const char *expr,
bool compute_ids)
{
- struct expr_scanner_ctx scanner_ctx = {
- .runtime = ctx->runtime,
- };
YY_BUFFER_STATE buffer;
void *scanner;
int ret;
pr_debug2("parsing metric: %s\n", expr);
- ret = expr_lex_init_extra(&scanner_ctx, &scanner);
+ ret = expr_lex_init_extra(&ctx->sctx, &scanner);
if (ret)
return ret;
}
#endif
-double expr__get_literal(const char *literal)
+double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx)
{
static struct cpu_topology *topology;
double result = NAN;
- if (!strcasecmp("#smt_on", literal)) {
- result = smt_on() > 0 ? 1.0 : 0.0;
- goto out;
- }
-
if (!strcmp("#num_cpus", literal)) {
result = cpu__max_present_cpu().cpu;
goto out;
goto out;
}
}
+ if (!strcasecmp("#smt_on", literal)) {
+ result = smt_on(topology) ? 1.0 : 0.0;
+ goto out;
+ }
+ if (!strcmp("#core_wide", literal)) {
+ result = core_wide(ctx->system_wide, ctx->user_requested_cpu_list, topology)
+ ? 1.0 : 0.0;
+ goto out;
+ }
if (!strcmp("#num_packages", literal)) {
result = topology->package_cpus_lists;
goto out;
#ifndef PARSE_CTX_H
#define PARSE_CTX_H 1
-// There are fixes that need to land upstream before we can use libbpf's headers,
-// for now use our copy unconditionally, since the data structures at this point
-// are exactly the same, no problem.
-//#ifdef HAVE_LIBBPF_SUPPORT
-//#include <bpf/hashmap.h>
-//#else
+#ifdef HAVE_LIBBPF_SUPPORT
+#include <bpf/hashmap.h>
+#else
#include "util/hashmap.h"
-//#endif
+#endif
struct metric_ref;
+struct expr_scanner_ctx {
+ char *user_requested_cpu_list;
+ int runtime;
+ bool system_wide;
+};
+
struct expr_parse_ctx {
struct hashmap *ids;
- int runtime;
+ struct expr_scanner_ctx sctx;
};
struct expr_id_data;
-struct expr_scanner_ctx {
- int runtime;
-};
-
struct hashmap *ids__new(void);
void ids__free(struct hashmap *ids);
int ids__insert(struct hashmap *ids, const char *id);
double expr_id_data__value(const struct expr_id_data *data);
double expr_id_data__source_count(const struct expr_id_data *data);
-double expr__get_literal(const char *literal);
+double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx);
#endif
return token;
}
-static int literal(yyscan_t scanner)
+static int literal(yyscan_t scanner, const struct expr_scanner_ctx *sctx)
{
YYSTYPE *yylval = expr_get_lval(scanner);
- yylval->num = expr__get_literal(expr_get_text(scanner));
+ yylval->num = expr__get_literal(expr_get_text(scanner), sctx);
if (isnan(yylval->num))
return EXPR_ERROR;
if { return IF; }
else { return ELSE; }
source_count { return SOURCE_COUNT; }
-{literal} { return literal(yyscanner); }
+{literal} { return literal(yyscanner, sctx); }
{number} { return value(yyscanner); }
{symbol} { return str(yyscanner, ID, sctx->runtime); }
"|" { return '|'; }
}
;
-if_expr: expr IF expr ELSE expr
+if_expr: expr IF expr ELSE if_expr
{
if (fpclassify($3.val) == FP_ZERO) {
/*
eh_frame_base_offset);
if (retval)
goto error;
+ retval = -1;
}
/*
#ifndef __GENELF_H__
#define __GENELF_H__
+#include <linux/math.h>
+
/* genelf.c */
int jit_write_elf(int fd, uint64_t code_addr, const char *sym,
const void *code, int csize, void *debug, int nr_debug_entries,
#endif
/* The .text section is directly after the ELF header */
-#define GEN_ELF_TEXT_OFFSET sizeof(Elf_Ehdr)
+#define GEN_ELF_TEXT_OFFSET round_up(sizeof(Elf_Ehdr) + sizeof(Elf_Phdr), 16)
#endif
size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp)
{
struct perf_record_event_update *ev = &event->event_update;
- struct perf_record_event_update_scale *ev_scale;
- struct perf_record_event_update_cpus *ev_cpus;
struct perf_cpu_map *map;
size_t ret;
switch (ev->type) {
case PERF_EVENT_UPDATE__SCALE:
- ev_scale = (struct perf_record_event_update_scale *)ev->data;
- ret += fprintf(fp, "... scale: %f\n", ev_scale->scale);
+ ret += fprintf(fp, "... scale: %f\n", ev->scale.scale);
break;
case PERF_EVENT_UPDATE__UNIT:
- ret += fprintf(fp, "... unit: %s\n", ev->data);
+ ret += fprintf(fp, "... unit: %s\n", ev->unit);
break;
case PERF_EVENT_UPDATE__NAME:
- ret += fprintf(fp, "... name: %s\n", ev->data);
+ ret += fprintf(fp, "... name: %s\n", ev->name);
break;
case PERF_EVENT_UPDATE__CPUS:
- ev_cpus = (struct perf_record_event_update_cpus *)ev->data;
ret += fprintf(fp, "... ");
- map = cpu_map__new_data(&ev_cpus->cpus);
+ map = cpu_map__new_data(&ev->cpus.cpus);
if (map)
ret += cpu_map__fprintf(map, fp);
else
struct evlist **pevlist)
{
struct perf_record_event_update *ev = &event->event_update;
- struct perf_record_event_update_scale *ev_scale;
- struct perf_record_event_update_cpus *ev_cpus;
struct evlist *evlist;
struct evsel *evsel;
struct perf_cpu_map *map;
switch (ev->type) {
case PERF_EVENT_UPDATE__UNIT:
free((char *)evsel->unit);
- evsel->unit = strdup(ev->data);
+ evsel->unit = strdup(ev->unit);
break;
case PERF_EVENT_UPDATE__NAME:
free(evsel->name);
- evsel->name = strdup(ev->data);
+ evsel->name = strdup(ev->name);
break;
case PERF_EVENT_UPDATE__SCALE:
- ev_scale = (struct perf_record_event_update_scale *)ev->data;
- evsel->scale = ev_scale->scale;
+ evsel->scale = ev->scale.scale;
break;
case PERF_EVENT_UPDATE__CPUS:
- ev_cpus = (struct perf_record_event_update_cpus *)ev->data;
- map = cpu_map__new_data(&ev_cpus->cpus);
+ map = cpu_map__new_data(&ev->cpus.cpus);
if (map) {
perf_cpu_map__put(evsel->core.own_cpus);
evsel->core.own_cpus = map;
--- /dev/null
+perf-$(CONFIG_AUXTRACE) += hisi-ptt-pkt-decoder.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * HiSilicon PCIe Trace and Tuning (PTT) support
+ * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <endian.h>
+#include <byteswap.h>
+#include <linux/bitops.h>
+#include <stdarg.h>
+
+#include "../color.h"
+#include "hisi-ptt-pkt-decoder.h"
+
+/*
+ * For 8DW format, the bit[31:11] of DW0 is always 0x1fffff, which can be
+ * used to distinguish the data format.
+ * 8DW format is like:
+ * bits [ 31:11 ][ 10:0 ]
+ * |---------------------------------------|-------------------|
+ * DW0 [ 0x1fffff ][ Reserved (0x7ff) ]
+ * DW1 [ Prefix ]
+ * DW2 [ Header DW0 ]
+ * DW3 [ Header DW1 ]
+ * DW4 [ Header DW2 ]
+ * DW5 [ Header DW3 ]
+ * DW6 [ Reserved (0x0) ]
+ * DW7 [ Time ]
+ *
+ * 4DW format is like:
+ * bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ]
+ * |-----|---------|---|---|---|---|-------------|-------------|
+ * DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ]
+ * DW1 [ Header DW1 ]
+ * DW2 [ Header DW2 ]
+ * DW3 [ Header DW3 ]
+ */
+
+enum hisi_ptt_8dw_pkt_field_type {
+ HISI_PTT_8DW_CHK_AND_RSV0,
+ HISI_PTT_8DW_PREFIX,
+ HISI_PTT_8DW_HEAD0,
+ HISI_PTT_8DW_HEAD1,
+ HISI_PTT_8DW_HEAD2,
+ HISI_PTT_8DW_HEAD3,
+ HISI_PTT_8DW_RSV1,
+ HISI_PTT_8DW_TIME,
+ HISI_PTT_8DW_TYPE_MAX
+};
+
+enum hisi_ptt_4dw_pkt_field_type {
+ HISI_PTT_4DW_HEAD1,
+ HISI_PTT_4DW_HEAD2,
+ HISI_PTT_4DW_HEAD3,
+ HISI_PTT_4DW_TYPE_MAX
+};
+
+static const char * const hisi_ptt_8dw_pkt_field_name[] = {
+ [HISI_PTT_8DW_PREFIX] = "Prefix",
+ [HISI_PTT_8DW_HEAD0] = "Header DW0",
+ [HISI_PTT_8DW_HEAD1] = "Header DW1",
+ [HISI_PTT_8DW_HEAD2] = "Header DW2",
+ [HISI_PTT_8DW_HEAD3] = "Header DW3",
+ [HISI_PTT_8DW_TIME] = "Time"
+};
+
+static const char * const hisi_ptt_4dw_pkt_field_name[] = {
+ [HISI_PTT_4DW_HEAD1] = "Header DW1",
+ [HISI_PTT_4DW_HEAD2] = "Header DW2",
+ [HISI_PTT_4DW_HEAD3] = "Header DW3",
+};
+
+union hisi_ptt_4dw {
+ struct {
+ uint32_t format : 2;
+ uint32_t type : 5;
+ uint32_t t9 : 1;
+ uint32_t t8 : 1;
+ uint32_t th : 1;
+ uint32_t so : 1;
+ uint32_t len : 10;
+ uint32_t time : 11;
+ };
+ uint32_t value;
+};
+
+static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc)
+{
+ const char *color = PERF_COLOR_BLUE;
+ int i;
+
+ printf(".");
+ color_fprintf(stdout, color, " %08x: ", pos);
+ for (i = 0; i < HISI_PTT_FIELD_LENTH; i++)
+ color_fprintf(stdout, color, "%02x ", buf[pos + i]);
+ for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++)
+ color_fprintf(stdout, color, " ");
+ color_fprintf(stdout, color, " %s\n", desc);
+}
+
+static int hisi_ptt_8dw_kpt_desc(const unsigned char *buf, int pos)
+{
+ int i;
+
+ for (i = 0; i < HISI_PTT_8DW_TYPE_MAX; i++) {
+ /* Do not show 8DW check field and reserved fields */
+ if (i == HISI_PTT_8DW_CHK_AND_RSV0 || i == HISI_PTT_8DW_RSV1) {
+ pos += HISI_PTT_FIELD_LENTH;
+ continue;
+ }
+
+ hisi_ptt_print_pkt(buf, pos, hisi_ptt_8dw_pkt_field_name[i]);
+ pos += HISI_PTT_FIELD_LENTH;
+ }
+
+ return hisi_ptt_pkt_size[HISI_PTT_8DW_PKT];
+}
+
+static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos)
+{
+ const char *color = PERF_COLOR_BLUE;
+ union hisi_ptt_4dw dw0;
+ int i;
+
+ dw0.value = *(uint32_t *)(buf + pos);
+ printf(".");
+ color_fprintf(stdout, color, " %08x: ", pos);
+ for (i = 0; i < HISI_PTT_FIELD_LENTH; i++)
+ color_fprintf(stdout, color, "%02x ", buf[pos + i]);
+ for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++)
+ color_fprintf(stdout, color, " ");
+
+ color_fprintf(stdout, color,
+ " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n",
+ "Format", dw0.format, "Type", dw0.type, "T9", dw0.t9,
+ "T8", dw0.t8, "TH", dw0.th, "SO", dw0.so, "Length",
+ dw0.len, "Time", dw0.time);
+}
+
+static int hisi_ptt_4dw_kpt_desc(const unsigned char *buf, int pos)
+{
+ int i;
+
+ hisi_ptt_4dw_print_dw0(buf, pos);
+ pos += HISI_PTT_FIELD_LENTH;
+
+ for (i = 0; i < HISI_PTT_4DW_TYPE_MAX; i++) {
+ hisi_ptt_print_pkt(buf, pos, hisi_ptt_4dw_pkt_field_name[i]);
+ pos += HISI_PTT_FIELD_LENTH;
+ }
+
+ return hisi_ptt_pkt_size[HISI_PTT_4DW_PKT];
+}
+
+int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type)
+{
+ if (type == HISI_PTT_8DW_PKT)
+ return hisi_ptt_8dw_kpt_desc(buf, pos);
+
+ return hisi_ptt_4dw_kpt_desc(buf, pos);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * HiSilicon PCIe Trace and Tuning (PTT) support
+ * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
+ */
+
+#ifndef INCLUDE__HISI_PTT_PKT_DECODER_H__
+#define INCLUDE__HISI_PTT_PKT_DECODER_H__
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11)
+#define HISI_PTT_IS_8DW_PKT GENMASK(31, 11)
+#define HISI_PTT_MAX_SPACE_LEN 10
+#define HISI_PTT_FIELD_LENTH 4
+
+enum hisi_ptt_pkt_type {
+ HISI_PTT_4DW_PKT,
+ HISI_PTT_8DW_PKT,
+ HISI_PTT_PKT_MAX
+};
+
+static int hisi_ptt_pkt_size[] = {
+ [HISI_PTT_4DW_PKT] = 16,
+ [HISI_PTT_8DW_PKT] = 32,
+};
+
+int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type);
+
+#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * HiSilicon PCIe Trace and Tuning (PTT) support
+ * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
+ */
+
+#include <byteswap.h>
+#include <endian.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/log2.h>
+#include <linux/types.h>
+#include <linux/zalloc.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "auxtrace.h"
+#include "color.h"
+#include "debug.h"
+#include "evsel.h"
+#include "hisi-ptt.h"
+#include "hisi-ptt-decoder/hisi-ptt-pkt-decoder.h"
+#include "machine.h"
+#include "session.h"
+#include "tool.h"
+#include <internal/lib.h>
+
+struct hisi_ptt {
+ struct auxtrace auxtrace;
+ u32 auxtrace_type;
+ struct perf_session *session;
+ struct machine *machine;
+ u32 pmu_type;
+};
+
+struct hisi_ptt_queue {
+ struct hisi_ptt *ptt;
+ struct auxtrace_buffer *buffer;
+};
+
+static enum hisi_ptt_pkt_type hisi_ptt_check_packet_type(unsigned char *buf)
+{
+ uint32_t head = *(uint32_t *)buf;
+
+ if ((HISI_PTT_8DW_CHECK_MASK & head) == HISI_PTT_IS_8DW_PKT)
+ return HISI_PTT_8DW_PKT;
+
+ return HISI_PTT_4DW_PKT;
+}
+
+static void hisi_ptt_dump(struct hisi_ptt *ptt __maybe_unused,
+ unsigned char *buf, size_t len)
+{
+ const char *color = PERF_COLOR_BLUE;
+ enum hisi_ptt_pkt_type type;
+ size_t pos = 0;
+ int pkt_len;
+
+ type = hisi_ptt_check_packet_type(buf);
+ len = round_down(len, hisi_ptt_pkt_size[type]);
+ color_fprintf(stdout, color, ". ... HISI PTT data: size %zu bytes\n",
+ len);
+
+ while (len > 0) {
+ pkt_len = hisi_ptt_pkt_desc(buf, pos, type);
+ if (!pkt_len)
+ color_fprintf(stdout, color, " Bad packet!\n");
+
+ pos += pkt_len;
+ len -= pkt_len;
+ }
+}
+
+static void hisi_ptt_dump_event(struct hisi_ptt *ptt, unsigned char *buf,
+ size_t len)
+{
+ printf(".\n");
+
+ hisi_ptt_dump(ptt, buf, len);
+}
+
+static int hisi_ptt_process_event(struct perf_session *session __maybe_unused,
+ union perf_event *event __maybe_unused,
+ struct perf_sample *sample __maybe_unused,
+ struct perf_tool *tool __maybe_unused)
+{
+ return 0;
+}
+
+static int hisi_ptt_process_auxtrace_event(struct perf_session *session,
+ union perf_event *event,
+ struct perf_tool *tool __maybe_unused)
+{
+ struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt,
+ auxtrace);
+ int fd = perf_data__fd(session->data);
+ int size = event->auxtrace.size;
+ void *data = malloc(size);
+ off_t data_offset;
+ int err;
+
+ if (!data)
+ return -errno;
+
+ if (perf_data__is_pipe(session->data)) {
+ data_offset = 0;
+ } else {
+ data_offset = lseek(fd, 0, SEEK_CUR);
+ if (data_offset == -1)
+ return -errno;
+ }
+
+ err = readn(fd, data, size);
+ if (err != (ssize_t)size) {
+ free(data);
+ return -errno;
+ }
+
+ if (dump_trace)
+ hisi_ptt_dump_event(ptt, data, size);
+
+ return 0;
+}
+
+static int hisi_ptt_flush(struct perf_session *session __maybe_unused,
+ struct perf_tool *tool __maybe_unused)
+{
+ return 0;
+}
+
+static void hisi_ptt_free_events(struct perf_session *session __maybe_unused)
+{
+}
+
+static void hisi_ptt_free(struct perf_session *session)
+{
+ struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt,
+ auxtrace);
+
+ session->auxtrace = NULL;
+ free(ptt);
+}
+
+static bool hisi_ptt_evsel_is_auxtrace(struct perf_session *session,
+ struct evsel *evsel)
+{
+ struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, auxtrace);
+
+ return evsel->core.attr.type == ptt->pmu_type;
+}
+
+static void hisi_ptt_print_info(__u64 type)
+{
+ if (!dump_trace)
+ return;
+
+ fprintf(stdout, " PMU Type %" PRId64 "\n", (s64) type);
+}
+
+int hisi_ptt_process_auxtrace_info(union perf_event *event,
+ struct perf_session *session)
+{
+ struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
+ struct hisi_ptt *ptt;
+
+ if (auxtrace_info->header.size < HISI_PTT_AUXTRACE_PRIV_SIZE +
+ sizeof(struct perf_record_auxtrace_info))
+ return -EINVAL;
+
+ ptt = zalloc(sizeof(*ptt));
+ if (!ptt)
+ return -ENOMEM;
+
+ ptt->session = session;
+ ptt->machine = &session->machines.host; /* No kvm support */
+ ptt->auxtrace_type = auxtrace_info->type;
+ ptt->pmu_type = auxtrace_info->priv[0];
+
+ ptt->auxtrace.process_event = hisi_ptt_process_event;
+ ptt->auxtrace.process_auxtrace_event = hisi_ptt_process_auxtrace_event;
+ ptt->auxtrace.flush_events = hisi_ptt_flush;
+ ptt->auxtrace.free_events = hisi_ptt_free_events;
+ ptt->auxtrace.free = hisi_ptt_free;
+ ptt->auxtrace.evsel_is_auxtrace = hisi_ptt_evsel_is_auxtrace;
+ session->auxtrace = &ptt->auxtrace;
+
+ hisi_ptt_print_info(auxtrace_info->priv[0]);
+
+ return 0;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * HiSilicon PCIe Trace and Tuning (PTT) support
+ * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
+ */
+
+#ifndef INCLUDE__PERF_HISI_PTT_H__
+#define INCLUDE__PERF_HISI_PTT_H__
+
+#define HISI_PTT_PMU_NAME "hisi_ptt"
+#define HISI_PTT_AUXTRACE_PRIV_SIZE sizeof(u64)
+
+struct auxtrace_record *hisi_ptt_recording_init(int *err,
+ struct perf_pmu *hisi_ptt_pmu);
+
+int hisi_ptt_process_auxtrace_info(union perf_event *event,
+ struct perf_session *session);
+
+#endif
hists__new_col_len(hists, HISTC_GLOBAL_INS_LAT, 13);
hists__new_col_len(hists, HISTC_LOCAL_P_STAGE_CYC, 13);
hists__new_col_len(hists, HISTC_GLOBAL_P_STAGE_CYC, 13);
+ hists__new_col_len(hists, HISTC_ADDR, BITS_PER_LONG / 4 + 2);
if (symbol_conf.nanosecs)
hists__new_col_len(hists, HISTC_TIME, 16);
{
struct rb_root_cached *root;
- pthread_mutex_lock(&hists->lock);
+ mutex_lock(&hists->lock);
root = hists->entries_in;
if (++hists->entries_in > &hists->entries_in_array[1])
hists->entries_in = &hists->entries_in_array[0];
- pthread_mutex_unlock(&hists->lock);
+ mutex_unlock(&hists->lock);
return root;
}
hists->stats.nr_non_filtered_samples++;
}
+void hists__inc_nr_lost_samples(struct hists *hists, u32 lost)
+{
+ hists->stats.nr_lost_samples += lost;
+}
+
static struct hist_entry *hists__add_dummy_entry(struct hists *hists,
struct hist_entry *pair)
{
evlist__for_each_entry(evlist, pos) {
struct hists *hists = evsel__hists(pos);
- if (skip_empty && !hists->stats.nr_samples)
+ if (skip_empty && !hists->stats.nr_samples && !hists->stats.nr_lost_samples)
continue;
ret += fprintf(fp, "%s stats:\n", evsel__name(pos));
- ret += fprintf(fp, "%16s events: %10d\n",
- "SAMPLE", hists->stats.nr_samples);
+ if (hists->stats.nr_samples)
+ ret += fprintf(fp, "%16s events: %10d\n",
+ "SAMPLE", hists->stats.nr_samples);
+ if (hists->stats.nr_lost_samples)
+ ret += fprintf(fp, "%16s events: %10d\n",
+ "LOST_SAMPLES", hists->stats.nr_lost_samples);
}
return ret;
hists->entries_in = &hists->entries_in_array[0];
hists->entries_collapsed = RB_ROOT_CACHED;
hists->entries = RB_ROOT_CACHED;
- pthread_mutex_init(&hists->lock, NULL);
+ mutex_init(&hists->lock);
hists->socket_filter = -1;
hists->hpp_list = hpp_list;
INIT_LIST_HEAD(&hists->hpp_formats);
#include <linux/rbtree.h>
#include <linux/types.h>
-#include <pthread.h>
#include "evsel.h"
#include "color.h"
#include "events_stats.h"
+#include "mutex.h"
struct hist_entry;
struct hist_entry_ops;
HISTC_GLOBAL_P_STAGE_CYC,
HISTC_ADDR_FROM,
HISTC_ADDR_TO,
+ HISTC_ADDR,
HISTC_NR_COLS, /* Last entry */
};
const struct dso *dso_filter;
const char *uid_filter_str;
const char *symbol_filter_str;
- pthread_mutex_t lock;
+ struct mutex lock;
struct hists_stats stats;
u64 event_stream;
u16 col_len[HISTC_NR_COLS];
void hists__inc_stats(struct hists *hists, struct hist_entry *h);
void hists__inc_nr_events(struct hists *hists);
void hists__inc_nr_samples(struct hists *hists, bool filtered);
+void hists__inc_nr_lost_samples(struct hists *hists, u32 lost);
size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows,
int max_cols, float min_pcnt, FILE *fp,
*/
#include <stdio.h>
+#include <stdlib.h>
#include <stdint.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdbool.h>
#include <string.h>
+#include <linux/zalloc.h>
+#include <linux/kernel.h>
+
#include "intel-pt-log.h"
#include "intel-pt-insn-decoder.h"
#define MAX_LOG_NAME 256
+#define DFLT_BUF_SZ (16 * 1024)
+
+struct log_buf {
+ char *buf;
+ size_t buf_sz;
+ size_t head;
+ bool wrapped;
+ FILE *backend;
+};
+
static FILE *f;
static char log_name[MAX_LOG_NAME];
bool intel_pt_enable_logging;
+static bool intel_pt_dump_log_on_error;
+static unsigned int intel_pt_log_on_error_size;
+static struct log_buf log_buf;
void *intel_pt_log_fp(void)
{
return f;
}
-void intel_pt_log_enable(void)
+void intel_pt_log_enable(bool dump_log_on_error, unsigned int log_on_error_size)
{
intel_pt_enable_logging = true;
+ intel_pt_dump_log_on_error = dump_log_on_error;
+ intel_pt_log_on_error_size = log_on_error_size;
}
void intel_pt_log_disable(void)
fprintf(f, " ");
}
+static ssize_t log_buf__write(void *cookie, const char *buf, size_t size)
+{
+ struct log_buf *b = cookie;
+ size_t sz = size;
+
+ if (!b->buf)
+ return size;
+
+ while (sz) {
+ size_t space = b->buf_sz - b->head;
+ size_t n = min(space, sz);
+
+ memcpy(b->buf + b->head, buf, n);
+ sz -= n;
+ buf += n;
+ b->head += n;
+ if (sz && b->head >= b->buf_sz) {
+ b->head = 0;
+ b->wrapped = true;
+ }
+ }
+ return size;
+}
+
+static int log_buf__close(void *cookie)
+{
+ struct log_buf *b = cookie;
+
+ zfree(&b->buf);
+ return 0;
+}
+
+static FILE *log_buf__open(struct log_buf *b, FILE *backend, unsigned int sz)
+{
+ cookie_io_functions_t fns = {
+ .write = log_buf__write,
+ .close = log_buf__close,
+ };
+ FILE *file;
+
+ memset(b, 0, sizeof(*b));
+ b->buf_sz = sz;
+ b->buf = malloc(b->buf_sz);
+ b->backend = backend;
+ file = fopencookie(b, "a", fns);
+ if (!file)
+ zfree(&b->buf);
+ return file;
+}
+
+static bool remove_first_line(const char **p, size_t *n)
+{
+ for (; *n && **p != '\n'; ++*p, --*n)
+ ;
+ if (*n) {
+ *p += 1;
+ *n -= 1;
+ return true;
+ }
+ return false;
+}
+
+static void write_lines(const char *p, size_t n, FILE *fp, bool *remove_first)
+{
+ if (*remove_first)
+ *remove_first = !remove_first_line(&p, &n);
+ fwrite(p, n, 1, fp);
+}
+
+static void log_buf__dump(struct log_buf *b)
+{
+ bool remove_first = false;
+
+ if (!b->buf)
+ return;
+
+ fflush(f); /* Could update b->head and b->wrapped */
+ fprintf(b->backend, "Dumping debug log buffer\n");
+ if (b->wrapped) {
+ remove_first = true;
+ write_lines(b->buf + b->head, b->buf_sz - b->head, b->backend, &remove_first);
+ }
+ write_lines(b->buf, b->head, b->backend, &remove_first);
+ fprintf(b->backend, "End of debug log buffer dump\n");
+
+ b->head = 0;
+ b->wrapped = false;
+}
+
+void intel_pt_log_dump_buf(void)
+{
+ log_buf__dump(&log_buf);
+}
+
static int intel_pt_log_open(void)
{
if (!intel_pt_enable_logging)
f = fopen(log_name, "w+");
else
f = stdout;
+ if (f && intel_pt_dump_log_on_error)
+ f = log_buf__open(&log_buf, f, intel_pt_log_on_error_size);
if (!f) {
intel_pt_enable_logging = false;
return -1;
struct intel_pt_pkt;
void *intel_pt_log_fp(void);
-void intel_pt_log_enable(void);
+void intel_pt_log_enable(bool dump_log_on_error, unsigned int log_on_error_size);
void intel_pt_log_disable(void);
void intel_pt_log_set_name(const char *name);
+void intel_pt_log_dump_buf(void);
void __intel_pt_log_packet(const struct intel_pt_pkt *packet, int pkt_len,
uint64_t pos, const unsigned char *buf);
offset, buf,
INTEL_PT_INSN_BUF_SZ);
if (len <= 0) {
- intel_pt_log("ERROR: failed to read at %" PRIu64 " ", offset);
+ intel_pt_log("ERROR: failed to read at offset %#" PRIx64 " ",
+ offset);
if (intel_pt_enable_logging)
dso__fprintf(al.map->dso, intel_pt_log_fp());
return -EINVAL;
pid_t pid, pid_t tid, u64 ip, u64 timestamp,
pid_t machine_pid, int vcpu)
{
+ bool dump_log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;
+ bool log_on_stdout = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT;
union perf_event event;
char msg[MAX_AUXTRACE_ERROR_MSG];
int err;
code, cpu, pid, tid, ip, msg, timestamp,
machine_pid, vcpu);
+ if (intel_pt_enable_logging && !log_on_stdout) {
+ FILE *fp = intel_pt_log_fp();
+
+ if (fp)
+ perf_event__fprintf_auxtrace_error(&event, fp);
+ }
+
+ if (code != INTEL_PT_ERR_LOST && dump_log_on_error)
+ intel_pt_log_dump_buf();
+
err = perf_session__deliver_synth_event(pt->session, &event, NULL);
if (err)
pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
[INTEL_PT_SNAPSHOT_MODE] = " Snapshot mode %"PRId64"\n",
[INTEL_PT_PER_CPU_MMAPS] = " Per-cpu maps %"PRId64"\n",
[INTEL_PT_MTC_BIT] = " MTC bit %#"PRIx64"\n",
+ [INTEL_PT_MTC_FREQ_BITS] = " MTC freq bits %#"PRIx64"\n",
[INTEL_PT_TSC_CTC_N] = " TSC:CTC numerator %"PRIu64"\n",
[INTEL_PT_TSC_CTC_D] = " TSC:CTC denominator %"PRIu64"\n",
[INTEL_PT_CYC_BIT] = " CYC bit %#"PRIx64"\n",
if (!dump_trace)
return;
- for (i = start; i <= finish; i++)
- fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
+ for (i = start; i <= finish; i++) {
+ const char *fmt = intel_pt_info_fmts[i];
+
+ if (fmt)
+ fprintf(stdout, fmt, arr[i]);
+ }
}
static void intel_pt_print_info_str(const char *name, const char *str)
goto err_delete_thread;
}
- if (pt->synth_opts.log)
- intel_pt_log_enable();
+ if (pt->synth_opts.log) {
+ bool log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;
+ unsigned int log_on_error_size = pt->synth_opts.log_on_error_size;
+
+ intel_pt_log_enable(log_on_error, log_on_error_size);
+ }
/* Maximum non-turbo ratio is TSC freq / 100 MHz */
if (pt->tc.time_mult) {
char dir[PATH_MAX];
};
-struct debug_line_info {
- unsigned long vma;
- unsigned int lineno;
- /* The filename format is unspecified, absolute path, relative etc. */
- char const filename[];
-};
-
struct jit_tool {
struct perf_tool tool;
struct perf_data output;
u64 addr; /* address of lockdep_map, used as ID */
char *name; /* for strcpy(), we cannot use const */
+ u64 *callstack;
unsigned int nr_acquire;
unsigned int nr_acquired;
struct machine *machine;
struct hlist_head *result;
unsigned long map_nr_entries;
- unsigned long lost;
+ int lost;
+ int max_stack;
+ int stack_skip;
};
#ifdef HAVE_BPF_SKEL
return kernel;
}
-struct process_args {
- u64 start;
-};
-
void machine__get_kallsyms_filename(struct machine *machine, char *buf,
size_t bufsz)
{
if (!(prot & PROT_EXEC))
dso__set_loaded(dso);
}
+ mutex_lock(&dso->lock);
+ nsinfo__put(dso->nsinfo);
dso->nsinfo = nsi;
+ mutex_unlock(&dso->lock);
if (build_id__is_defined(bid)) {
dso__set_build_id(dso, bid);
for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) {
struct perf_mem_event *e = perf_mem_events__ptr(j);
- fprintf(stderr, "%-13s%-*s%s\n",
- e->tag ?: "",
- verbose > 0 ? 25 : 0,
- verbose > 0 ? perf_mem_events__name(j, NULL) : "",
- e->supported ? ": available" : "");
+ fprintf(stderr, "%-*s%-*s%s",
+ e->tag ? 13 : 0,
+ e->tag ? : "",
+ e->tag && verbose > 0 ? 25 : 0,
+ e->tag && verbose > 0 ? perf_mem_events__name(j, NULL) : "",
+ e->supported ? ": available\n" : "");
}
}
"HIT",
"MISS",
"L1",
- "LFB",
+ "LFB/MAB",
"L2",
"L3",
"Local RAM",
};
static const char * const mem_lvlnum[] = {
+ [PERF_MEM_LVLNUM_CXL] = "CXL",
+ [PERF_MEM_LVLNUM_IO] = "I/O",
[PERF_MEM_LVLNUM_ANY_CACHE] = "Any cache",
- [PERF_MEM_LVLNUM_LFB] = "LFB",
+ [PERF_MEM_LVLNUM_LFB] = "LFB/MAB",
[PERF_MEM_LVLNUM_RAM] = "RAM",
[PERF_MEM_LVLNUM_PMEM] = "PMEM",
[PERF_MEM_LVLNUM_NA] = "N/A",
#include <linux/list_sort.h>
#include <linux/string.h>
#include <linux/zalloc.h>
+#include <perf/cpumap.h>
#include <subcmd/parse-options.h>
#include <api/fs/fs.h>
#include "util.h"
rblist__exit(metric_events);
}
-/*
- * A node in the list of referenced metrics. metric_expr
- * is held as a convenience to avoid a search through the
- * metric list.
- */
-struct metric_ref_node {
- const char *metric_name;
- const char *metric_expr;
- struct list_head list;
-};
-
/**
* The metric under construction. The data held here will be placed in a
* metric_expr.
return false;
}
+static void metric__free(struct metric *m)
+{
+ if (!m)
+ return;
+
+ free(m->metric_refs);
+ expr__ctx_free(m->pctx);
+ free((char *)m->modifier);
+ evlist__delete(m->evlist);
+ free(m);
+}
+
static struct metric *metric__new(const struct pmu_event *pe,
const char *modifier,
bool metric_no_group,
- int runtime)
+ int runtime,
+ const char *user_requested_cpu_list,
+ bool system_wide)
{
struct metric *m;
return NULL;
m->pctx = expr__ctx_new();
- if (!m->pctx) {
- free(m);
- return NULL;
- }
+ if (!m->pctx)
+ goto out_err;
m->metric_name = pe->metric_name;
- m->modifier = modifier ? strdup(modifier) : NULL;
- if (modifier && !m->modifier) {
- expr__ctx_free(m->pctx);
- free(m);
- return NULL;
+ m->modifier = NULL;
+ if (modifier) {
+ m->modifier = strdup(modifier);
+ if (!m->modifier)
+ goto out_err;
}
m->metric_expr = pe->metric_expr;
m->metric_unit = pe->unit;
- m->pctx->runtime = runtime;
+ m->pctx->sctx.user_requested_cpu_list = NULL;
+ if (user_requested_cpu_list) {
+ m->pctx->sctx.user_requested_cpu_list = strdup(user_requested_cpu_list);
+ if (!m->pctx->sctx.user_requested_cpu_list)
+ goto out_err;
+ }
+ m->pctx->sctx.runtime = runtime;
+ m->pctx->sctx.system_wide = system_wide;
m->has_constraint = metric_no_group || metricgroup__has_constraint(pe);
m->metric_refs = NULL;
m->evlist = NULL;
return m;
-}
-
-static void metric__free(struct metric *m)
-{
- free(m->metric_refs);
- expr__ctx_free(m->pctx);
- free((char *)m->modifier);
- evlist__delete(m->evlist);
- free(m);
+out_err:
+ metric__free(m);
+ return NULL;
}
static bool contains_metric_id(struct evsel **metric_events, int num_events,
int *ret;
bool *has_match;
bool metric_no_group;
+ const char *user_requested_cpu_list;
+ bool system_wide;
struct metric *root_metric;
const struct visited_metric *visited;
const struct pmu_events_table *table;
const struct pmu_event *pe,
const char *modifier,
bool metric_no_group,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct metric *root_metric,
const struct visited_metric *visited,
const struct pmu_events_table *table);
* @metric_no_group: Should events written to events be grouped "{}" or
* global. Grouping is the default but due to multiplexing the
* user may override.
+ * @user_requested_cpu_list: Command line specified CPUs to record on.
+ * @system_wide: Are events for all processes recorded.
* @root_metric: Metrics may reference other metrics to form a tree. In this
* case the root_metric holds all the IDs and a list of referenced
* metrics. When adding a root this argument is NULL.
static int resolve_metric(struct list_head *metric_list,
const char *modifier,
bool metric_no_group,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct metric *root_metric,
const struct visited_metric *visited,
const struct pmu_events_table *table)
*/
for (i = 0; i < pending_cnt; i++) {
ret = add_metric(metric_list, &pending[i].pe, modifier, metric_no_group,
- root_metric, visited, table);
+ user_requested_cpu_list, system_wide, root_metric, visited,
+ table);
if (ret)
break;
}
* global. Grouping is the default but due to multiplexing the
* user may override.
* @runtime: A special argument for the parser only known at runtime.
+ * @user_requested_cpu_list: Command line specified CPUs to record on.
+ * @system_wide: Are events for all processes recorded.
* @root_metric: Metrics may reference other metrics to form a tree. In this
* case the root_metric holds all the IDs and a list of referenced
* metrics. When adding a root this argument is NULL.
const char *modifier,
bool metric_no_group,
int runtime,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct metric *root_metric,
const struct visited_metric *visited,
const struct pmu_events_table *table)
* This metric is the root of a tree and may reference other
* metrics that are added recursively.
*/
- root_metric = metric__new(pe, modifier, metric_no_group, runtime);
+ root_metric = metric__new(pe, modifier, metric_no_group, runtime,
+ user_requested_cpu_list, system_wide);
if (!root_metric)
return -ENOMEM;
ret = -EINVAL;
} else {
/* Resolve referenced metrics. */
- ret = resolve_metric(metric_list, modifier, metric_no_group, root_metric,
- &visited_node, table);
+ ret = resolve_metric(metric_list, modifier, metric_no_group,
+ user_requested_cpu_list, system_wide,
+ root_metric, &visited_node, table);
}
if (ret) {
const struct pmu_event *pe,
const char *modifier,
bool metric_no_group,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct metric *root_metric,
const struct visited_metric *visited,
const struct pmu_events_table *table)
if (!strstr(pe->metric_expr, "?")) {
ret = __add_metric(metric_list, pe, modifier, metric_no_group, 0,
- root_metric, visited, table);
+ user_requested_cpu_list, system_wide, root_metric,
+ visited, table);
} else {
int j, count;
for (j = 0; j < count && !ret; j++)
ret = __add_metric(metric_list, pe, modifier, metric_no_group, j,
- root_metric, visited, table);
+ user_requested_cpu_list, system_wide,
+ root_metric, visited, table);
}
return ret;
return 0;
ret = add_metric(d->metric_list, pe, d->modifier, d->metric_no_group,
+ d->user_requested_cpu_list, d->system_wide,
d->root_metric, d->visited, d->table);
if (ret)
goto out;
struct list_head *list;
const char *metric_name;
const char *modifier;
+ const char *user_requested_cpu_list;
bool metric_no_group;
+ bool system_wide;
bool has_match;
};
data->has_match = true;
ret = add_metric(data->list, pe, data->modifier, data->metric_no_group,
- /*root_metric=*/NULL,
- /*visited_metrics=*/NULL, table);
+ data->user_requested_cpu_list, data->system_wide,
+ /*root_metric=*/NULL, /*visited_metrics=*/NULL, table);
}
return ret;
}
* @metric_no_group: Should events written to events be grouped "{}" or
* global. Grouping is the default but due to multiplexing the
* user may override.
+ * @user_requested_cpu_list: Command line specified CPUs to record on.
+ * @system_wide: Are events for all processes recorded.
* @metric_list: The list that the metric or metric group are added to.
* @table: The table that is searched for metrics, most commonly the table for the
* architecture perf is running upon.
*/
static int metricgroup__add_metric(const char *metric_name, const char *modifier,
bool metric_no_group,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct list_head *metric_list,
const struct pmu_events_table *table)
{
.metric_name = metric_name,
.modifier = modifier,
.metric_no_group = metric_no_group,
+ .user_requested_cpu_list = user_requested_cpu_list,
+ .system_wide = system_wide,
.has_match = false,
};
/*
.metric_name = metric_name,
.modifier = modifier,
.metric_no_group = metric_no_group,
+ .user_requested_cpu_list = user_requested_cpu_list,
+ .system_wide = system_wide,
.has_match = &has_match,
.ret = &ret,
.table = table,
* @metric_no_group: Should events written to events be grouped "{}" or
* global. Grouping is the default but due to multiplexing the
* user may override.
+ * @user_requested_cpu_list: Command line specified CPUs to record on.
+ * @system_wide: Are events for all processes recorded.
* @metric_list: The list that metrics are added to.
* @table: The table that is searched for metrics, most commonly the table for the
* architecture perf is running upon.
*/
static int metricgroup__add_metric_list(const char *list, bool metric_no_group,
- struct list_head *metric_list,
+ const char *user_requested_cpu_list,
+ bool system_wide, struct list_head *metric_list,
const struct pmu_events_table *table)
{
char *list_itr, *list_copy, *metric_name, *modifier;
*modifier++ = '\0';
ret = metricgroup__add_metric(metric_name, modifier,
- metric_no_group, metric_list,
- table);
+ metric_no_group, user_requested_cpu_list,
+ system_wide, metric_list, table);
if (ret == -EINVAL)
pr_err("Cannot find metric or group `%s'\n", metric_name);
static int parse_groups(struct evlist *perf_evlist, const char *str,
bool metric_no_group,
bool metric_no_merge,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct perf_pmu *fake_pmu,
struct rblist *metric_events_list,
const struct pmu_events_table *table)
if (metric_events_list->nr_entries == 0)
metricgroup__rblist_init(metric_events_list);
ret = metricgroup__add_metric_list(str, metric_no_group,
- &metric_list, table);
+ user_requested_cpu_list,
+ system_wide, &metric_list, table);
if (ret)
goto out;
}
expr->metric_unit = m->metric_unit;
expr->metric_events = metric_events;
- expr->runtime = m->pctx->runtime;
+ expr->runtime = m->pctx->sctx.runtime;
list_add(&expr->nd, &me->head);
}
return ret;
}
-int metricgroup__parse_groups(const struct option *opt,
+int metricgroup__parse_groups(struct evlist *perf_evlist,
const char *str,
bool metric_no_group,
bool metric_no_merge,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct rblist *metric_events)
{
- struct evlist *perf_evlist = *(struct evlist **)opt->value;
const struct pmu_events_table *table = pmu_events_table__find();
if (!table)
return -EINVAL;
- return parse_groups(perf_evlist, str, metric_no_group,
- metric_no_merge, NULL, metric_events, table);
+ return parse_groups(perf_evlist, str, metric_no_group, metric_no_merge,
+ user_requested_cpu_list, system_wide,
+ /*fake_pmu=*/NULL, metric_events, table);
}
int metricgroup__parse_groups_test(struct evlist *evlist,
bool metric_no_merge,
struct rblist *metric_events)
{
- return parse_groups(evlist, str, metric_no_group,
- metric_no_merge, &perf_pmu__fake, metric_events, table);
+ return parse_groups(evlist, str, metric_no_group, metric_no_merge,
+ /*user_requested_cpu_list=*/NULL,
+ /*system_wide=*/false,
+ &perf_pmu__fake, metric_events, table);
}
static int metricgroup__has_metric_callback(const struct pmu_event *pe,
struct rblist *new_metric_events,
struct rblist *old_metric_events)
{
- unsigned i;
+ unsigned int i;
for (i = 0; i < rblist__nr_entries(old_metric_events); i++) {
struct rb_node *nd;
struct metric_event *metricgroup__lookup(struct rblist *metric_events,
struct evsel *evsel,
bool create);
-int metricgroup__parse_groups(const struct option *opt,
+int metricgroup__parse_groups(struct evlist *perf_evlist,
const char *str,
bool metric_no_group,
bool metric_no_merge,
+ const char *user_requested_cpu_list,
+ bool system_wide,
struct rblist *metric_events);
int metricgroup__parse_groups_test(struct evlist *evlist,
const struct pmu_events_table *table,
#include <linux/bitops.h>
#include <perf/cpumap.h>
#include <stdbool.h>
-#include <pthread.h> // for cpu_set_t
#ifdef HAVE_AIO_SUPPORT
#include <aio.h>
#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include "mutex.h"
+
+#include "debug.h"
+#include <linux/string.h>
+#include <errno.h>
+
+static void check_err(const char *fn, int err)
+{
+ char sbuf[STRERR_BUFSIZE];
+
+ if (err == 0)
+ return;
+
+ pr_err("%s error: '%s'\n", fn, str_error_r(err, sbuf, sizeof(sbuf)));
+}
+
+#define CHECK_ERR(err) check_err(__func__, err)
+
+static void __mutex_init(struct mutex *mtx, bool pshared)
+{
+ pthread_mutexattr_t attr;
+
+ CHECK_ERR(pthread_mutexattr_init(&attr));
+
+#ifndef NDEBUG
+ /* In normal builds enable error checking, such as recursive usage. */
+ CHECK_ERR(pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK));
+#endif
+ if (pshared)
+ CHECK_ERR(pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED));
+
+ CHECK_ERR(pthread_mutex_init(&mtx->lock, &attr));
+ CHECK_ERR(pthread_mutexattr_destroy(&attr));
+}
+
+void mutex_init(struct mutex *mtx)
+{
+ __mutex_init(mtx, /*pshared=*/false);
+}
+
+void mutex_init_pshared(struct mutex *mtx)
+{
+ __mutex_init(mtx, /*pshared=*/true);
+}
+
+void mutex_destroy(struct mutex *mtx)
+{
+ CHECK_ERR(pthread_mutex_destroy(&mtx->lock));
+}
+
+void mutex_lock(struct mutex *mtx)
+ NO_THREAD_SAFETY_ANALYSIS
+{
+ CHECK_ERR(pthread_mutex_lock(&mtx->lock));
+}
+
+void mutex_unlock(struct mutex *mtx)
+ NO_THREAD_SAFETY_ANALYSIS
+{
+ CHECK_ERR(pthread_mutex_unlock(&mtx->lock));
+}
+
+bool mutex_trylock(struct mutex *mtx)
+{
+ int ret = pthread_mutex_trylock(&mtx->lock);
+
+ if (ret == 0)
+ return true; /* Lock acquired. */
+
+ if (ret == EBUSY)
+ return false; /* Lock busy. */
+
+ /* Print error. */
+ CHECK_ERR(ret);
+ return false;
+}
+
+static void __cond_init(struct cond *cnd, bool pshared)
+{
+ pthread_condattr_t attr;
+
+ CHECK_ERR(pthread_condattr_init(&attr));
+ if (pshared)
+ CHECK_ERR(pthread_condattr_setpshared(&attr, PTHREAD_PROCESS_SHARED));
+
+ CHECK_ERR(pthread_cond_init(&cnd->cond, &attr));
+ CHECK_ERR(pthread_condattr_destroy(&attr));
+}
+
+void cond_init(struct cond *cnd)
+{
+ __cond_init(cnd, /*pshared=*/false);
+}
+
+void cond_init_pshared(struct cond *cnd)
+{
+ __cond_init(cnd, /*pshared=*/true);
+}
+
+void cond_destroy(struct cond *cnd)
+{
+ CHECK_ERR(pthread_cond_destroy(&cnd->cond));
+}
+
+void cond_wait(struct cond *cnd, struct mutex *mtx)
+{
+ CHECK_ERR(pthread_cond_wait(&cnd->cond, &mtx->lock));
+}
+
+void cond_signal(struct cond *cnd)
+{
+ CHECK_ERR(pthread_cond_signal(&cnd->cond));
+}
+
+void cond_broadcast(struct cond *cnd)
+{
+ CHECK_ERR(pthread_cond_broadcast(&cnd->cond));
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __PERF_MUTEX_H
+#define __PERF_MUTEX_H
+
+#include <pthread.h>
+#include <stdbool.h>
+
+/*
+ * A function-like feature checking macro that is a wrapper around
+ * `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a
+ * nonzero constant integer if the attribute is supported or 0 if not.
+ */
+#ifdef __has_attribute
+#define HAVE_ATTRIBUTE(x) __has_attribute(x)
+#else
+#define HAVE_ATTRIBUTE(x) 0
+#endif
+
+#if HAVE_ATTRIBUTE(guarded_by) && HAVE_ATTRIBUTE(pt_guarded_by) && \
+ HAVE_ATTRIBUTE(lockable) && HAVE_ATTRIBUTE(exclusive_lock_function) && \
+ HAVE_ATTRIBUTE(exclusive_trylock_function) && HAVE_ATTRIBUTE(exclusive_locks_required) && \
+ HAVE_ATTRIBUTE(no_thread_safety_analysis)
+
+/* Documents if a shared field or global variable needs to be protected by a mutex. */
+#define GUARDED_BY(x) __attribute__((guarded_by(x)))
+
+/*
+ * Documents if the memory location pointed to by a pointer should be guarded by
+ * a mutex when dereferencing the pointer.
+ */
+#define PT_GUARDED_BY(x) __attribute__((pt_guarded_by(x)))
+
+/* Documents if a type is a lockable type. */
+#define LOCKABLE __attribute__((lockable))
+
+/* Documents functions that acquire a lock in the body of a function, and do not release it. */
+#define EXCLUSIVE_LOCK_FUNCTION(...) __attribute__((exclusive_lock_function(__VA_ARGS__)))
+
+/*
+ * Documents functions that expect a lock to be held on entry to the function,
+ * and release it in the body of the function.
+ */
+#define UNLOCK_FUNCTION(...) __attribute__((unlock_function(__VA_ARGS__)))
+
+/* Documents functions that try to acquire a lock, and return success or failure. */
+#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \
+ __attribute__((exclusive_trylock_function(__VA_ARGS__)))
+
+/* Documents a function that expects a mutex to be held prior to entry. */
+#define EXCLUSIVE_LOCKS_REQUIRED(...) __attribute__((exclusive_locks_required(__VA_ARGS__)))
+
+/* Turns off thread safety checking within the body of a particular function. */
+#define NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis))
+
+#else
+
+#define GUARDED_BY(x)
+#define PT_GUARDED_BY(x)
+#define LOCKABLE
+#define EXCLUSIVE_LOCK_FUNCTION(...)
+#define UNLOCK_FUNCTION(...)
+#define EXCLUSIVE_TRYLOCK_FUNCTION(...)
+#define EXCLUSIVE_LOCKS_REQUIRED(...)
+#define NO_THREAD_SAFETY_ANALYSIS
+
+#endif
+
+/*
+ * A wrapper around the mutex implementation that allows perf to error check
+ * usage, etc.
+ */
+struct LOCKABLE mutex {
+ pthread_mutex_t lock;
+};
+
+/* A wrapper around the condition variable implementation. */
+struct cond {
+ pthread_cond_t cond;
+};
+
+/* Default initialize the mtx struct. */
+void mutex_init(struct mutex *mtx);
+/*
+ * Initialize the mtx struct and set the process-shared rather than default
+ * process-private attribute.
+ */
+void mutex_init_pshared(struct mutex *mtx);
+void mutex_destroy(struct mutex *mtx);
+
+void mutex_lock(struct mutex *mtx) EXCLUSIVE_LOCK_FUNCTION(*mtx);
+void mutex_unlock(struct mutex *mtx) UNLOCK_FUNCTION(*mtx);
+/* Tries to acquire the lock and returns true on success. */
+bool mutex_trylock(struct mutex *mtx) EXCLUSIVE_TRYLOCK_FUNCTION(true, *mtx);
+
+/* Default initialize the cond struct. */
+void cond_init(struct cond *cnd);
+/*
+ * Initialize the cond struct and specify the process-shared rather than default
+ * process-private attribute.
+ */
+void cond_init_pshared(struct cond *cnd);
+void cond_destroy(struct cond *cnd);
+
+void cond_wait(struct cond *cnd, struct mutex *mtx) EXCLUSIVE_LOCKS_REQUIRED(mtx);
+void cond_signal(struct cond *cnd);
+void cond_broadcast(struct cond *cnd);
+
+#endif /* __PERF_MUTEX_H */
BRANCH_OPT("call", PERF_SAMPLE_BRANCH_CALL),
BRANCH_OPT("save_type", PERF_SAMPLE_BRANCH_TYPE_SAVE),
BRANCH_OPT("stack", PERF_SAMPLE_BRANCH_CALL_STACK),
+ BRANCH_OPT("priv", PERF_SAMPLE_BRANCH_PRIV_SAVE),
BRANCH_END
};
},
};
-#define __PERF_EVENT_FIELD(config, name) \
- ((config & PERF_EVENT_##name##_MASK) >> PERF_EVENT_##name##_SHIFT)
-
-#define PERF_EVENT_RAW(config) __PERF_EVENT_FIELD(config, RAW)
-#define PERF_EVENT_CONFIG(config) __PERF_EVENT_FIELD(config, CONFIG)
-#define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE)
-#define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT)
-
bool is_event_supported(u8 type, u64 config)
{
bool ret = true;
struct perf_cpu_map *cpus = pmu ? perf_cpu_map__get(pmu->cpus) :
cpu_list ? perf_cpu_map__new(cpu_list) : NULL;
+ if (pmu)
+ perf_pmu__warn_invalid_formats(pmu);
+
if (pmu && attr->type == PERF_TYPE_RAW)
perf_pmu__warn_invalid_config(pmu, attr->config, name);
bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX),
bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
- bit_name(TYPE_SAVE), bit_name(HW_INDEX),
+ bit_name(TYPE_SAVE), bit_name(HW_INDEX), bit_name(PRIV_SAVE),
{ .name = NULL, }
};
#undef bit_name
#define bit_name(n) { PERF_FORMAT_##n, #n }
struct bit_names bits[] = {
bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING),
- bit_name(ID), bit_name(GROUP),
+ bit_name(ID), bit_name(GROUP), bit_name(LOST),
{ .name = NULL, }
};
#undef bit_name
return NULL;
}
+void perf_pmu__warn_invalid_formats(struct perf_pmu *pmu)
+{
+ struct perf_pmu_format *format;
+
+ /* fake pmu doesn't have format list */
+ if (pmu == &perf_pmu__fake)
+ return;
+
+ list_for_each_entry(format, &pmu->format, list)
+ if (format->value >= PERF_PMU_FORMAT_VALUE_CONFIG_END) {
+ pr_warning("WARNING: '%s' format '%s' requires 'perf_event_attr::config%d'"
+ "which is not supported by this version of perf!\n",
+ pmu->name, format->name, format->value);
+ return;
+ }
+}
+
static struct perf_pmu *pmu_find(const char *name)
{
struct perf_pmu *pmu;
struct perf_pmu_format *format;
char *str = NULL;
struct strbuf buf = STRBUF_INIT;
- unsigned i = 0;
+ unsigned int i = 0;
if (!formats)
return NULL;
PERF_PMU_FORMAT_VALUE_CONFIG,
PERF_PMU_FORMAT_VALUE_CONFIG1,
PERF_PMU_FORMAT_VALUE_CONFIG2,
+ PERF_PMU_FORMAT_VALUE_CONFIG_END,
};
#define PERF_PMU_FORMAT_BITS 64
void perf_pmu__warn_invalid_config(struct perf_pmu *pmu, __u64 config,
const char *name);
+void perf_pmu__warn_invalid_formats(struct perf_pmu *pmu);
bool perf_pmu__has_hybrid(void);
int perf_pmu__match(char *pattern, char *name, char *tok);
{num_dec} { return value(10); }
config { return PP_CONFIG; }
-config1 { return PP_CONFIG1; }
-config2 { return PP_CONFIG2; }
- { return '-'; }
: { return ':'; }
, { return ','; }
#include <string.h>
#include "pmu.h"
-extern int perf_pmu_lex (void);
-
#define ABORT_ON(val) \
do { \
if (val) \
%}
-%token PP_CONFIG PP_CONFIG1 PP_CONFIG2
+%token PP_CONFIG
%token PP_VALUE PP_ERROR
%type <num> PP_VALUE
%type <bits> bit_term
$3));
}
|
-PP_CONFIG1 ':' bits
+PP_CONFIG PP_VALUE ':' bits
{
ABORT_ON(perf_pmu__new_format(format, name,
- PERF_PMU_FORMAT_VALUE_CONFIG1,
- $3));
-}
-|
-PP_CONFIG2 ':' bits
-{
- ABORT_ON(perf_pmu__new_format(format, name,
- PERF_PMU_FORMAT_VALUE_CONFIG2,
- $3));
+ $2,
+ $4));
}
bits:
#include "color.h"
#include "map.h"
#include "maps.h"
+#include "mutex.h"
#include "symbol.h"
#include <api/fs/fs.h>
#include "trace-event.h" /* For __maybe_unused */
map = dso__new_map(target);
if (map && map->dso) {
+ mutex_lock(&map->dso->lock);
nsinfo__put(map->dso->nsinfo);
map->dso->nsinfo = nsinfo__get(nsi);
+ mutex_unlock(&map->dso->lock);
}
return map;
} else {
default:
pr_err("cpu_map swap: unsupported long size\n");
}
+ break;
+ case PERF_CPU_MAP__RANGE_CPUS:
+ data->range_cpu_data.start_cpu = bswap_16(data->range_cpu_data.start_cpu);
+ data->range_cpu_data.end_cpu = bswap_16(data->range_cpu_data.end_cpu);
+ break;
default:
break;
}
e->flags.abort ? "A" : " ",
e->flags.in_tx ? "T" : " ",
(unsigned)e->flags.reserved,
- e->flags.type ? branch_type_name(e->flags.type) : "");
+ get_branch_type(e));
} else {
if (i == 0) {
printf("..... %2"PRIu64": %016" PRIx64 "\n"
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <linux/bitops.h>
+// SPDX-License-Identifier: GPL-2.0-only
+#include <string.h>
#include "api/fs/fs.h"
+#include "cputopo.h"
#include "smt.h"
-/**
- * hweight_str - Returns the number of bits set in str. Stops at first non-hex
- * or ',' character.
- */
-static int hweight_str(char *str)
-{
- int result = 0;
-
- while (*str) {
- switch (*str++) {
- case '0':
- case ',':
- break;
- case '1':
- case '2':
- case '4':
- case '8':
- result++;
- break;
- case '3':
- case '5':
- case '6':
- case '9':
- case 'a':
- case 'A':
- case 'c':
- case 'C':
- result += 2;
- break;
- case '7':
- case 'b':
- case 'B':
- case 'd':
- case 'D':
- case 'e':
- case 'E':
- result += 3;
- break;
- case 'f':
- case 'F':
- result += 4;
- break;
- default:
- goto done;
- }
- }
-done:
- return result;
-}
-
-int smt_on(void)
+bool smt_on(const struct cpu_topology *topology)
{
static bool cached;
- static int cached_result;
- int cpu;
- int ncpu;
+ static bool cached_result;
+ int fs_value;
if (cached)
return cached_result;
- if (sysfs__read_int("devices/system/cpu/smt/active", &cached_result) >= 0) {
- cached = true;
- return cached_result;
- }
-
- cached_result = 0;
- ncpu = sysconf(_SC_NPROCESSORS_CONF);
- for (cpu = 0; cpu < ncpu; cpu++) {
- unsigned long long siblings;
- char *str;
- size_t strlen;
- char fn[256];
+ if (sysfs__read_int("devices/system/cpu/smt/active", &fs_value) >= 0)
+ cached_result = (fs_value == 1);
+ else
+ cached_result = cpu_topology__smt_on(topology);
- snprintf(fn, sizeof fn,
- "devices/system/cpu/cpu%d/topology/thread_siblings", cpu);
- if (sysfs__read_str(fn, &str, &strlen) < 0) {
- snprintf(fn, sizeof fn,
- "devices/system/cpu/cpu%d/topology/core_cpus", cpu);
- if (sysfs__read_str(fn, &str, &strlen) < 0)
- continue;
- }
- /* Entry is hex, but does not have 0x, so need custom parser */
- siblings = hweight_str(str);
- free(str);
- if (siblings > 1) {
- cached_result = 1;
- break;
- }
- }
cached = true;
return cached_result;
}
+
+bool core_wide(bool system_wide, const char *user_requested_cpu_list,
+ const struct cpu_topology *topology)
+{
+ /* If not everything running on a core is being recorded then we can't use core_wide. */
+ if (!system_wide)
+ return false;
+
+ /* Cheap case that SMT is disabled and therefore we're inherently core_wide. */
+ if (!smt_on(topology))
+ return true;
+
+ return cpu_topology__core_wide(topology, user_requested_cpu_list);
+}
-#ifndef SMT_H
-#define SMT_H 1
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __SMT_H
+#define __SMT_H 1
-int smt_on(void);
+struct cpu_topology;
-#endif
+/* Returns true if SMT (aka hyperthreading) is enabled. */
+bool smt_on(const struct cpu_topology *topology);
+
+/*
+ * Returns true when system wide and all SMT threads for a core are in the
+ * user_requested_cpus map.
+ */
+bool core_wide(bool system_wide, const char *user_requested_cpu_list,
+ const struct cpu_topology *topology);
+
+#endif /* __SMT_H */
.se_width_idx = HISTC_DSO_SIZE,
};
+/* --sort dso_size */
+
+static int64_t
+sort__addr_cmp(struct hist_entry *left, struct hist_entry *right)
+{
+ u64 left_ip = left->ip;
+ u64 right_ip = right->ip;
+ struct map *left_map = left->ms.map;
+ struct map *right_map = right->ms.map;
+
+ if (left_map)
+ left_ip = left_map->unmap_ip(left_map, left_ip);
+ if (right_map)
+ right_ip = right_map->unmap_ip(right_map, right_ip);
+
+ return _sort__addr_cmp(left_ip, right_ip);
+}
+
+static int hist_entry__addr_snprintf(struct hist_entry *he, char *bf,
+ size_t size, unsigned int width)
+{
+ u64 ip = he->ip;
+ struct map *map = he->ms.map;
+
+ if (map)
+ ip = map->unmap_ip(map, ip);
+
+ return repsep_snprintf(bf, size, "%-#*llx", width, ip);
+}
+
+struct sort_entry sort_addr = {
+ .se_header = "Address",
+ .se_cmp = sort__addr_cmp,
+ .se_snprintf = hist_entry__addr_snprintf,
+ .se_width_idx = HISTC_ADDR,
+};
+
struct sort_dimension {
const char *name;
DIM(SORT_GLOBAL_INS_LAT, "ins_lat", sort_global_ins_lat),
DIM(SORT_LOCAL_PIPELINE_STAGE_CYC, "local_p_stage_cyc", sort_local_p_stage_cyc),
DIM(SORT_GLOBAL_PIPELINE_STAGE_CYC, "p_stage_cyc", sort_global_p_stage_cyc),
+ DIM(SORT_ADDR, "addr", sort_addr),
};
#undef DIM
extern struct sort_entry sort_sym_from;
extern struct sort_entry sort_sym_to;
extern struct sort_entry sort_srcline;
-extern enum sort_type sort__first_dimension;
extern const char default_mem_sort_order[];
struct res_sample {
SORT_GLOBAL_INS_LAT,
SORT_LOCAL_PIPELINE_STAGE_CYC,
SORT_GLOBAL_PIPELINE_STAGE_CYC,
+ SORT_ADDR,
/* branch stack specific sort keys */
__SORT_BRANCH_STACK,
};
extern struct sort_entry sort_thread;
-extern struct list_head hist_entry__sort_list;
struct evlist;
struct tep_handle;
return;
ps = evsel->stats;
- print_noise_pct(config, stddev_stats(&ps->res_stats[0]), avg);
+ print_noise_pct(config, stddev_stats(&ps->res_stats), avg);
}
static void print_cgroup(struct perf_stat_config *config, struct evsel *evsel)
id.socket,
id.die,
id.core);
- } else if (id.core > -1) {
+ } else if (id.cpu.cpu > -1) {
fprintf(config->output, "\"cpu\" : \"%d\", ",
id.cpu.cpu);
}
id.die,
config->csv_output ? 0 : -3,
id.core, config->csv_sep);
- } else if (id.core > -1) {
+ } else if (id.cpu.cpu > -1) {
fprintf(config->output, "CPU%*d%s",
config->csv_output ? 0 : -7,
id.cpu.cpu, config->csv_sep);
case AGGR_THREAD:
if (config->json_output) {
fprintf(config->output, "\"thread\" : \"%s-%d\", ",
- perf_thread_map__comm(evsel->core.threads, id.thread),
- perf_thread_map__pid(evsel->core.threads, id.thread));
+ perf_thread_map__comm(evsel->core.threads, id.thread_idx),
+ perf_thread_map__pid(evsel->core.threads, id.thread_idx));
} else {
fprintf(config->output, "%*s-%*d%s",
config->csv_output ? 0 : 16,
- perf_thread_map__comm(evsel->core.threads, id.thread),
+ perf_thread_map__comm(evsel->core.threads, id.thread_idx),
config->csv_output ? 0 : -8,
- perf_thread_map__pid(evsel->core.threads, id.thread),
+ perf_thread_map__pid(evsel->core.threads, id.thread_idx),
config->csv_sep);
}
break;
fprintf(os->fh, "%*s ", config->metric_only_len, unit);
}
-static int first_shadow_cpu_map_idx(struct perf_stat_config *config,
+static int first_shadow_map_idx(struct perf_stat_config *config,
struct evsel *evsel, const struct aggr_cpu_id *id)
{
struct perf_cpu_map *cpus = evsel__cpus(evsel);
if (config->aggr_mode == AGGR_NONE)
return perf_cpu_map__idx(cpus, id->cpu);
+ if (config->aggr_mode == AGGR_THREAD)
+ return id->thread_idx;
+
if (!config->aggr_get_id)
return 0;
}
perf_stat__print_shadow_stats(config, counter, uval,
- first_shadow_cpu_map_idx(config, counter, &id),
+ first_shadow_map_idx(config, counter, &id),
&out, &config->metric_events, st);
if (!config->csv_output && !config->metric_only && !config->json_output) {
print_noise(config, counter, noise);
val += perf_counts(counter->counts, idx, 0)->val;
}
perf_stat__update_shadow_stats(counter, val,
- first_shadow_cpu_map_idx(config, counter, &id),
+ first_shadow_map_idx(config, counter, &id),
&rt_stat);
}
}
buf[i].counter = counter;
buf[i].id = aggr_cpu_id__empty();
- buf[i].id.thread = thread;
+ buf[i].id.thread_idx = thread;
buf[i].uval = uval;
buf[i].val = val;
buf[i].run = run;
fprintf(output, "%s", prefix);
id = buf[thread].id;
- if (config->stats)
- printout(config, id, 0, buf[thread].counter, buf[thread].uval,
- prefix, buf[thread].run, buf[thread].ena, 1.0,
- &config->stats[id.thread]);
- else
- printout(config, id, 0, buf[thread].counter, buf[thread].uval,
- prefix, buf[thread].run, buf[thread].ena, 1.0,
- &rt_stat);
+ printout(config, id, 0, buf[thread].counter, buf[thread].uval,
+ prefix, buf[thread].run, buf[thread].ena, 1.0,
+ &rt_stat);
fputc('\n', output);
}
struct evsel *evsel;
enum stat_type type;
int ctx;
- int cpu_map_idx;
+ int map_idx; /* cpu or thread map index */
struct cgroup *cgrp;
- struct runtime_stat *stat;
struct stats stats;
u64 metric_total;
int metric_other;
rb_node);
const struct saved_value *b = entry;
- if (a->cpu_map_idx != b->cpu_map_idx)
- return a->cpu_map_idx - b->cpu_map_idx;
+ if (a->map_idx != b->map_idx)
+ return a->map_idx - b->map_idx;
/*
* Previously the rbtree was used to link generic metrics.
if (a->cgrp != b->cgrp)
return (char *)a->cgrp < (char *)b->cgrp ? -1 : +1;
- if (a->evsel == NULL && b->evsel == NULL) {
- if (a->stat == b->stat)
- return 0;
-
- if ((char *)a->stat < (char *)b->stat)
- return -1;
-
- return 1;
- }
-
if (a->evsel == b->evsel)
return 0;
if ((char *)a->evsel < (char *)b->evsel)
}
static struct saved_value *saved_value_lookup(struct evsel *evsel,
- int cpu_map_idx,
+ int map_idx,
bool create,
enum stat_type type,
int ctx,
struct rblist *rblist;
struct rb_node *nd;
struct saved_value dm = {
- .cpu_map_idx = cpu_map_idx,
+ .map_idx = map_idx,
.evsel = evsel,
.type = type,
.ctx = ctx,
- .stat = st,
.cgrp = cgrp,
};
static void update_runtime_stat(struct runtime_stat *st,
enum stat_type type,
- int cpu_map_idx, u64 count,
+ int map_idx, u64 count,
struct runtime_stat_data *rsd)
{
- struct saved_value *v = saved_value_lookup(NULL, cpu_map_idx, true, type,
+ struct saved_value *v = saved_value_lookup(NULL, map_idx, true, type,
rsd->ctx, st, rsd->cgrp);
if (v)
* instruction rates, etc:
*/
void perf_stat__update_shadow_stats(struct evsel *counter, u64 count,
- int cpu_map_idx, struct runtime_stat *st)
+ int map_idx, struct runtime_stat *st)
{
u64 count_ns = count;
struct saved_value *v;
count *= counter->scale;
if (evsel__is_clock(counter))
- update_runtime_stat(st, STAT_NSECS, cpu_map_idx, count_ns, &rsd);
+ update_runtime_stat(st, STAT_NSECS, map_idx, count_ns, &rsd);
else if (evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
- update_runtime_stat(st, STAT_CYCLES, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_CYCLES, map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, CYCLES_IN_TX))
- update_runtime_stat(st, STAT_CYCLES_IN_TX, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_CYCLES_IN_TX, map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TRANSACTION_START))
- update_runtime_stat(st, STAT_TRANSACTION, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_TRANSACTION, map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, ELISION_START))
- update_runtime_stat(st, STAT_ELISION, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_ELISION, map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_TOTAL_SLOTS))
update_runtime_stat(st, STAT_TOPDOWN_TOTAL_SLOTS,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_SLOTS_ISSUED))
update_runtime_stat(st, STAT_TOPDOWN_SLOTS_ISSUED,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_SLOTS_RETIRED))
update_runtime_stat(st, STAT_TOPDOWN_SLOTS_RETIRED,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_FETCH_BUBBLES))
update_runtime_stat(st, STAT_TOPDOWN_FETCH_BUBBLES,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_RECOVERY_BUBBLES))
update_runtime_stat(st, STAT_TOPDOWN_RECOVERY_BUBBLES,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_RETIRING))
update_runtime_stat(st, STAT_TOPDOWN_RETIRING,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_BAD_SPEC))
update_runtime_stat(st, STAT_TOPDOWN_BAD_SPEC,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_FE_BOUND))
update_runtime_stat(st, STAT_TOPDOWN_FE_BOUND,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_BE_BOUND))
update_runtime_stat(st, STAT_TOPDOWN_BE_BOUND,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_HEAVY_OPS))
update_runtime_stat(st, STAT_TOPDOWN_HEAVY_OPS,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_BR_MISPREDICT))
update_runtime_stat(st, STAT_TOPDOWN_BR_MISPREDICT,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_FETCH_LAT))
update_runtime_stat(st, STAT_TOPDOWN_FETCH_LAT,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, TOPDOWN_MEM_BOUND))
update_runtime_stat(st, STAT_TOPDOWN_MEM_BOUND,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_FRONTEND))
update_runtime_stat(st, STAT_STALLED_CYCLES_FRONT,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_BACKEND))
update_runtime_stat(st, STAT_STALLED_CYCLES_BACK,
- cpu_map_idx, count, &rsd);
+ map_idx, count, &rsd);
else if (evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
- update_runtime_stat(st, STAT_BRANCHES, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_BRANCHES, map_idx, count, &rsd);
else if (evsel__match(counter, HARDWARE, HW_CACHE_REFERENCES))
- update_runtime_stat(st, STAT_CACHEREFS, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_CACHEREFS, map_idx, count, &rsd);
else if (evsel__match(counter, HW_CACHE, HW_CACHE_L1D))
- update_runtime_stat(st, STAT_L1_DCACHE, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_L1_DCACHE, map_idx, count, &rsd);
else if (evsel__match(counter, HW_CACHE, HW_CACHE_L1I))
- update_runtime_stat(st, STAT_L1_ICACHE, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_L1_ICACHE, map_idx, count, &rsd);
else if (evsel__match(counter, HW_CACHE, HW_CACHE_LL))
- update_runtime_stat(st, STAT_LL_CACHE, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_LL_CACHE, map_idx, count, &rsd);
else if (evsel__match(counter, HW_CACHE, HW_CACHE_DTLB))
- update_runtime_stat(st, STAT_DTLB_CACHE, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_DTLB_CACHE, map_idx, count, &rsd);
else if (evsel__match(counter, HW_CACHE, HW_CACHE_ITLB))
- update_runtime_stat(st, STAT_ITLB_CACHE, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_ITLB_CACHE, map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, SMI_NUM))
- update_runtime_stat(st, STAT_SMI_NUM, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_SMI_NUM, map_idx, count, &rsd);
else if (perf_stat_evsel__is(counter, APERF))
- update_runtime_stat(st, STAT_APERF, cpu_map_idx, count, &rsd);
+ update_runtime_stat(st, STAT_APERF, map_idx, count, &rsd);
if (counter->collect_stat) {
- v = saved_value_lookup(counter, cpu_map_idx, true, STAT_NONE, 0, st,
+ v = saved_value_lookup(counter, map_idx, true, STAT_NONE, 0, st,
rsd.cgrp);
update_stats(&v->stats, count);
if (counter->metric_leader)
v->metric_total += count;
} else if (counter->metric_leader) {
v = saved_value_lookup(counter->metric_leader,
- cpu_map_idx, true, STAT_NONE, 0, st, rsd.cgrp);
+ map_idx, true, STAT_NONE, 0, st, rsd.cgrp);
v->metric_total += count;
v->metric_other++;
}
}
static double runtime_stat_avg(struct runtime_stat *st,
- enum stat_type type, int cpu_map_idx,
+ enum stat_type type, int map_idx,
struct runtime_stat_data *rsd)
{
struct saved_value *v;
- v = saved_value_lookup(NULL, cpu_map_idx, false, type, rsd->ctx, st, rsd->cgrp);
+ v = saved_value_lookup(NULL, map_idx, false, type, rsd->ctx, st, rsd->cgrp);
if (!v)
return 0.0;
}
static double runtime_stat_n(struct runtime_stat *st,
- enum stat_type type, int cpu_map_idx,
+ enum stat_type type, int map_idx,
struct runtime_stat_data *rsd)
{
struct saved_value *v;
- v = saved_value_lookup(NULL, cpu_map_idx, false, type, rsd->ctx, st, rsd->cgrp);
+ v = saved_value_lookup(NULL, map_idx, false, type, rsd->ctx, st, rsd->cgrp);
if (!v)
return 0.0;
}
static void print_stalled_cycles_frontend(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_stalled_cycles_backend(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_branch_misses(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_BRANCHES, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_BRANCHES, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_l1_dcache_misses(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_L1_DCACHE, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_L1_DCACHE, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_l1_icache_misses(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_L1_ICACHE, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_L1_ICACHE, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_dtlb_cache_misses(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_DTLB_CACHE, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_DTLB_CACHE, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_itlb_cache_misses(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_ITLB_CACHE, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_ITLB_CACHE, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
}
static void print_ll_cache_misses(struct perf_stat_config *config,
- int cpu_map_idx, double avg,
+ int map_idx, double avg,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double total, ratio = 0.0;
const char *color;
- total = runtime_stat_avg(st, STAT_LL_CACHE, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_LL_CACHE, map_idx, rsd);
if (total)
ratio = avg / total * 100.0;
return x;
}
-static double td_total_slots(int cpu_map_idx, struct runtime_stat *st,
+static double td_total_slots(int map_idx, struct runtime_stat *st,
struct runtime_stat_data *rsd)
{
- return runtime_stat_avg(st, STAT_TOPDOWN_TOTAL_SLOTS, cpu_map_idx, rsd);
+ return runtime_stat_avg(st, STAT_TOPDOWN_TOTAL_SLOTS, map_idx, rsd);
}
-static double td_bad_spec(int cpu_map_idx, struct runtime_stat *st,
+static double td_bad_spec(int map_idx, struct runtime_stat *st,
struct runtime_stat_data *rsd)
{
double bad_spec = 0;
double total_slots;
double total;
- total = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_ISSUED, cpu_map_idx, rsd) -
- runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED, cpu_map_idx, rsd) +
- runtime_stat_avg(st, STAT_TOPDOWN_RECOVERY_BUBBLES, cpu_map_idx, rsd);
+ total = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_ISSUED, map_idx, rsd) -
+ runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED, map_idx, rsd) +
+ runtime_stat_avg(st, STAT_TOPDOWN_RECOVERY_BUBBLES, map_idx, rsd);
- total_slots = td_total_slots(cpu_map_idx, st, rsd);
+ total_slots = td_total_slots(map_idx, st, rsd);
if (total_slots)
bad_spec = total / total_slots;
return sanitize_val(bad_spec);
}
-static double td_retiring(int cpu_map_idx, struct runtime_stat *st,
+static double td_retiring(int map_idx, struct runtime_stat *st,
struct runtime_stat_data *rsd)
{
double retiring = 0;
- double total_slots = td_total_slots(cpu_map_idx, st, rsd);
+ double total_slots = td_total_slots(map_idx, st, rsd);
double ret_slots = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED,
- cpu_map_idx, rsd);
+ map_idx, rsd);
if (total_slots)
retiring = ret_slots / total_slots;
return retiring;
}
-static double td_fe_bound(int cpu_map_idx, struct runtime_stat *st,
+static double td_fe_bound(int map_idx, struct runtime_stat *st,
struct runtime_stat_data *rsd)
{
double fe_bound = 0;
- double total_slots = td_total_slots(cpu_map_idx, st, rsd);
+ double total_slots = td_total_slots(map_idx, st, rsd);
double fetch_bub = runtime_stat_avg(st, STAT_TOPDOWN_FETCH_BUBBLES,
- cpu_map_idx, rsd);
+ map_idx, rsd);
if (total_slots)
fe_bound = fetch_bub / total_slots;
return fe_bound;
}
-static double td_be_bound(int cpu_map_idx, struct runtime_stat *st,
+static double td_be_bound(int map_idx, struct runtime_stat *st,
struct runtime_stat_data *rsd)
{
- double sum = (td_fe_bound(cpu_map_idx, st, rsd) +
- td_bad_spec(cpu_map_idx, st, rsd) +
- td_retiring(cpu_map_idx, st, rsd));
+ double sum = (td_fe_bound(map_idx, st, rsd) +
+ td_bad_spec(map_idx, st, rsd) +
+ td_retiring(map_idx, st, rsd));
if (sum == 0)
return 0;
return sanitize_val(1.0 - sum);
* the ratios we need to recreate the sum.
*/
-static double td_metric_ratio(int cpu_map_idx, enum stat_type type,
+static double td_metric_ratio(int map_idx, enum stat_type type,
struct runtime_stat *stat,
struct runtime_stat_data *rsd)
{
- double sum = runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, cpu_map_idx, rsd) +
- runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, cpu_map_idx, rsd) +
- runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, cpu_map_idx, rsd) +
- runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, cpu_map_idx, rsd);
- double d = runtime_stat_avg(stat, type, cpu_map_idx, rsd);
+ double sum = runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, map_idx, rsd) +
+ runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, map_idx, rsd) +
+ runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, map_idx, rsd) +
+ runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, map_idx, rsd);
+ double d = runtime_stat_avg(stat, type, map_idx, rsd);
if (sum)
return d / sum;
* We allow two missing.
*/
-static bool full_td(int cpu_map_idx, struct runtime_stat *stat,
+static bool full_td(int map_idx, struct runtime_stat *stat,
struct runtime_stat_data *rsd)
{
int c = 0;
- if (runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, cpu_map_idx, rsd) > 0)
+ if (runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, map_idx, rsd) > 0)
c++;
- if (runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, cpu_map_idx, rsd) > 0)
+ if (runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, map_idx, rsd) > 0)
c++;
- if (runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, cpu_map_idx, rsd) > 0)
+ if (runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, map_idx, rsd) > 0)
c++;
- if (runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, cpu_map_idx, rsd) > 0)
+ if (runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, map_idx, rsd) > 0)
c++;
return c >= 2;
}
-static void print_smi_cost(struct perf_stat_config *config, int cpu_map_idx,
+static void print_smi_cost(struct perf_stat_config *config, int map_idx,
struct perf_stat_output_ctx *out,
struct runtime_stat *st,
struct runtime_stat_data *rsd)
double smi_num, aperf, cycles, cost = 0.0;
const char *color = NULL;
- smi_num = runtime_stat_avg(st, STAT_SMI_NUM, cpu_map_idx, rsd);
- aperf = runtime_stat_avg(st, STAT_APERF, cpu_map_idx, rsd);
- cycles = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, rsd);
+ smi_num = runtime_stat_avg(st, STAT_SMI_NUM, map_idx, rsd);
+ aperf = runtime_stat_avg(st, STAT_APERF, map_idx, rsd);
+ cycles = runtime_stat_avg(st, STAT_CYCLES, map_idx, rsd);
if ((cycles == 0) || (aperf == 0))
return;
static int prepare_metric(struct evsel **metric_events,
struct metric_ref *metric_refs,
struct expr_parse_ctx *pctx,
- int cpu_map_idx,
+ int map_idx,
struct runtime_stat *st)
{
double scale;
abort();
}
} else {
- v = saved_value_lookup(metric_events[i], cpu_map_idx, false,
+ v = saved_value_lookup(metric_events[i], map_idx, false,
STAT_NONE, 0, st,
metric_events[i]->cgrp);
if (!v)
break;
stats = &v->stats;
- scale = 1.0;
+ /*
+ * If an event was scaled during stat gathering, reverse
+ * the scale before computing the metric.
+ */
+ scale = 1.0 / metric_events[i]->scale;
+
source_count = evsel__source_count(metric_events[i]);
if (v->metric_other)
- metric_total = v->metric_total;
+ metric_total = v->metric_total * scale;
}
n = strdup(evsel__metric_id(metric_events[i]));
if (!n)
const char *metric_name,
const char *metric_unit,
int runtime,
- int cpu_map_idx,
+ int map_idx,
struct perf_stat_output_ctx *out,
struct runtime_stat *st)
{
if (!pctx)
return;
- pctx->runtime = runtime;
- i = prepare_metric(metric_events, metric_refs, pctx, cpu_map_idx, st);
+ if (config->user_requested_cpu_list)
+ pctx->sctx.user_requested_cpu_list = strdup(config->user_requested_cpu_list);
+ pctx->sctx.runtime = runtime;
+ pctx->sctx.system_wide = config->system_wide;
+ i = prepare_metric(metric_events, metric_refs, pctx, map_idx, st);
if (i < 0) {
expr__ctx_free(pctx);
return;
expr__ctx_free(pctx);
}
-double test_generic_metric(struct metric_expr *mexp, int cpu_map_idx, struct runtime_stat *st)
+double test_generic_metric(struct metric_expr *mexp, int map_idx, struct runtime_stat *st)
{
struct expr_parse_ctx *pctx;
double ratio = 0.0;
if (!pctx)
return NAN;
- if (prepare_metric(mexp->metric_events, mexp->metric_refs, pctx, cpu_map_idx, st) < 0)
+ if (prepare_metric(mexp->metric_events, mexp->metric_refs, pctx, map_idx, st) < 0)
goto out;
if (expr__parse(&ratio, pctx, mexp->metric_expr))
void perf_stat__print_shadow_stats(struct perf_stat_config *config,
struct evsel *evsel,
- double avg, int cpu_map_idx,
+ double avg, int map_idx,
struct perf_stat_output_ctx *out,
struct rblist *metric_events,
struct runtime_stat *st)
if (config->iostat_run) {
iostat_print_metric(config, evsel, out);
} else if (evsel__match(evsel, HARDWARE, HW_INSTRUCTIONS)) {
- total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES, map_idx, &rsd);
if (total) {
ratio = avg / total;
print_metric(config, ctxp, NULL, NULL, "insn per cycle", 0);
}
- total = runtime_stat_avg(st, STAT_STALLED_CYCLES_FRONT, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_STALLED_CYCLES_FRONT, map_idx, &rsd);
total = max(total, runtime_stat_avg(st,
STAT_STALLED_CYCLES_BACK,
- cpu_map_idx, &rsd));
+ map_idx, &rsd));
if (total && avg) {
out->new_line(config, ctxp);
ratio);
}
} else if (evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES)) {
- if (runtime_stat_n(st, STAT_BRANCHES, cpu_map_idx, &rsd) != 0)
- print_branch_misses(config, cpu_map_idx, avg, out, st, &rsd);
+ if (runtime_stat_n(st, STAT_BRANCHES, map_idx, &rsd) != 0)
+ print_branch_misses(config, map_idx, avg, out, st, &rsd);
else
print_metric(config, ctxp, NULL, NULL, "of all branches", 0);
} else if (
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
- if (runtime_stat_n(st, STAT_L1_DCACHE, cpu_map_idx, &rsd) != 0)
- print_l1_dcache_misses(config, cpu_map_idx, avg, out, st, &rsd);
+ if (runtime_stat_n(st, STAT_L1_DCACHE, map_idx, &rsd) != 0)
+ print_l1_dcache_misses(config, map_idx, avg, out, st, &rsd);
else
print_metric(config, ctxp, NULL, NULL, "of all L1-dcache accesses", 0);
} else if (
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
- if (runtime_stat_n(st, STAT_L1_ICACHE, cpu_map_idx, &rsd) != 0)
- print_l1_icache_misses(config, cpu_map_idx, avg, out, st, &rsd);
+ if (runtime_stat_n(st, STAT_L1_ICACHE, map_idx, &rsd) != 0)
+ print_l1_icache_misses(config, map_idx, avg, out, st, &rsd);
else
print_metric(config, ctxp, NULL, NULL, "of all L1-icache accesses", 0);
} else if (
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
- if (runtime_stat_n(st, STAT_DTLB_CACHE, cpu_map_idx, &rsd) != 0)
- print_dtlb_cache_misses(config, cpu_map_idx, avg, out, st, &rsd);
+ if (runtime_stat_n(st, STAT_DTLB_CACHE, map_idx, &rsd) != 0)
+ print_dtlb_cache_misses(config, map_idx, avg, out, st, &rsd);
else
print_metric(config, ctxp, NULL, NULL, "of all dTLB cache accesses", 0);
} else if (
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
- if (runtime_stat_n(st, STAT_ITLB_CACHE, cpu_map_idx, &rsd) != 0)
- print_itlb_cache_misses(config, cpu_map_idx, avg, out, st, &rsd);
+ if (runtime_stat_n(st, STAT_ITLB_CACHE, map_idx, &rsd) != 0)
+ print_itlb_cache_misses(config, map_idx, avg, out, st, &rsd);
else
print_metric(config, ctxp, NULL, NULL, "of all iTLB cache accesses", 0);
} else if (
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) {
- if (runtime_stat_n(st, STAT_LL_CACHE, cpu_map_idx, &rsd) != 0)
- print_ll_cache_misses(config, cpu_map_idx, avg, out, st, &rsd);
+ if (runtime_stat_n(st, STAT_LL_CACHE, map_idx, &rsd) != 0)
+ print_ll_cache_misses(config, map_idx, avg, out, st, &rsd);
else
print_metric(config, ctxp, NULL, NULL, "of all LL-cache accesses", 0);
} else if (evsel__match(evsel, HARDWARE, HW_CACHE_MISSES)) {
- total = runtime_stat_avg(st, STAT_CACHEREFS, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_CACHEREFS, map_idx, &rsd);
if (total)
ratio = avg * 100 / total;
- if (runtime_stat_n(st, STAT_CACHEREFS, cpu_map_idx, &rsd) != 0)
+ if (runtime_stat_n(st, STAT_CACHEREFS, map_idx, &rsd) != 0)
print_metric(config, ctxp, NULL, "%8.3f %%",
"of all cache refs", ratio);
else
print_metric(config, ctxp, NULL, NULL, "of all cache refs", 0);
} else if (evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) {
- print_stalled_cycles_frontend(config, cpu_map_idx, avg, out, st, &rsd);
+ print_stalled_cycles_frontend(config, map_idx, avg, out, st, &rsd);
} else if (evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_BACKEND)) {
- print_stalled_cycles_backend(config, cpu_map_idx, avg, out, st, &rsd);
+ print_stalled_cycles_backend(config, map_idx, avg, out, st, &rsd);
} else if (evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) {
- total = runtime_stat_avg(st, STAT_NSECS, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_NSECS, map_idx, &rsd);
if (total) {
ratio = avg / total;
print_metric(config, ctxp, NULL, NULL, "Ghz", 0);
}
} else if (perf_stat_evsel__is(evsel, CYCLES_IN_TX)) {
- total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES, map_idx, &rsd);
if (total)
print_metric(config, ctxp, NULL,
print_metric(config, ctxp, NULL, NULL, "transactional cycles",
0);
} else if (perf_stat_evsel__is(evsel, CYCLES_IN_TX_CP)) {
- total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, &rsd);
- total2 = runtime_stat_avg(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES, map_idx, &rsd);
+ total2 = runtime_stat_avg(st, STAT_CYCLES_IN_TX, map_idx, &rsd);
if (total2 < avg)
total2 = avg;
else
print_metric(config, ctxp, NULL, NULL, "aborted cycles", 0);
} else if (perf_stat_evsel__is(evsel, TRANSACTION_START)) {
- total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, map_idx, &rsd);
if (avg)
ratio = total / avg;
- if (runtime_stat_n(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd) != 0)
+ if (runtime_stat_n(st, STAT_CYCLES_IN_TX, map_idx, &rsd) != 0)
print_metric(config, ctxp, NULL, "%8.0f",
"cycles / transaction", ratio);
else
print_metric(config, ctxp, NULL, NULL, "cycles / transaction",
0);
} else if (perf_stat_evsel__is(evsel, ELISION_START)) {
- total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, map_idx, &rsd);
if (avg)
ratio = total / avg;
else
print_metric(config, ctxp, NULL, NULL, "CPUs utilized", 0);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_FETCH_BUBBLES)) {
- double fe_bound = td_fe_bound(cpu_map_idx, st, &rsd);
+ double fe_bound = td_fe_bound(map_idx, st, &rsd);
if (fe_bound > 0.2)
color = PERF_COLOR_RED;
print_metric(config, ctxp, color, "%8.1f%%", "frontend bound",
fe_bound * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_SLOTS_RETIRED)) {
- double retiring = td_retiring(cpu_map_idx, st, &rsd);
+ double retiring = td_retiring(map_idx, st, &rsd);
if (retiring > 0.7)
color = PERF_COLOR_GREEN;
print_metric(config, ctxp, color, "%8.1f%%", "retiring",
retiring * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_RECOVERY_BUBBLES)) {
- double bad_spec = td_bad_spec(cpu_map_idx, st, &rsd);
+ double bad_spec = td_bad_spec(map_idx, st, &rsd);
if (bad_spec > 0.1)
color = PERF_COLOR_RED;
print_metric(config, ctxp, color, "%8.1f%%", "bad speculation",
bad_spec * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_SLOTS_ISSUED)) {
- double be_bound = td_be_bound(cpu_map_idx, st, &rsd);
+ double be_bound = td_be_bound(map_idx, st, &rsd);
const char *name = "backend bound";
static int have_recovery_bubbles = -1;
if (be_bound > 0.2)
color = PERF_COLOR_RED;
- if (td_total_slots(cpu_map_idx, st, &rsd) > 0)
+ if (td_total_slots(map_idx, st, &rsd) > 0)
print_metric(config, ctxp, color, "%8.1f%%", name,
be_bound * 100.);
else
print_metric(config, ctxp, NULL, NULL, name, 0);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_RETIRING) &&
- full_td(cpu_map_idx, st, &rsd)) {
- double retiring = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd)) {
+ double retiring = td_metric_ratio(map_idx,
STAT_TOPDOWN_RETIRING, st,
&rsd);
if (retiring > 0.7)
print_metric(config, ctxp, color, "%8.1f%%", "Retiring",
retiring * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_FE_BOUND) &&
- full_td(cpu_map_idx, st, &rsd)) {
- double fe_bound = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd)) {
+ double fe_bound = td_metric_ratio(map_idx,
STAT_TOPDOWN_FE_BOUND, st,
&rsd);
if (fe_bound > 0.2)
print_metric(config, ctxp, color, "%8.1f%%", "Frontend Bound",
fe_bound * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_BE_BOUND) &&
- full_td(cpu_map_idx, st, &rsd)) {
- double be_bound = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd)) {
+ double be_bound = td_metric_ratio(map_idx,
STAT_TOPDOWN_BE_BOUND, st,
&rsd);
if (be_bound > 0.2)
print_metric(config, ctxp, color, "%8.1f%%", "Backend Bound",
be_bound * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_BAD_SPEC) &&
- full_td(cpu_map_idx, st, &rsd)) {
- double bad_spec = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd)) {
+ double bad_spec = td_metric_ratio(map_idx,
STAT_TOPDOWN_BAD_SPEC, st,
&rsd);
if (bad_spec > 0.1)
print_metric(config, ctxp, color, "%8.1f%%", "Bad Speculation",
bad_spec * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_HEAVY_OPS) &&
- full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) {
- double retiring = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) {
+ double retiring = td_metric_ratio(map_idx,
STAT_TOPDOWN_RETIRING, st,
&rsd);
- double heavy_ops = td_metric_ratio(cpu_map_idx,
+ double heavy_ops = td_metric_ratio(map_idx,
STAT_TOPDOWN_HEAVY_OPS, st,
&rsd);
double light_ops = retiring - heavy_ops;
print_metric(config, ctxp, color, "%8.1f%%", "Light Operations",
light_ops * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_BR_MISPREDICT) &&
- full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) {
- double bad_spec = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) {
+ double bad_spec = td_metric_ratio(map_idx,
STAT_TOPDOWN_BAD_SPEC, st,
&rsd);
- double br_mis = td_metric_ratio(cpu_map_idx,
+ double br_mis = td_metric_ratio(map_idx,
STAT_TOPDOWN_BR_MISPREDICT, st,
&rsd);
double m_clears = bad_spec - br_mis;
print_metric(config, ctxp, color, "%8.1f%%", "Machine Clears",
m_clears * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_FETCH_LAT) &&
- full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) {
- double fe_bound = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) {
+ double fe_bound = td_metric_ratio(map_idx,
STAT_TOPDOWN_FE_BOUND, st,
&rsd);
- double fetch_lat = td_metric_ratio(cpu_map_idx,
+ double fetch_lat = td_metric_ratio(map_idx,
STAT_TOPDOWN_FETCH_LAT, st,
&rsd);
double fetch_bw = fe_bound - fetch_lat;
print_metric(config, ctxp, color, "%8.1f%%", "Fetch Bandwidth",
fetch_bw * 100.);
} else if (perf_stat_evsel__is(evsel, TOPDOWN_MEM_BOUND) &&
- full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) {
- double be_bound = td_metric_ratio(cpu_map_idx,
+ full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) {
+ double be_bound = td_metric_ratio(map_idx,
STAT_TOPDOWN_BE_BOUND, st,
&rsd);
- double mem_bound = td_metric_ratio(cpu_map_idx,
+ double mem_bound = td_metric_ratio(map_idx,
STAT_TOPDOWN_MEM_BOUND, st,
&rsd);
double core_bound = be_bound - mem_bound;
core_bound * 100.);
} else if (evsel->metric_expr) {
generic_metric(config, evsel->metric_expr, evsel->metric_events, NULL,
- evsel->name, evsel->metric_name, NULL, 1, cpu_map_idx, out, st);
- } else if (runtime_stat_n(st, STAT_NSECS, cpu_map_idx, &rsd) != 0) {
+ evsel->name, evsel->metric_name, NULL, 1,
+ map_idx, out, st);
+ } else if (runtime_stat_n(st, STAT_NSECS, map_idx, &rsd) != 0) {
char unit = ' ';
char unit_buf[10] = "/sec";
- total = runtime_stat_avg(st, STAT_NSECS, cpu_map_idx, &rsd);
+ total = runtime_stat_avg(st, STAT_NSECS, map_idx, &rsd);
if (total)
ratio = convert_unit_double(1000000000.0 * avg / total, &unit);
snprintf(unit_buf, sizeof(unit_buf), "%c/sec", unit);
print_metric(config, ctxp, NULL, "%8.3f", unit_buf, ratio);
} else if (perf_stat_evsel__is(evsel, SMI_NUM)) {
- print_smi_cost(config, cpu_map_idx, out, st, &rsd);
+ print_smi_cost(config, map_idx, out, st, &rsd);
} else {
num = 0;
}
if (num++ > 0)
out->new_line(config, ctxp);
generic_metric(config, mexp->metric_expr, mexp->metric_events,
- mexp->metric_refs, evsel->name, mexp->metric_name,
- mexp->metric_unit, mexp->runtime, cpu_map_idx, out, st);
+ mexp->metric_refs, evsel->name, mexp->metric_name,
+ mexp->metric_unit, mexp->runtime,
+ map_idx, out, st);
}
}
if (num == 0)
#include "evlist.h"
#include "evsel.h"
#include "thread_map.h"
-#include "hashmap.h"
+#ifdef HAVE_LIBBPF_SUPPORT
+#include <bpf/hashmap.h>
+#else
+#include "util/hashmap.h"
+#endif
#include <linux/zalloc.h>
void update_stats(struct stats *stats, u64 val)
static void evsel__reset_stat_priv(struct evsel *evsel)
{
- int i;
struct perf_stat_evsel *ps = evsel->stats;
- for (i = 0; i < 3; i++)
- init_stats(&ps->res_stats[i]);
-
- perf_stat_evsel_id_init(evsel);
+ init_stats(&ps->res_stats);
}
static int evsel__alloc_stat_priv(struct evsel *evsel)
evsel->stats = zalloc(sizeof(struct perf_stat_evsel));
if (evsel->stats == NULL)
return -ENOMEM;
+ perf_stat_evsel_id_init(evsel);
evsel__reset_stat_priv(evsel);
return 0;
}
}
if (config->aggr_mode == AGGR_THREAD) {
- if (config->stats)
- perf_stat__update_shadow_stats(evsel,
- count->val, 0, &config->stats[thread]);
- else
- perf_stat__update_shadow_stats(evsel,
- count->val, 0, &rt_stat);
+ perf_stat__update_shadow_stats(evsel, count->val,
+ thread, &rt_stat);
}
break;
case AGGR_GLOBAL:
int ncpus = evsel__nr_cpus(counter);
int idx, thread;
- if (counter->core.system_wide)
- nthreads = 1;
-
for (thread = 0; thread < nthreads; thread++) {
for (idx = 0; idx < ncpus; idx++) {
if (process_counter_values(config, counter, idx, thread,
struct perf_counts_values *aggr = &counter->counts->aggr;
struct perf_stat_evsel *ps = counter->stats;
u64 *count = counter->counts->aggr.values;
- int i, ret;
+ int ret;
aggr->val = aggr->ena = aggr->run = 0;
evsel__compute_deltas(counter, -1, -1, aggr);
perf_counts_values__scale(aggr, config->scale, &counter->counts->scaled);
- for (i = 0; i < 3; i++)
- update_stats(&ps->res_stats[i], count[i]);
+ update_stats(&ps->res_stats, *count);
if (verbose > 0) {
fprintf(config->output, "%s: %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",
};
struct perf_stat_evsel {
- struct stats res_stats[3];
+ struct stats res_stats;
enum perf_stat_evsel_id id;
u64 *group_data;
};
bool stop_read_counter;
bool quiet;
bool iostat_run;
+ char *user_requested_cpu_list;
+ bool system_wide;
FILE *output;
unsigned int interval;
unsigned int timeout;
int run_count;
int print_free_counters_hint;
int print_mixed_hw_group_error;
- struct runtime_stat *stats;
- int stats_num;
const char *csv_sep;
struct stats *walltime_nsecs_stats;
struct rusage ru_data;
void perf_stat__reset_shadow_stats(void);
void perf_stat__reset_shadow_per_stat(struct runtime_stat *st);
void perf_stat__update_shadow_stats(struct evsel *counter, u64 count,
- int cpu_map_idx, struct runtime_stat *st);
+ int map_idx, struct runtime_stat *st);
struct perf_stat_output_ctx {
void *ctx;
print_metric_t print_metric;
void perf_stat__print_shadow_stats(struct perf_stat_config *config,
struct evsel *evsel,
- double avg, int cpu,
+ double avg, int map_idx,
struct perf_stat_output_ctx *out,
struct rblist *metric_events,
struct runtime_stat *st);
struct target *_target, struct timespec *ts, int argc, const char **argv);
struct metric_expr;
-double test_generic_metric(struct metric_expr *mexp, int cpu_map_idx, struct runtime_stat *st);
+double test_generic_metric(struct metric_expr *mexp, int map_idx, struct runtime_stat *st);
#endif
"....................................................................."
".....................................................................";
-#define K 1024LL
/*
* perf_atoll()
* Parse (\d+)(b|B|kb|KB|mb|MB|gb|GB|tb|TB) (e.g. "256MB")
char newmapname[PATH_MAX];
const char *map_path = dso->long_name;
+ mutex_lock(&dso->lock);
perfmap = strncmp(dso->name, "/tmp/perf-", 10) == 0;
if (perfmap) {
if (dso->nsinfo && (dso__find_perf_map(newmapname,
}
nsinfo__mountns_enter(dso->nsinfo, &nsc);
- pthread_mutex_lock(&dso->lock);
/* check again under the dso->lock */
if (dso__loaded(dso)) {
ret = 0;
out:
dso__set_loaded(dso);
- pthread_mutex_unlock(&dso->lock);
+ mutex_unlock(&dso->lock);
nsinfo__mountns_exit(&nsc);
return ret;
}
static void perf_record_mmap2__read_build_id(struct perf_record_mmap2 *event,
+ struct machine *machine,
bool is_kernel)
{
struct build_id bid;
struct nsinfo *nsi;
struct nscookie nc;
+ struct dso *dso = NULL;
+ struct dso_id id;
int rc;
if (is_kernel) {
goto out;
}
+ id.maj = event->maj;
+ id.min = event->min;
+ id.ino = event->ino;
+ id.ino_generation = event->ino_generation;
+
+ dso = dsos__findnew_id(&machine->dsos, event->filename, &id);
+ if (dso && dso->has_build_id) {
+ bid = dso->bid;
+ rc = 0;
+ goto out;
+ }
+
nsi = nsinfo__new(event->pid);
nsinfo__mountns_enter(nsi, &nc);
event->header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID;
event->__reserved_1 = 0;
event->__reserved_2 = 0;
+
+ if (dso && !dso->has_build_id)
+ dso__set_build_id(dso, &bid);
} else {
if (event->filename[0] == '/') {
pr_debug2("Failed to read build ID for %s\n",
event->filename);
}
}
+ dso__put(dso);
}
int perf_event__synthesize_mmap_events(struct perf_tool *tool,
event->mmap2.tid = pid;
if (symbol_conf.buildid_mmap2)
- perf_record_mmap2__read_build_id(&event->mmap2, false);
+ perf_record_mmap2__read_build_id(&event->mmap2, machine, false);
if (perf_tool__process_synth_event(tool, event, machine, process) != 0) {
rc = -1;
memcpy(event->mmap2.filename, pos->dso->long_name,
pos->dso->long_name_len + 1);
- perf_record_mmap2__read_build_id(&event->mmap2, false);
+ perf_record_mmap2__read_build_id(&event->mmap2, machine, false);
} else {
size = PERF_ALIGN(pos->dso->long_name_len + 1, sizeof(u64));
event->mmap.header.type = PERF_RECORD_MMAP;
event->mmap2.len = map->end - event->mmap.start;
event->mmap2.pid = machine->pid;
- perf_record_mmap2__read_build_id(&event->mmap2, true);
+ perf_record_mmap2__read_build_id(&event->mmap2, machine, true);
} else {
size = snprintf(event->mmap.filename, sizeof(event->mmap.filename),
"%s%s", machine->mmap_name, kmap->ref_reloc_sym->name) + 1;
return err;
}
-static void synthesize_cpus(struct perf_record_cpu_map_data *data,
- const struct perf_cpu_map *map)
-{
- int i, map_nr = perf_cpu_map__nr(map);
-
- data->cpus_data.nr = map_nr;
+struct synthesize_cpu_map_data {
+ const struct perf_cpu_map *map;
+ int nr;
+ int min_cpu;
+ int max_cpu;
+ int has_any_cpu;
+ int type;
+ size_t size;
+ struct perf_record_cpu_map_data *data;
+};
- for (i = 0; i < map_nr; i++)
- data->cpus_data.cpu[i] = perf_cpu_map__cpu(map, i).cpu;
+static void synthesize_cpus(struct synthesize_cpu_map_data *data)
+{
+ data->data->type = PERF_CPU_MAP__CPUS;
+ data->data->cpus_data.nr = data->nr;
+ for (int i = 0; i < data->nr; i++)
+ data->data->cpus_data.cpu[i] = perf_cpu_map__cpu(data->map, i).cpu;
}
-static void synthesize_mask(struct perf_record_cpu_map_data *data,
- const struct perf_cpu_map *map, int max)
+static void synthesize_mask(struct synthesize_cpu_map_data *data)
{
int idx;
struct perf_cpu cpu;
/* Due to padding, the 4bytes per entry mask variant is always smaller. */
- data->mask32_data.nr = BITS_TO_U32(max);
- data->mask32_data.long_size = 4;
+ data->data->type = PERF_CPU_MAP__MASK;
+ data->data->mask32_data.nr = BITS_TO_U32(data->max_cpu);
+ data->data->mask32_data.long_size = 4;
- perf_cpu_map__for_each_cpu(cpu, idx, map) {
+ perf_cpu_map__for_each_cpu(cpu, idx, data->map) {
int bit_word = cpu.cpu / 32;
- __u32 bit_mask = 1U << (cpu.cpu & 31);
+ u32 bit_mask = 1U << (cpu.cpu & 31);
- data->mask32_data.mask[bit_word] |= bit_mask;
+ data->data->mask32_data.mask[bit_word] |= bit_mask;
}
}
-static size_t cpus_size(const struct perf_cpu_map *map)
-{
- return sizeof(struct cpu_map_entries) + perf_cpu_map__nr(map) * sizeof(u16);
-}
-
-static size_t mask_size(const struct perf_cpu_map *map, int *max)
+static void synthesize_range_cpus(struct synthesize_cpu_map_data *data)
{
- *max = perf_cpu_map__max(map).cpu;
- return sizeof(struct perf_record_mask_cpu_map32) + BITS_TO_U32(*max) * sizeof(__u32);
+ data->data->type = PERF_CPU_MAP__RANGE_CPUS;
+ data->data->range_cpu_data.any_cpu = data->has_any_cpu;
+ data->data->range_cpu_data.start_cpu = data->min_cpu;
+ data->data->range_cpu_data.end_cpu = data->max_cpu;
}
-static void *cpu_map_data__alloc(const struct perf_cpu_map *map, size_t *size,
- u16 *type, int *max)
+static void *cpu_map_data__alloc(struct synthesize_cpu_map_data *syn_data,
+ size_t header_size)
{
size_t size_cpus, size_mask;
- bool is_dummy = perf_cpu_map__empty(map);
- /*
- * Both array and mask data have variable size based
- * on the number of cpus and their actual values.
- * The size of the 'struct perf_record_cpu_map_data' is:
- *
- * array = size of 'struct cpu_map_entries' +
- * number of cpus * sizeof(u64)
- *
- * mask = size of 'struct perf_record_record_cpu_map' +
- * maximum cpu bit converted to size of longs
- *
- * and finally + the size of 'struct perf_record_cpu_map_data'.
- */
- size_cpus = cpus_size(map);
- size_mask = mask_size(map, max);
+ syn_data->nr = perf_cpu_map__nr(syn_data->map);
+ syn_data->has_any_cpu = (perf_cpu_map__cpu(syn_data->map, 0).cpu == -1) ? 1 : 0;
- if (is_dummy || (size_cpus < size_mask)) {
- *size += size_cpus;
- *type = PERF_CPU_MAP__CPUS;
- } else {
- *size += size_mask;
- *type = PERF_CPU_MAP__MASK;
+ syn_data->min_cpu = perf_cpu_map__cpu(syn_data->map, syn_data->has_any_cpu).cpu;
+ syn_data->max_cpu = perf_cpu_map__max(syn_data->map).cpu;
+ if (syn_data->max_cpu - syn_data->min_cpu + 1 == syn_data->nr - syn_data->has_any_cpu) {
+ /* A consecutive range of CPUs can be encoded using a range. */
+ assert(sizeof(u16) + sizeof(struct perf_record_range_cpu_map) == sizeof(u64));
+ syn_data->type = PERF_CPU_MAP__RANGE_CPUS;
+ syn_data->size = header_size + sizeof(u64);
+ return zalloc(syn_data->size);
}
- *size += sizeof(__u16); /* For perf_record_cpu_map_data.type. */
- *size = PERF_ALIGN(*size, sizeof(u64));
- return zalloc(*size);
+ size_cpus = sizeof(u16) + sizeof(struct cpu_map_entries) + syn_data->nr * sizeof(u16);
+ /* Due to padding, the 4bytes per entry mask variant is always smaller. */
+ size_mask = sizeof(u16) + sizeof(struct perf_record_mask_cpu_map32) +
+ BITS_TO_U32(syn_data->max_cpu) * sizeof(__u32);
+ if (syn_data->has_any_cpu || size_cpus < size_mask) {
+ /* Follow the CPU map encoding. */
+ syn_data->type = PERF_CPU_MAP__CPUS;
+ syn_data->size = header_size + PERF_ALIGN(size_cpus, sizeof(u64));
+ return zalloc(syn_data->size);
+ }
+ /* Encode using a bitmask. */
+ syn_data->type = PERF_CPU_MAP__MASK;
+ syn_data->size = header_size + PERF_ALIGN(size_mask, sizeof(u64));
+ return zalloc(syn_data->size);
}
-static void cpu_map_data__synthesize(struct perf_record_cpu_map_data *data,
- const struct perf_cpu_map *map,
- u16 type, int max)
+static void cpu_map_data__synthesize(struct synthesize_cpu_map_data *data)
{
- data->type = type;
-
- switch (type) {
+ switch (data->type) {
case PERF_CPU_MAP__CPUS:
- synthesize_cpus(data, map);
+ synthesize_cpus(data);
break;
case PERF_CPU_MAP__MASK:
- synthesize_mask(data, map, max);
+ synthesize_mask(data);
+ break;
+ case PERF_CPU_MAP__RANGE_CPUS:
+ synthesize_range_cpus(data);
+ break;
default:
break;
}
static struct perf_record_cpu_map *cpu_map_event__new(const struct perf_cpu_map *map)
{
- size_t size = sizeof(struct perf_event_header);
+ struct synthesize_cpu_map_data syn_data = { .map = map };
struct perf_record_cpu_map *event;
- int max;
- u16 type;
- event = cpu_map_data__alloc(map, &size, &type, &max);
+
+ event = cpu_map_data__alloc(&syn_data, sizeof(struct perf_event_header));
if (!event)
return NULL;
+ syn_data.data = &event->data;
event->header.type = PERF_RECORD_CPU_MAP;
- event->header.size = size;
- event->data.type = type;
-
- cpu_map_data__synthesize(&event->data, map, type, max);
+ event->header.size = syn_data.size;
+ cpu_map_data__synthesize(&syn_data);
return event;
}
+
int perf_event__synthesize_cpu_map(struct perf_tool *tool,
const struct perf_cpu_map *map,
perf_event__handler_t process,
if (ev == NULL)
return -ENOMEM;
- strlcpy(ev->data, evsel->unit, size + 1);
+ strlcpy(ev->unit, evsel->unit, size + 1);
err = process(tool, (union perf_event *)ev, NULL, NULL);
free(ev);
return err;
if (ev == NULL)
return -ENOMEM;
- ev_data = (struct perf_record_event_update_scale *)ev->data;
- ev_data->scale = evsel->scale;
+ ev->scale.scale = evsel->scale;
err = process(tool, (union perf_event *)ev, NULL, NULL);
free(ev);
return err;
if (ev == NULL)
return -ENOMEM;
- strlcpy(ev->data, evsel->name, len + 1);
+ strlcpy(ev->name, evsel->name, len + 1);
err = process(tool, (union perf_event *)ev, NULL, NULL);
free(ev);
return err;
int perf_event__synthesize_event_update_cpus(struct perf_tool *tool, struct evsel *evsel,
perf_event__handler_t process)
{
- size_t size = sizeof(struct perf_record_event_update);
+ struct synthesize_cpu_map_data syn_data = { .map = evsel->core.own_cpus };
struct perf_record_event_update *ev;
- int max, err;
- u16 type;
-
- if (!evsel->core.own_cpus)
- return 0;
+ int err;
- ev = cpu_map_data__alloc(evsel->core.own_cpus, &size, &type, &max);
+ ev = cpu_map_data__alloc(&syn_data, sizeof(struct perf_event_header) + 2 * sizeof(u64));
if (!ev)
return -ENOMEM;
+ syn_data.data = &ev->cpus.cpus;
ev->header.type = PERF_RECORD_EVENT_UPDATE;
- ev->header.size = (u16)size;
+ ev->header.size = (u16)syn_data.size;
ev->type = PERF_EVENT_UPDATE__CPUS;
ev->id = evsel->core.id[0];
-
- cpu_map_data__synthesize((struct perf_record_cpu_map_data *)ev->data,
- evsel->core.own_cpus, type, max);
+ cpu_map_data__synthesize(&syn_data);
err = process(tool, (union perf_event *)ev, NULL, NULL);
free(ev);
#include "tool.h"
#include "evswitch.h"
#include "annotate.h"
+#include "mutex.h"
#include "ordered-events.h"
#include "record.h"
#include <linux/types.h>
struct ordered_events *in;
struct ordered_events data[2];
bool rotate;
- pthread_mutex_t mutex;
- pthread_cond_t cond;
+ struct mutex mutex;
+ struct cond cond;
} qe;
};
unsigned int use_c1_residency_msr;
unsigned int has_aperf;
unsigned int has_epb;
+unsigned int has_turbo;
unsigned int is_hybrid;
unsigned int do_irtl_snb;
unsigned int do_irtl_hsw;
*to = 0;
}
-static void dump_cstate_pstate_config_info(unsigned int family, unsigned int model)
+static void dump_turbo_ratio_info(unsigned int family, unsigned int model)
{
- if (!do_nhm_platform_info)
+ if (!has_turbo)
return;
- dump_nhm_platform_info();
-
if (has_hsw_turbo_ratio_limit(family, model))
dump_hsw_turbo_ratio_limits();
if (has_config_tdp(family, model))
dump_config_tdp();
+}
+
+static void dump_cstate_pstate_config_info(unsigned int family, unsigned int model)
+{
+ if (!do_nhm_platform_info)
+ return;
+ dump_nhm_platform_info();
+ dump_turbo_ratio_info(family, model);
dump_nhm_cst_cfg();
}
case INTEL_FAM6_SKYLAKE_X: /* SKX */
case INTEL_FAM6_XEON_PHI_KNL: /* KNL */
case INTEL_FAM6_ICELAKE_X: /* ICX */
- case INTEL_FAM6_SAPPHIRERAPIDS_X: /* SPR */
return (rapl_dram_energy_units = 15.3 / 1000000);
default:
return (rapl_energy_units);
case INTEL_FAM6_ALDERLAKE_N:
case INTEL_FAM6_RAPTORLAKE:
case INTEL_FAM6_RAPTORLAKE_P:
+ case INTEL_FAM6_RAPTORLAKE_S:
+ case INTEL_FAM6_METEORLAKE:
+ case INTEL_FAM6_METEORLAKE_L:
return INTEL_FAM6_CANNONLAKE_L;
case INTEL_FAM6_ATOM_TREMONT_L:
{
unsigned int eax, ebx, ecx, edx;
unsigned int fms, family, model, stepping, ecx_flags, edx_flags;
- unsigned int has_turbo;
unsigned long long ucode_patch = 0;
eax = ebx = ecx = edx = 0;
void print_version()
{
- fprintf(outf, "turbostat version 2022.07.28 - Len Brown <lenb@kernel.org>\n");
+ fprintf(outf, "turbostat version 2022.10.04 - Len Brown <lenb@kernel.org>\n");
}
#define COMMAND_LINE_SIZE 2048
import os.path
import sys
-GITHUB_OPENSBI_URL = 'https://github.com/qemu/qemu/raw/master/pc-bios/opensbi-riscv64-generic-fw_dynamic.bin'
-OPENSBI_FILE = os.path.basename(GITHUB_OPENSBI_URL)
+OPENSBI_FILE = 'opensbi-riscv64-generic-fw_dynamic.bin'
+OPENSBI_PATH = '/usr/share/qemu/' + OPENSBI_FILE
-if not os.path.isfile(OPENSBI_FILE):
- print('\n\nOpenSBI file is not in the current working directory.\n'
- 'Would you like me to download it for you from:\n' + GITHUB_OPENSBI_URL + ' ?\n')
- response = input('yes/[no]: ')
- if response.strip() == 'yes':
- os.system('wget ' + GITHUB_OPENSBI_URL)
- else:
- sys.exit()
+if not os.path.isfile(OPENSBI_PATH):
+ print('\n\nOpenSBI bios was not found in "' + OPENSBI_PATH + '".\n'
+ 'Please ensure that qemu-system-riscv is installed, or edit the path in "qemu_configs/riscv.py"\n')
+ sys.exit()
QEMU_ARCH = QemuArchParams(linux_arch='riscv',
kconfig='''
extra_qemu_params=[
'-machine', 'virt',
'-cpu', 'rv64',
- '-bios', 'opensbi-riscv64-generic-fw_dynamic.bin'])
+ '-bios', OPENSBI_PATH])
#define pageblock_order (MAX_ORDER - 1)
#define pageblock_nr_pages BIT(pageblock_order)
+#define pageblock_align(pfn) ALIGN((pfn), pageblock_nr_pages)
+#define pageblock_start_pfn(pfn) ALIGN_DOWN((pfn), pageblock_nr_pages)
struct zone {
atomic_long_t managed_pages;
# Simulate CONFIG_NUMA=y
ifeq ($(NUMA), 1)
- CFLAGS += -D CONFIG_NUMA
+ CFLAGS += -D CONFIG_NUMA -D CONFIG_NODES_SHIFT=4
endif
# Use 32 bit physical addresses.
// SPDX-License-Identifier: GPL-2.0-or-later
#include "alloc_api.h"
+static int alloc_test_flags = TEST_F_NONE;
+
+static inline const char * const get_memblock_alloc_name(int flags)
+{
+ if (flags & TEST_F_RAW)
+ return "memblock_alloc_raw";
+ return "memblock_alloc";
+}
+
+static inline void *run_memblock_alloc(phys_addr_t size, phys_addr_t align)
+{
+ if (alloc_test_flags & TEST_F_RAW)
+ return memblock_alloc_raw(size, align);
+ return memblock_alloc(size, align);
+}
+
/*
* A simple test that tries to allocate a small memory region.
* Expect to allocate an aligned region near the end of the available memory.
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_2;
phys_addr_t expected_start;
+ PREFIX_PUSH();
setup_memblock();
expected_start = memblock_end_of_DRAM() - SMP_CACHE_BYTES;
- allocated_ptr = memblock_alloc(size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, expected_start);
struct memblock_region *rgn2 = &memblock.reserved.regions[0];
struct region r1;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r2_size = SZ_16;
/* Use custom alignment */
phys_addr_t alignment = SMP_CACHE_BYTES * 2;
phys_addr_t total_size;
phys_addr_t expected_start;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SZ_2;
memblock_reserve(r1.base, r1.size);
- allocated_ptr = memblock_alloc(r2_size, alignment);
+ allocated_ptr = run_memblock_alloc(r2_size, alignment);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r2_size, alloc_test_flags);
+
ASSERT_EQ(rgn1->size, r1.size);
ASSERT_EQ(rgn1->base, r1.base);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
/*
* The first region ends at the aligned address to test region merging
*/
phys_addr_t r2_size = SZ_512;
phys_addr_t total_size = r1_size + r2_size;
+ PREFIX_PUSH();
setup_memblock();
memblock_reserve(memblock_end_of_DRAM() - total_size, r1_size);
- allocated_ptr = memblock_alloc(r2_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r2_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r2_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - total_size);
struct memblock_region *rgn = &memblock.reserved.regions[0];
struct region r1;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r2_size = SZ_512;
phys_addr_t total_size;
+ PREFIX_PUSH();
setup_memblock();
/*
memblock_reserve(r1.base, r1.size);
- allocated_ptr = memblock_alloc(r2_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r2_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r2_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, r1.base - r2_size);
struct memblock_region *rgn = &memblock.reserved.regions[0];
struct region r1, r2;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_1K;
phys_addr_t total_size;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SZ_512;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc(r3_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r3_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r3_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, r2.size + r3_size);
ASSERT_EQ(rgn->base, r2.base - r3_size);
struct memblock_region *rgn = &memblock.reserved.regions[0];
struct region r1, r2;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t gap_size = SMP_CACHE_BYTES;
phys_addr_t r3_size = SZ_64;
/*
phys_addr_t rgn_size = (MEM_SIZE - (2 * gap_size + r3_size)) / 2;
phys_addr_t total_size;
+ PREFIX_PUSH();
setup_memblock();
r1.size = rgn_size;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc(r3_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r3_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r3_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, r1.base - r2.size - r3_size);
static int alloc_small_gaps_generic_check(void)
{
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t region_size = SZ_1K;
phys_addr_t gap_size = SZ_256;
phys_addr_t region_end;
+ PREFIX_PUSH();
setup_memblock();
region_end = memblock_start_of_DRAM();
region_end += gap_size + region_size;
}
- allocated_ptr = memblock_alloc(region_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(region_size, SMP_CACHE_BYTES);
ASSERT_EQ(allocated_ptr, NULL);
void *allocated_ptr = NULL;
PREFIX_PUSH();
-
setup_memblock();
/* Simulate full memory */
memblock_reserve(memblock_start_of_DRAM(), MEM_SIZE);
- allocated_ptr = memblock_alloc(SZ_256, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(SZ_256, SMP_CACHE_BYTES);
ASSERT_EQ(allocated_ptr, NULL);
static int alloc_no_space_generic_check(void)
{
void *allocated_ptr = NULL;
+ phys_addr_t available_size = SZ_256;
+ phys_addr_t reserved_size = MEM_SIZE - available_size;
PREFIX_PUSH();
-
setup_memblock();
- phys_addr_t available_size = SZ_256;
- phys_addr_t reserved_size = MEM_SIZE - available_size;
-
/* Simulate almost-full memory */
memblock_reserve(memblock_start_of_DRAM(), reserved_size);
- allocated_ptr = memblock_alloc(SZ_1K, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(SZ_1K, SMP_CACHE_BYTES);
ASSERT_EQ(allocated_ptr, NULL);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t available_size = SZ_256;
phys_addr_t reserved_size = MEM_SIZE - available_size;
+ PREFIX_PUSH();
setup_memblock();
/* Simulate almost-full memory */
memblock_reserve(memblock_start_of_DRAM(), reserved_size);
- allocated_ptr = memblock_alloc(available_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(available_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, available_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, MEM_SIZE);
ASSERT_EQ(rgn->base, memblock_start_of_DRAM());
reset_memblock_regions();
- allocated_ptr = memblock_alloc(SZ_1K, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(SZ_1K, SMP_CACHE_BYTES);
+
+ ASSERT_EQ(allocated_ptr, NULL);
+ ASSERT_EQ(rgn->size, 0);
+ ASSERT_EQ(rgn->base, 0);
+ ASSERT_EQ(memblock.reserved.total_size, 0);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a region that is larger than the total size of
+ * available memory (memblock.memory):
+ *
+ * +-----------------------------------+
+ * | new |
+ * +-----------------------------------+
+ * | |
+ * | |
+ * +---------------------------------+
+ *
+ * Expect no allocation to happen.
+ */
+static int alloc_too_large_generic_check(void)
+{
+ struct memblock_region *rgn = &memblock.reserved.regions[0];
+ void *allocated_ptr = NULL;
+
+ PREFIX_PUSH();
+ setup_memblock();
+
+ allocated_ptr = run_memblock_alloc(MEM_SIZE + SZ_2, SMP_CACHE_BYTES);
ASSERT_EQ(allocated_ptr, NULL);
ASSERT_EQ(rgn->size, 0);
void *allocated_ptr = NULL;
PREFIX_PUSH();
-
setup_memblock();
- allocated_ptr = memblock_alloc(SZ_2, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(SZ_2, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, SZ_2, alloc_test_flags);
+
ASSERT_EQ(rgn->size, SZ_2);
ASSERT_EQ(rgn->base, memblock_start_of_DRAM());
struct memblock_region *rgn2 = &memblock.reserved.regions[1];
struct region r1;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r2_size = SZ_16;
/* Use custom alignment */
phys_addr_t alignment = SMP_CACHE_BYTES * 2;
phys_addr_t total_size;
phys_addr_t expected_start;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_start_of_DRAM() + SZ_2;
memblock_reserve(r1.base, r1.size);
- allocated_ptr = memblock_alloc(r2_size, alignment);
+ allocated_ptr = run_memblock_alloc(r2_size, alignment);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r2_size, alloc_test_flags);
ASSERT_EQ(rgn1->size, r1.size);
ASSERT_EQ(rgn1->base, r1.base);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_512;
phys_addr_t r2_size = SZ_128;
phys_addr_t total_size = r1_size + r2_size;
+ PREFIX_PUSH();
setup_memblock();
memblock_reserve(memblock_start_of_DRAM() + r1_size, r2_size);
- allocated_ptr = memblock_alloc(r1_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r1_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r1_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, memblock_start_of_DRAM());
struct memblock_region *rgn = &memblock.reserved.regions[0];
struct region r1;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r2_size = SZ_512;
phys_addr_t total_size;
+ PREFIX_PUSH();
setup_memblock();
/*
memblock_reserve(r1.base, r1.size);
- allocated_ptr = memblock_alloc(r2_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r2_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r2_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, r1.base);
struct memblock_region *rgn = &memblock.reserved.regions[1];
struct region r1, r2;
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_1K;
phys_addr_t total_size;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_start_of_DRAM();
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc(r3_size, SMP_CACHE_BYTES);
+ allocated_ptr = run_memblock_alloc(r3_size, SMP_CACHE_BYTES);
ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, r3_size, alloc_test_flags);
+
ASSERT_EQ(rgn->size, r2.size + r3_size);
ASSERT_EQ(rgn->base, r2.base);
static int alloc_in_between_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_in_between_generic_check();
- memblock_set_bottom_up(true);
- alloc_in_between_generic_check();
+ run_top_down(alloc_in_between_generic_check);
+ run_bottom_up(alloc_in_between_generic_check);
return 0;
}
static int alloc_small_gaps_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_small_gaps_generic_check();
- memblock_set_bottom_up(true);
- alloc_small_gaps_generic_check();
+ run_top_down(alloc_small_gaps_generic_check);
+ run_bottom_up(alloc_small_gaps_generic_check);
return 0;
}
static int alloc_all_reserved_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_all_reserved_generic_check();
- memblock_set_bottom_up(true);
- alloc_all_reserved_generic_check();
+ run_top_down(alloc_all_reserved_generic_check);
+ run_bottom_up(alloc_all_reserved_generic_check);
return 0;
}
static int alloc_no_space_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_no_space_generic_check();
- memblock_set_bottom_up(true);
- alloc_no_space_generic_check();
+ run_top_down(alloc_no_space_generic_check);
+ run_bottom_up(alloc_no_space_generic_check);
return 0;
}
static int alloc_limited_space_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_limited_space_generic_check();
- memblock_set_bottom_up(true);
- alloc_limited_space_generic_check();
+ run_top_down(alloc_limited_space_generic_check);
+ run_bottom_up(alloc_limited_space_generic_check);
return 0;
}
static int alloc_no_memory_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_no_memory_generic_check();
- memblock_set_bottom_up(true);
- alloc_no_memory_generic_check();
+ run_top_down(alloc_no_memory_generic_check);
+ run_bottom_up(alloc_no_memory_generic_check);
return 0;
}
-int memblock_alloc_checks(void)
+static int alloc_too_large_check(void)
{
- const char *func_testing = "memblock_alloc";
+ test_print("\tRunning %s...\n", __func__);
+ run_top_down(alloc_too_large_generic_check);
+ run_bottom_up(alloc_too_large_generic_check);
+ return 0;
+}
+
+static int memblock_alloc_checks_internal(int flags)
+{
+ const char *func = get_memblock_alloc_name(flags);
+
+ alloc_test_flags = flags;
prefix_reset();
- prefix_push(func_testing);
- test_print("Running %s tests...\n", func_testing);
+ prefix_push(func);
+ test_print("Running %s tests...\n", func);
reset_memblock_attributes();
dummy_physical_memory_init();
alloc_no_space_check();
alloc_limited_space_check();
alloc_no_memory_check();
+ alloc_too_large_check();
dummy_physical_memory_cleanup();
return 0;
}
+
+int memblock_alloc_checks(void)
+{
+ memblock_alloc_checks_internal(TEST_F_NONE);
+ memblock_alloc_checks_internal(TEST_F_RAW);
+
+ return 0;
+}
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_16;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_end_of_DRAM() - SMP_CACHE_BYTES;
allocated_ptr = memblock_alloc_from(size, SMP_CACHE_BYTES, min_addr);
- b = (char *)allocated_ptr;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ ASSERT_MEM_EQ(allocated_ptr, 0, size);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, min_addr);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_32;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
/* A misaligned address */
min_addr = memblock_end_of_DRAM() - (SMP_CACHE_BYTES * 2 - 1);
allocated_ptr = memblock_alloc_from(size, SMP_CACHE_BYTES, min_addr);
- b = (char *)allocated_ptr;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ ASSERT_MEM_EQ(allocated_ptr, 0, size);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - SMP_CACHE_BYTES);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_32;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
/* The address is too close to the end of the memory */
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_64;
phys_addr_t r2_size = SZ_2;
phys_addr_t total_size = r1_size + r2_size;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2;
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_64;
phys_addr_t min_addr;
phys_addr_t start_addr;
+ PREFIX_PUSH();
setup_memblock();
start_addr = (phys_addr_t)memblock_start_of_DRAM();
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_32;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
/* The address is too close to the end of the memory */
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_64;
phys_addr_t min_addr;
phys_addr_t r2_size;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SZ_128;
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_64;
phys_addr_t min_addr;
phys_addr_t start_addr;
+ PREFIX_PUSH();
setup_memblock();
start_addr = (phys_addr_t)memblock_start_of_DRAM();
static int alloc_from_simple_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_from_simple_generic_check();
- memblock_set_bottom_up(true);
- alloc_from_simple_generic_check();
+ run_top_down(alloc_from_simple_generic_check);
+ run_bottom_up(alloc_from_simple_generic_check);
return 0;
}
static int alloc_from_misaligned_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_from_misaligned_generic_check();
- memblock_set_bottom_up(true);
- alloc_from_misaligned_generic_check();
+ run_top_down(alloc_from_misaligned_generic_check);
+ run_bottom_up(alloc_from_misaligned_generic_check);
return 0;
}
// SPDX-License-Identifier: GPL-2.0-or-later
#include "alloc_nid_api.h"
+static int alloc_nid_test_flags = TEST_F_NONE;
+
+/*
+ * contains the fraction of MEM_SIZE contained in each node in basis point
+ * units (one hundredth of 1% or 1/10000)
+ */
+static const unsigned int node_fractions[] = {
+ 2500, /* 1/4 */
+ 625, /* 1/16 */
+ 1250, /* 1/8 */
+ 1250, /* 1/8 */
+ 625, /* 1/16 */
+ 625, /* 1/16 */
+ 2500, /* 1/4 */
+ 625, /* 1/16 */
+};
+
+static inline const char * const get_memblock_alloc_try_nid_name(int flags)
+{
+ if (flags & TEST_F_RAW)
+ return "memblock_alloc_try_nid_raw";
+ return "memblock_alloc_try_nid";
+}
+
+static inline void *run_memblock_alloc_try_nid(phys_addr_t size,
+ phys_addr_t align,
+ phys_addr_t min_addr,
+ phys_addr_t max_addr, int nid)
+{
+ if (alloc_nid_test_flags & TEST_F_RAW)
+ return memblock_alloc_try_nid_raw(size, align, min_addr,
+ max_addr, nid);
+ return memblock_alloc_try_nid(size, align, min_addr, max_addr, nid);
+}
+
/*
* A simple test that tries to allocate a memory region within min_addr and
* max_addr range:
* | |
* min_addr max_addr
*
- * Expect to allocate a cleared region that ends at max_addr.
+ * Expect to allocate a region that ends at max_addr.
*/
static int alloc_try_nid_top_down_simple_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_128;
phys_addr_t min_addr;
phys_addr_t max_addr;
phys_addr_t rgn_end;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES * 2;
max_addr = min_addr + SZ_512;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
rgn_end = rgn->base + rgn->size;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, max_addr - size);
* Aligned address
* boundary
*
- * Expect to allocate a cleared, aligned region that ends before max_addr.
+ * Expect to allocate an aligned region that ends before max_addr.
*/
static int alloc_try_nid_top_down_end_misaligned_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_128;
phys_addr_t misalign = SZ_2;
phys_addr_t min_addr;
phys_addr_t max_addr;
phys_addr_t rgn_end;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES * 2;
max_addr = min_addr + SZ_512 + misalign;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
rgn_end = rgn->base + rgn->size;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, max_addr - size - misalign);
* | |
* min_addr max_addr
*
- * Expect to allocate a cleared region that starts at min_addr and ends at
+ * Expect to allocate a region that starts at min_addr and ends at
* max_addr, given that min_addr is aligned.
*/
static int alloc_try_nid_exact_address_generic_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_1K;
phys_addr_t min_addr;
phys_addr_t max_addr;
phys_addr_t rgn_end;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES;
max_addr = min_addr + size;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
rgn_end = rgn->base + rgn->size;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, min_addr);
* address |
* boundary min_add
*
- * Expect to drop the lower limit and allocate a cleared memory region which
+ * Expect to drop the lower limit and allocate a memory region which
* ends at max_addr (if the address is aligned).
*/
static int alloc_try_nid_top_down_narrow_range_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_256;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SZ_512;
max_addr = min_addr + SMP_CACHE_BYTES;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, max_addr - size);
static int alloc_try_nid_low_max_generic_check(void)
{
void *allocated_ptr = NULL;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_1K;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM();
max_addr = min_addr + SMP_CACHE_BYTES;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_EQ(allocated_ptr, NULL);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_128;
phys_addr_t r2_size = SZ_64;
phys_addr_t total_size = r1_size + r2_size;
phys_addr_t max_addr;
phys_addr_t reserved_base;
+ PREFIX_PUSH();
setup_memblock();
max_addr = memblock_end_of_DRAM();
memblock_reserve(reserved_base, r1_size);
- allocated_ptr = memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r2_size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, reserved_base);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t r1_size = SZ_64;
phys_addr_t r2_size = SZ_128;
phys_addr_t total_size = r1_size + r2_size;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
max_addr = memblock_end_of_DRAM() - r1_size;
memblock_reserve(max_addr, r1_size);
- allocated_ptr = memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r2_size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, min_addr);
struct memblock_region *rgn1 = &memblock.reserved.regions[1];
struct memblock_region *rgn2 = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
struct region r1, r2;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_64;
phys_addr_t gap_size = SMP_CACHE_BYTES;
phys_addr_t total_size;
phys_addr_t max_addr;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags);
ASSERT_EQ(rgn1->size, r1.size + r3_size);
ASSERT_EQ(rgn1->base, max_addr - r3_size);
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
struct region r1, r2;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_64;
phys_addr_t total_size;
phys_addr_t max_addr;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, total_size);
ASSERT_EQ(rgn->base, r2.base);
struct memblock_region *rgn1 = &memblock.reserved.regions[1];
struct memblock_region *rgn2 = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
struct region r1, r2;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_256;
phys_addr_t gap_size = SMP_CACHE_BYTES;
phys_addr_t total_size;
phys_addr_t max_addr;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags);
ASSERT_EQ(rgn1->size, r1.size);
ASSERT_EQ(rgn1->base, r1.base);
{
void *allocated_ptr = NULL;
struct region r1, r2;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_256;
phys_addr_t gap_size = SMP_CACHE_BYTES;
phys_addr_t max_addr;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
+ allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_EQ(allocated_ptr, NULL);
/*
* A test that tries to allocate a memory region, where max_addr is
* bigger than the end address of the available memory. Expect to allocate
- * a cleared region that ends before the end of the memory.
+ * a region that ends before the end of the memory.
*/
static int alloc_try_nid_top_down_cap_max_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_256;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_end_of_DRAM() - SZ_1K;
max_addr = memblock_end_of_DRAM() + SZ_256;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - size);
/*
* A test that tries to allocate a memory region, where min_addr is
* smaller than the start address of the available memory. Expect to allocate
- * a cleared region that ends before the end of the memory.
+ * a region that ends before the end of the memory.
*/
static int alloc_try_nid_top_down_cap_min_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_1K;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() - SZ_256;
max_addr = memblock_end_of_DRAM();
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr, NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - size);
* | |
* min_addr max_addr
*
- * Expect to allocate a cleared region that ends before max_addr.
+ * Expect to allocate a region that ends before max_addr.
*/
static int alloc_try_nid_bottom_up_simple_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_128;
phys_addr_t min_addr;
phys_addr_t max_addr;
phys_addr_t rgn_end;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES * 2;
max_addr = min_addr + SZ_512;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
rgn_end = rgn->base + rgn->size;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, min_addr);
* Aligned address
* boundary
*
- * Expect to allocate a cleared, aligned region that ends before max_addr.
+ * Expect to allocate an aligned region that ends before max_addr.
*/
static int alloc_try_nid_bottom_up_start_misaligned_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_128;
phys_addr_t misalign = SZ_2;
phys_addr_t min_addr;
phys_addr_t max_addr;
phys_addr_t rgn_end;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + misalign;
max_addr = min_addr + SZ_512;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
rgn_end = rgn->base + rgn->size;
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, min_addr + (SMP_CACHE_BYTES - misalign));
* |
* min_add
*
- * Expect to drop the lower limit and allocate a cleared memory region which
+ * Expect to drop the lower limit and allocate a memory region which
* starts at the beginning of the available memory.
*/
static int alloc_try_nid_bottom_up_narrow_range_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_256;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SZ_512;
max_addr = min_addr + SMP_CACHE_BYTES;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, memblock_start_of_DRAM());
struct memblock_region *rgn1 = &memblock.reserved.regions[1];
struct memblock_region *rgn2 = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
struct region r1, r2;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_64;
phys_addr_t gap_size = SMP_CACHE_BYTES;
phys_addr_t total_size;
phys_addr_t max_addr;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags);
ASSERT_EQ(rgn1->size, r1.size);
ASSERT_EQ(rgn1->base, max_addr);
struct memblock_region *rgn2 = &memblock.reserved.regions[1];
struct memblock_region *rgn3 = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
struct region r1, r2;
-
- PREFIX_PUSH();
-
phys_addr_t r3_size = SZ_256;
phys_addr_t gap_size = SMP_CACHE_BYTES;
phys_addr_t total_size;
phys_addr_t max_addr;
phys_addr_t min_addr;
+ PREFIX_PUSH();
setup_memblock();
r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2;
memblock_reserve(r1.base, r1.size);
memblock_reserve(r2.base, r2.size);
- allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags);
ASSERT_EQ(rgn3->size, r3_size);
ASSERT_EQ(rgn3->base, memblock_start_of_DRAM());
/*
* A test that tries to allocate a memory region, where max_addr is
* bigger than the end address of the available memory. Expect to allocate
- * a cleared region that starts at the min_addr
+ * a region that starts at the min_addr.
*/
static int alloc_try_nid_bottom_up_cap_max_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_256;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM() + SZ_1K;
max_addr = memblock_end_of_DRAM() + SZ_256;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, min_addr);
/*
* A test that tries to allocate a memory region, where min_addr is
* smaller than the start address of the available memory. Expect to allocate
- * a cleared region at the beginning of the available memory.
+ * a region at the beginning of the available memory.
*/
static int alloc_try_nid_bottom_up_cap_min_check(void)
{
struct memblock_region *rgn = &memblock.reserved.regions[0];
void *allocated_ptr = NULL;
- char *b;
-
- PREFIX_PUSH();
-
phys_addr_t size = SZ_1K;
phys_addr_t min_addr;
phys_addr_t max_addr;
+ PREFIX_PUSH();
setup_memblock();
min_addr = memblock_start_of_DRAM();
max_addr = memblock_end_of_DRAM() - SZ_256;
- allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
- min_addr, max_addr,
- NUMA_NO_NODE);
- b = (char *)allocated_ptr;
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
ASSERT_NE(allocated_ptr, NULL);
- ASSERT_EQ(*b, 0);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
ASSERT_EQ(rgn->size, size);
ASSERT_EQ(rgn->base, memblock_start_of_DRAM());
return 0;
}
-/* Test case wrappers */
+/* Test case wrappers for range tests */
static int alloc_try_nid_simple_check(void)
{
test_print("\tRunning %s...\n", __func__);
static int alloc_try_nid_min_reserved_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_try_nid_min_reserved_generic_check();
- memblock_set_bottom_up(true);
- alloc_try_nid_min_reserved_generic_check();
+ run_top_down(alloc_try_nid_min_reserved_generic_check);
+ run_bottom_up(alloc_try_nid_min_reserved_generic_check);
return 0;
}
static int alloc_try_nid_max_reserved_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_try_nid_max_reserved_generic_check();
- memblock_set_bottom_up(true);
- alloc_try_nid_max_reserved_generic_check();
+ run_top_down(alloc_try_nid_max_reserved_generic_check);
+ run_bottom_up(alloc_try_nid_max_reserved_generic_check);
return 0;
}
static int alloc_try_nid_exact_address_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_try_nid_exact_address_generic_check();
- memblock_set_bottom_up(true);
- alloc_try_nid_exact_address_generic_check();
+ run_top_down(alloc_try_nid_exact_address_generic_check);
+ run_bottom_up(alloc_try_nid_exact_address_generic_check);
return 0;
}
static int alloc_try_nid_reserved_full_merge_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_try_nid_reserved_full_merge_generic_check();
- memblock_set_bottom_up(true);
- alloc_try_nid_reserved_full_merge_generic_check();
+ run_top_down(alloc_try_nid_reserved_full_merge_generic_check);
+ run_bottom_up(alloc_try_nid_reserved_full_merge_generic_check);
return 0;
}
static int alloc_try_nid_reserved_all_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_try_nid_reserved_all_generic_check();
- memblock_set_bottom_up(true);
- alloc_try_nid_reserved_all_generic_check();
+ run_top_down(alloc_try_nid_reserved_all_generic_check);
+ run_bottom_up(alloc_try_nid_reserved_all_generic_check);
return 0;
}
static int alloc_try_nid_low_max_check(void)
{
test_print("\tRunning %s...\n", __func__);
- memblock_set_bottom_up(false);
- alloc_try_nid_low_max_generic_check();
- memblock_set_bottom_up(true);
- alloc_try_nid_low_max_generic_check();
+ run_top_down(alloc_try_nid_low_max_generic_check);
+ run_bottom_up(alloc_try_nid_low_max_generic_check);
return 0;
}
-int memblock_alloc_nid_checks(void)
+static int memblock_alloc_nid_range_checks(void)
{
- const char *func_testing = "memblock_alloc_try_nid";
-
- prefix_reset();
- prefix_push(func_testing);
- test_print("Running %s tests...\n", func_testing);
-
- reset_memblock_attributes();
- dummy_physical_memory_init();
+ test_print("Running %s range tests...\n",
+ get_memblock_alloc_try_nid_name(alloc_nid_test_flags));
alloc_try_nid_simple_check();
alloc_try_nid_misaligned_check();
alloc_try_nid_reserved_all_check();
alloc_try_nid_low_max_check();
- dummy_physical_memory_cleanup();
+ return 0;
+}
- prefix_pop();
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * has enough memory to allocate a region of the requested size.
+ * Expect to allocate an aligned region at the end of the requested node.
+ */
+static int alloc_try_nid_top_down_numa_simple_check(void)
+{
+ int nid_req = 3;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ ASSERT_LE(SZ_4, req_node->size);
+ size = req_node->size / SZ_4;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, region_end(req_node) - size);
+ ASSERT_LE(req_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * does not have enough memory to allocate a region of the requested size:
+ *
+ * | +-----+ +------------------+ |
+ * | | req | | expected | |
+ * +---+-----+----------+------------------+-----+
+ *
+ * | +---------+ |
+ * | | rgn | |
+ * +-----------------------------+---------+-----+
+ *
+ * Expect to allocate an aligned region at the end of the last node that has
+ * enough memory (in this case, nid = 6) after falling back to NUMA_NO_NODE.
+ */
+static int alloc_try_nid_top_down_numa_small_node_check(void)
+{
+ int nid_req = 1;
+ int nid_exp = 6;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ size = SZ_2 * req_node->size;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, region_end(exp_node) - size);
+ ASSERT_LE(exp_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * is fully reserved:
+ *
+ * | +---------+ +------------------+ |
+ * | |requested| | expected | |
+ * +--------------+---------+------------+------------------+-----+
+ *
+ * | +---------+ +---------+ |
+ * | | reserved| | new | |
+ * +--------------+---------+---------------------+---------+-----+
+ *
+ * Expect to allocate an aligned region at the end of the last node that is
+ * large enough and has enough unreserved memory (in this case, nid = 6) after
+ * falling back to NUMA_NO_NODE. The region count and total size get updated.
+ */
+static int alloc_try_nid_top_down_numa_node_reserved_check(void)
+{
+ int nid_req = 2;
+ int nid_exp = 6;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[1];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ size = req_node->size;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ memblock_reserve(req_node->base, req_node->size);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, region_end(exp_node) - size);
+ ASSERT_LE(exp_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 2);
+ ASSERT_EQ(memblock.reserved.total_size, size + req_node->size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * is partially reserved but has enough memory for the allocated region:
+ *
+ * | +---------------------------------------+ |
+ * | | requested | |
+ * +-----------+---------------------------------------+----------+
+ *
+ * | +------------------+ +-----+ |
+ * | | reserved | | new | |
+ * +-----------+------------------+--------------+-----+----------+
+ *
+ * Expect to allocate an aligned region at the end of the requested node. The
+ * region count and total size get updated.
+ */
+static int alloc_try_nid_top_down_numa_part_reserved_check(void)
+{
+ int nid_req = 4;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[1];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ void *allocated_ptr = NULL;
+ struct region r1;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ ASSERT_LE(SZ_8, req_node->size);
+ r1.base = req_node->base;
+ r1.size = req_node->size / SZ_2;
+ size = r1.size / SZ_4;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ memblock_reserve(r1.base, r1.size);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, region_end(req_node) - size);
+ ASSERT_LE(req_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 2);
+ ASSERT_EQ(memblock.reserved.total_size, size + r1.size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * is partially reserved and does not have enough contiguous memory for the
+ * allocated region:
+ *
+ * | +-----------------------+ +----------------------|
+ * | | requested | | expected |
+ * +-----------+-----------------------+---------+----------------------+
+ *
+ * | +----------+ +-----------|
+ * | | reserved | | new |
+ * +-----------------+----------+---------------------------+-----------+
+ *
+ * Expect to allocate an aligned region at the end of the last node that is
+ * large enough and has enough unreserved memory (in this case,
+ * nid = NUMA_NODES - 1) after falling back to NUMA_NO_NODE. The region count
+ * and total size get updated.
+ */
+static int alloc_try_nid_top_down_numa_part_reserved_fallback_check(void)
+{
+ int nid_req = 4;
+ int nid_exp = NUMA_NODES - 1;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[1];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ struct region r1;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ ASSERT_LE(SZ_4, req_node->size);
+ size = req_node->size / SZ_2;
+ r1.base = req_node->base + (size / SZ_2);
+ r1.size = size;
+
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ memblock_reserve(r1.base, r1.size);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, region_end(exp_node) - size);
+ ASSERT_LE(exp_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 2);
+ ASSERT_EQ(memblock.reserved.total_size, size + r1.size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region that spans over the min_addr
+ * and max_addr range and overlaps with two different nodes, where the first
+ * node is the requested node:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +-----------------------+-----------+ |
+ * | | requested | node3 | |
+ * +-----------+-----------------------+-----------+--------------+
+ * + +
+ * | +-----------+ |
+ * | | rgn | |
+ * +-----------------------+-----------+--------------------------+
+ *
+ * Expect to drop the lower limit and allocate a memory region that ends at
+ * the end of the requested node.
+ */
+static int alloc_try_nid_top_down_numa_split_range_low_check(void)
+{
+ int nid_req = 2;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_512;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+ phys_addr_t req_node_end;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ req_node_end = region_end(req_node);
+ min_addr = req_node_end - SZ_256;
+ max_addr = min_addr + size;
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, req_node_end - size);
+ ASSERT_LE(req_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region that spans over the min_addr
+ * and max_addr range and overlaps with two different nodes, where the second
+ * node is the requested node:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +--------------------------+---------+ |
+ * | | expected |requested| |
+ * +------+--------------------------+---------+----------------+
+ * + +
+ * | +---------+ |
+ * | | rgn | |
+ * +-----------------------+---------+--------------------------+
+ *
+ * Expect to drop the lower limit and allocate a memory region that
+ * ends at the end of the first node that overlaps with the range.
+ */
+static int alloc_try_nid_top_down_numa_split_range_high_check(void)
+{
+ int nid_req = 3;
+ int nid_exp = nid_req - 1;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_512;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+ phys_addr_t exp_node_end;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ exp_node_end = region_end(exp_node);
+ min_addr = exp_node_end - SZ_256;
+ max_addr = min_addr + size;
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, exp_node_end - size);
+ ASSERT_LE(exp_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region that spans over the min_addr
+ * and max_addr range and overlaps with two different nodes, where the requested
+ * node ends before min_addr:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +---------------+ +-------------+---------+ |
+ * | | requested | | node1 | node2 | |
+ * +----+---------------+--------+-------------+---------+----------+
+ * + +
+ * | +---------+ |
+ * | | rgn | |
+ * +----------+---------+-------------------------------------------+
+ *
+ * Expect to drop the lower limit and allocate a memory region that ends at
+ * the end of the requested node.
+ */
+static int alloc_try_nid_top_down_numa_no_overlap_split_check(void)
+{
+ int nid_req = 2;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *node2 = &memblock.memory.regions[6];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ size = SZ_512;
+ min_addr = node2->base - SZ_256;
+ max_addr = min_addr + size;
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, region_end(req_node) - size);
+ ASSERT_LE(req_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate memory within min_addr and max_add range when
+ * the requested node and the range do not overlap, and requested node ends
+ * before min_addr. The range overlaps with multiple nodes along node
+ * boundaries:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * |-----------+ +----------+----...----+----------+ |
+ * | requested | | min node | ... | max node | |
+ * +-----------+-----------+----------+----...----+----------+------+
+ * + +
+ * | +-----+ |
+ * | | rgn | |
+ * +---------------------------------------------------+-----+------+
+ *
+ * Expect to allocate a memory region at the end of the final node in
+ * the range after falling back to NUMA_NO_NODE.
+ */
+static int alloc_try_nid_top_down_numa_no_overlap_low_check(void)
+{
+ int nid_req = 0;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *min_node = &memblock.memory.regions[2];
+ struct memblock_region *max_node = &memblock.memory.regions[5];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_64;
+ phys_addr_t max_addr;
+ phys_addr_t min_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ min_addr = min_node->base;
+ max_addr = region_end(max_node);
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, max_addr - size);
+ ASSERT_LE(max_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate memory within min_addr and max_add range when
+ * the requested node and the range do not overlap, and requested node starts
+ * after max_addr. The range overlaps with multiple nodes along node
+ * boundaries:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +----------+----...----+----------+ +-----------+ |
+ * | | min node | ... | max node | | requested | |
+ * +-----+----------+----...----+----------+--------+-----------+---+
+ * + +
+ * | +-----+ |
+ * | | rgn | |
+ * +---------------------------------+-----+------------------------+
+ *
+ * Expect to allocate a memory region at the end of the final node in
+ * the range after falling back to NUMA_NO_NODE.
+ */
+static int alloc_try_nid_top_down_numa_no_overlap_high_check(void)
+{
+ int nid_req = 7;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *min_node = &memblock.memory.regions[2];
+ struct memblock_region *max_node = &memblock.memory.regions[5];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_64;
+ phys_addr_t max_addr;
+ phys_addr_t min_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ min_addr = min_node->base;
+ max_addr = region_end(max_node);
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, max_addr - size);
+ ASSERT_LE(max_node->base, new_rgn->base);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * has enough memory to allocate a region of the requested size.
+ * Expect to allocate an aligned region at the beginning of the requested node.
+ */
+static int alloc_try_nid_bottom_up_numa_simple_check(void)
+{
+ int nid_req = 3;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ ASSERT_LE(SZ_4, req_node->size);
+ size = req_node->size / SZ_4;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, req_node->base);
+ ASSERT_LE(region_end(new_rgn), region_end(req_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * does not have enough memory to allocate a region of the requested size:
+ *
+ * |----------------------+-----+ |
+ * | expected | req | |
+ * +----------------------+-----+----------------+
+ *
+ * |---------+ |
+ * | rgn | |
+ * +---------+-----------------------------------+
+ *
+ * Expect to allocate an aligned region at the beginning of the first node that
+ * has enough memory (in this case, nid = 0) after falling back to NUMA_NO_NODE.
+ */
+static int alloc_try_nid_bottom_up_numa_small_node_check(void)
+{
+ int nid_req = 1;
+ int nid_exp = 0;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ size = SZ_2 * req_node->size;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, exp_node->base);
+ ASSERT_LE(region_end(new_rgn), region_end(exp_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * is fully reserved:
+ *
+ * |----------------------+ +-----------+ |
+ * | expected | | requested | |
+ * +----------------------+-----+-----------+--------------------+
+ *
+ * |-----------+ +-----------+ |
+ * | new | | reserved | |
+ * +-----------+----------------+-----------+--------------------+
+ *
+ * Expect to allocate an aligned region at the beginning of the first node that
+ * is large enough and has enough unreserved memory (in this case, nid = 0)
+ * after falling back to NUMA_NO_NODE. The region count and total size get
+ * updated.
+ */
+static int alloc_try_nid_bottom_up_numa_node_reserved_check(void)
+{
+ int nid_req = 2;
+ int nid_exp = 0;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ size = req_node->size;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ memblock_reserve(req_node->base, req_node->size);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, exp_node->base);
+ ASSERT_LE(region_end(new_rgn), region_end(exp_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 2);
+ ASSERT_EQ(memblock.reserved.total_size, size + req_node->size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * is partially reserved but has enough memory for the allocated region:
+ *
+ * | +---------------------------------------+ |
+ * | | requested | |
+ * +-----------+---------------------------------------+---------+
+ *
+ * | +------------------+-----+ |
+ * | | reserved | new | |
+ * +-----------+------------------+-----+------------------------+
+ *
+ * Expect to allocate an aligned region in the requested node that merges with
+ * the existing reserved region. The total size gets updated.
+ */
+static int alloc_try_nid_bottom_up_numa_part_reserved_check(void)
+{
+ int nid_req = 4;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ void *allocated_ptr = NULL;
+ struct region r1;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+ phys_addr_t total_size;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ ASSERT_LE(SZ_8, req_node->size);
+ r1.base = req_node->base;
+ r1.size = req_node->size / SZ_2;
+ size = r1.size / SZ_4;
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+ total_size = size + r1.size;
+
+ memblock_reserve(r1.base, r1.size);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, total_size);
+ ASSERT_EQ(new_rgn->base, req_node->base);
+ ASSERT_LE(region_end(new_rgn), region_end(req_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * is partially reserved and does not have enough contiguous memory for the
+ * allocated region:
+ *
+ * |----------------------+ +-----------------------+ |
+ * | expected | | requested | |
+ * +----------------------+-------+-----------------------+---------+
+ *
+ * |-----------+ +----------+ |
+ * | new | | reserved | |
+ * +-----------+------------------------+----------+----------------+
+ *
+ * Expect to allocate an aligned region at the beginning of the first
+ * node that is large enough and has enough unreserved memory (in this case,
+ * nid = 0) after falling back to NUMA_NO_NODE. The region count and total size
+ * get updated.
+ */
+static int alloc_try_nid_bottom_up_numa_part_reserved_fallback_check(void)
+{
+ int nid_req = 4;
+ int nid_exp = 0;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ struct region r1;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ ASSERT_LE(SZ_4, req_node->size);
+ size = req_node->size / SZ_2;
+ r1.base = req_node->base + (size / SZ_2);
+ r1.size = size;
+
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ memblock_reserve(r1.base, r1.size);
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, exp_node->base);
+ ASSERT_LE(region_end(new_rgn), region_end(exp_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 2);
+ ASSERT_EQ(memblock.reserved.total_size, size + r1.size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region that spans over the min_addr
+ * and max_addr range and overlaps with two different nodes, where the first
+ * node is the requested node:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +-----------------------+-----------+ |
+ * | | requested | node3 | |
+ * +-----------+-----------------------+-----------+--------------+
+ * + +
+ * | +-----------+ |
+ * | | rgn | |
+ * +-----------+-----------+--------------------------------------+
+ *
+ * Expect to drop the lower limit and allocate a memory region at the beginning
+ * of the requested node.
+ */
+static int alloc_try_nid_bottom_up_numa_split_range_low_check(void)
+{
+ int nid_req = 2;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_512;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+ phys_addr_t req_node_end;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ req_node_end = region_end(req_node);
+ min_addr = req_node_end - SZ_256;
+ max_addr = min_addr + size;
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, req_node->base);
+ ASSERT_LE(region_end(new_rgn), req_node_end);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region that spans over the min_addr
+ * and max_addr range and overlaps with two different nodes, where the second
+ * node is the requested node:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * |------------------+ +----------------------+---------+ |
+ * | expected | | previous |requested| |
+ * +------------------+--------+----------------------+---------+------+
+ * + +
+ * |---------+ |
+ * | rgn | |
+ * +---------+---------------------------------------------------------+
+ *
+ * Expect to drop the lower limit and allocate a memory region at the beginning
+ * of the first node that has enough memory.
+ */
+static int alloc_try_nid_bottom_up_numa_split_range_high_check(void)
+{
+ int nid_req = 3;
+ int nid_exp = 0;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *exp_node = &memblock.memory.regions[nid_exp];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_512;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+ phys_addr_t exp_node_end;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ exp_node_end = region_end(req_node);
+ min_addr = req_node->base - SZ_256;
+ max_addr = min_addr + size;
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, exp_node->base);
+ ASSERT_LE(region_end(new_rgn), exp_node_end);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region that spans over the min_addr
+ * and max_addr range and overlaps with two different nodes, where the requested
+ * node ends before min_addr:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +---------------+ +-------------+---------+ |
+ * | | requested | | node1 | node2 | |
+ * +----+---------------+--------+-------------+---------+---------+
+ * + +
+ * | +---------+ |
+ * | | rgn | |
+ * +----+---------+------------------------------------------------+
+ *
+ * Expect to drop the lower limit and allocate a memory region that starts at
+ * the beginning of the requested node.
+ */
+static int alloc_try_nid_bottom_up_numa_no_overlap_split_check(void)
+{
+ int nid_req = 2;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *node2 = &memblock.memory.regions[6];
+ void *allocated_ptr = NULL;
+ phys_addr_t size;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ size = SZ_512;
+ min_addr = node2->base - SZ_256;
+ max_addr = min_addr + size;
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, req_node->base);
+ ASSERT_LE(region_end(new_rgn), region_end(req_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate memory within min_addr and max_add range when
+ * the requested node and the range do not overlap, and requested node ends
+ * before min_addr. The range overlaps with multiple nodes along node
+ * boundaries:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * |-----------+ +----------+----...----+----------+ |
+ * | requested | | min node | ... | max node | |
+ * +-----------+-----------+----------+----...----+----------+------+
+ * + +
+ * | +-----+ |
+ * | | rgn | |
+ * +-----------------------+-----+----------------------------------+
+ *
+ * Expect to allocate a memory region at the beginning of the first node
+ * in the range after falling back to NUMA_NO_NODE.
+ */
+static int alloc_try_nid_bottom_up_numa_no_overlap_low_check(void)
+{
+ int nid_req = 0;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *min_node = &memblock.memory.regions[2];
+ struct memblock_region *max_node = &memblock.memory.regions[5];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_64;
+ phys_addr_t max_addr;
+ phys_addr_t min_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ min_addr = min_node->base;
+ max_addr = region_end(max_node);
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, min_addr);
+ ASSERT_LE(region_end(new_rgn), region_end(min_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate memory within min_addr and max_add range when
+ * the requested node and the range do not overlap, and requested node starts
+ * after max_addr. The range overlaps with multiple nodes along node
+ * boundaries:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +----------+----...----+----------+ +---------+ |
+ * | | min node | ... | max node | |requested| |
+ * +-----+----------+----...----+----------+---------+---------+---+
+ * + +
+ * | +-----+ |
+ * | | rgn | |
+ * +-----+-----+---------------------------------------------------+
+ *
+ * Expect to allocate a memory region at the beginning of the first node
+ * in the range after falling back to NUMA_NO_NODE.
+ */
+static int alloc_try_nid_bottom_up_numa_no_overlap_high_check(void)
+{
+ int nid_req = 7;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *min_node = &memblock.memory.regions[2];
+ struct memblock_region *max_node = &memblock.memory.regions[5];
+ void *allocated_ptr = NULL;
+ phys_addr_t size = SZ_64;
+ phys_addr_t max_addr;
+ phys_addr_t min_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ min_addr = min_node->base;
+ max_addr = region_end(max_node);
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, size);
+ ASSERT_EQ(new_rgn->base, min_addr);
+ ASSERT_LE(region_end(new_rgn), region_end(min_node));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate a memory region in a specific NUMA node that
+ * does not have enough memory to allocate a region of the requested size.
+ * Additionally, none of the nodes have enough memory to allocate the region:
+ *
+ * +-----------------------------------+
+ * | new |
+ * +-----------------------------------+
+ * |-------+-------+-------+-------+-------+-------+-------+-------|
+ * | node0 | node1 | node2 | node3 | node4 | node5 | node6 | node7 |
+ * +-------+-------+-------+-------+-------+-------+-------+-------+
+ *
+ * Expect no allocation to happen.
+ */
+static int alloc_try_nid_numa_large_region_generic_check(void)
+{
+ int nid_req = 3;
+ void *allocated_ptr = NULL;
+ phys_addr_t size = MEM_SIZE / SZ_2;
+ phys_addr_t min_addr;
+ phys_addr_t max_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ min_addr = memblock_start_of_DRAM();
+ max_addr = memblock_end_of_DRAM();
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+ ASSERT_EQ(allocated_ptr, NULL);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate memory within min_addr and max_addr range when
+ * there are two reserved regions at the borders. The requested node starts at
+ * min_addr and ends at max_addr and is the same size as the region to be
+ * allocated:
+ *
+ * min_addr
+ * | max_addr
+ * | |
+ * v v
+ * | +-----------+-----------------------+-----------------------|
+ * | | node5 | requested | node7 |
+ * +------+-----------+-----------------------+-----------------------+
+ * + +
+ * | +----+-----------------------+----+ |
+ * | | r2 | new | r1 | |
+ * +-------------+----+-----------------------+----+------------------+
+ *
+ * Expect to merge all of the regions into one. The region counter and total
+ * size fields get updated.
+ */
+static int alloc_try_nid_numa_reserved_full_merge_generic_check(void)
+{
+ int nid_req = 6;
+ int nid_next = nid_req + 1;
+ struct memblock_region *new_rgn = &memblock.reserved.regions[0];
+ struct memblock_region *req_node = &memblock.memory.regions[nid_req];
+ struct memblock_region *next_node = &memblock.memory.regions[nid_next];
+ void *allocated_ptr = NULL;
+ struct region r1, r2;
+ phys_addr_t size = req_node->size;
+ phys_addr_t total_size;
+ phys_addr_t max_addr;
+ phys_addr_t min_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ r1.base = next_node->base;
+ r1.size = SZ_128;
+
+ r2.size = SZ_128;
+ r2.base = r1.base - (size + r2.size);
+
+ total_size = r1.size + r2.size + size;
+ min_addr = r2.base + r2.size;
+ max_addr = r1.base;
+
+ memblock_reserve(r1.base, r1.size);
+ memblock_reserve(r2.base, r2.size);
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr, nid_req);
+
+ ASSERT_NE(allocated_ptr, NULL);
+ assert_mem_content(allocated_ptr, size, alloc_nid_test_flags);
+
+ ASSERT_EQ(new_rgn->size, total_size);
+ ASSERT_EQ(new_rgn->base, r2.base);
+
+ ASSERT_LE(new_rgn->base, req_node->base);
+ ASSERT_LE(region_end(req_node), region_end(new_rgn));
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to allocate memory within min_addr and max_add range,
+ * where the total range can fit the region, but it is split between two nodes
+ * and everything else is reserved. Additionally, nid is set to NUMA_NO_NODE
+ * instead of requesting a specific node:
+ *
+ * +-----------+
+ * | new |
+ * +-----------+
+ * | +---------------------+-----------|
+ * | | prev node | next node |
+ * +------+---------------------+-----------+
+ * + +
+ * |----------------------+ +-----|
+ * | r1 | | r2 |
+ * +----------------------+-----------+-----+
+ * ^ ^
+ * | |
+ * | max_addr
+ * |
+ * min_addr
+ *
+ * Expect no allocation to happen.
+ */
+static int alloc_try_nid_numa_split_all_reserved_generic_check(void)
+{
+ void *allocated_ptr = NULL;
+ struct memblock_region *next_node = &memblock.memory.regions[7];
+ struct region r1, r2;
+ phys_addr_t size = SZ_256;
+ phys_addr_t max_addr;
+ phys_addr_t min_addr;
+
+ PREFIX_PUSH();
+ setup_numa_memblock(node_fractions);
+
+ r2.base = next_node->base + SZ_128;
+ r2.size = memblock_end_of_DRAM() - r2.base;
+
+ r1.size = MEM_SIZE - (r2.size + size);
+ r1.base = memblock_start_of_DRAM();
+
+ min_addr = r1.base + r1.size;
+ max_addr = r2.base;
+
+ memblock_reserve(r1.base, r1.size);
+ memblock_reserve(r2.base, r2.size);
+
+ allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES,
+ min_addr, max_addr,
+ NUMA_NO_NODE);
+
+ ASSERT_EQ(allocated_ptr, NULL);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/* Test case wrappers for NUMA tests */
+static int alloc_try_nid_numa_simple_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_simple_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_simple_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_small_node_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_small_node_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_small_node_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_node_reserved_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_node_reserved_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_node_reserved_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_part_reserved_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_part_reserved_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_part_reserved_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_part_reserved_fallback_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_part_reserved_fallback_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_part_reserved_fallback_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_split_range_low_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_split_range_low_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_split_range_low_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_split_range_high_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_split_range_high_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_split_range_high_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_no_overlap_split_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_no_overlap_split_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_no_overlap_split_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_no_overlap_low_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_no_overlap_low_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_no_overlap_low_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_no_overlap_high_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ memblock_set_bottom_up(false);
+ alloc_try_nid_top_down_numa_no_overlap_high_check();
+ memblock_set_bottom_up(true);
+ alloc_try_nid_bottom_up_numa_no_overlap_high_check();
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_large_region_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ run_top_down(alloc_try_nid_numa_large_region_generic_check);
+ run_bottom_up(alloc_try_nid_numa_large_region_generic_check);
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_reserved_full_merge_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ run_top_down(alloc_try_nid_numa_reserved_full_merge_generic_check);
+ run_bottom_up(alloc_try_nid_numa_reserved_full_merge_generic_check);
+
+ return 0;
+}
+
+static int alloc_try_nid_numa_split_all_reserved_check(void)
+{
+ test_print("\tRunning %s...\n", __func__);
+ run_top_down(alloc_try_nid_numa_split_all_reserved_generic_check);
+ run_bottom_up(alloc_try_nid_numa_split_all_reserved_generic_check);
+
+ return 0;
+}
+
+int __memblock_alloc_nid_numa_checks(void)
+{
+ test_print("Running %s NUMA tests...\n",
+ get_memblock_alloc_try_nid_name(alloc_nid_test_flags));
+
+ alloc_try_nid_numa_simple_check();
+ alloc_try_nid_numa_small_node_check();
+ alloc_try_nid_numa_node_reserved_check();
+ alloc_try_nid_numa_part_reserved_check();
+ alloc_try_nid_numa_part_reserved_fallback_check();
+ alloc_try_nid_numa_split_range_low_check();
+ alloc_try_nid_numa_split_range_high_check();
+
+ alloc_try_nid_numa_no_overlap_split_check();
+ alloc_try_nid_numa_no_overlap_low_check();
+ alloc_try_nid_numa_no_overlap_high_check();
+ alloc_try_nid_numa_large_region_check();
+ alloc_try_nid_numa_reserved_full_merge_check();
+ alloc_try_nid_numa_split_all_reserved_check();
+
+ return 0;
+}
+
+static int memblock_alloc_nid_checks_internal(int flags)
+{
+ alloc_nid_test_flags = flags;
+
+ prefix_reset();
+ prefix_push(get_memblock_alloc_try_nid_name(flags));
+
+ reset_memblock_attributes();
+ dummy_physical_memory_init();
+
+ memblock_alloc_nid_range_checks();
+ memblock_alloc_nid_numa_checks();
+
+ dummy_physical_memory_cleanup();
+
+ prefix_pop();
+
+ return 0;
+}
+
+int memblock_alloc_nid_checks(void)
+{
+ memblock_alloc_nid_checks_internal(TEST_F_NONE);
+ memblock_alloc_nid_checks_internal(TEST_F_RAW);
return 0;
}
#include "common.h"
int memblock_alloc_nid_checks(void);
+int __memblock_alloc_nid_numa_checks(void);
+
+#ifdef CONFIG_NUMA
+static inline int memblock_alloc_nid_numa_checks(void)
+{
+ __memblock_alloc_nid_numa_checks();
+ return 0;
+}
+
+#else
+static inline int memblock_alloc_nid_numa_checks(void)
+{
+ return 0;
+}
+
+#endif /* CONFIG_NUMA */
#endif
#define FUNC_RESERVE "memblock_reserve"
#define FUNC_REMOVE "memblock_remove"
#define FUNC_FREE "memblock_free"
+#define FUNC_TRIM "memblock_trim_memory"
static int memblock_initialization_check(void)
{
return 0;
}
+/*
+ * A test that tries to add two memory blocks that don't overlap with one
+ * another and then add a third memory block in the space between the first two:
+ *
+ * | +--------+--------+--------+ |
+ * | | r1 | r3 | r2 | |
+ * +--------+--------+--------+--------+--+
+ *
+ * Expect to merge the three entries into one region that starts at r1.base
+ * and has size of r1.size + r2.size + r3.size. The region counter and total
+ * size of the available memory are updated.
+ */
+static int memblock_add_between_check(void)
+{
+ struct memblock_region *rgn;
+ phys_addr_t total_size;
+
+ rgn = &memblock.memory.regions[0];
+
+ struct region r1 = {
+ .base = SZ_1G,
+ .size = SZ_8K
+ };
+ struct region r2 = {
+ .base = SZ_1G + SZ_16K,
+ .size = SZ_8K
+ };
+ struct region r3 = {
+ .base = SZ_1G + SZ_8K,
+ .size = SZ_8K
+ };
+
+ PREFIX_PUSH();
+
+ total_size = r1.size + r2.size + r3.size;
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_add(r2.base, r2.size);
+ memblock_add(r3.base, r3.size);
+
+ ASSERT_EQ(rgn->base, r1.base);
+ ASSERT_EQ(rgn->size, total_size);
+
+ ASSERT_EQ(memblock.memory.cnt, 1);
+ ASSERT_EQ(memblock.memory.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A simple test that tries to add a memory block r when r extends past
+ * PHYS_ADDR_MAX:
+ *
+ * +--------+
+ * | r |
+ * +--------+
+ * | +----+
+ * | | rgn|
+ * +----------------------------+----+
+ *
+ * Expect to add a memory block of size PHYS_ADDR_MAX - r.base. Expect the
+ * total size of available memory and the counter to be updated.
+ */
+static int memblock_add_near_max_check(void)
+{
+ struct memblock_region *rgn;
+ phys_addr_t total_size;
+
+ rgn = &memblock.memory.regions[0];
+
+ struct region r = {
+ .base = PHYS_ADDR_MAX - SZ_1M,
+ .size = SZ_2M
+ };
+
+ PREFIX_PUSH();
+
+ total_size = PHYS_ADDR_MAX - r.base;
+
+ reset_memblock_regions();
+ memblock_add(r.base, r.size);
+
+ ASSERT_EQ(rgn->base, r.base);
+ ASSERT_EQ(rgn->size, total_size);
+
+ ASSERT_EQ(memblock.memory.cnt, 1);
+ ASSERT_EQ(memblock.memory.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
static int memblock_add_checks(void)
{
prefix_reset();
memblock_add_overlap_bottom_check();
memblock_add_within_check();
memblock_add_twice_check();
+ memblock_add_between_check();
+ memblock_add_near_max_check();
prefix_pop();
return 0;
}
+/*
+ * A test that tries to mark two memory blocks that don't overlap as reserved
+ * and then reserve a third memory block in the space between the first two:
+ *
+ * | +--------+--------+--------+ |
+ * | | r1 | r3 | r2 | |
+ * +--------+--------+--------+--------+--+
+ *
+ * Expect to merge the three entries into one reserved region that starts at
+ * r1.base and has size of r1.size + r2.size + r3.size. The region counter and
+ * total for memblock.reserved are updated.
+ */
+static int memblock_reserve_between_check(void)
+{
+ struct memblock_region *rgn;
+ phys_addr_t total_size;
+
+ rgn = &memblock.reserved.regions[0];
+
+ struct region r1 = {
+ .base = SZ_1G,
+ .size = SZ_8K
+ };
+ struct region r2 = {
+ .base = SZ_1G + SZ_16K,
+ .size = SZ_8K
+ };
+ struct region r3 = {
+ .base = SZ_1G + SZ_8K,
+ .size = SZ_8K
+ };
+
+ PREFIX_PUSH();
+
+ total_size = r1.size + r2.size + r3.size;
+
+ reset_memblock_regions();
+ memblock_reserve(r1.base, r1.size);
+ memblock_reserve(r2.base, r2.size);
+ memblock_reserve(r3.base, r3.size);
+
+ ASSERT_EQ(rgn->base, r1.base);
+ ASSERT_EQ(rgn->size, total_size);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A simple test that tries to reserve a memory block r when r extends past
+ * PHYS_ADDR_MAX:
+ *
+ * +--------+
+ * | r |
+ * +--------+
+ * | +----+
+ * | | rgn|
+ * +----------------------------+----+
+ *
+ * Expect to reserve a memory block of size PHYS_ADDR_MAX - r.base. Expect the
+ * total size of reserved memory and the counter to be updated.
+ */
+static int memblock_reserve_near_max_check(void)
+{
+ struct memblock_region *rgn;
+ phys_addr_t total_size;
+
+ rgn = &memblock.reserved.regions[0];
+
+ struct region r = {
+ .base = PHYS_ADDR_MAX - SZ_1M,
+ .size = SZ_2M
+ };
+
+ PREFIX_PUSH();
+
+ total_size = PHYS_ADDR_MAX - r.base;
+
+ reset_memblock_regions();
+ memblock_reserve(r.base, r.size);
+
+ ASSERT_EQ(rgn->base, r.base);
+ ASSERT_EQ(rgn->size, total_size);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
static int memblock_reserve_checks(void)
{
prefix_reset();
memblock_reserve_overlap_bottom_check();
memblock_reserve_within_check();
memblock_reserve_twice_check();
+ memblock_reserve_between_check();
+ memblock_reserve_near_max_check();
prefix_pop();
return 0;
}
+/*
+ * A simple test that tries to remove a region r1 from the array of
+ * available memory regions when r1 is the only available region.
+ * Expect to add a memory block r1 and then remove r1 so that a dummy
+ * region is added. The region counter stays the same, and the total size
+ * is updated.
+ */
+static int memblock_remove_only_region_check(void)
+{
+ struct memblock_region *rgn;
+
+ rgn = &memblock.memory.regions[0];
+
+ struct region r1 = {
+ .base = SZ_2K,
+ .size = SZ_4K
+ };
+
+ PREFIX_PUSH();
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_remove(r1.base, r1.size);
+
+ ASSERT_EQ(rgn->base, 0);
+ ASSERT_EQ(rgn->size, 0);
+
+ ASSERT_EQ(memblock.memory.cnt, 1);
+ ASSERT_EQ(memblock.memory.total_size, 0);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A simple test that tries remove a region r2 from the array of available
+ * memory regions when r2 extends past PHYS_ADDR_MAX:
+ *
+ * +--------+
+ * | r2 |
+ * +--------+
+ * | +---+....+
+ * | |rgn| |
+ * +------------------------+---+----+
+ *
+ * Expect that only the portion between PHYS_ADDR_MAX and r2.base is removed.
+ * Expect the total size of available memory to be updated and the counter to
+ * not be updated.
+ */
+static int memblock_remove_near_max_check(void)
+{
+ struct memblock_region *rgn;
+ phys_addr_t total_size;
+
+ rgn = &memblock.memory.regions[0];
+
+ struct region r1 = {
+ .base = PHYS_ADDR_MAX - SZ_2M,
+ .size = SZ_2M
+ };
+
+ struct region r2 = {
+ .base = PHYS_ADDR_MAX - SZ_1M,
+ .size = SZ_2M
+ };
+
+ PREFIX_PUSH();
+
+ total_size = r1.size - (PHYS_ADDR_MAX - r2.base);
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_remove(r2.base, r2.size);
+
+ ASSERT_EQ(rgn->base, r1.base);
+ ASSERT_EQ(rgn->size, total_size);
+
+ ASSERT_EQ(memblock.memory.cnt, 1);
+ ASSERT_EQ(memblock.memory.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to remove a region r3 that overlaps with two existing
+ * regions r1 and r2:
+ *
+ * +----------------+
+ * | r3 |
+ * +----------------+
+ * | +----+..... ........+--------+
+ * | | |r1 : : |r2 | |
+ * +----+----+----+---+-------+--------+-----+
+ *
+ * Expect that only the intersections of r1 with r3 and r2 with r3 are removed
+ * from the available memory pool. Expect the total size of available memory to
+ * be updated and the counter to not be updated.
+ */
+static int memblock_remove_overlap_two_check(void)
+{
+ struct memblock_region *rgn1, *rgn2;
+ phys_addr_t new_r1_size, new_r2_size, r2_end, r3_end, total_size;
+
+ rgn1 = &memblock.memory.regions[0];
+ rgn2 = &memblock.memory.regions[1];
+
+ struct region r1 = {
+ .base = SZ_16M,
+ .size = SZ_32M
+ };
+ struct region r2 = {
+ .base = SZ_64M,
+ .size = SZ_64M
+ };
+ struct region r3 = {
+ .base = SZ_32M,
+ .size = SZ_64M
+ };
+
+ PREFIX_PUSH();
+
+ r2_end = r2.base + r2.size;
+ r3_end = r3.base + r3.size;
+ new_r1_size = r3.base - r1.base;
+ new_r2_size = r2_end - r3_end;
+ total_size = new_r1_size + new_r2_size;
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_add(r2.base, r2.size);
+ memblock_remove(r3.base, r3.size);
+
+ ASSERT_EQ(rgn1->base, r1.base);
+ ASSERT_EQ(rgn1->size, new_r1_size);
+
+ ASSERT_EQ(rgn2->base, r3_end);
+ ASSERT_EQ(rgn2->size, new_r2_size);
+
+ ASSERT_EQ(memblock.memory.cnt, 2);
+ ASSERT_EQ(memblock.memory.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
static int memblock_remove_checks(void)
{
prefix_reset();
memblock_remove_overlap_top_check();
memblock_remove_overlap_bottom_check();
memblock_remove_within_check();
+ memblock_remove_only_region_check();
+ memblock_remove_near_max_check();
+ memblock_remove_overlap_two_check();
prefix_pop();
return 0;
}
+/*
+ * A simple test that tries to free a memory block r1 that was marked
+ * earlier as reserved when r1 is the only available region.
+ * Expect to reserve a memory block r1 and then free r1 so that r1 is
+ * overwritten with a dummy region. The region counter stays the same,
+ * and the total size is updated.
+ */
+static int memblock_free_only_region_check(void)
+{
+ struct memblock_region *rgn;
+
+ rgn = &memblock.reserved.regions[0];
+
+ struct region r1 = {
+ .base = SZ_2K,
+ .size = SZ_4K
+ };
+
+ PREFIX_PUSH();
+
+ reset_memblock_regions();
+ memblock_reserve(r1.base, r1.size);
+ memblock_free((void *)r1.base, r1.size);
+
+ ASSERT_EQ(rgn->base, 0);
+ ASSERT_EQ(rgn->size, 0);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, 0);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A simple test that tries free a region r2 when r2 extends past PHYS_ADDR_MAX:
+ *
+ * +--------+
+ * | r2 |
+ * +--------+
+ * | +---+....+
+ * | |rgn| |
+ * +------------------------+---+----+
+ *
+ * Expect that only the portion between PHYS_ADDR_MAX and r2.base is freed.
+ * Expect the total size of reserved memory to be updated and the counter to
+ * not be updated.
+ */
+static int memblock_free_near_max_check(void)
+{
+ struct memblock_region *rgn;
+ phys_addr_t total_size;
+
+ rgn = &memblock.reserved.regions[0];
+
+ struct region r1 = {
+ .base = PHYS_ADDR_MAX - SZ_2M,
+ .size = SZ_2M
+ };
+
+ struct region r2 = {
+ .base = PHYS_ADDR_MAX - SZ_1M,
+ .size = SZ_2M
+ };
+
+ PREFIX_PUSH();
+
+ total_size = r1.size - (PHYS_ADDR_MAX - r2.base);
+
+ reset_memblock_regions();
+ memblock_reserve(r1.base, r1.size);
+ memblock_free((void *)r2.base, r2.size);
+
+ ASSERT_EQ(rgn->base, r1.base);
+ ASSERT_EQ(rgn->size, total_size);
+
+ ASSERT_EQ(memblock.reserved.cnt, 1);
+ ASSERT_EQ(memblock.reserved.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to free a reserved region r3 that overlaps with two
+ * existing reserved regions r1 and r2:
+ *
+ * +----------------+
+ * | r3 |
+ * +----------------+
+ * | +----+..... ........+--------+
+ * | | |r1 : : |r2 | |
+ * +----+----+----+---+-------+--------+-----+
+ *
+ * Expect that only the intersections of r1 with r3 and r2 with r3 are freed
+ * from the collection of reserved memory. Expect the total size of reserved
+ * memory to be updated and the counter to not be updated.
+ */
+static int memblock_free_overlap_two_check(void)
+{
+ struct memblock_region *rgn1, *rgn2;
+ phys_addr_t new_r1_size, new_r2_size, r2_end, r3_end, total_size;
+
+ rgn1 = &memblock.reserved.regions[0];
+ rgn2 = &memblock.reserved.regions[1];
+
+ struct region r1 = {
+ .base = SZ_16M,
+ .size = SZ_32M
+ };
+ struct region r2 = {
+ .base = SZ_64M,
+ .size = SZ_64M
+ };
+ struct region r3 = {
+ .base = SZ_32M,
+ .size = SZ_64M
+ };
+
+ PREFIX_PUSH();
+
+ r2_end = r2.base + r2.size;
+ r3_end = r3.base + r3.size;
+ new_r1_size = r3.base - r1.base;
+ new_r2_size = r2_end - r3_end;
+ total_size = new_r1_size + new_r2_size;
+
+ reset_memblock_regions();
+ memblock_reserve(r1.base, r1.size);
+ memblock_reserve(r2.base, r2.size);
+ memblock_free((void *)r3.base, r3.size);
+
+ ASSERT_EQ(rgn1->base, r1.base);
+ ASSERT_EQ(rgn1->size, new_r1_size);
+
+ ASSERT_EQ(rgn2->base, r3_end);
+ ASSERT_EQ(rgn2->size, new_r2_size);
+
+ ASSERT_EQ(memblock.reserved.cnt, 2);
+ ASSERT_EQ(memblock.reserved.total_size, total_size);
+
+ test_pass_pop();
+
+ return 0;
+}
+
static int memblock_free_checks(void)
{
prefix_reset();
memblock_free_overlap_top_check();
memblock_free_overlap_bottom_check();
memblock_free_within_check();
+ memblock_free_only_region_check();
+ memblock_free_near_max_check();
+ memblock_free_overlap_two_check();
+
+ prefix_pop();
+
+ return 0;
+}
+
+static int memblock_set_bottom_up_check(void)
+{
+ prefix_push("memblock_set_bottom_up");
+
+ memblock_set_bottom_up(false);
+ ASSERT_EQ(memblock.bottom_up, false);
+ memblock_set_bottom_up(true);
+ ASSERT_EQ(memblock.bottom_up, true);
+
+ reset_memblock_attributes();
+ test_pass_pop();
+
+ return 0;
+}
+
+static int memblock_bottom_up_check(void)
+{
+ prefix_push("memblock_bottom_up");
+
+ memblock_set_bottom_up(false);
+ ASSERT_EQ(memblock_bottom_up(), memblock.bottom_up);
+ ASSERT_EQ(memblock_bottom_up(), false);
+ memblock_set_bottom_up(true);
+ ASSERT_EQ(memblock_bottom_up(), memblock.bottom_up);
+ ASSERT_EQ(memblock_bottom_up(), true);
+
+ reset_memblock_attributes();
+ test_pass_pop();
+
+ return 0;
+}
+
+static int memblock_bottom_up_checks(void)
+{
+ test_print("Running memblock_*bottom_up tests...\n");
+
+ prefix_reset();
+ memblock_set_bottom_up_check();
+ prefix_reset();
+ memblock_bottom_up_check();
+
+ return 0;
+}
+
+/*
+ * A test that tries to trim memory when both ends of the memory region are
+ * aligned. Expect that the memory will not be trimmed. Expect the counter to
+ * not be updated.
+ */
+static int memblock_trim_memory_aligned_check(void)
+{
+ struct memblock_region *rgn;
+ const phys_addr_t alignment = SMP_CACHE_BYTES;
+
+ rgn = &memblock.memory.regions[0];
+
+ struct region r = {
+ .base = alignment,
+ .size = alignment * 4
+ };
+
+ PREFIX_PUSH();
+
+ reset_memblock_regions();
+ memblock_add(r.base, r.size);
+ memblock_trim_memory(alignment);
+
+ ASSERT_EQ(rgn->base, r.base);
+ ASSERT_EQ(rgn->size, r.size);
+
+ ASSERT_EQ(memblock.memory.cnt, 1);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to trim memory when there are two available regions, r1 and
+ * r2. Region r1 is aligned on both ends and region r2 is unaligned on one end
+ * and smaller than the alignment:
+ *
+ * alignment
+ * |--------|
+ * | +-----------------+ +------+ |
+ * | | r1 | | r2 | |
+ * +--------+-----------------+--------+------+---+
+ * ^ ^ ^ ^ ^
+ * |________|________|________| |
+ * | Unaligned address
+ * Aligned addresses
+ *
+ * Expect that r1 will not be trimmed and r2 will be removed. Expect the
+ * counter to be updated.
+ */
+static int memblock_trim_memory_too_small_check(void)
+{
+ struct memblock_region *rgn;
+ const phys_addr_t alignment = SMP_CACHE_BYTES;
+
+ rgn = &memblock.memory.regions[0];
+
+ struct region r1 = {
+ .base = alignment,
+ .size = alignment * 2
+ };
+ struct region r2 = {
+ .base = alignment * 4,
+ .size = alignment - SZ_2
+ };
+
+ PREFIX_PUSH();
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_add(r2.base, r2.size);
+ memblock_trim_memory(alignment);
+
+ ASSERT_EQ(rgn->base, r1.base);
+ ASSERT_EQ(rgn->size, r1.size);
+
+ ASSERT_EQ(memblock.memory.cnt, 1);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to trim memory when there are two available regions, r1 and
+ * r2. Region r1 is aligned on both ends and region r2 is unaligned at the base
+ * and aligned at the end:
+ *
+ * Unaligned address
+ * |
+ * v
+ * | +-----------------+ +---------------+ |
+ * | | r1 | | r2 | |
+ * +--------+-----------------+----------+---------------+---+
+ * ^ ^ ^ ^ ^ ^
+ * |________|________|________|________|________|
+ * |
+ * Aligned addresses
+ *
+ * Expect that r1 will not be trimmed and r2 will be trimmed at the base.
+ * Expect the counter to not be updated.
+ */
+static int memblock_trim_memory_unaligned_base_check(void)
+{
+ struct memblock_region *rgn1, *rgn2;
+ const phys_addr_t alignment = SMP_CACHE_BYTES;
+ phys_addr_t offset = SZ_2;
+ phys_addr_t new_r2_base, new_r2_size;
+
+ rgn1 = &memblock.memory.regions[0];
+ rgn2 = &memblock.memory.regions[1];
+
+ struct region r1 = {
+ .base = alignment,
+ .size = alignment * 2
+ };
+ struct region r2 = {
+ .base = alignment * 4 + offset,
+ .size = alignment * 2 - offset
+ };
+
+ PREFIX_PUSH();
+
+ new_r2_base = r2.base + (alignment - offset);
+ new_r2_size = r2.size - (alignment - offset);
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_add(r2.base, r2.size);
+ memblock_trim_memory(alignment);
+
+ ASSERT_EQ(rgn1->base, r1.base);
+ ASSERT_EQ(rgn1->size, r1.size);
+
+ ASSERT_EQ(rgn2->base, new_r2_base);
+ ASSERT_EQ(rgn2->size, new_r2_size);
+
+ ASSERT_EQ(memblock.memory.cnt, 2);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+/*
+ * A test that tries to trim memory when there are two available regions, r1 and
+ * r2. Region r1 is aligned on both ends and region r2 is aligned at the base
+ * and unaligned at the end:
+ *
+ * Unaligned address
+ * |
+ * v
+ * | +-----------------+ +---------------+ |
+ * | | r1 | | r2 | |
+ * +--------+-----------------+--------+---------------+---+
+ * ^ ^ ^ ^ ^ ^
+ * |________|________|________|________|________|
+ * |
+ * Aligned addresses
+ *
+ * Expect that r1 will not be trimmed and r2 will be trimmed at the end.
+ * Expect the counter to not be updated.
+ */
+static int memblock_trim_memory_unaligned_end_check(void)
+{
+ struct memblock_region *rgn1, *rgn2;
+ const phys_addr_t alignment = SMP_CACHE_BYTES;
+ phys_addr_t offset = SZ_2;
+ phys_addr_t new_r2_size;
+
+ rgn1 = &memblock.memory.regions[0];
+ rgn2 = &memblock.memory.regions[1];
+
+ struct region r1 = {
+ .base = alignment,
+ .size = alignment * 2
+ };
+ struct region r2 = {
+ .base = alignment * 4,
+ .size = alignment * 2 - offset
+ };
+
+ PREFIX_PUSH();
+
+ new_r2_size = r2.size - (alignment - offset);
+
+ reset_memblock_regions();
+ memblock_add(r1.base, r1.size);
+ memblock_add(r2.base, r2.size);
+ memblock_trim_memory(alignment);
+
+ ASSERT_EQ(rgn1->base, r1.base);
+ ASSERT_EQ(rgn1->size, r1.size);
+
+ ASSERT_EQ(rgn2->base, r2.base);
+ ASSERT_EQ(rgn2->size, new_r2_size);
+
+ ASSERT_EQ(memblock.memory.cnt, 2);
+
+ test_pass_pop();
+
+ return 0;
+}
+
+static int memblock_trim_memory_checks(void)
+{
+ prefix_reset();
+ prefix_push(FUNC_TRIM);
+ test_print("Running %s tests...\n", FUNC_TRIM);
+
+ memblock_trim_memory_aligned_check();
+ memblock_trim_memory_too_small_check();
+ memblock_trim_memory_unaligned_base_check();
+ memblock_trim_memory_unaligned_end_check();
prefix_pop();
memblock_reserve_checks();
memblock_remove_checks();
memblock_free_checks();
+ memblock_bottom_up_checks();
+ memblock_trim_memory_checks();
return 0;
}
#define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
#define PREFIXES_MAX 15
#define DELIM ": "
+#define BASIS 10000
static struct test_memory memory_block;
static const char __maybe_unused *prefixes[PREFIXES_MAX];
static int __maybe_unused nr_prefixes;
-static const char *short_opts = "mv";
+static const char *short_opts = "hmv";
static const struct option long_opts[] = {
+ {"help", 0, NULL, 'h'},
{"movable-node", 0, NULL, 'm'},
{"verbose", 0, NULL, 'v'},
{NULL, 0, NULL, 0}
};
static const char * const help_opts[] = {
+ "display this help message and exit",
"disallow allocations from regions marked as hotplugged\n\t\t\t"
"by simulating enabling the \"movable_node\" kernel\n\t\t\t"
"parameter",
memblock.current_limit = MEMBLOCK_ALLOC_ANYWHERE;
}
+static inline void fill_memblock(void)
+{
+ memset(memory_block.base, 1, MEM_SIZE);
+}
+
void setup_memblock(void)
{
reset_memblock_regions();
memblock_add((phys_addr_t)memory_block.base, MEM_SIZE);
+ fill_memblock();
+}
+
+/**
+ * setup_numa_memblock:
+ * Set up a memory layout with multiple NUMA nodes in a previously allocated
+ * dummy physical memory.
+ * @node_fracs: an array representing the fraction of MEM_SIZE contained in
+ * each node in basis point units (one hundredth of 1% or 1/10000).
+ * For example, if node 0 should contain 1/8 of MEM_SIZE,
+ * node_fracs[0] = 1250.
+ *
+ * The nids will be set to 0 through NUMA_NODES - 1.
+ */
+void setup_numa_memblock(const unsigned int node_fracs[])
+{
+ phys_addr_t base;
+ int flags;
+
+ reset_memblock_regions();
+ base = (phys_addr_t)memory_block.base;
+ flags = (movable_node_is_enabled()) ? MEMBLOCK_NONE : MEMBLOCK_HOTPLUG;
+
+ for (int i = 0; i < NUMA_NODES; i++) {
+ assert(node_fracs[i] <= BASIS);
+ phys_addr_t size = MEM_SIZE * node_fracs[i] / BASIS;
+
+ memblock_add_node(base, size, i, flags);
+ base += size;
+ }
+ fill_memblock();
}
void dummy_physical_memory_init(void)
{
memory_block.base = malloc(MEM_SIZE);
assert(memory_block.base);
+ fill_memblock();
}
void dummy_physical_memory_cleanup(void)
#include <linux/printk.h>
#include <../selftests/kselftest.h>
-#define MEM_SIZE SZ_16K
+#define MEM_SIZE SZ_16K
+#define NUMA_NODES 8
+
+enum test_flags {
+ /* No special request. */
+ TEST_F_NONE = 0x0,
+ /* Perform raw allocations (no zeroing of memory). */
+ TEST_F_RAW = 0x1,
+};
/**
* ASSERT_EQ():
* Check the condition
* @_expected == @_seen
- * If false, print failed test message (if in VERBOSE mode) and then assert
+ * If false, print failed test message (if running with --verbose) and then
+ * assert.
*/
#define ASSERT_EQ(_expected, _seen) do { \
if ((_expected) != (_seen)) \
* ASSERT_NE():
* Check the condition
* @_expected != @_seen
- * If false, print failed test message (if in VERBOSE mode) and then assert
+ * If false, print failed test message (if running with --verbose) and then
+ * assert.
*/
#define ASSERT_NE(_expected, _seen) do { \
if ((_expected) == (_seen)) \
* ASSERT_LT():
* Check the condition
* @_expected < @_seen
- * If false, print failed test message (if in VERBOSE mode) and then assert
+ * If false, print failed test message (if running with --verbose) and then
+ * assert.
*/
#define ASSERT_LT(_expected, _seen) do { \
if ((_expected) >= (_seen)) \
assert((_expected) < (_seen)); \
} while (0)
+/**
+ * ASSERT_LE():
+ * Check the condition
+ * @_expected <= @_seen
+ * If false, print failed test message (if running with --verbose) and then
+ * assert.
+ */
+#define ASSERT_LE(_expected, _seen) do { \
+ if ((_expected) > (_seen)) \
+ test_fail(); \
+ assert((_expected) <= (_seen)); \
+} while (0)
+
+/**
+ * ASSERT_MEM_EQ():
+ * Check that the first @_size bytes of @_seen are all equal to @_expected.
+ * If false, print failed test message (if running with --verbose) and then
+ * assert.
+ */
+#define ASSERT_MEM_EQ(_seen, _expected, _size) do { \
+ for (int _i = 0; _i < (_size); _i++) { \
+ ASSERT_EQ(((char *)_seen)[_i], (_expected)); \
+ } \
+} while (0)
+
+/**
+ * ASSERT_MEM_NE():
+ * Check that none of the first @_size bytes of @_seen are equal to @_expected.
+ * If false, print failed test message (if running with --verbose) and then
+ * assert.
+ */
+#define ASSERT_MEM_NE(_seen, _expected, _size) do { \
+ for (int _i = 0; _i < (_size); _i++) { \
+ ASSERT_NE(((char *)_seen)[_i], (_expected)); \
+ } \
+} while (0)
+
#define PREFIX_PUSH() prefix_push(__func__)
/*
phys_addr_t size;
};
+static inline phys_addr_t __maybe_unused region_end(struct memblock_region *rgn)
+{
+ return rgn->base + rgn->size;
+}
+
void reset_memblock_regions(void);
void reset_memblock_attributes(void);
void setup_memblock(void);
+void setup_numa_memblock(const unsigned int node_fracs[]);
void dummy_physical_memory_init(void);
void dummy_physical_memory_cleanup(void);
void parse_args(int argc, char **argv);
prefix_pop();
}
+static inline void run_top_down(int (*func)())
+{
+ memblock_set_bottom_up(false);
+ prefix_push("top-down");
+ func();
+ prefix_pop();
+}
+
+static inline void run_bottom_up(int (*func)())
+{
+ memblock_set_bottom_up(true);
+ prefix_push("bottom-up");
+ func();
+ prefix_pop();
+}
+
+static inline void assert_mem_content(void *mem, int size, int flags)
+{
+ if (flags & TEST_F_RAW)
+ ASSERT_MEM_NE(mem, 0, size);
+ else
+ ASSERT_MEM_EQ(mem, 0, size);
+}
+
#endif
multiorder
radix-tree.c
xarray
+maple
+ma_xa_benchmark
-fsanitize=undefined
LDFLAGS += -fsanitize=address -fsanitize=undefined
LDLIBS+= -lpthread -lurcu
-TARGETS = main idr-test multiorder xarray
+TARGETS = main idr-test multiorder xarray maple
CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \
- slab.o
+ slab.o maple.o
OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \
regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \
iteration_check_2.o benchmark.o
xarray: $(CORE_OFILES)
+maple: $(CORE_OFILES)
+
multiorder: multiorder.o $(CORE_OFILES)
clean:
../../include/linux/*.h \
../../include/asm/*.h \
../../../include/linux/xarray.h \
+ ../../../include/linux/maple_tree.h \
../../../include/linux/radix-tree.h \
../../../include/linux/idr.h
xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c
+maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c
+
generated/map-shift.h:
@if ! grep -qws $(SHIFT) generated/map-shift.h; then \
echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \
#define CONFIG_XARRAY_MULTI 1
+#define CONFIG_64BIT 1
int nr_objs;
void *objs;
void (*ctor)(void *);
+ unsigned int non_kernel;
+ unsigned long nr_allocated;
+ unsigned long nr_tallocated;
};
+void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val)
+{
+ cachep->non_kernel = val;
+}
+
+unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep)
+{
+ return cachep->size * cachep->nr_allocated;
+}
+
+unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep)
+{
+ return cachep->nr_allocated;
+}
+
+unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep)
+{
+ return cachep->nr_tallocated;
+}
+
+void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep)
+{
+ cachep->nr_tallocated = 0;
+}
+
void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru,
int gfp)
{
void *p;
- if (!(gfp & __GFP_DIRECT_RECLAIM))
- return NULL;
+ if (!(gfp & __GFP_DIRECT_RECLAIM)) {
+ if (!cachep->non_kernel)
+ return NULL;
+
+ cachep->non_kernel--;
+ }
pthread_mutex_lock(&cachep->lock);
if (cachep->nr_objs) {
memset(p, 0, cachep->size);
}
+ uatomic_inc(&cachep->nr_allocated);
uatomic_inc(&nr_allocated);
+ uatomic_inc(&cachep->nr_tallocated);
if (kmalloc_verbose)
printf("Allocating %p from slab\n", p);
return p;
}
-void kmem_cache_free(struct kmem_cache *cachep, void *objp)
+void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)
{
assert(objp);
uatomic_dec(&nr_allocated);
+ uatomic_dec(&cachep->nr_allocated);
if (kmalloc_verbose)
printf("Freeing %p to slab\n", objp);
- pthread_mutex_lock(&cachep->lock);
if (cachep->nr_objs > 10 || cachep->align) {
memset(objp, POISON_FREE, cachep->size);
free(objp);
node->parent = cachep->objs;
cachep->objs = node;
}
+}
+
+void kmem_cache_free(struct kmem_cache *cachep, void *objp)
+{
+ pthread_mutex_lock(&cachep->lock);
+ kmem_cache_free_locked(cachep, objp);
pthread_mutex_unlock(&cachep->lock);
}
+void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list)
+{
+ if (kmalloc_verbose)
+ pr_debug("Bulk free %p[0-%lu]\n", list, size - 1);
+
+ pthread_mutex_lock(&cachep->lock);
+ for (int i = 0; i < size; i++)
+ kmem_cache_free_locked(cachep, list[i]);
+ pthread_mutex_unlock(&cachep->lock);
+}
+
+int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
+ void **p)
+{
+ size_t i;
+
+ if (kmalloc_verbose)
+ pr_debug("Bulk alloc %lu\n", size);
+
+ if (!(gfp & __GFP_DIRECT_RECLAIM)) {
+ if (cachep->non_kernel < size)
+ return 0;
+
+ cachep->non_kernel -= size;
+ }
+
+ pthread_mutex_lock(&cachep->lock);
+ if (cachep->nr_objs >= size) {
+ struct radix_tree_node *node;
+
+ for (i = 0; i < size; i++) {
+ node = cachep->objs;
+ cachep->nr_objs--;
+ cachep->objs = node->parent;
+ p[i] = node;
+ node->parent = NULL;
+ }
+ pthread_mutex_unlock(&cachep->lock);
+ } else {
+ pthread_mutex_unlock(&cachep->lock);
+ for (i = 0; i < size; i++) {
+ if (cachep->align) {
+ posix_memalign(&p[i], cachep->align,
+ cachep->size * size);
+ } else {
+ p[i] = malloc(cachep->size * size);
+ }
+ if (cachep->ctor)
+ cachep->ctor(p[i]);
+ else if (gfp & __GFP_ZERO)
+ memset(p[i], 0, cachep->size);
+ }
+ }
+
+ for (i = 0; i < size; i++) {
+ uatomic_inc(&nr_allocated);
+ uatomic_inc(&cachep->nr_allocated);
+ uatomic_inc(&cachep->nr_tallocated);
+ if (kmalloc_verbose)
+ printf("Allocating %p from slab\n", p[i]);
+ }
+
+ return size;
+}
+
struct kmem_cache *
kmem_cache_create(const char *name, unsigned int size, unsigned int align,
unsigned int flags, void (*ctor)(void *))
ret->size = size;
ret->align = align;
ret->nr_objs = 0;
+ ret->nr_allocated = 0;
+ ret->nr_tallocated = 0;
ret->objs = NULL;
ret->ctor = ctor;
+ ret->non_kernel = 0;
return ret;
}
+
+/*
+ * Test the test infrastructure for kem_cache_alloc/free and bulk counterparts.
+ */
+void test_kmem_cache_bulk(void)
+{
+ int i;
+ void *list[12];
+ static struct kmem_cache *test_cache, *test_cache2;
+
+ /*
+ * Testing the bulk allocators without aligned kmem_cache to force the
+ * bulk alloc/free to reuse
+ */
+ test_cache = kmem_cache_create("test_cache", 256, 0, SLAB_PANIC, NULL);
+
+ for (i = 0; i < 5; i++)
+ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM);
+
+ for (i = 0; i < 5; i++)
+ kmem_cache_free(test_cache, list[i]);
+ assert(test_cache->nr_objs == 5);
+
+ kmem_cache_alloc_bulk(test_cache, __GFP_DIRECT_RECLAIM, 5, list);
+ kmem_cache_free_bulk(test_cache, 5, list);
+
+ for (i = 0; i < 12 ; i++)
+ list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM);
+
+ for (i = 0; i < 12; i++)
+ kmem_cache_free(test_cache, list[i]);
+
+ /* The last free will not be kept around */
+ assert(test_cache->nr_objs == 11);
+
+ /* Aligned caches will immediately free */
+ test_cache2 = kmem_cache_create("test_cache2", 128, 128, SLAB_PANIC, NULL);
+
+ kmem_cache_alloc_bulk(test_cache2, __GFP_DIRECT_RECLAIM, 10, list);
+ kmem_cache_free_bulk(test_cache2, 10, list);
+ assert(!test_cache2->nr_objs);
+
+
+}
#include "../../../include/linux/kconfig.h"
#define printk printf
+#define pr_err printk
#define pr_info printk
#define pr_debug printk
#define pr_cont printk
struct lock_class_key *key)
{
}
+
+extern int lockdep_is_held(const void *);
#endif /* _LINUX_LOCKDEP_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0+ */
+#define atomic_t int32_t
+#include "../../../../include/linux/maple_tree.h"
+#define atomic_inc(x) uatomic_inc(x)
+#define atomic_read(x) uatomic_read(x)
+#define atomic_set(x, y) do {} while (0)
+#define U8_MAX UCHAR_MAX
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * maple_tree.c: Userspace shim for maple tree test-suite
+ * Copyright (c) 2018 Liam R. Howlett <Liam.Howlett@Oracle.com>
+ */
+
+#define CONFIG_DEBUG_MAPLE_TREE
+#define CONFIG_MAPLE_SEARCH
+#include "test.h"
+
+#define module_init(x)
+#define module_exit(x)
+#define MODULE_AUTHOR(x)
+#define MODULE_LICENSE(x)
+#define dump_stack() assert(0)
+
+#include "../../../lib/maple_tree.c"
+#undef CONFIG_DEBUG_MAPLE_TREE
+#include "../../../lib/test_maple_tree.c"
+
+void farmer_tests(void)
+{
+ struct maple_node *node;
+ DEFINE_MTREE(tree);
+
+ mt_dump(&tree);
+
+ tree.ma_root = xa_mk_value(0);
+ mt_dump(&tree);
+
+ node = mt_alloc_one(GFP_KERNEL);
+ node->parent = (void *)((unsigned long)(&tree) | 1);
+ node->slot[0] = xa_mk_value(0);
+ node->slot[1] = xa_mk_value(1);
+ node->mr64.pivot[0] = 0;
+ node->mr64.pivot[1] = 1;
+ node->mr64.pivot[2] = 0;
+ tree.ma_root = mt_mk_node(node, maple_leaf_64);
+ mt_dump(&tree);
+
+ ma_free_rcu(node);
+}
+
+void maple_tree_tests(void)
+{
+ farmer_tests();
+ maple_tree_seed();
+ maple_tree_harvest();
+}
+
+int __weak main(void)
+{
+ maple_tree_init();
+ maple_tree_tests();
+ rcu_barrier();
+ if (nr_allocated)
+ printf("nr_allocated = %d\n", nr_allocated);
+ return 0;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#define trace_ma_op(a, b) do {} while (0)
+#define trace_ma_read(a, b) do {} while (0)
+#define trace_ma_write(a, b, c, d) do {} while (0)
CONFIG_CGROUP_SCHED=y
CONFIG_MEMCG=y
CONFIG_MEMCG_KMEM=y
-CONFIG_MEMCG_SWAP=y
CONFIG_PAGE_COUNTER=y
TEST_FILES = _chk_dependency.sh _debugfs_common.sh
TEST_PROGS = debugfs_attrs.sh debugfs_schemes.sh debugfs_target_ids.sh
TEST_PROGS += debugfs_empty_targets.sh debugfs_huge_count_read_write.sh
+TEST_PROGS += debugfs_duplicate_context_creation.sh
TEST_PROGS += sysfs.sh
include ../lib.mk
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+source _debugfs_common.sh
+
+# Test duplicated context creation
+# ================================
+
+if ! echo foo > "$DBGFS/mk_contexts"
+then
+ echo "context creation failed"
+ exit 1
+fi
+
+if echo foo > "$DBGFS/mk_contexts"
+then
+ echo "duplicate context creation success"
+ exit 1
+fi
+
+if ! echo foo > "$DBGFS/rm_contexts"
+then
+ echo "context deletion failed"
+ exit 1
+fi
+
+exit 0
echo $check_disable > $EVENT_ENABLE
done
sleep $SLEEP_TIME
- echo " make sure it's still works"
+ echo " make sure it still works"
test_event_enabled $check_enable_star
reset_ftrace_filter
# SPDX-License-Identifier: GPL-2.0-only
+/aarch64/aarch32_id_regs
/aarch64/arch_timer
/aarch64/debug-exceptions
/aarch64/get-reg-list
# Compiled outputs used by test targets
TEST_GEN_PROGS_EXTENDED_x86_64 += x86_64/nx_huge_pages_test
+TEST_GEN_PROGS_aarch64 += aarch64/aarch32_id_regs
TEST_GEN_PROGS_aarch64 += aarch64/arch_timer
TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions
TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * aarch32_id_regs - Test for ID register behavior on AArch64-only systems
+ *
+ * Copyright (c) 2022 Google LLC.
+ *
+ * Test that KVM handles the AArch64 views of the AArch32 ID registers as RAZ
+ * and WI from userspace.
+ */
+
+#include <stdint.h>
+
+#include "kvm_util.h"
+#include "processor.h"
+#include "test_util.h"
+
+#define BAD_ID_REG_VAL 0x1badc0deul
+
+#define GUEST_ASSERT_REG_RAZ(reg) GUEST_ASSERT_EQ(read_sysreg_s(reg), 0)
+
+static void guest_main(void)
+{
+ GUEST_ASSERT_REG_RAZ(SYS_ID_PFR0_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_PFR1_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_DFR0_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_AFR0_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR0_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR1_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR2_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR3_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR0_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR1_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR2_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR3_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR4_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR5_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR4_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR6_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_MVFR0_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_MVFR1_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_MVFR2_EL1);
+ GUEST_ASSERT_REG_RAZ(sys_reg(3, 0, 0, 3, 3));
+ GUEST_ASSERT_REG_RAZ(SYS_ID_PFR2_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_DFR1_EL1);
+ GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR5_EL1);
+ GUEST_ASSERT_REG_RAZ(sys_reg(3, 0, 0, 3, 7));
+
+ GUEST_DONE();
+}
+
+static void test_guest_raz(struct kvm_vcpu *vcpu)
+{
+ struct ucall uc;
+
+ vcpu_run(vcpu);
+
+ switch (get_ucall(vcpu, &uc)) {
+ case UCALL_ABORT:
+ REPORT_GUEST_ASSERT(uc);
+ break;
+ case UCALL_DONE:
+ break;
+ default:
+ TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
+ }
+}
+
+static uint64_t raz_wi_reg_ids[] = {
+ KVM_ARM64_SYS_REG(SYS_ID_PFR0_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_PFR1_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_DFR0_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_MMFR0_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_MMFR1_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_MMFR2_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_MMFR3_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR0_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR1_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR2_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR3_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR4_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR5_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_MMFR4_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_ISAR6_EL1),
+ KVM_ARM64_SYS_REG(SYS_MVFR0_EL1),
+ KVM_ARM64_SYS_REG(SYS_MVFR1_EL1),
+ KVM_ARM64_SYS_REG(SYS_MVFR2_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_PFR2_EL1),
+ KVM_ARM64_SYS_REG(SYS_ID_MMFR5_EL1),
+};
+
+static void test_user_raz_wi(struct kvm_vcpu *vcpu)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(raz_wi_reg_ids); i++) {
+ uint64_t reg_id = raz_wi_reg_ids[i];
+ uint64_t val;
+
+ vcpu_get_reg(vcpu, reg_id, &val);
+ ASSERT_EQ(val, 0);
+
+ /*
+ * Expect the ioctl to succeed with no effect on the register
+ * value.
+ */
+ vcpu_set_reg(vcpu, reg_id, BAD_ID_REG_VAL);
+
+ vcpu_get_reg(vcpu, reg_id, &val);
+ ASSERT_EQ(val, 0);
+ }
+}
+
+static uint64_t raz_invariant_reg_ids[] = {
+ KVM_ARM64_SYS_REG(SYS_ID_AFR0_EL1),
+ KVM_ARM64_SYS_REG(sys_reg(3, 0, 0, 3, 3)),
+ KVM_ARM64_SYS_REG(SYS_ID_DFR1_EL1),
+ KVM_ARM64_SYS_REG(sys_reg(3, 0, 0, 3, 7)),
+};
+
+static void test_user_raz_invariant(struct kvm_vcpu *vcpu)
+{
+ int i, r;
+
+ for (i = 0; i < ARRAY_SIZE(raz_invariant_reg_ids); i++) {
+ uint64_t reg_id = raz_invariant_reg_ids[i];
+ uint64_t val;
+
+ vcpu_get_reg(vcpu, reg_id, &val);
+ ASSERT_EQ(val, 0);
+
+ r = __vcpu_set_reg(vcpu, reg_id, BAD_ID_REG_VAL);
+ TEST_ASSERT(r < 0 && errno == EINVAL,
+ "unexpected KVM_SET_ONE_REG error: r=%d, errno=%d", r, errno);
+
+ vcpu_get_reg(vcpu, reg_id, &val);
+ ASSERT_EQ(val, 0);
+ }
+}
+
+
+
+static bool vcpu_aarch64_only(struct kvm_vcpu *vcpu)
+{
+ uint64_t val, el0;
+
+ vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1), &val);
+
+ el0 = (val & ARM64_FEATURE_MASK(ID_AA64PFR0_EL0)) >> ID_AA64PFR0_EL0_SHIFT;
+ return el0 == ID_AA64PFR0_ELx_64BIT_ONLY;
+}
+
+int main(void)
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+
+ vm = vm_create_with_one_vcpu(&vcpu, guest_main);
+
+ TEST_REQUIRE(vcpu_aarch64_only(vcpu));
+
+ ucall_init(vm, NULL);
+
+ test_user_raz_wi(vcpu);
+ test_user_raz_invariant(vcpu);
+ test_guest_raz(vcpu);
+
+ ucall_uninit(vm);
+ kvm_vm_free(vm);
+}
#define SPSR_SS (1 << 21)
extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start;
+extern unsigned char iter_ss_begin, iter_ss_end;
static volatile uint64_t sw_bp_addr, hw_bp_addr;
static volatile uint64_t wp_addr, wp_data_addr;
static volatile uint64_t svc_addr;
svc_addr = regs->pc;
}
+enum single_step_op {
+ SINGLE_STEP_ENABLE = 0,
+ SINGLE_STEP_DISABLE = 1,
+};
+
+static void guest_code_ss(int test_cnt)
+{
+ uint64_t i;
+ uint64_t bvr, wvr, w_bvr, w_wvr;
+
+ for (i = 0; i < test_cnt; i++) {
+ /* Bits [1:0] of dbg{b,w}vr are RES0 */
+ w_bvr = i << 2;
+ w_wvr = i << 2;
+
+ /* Enable Single Step execution */
+ GUEST_SYNC(SINGLE_STEP_ENABLE);
+
+ /*
+ * The userspace will veriry that the pc is as expected during
+ * single step execution between iter_ss_begin and iter_ss_end.
+ */
+ asm volatile("iter_ss_begin:nop\n");
+
+ write_sysreg(w_bvr, dbgbvr0_el1);
+ write_sysreg(w_wvr, dbgwvr0_el1);
+ bvr = read_sysreg(dbgbvr0_el1);
+ wvr = read_sysreg(dbgwvr0_el1);
+
+ asm volatile("iter_ss_end:\n");
+
+ /* Disable Single Step execution */
+ GUEST_SYNC(SINGLE_STEP_DISABLE);
+
+ GUEST_ASSERT(bvr == w_bvr);
+ GUEST_ASSERT(wvr == w_wvr);
+ }
+ GUEST_DONE();
+}
+
static int debug_version(struct kvm_vcpu *vcpu)
{
uint64_t id_aa64dfr0;
return id_aa64dfr0 & 0xf;
}
-int main(int argc, char *argv[])
+static void test_guest_debug_exceptions(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
vm_init_descriptor_tables(vm);
vcpu_init_descriptor_tables(vcpu);
- __TEST_REQUIRE(debug_version(vcpu) >= 6,
- "Armv8 debug architecture not supported.");
-
vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
ESR_EC_BRK_INS, guest_sw_bp_handler);
vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
done:
kvm_vm_free(vm);
+}
+
+void test_single_step_from_userspace(int test_cnt)
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ struct ucall uc;
+ struct kvm_run *run;
+ uint64_t pc, cmd;
+ uint64_t test_pc = 0;
+ bool ss_enable = false;
+ struct kvm_guest_debug debug = {};
+
+ vm = vm_create_with_one_vcpu(&vcpu, guest_code_ss);
+ ucall_init(vm, NULL);
+ run = vcpu->run;
+ vcpu_args_set(vcpu, 1, test_cnt);
+
+ while (1) {
+ vcpu_run(vcpu);
+ if (run->exit_reason != KVM_EXIT_DEBUG) {
+ cmd = get_ucall(vcpu, &uc);
+ if (cmd == UCALL_ABORT) {
+ REPORT_GUEST_ASSERT(uc);
+ /* NOT REACHED */
+ } else if (cmd == UCALL_DONE) {
+ break;
+ }
+
+ TEST_ASSERT(cmd == UCALL_SYNC,
+ "Unexpected ucall cmd 0x%lx", cmd);
+
+ if (uc.args[1] == SINGLE_STEP_ENABLE) {
+ debug.control = KVM_GUESTDBG_ENABLE |
+ KVM_GUESTDBG_SINGLESTEP;
+ ss_enable = true;
+ } else {
+ debug.control = SINGLE_STEP_DISABLE;
+ ss_enable = false;
+ }
+
+ vcpu_guest_debug_set(vcpu, &debug);
+ continue;
+ }
+
+ TEST_ASSERT(ss_enable, "Unexpected KVM_EXIT_DEBUG");
+
+ /* Check if the current pc is expected. */
+ vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pc), &pc);
+ TEST_ASSERT(!test_pc || pc == test_pc,
+ "Unexpected pc 0x%lx (expected 0x%lx)",
+ pc, test_pc);
+
+ /*
+ * If the current pc is between iter_ss_bgin and
+ * iter_ss_end, the pc for the next KVM_EXIT_DEBUG should
+ * be the current pc + 4.
+ */
+ if ((pc >= (uint64_t)&iter_ss_begin) &&
+ (pc < (uint64_t)&iter_ss_end))
+ test_pc = pc + 4;
+ else
+ test_pc = 0;
+ }
+
+ kvm_vm_free(vm);
+}
+
+static void help(char *name)
+{
+ puts("");
+ printf("Usage: %s [-h] [-i iterations of the single step test]\n", name);
+ puts("");
+ exit(0);
+}
+
+int main(int argc, char *argv[])
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_vm *vm;
+ int opt;
+ int ss_iteration = 10000;
+
+ vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+ __TEST_REQUIRE(debug_version(vcpu) >= 6,
+ "Armv8 debug architecture not supported.");
+ kvm_vm_free(vm);
+
+ while ((opt = getopt(argc, argv, "i:")) != -1) {
+ switch (opt) {
+ case 'i':
+ ss_iteration = atoi(optarg);
+ break;
+ case 'h':
+ default:
+ help(argv[0]);
+ break;
+ }
+ }
+
+ test_guest_debug_exceptions();
+ test_single_step_from_userspace(ss_iteration);
+
return 0;
}
// SPDX-License-Identifier: GPL-2.0-only
/*
- * psci_cpu_on_test - Test that the observable state of a vCPU targeted by the
- * CPU_ON PSCI call matches what the caller requested.
+ * psci_test - Tests relating to KVM's PSCI implementation.
*
* Copyright (c) 2021 Google LLC.
*
- * This is a regression test for a race between KVM servicing the PSCI call and
- * userspace reading the vCPUs registers.
+ * This test includes:
+ * - A regression test for a race between KVM servicing the PSCI CPU_ON call
+ * and userspace reading the targeted vCPU's registers.
+ * - A test for KVM's handling of PSCI SYSTEM_SUSPEND and the associated
+ * KVM_SYSTEM_EVENT_SUSPEND UAPI.
*/
#define _GNU_SOURCE
#include <linux/bitmap.h>
#include <linux/bitops.h>
#include <linux/atomic.h>
+#include <asm/barrier.h>
#include "kvm_util.h"
#include "test_util.h"
static bool dirty_ring_supported(void)
{
- return kvm_has_cap(KVM_CAP_DIRTY_LOG_RING);
+ return (kvm_has_cap(KVM_CAP_DIRTY_LOG_RING) ||
+ kvm_has_cap(KVM_CAP_DIRTY_LOG_RING_ACQ_REL));
}
static void dirty_ring_create_vm_done(struct kvm_vm *vm)
static inline bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn)
{
- return gfn->flags == KVM_DIRTY_GFN_F_DIRTY;
+ return smp_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY;
}
static inline void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn)
{
- gfn->flags = KVM_DIRTY_GFN_F_RESET;
+ smp_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET);
}
static uint32_t dirty_ring_collect_one(struct kvm_dirty_gfn *dirty_gfns,
int open_path_or_exit(const char *path, int flags);
int open_kvm_dev_path_or_exit(void);
+
+bool get_kvm_intel_param_bool(const char *param);
+bool get_kvm_amd_param_bool(const char *param);
+
unsigned int kvm_check_cap(long cap);
static inline bool kvm_has_cap(long cap)
#a, #b, #a, (unsigned long) __a, #b, (unsigned long) __b); \
} while (0)
-#define TEST_FAIL(fmt, ...) \
- TEST_ASSERT(false, fmt, ##__VA_ARGS__)
+#define TEST_FAIL(fmt, ...) do { \
+ TEST_ASSERT(false, fmt, ##__VA_ARGS__); \
+ __builtin_unreachable(); \
+} while (0)
size_t parse_size(const char *size);
return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr));
}
+bool kvm_is_tdp_enabled(void);
+
uint64_t vm_get_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
uint64_t vaddr);
void vm_set_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
#define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G)
void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level);
+void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
+ uint64_t nr_bytes, int level);
/*
* Basic CPU control in CR0
return _open_kvm_dev_path_or_exit(O_RDONLY);
}
+static bool get_module_param_bool(const char *module_name, const char *param)
+{
+ const int path_size = 128;
+ char path[path_size];
+ char value;
+ ssize_t r;
+ int fd;
+
+ r = snprintf(path, path_size, "/sys/module/%s/parameters/%s",
+ module_name, param);
+ TEST_ASSERT(r < path_size,
+ "Failed to construct sysfs path in %d bytes.", path_size);
+
+ fd = open_path_or_exit(path, O_RDONLY);
+
+ r = read(fd, &value, 1);
+ TEST_ASSERT(r == 1, "read(%s) failed", path);
+
+ r = close(fd);
+ TEST_ASSERT(!r, "close(%s) failed", path);
+
+ if (value == 'Y')
+ return true;
+ else if (value == 'N')
+ return false;
+
+ TEST_FAIL("Unrecognized value '%c' for boolean module param", value);
+}
+
+bool get_kvm_intel_param_bool(const char *param)
+{
+ return get_module_param_bool("kvm_intel", param);
+}
+
+bool get_kvm_amd_param_bool(const char *param)
+{
+ return get_module_param_bool("kvm_amd", param);
+}
+
/*
* Capability
*
void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
{
- vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size);
+ if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL))
+ vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size);
+ else
+ vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size);
vm->dirty_ring_size = ring_size;
}
}
}
+bool kvm_is_tdp_enabled(void)
+{
+ if (is_intel_cpu())
+ return get_kvm_intel_param_bool("ept");
+ else
+ return get_kvm_amd_param_bool("npt");
+}
+
void virt_arch_pgd_alloc(struct kvm_vm *vm)
{
TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
__virt_pg_map(vm, vaddr, paddr, PG_LEVEL_4K);
}
+void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
+ uint64_t nr_bytes, int level)
+{
+ uint64_t pg_size = PG_LEVEL_SIZE(level);
+ uint64_t nr_pages = nr_bytes / pg_size;
+ int i;
+
+ TEST_ASSERT(nr_bytes % pg_size == 0,
+ "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx",
+ nr_bytes, pg_size);
+
+ for (i = 0; i < nr_pages; i++) {
+ __virt_pg_map(vm, vaddr, paddr, level);
+
+ vaddr += pg_size;
+ paddr += pg_size;
+ }
+}
+
static uint64_t *_vm_get_page_table_entry(struct kvm_vm *vm,
struct kvm_vcpu *vcpu,
uint64_t vaddr)
/* Returns true if kvm_intel was loaded with unrestricted_guest=1. */
bool vm_is_unrestricted_guest(struct kvm_vm *vm)
{
- char val = 'N';
- size_t count;
- FILE *f;
-
/* Ensure that a KVM vendor-specific module is loaded. */
if (vm == NULL)
close(open_kvm_dev_path_or_exit());
- f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
- if (f) {
- count = fread(&val, sizeof(char), 1, f);
- TEST_ASSERT(count == 1, "Unable to read from param file.");
- fclose(f);
- }
-
- return val == 'Y';
+ return get_kvm_intel_param_bool("unrestricted_guest");
}
seg->base = base;
}
-/*
- * Avoid using memset to clear the vmcb, since libc may not be
- * available in L1 (and, even if it is, features that libc memset may
- * want to use, like AVX, may not be enabled).
- */
-static void clear_vmcb(struct vmcb *vmcb)
-{
- int n = sizeof(*vmcb) / sizeof(u32);
-
- asm volatile ("rep stosl" : "+c"(n), "+D"(vmcb) : "a"(0) : "memory");
-}
-
void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp)
{
struct vmcb *vmcb = svm->vmcb;
wrmsr(MSR_EFER, efer | EFER_SVME);
wrmsr(MSR_VM_HSAVE_PA, svm->save_area_gpa);
- clear_vmcb(vmcb);
+ memset(vmcb, 0, sizeof(*vmcb));
asm volatile ("vmsave %0\n\t" : : "a" (vmcb_gpa) : "memory");
vmcb_set_seg(&save->es, get_es(), 0, -1U, data_seg_attr);
vmcb_set_seg(&save->cs, get_cs(), 0, -1U, code_seg_attr);
/* VMCALL and VMMCALL are both 3-byte opcodes. */
#define HYPERCALL_INSN_SIZE 3
-static bool ud_expected;
+static bool quirk_disabled;
static void guest_ud_handler(struct ex_regs *regs)
{
- GUEST_ASSERT(ud_expected);
- GUEST_DONE();
+ regs->rax = -EFAULT;
+ regs->rip += HYPERCALL_INSN_SIZE;
}
-extern uint8_t svm_hypercall_insn[HYPERCALL_INSN_SIZE];
-static uint64_t svm_do_sched_yield(uint8_t apic_id)
-{
- uint64_t ret;
+static const uint8_t vmx_vmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xc1 };
+static const uint8_t svm_vmmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xd9 };
- asm volatile("mov %1, %%rax\n\t"
- "mov %2, %%rbx\n\t"
- "svm_hypercall_insn:\n\t"
- "vmmcall\n\t"
- "mov %%rax, %0\n\t"
- : "=r"(ret)
- : "r"((uint64_t)KVM_HC_SCHED_YIELD), "r"((uint64_t)apic_id)
- : "rax", "rbx", "memory");
-
- return ret;
-}
-
-extern uint8_t vmx_hypercall_insn[HYPERCALL_INSN_SIZE];
-static uint64_t vmx_do_sched_yield(uint8_t apic_id)
+extern uint8_t hypercall_insn[HYPERCALL_INSN_SIZE];
+static uint64_t do_sched_yield(uint8_t apic_id)
{
uint64_t ret;
- asm volatile("mov %1, %%rax\n\t"
- "mov %2, %%rbx\n\t"
- "vmx_hypercall_insn:\n\t"
- "vmcall\n\t"
- "mov %%rax, %0\n\t"
- : "=r"(ret)
- : "r"((uint64_t)KVM_HC_SCHED_YIELD), "r"((uint64_t)apic_id)
- : "rax", "rbx", "memory");
+ asm volatile("hypercall_insn:\n\t"
+ ".byte 0xcc,0xcc,0xcc\n\t"
+ : "=a"(ret)
+ : "a"((uint64_t)KVM_HC_SCHED_YIELD), "b"((uint64_t)apic_id)
+ : "memory");
return ret;
}
static void guest_main(void)
{
- uint8_t *native_hypercall_insn, *hypercall_insn;
- uint8_t apic_id;
-
- apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID));
+ const uint8_t *native_hypercall_insn;
+ const uint8_t *other_hypercall_insn;
+ uint64_t ret;
if (is_intel_cpu()) {
- native_hypercall_insn = vmx_hypercall_insn;
- hypercall_insn = svm_hypercall_insn;
- svm_do_sched_yield(apic_id);
+ native_hypercall_insn = vmx_vmcall;
+ other_hypercall_insn = svm_vmmcall;
} else if (is_amd_cpu()) {
- native_hypercall_insn = svm_hypercall_insn;
- hypercall_insn = vmx_hypercall_insn;
- vmx_do_sched_yield(apic_id);
+ native_hypercall_insn = svm_vmmcall;
+ other_hypercall_insn = vmx_vmcall;
} else {
GUEST_ASSERT(0);
/* unreachable */
return;
}
+ memcpy(hypercall_insn, other_hypercall_insn, HYPERCALL_INSN_SIZE);
+
+ ret = do_sched_yield(GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID)));
+
/*
- * The hypercall didn't #UD (guest_ud_handler() signals "done" if a #UD
- * occurs). Verify that a #UD is NOT expected and that KVM patched in
- * the native hypercall.
+ * If the quirk is disabled, verify that guest_ud_handler() "returned"
+ * -EFAULT and that KVM did NOT patch the hypercall. If the quirk is
+ * enabled, verify that the hypercall succeeded and that KVM patched in
+ * the "right" hypercall.
*/
- GUEST_ASSERT(!ud_expected);
- GUEST_ASSERT(!memcmp(native_hypercall_insn, hypercall_insn, HYPERCALL_INSN_SIZE));
- GUEST_DONE();
-}
+ if (quirk_disabled) {
+ GUEST_ASSERT(ret == (uint64_t)-EFAULT);
+ GUEST_ASSERT(!memcmp(other_hypercall_insn, hypercall_insn,
+ HYPERCALL_INSN_SIZE));
+ } else {
+ GUEST_ASSERT(!ret);
+ GUEST_ASSERT(!memcmp(native_hypercall_insn, hypercall_insn,
+ HYPERCALL_INSN_SIZE));
+ }
-static void setup_ud_vector(struct kvm_vcpu *vcpu)
-{
- vm_init_descriptor_tables(vcpu->vm);
- vcpu_init_descriptor_tables(vcpu);
- vm_install_exception_handler(vcpu->vm, UD_VECTOR, guest_ud_handler);
+ GUEST_DONE();
}
static void enter_guest(struct kvm_vcpu *vcpu)
}
}
-static void test_fix_hypercall(void)
+static void test_fix_hypercall(bool disable_quirk)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
vm = vm_create_with_one_vcpu(&vcpu, guest_main);
- setup_ud_vector(vcpu);
-
- ud_expected = false;
- sync_global_to_guest(vm, ud_expected);
-
- virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA);
-
- enter_guest(vcpu);
-}
-static void test_fix_hypercall_disabled(void)
-{
- struct kvm_vcpu *vcpu;
- struct kvm_vm *vm;
-
- vm = vm_create_with_one_vcpu(&vcpu, guest_main);
- setup_ud_vector(vcpu);
+ vm_init_descriptor_tables(vcpu->vm);
+ vcpu_init_descriptor_tables(vcpu);
+ vm_install_exception_handler(vcpu->vm, UD_VECTOR, guest_ud_handler);
- vm_enable_cap(vm, KVM_CAP_DISABLE_QUIRKS2,
- KVM_X86_QUIRK_FIX_HYPERCALL_INSN);
+ if (disable_quirk)
+ vm_enable_cap(vm, KVM_CAP_DISABLE_QUIRKS2,
+ KVM_X86_QUIRK_FIX_HYPERCALL_INSN);
- ud_expected = true;
- sync_global_to_guest(vm, ud_expected);
+ quirk_disabled = disable_quirk;
+ sync_global_to_guest(vm, quirk_disabled);
virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA);
{
TEST_REQUIRE(kvm_check_cap(KVM_CAP_DISABLE_QUIRKS2) & KVM_X86_QUIRK_FIX_HYPERCALL_INSN);
- test_fix_hypercall();
- test_fix_hypercall_disabled();
+ test_fix_hypercall(false);
+ test_fix_hypercall(true);
}
: "=a" (*hv_status),
"+c" (control), "+d" (input_address),
KVM_ASM_SAFE_OUTPUTS(vector)
- : [output_address] "r"(output_address)
+ : [output_address] "r"(output_address),
+ "a" (-EFAULT)
: "cc", "memory", "r8", KVM_ASM_SAFE_CLOBBERS);
return vector;
}
}
vector = hypercall(hcall->control, input, output, &res);
- if (hcall->ud_expected)
+ if (hcall->ud_expected) {
GUEST_ASSERT_2(vector == UD_VECTOR, hcall->control, vector);
- else
+ } else {
GUEST_ASSERT_2(!vector, hcall->control, vector);
+ GUEST_ASSERT_2(res == hcall->expect, hcall->expect, res);
+ }
- GUEST_ASSERT_2(!hcall->ud_expected || res == hcall->expect,
- hcall->expect, res);
GUEST_DONE();
}
switch (stage) {
case 0:
feat->eax |= HV_MSR_HYPERCALL_AVAILABLE;
- hcall->control = 0xdeadbeef;
+ hcall->control = 0xbeef;
hcall->expect = HV_STATUS_INVALID_HYPERCALL_CODE;
break;
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
+ uint64_t nr_bytes;
void *hva;
int r;
HPAGE_GPA, HPAGE_SLOT,
HPAGE_SLOT_NPAGES, 0);
- virt_map(vm, HPAGE_GVA, HPAGE_GPA, HPAGE_SLOT_NPAGES);
+ nr_bytes = HPAGE_SLOT_NPAGES * vm->page_size;
+
+ /*
+ * Ensure that KVM can map HPAGE_SLOT with huge pages by mapping the
+ * region into the guest with 2MiB pages whenever TDP is disabled (i.e.
+ * whenever KVM is shadowing the guest page tables).
+ *
+ * When TDP is enabled, KVM should be able to map HPAGE_SLOT with huge
+ * pages irrespective of the guest page size, so map with 4KiB pages
+ * to test that that is the case.
+ */
+ if (kvm_is_tdp_enabled())
+ virt_map_level(vm, HPAGE_GVA, HPAGE_GPA, nr_bytes, PG_LEVEL_4K);
+ else
+ virt_map_level(vm, HPAGE_GVA, HPAGE_GPA, nr_bytes, PG_LEVEL_2M);
hva = addr_gpa2hva(vm, HPAGE_GPA);
- memset(hva, RETURN_OPCODE, HPAGE_SLOT_NPAGES * PAGE_SIZE);
+ memset(hva, RETURN_OPCODE, nr_bytes);
check_2m_page_count(vm, 0);
check_split_count(vm, 0);
return 0
}
+online_all_offline_memory()
+{
+ for memory in `hotpluggable_offline_memory`; do
+ if ! online_memory_expect_success $memory; then
+ echo "$FUNCNAME $memory: unexpected fail" >&2
+ retval=1
+ fi
+ done
+}
+
error=-12
priority=0
# Run with default of ratio=2 for Kselftest run
for memory in `hotpluggable_online_memory`; do
if [ "$target" -gt 0 ]; then
echo "online->offline memory$memory"
- if offline_memory_expect_success $memory; then
+ if offline_memory_expect_success $memory &>/dev/null; then
target=$(($target - 1))
+ echo "-> Success"
+ else
+ echo "-> Failure"
fi
fi
done
echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_OFFLINE/error
for memory in `hotpluggable_online_memory`; do
if [ $((RANDOM % 100)) -lt $ratio ]; then
- offline_memory_expect_success $memory
+ offline_memory_expect_success $memory &>/dev/null
fi
done
#
echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_ONLINE/error
for memory in `hotpluggable_offline_memory`; do
- online_memory_expect_fail $memory
+ if ! online_memory_expect_fail $memory; then
+ retval=1
+ fi
done
#
# Online all hot-pluggable memory
#
echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_ONLINE/error
-for memory in `hotpluggable_offline_memory`; do
- online_memory_expect_success $memory
-done
+online_all_offline_memory
#
# Test memory hot-remove error handling (online => offline)
echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_OFFLINE/error
for memory in `hotpluggable_online_memory`; do
if [ $((RANDOM % 100)) -lt $ratio ]; then
- offline_memory_expect_fail $memory
+ if ! offline_memory_expect_fail $memory; then
+ retval=1
+ fi
fi
done
echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_OFFLINE/error
/sbin/modprobe -q -r memory-notifier-error-inject
+#
+# Restore memory before exit
+#
+online_all_offline_memory
+
exit $retval
log_test $rc 0 "Delete nexthop route warning"
run_cmd "$IP route delete 172.16.101.1/32 nhid 12"
run_cmd "$IP nexthop del id 12"
+
+ run_cmd "$IP nexthop add id 21 via 172.16.1.6 dev veth1"
+ run_cmd "$IP ro add 172.16.101.0/24 nhid 21"
+ run_cmd "$IP ro del 172.16.101.0/24 nexthop via 172.16.1.7 dev veth1 nexthop via 172.16.1.8 dev veth1"
+ log_test $? 2 "Delete multipath route with only nh id based entry"
}
ipv4_grp_fcnal()
nft_concat_range.sh nft_conntrack_helper.sh \
nft_queue.sh nft_meta.sh nf_nat_edemux.sh \
ipip-conntrack-mtu.sh conntrack_tcp_unreplied.sh \
- conntrack_vrf.sh nft_synproxy.sh
+ conntrack_vrf.sh nft_synproxy.sh rpath.sh
CFLAGS += $(shell pkg-config --cflags libmnl 2>/dev/null || echo "-I/usr/include/libmnl")
LDLIBS = -lmnl
ip netns exec ${nsrouter} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth0.forwarding=1 > /dev/null
ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth1.forwarding=1 > /dev/null
+ip netns exec ${nsrouter} sysctl net.ipv4.conf.all.rp_filter=0 > /dev/null
ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth0.rp_filter=0 > /dev/null
sleep 3
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# return code to signal skipped test
+ksft_skip=4
+
+# search for legacy iptables (it uses the xtables extensions
+if iptables-legacy --version >/dev/null 2>&1; then
+ iptables='iptables-legacy'
+elif iptables --version >/dev/null 2>&1; then
+ iptables='iptables'
+else
+ iptables=''
+fi
+
+if ip6tables-legacy --version >/dev/null 2>&1; then
+ ip6tables='ip6tables-legacy'
+elif ! ip6tables --version >/dev/null 2>&1; then
+ ip6tables='ip6tables'
+else
+ ip6tables=''
+fi
+
+if nft --version >/dev/null 2>&1; then
+ nft='nft'
+else
+ nft=''
+fi
+
+if [ -z "$iptables$ip6tables$nft" ]; then
+ echo "SKIP: Test needs iptables, ip6tables or nft"
+ exit $ksft_skip
+fi
+
+sfx=$(mktemp -u "XXXXXXXX")
+ns1="ns1-$sfx"
+ns2="ns2-$sfx"
+trap "ip netns del $ns1; ip netns del $ns2" EXIT
+
+# create two netns, disable rp_filter in ns2 and
+# keep IPv6 address when moving into VRF
+ip netns add "$ns1"
+ip netns add "$ns2"
+ip netns exec "$ns2" sysctl -q net.ipv4.conf.all.rp_filter=0
+ip netns exec "$ns2" sysctl -q net.ipv4.conf.default.rp_filter=0
+ip netns exec "$ns2" sysctl -q net.ipv6.conf.all.keep_addr_on_down=1
+
+# a standard connection between the netns, should not trigger rp filter
+ip -net "$ns1" link add v0 type veth peer name v0 netns "$ns2"
+ip -net "$ns1" link set v0 up; ip -net "$ns2" link set v0 up
+ip -net "$ns1" a a 192.168.23.2/24 dev v0
+ip -net "$ns2" a a 192.168.23.1/24 dev v0
+ip -net "$ns1" a a fec0:23::2/64 dev v0 nodad
+ip -net "$ns2" a a fec0:23::1/64 dev v0 nodad
+
+# rp filter testing: ns1 sends packets via v0 which ns2 would route back via d0
+ip -net "$ns2" link add d0 type dummy
+ip -net "$ns2" link set d0 up
+ip -net "$ns1" a a 192.168.42.2/24 dev v0
+ip -net "$ns2" a a 192.168.42.1/24 dev d0
+ip -net "$ns1" a a fec0:42::2/64 dev v0 nodad
+ip -net "$ns2" a a fec0:42::1/64 dev d0 nodad
+
+# firewall matches to test
+ip netns exec "$ns2" "$iptables" -t raw -A PREROUTING -s 192.168.0.0/16 -m rpfilter
+ip netns exec "$ns2" "$ip6tables" -t raw -A PREROUTING -s fec0::/16 -m rpfilter
+ip netns exec "$ns2" nft -f - <<EOF
+table inet t {
+ chain c {
+ type filter hook prerouting priority raw;
+ ip saddr 192.168.0.0/16 fib saddr . iif oif exists counter
+ ip6 saddr fec0::/16 fib saddr . iif oif exists counter
+ }
+}
+EOF
+
+die() {
+ echo "FAIL: $*"
+ #ip netns exec "$ns2" "$iptables" -t raw -vS
+ #ip netns exec "$ns2" "$ip6tables" -t raw -vS
+ #ip netns exec "$ns2" nft list ruleset
+ exit 1
+}
+
+# check rule counters, return true if rule did not match
+ipt_zero_rule() { # (command)
+ [ -n "$1" ] || return 0
+ ip netns exec "$ns2" "$1" -t raw -vS | grep -q -- "-m rpfilter -c 0 0"
+}
+nft_zero_rule() { # (family)
+ [ -n "$nft" ] || return 0
+ ip netns exec "$ns2" "$nft" list chain inet t c | \
+ grep -q "$1 saddr .* counter packets 0 bytes 0"
+}
+
+netns_ping() { # (netns, args...)
+ local netns="$1"
+ shift
+ ip netns exec "$netns" ping -q -c 1 -W 1 "$@" >/dev/null
+}
+
+testrun() {
+ # clear counters first
+ [ -n "$iptables" ] && ip netns exec "$ns2" "$iptables" -t raw -Z
+ [ -n "$ip6tables" ] && ip netns exec "$ns2" "$ip6tables" -t raw -Z
+ if [ -n "$nft" ]; then
+ (
+ echo "delete table inet t";
+ ip netns exec "$ns2" nft -s list table inet t;
+ ) | ip netns exec "$ns2" nft -f -
+ fi
+
+ # test 1: martian traffic should fail rpfilter matches
+ netns_ping "$ns1" -I v0 192.168.42.1 && \
+ die "martian ping 192.168.42.1 succeeded"
+ netns_ping "$ns1" -I v0 fec0:42::1 && \
+ die "martian ping fec0:42::1 succeeded"
+
+ ipt_zero_rule "$iptables" || die "iptables matched martian"
+ ipt_zero_rule "$ip6tables" || die "ip6tables matched martian"
+ nft_zero_rule ip || die "nft IPv4 matched martian"
+ nft_zero_rule ip6 || die "nft IPv6 matched martian"
+
+ # test 2: rpfilter match should pass for regular traffic
+ netns_ping "$ns1" 192.168.23.1 || \
+ die "regular ping 192.168.23.1 failed"
+ netns_ping "$ns1" fec0:23::1 || \
+ die "regular ping fec0:23::1 failed"
+
+ ipt_zero_rule "$iptables" && die "iptables match not effective"
+ ipt_zero_rule "$ip6tables" && die "ip6tables match not effective"
+ nft_zero_rule ip && die "nft IPv4 match not effective"
+ nft_zero_rule ip6 && die "nft IPv6 match not effective"
+
+}
+
+testrun
+
+# repeat test with vrf device in $ns2
+ip -net "$ns2" link add vrf0 type vrf table 10
+ip -net "$ns2" link set vrf0 up
+ip -net "$ns2" link set v0 master vrf0
+
+testrun
+
+echo "PASS: netfilter reverse path match works as intended"
+exit 0
/proc-fsconfig-hidepid
/proc-loadavg-001
/proc-multiple-procfs
+/proc-empty-vm
/proc-pid-vm
/proc-self-map-files-001
/proc-self-map-files-002
TEST_GEN_PROGS += fd-002-posix-eq
TEST_GEN_PROGS += fd-003-kthread
TEST_GEN_PROGS += proc-loadavg-001
+TEST_GEN_PROGS += proc-empty-vm
TEST_GEN_PROGS += proc-pid-vm
TEST_GEN_PROGS += proc-self-map-files-001
TEST_GEN_PROGS += proc-self-map-files-002
--- /dev/null
+/*
+ * Copyright (c) 2022 Alexey Dobriyan <adobriyan@gmail.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+/*
+ * Create a process without mappings by unmapping everything at once and
+ * holding it with ptrace(2). See what happens to
+ *
+ * /proc/${pid}/maps
+ * /proc/${pid}/numa_maps
+ * /proc/${pid}/smaps
+ * /proc/${pid}/smaps_rollup
+ */
+#undef NDEBUG
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/ptrace.h>
+#include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+/*
+ * 0: vsyscall VMA doesn't exist vsyscall=none
+ * 1: vsyscall VMA is --xp vsyscall=xonly
+ * 2: vsyscall VMA is r-xp vsyscall=emulate
+ */
+static int g_vsyscall;
+static const char *g_proc_pid_maps_vsyscall;
+static const char *g_proc_pid_smaps_vsyscall;
+
+static const char proc_pid_maps_vsyscall_0[] = "";
+static const char proc_pid_maps_vsyscall_1[] =
+"ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]\n";
+static const char proc_pid_maps_vsyscall_2[] =
+"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n";
+
+static const char proc_pid_smaps_vsyscall_0[] = "";
+
+static const char proc_pid_smaps_vsyscall_1[] =
+"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"
+"Size: 4 kB\n"
+"KernelPageSize: 4 kB\n"
+"MMUPageSize: 4 kB\n"
+"Rss: 0 kB\n"
+"Pss: 0 kB\n"
+"Pss_Dirty: 0 kB\n"
+"Shared_Clean: 0 kB\n"
+"Shared_Dirty: 0 kB\n"
+"Private_Clean: 0 kB\n"
+"Private_Dirty: 0 kB\n"
+"Referenced: 0 kB\n"
+"Anonymous: 0 kB\n"
+"LazyFree: 0 kB\n"
+"AnonHugePages: 0 kB\n"
+"ShmemPmdMapped: 0 kB\n"
+"FilePmdMapped: 0 kB\n"
+"Shared_Hugetlb: 0 kB\n"
+"Private_Hugetlb: 0 kB\n"
+"Swap: 0 kB\n"
+"SwapPss: 0 kB\n"
+"Locked: 0 kB\n"
+"THPeligible: 0\n"
+/*
+ * "ProtectionKey:" field is conditional. It is possible to check it as well,
+ * but I don't have such machine.
+ */
+;
+
+static const char proc_pid_smaps_vsyscall_2[] =
+"ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]\n"
+"Size: 4 kB\n"
+"KernelPageSize: 4 kB\n"
+"MMUPageSize: 4 kB\n"
+"Rss: 0 kB\n"
+"Pss: 0 kB\n"
+"Pss_Dirty: 0 kB\n"
+"Shared_Clean: 0 kB\n"
+"Shared_Dirty: 0 kB\n"
+"Private_Clean: 0 kB\n"
+"Private_Dirty: 0 kB\n"
+"Referenced: 0 kB\n"
+"Anonymous: 0 kB\n"
+"LazyFree: 0 kB\n"
+"AnonHugePages: 0 kB\n"
+"ShmemPmdMapped: 0 kB\n"
+"FilePmdMapped: 0 kB\n"
+"Shared_Hugetlb: 0 kB\n"
+"Private_Hugetlb: 0 kB\n"
+"Swap: 0 kB\n"
+"SwapPss: 0 kB\n"
+"Locked: 0 kB\n"
+"THPeligible: 0\n"
+/*
+ * "ProtectionKey:" field is conditional. It is possible to check it as well,
+ * but I'm too tired.
+ */
+;
+
+static void sigaction_SIGSEGV(int _, siginfo_t *__, void *___)
+{
+ _exit(EXIT_FAILURE);
+}
+
+static void sigaction_SIGSEGV_vsyscall(int _, siginfo_t *__, void *___)
+{
+ _exit(g_vsyscall);
+}
+
+/*
+ * vsyscall page can't be unmapped, probe it directly.
+ */
+static void vsyscall(void)
+{
+ pid_t pid;
+ int wstatus;
+
+ pid = fork();
+ if (pid < 0) {
+ fprintf(stderr, "fork, errno %d\n", errno);
+ exit(1);
+ }
+ if (pid == 0) {
+ setrlimit(RLIMIT_CORE, &(struct rlimit){});
+
+ /* Hide "segfault at ffffffffff600000" messages. */
+ struct sigaction act = {};
+ act.sa_flags = SA_SIGINFO;
+ act.sa_sigaction = sigaction_SIGSEGV_vsyscall;
+ sigaction(SIGSEGV, &act, NULL);
+
+ g_vsyscall = 0;
+ /* gettimeofday(NULL, NULL); */
+ asm volatile (
+ "call %P0"
+ :
+ : "i" (0xffffffffff600000), "D" (NULL), "S" (NULL)
+ : "rax", "rcx", "r11"
+ );
+
+ g_vsyscall = 1;
+ *(volatile int *)0xffffffffff600000UL;
+
+ g_vsyscall = 2;
+ exit(g_vsyscall);
+ }
+ waitpid(pid, &wstatus, 0);
+ if (WIFEXITED(wstatus)) {
+ g_vsyscall = WEXITSTATUS(wstatus);
+ } else {
+ fprintf(stderr, "error: vsyscall wstatus %08x\n", wstatus);
+ exit(1);
+ }
+}
+
+static int test_proc_pid_maps(pid_t pid)
+{
+ char buf[4096];
+ snprintf(buf, sizeof(buf), "/proc/%u/maps", pid);
+ int fd = open(buf, O_RDONLY);
+ if (fd == -1) {
+ perror("open /proc/${pid}/maps");
+ return EXIT_FAILURE;
+ } else {
+ ssize_t rv = read(fd, buf, sizeof(buf));
+ close(fd);
+ if (g_vsyscall == 0) {
+ assert(rv == 0);
+ } else {
+ size_t len = strlen(g_proc_pid_maps_vsyscall);
+ assert(rv == len);
+ assert(memcmp(buf, g_proc_pid_maps_vsyscall, len) == 0);
+ }
+ return EXIT_SUCCESS;
+ }
+}
+
+static int test_proc_pid_numa_maps(pid_t pid)
+{
+ char buf[4096];
+ snprintf(buf, sizeof(buf), "/proc/%u/numa_maps", pid);
+ int fd = open(buf, O_RDONLY);
+ if (fd == -1) {
+ if (errno == ENOENT) {
+ /*
+ * /proc/${pid}/numa_maps is under CONFIG_NUMA,
+ * it doesn't necessarily exist.
+ */
+ return EXIT_SUCCESS;
+ }
+ perror("open /proc/${pid}/numa_maps");
+ return EXIT_FAILURE;
+ } else {
+ ssize_t rv = read(fd, buf, sizeof(buf));
+ close(fd);
+ assert(rv == 0);
+ return EXIT_SUCCESS;
+ }
+}
+
+static int test_proc_pid_smaps(pid_t pid)
+{
+ char buf[4096];
+ snprintf(buf, sizeof(buf), "/proc/%u/smaps", pid);
+ int fd = open(buf, O_RDONLY);
+ if (fd == -1) {
+ if (errno == ENOENT) {
+ /*
+ * /proc/${pid}/smaps is under CONFIG_PROC_PAGE_MONITOR,
+ * it doesn't necessarily exist.
+ */
+ return EXIT_SUCCESS;
+ }
+ perror("open /proc/${pid}/smaps");
+ return EXIT_FAILURE;
+ } else {
+ ssize_t rv = read(fd, buf, sizeof(buf));
+ close(fd);
+ if (g_vsyscall == 0) {
+ assert(rv == 0);
+ } else {
+ size_t len = strlen(g_proc_pid_maps_vsyscall);
+ /* TODO "ProtectionKey:" */
+ assert(rv > len);
+ assert(memcmp(buf, g_proc_pid_maps_vsyscall, len) == 0);
+ }
+ return EXIT_SUCCESS;
+ }
+}
+
+static const char g_smaps_rollup[] =
+"00000000-00000000 ---p 00000000 00:00 0 [rollup]\n"
+"Rss: 0 kB\n"
+"Pss: 0 kB\n"
+"Pss_Dirty: 0 kB\n"
+"Pss_Anon: 0 kB\n"
+"Pss_File: 0 kB\n"
+"Pss_Shmem: 0 kB\n"
+"Shared_Clean: 0 kB\n"
+"Shared_Dirty: 0 kB\n"
+"Private_Clean: 0 kB\n"
+"Private_Dirty: 0 kB\n"
+"Referenced: 0 kB\n"
+"Anonymous: 0 kB\n"
+"LazyFree: 0 kB\n"
+"AnonHugePages: 0 kB\n"
+"ShmemPmdMapped: 0 kB\n"
+"FilePmdMapped: 0 kB\n"
+"Shared_Hugetlb: 0 kB\n"
+"Private_Hugetlb: 0 kB\n"
+"Swap: 0 kB\n"
+"SwapPss: 0 kB\n"
+"Locked: 0 kB\n"
+;
+
+static int test_proc_pid_smaps_rollup(pid_t pid)
+{
+ char buf[4096];
+ snprintf(buf, sizeof(buf), "/proc/%u/smaps_rollup", pid);
+ int fd = open(buf, O_RDONLY);
+ if (fd == -1) {
+ if (errno == ENOENT) {
+ /*
+ * /proc/${pid}/smaps_rollup is under CONFIG_PROC_PAGE_MONITOR,
+ * it doesn't necessarily exist.
+ */
+ return EXIT_SUCCESS;
+ }
+ perror("open /proc/${pid}/smaps_rollup");
+ return EXIT_FAILURE;
+ } else {
+ ssize_t rv = read(fd, buf, sizeof(buf));
+ close(fd);
+ assert(rv == sizeof(g_smaps_rollup) - 1);
+ assert(memcmp(buf, g_smaps_rollup, sizeof(g_smaps_rollup) - 1) == 0);
+ return EXIT_SUCCESS;
+ }
+}
+
+int main(void)
+{
+ int rv = EXIT_SUCCESS;
+
+ vsyscall();
+
+ switch (g_vsyscall) {
+ case 0:
+ g_proc_pid_maps_vsyscall = proc_pid_maps_vsyscall_0;
+ g_proc_pid_smaps_vsyscall = proc_pid_smaps_vsyscall_0;
+ break;
+ case 1:
+ g_proc_pid_maps_vsyscall = proc_pid_maps_vsyscall_1;
+ g_proc_pid_smaps_vsyscall = proc_pid_smaps_vsyscall_1;
+ break;
+ case 2:
+ g_proc_pid_maps_vsyscall = proc_pid_maps_vsyscall_2;
+ g_proc_pid_smaps_vsyscall = proc_pid_smaps_vsyscall_2;
+ break;
+ default:
+ abort();
+ }
+
+ pid_t pid = fork();
+ if (pid == -1) {
+ perror("fork");
+ return EXIT_FAILURE;
+ } else if (pid == 0) {
+ rv = ptrace(PTRACE_TRACEME, 0, NULL, NULL);
+ if (rv != 0) {
+ if (errno == EPERM) {
+ fprintf(stderr,
+"Did you know? ptrace(PTRACE_TRACEME) doesn't work under strace.\n"
+ );
+ kill(getppid(), SIGTERM);
+ return EXIT_FAILURE;
+ }
+ perror("ptrace PTRACE_TRACEME");
+ return EXIT_FAILURE;
+ }
+
+ /*
+ * Hide "segfault at ..." messages. Signal handler won't run.
+ */
+ struct sigaction act = {};
+ act.sa_flags = SA_SIGINFO;
+ act.sa_sigaction = sigaction_SIGSEGV;
+ sigaction(SIGSEGV, &act, NULL);
+
+#ifdef __amd64__
+ munmap(NULL, ((size_t)1 << 47) - 4096);
+#else
+#error "implement 'unmap everything'"
+#endif
+ return EXIT_FAILURE;
+ } else {
+ /*
+ * TODO find reliable way to signal parent that munmap(2) completed.
+ * Child can't do it directly because it effectively doesn't exist
+ * anymore. Looking at child's VM files isn't 100% reliable either:
+ * due to a bug they may not become empty or empty-like.
+ */
+ sleep(1);
+
+ if (rv == EXIT_SUCCESS) {
+ rv = test_proc_pid_maps(pid);
+ }
+ if (rv == EXIT_SUCCESS) {
+ rv = test_proc_pid_numa_maps(pid);
+ }
+ if (rv == EXIT_SUCCESS) {
+ rv = test_proc_pid_smaps(pid);
+ }
+ if (rv == EXIT_SUCCESS) {
+ rv = test_proc_pid_smaps_rollup(pid);
+ }
+ /*
+ * TODO test /proc/${pid}/statm, task_statm()
+ * ->start_code, ->end_code aren't updated by munmap().
+ * Output can be "0 0 0 2 0 0 0\n" where "2" can be anything.
+ */
+
+ /* Cut the rope. */
+ int wstatus;
+ waitpid(pid, &wstatus, 0);
+ assert(WIFSTOPPED(wstatus));
+ assert(WSTOPSIG(wstatus) == SIGSEGV);
+ }
+
+ return rv;
+}
/*
* 0: vsyscall VMA doesn't exist vsyscall=none
- * 1: vsyscall VMA is r-xp vsyscall=emulate
- * 2: vsyscall VMA is --xp vsyscall=xonly
+ * 1: vsyscall VMA is --xp vsyscall=xonly
+ * 2: vsyscall VMA is r-xp vsyscall=emulate
*/
-static int g_vsyscall;
+static volatile int g_vsyscall;
static const char *str_vsyscall;
static const char str_vsyscall_0[] = "";
static const char str_vsyscall_1[] =
-"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n";
-static const char str_vsyscall_2[] =
"ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]\n";
+static const char str_vsyscall_2[] =
+"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n";
#ifdef __x86_64__
static void sigaction_SIGSEGV(int _, siginfo_t *__, void *___)
{
- _exit(1);
+ _exit(g_vsyscall);
}
/*
act.sa_sigaction = sigaction_SIGSEGV;
(void)sigaction(SIGSEGV, &act, NULL);
+ g_vsyscall = 0;
/* gettimeofday(NULL, NULL); */
asm volatile (
"call %P0"
: "i" (0xffffffffff600000), "D" (NULL), "S" (NULL)
: "rax", "rcx", "r11"
);
- exit(0);
- }
- waitpid(pid, &wstatus, 0);
- if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0) {
- /* vsyscall page exists and is executable. */
- } else {
- /* vsyscall page doesn't exist. */
- g_vsyscall = 0;
- return;
- }
-
- pid = fork();
- if (pid < 0) {
- fprintf(stderr, "fork, errno %d\n", errno);
- exit(1);
- }
- if (pid == 0) {
- struct rlimit rlim = {0, 0};
- (void)setrlimit(RLIMIT_CORE, &rlim);
-
- /* Hide "segfault at ffffffffff600000" messages. */
- struct sigaction act;
- memset(&act, 0, sizeof(struct sigaction));
- act.sa_flags = SA_SIGINFO;
- act.sa_sigaction = sigaction_SIGSEGV;
- (void)sigaction(SIGSEGV, &act, NULL);
+ g_vsyscall = 1;
*(volatile int *)0xffffffffff600000UL;
- exit(0);
+
+ g_vsyscall = 2;
+ exit(g_vsyscall);
}
waitpid(pid, &wstatus, 0);
- if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0) {
- /* vsyscall page is readable and executable. */
- g_vsyscall = 1;
- return;
+ if (WIFEXITED(wstatus)) {
+ g_vsyscall = WEXITSTATUS(wstatus);
+ } else {
+ fprintf(stderr, "error: wstatus %08x\n", wstatus);
+ exit(1);
}
-
- /* vsyscall page is executable but unreadable. */
- g_vsyscall = 2;
}
int main(void)
write_to_hugetlbfs
hmm-tests
memfd_secret
-local_config.*
soft-dirty
split_huge_page_test
ksm_tests
# SPDX-License-Identifier: GPL-2.0
# Makefile for vm selftests
-LOCAL_HDRS += $(selfdir)/vm/local_config.h $(top_srcdir)/mm/gup_test.h
-
-include local_config.mk
+LOCAL_HDRS += $(top_srcdir)/mm/gup_test.h
uname_M := $(shell uname -m 2>/dev/null || echo not)
MACHINE ?= $(shell echo $(uname_M) | sed -e 's/aarch64.*/arm64/' -e 's/ppc64.*/ppc64/')
include ../lib.mk
+$(OUTPUT)/khugepaged: vm_util.c
$(OUTPUT)/madv_populate: vm_util.c
$(OUTPUT)/soft-dirty: vm_util.c
$(OUTPUT)/split_huge_page_test: vm_util.c
+$(OUTPUT)/userfaultfd: vm_util.c
ifeq ($(MACHINE),x86_64)
BINARIES_32 := $(patsubst %,$(OUTPUT)/%,$(BINARIES_32))
$(OUTPUT)/mlock-random-test $(OUTPUT)/memfd_secret: LDLIBS += -lcap
-# HMM_EXTRA_LIBS may get set in local_config.mk, or it may be left empty.
-$(OUTPUT)/hmm-tests: LDLIBS += $(HMM_EXTRA_LIBS)
-
$(OUTPUT)/ksm_tests: LDLIBS += -lnuma
$(OUTPUT)/migration: LDLIBS += -lnuma
-
-local_config.mk local_config.h: check_config.sh
- /bin/sh ./check_config.sh $(CC)
-
-EXTRA_CLEAN += local_config.mk local_config.h
-
-ifeq ($(HMM_EXTRA_LIBS),)
-all: warn_missing_hugelibs
-
-warn_missing_hugelibs:
- @echo ; \
- echo "Warning: missing libhugetlbfs support. Some HMM tests will be skipped." ; \
- echo
-endif
+++ /dev/null
-#!/bin/sh
-# SPDX-License-Identifier: GPL-2.0
-#
-# Probe for libraries and create header files to record the results. Both C
-# header files and Makefile include fragments are created.
-
-OUTPUT_H_FILE=local_config.h
-OUTPUT_MKFILE=local_config.mk
-
-# libhugetlbfs
-tmpname=$(mktemp)
-tmpfile_c=${tmpname}.c
-tmpfile_o=${tmpname}.o
-
-echo "#include <sys/types.h>" > $tmpfile_c
-echo "#include <hugetlbfs.h>" >> $tmpfile_c
-echo "int func(void) { return 0; }" >> $tmpfile_c
-
-CC=${1:?"Usage: $0 <compiler> # example compiler: gcc"}
-$CC -c $tmpfile_c -o $tmpfile_o >/dev/null 2>&1
-
-if [ -f $tmpfile_o ]; then
- echo "#define LOCAL_CONFIG_HAVE_LIBHUGETLBFS 1" > $OUTPUT_H_FILE
- echo "HMM_EXTRA_LIBS = -lhugetlbfs" > $OUTPUT_MKFILE
-else
- echo "// No libhugetlbfs support found" > $OUTPUT_H_FILE
- echo "# No libhugetlbfs support found, so:" > $OUTPUT_MKFILE
- echo "HMM_EXTRA_LIBS = " >> $OUTPUT_MKFILE
-fi
-
-rm ${tmpname}.*
#include <sys/mman.h>
#include <sys/ioctl.h>
-#include "./local_config.h"
-#ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS
-#include <hugetlbfs.h>
-#endif
/*
* This is a private UAPI to the kernel test module so it isn't exported
hmm_buffer_free(buffer);
}
-#ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS
+/*
+ * Read numeric data from raw and tagged kernel status files. Used to read
+ * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).
+ */
+static long file_read_ulong(char *file, const char *tag)
+{
+ int fd;
+ char buf[2048];
+ int len;
+ char *p, *q;
+ long val;
+
+ fd = open(file, O_RDONLY);
+ if (fd < 0) {
+ /* Error opening the file */
+ return -1;
+ }
+
+ len = read(fd, buf, sizeof(buf));
+ close(fd);
+ if (len < 0) {
+ /* Error in reading the file */
+ return -1;
+ }
+ if (len == sizeof(buf)) {
+ /* Error file is too large */
+ return -1;
+ }
+ buf[len] = '\0';
+
+ /* Search for a tag if provided */
+ if (tag) {
+ p = strstr(buf, tag);
+ if (!p)
+ return -1; /* looks like the line we want isn't there */
+ p += strlen(tag);
+ } else
+ p = buf;
+
+ val = strtol(p, &q, 0);
+ if (*q != ' ') {
+ /* Error parsing the file */
+ return -1;
+ }
+
+ return val;
+}
+
/*
* Write huge TLBFS page.
*/
struct hmm_buffer *buffer;
unsigned long npages;
unsigned long size;
+ unsigned long default_hsize;
unsigned long i;
int *ptr;
int ret;
- long pagesizes[4];
- int n, idx;
- /* Skip test if we can't allocate a hugetlbfs page. */
-
- n = gethugepagesizes(pagesizes, 4);
- if (n <= 0)
+ default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");
+ if (default_hsize < 0 || default_hsize*1024 < default_hsize)
SKIP(return, "Huge page size could not be determined");
- for (idx = 0; --n > 0; ) {
- if (pagesizes[n] < pagesizes[idx])
- idx = n;
- }
- size = ALIGN(TWOMEG, pagesizes[idx]);
+ default_hsize = default_hsize*1024; /* KB to B */
+
+ size = ALIGN(TWOMEG, default_hsize);
npages = size >> self->page_shift;
buffer = malloc(sizeof(*buffer));
ASSERT_NE(buffer, NULL);
- buffer->ptr = get_hugepage_region(size, GHR_STRICT);
- if (buffer->ptr == NULL) {
+ buffer->ptr = mmap(NULL, size,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
+ -1, 0);
+ if (buffer->ptr == MAP_FAILED) {
free(buffer);
SKIP(return, "Huge page could not be allocated");
}
for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
ASSERT_EQ(ptr[i], i);
- free_hugepage_region(buffer->ptr);
+ munmap(buffer->ptr, buffer->size);
buffer->ptr = NULL;
hmm_buffer_free(buffer);
}
-#endif /* LOCAL_CONFIG_HAVE_LIBHUGETLBFS */
/*
* Read mmap'ed file memory.
hmm_buffer_free(buffer);
}
+TEST_F(hmm, migrate_release)
+{
+ struct hmm_buffer *buffer;
+ unsigned long npages;
+ unsigned long size;
+ unsigned long i;
+ int *ptr;
+ int ret;
+
+ npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
+ ASSERT_NE(npages, 0);
+ size = npages << self->page_shift;
+
+ buffer = malloc(sizeof(*buffer));
+ ASSERT_NE(buffer, NULL);
+
+ buffer->fd = -1;
+ buffer->size = size;
+ buffer->mirror = malloc(size);
+ ASSERT_NE(buffer->mirror, NULL);
+
+ buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
+ ASSERT_NE(buffer->ptr, MAP_FAILED);
+
+ /* Initialize buffer in system memory. */
+ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
+ ptr[i] = i;
+
+ /* Migrate memory to device. */
+ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(buffer->cpages, npages);
+
+ /* Check what the device read. */
+ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
+ ASSERT_EQ(ptr[i], i);
+
+ /* Release device memory. */
+ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages);
+ ASSERT_EQ(ret, 0);
+
+ /* Fault pages back to system memory and check them. */
+ for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
+ ASSERT_EQ(ptr[i], i);
+
+ hmm_buffer_free(buffer);
+}
+
/*
* Migrate anonymous shared memory to device private memory.
*/
hmm_buffer_free(buffer);
}
-#ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS
/*
* Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
* should be mapped by a large page table entry.
struct hmm_buffer *buffer;
unsigned long npages;
unsigned long size;
+ unsigned long default_hsize;
int *ptr;
unsigned char *m;
int ret;
- long pagesizes[4];
- int n, idx;
unsigned long i;
/* Skip test if we can't allocate a hugetlbfs page. */
- n = gethugepagesizes(pagesizes, 4);
- if (n <= 0)
- return;
- for (idx = 0; --n > 0; ) {
- if (pagesizes[n] < pagesizes[idx])
- idx = n;
- }
- size = ALIGN(TWOMEG, pagesizes[idx]);
+ default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");
+ if (default_hsize < 0 || default_hsize*1024 < default_hsize)
+ SKIP(return, "Huge page size could not be determined");
+ default_hsize = default_hsize*1024; /* KB to B */
+
+ size = ALIGN(TWOMEG, default_hsize);
npages = size >> self->page_shift;
buffer = malloc(sizeof(*buffer));
ASSERT_NE(buffer, NULL);
- buffer->ptr = get_hugepage_region(size, GHR_STRICT);
- if (buffer->ptr == NULL) {
+ buffer->ptr = mmap(NULL, size,
+ PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
+ -1, 0);
+ if (buffer->ptr == MAP_FAILED) {
free(buffer);
return;
}
ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
HMM_DMIRROR_PROT_PMD);
- free_hugepage_region(buffer->ptr);
+ munmap(buffer->ptr, buffer->size);
buffer->ptr = NULL;
hmm_buffer_free(buffer);
}
-#endif /* LOCAL_CONFIG_HAVE_LIBHUGETLBFS */
/*
* Test two devices reading the same memory (double mapped).
#define _GNU_SOURCE
+#include <ctype.h>
+#include <errno.h>
#include <fcntl.h>
#include <limits.h>
+#include <dirent.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/wait.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/sysmacros.h>
+#include <sys/vfs.h>
+
+#include "linux/magic.h"
+
+#include "vm_util.h"
#ifndef MADV_PAGEOUT
#define MADV_PAGEOUT 21
#endif
+#ifndef MADV_POPULATE_READ
+#define MADV_POPULATE_READ 22
+#endif
+#ifndef MADV_COLLAPSE
+#define MADV_COLLAPSE 25
+#endif
#define BASE_ADDR ((void *)(1UL << 30))
static unsigned long hpage_pmd_size;
#define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"
#define PID_SMAPS "/proc/self/smaps"
+#define TEST_FILE "collapse_test_file"
+
+#define MAX_LINE_LENGTH 500
+
+enum vma_type {
+ VMA_ANON,
+ VMA_FILE,
+ VMA_SHMEM,
+};
+
+struct mem_ops {
+ void *(*setup_area)(int nr_hpages);
+ void (*cleanup_area)(void *p, unsigned long size);
+ void (*fault)(void *p, unsigned long start, unsigned long end);
+ bool (*check_huge)(void *addr, int nr_hpages);
+ const char *name;
+};
+
+static struct mem_ops *file_ops;
+static struct mem_ops *anon_ops;
+static struct mem_ops *shmem_ops;
+
+struct collapse_context {
+ void (*collapse)(const char *msg, char *p, int nr_hpages,
+ struct mem_ops *ops, bool expect);
+ bool enforce_pte_scan_limits;
+ const char *name;
+};
+
+static struct collapse_context *khugepaged_context;
+static struct collapse_context *madvise_context;
+
+struct file_info {
+ const char *dir;
+ char path[PATH_MAX];
+ enum vma_type type;
+ int fd;
+ char dev_queue_read_ahead_path[PATH_MAX];
+};
+
+static struct file_info finfo;
enum thp_enabled {
THP_ALWAYS,
enum shmem_enabled shmem_enabled;
bool use_zero_page;
struct khugepaged_settings khugepaged;
-};
-
-static struct settings default_settings = {
- .thp_enabled = THP_MADVISE,
- .thp_defrag = THP_DEFRAG_ALWAYS,
- .shmem_enabled = SHMEM_NEVER,
- .use_zero_page = 0,
- .khugepaged = {
- .defrag = 1,
- .alloc_sleep_millisecs = 10,
- .scan_sleep_millisecs = 10,
- },
+ unsigned long read_ahead_kb;
};
static struct settings saved_settings;
exit_status++;
}
+static void skip(const char *msg)
+{
+ printf(" \e[33m%s\e[0m\n", msg);
+}
+
static int read_file(const char *path, char *buf, size_t buflen)
{
int fd;
ssize_t numwritten;
fd = open(path, O_WRONLY);
- if (fd == -1)
+ if (fd == -1) {
+ printf("open(%s)\n", path);
+ exit(EXIT_FAILURE);
return 0;
+ }
numwritten = write(fd, buf, buflen - 1);
close(fd);
- if (numwritten < 1)
+ if (numwritten < 1) {
+ printf("write(%s)\n", buf);
+ exit(EXIT_FAILURE);
return 0;
+ }
return (unsigned int) numwritten;
}
}
}
-static const unsigned long read_num(const char *name)
+static const unsigned long _read_num(const char *path)
{
- char path[PATH_MAX];
char buf[21];
- int ret;
-
- ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
- if (ret >= PATH_MAX) {
- printf("%s: Pathname is too long\n", __func__);
- exit(EXIT_FAILURE);
- }
- ret = read_file(path, buf, sizeof(buf));
- if (ret < 0) {
+ if (read_file(path, buf, sizeof(buf)) < 0) {
perror("read_file(read_num)");
exit(EXIT_FAILURE);
}
return strtoul(buf, NULL, 10);
}
-static void write_num(const char *name, unsigned long num)
+static const unsigned long read_num(const char *name)
{
char path[PATH_MAX];
- char buf[21];
int ret;
ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
printf("%s: Pathname is too long\n", __func__);
exit(EXIT_FAILURE);
}
+ return _read_num(path);
+}
+
+static void _write_num(const char *path, unsigned long num)
+{
+ char buf[21];
sprintf(buf, "%ld", num);
if (!write_file(path, buf, strlen(buf) + 1)) {
}
}
+static void write_num(const char *name, unsigned long num)
+{
+ char path[PATH_MAX];
+ int ret;
+
+ ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
+ if (ret >= PATH_MAX) {
+ printf("%s: Pathname is too long\n", __func__);
+ exit(EXIT_FAILURE);
+ }
+ _write_num(path, num);
+}
+
static void write_settings(struct settings *settings)
{
struct khugepaged_settings *khugepaged = &settings->khugepaged;
write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);
write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);
write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);
+
+ if (file_ops && finfo.type == VMA_FILE)
+ _write_num(finfo.dev_queue_read_ahead_path,
+ settings->read_ahead_kb);
+}
+
+#define MAX_SETTINGS_DEPTH 4
+static struct settings settings_stack[MAX_SETTINGS_DEPTH];
+static int settings_index;
+
+static struct settings *current_settings(void)
+{
+ if (!settings_index) {
+ printf("Fail: No settings set");
+ exit(EXIT_FAILURE);
+ }
+ return settings_stack + settings_index - 1;
+}
+
+static void push_settings(struct settings *settings)
+{
+ if (settings_index >= MAX_SETTINGS_DEPTH) {
+ printf("Fail: Settings stack exceeded");
+ exit(EXIT_FAILURE);
+ }
+ settings_stack[settings_index++] = *settings;
+ write_settings(current_settings());
+}
+
+static void pop_settings(void)
+{
+ if (settings_index <= 0) {
+ printf("Fail: Settings stack empty");
+ exit(EXIT_FAILURE);
+ }
+ --settings_index;
+ write_settings(current_settings());
}
static void restore_settings(int sig)
.max_ptes_shared = read_num("khugepaged/max_ptes_shared"),
.pages_to_scan = read_num("khugepaged/pages_to_scan"),
};
+ if (file_ops && finfo.type == VMA_FILE)
+ saved_settings.read_ahead_kb =
+ _read_num(finfo.dev_queue_read_ahead_path);
+
success("OK");
signal(SIGTERM, restore_settings);
signal(SIGQUIT, restore_settings);
}
-static void adjust_settings(void)
+static void get_finfo(const char *dir)
{
+ struct stat path_stat;
+ struct statfs fs;
+ char buf[1 << 10];
+ char path[PATH_MAX];
+ char *str, *end;
- printf("Adjust settings...");
- write_settings(&default_settings);
- success("OK");
-}
-
-#define MAX_LINE_LENGTH 500
-
-static bool check_for_pattern(FILE *fp, char *pattern, char *buf)
-{
- while (fgets(buf, MAX_LINE_LENGTH, fp) != NULL) {
- if (!strncmp(buf, pattern, strlen(pattern)))
- return true;
+ finfo.dir = dir;
+ stat(finfo.dir, &path_stat);
+ if (!S_ISDIR(path_stat.st_mode)) {
+ printf("%s: Not a directory (%s)\n", __func__, finfo.dir);
+ exit(EXIT_FAILURE);
}
- return false;
-}
-
-static bool check_huge(void *addr)
-{
- bool thp = false;
- int ret;
- FILE *fp;
- char buffer[MAX_LINE_LENGTH];
- char addr_pattern[MAX_LINE_LENGTH];
-
- ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
- (unsigned long) addr);
- if (ret >= MAX_LINE_LENGTH) {
- printf("%s: Pattern is too long\n", __func__);
+ if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE,
+ finfo.dir) >= sizeof(finfo.path)) {
+ printf("%s: Pathname is too long\n", __func__);
exit(EXIT_FAILURE);
}
-
-
- fp = fopen(PID_SMAPS, "r");
- if (!fp) {
- printf("%s: Failed to open file %s\n", __func__, PID_SMAPS);
+ if (statfs(finfo.dir, &fs)) {
+ perror("statfs()");
exit(EXIT_FAILURE);
}
- if (!check_for_pattern(fp, addr_pattern, buffer))
- goto err_out;
-
- ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB",
- hpage_pmd_size >> 10);
- if (ret >= MAX_LINE_LENGTH) {
- printf("%s: Pattern is too long\n", __func__);
+ finfo.type = fs.f_type == TMPFS_MAGIC ? VMA_SHMEM : VMA_FILE;
+ if (finfo.type == VMA_SHMEM)
+ return;
+
+ /* Find owning device's queue/read_ahead_kb control */
+ if (snprintf(path, sizeof(path), "/sys/dev/block/%d:%d/uevent",
+ major(path_stat.st_dev), minor(path_stat.st_dev))
+ >= sizeof(path)) {
+ printf("%s: Pathname is too long\n", __func__);
+ exit(EXIT_FAILURE);
+ }
+ if (read_file(path, buf, sizeof(buf)) < 0) {
+ perror("read_file(read_num)");
+ exit(EXIT_FAILURE);
+ }
+ if (strstr(buf, "DEVTYPE=disk")) {
+ /* Found it */
+ if (snprintf(finfo.dev_queue_read_ahead_path,
+ sizeof(finfo.dev_queue_read_ahead_path),
+ "/sys/dev/block/%d:%d/queue/read_ahead_kb",
+ major(path_stat.st_dev), minor(path_stat.st_dev))
+ >= sizeof(finfo.dev_queue_read_ahead_path)) {
+ printf("%s: Pathname is too long\n", __func__);
+ exit(EXIT_FAILURE);
+ }
+ return;
+ }
+ if (!strstr(buf, "DEVTYPE=partition")) {
+ printf("%s: Unknown device type: %s\n", __func__, path);
exit(EXIT_FAILURE);
}
/*
- * Fetch the AnonHugePages: in the same block and check whether it got
- * the expected number of hugeepages next.
+ * Partition of block device - need to find actual device.
+ * Using naming convention that devnameN is partition of
+ * device devname.
*/
- if (!check_for_pattern(fp, "AnonHugePages:", buffer))
- goto err_out;
-
- if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
- goto err_out;
-
- thp = true;
-err_out:
- fclose(fp);
- return thp;
+ str = strstr(buf, "DEVNAME=");
+ if (!str) {
+ printf("%s: Could not read: %s", __func__, path);
+ exit(EXIT_FAILURE);
+ }
+ str += 8;
+ end = str;
+ while (*end) {
+ if (isdigit(*end)) {
+ *end = '\0';
+ if (snprintf(finfo.dev_queue_read_ahead_path,
+ sizeof(finfo.dev_queue_read_ahead_path),
+ "/sys/block/%s/queue/read_ahead_kb",
+ str) >= sizeof(finfo.dev_queue_read_ahead_path)) {
+ printf("%s: Pathname is too long\n", __func__);
+ exit(EXIT_FAILURE);
+ }
+ return;
+ }
+ ++end;
+ }
+ printf("%s: Could not read: %s\n", __func__, path);
+ exit(EXIT_FAILURE);
}
-
static bool check_swap(void *addr, unsigned long size)
{
bool swap = false;
printf("%s: Failed to open file %s\n", __func__, PID_SMAPS);
exit(EXIT_FAILURE);
}
- if (!check_for_pattern(fp, addr_pattern, buffer))
+ if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
goto err_out;
ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB",
* Fetch the Swap: in the same block and check whether it got
* the expected number of hugeepages next.
*/
- if (!check_for_pattern(fp, "Swap:", buffer))
+ if (!check_for_pattern(fp, "Swap:", buffer, sizeof(buffer)))
goto err_out;
if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
return swap;
}
-static void *alloc_mapping(void)
+static void *alloc_mapping(int nr)
{
void *p;
- p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (p != BASE_ADDR) {
printf("Failed to allocate VMA at %p\n", BASE_ADDR);
exit(EXIT_FAILURE);
p[i * page_size / sizeof(*p)] = i + 0xdead0000;
}
+/*
+ * MADV_COLLAPSE is a best-effort request and may fail if an internal
+ * resource is temporarily unavailable, in which case it will set errno to
+ * EAGAIN. In such a case, immediately reattempt the operation one more
+ * time.
+ */
+static int madvise_collapse_retry(void *p, unsigned long size)
+{
+ bool retry = true;
+ int ret;
+
+retry:
+ ret = madvise(p, size, MADV_COLLAPSE);
+ if (ret && errno == EAGAIN && retry) {
+ retry = false;
+ goto retry;
+ }
+ return ret;
+}
+
+/*
+ * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with
+ * validate_memory()'able contents.
+ */
+static void *alloc_hpage(struct mem_ops *ops)
+{
+ void *p = ops->setup_area(1);
+
+ ops->fault(p, 0, hpage_pmd_size);
+
+ /*
+ * VMA should be neither VM_HUGEPAGE nor VM_NOHUGEPAGE.
+ * The latter is ineligible for collapse by MADV_COLLAPSE
+ * while the former might cause MADV_COLLAPSE to race with
+ * khugepaged on low-load system (like a test machine), which
+ * would cause MADV_COLLAPSE to fail with EAGAIN.
+ */
+ printf("Allocate huge page...");
+ if (madvise_collapse_retry(p, hpage_pmd_size)) {
+ perror("madvise(MADV_COLLAPSE)");
+ exit(EXIT_FAILURE);
+ }
+ if (!ops->check_huge(p, 1)) {
+ perror("madvise(MADV_COLLAPSE)");
+ exit(EXIT_FAILURE);
+ }
+ if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) {
+ perror("madvise(MADV_HUGEPAGE)");
+ exit(EXIT_FAILURE);
+ }
+ success("OK");
+ return p;
+}
+
static void validate_memory(int *p, unsigned long start, unsigned long end)
{
int i;
}
}
+static void *anon_setup_area(int nr_hpages)
+{
+ return alloc_mapping(nr_hpages);
+}
+
+static void anon_cleanup_area(void *p, unsigned long size)
+{
+ munmap(p, size);
+}
+
+static void anon_fault(void *p, unsigned long start, unsigned long end)
+{
+ fill_memory(p, start, end);
+}
+
+static bool anon_check_huge(void *addr, int nr_hpages)
+{
+ return check_huge_anon(addr, nr_hpages, hpage_pmd_size);
+}
+
+static void *file_setup_area(int nr_hpages)
+{
+ int fd;
+ void *p;
+ unsigned long size;
+
+ unlink(finfo.path); /* Cleanup from previous failed tests */
+ printf("Creating %s for collapse%s...", finfo.path,
+ finfo.type == VMA_SHMEM ? " (tmpfs)" : "");
+ fd = open(finfo.path, O_DSYNC | O_CREAT | O_RDWR | O_TRUNC | O_EXCL,
+ 777);
+ if (fd < 0) {
+ perror("open()");
+ exit(EXIT_FAILURE);
+ }
+
+ size = nr_hpages * hpage_pmd_size;
+ p = alloc_mapping(nr_hpages);
+ fill_memory(p, 0, size);
+ write(fd, p, size);
+ close(fd);
+ munmap(p, size);
+ success("OK");
+
+ printf("Opening %s read only for collapse...", finfo.path);
+ finfo.fd = open(finfo.path, O_RDONLY, 777);
+ if (finfo.fd < 0) {
+ perror("open()");
+ exit(EXIT_FAILURE);
+ }
+ p = mmap(BASE_ADDR, size, PROT_READ | PROT_EXEC,
+ MAP_PRIVATE, finfo.fd, 0);
+ if (p == MAP_FAILED || p != BASE_ADDR) {
+ perror("mmap()");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Drop page cache */
+ write_file("/proc/sys/vm/drop_caches", "3", 2);
+ success("OK");
+ return p;
+}
+
+static void file_cleanup_area(void *p, unsigned long size)
+{
+ munmap(p, size);
+ close(finfo.fd);
+ unlink(finfo.path);
+}
+
+static void file_fault(void *p, unsigned long start, unsigned long end)
+{
+ if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) {
+ perror("madvise(MADV_POPULATE_READ");
+ exit(EXIT_FAILURE);
+ }
+}
+
+static bool file_check_huge(void *addr, int nr_hpages)
+{
+ switch (finfo.type) {
+ case VMA_FILE:
+ return check_huge_file(addr, nr_hpages, hpage_pmd_size);
+ case VMA_SHMEM:
+ return check_huge_shmem(addr, nr_hpages, hpage_pmd_size);
+ default:
+ exit(EXIT_FAILURE);
+ return false;
+ }
+}
+
+static void *shmem_setup_area(int nr_hpages)
+{
+ void *p;
+ unsigned long size = nr_hpages * hpage_pmd_size;
+
+ finfo.fd = memfd_create("khugepaged-selftest-collapse-shmem", 0);
+ if (finfo.fd < 0) {
+ perror("memfd_create()");
+ exit(EXIT_FAILURE);
+ }
+ if (ftruncate(finfo.fd, size)) {
+ perror("ftruncate()");
+ exit(EXIT_FAILURE);
+ }
+ p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd,
+ 0);
+ if (p != BASE_ADDR) {
+ perror("mmap()");
+ exit(EXIT_FAILURE);
+ }
+ return p;
+}
+
+static void shmem_cleanup_area(void *p, unsigned long size)
+{
+ munmap(p, size);
+ close(finfo.fd);
+}
+
+static bool shmem_check_huge(void *addr, int nr_hpages)
+{
+ return check_huge_shmem(addr, nr_hpages, hpage_pmd_size);
+}
+
+static struct mem_ops __anon_ops = {
+ .setup_area = &anon_setup_area,
+ .cleanup_area = &anon_cleanup_area,
+ .fault = &anon_fault,
+ .check_huge = &anon_check_huge,
+ .name = "anon",
+};
+
+static struct mem_ops __file_ops = {
+ .setup_area = &file_setup_area,
+ .cleanup_area = &file_cleanup_area,
+ .fault = &file_fault,
+ .check_huge = &file_check_huge,
+ .name = "file",
+};
+
+static struct mem_ops __shmem_ops = {
+ .setup_area = &shmem_setup_area,
+ .cleanup_area = &shmem_cleanup_area,
+ .fault = &anon_fault,
+ .check_huge = &shmem_check_huge,
+ .name = "shmem",
+};
+
+static void __madvise_collapse(const char *msg, char *p, int nr_hpages,
+ struct mem_ops *ops, bool expect)
+{
+ int ret;
+ struct settings settings = *current_settings();
+
+ printf("%s...", msg);
+
+ /*
+ * Prevent khugepaged interference and tests that MADV_COLLAPSE
+ * ignores /sys/kernel/mm/transparent_hugepage/enabled
+ */
+ settings.thp_enabled = THP_NEVER;
+ settings.shmem_enabled = SHMEM_NEVER;
+ push_settings(&settings);
+
+ /* Clear VM_NOHUGEPAGE */
+ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE);
+ ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size);
+ if (((bool)ret) == expect)
+ fail("Fail: Bad return value");
+ else if (!ops->check_huge(p, expect ? nr_hpages : 0))
+ fail("Fail: check_huge()");
+ else
+ success("OK");
+
+ pop_settings();
+}
+
+static void madvise_collapse(const char *msg, char *p, int nr_hpages,
+ struct mem_ops *ops, bool expect)
+{
+ /* Sanity check */
+ if (!ops->check_huge(p, 0)) {
+ printf("Unexpected huge page\n");
+ exit(EXIT_FAILURE);
+ }
+ __madvise_collapse(msg, p, nr_hpages, ops, expect);
+}
+
#define TICK 500000
-static bool wait_for_scan(const char *msg, char *p)
+static bool wait_for_scan(const char *msg, char *p, int nr_hpages,
+ struct mem_ops *ops)
{
int full_scans;
int timeout = 6; /* 3 seconds */
/* Sanity check */
- if (check_huge(p)) {
+ if (!ops->check_huge(p, 0)) {
printf("Unexpected huge page\n");
exit(EXIT_FAILURE);
}
- madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
+ madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE);
/* Wait until the second full_scan completed */
full_scans = read_num("khugepaged/full_scans") + 2;
printf("%s...", msg);
while (timeout--) {
- if (check_huge(p))
+ if (ops->check_huge(p, nr_hpages))
break;
if (read_num("khugepaged/full_scans") >= full_scans)
break;
usleep(TICK);
}
- madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
+ madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE);
return timeout == -1;
}
+static void khugepaged_collapse(const char *msg, char *p, int nr_hpages,
+ struct mem_ops *ops, bool expect)
+{
+ if (wait_for_scan(msg, p, nr_hpages, ops)) {
+ if (expect)
+ fail("Timeout");
+ else
+ success("OK");
+ return;
+ }
+
+ /*
+ * For file and shmem memory, khugepaged only retracts pte entries after
+ * putting the new hugepage in the page cache. The hugepage must be
+ * subsequently refaulted to install the pmd mapping for the mm.
+ */
+ if (ops != &__anon_ops)
+ ops->fault(p, 0, nr_hpages * hpage_pmd_size);
+
+ if (ops->check_huge(p, expect ? nr_hpages : 0))
+ success("OK");
+ else
+ fail("Fail");
+}
+
+static struct collapse_context __khugepaged_context = {
+ .collapse = &khugepaged_collapse,
+ .enforce_pte_scan_limits = true,
+ .name = "khugepaged",
+};
+
+static struct collapse_context __madvise_context = {
+ .collapse = &madvise_collapse,
+ .enforce_pte_scan_limits = false,
+ .name = "madvise",
+};
+
+static bool is_tmpfs(struct mem_ops *ops)
+{
+ return ops == &__file_ops && finfo.type == VMA_SHMEM;
+}
+
static void alloc_at_fault(void)
{
- struct settings settings = default_settings;
+ struct settings settings = *current_settings();
char *p;
settings.thp_enabled = THP_ALWAYS;
- write_settings(&settings);
+ push_settings(&settings);
- p = alloc_mapping();
+ p = alloc_mapping(1);
*p = 1;
printf("Allocate huge page on fault...");
- if (check_huge(p))
+ if (check_huge_anon(p, 1, hpage_pmd_size))
success("OK");
else
fail("Fail");
- write_settings(&default_settings);
+ pop_settings();
madvise(p, page_size, MADV_DONTNEED);
printf("Split huge PMD on MADV_DONTNEED...");
- if (!check_huge(p))
+ if (check_huge_anon(p, 0, hpage_pmd_size))
success("OK");
else
fail("Fail");
munmap(p, hpage_pmd_size);
}
-static void collapse_full(void)
+static void collapse_full(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
-
- p = alloc_mapping();
- fill_memory(p, 0, hpage_pmd_size);
- if (wait_for_scan("Collapse fully populated PTE table", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
- validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ int nr_hpages = 4;
+ unsigned long size = nr_hpages * hpage_pmd_size;
+
+ p = ops->setup_area(nr_hpages);
+ ops->fault(p, 0, size);
+ c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages,
+ ops, true);
+ validate_memory(p, 0, size);
+ ops->cleanup_area(p, size);
}
-static void collapse_empty(void)
+static void collapse_empty(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
- p = alloc_mapping();
- if (wait_for_scan("Do not collapse empty PTE table", p))
- fail("Timeout");
- else if (check_huge(p))
- fail("Fail");
- else
- success("OK");
- munmap(p, hpage_pmd_size);
+ p = ops->setup_area(1);
+ c->collapse("Do not collapse empty PTE table", p, 1, ops, false);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_single_pte_entry(void)
+static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
- p = alloc_mapping();
- fill_memory(p, 0, page_size);
- if (wait_for_scan("Collapse PTE table with single PTE entry present", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
- validate_memory(p, 0, page_size);
- munmap(p, hpage_pmd_size);
+ p = ops->setup_area(1);
+ ops->fault(p, 0, page_size);
+ c->collapse("Collapse PTE table with single PTE entry present", p,
+ 1, ops, true);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_max_ptes_none(void)
+static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops)
{
int max_ptes_none = hpage_pmd_nr / 2;
- struct settings settings = default_settings;
+ struct settings settings = *current_settings();
void *p;
settings.khugepaged.max_ptes_none = max_ptes_none;
- write_settings(&settings);
+ push_settings(&settings);
- p = alloc_mapping();
+ p = ops->setup_area(1);
- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
- if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p))
- fail("Timeout");
- else if (check_huge(p))
- fail("Fail");
- else
- success("OK");
- validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
+ if (is_tmpfs(ops)) {
+ /* shmem pages always in the page cache */
+ printf("tmpfs...");
+ skip("Skip");
+ goto skip;
+ }
- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
- if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
- validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
+ ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
+ c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1,
+ ops, !c->enforce_pte_scan_limits);
+ validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
- munmap(p, hpage_pmd_size);
- write_settings(&default_settings);
+ if (c->enforce_pte_scan_limits) {
+ ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
+ c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops,
+ true);
+ validate_memory(p, 0,
+ (hpage_pmd_nr - max_ptes_none) * page_size);
+ }
+skip:
+ ops->cleanup_area(p, hpage_pmd_size);
+ pop_settings();
}
-static void collapse_swapin_single_pte(void)
+static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
- p = alloc_mapping();
- fill_memory(p, 0, hpage_pmd_size);
+
+ p = ops->setup_area(1);
+ ops->fault(p, 0, hpage_pmd_size);
printf("Swapout one page...");
if (madvise(p, page_size, MADV_PAGEOUT)) {
goto out;
}
- if (wait_for_scan("Collapse with swapping in single PTE entry", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ c->collapse("Collapse with swapping in single PTE entry", p, 1, ops,
+ true);
validate_memory(p, 0, hpage_pmd_size);
out:
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_max_ptes_swap(void)
+static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops)
{
int max_ptes_swap = read_num("khugepaged/max_ptes_swap");
void *p;
- p = alloc_mapping();
+ p = ops->setup_area(1);
+ ops->fault(p, 0, hpage_pmd_size);
- fill_memory(p, 0, hpage_pmd_size);
printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr);
if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) {
perror("madvise(MADV_PAGEOUT)");
goto out;
}
- if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p))
- fail("Timeout");
- else if (check_huge(p))
- fail("Fail");
- else
- success("OK");
+ c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops,
+ !c->enforce_pte_scan_limits);
validate_memory(p, 0, hpage_pmd_size);
- fill_memory(p, 0, hpage_pmd_size);
- printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr);
- if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) {
- perror("madvise(MADV_PAGEOUT)");
- exit(EXIT_FAILURE);
- }
- if (check_swap(p, max_ptes_swap * page_size)) {
- success("OK");
- } else {
- fail("Fail");
- goto out;
- }
+ if (c->enforce_pte_scan_limits) {
+ ops->fault(p, 0, hpage_pmd_size);
+ printf("Swapout %d of %d pages...", max_ptes_swap,
+ hpage_pmd_nr);
+ if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) {
+ perror("madvise(MADV_PAGEOUT)");
+ exit(EXIT_FAILURE);
+ }
+ if (check_swap(p, max_ptes_swap * page_size)) {
+ success("OK");
+ } else {
+ fail("Fail");
+ goto out;
+ }
- if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
- validate_memory(p, 0, hpage_pmd_size);
+ c->collapse("Collapse with max_ptes_swap pages swapped out", p,
+ 1, ops, true);
+ validate_memory(p, 0, hpage_pmd_size);
+ }
out:
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_single_pte_entry_compound(void)
+static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
- p = alloc_mapping();
+ p = alloc_hpage(ops);
- printf("Allocate huge page...");
- madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
- fill_memory(p, 0, hpage_pmd_size);
- if (check_huge(p))
- success("OK");
- else
- fail("Fail");
- madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
+ if (is_tmpfs(ops)) {
+ /* MADV_DONTNEED won't evict tmpfs pages */
+ printf("tmpfs...");
+ skip("Skip");
+ goto skip;
+ }
+ madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
printf("Split huge page leaving single PTE mapping compound page...");
madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED);
- if (!check_huge(p))
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
- if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ c->collapse("Collapse PTE table with single PTE mapping compound page",
+ p, 1, ops, true);
validate_memory(p, 0, page_size);
- munmap(p, hpage_pmd_size);
+skip:
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_full_of_compound(void)
+static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
- p = alloc_mapping();
-
- printf("Allocate huge page...");
- madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
- fill_memory(p, 0, hpage_pmd_size);
- if (check_huge(p))
- success("OK");
- else
- fail("Fail");
-
+ p = alloc_hpage(ops);
printf("Split huge page leaving single PTE page table full of compound pages...");
madvise(p, page_size, MADV_NOHUGEPAGE);
madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
- if (!check_huge(p))
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
- if (wait_for_scan("Collapse PTE table full of compound pages", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ c->collapse("Collapse PTE table full of compound pages", p, 1, ops,
+ true);
validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_compound_extreme(void)
+static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops)
{
void *p;
int i;
- p = alloc_mapping();
+ p = ops->setup_area(1);
for (i = 0; i < hpage_pmd_nr; i++) {
printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...",
i + 1, hpage_pmd_nr);
madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE);
- fill_memory(BASE_ADDR, 0, hpage_pmd_size);
- if (!check_huge(BASE_ADDR)) {
+ ops->fault(BASE_ADDR, 0, hpage_pmd_size);
+ if (!ops->check_huge(BASE_ADDR, 1)) {
printf("Failed to allocate huge page\n");
exit(EXIT_FAILURE);
}
}
}
- munmap(BASE_ADDR, hpage_pmd_size);
- fill_memory(p, 0, hpage_pmd_size);
- if (!check_huge(p))
+ ops->cleanup_area(BASE_ADDR, hpage_pmd_size);
+ ops->fault(p, 0, hpage_pmd_size);
+ if (!ops->check_huge(p, 1))
success("OK");
else
fail("Fail");
- if (wait_for_scan("Collapse PTE table full of different compound pages", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ c->collapse("Collapse PTE table full of different compound pages", p, 1,
+ ops, true);
validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_fork(void)
+static void collapse_fork(struct collapse_context *c, struct mem_ops *ops)
{
int wstatus;
void *p;
- p = alloc_mapping();
+ p = ops->setup_area(1);
printf("Allocate small page...");
- fill_memory(p, 0, page_size);
- if (!check_huge(p))
+ ops->fault(p, 0, page_size);
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
skip_settings_restore = true;
exit_status = 0;
- if (!check_huge(p))
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
- fill_memory(p, page_size, 2 * page_size);
-
- if (wait_for_scan("Collapse PTE table with single page shared with parent process", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ ops->fault(p, page_size, 2 * page_size);
+ c->collapse("Collapse PTE table with single page shared with parent process",
+ p, 1, ops, true);
validate_memory(p, 0, page_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
exit(exit_status);
}
exit_status += WEXITSTATUS(wstatus);
printf("Check if parent still has small page...");
- if (!check_huge(p))
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
validate_memory(p, 0, page_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_fork_compound(void)
+static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops)
{
int wstatus;
void *p;
- p = alloc_mapping();
-
- printf("Allocate huge page...");
- madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
- fill_memory(p, 0, hpage_pmd_size);
- if (check_huge(p))
- success("OK");
- else
- fail("Fail");
-
+ p = alloc_hpage(ops);
printf("Share huge page over fork()...");
if (!fork()) {
/* Do not touch settings on child exit */
skip_settings_restore = true;
exit_status = 0;
- if (check_huge(p))
+ if (ops->check_huge(p, 1))
success("OK");
else
fail("Fail");
printf("Split huge page PMD in child process...");
madvise(p, page_size, MADV_NOHUGEPAGE);
madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
- if (!check_huge(p))
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
- fill_memory(p, 0, page_size);
+ ops->fault(p, 0, page_size);
write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1);
- if (wait_for_scan("Collapse PTE table full of compound pages in child", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ c->collapse("Collapse PTE table full of compound pages in child",
+ p, 1, ops, true);
write_num("khugepaged/max_ptes_shared",
- default_settings.khugepaged.max_ptes_shared);
+ current_settings()->khugepaged.max_ptes_shared);
validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
exit(exit_status);
}
exit_status += WEXITSTATUS(wstatus);
printf("Check if parent still has huge page...");
- if (check_huge(p))
+ if (ops->check_huge(p, 1))
success("OK");
else
fail("Fail");
validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-static void collapse_max_ptes_shared()
+static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops)
{
int max_ptes_shared = read_num("khugepaged/max_ptes_shared");
int wstatus;
void *p;
- p = alloc_mapping();
-
- printf("Allocate huge page...");
- madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
- fill_memory(p, 0, hpage_pmd_size);
- if (check_huge(p))
- success("OK");
- else
- fail("Fail");
-
+ p = alloc_hpage(ops);
printf("Share huge page over fork()...");
if (!fork()) {
/* Do not touch settings on child exit */
skip_settings_restore = true;
exit_status = 0;
- if (check_huge(p))
+ if (ops->check_huge(p, 1))
success("OK");
else
fail("Fail");
printf("Trigger CoW on page %d of %d...",
hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr);
- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
- if (!check_huge(p))
- success("OK");
- else
- fail("Fail");
-
- if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p))
- fail("Timeout");
- else if (!check_huge(p))
- success("OK");
- else
- fail("Fail");
-
- printf("Trigger CoW on page %d of %d...",
- hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr);
- fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size);
- if (!check_huge(p))
+ ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
+ if (ops->check_huge(p, 0))
success("OK");
else
fail("Fail");
-
- if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p))
- fail("Timeout");
- else if (check_huge(p))
- success("OK");
- else
- fail("Fail");
+ c->collapse("Maybe collapse with max_ptes_shared exceeded", p,
+ 1, ops, !c->enforce_pte_scan_limits);
+
+ if (c->enforce_pte_scan_limits) {
+ printf("Trigger CoW on page %d of %d...",
+ hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr);
+ ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) *
+ page_size);
+ if (ops->check_huge(p, 0))
+ success("OK");
+ else
+ fail("Fail");
+
+ c->collapse("Collapse with max_ptes_shared PTEs shared",
+ p, 1, ops, true);
+ }
validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
exit(exit_status);
}
exit_status += WEXITSTATUS(wstatus);
printf("Check if parent still has huge page...");
- if (check_huge(p))
+ if (ops->check_huge(p, 1))
success("OK");
else
fail("Fail");
validate_memory(p, 0, hpage_pmd_size);
- munmap(p, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
}
-int main(void)
+static void madvise_collapse_existing_thps(struct collapse_context *c,
+ struct mem_ops *ops)
{
+ void *p;
+
+ p = ops->setup_area(1);
+ ops->fault(p, 0, hpage_pmd_size);
+ c->collapse("Collapse fully populated PTE table...", p, 1, ops, true);
+ validate_memory(p, 0, hpage_pmd_size);
+
+ /* c->collapse() will find a hugepage and complain - call directly. */
+ __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true);
+ validate_memory(p, 0, hpage_pmd_size);
+ ops->cleanup_area(p, hpage_pmd_size);
+}
+
+/*
+ * Test race with khugepaged where page tables have been retracted and
+ * pmd cleared.
+ */
+static void madvise_retracted_page_tables(struct collapse_context *c,
+ struct mem_ops *ops)
+{
+ void *p;
+ int nr_hpages = 1;
+ unsigned long size = nr_hpages * hpage_pmd_size;
+
+ p = ops->setup_area(nr_hpages);
+ ops->fault(p, 0, size);
+
+ /* Let khugepaged collapse and leave pmd cleared */
+ if (wait_for_scan("Collapse and leave PMD cleared", p, nr_hpages,
+ ops)) {
+ fail("Timeout");
+ return;
+ }
+ success("OK");
+ c->collapse("Install huge PMD from page cache", p, nr_hpages, ops,
+ true);
+ validate_memory(p, 0, size);
+ ops->cleanup_area(p, size);
+}
+
+static void usage(void)
+{
+ fprintf(stderr, "\nUsage: ./khugepaged <test type> [dir]\n\n");
+ fprintf(stderr, "\t<test type>\t: <context>:<mem_type>\n");
+ fprintf(stderr, "\t<context>\t: [all|khugepaged|madvise]\n");
+ fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n");
+ fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n");
+ fprintf(stderr, "\n\t\"file,all\" mem_type requires kernel built with\n");
+ fprintf(stderr, "\tCONFIG_READ_ONLY_THP_FOR_FS=y\n");
+ fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n");
+ fprintf(stderr, "\tmounted with huge=madvise option for khugepaged tests to work\n");
+ exit(1);
+}
+
+static void parse_test_type(int argc, const char **argv)
+{
+ char *buf;
+ const char *token;
+
+ if (argc == 1) {
+ /* Backwards compatibility */
+ khugepaged_context = &__khugepaged_context;
+ madvise_context = &__madvise_context;
+ anon_ops = &__anon_ops;
+ return;
+ }
+
+ buf = strdup(argv[1]);
+ token = strsep(&buf, ":");
+
+ if (!strcmp(token, "all")) {
+ khugepaged_context = &__khugepaged_context;
+ madvise_context = &__madvise_context;
+ } else if (!strcmp(token, "khugepaged")) {
+ khugepaged_context = &__khugepaged_context;
+ } else if (!strcmp(token, "madvise")) {
+ madvise_context = &__madvise_context;
+ } else {
+ usage();
+ }
+
+ if (!buf)
+ usage();
+
+ if (!strcmp(buf, "all")) {
+ file_ops = &__file_ops;
+ anon_ops = &__anon_ops;
+ shmem_ops = &__shmem_ops;
+ } else if (!strcmp(buf, "anon")) {
+ anon_ops = &__anon_ops;
+ } else if (!strcmp(buf, "file")) {
+ file_ops = &__file_ops;
+ } else if (!strcmp(buf, "shmem")) {
+ shmem_ops = &__shmem_ops;
+ } else {
+ usage();
+ }
+
+ if (!file_ops)
+ return;
+
+ if (argc != 3)
+ usage();
+}
+
+int main(int argc, const char **argv)
+{
+ struct settings default_settings = {
+ .thp_enabled = THP_MADVISE,
+ .thp_defrag = THP_DEFRAG_ALWAYS,
+ .shmem_enabled = SHMEM_ADVISE,
+ .use_zero_page = 0,
+ .khugepaged = {
+ .defrag = 1,
+ .alloc_sleep_millisecs = 10,
+ .scan_sleep_millisecs = 10,
+ },
+ /*
+ * When testing file-backed memory, the collapse path
+ * looks at how many pages are found in the page cache, not
+ * what pages are mapped. Disable read ahead optimization so
+ * pages don't find their way into the page cache unless
+ * we mem_ops->fault() them in.
+ */
+ .read_ahead_kb = 0,
+ };
+
+ parse_test_type(argc, argv);
+
+ if (file_ops)
+ get_finfo(argv[2]);
+
setbuf(stdout, NULL);
page_size = getpagesize();
- hpage_pmd_size = read_num("hpage_pmd_size");
+ hpage_pmd_size = read_pmd_pagesize();
hpage_pmd_nr = hpage_pmd_size / page_size;
default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1;
default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8;
save_settings();
- adjust_settings();
+ push_settings(&default_settings);
alloc_at_fault();
- collapse_full();
- collapse_empty();
- collapse_single_pte_entry();
- collapse_max_ptes_none();
- collapse_swapin_single_pte();
- collapse_max_ptes_swap();
- collapse_single_pte_entry_compound();
- collapse_full_of_compound();
- collapse_compound_extreme();
- collapse_fork();
- collapse_fork_compound();
- collapse_max_ptes_shared();
+
+#define TEST(t, c, o) do { \
+ if (c && o) { \
+ printf("\nRun test: " #t " (%s:%s)\n", c->name, o->name); \
+ t(c, o); \
+ } \
+ } while (0)
+
+ TEST(collapse_full, khugepaged_context, anon_ops);
+ TEST(collapse_full, khugepaged_context, file_ops);
+ TEST(collapse_full, khugepaged_context, shmem_ops);
+ TEST(collapse_full, madvise_context, anon_ops);
+ TEST(collapse_full, madvise_context, file_ops);
+ TEST(collapse_full, madvise_context, shmem_ops);
+
+ TEST(collapse_empty, khugepaged_context, anon_ops);
+ TEST(collapse_empty, madvise_context, anon_ops);
+
+ TEST(collapse_single_pte_entry, khugepaged_context, anon_ops);
+ TEST(collapse_single_pte_entry, khugepaged_context, file_ops);
+ TEST(collapse_single_pte_entry, khugepaged_context, shmem_ops);
+ TEST(collapse_single_pte_entry, madvise_context, anon_ops);
+ TEST(collapse_single_pte_entry, madvise_context, file_ops);
+ TEST(collapse_single_pte_entry, madvise_context, shmem_ops);
+
+ TEST(collapse_max_ptes_none, khugepaged_context, anon_ops);
+ TEST(collapse_max_ptes_none, khugepaged_context, file_ops);
+ TEST(collapse_max_ptes_none, madvise_context, anon_ops);
+ TEST(collapse_max_ptes_none, madvise_context, file_ops);
+
+ TEST(collapse_single_pte_entry_compound, khugepaged_context, anon_ops);
+ TEST(collapse_single_pte_entry_compound, khugepaged_context, file_ops);
+ TEST(collapse_single_pte_entry_compound, madvise_context, anon_ops);
+ TEST(collapse_single_pte_entry_compound, madvise_context, file_ops);
+
+ TEST(collapse_full_of_compound, khugepaged_context, anon_ops);
+ TEST(collapse_full_of_compound, khugepaged_context, file_ops);
+ TEST(collapse_full_of_compound, khugepaged_context, shmem_ops);
+ TEST(collapse_full_of_compound, madvise_context, anon_ops);
+ TEST(collapse_full_of_compound, madvise_context, file_ops);
+ TEST(collapse_full_of_compound, madvise_context, shmem_ops);
+
+ TEST(collapse_compound_extreme, khugepaged_context, anon_ops);
+ TEST(collapse_compound_extreme, madvise_context, anon_ops);
+
+ TEST(collapse_swapin_single_pte, khugepaged_context, anon_ops);
+ TEST(collapse_swapin_single_pte, madvise_context, anon_ops);
+
+ TEST(collapse_max_ptes_swap, khugepaged_context, anon_ops);
+ TEST(collapse_max_ptes_swap, madvise_context, anon_ops);
+
+ TEST(collapse_fork, khugepaged_context, anon_ops);
+ TEST(collapse_fork, madvise_context, anon_ops);
+
+ TEST(collapse_fork_compound, khugepaged_context, anon_ops);
+ TEST(collapse_fork_compound, madvise_context, anon_ops);
+
+ TEST(collapse_max_ptes_shared, khugepaged_context, anon_ops);
+ TEST(collapse_max_ptes_shared, madvise_context, anon_ops);
+
+ TEST(madvise_collapse_existing_thps, madvise_context, anon_ops);
+ TEST(madvise_collapse_existing_thps, madvise_context, file_ops);
+ TEST(madvise_collapse_existing_thps, madvise_context, shmem_ops);
+
+ TEST(madvise_retracted_page_tables, madvise_context, file_ops);
+ TEST(madvise_retracted_page_tables, madvise_context, shmem_ops);
restore_settings(0);
}
return addr;
}
+/*
+ * This test validates that merge is called when expanding a mapping.
+ * Mapping containing three pages is created, middle page is unmapped
+ * and then the mapping containing the first page is expanded so that
+ * it fills the created hole. The two parts should merge creating
+ * single mapping with three pages.
+ */
+static void mremap_expand_merge(unsigned long page_size)
+{
+ char *test_name = "mremap expand merge";
+ FILE *fp;
+ char *line = NULL;
+ size_t len = 0;
+ bool success = false;
+ char *start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ munmap(start + page_size, page_size);
+ mremap(start, page_size, 2 * page_size, 0);
+
+ fp = fopen("/proc/self/maps", "r");
+ if (fp == NULL) {
+ ksft_test_result_fail("%s\n", test_name);
+ return;
+ }
+
+ while (getline(&line, &len, fp) != -1) {
+ char *first = strtok(line, "- ");
+ void *first_val = (void *)strtol(first, NULL, 16);
+ char *second = strtok(NULL, "- ");
+ void *second_val = (void *) strtol(second, NULL, 16);
+
+ if (first_val == start && second_val == start + 3 * page_size) {
+ success = true;
+ break;
+ }
+ }
+ if (success)
+ ksft_test_result_pass("%s\n", test_name);
+ else
+ ksft_test_result_fail("%s\n", test_name);
+ fclose(fp);
+}
+
/*
* Returns the start address of the mapping on success, else returns
* NULL on failure.
int i, run_perf_tests;
unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
unsigned int pattern_seed;
+ int num_expand_tests = 1;
struct test test_cases[MAX_TEST];
struct test perf_test_cases[MAX_PERF_TEST];
int page_size;
(threshold_mb * _1MB >= _1GB);
ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
- ARRAY_SIZE(perf_test_cases) : 0));
+ ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests);
for (i = 0; i < ARRAY_SIZE(test_cases); i++)
run_mremap_test_case(test_cases[i], &failures, threshold_mb,
pattern_seed);
+ mremap_expand_merge(page_size);
+
if (run_perf_tests) {
ksft_print_msg("\n%s\n",
"mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
# Dump pages 0, 19, and 4096, using pin_user_pages:
run_test ./gup_test -ct -F 0x1 0 19 0x1000
-run_test ./userfaultfd anon 20 16
-# Test requires source and destination huge pages. Size of source
-# (half_ufd_size_MB) is passed as argument to test.
-run_test ./userfaultfd hugetlb "$half_ufd_size_MB" 32
-run_test ./userfaultfd shmem 20 16
+uffd_mods=("" ":dev")
+for mod in "${uffd_mods[@]}"; do
+ run_test ./userfaultfd anon${mod} 20 16
+ # Hugetlb tests require source and destination huge pages. Pass in half
+ # the size ($half_ufd_size_MB), which is used for *each*.
+ run_test ./userfaultfd hugetlb${mod} "$half_ufd_size_MB" 32
+ run_test ./userfaultfd hugetlb_shared${mod} "$half_ufd_size_MB" 32 "$mnt"/uffd-test
+ rm -f "$mnt"/uffd-test
+ run_test ./userfaultfd shmem${mod} 20 16
+done
#cleanup
umount "$mnt"
for (i = 0; i < hpage_len; i++)
map[i] = (char)i;
- if (check_huge(map)) {
+ if (check_huge_anon(map, 1, hpage_len)) {
ksft_test_result_pass("Test %s huge page allocation\n", __func__);
clear_softdirty();
{
char *one_page;
size_t len = 4 * pmd_pagesize;
- uint64_t thp_size;
size_t i;
one_page = memalign(pmd_pagesize, len);
for (i = 0; i < len; i++)
one_page[i] = (char)i;
- thp_size = check_huge(one_page);
- if (!thp_size) {
+ if (!check_huge_anon(one_page, 1, pmd_pagesize)) {
printf("No THP is allocated\n");
exit(EXIT_FAILURE);
}
}
- thp_size = check_huge(one_page);
- if (thp_size) {
- printf("Still %ld kB AnonHugePages not split\n", thp_size);
+ if (check_huge_anon(one_page, 0, pmd_pagesize)) {
+ printf("Still AnonHugePages not split\n");
exit(EXIT_FAILURE);
}
for (i = 0; i < len; i++)
one_page[i] = (char)i;
- thp_size = check_huge(one_page);
- if (!thp_size) {
+ if (!check_huge_anon(one_page, 1, pmd_pagesize)) {
printf("No THP is allocated\n");
exit(EXIT_FAILURE);
}
usage
fi
fi
- if [ $? == 0 ]; then
- major=$(awk "\$2==\"HMM_DMIRROR\" {print \$1}" /proc/devices)
- mknod /dev/hmm_dmirror0 c $major 0
- mknod /dev/hmm_dmirror1 c $major 1
- if [ $# -eq 2 ]; then
- mknod /dev/hmm_dmirror2 c $major 2
- mknod /dev/hmm_dmirror3 c $major 3
- fi
- fi
}
unload_driver()
{
modprobe -r $DRIVER > /dev/null 2>&1
- rm -f /dev/hmm_dmirror?
}
run_smoke()
#include <sys/random.h>
#include "../kselftest.h"
+#include "vm_util.h"
#ifdef __NR_userfaultfd
-static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
+static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size, hpage_size;
#define BOUNCE_RANDOM (1<<0)
#define BOUNCE_RACINGFAULTS (1<<1)
#define TEST_SHMEM 3
static int test_type;
+#define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY)
+
+#define BASE_PMD_ADDR ((void *)(1UL << 30))
+
+/* test using /dev/userfaultfd, instead of userfaultfd(2) */
+static bool test_dev_userfaultfd;
+
/* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
#define ALARM_INTERVAL_SECS 10
static volatile bool test_uffdio_copy_eexist = true;
static unsigned long long *count_verify;
static int uffd = -1;
static int uffd_flags, finished, *pipefd;
-static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
+static char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap;
static char *zeropage;
pthread_attr_t attr;
+static bool test_collapse;
/* Userfaultfd test statistics */
struct uffd_stats {
#define swap(a, b) \
do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
+#define factor_of_2(x) ((x) ^ ((x) & ((x) - 1)))
+
const char *examples =
"# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
"./userfaultfd anon 100 99999\n\n"
+ "# Run the same anonymous memory test, but using /dev/userfaultfd:\n"
+ "./userfaultfd anon:dev 100 99999\n\n"
"# Run share memory test on 1GiB region with 99 bounces:\n"
"./userfaultfd shmem 1000 99\n\n"
"# Run hugetlb memory test on 256MiB region with 50 bounces:\n"
"[hugetlbfs_file]\n\n");
fprintf(stderr, "Supported <test type>: anon, hugetlb, "
"hugetlb_shared, shmem\n\n");
+ fprintf(stderr, "'Test mods' can be joined to the test type string with a ':'. "
+ "Supported mods:\n");
+ fprintf(stderr, "\tsyscall - Use userfaultfd(2) (default)\n");
+ fprintf(stderr, "\tdev - Use /dev/userfaultfd instead of userfaultfd(2)\n");
+ fprintf(stderr, "\tcollapse - Test MADV_COLLAPSE of UFFDIO_REGISTER_MODE_MINOR\n"
+ "memory\n");
+ fprintf(stderr, "\nExample test mod usage:\n");
+ fprintf(stderr, "# Run anonymous memory test with /dev/userfaultfd:\n");
+ fprintf(stderr, "./userfaultfd anon:dev 100 99999\n\n");
+
fprintf(stderr, "Examples:\n\n");
fprintf(stderr, "%s", examples);
exit(1);
ret, __LINE__); \
} while (0)
-#define err(fmt, ...) \
+#define errexit(exitcode, fmt, ...) \
do { \
_err(fmt, ##__VA_ARGS__); \
- exit(1); \
+ exit(exitcode); \
} while (0)
+#define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__)
+
static void uffd_stats_reset(struct uffd_stats *uffd_stats,
unsigned long n_cpus)
{
err("madvise(MADV_DONTNEED) failed");
}
-static void anon_allocate_area(void **alloc_area)
+static void anon_allocate_area(void **alloc_area, bool is_src)
{
*alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- if (*alloc_area == MAP_FAILED)
- err("mmap of anonymous memory failed");
}
static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
}
}
-static void hugetlb_allocate_area(void **alloc_area)
+static void hugetlb_allocate_area(void **alloc_area, bool is_src)
{
void *area_alias = NULL;
char **alloc_area_alias;
nr_pages * page_size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB |
- (*alloc_area == area_src ? 0 : MAP_NORESERVE),
+ (is_src ? 0 : MAP_NORESERVE),
-1,
0);
else
nr_pages * page_size,
PROT_READ | PROT_WRITE,
MAP_SHARED |
- (*alloc_area == area_src ? 0 : MAP_NORESERVE),
+ (is_src ? 0 : MAP_NORESERVE),
huge_fd,
- *alloc_area == area_src ? 0 : nr_pages * page_size);
+ is_src ? 0 : nr_pages * page_size);
if (*alloc_area == MAP_FAILED)
err("mmap of hugetlbfs file failed");
PROT_READ | PROT_WRITE,
MAP_SHARED,
huge_fd,
- *alloc_area == area_src ? 0 : nr_pages * page_size);
+ is_src ? 0 : nr_pages * page_size);
if (area_alias == MAP_FAILED)
err("mmap of hugetlb file alias failed");
}
- if (*alloc_area == area_src) {
+ if (is_src) {
alloc_area_alias = &area_src_alias;
} else {
alloc_area_alias = &area_dst_alias;
err("madvise(MADV_REMOVE) failed");
}
-static void shmem_allocate_area(void **alloc_area)
+static void shmem_allocate_area(void **alloc_area, bool is_src)
{
void *area_alias = NULL;
- bool is_src = alloc_area == (void **)&area_src;
- unsigned long offset = is_src ? 0 : nr_pages * page_size;
+ size_t bytes = nr_pages * page_size;
+ unsigned long offset = is_src ? 0 : bytes;
+ char *p = NULL, *p_alias = NULL;
+
+ if (test_collapse) {
+ p = BASE_PMD_ADDR;
+ if (!is_src)
+ /* src map + alias + interleaved hpages */
+ p += 2 * (bytes + hpage_size);
+ p_alias = p;
+ p_alias += bytes;
+ p_alias += hpage_size; /* Prevent src/dst VMA merge */
+ }
- *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, shm_fd, offset);
+ *alloc_area = mmap(p, bytes, PROT_READ | PROT_WRITE, MAP_SHARED,
+ shm_fd, offset);
if (*alloc_area == MAP_FAILED)
err("mmap of memfd failed");
+ if (test_collapse && *alloc_area != p)
+ err("mmap of memfd failed at %p", p);
- area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, shm_fd, offset);
+ area_alias = mmap(p_alias, bytes, PROT_READ | PROT_WRITE, MAP_SHARED,
+ shm_fd, offset);
if (area_alias == MAP_FAILED)
err("mmap of memfd alias failed");
+ if (test_collapse && area_alias != p_alias)
+ err("mmap of anonymous memory failed at %p", p_alias);
if (is_src)
area_src_alias = area_alias;
*start = (unsigned long)area_dst_alias + offset;
}
+static void shmem_check_pmd_mapping(void *p, int expect_nr_hpages)
+{
+ if (!check_huge_shmem(area_dst_alias, expect_nr_hpages, hpage_size))
+ err("Did not find expected %d number of hugepages",
+ expect_nr_hpages);
+}
+
struct uffd_test_ops {
- void (*allocate_area)(void **alloc_area);
+ void (*allocate_area)(void **alloc_area, bool is_src);
void (*release_pages)(char *rel_area);
void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
+ void (*check_pmd_mapping)(void *p, int expect_nr_hpages);
};
static struct uffd_test_ops anon_uffd_test_ops = {
.allocate_area = anon_allocate_area,
.release_pages = anon_release_pages,
.alias_mapping = noop_alias_mapping,
+ .check_pmd_mapping = NULL,
};
static struct uffd_test_ops shmem_uffd_test_ops = {
.allocate_area = shmem_allocate_area,
.release_pages = shmem_release_pages,
.alias_mapping = shmem_alias_mapping,
+ .check_pmd_mapping = shmem_check_pmd_mapping,
};
static struct uffd_test_ops hugetlb_uffd_test_ops = {
.allocate_area = hugetlb_allocate_area,
.release_pages = hugetlb_release_pages,
.alias_mapping = hugetlb_alias_mapping,
+ .check_pmd_mapping = NULL,
};
static struct uffd_test_ops *uffd_test_ops;
}
}
+static int __userfaultfd_open_dev(void)
+{
+ int fd, _uffd;
+
+ fd = open("/dev/userfaultfd", O_RDWR | O_CLOEXEC);
+ if (fd < 0)
+ errexit(KSFT_SKIP, "opening /dev/userfaultfd failed");
+
+ _uffd = ioctl(fd, USERFAULTFD_IOC_NEW, UFFD_FLAGS);
+ if (_uffd < 0)
+ errexit(errno == ENOTTY ? KSFT_SKIP : 1,
+ "creating userfaultfd failed");
+ close(fd);
+ return _uffd;
+}
+
static void userfaultfd_open(uint64_t *features)
{
struct uffdio_api uffdio_api;
- uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
- if (uffd < 0)
- err("userfaultfd syscall not available in this kernel");
+ if (test_dev_userfaultfd)
+ uffd = __userfaultfd_open_dev();
+ else {
+ uffd = syscall(__NR_userfaultfd, UFFD_FLAGS);
+ if (uffd < 0)
+ errexit(errno == ENOSYS ? KSFT_SKIP : 1,
+ "creating userfaultfd failed");
+ }
uffd_flags = fcntl(uffd, F_GETFD, NULL);
uffdio_api.api = UFFD_API;
munmap_area((void **)&area_src_alias);
munmap_area((void **)&area_dst);
munmap_area((void **)&area_dst_alias);
+ munmap_area((void **)&area_remap);
}
static void uffd_test_ctx_init(uint64_t features)
uffd_test_ctx_clear();
- uffd_test_ops->allocate_area((void **)&area_src);
- uffd_test_ops->allocate_area((void **)&area_dst);
+ uffd_test_ops->allocate_area((void **)&area_src, true);
+ uffd_test_ops->allocate_area((void **)&area_dst, false);
userfaultfd_open(&features);
continue_range(uffd, msg->arg.pagefault.address, page_size);
stats->minor_faults++;
} else {
- /* Missing page faults */
+ /*
+ * Missing page faults.
+ *
+ * Here we force a write check for each of the missing mode
+ * faults. It's guaranteed because the only threads that
+ * will trigger uffd faults are the locking threads, and
+ * their first instruction to touch the missing page will
+ * always be pthread_mutex_lock().
+ *
+ * Note that here we relied on an NPTL glibc impl detail to
+ * always read the lock type at the entry of the lock op
+ * (pthread_mutex_t.__data.__type, offset 0x10) before
+ * doing any locking operations to guarantee that. It's
+ * actually not good to rely on this impl detail because
+ * logically a pthread-compatible lib can implement the
+ * locks without types and we can fail when linking with
+ * them. However since we used to find bugs with this
+ * strict check we still keep it around. Hopefully this
+ * could be a good hint when it fails again. If one day
+ * it'll break on some other impl of glibc we'll revisit.
+ */
if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
err("unexpected write fault");
err("remove failure");
break;
case UFFD_EVENT_REMAP:
+ area_remap = area_dst; /* save for later unmap */
area_dst = (char *)(unsigned long)msg.arg.remap.to;
break;
}
return userfaults != 0;
}
+void check_memory_contents(char *p)
+{
+ unsigned long i;
+ uint8_t expected_byte;
+ void *expected_page;
+
+ if (posix_memalign(&expected_page, page_size, page_size))
+ err("out of memory");
+
+ for (i = 0; i < nr_pages; ++i) {
+ expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
+ memset(expected_page, expected_byte, page_size);
+ if (my_bcmp(expected_page, p + (i * page_size), page_size))
+ err("unexpected page contents after minor fault");
+ }
+
+ free(expected_page);
+}
+
static int userfaultfd_minor_test(void)
{
- struct uffdio_register uffdio_register;
unsigned long p;
+ struct uffdio_register uffdio_register;
pthread_t uffd_mon;
- uint8_t expected_byte;
- void *expected_page;
char c;
struct uffd_stats stats = { 0 };
* fault. uffd_poll_thread will resolve the fault by bit-flipping the
* page's contents, and then issuing a CONTINUE ioctl.
*/
-
- if (posix_memalign(&expected_page, page_size, page_size))
- err("out of memory");
-
- for (p = 0; p < nr_pages; ++p) {
- expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));
- memset(expected_page, expected_byte, page_size);
- if (my_bcmp(expected_page, area_dst_alias + (p * page_size),
- page_size))
- err("unexpected page contents after minor fault");
- }
+ check_memory_contents(area_dst_alias);
if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
uffd_stats_report(&stats, 1);
+ if (test_collapse) {
+ printf("testing collapse of uffd memory into PMD-mapped THPs:");
+ if (madvise(area_dst_alias, nr_pages * page_size,
+ MADV_COLLAPSE))
+ err("madvise(MADV_COLLAPSE)");
+
+ uffd_test_ops->check_pmd_mapping(area_dst,
+ nr_pages * page_size /
+ hpage_size);
+ /*
+ * This won't cause uffd-fault - it purely just makes sure there
+ * was no corruption.
+ */
+ check_memory_contents(area_dst_alias);
+ printf(" done.\n");
+ }
+
return stats.missing_faults != 0 || stats.minor_faults != nr_pages;
}
static void set_test_type(const char *type)
{
- uint64_t features = UFFD_API_FEATURES;
-
if (!strcmp(type, "anon")) {
test_type = TEST_ANON;
uffd_test_ops = &anon_uffd_test_ops;
test_type = TEST_SHMEM;
uffd_test_ops = &shmem_uffd_test_ops;
test_uffdio_minor = true;
- } else {
- err("Unknown test type: %s", type);
}
+}
+
+static void parse_test_type_arg(const char *raw_type)
+{
+ char *buf = strdup(raw_type);
+ uint64_t features = UFFD_API_FEATURES;
+
+ while (buf) {
+ const char *token = strsep(&buf, ":");
+
+ if (!test_type)
+ set_test_type(token);
+ else if (!strcmp(token, "dev"))
+ test_dev_userfaultfd = true;
+ else if (!strcmp(token, "syscall"))
+ test_dev_userfaultfd = false;
+ else if (!strcmp(token, "collapse"))
+ test_collapse = true;
+ else
+ err("unrecognized test mod '%s'", token);
+ }
+
+ if (!test_type)
+ err("failed to parse test type argument: '%s'", raw_type);
+
+ if (test_collapse && test_type != TEST_SHMEM)
+ err("Unsupported test: %s", raw_type);
if (test_type == TEST_HUGETLB)
- page_size = default_huge_page_size();
+ page_size = hpage_size;
else
page_size = sysconf(_SC_PAGE_SIZE);
int main(int argc, char **argv)
{
+ size_t bytes;
+
if (argc < 4)
usage();
err("failed to arm SIGALRM");
alarm(ALARM_INTERVAL_SECS);
- set_test_type(argv[1]);
+ hpage_size = default_huge_page_size();
+ parse_test_type_arg(argv[1]);
+ bytes = atol(argv[2]) * 1024 * 1024;
+
+ if (test_collapse && bytes & (hpage_size - 1))
+ err("MiB must be multiple of %lu if :collapse mod set",
+ hpage_size >> 20);
nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
- nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
- nr_cpus;
+
+ if (test_collapse) {
+ /* nr_cpus must divide (bytes / page_size), otherwise,
+ * area allocations of (nr_pages * paze_size) won't be a
+ * multiple of hpage_size, even if bytes is a multiple of
+ * hpage_size.
+ *
+ * This means that nr_cpus must divide (N * (2 << (H-P))
+ * where:
+ * bytes = hpage_size * N
+ * hpage_size = 2 << H
+ * page_size = 2 << P
+ *
+ * And we want to chose nr_cpus to be the largest value
+ * satisfying this constraint, not larger than the number
+ * of online CPUs. Unfortunately, prime factorization of
+ * N and nr_cpus may be arbitrary, so have to search for it.
+ * Instead, just use the highest power of 2 dividing both
+ * nr_cpus and (bytes / page_size).
+ */
+ int x = factor_of_2(nr_cpus);
+ int y = factor_of_2(bytes / page_size);
+
+ nr_cpus = x < y ? x : y;
+ }
+ nr_pages_per_cpu = bytes / page_size / nr_cpus;
if (!nr_pages_per_cpu) {
_err("invalid MiB");
usage();
ksft_exit_fail_msg("writing clear_refs failed\n");
}
-static bool check_for_pattern(FILE *fp, const char *pattern, char *buf)
+bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
{
- while (fgets(buf, MAX_LINE_LENGTH, fp) != NULL) {
+ while (fgets(buf, len, fp)) {
if (!strncmp(buf, pattern, strlen(pattern)))
return true;
}
return strtoul(buf, NULL, 10);
}
-uint64_t check_huge(void *addr)
+bool __check_huge(void *addr, char *pattern, int nr_hpages,
+ uint64_t hpage_size)
{
- uint64_t thp = 0;
+ uint64_t thp = -1;
int ret;
FILE *fp;
char buffer[MAX_LINE_LENGTH];
if (!fp)
ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
- if (!check_for_pattern(fp, addr_pattern, buffer))
+ if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
goto err_out;
/*
- * Fetch the AnonHugePages: in the same block and check the number of
+ * Fetch the pattern in the same block and check the number of
* hugepages.
*/
- if (!check_for_pattern(fp, "AnonHugePages:", buffer))
+ if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
goto err_out;
- if (sscanf(buffer, "AnonHugePages:%10ld kB", &thp) != 1)
+ snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
+
+ if (sscanf(buffer, addr_pattern, &thp) != 1)
ksft_exit_fail_msg("Reading smap error\n");
err_out:
fclose(fp);
- return thp;
+ return thp == (nr_hpages * (hpage_size >> 10));
+}
+
+bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
+{
+ return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
+}
+
+bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
+{
+ return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
+}
+
+bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
+{
+ return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
}
uint64_t pagemap_get_entry(int fd, char *start);
bool pagemap_is_softdirty(int fd, char *start);
void clear_softdirty(void);
+bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len);
uint64_t read_pmd_pagesize(void);
-uint64_t check_huge(void *addr);
+bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size);
+bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size);
+bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size);
static bool is_need(char *buf)
{
- if ((filter & FILTER_UNRELEASE) && get_free_ts_nsec(buf) != 0)
+ __u64 ts_nsec, free_ts_nsec;
+
+ ts_nsec = get_ts_nsec(buf);
+ free_ts_nsec = get_free_ts_nsec(buf);
+
+ if ((filter & FILTER_UNRELEASE) && free_ts_nsec != 0 && ts_nsec < free_ts_nsec)
return false;
if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size))
return false;
char s[256];
struct stat buf;
unsigned long size;
- int file = -1;
+ int file;
int retval;
int rc = -1;
int namesize;
config HAVE_KVM_DIRTY_RING
bool
+# Only strongly ordered architectures can select this, as it doesn't
+# put any explicit constraint on userspace ordering. They can also
+# select the _ACQ_REL version.
+config HAVE_KVM_DIRTY_RING_TSO
+ bool
+ select HAVE_KVM_DIRTY_RING
+ depends on X86
+
+# Weakly ordered architectures can only select this, advertising
+# to userspace the additional ordering requirements.
+config HAVE_KVM_DIRTY_RING_ACQ_REL
+ bool
+ select HAVE_KVM_DIRTY_RING
+
config HAVE_KVM_EVENTFD
bool
select EVENTFD
static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn)
{
- gfn->flags = 0;
+ smp_store_release(&gfn->flags, 0);
}
static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn)
static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn)
{
- return gfn->flags & KVM_DIRTY_GFN_F_RESET;
+ return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET;
}
int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring)
case KVM_CAP_NR_MEMSLOTS:
return KVM_USER_MEM_SLOTS;
case KVM_CAP_DIRTY_LOG_RING:
-#ifdef CONFIG_HAVE_KVM_DIRTY_RING
+#ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO
+ return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
+#else
+ return 0;
+#endif
+ case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
+#ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL
return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
#else
return 0;
return 0;
}
case KVM_CAP_DIRTY_LOG_RING:
+ case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
default:
return kvm_vm_ioctl_enable_cap(kvm, cap);
struct kvm_vfio_group {
struct list_head node;
struct file *file;
+#ifdef CONFIG_SPAPR_TCE_IOMMU
+ struct iommu_group *iommu_group;
+#endif
};
struct kvm_vfio {
return ret;
}
+static bool kvm_vfio_file_is_group(struct file *file)
+{
+ bool (*fn)(struct file *file);
+ bool ret;
+
+ fn = symbol_get(vfio_file_is_group);
+ if (!fn)
+ return false;
+
+ ret = fn(file);
+
+ symbol_put(vfio_file_is_group);
+
+ return ret;
+}
+
+#ifdef CONFIG_SPAPR_TCE_IOMMU
static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file)
{
struct iommu_group *(*fn)(struct file *file);
return ret;
}
-#ifdef CONFIG_SPAPR_TCE_IOMMU
static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm,
struct kvm_vfio_group *kvg)
{
- struct iommu_group *grp = kvm_vfio_file_iommu_group(kvg->file);
-
- if (WARN_ON_ONCE(!grp))
+ if (WARN_ON_ONCE(!kvg->iommu_group))
return;
- kvm_spapr_tce_release_iommu_group(kvm, grp);
+ kvm_spapr_tce_release_iommu_group(kvm, kvg->iommu_group);
+ iommu_group_put(kvg->iommu_group);
+ kvg->iommu_group = NULL;
}
#endif
return -EBADF;
/* Ensure the FD is a vfio group FD.*/
- if (!kvm_vfio_file_iommu_group(filp)) {
+ if (!kvm_vfio_file_is_group(filp)) {
ret = -EINVAL;
goto err_fput;
}
mutex_lock(&kv->lock);
list_for_each_entry(kvg, &kv->group_list, node) {
- struct iommu_group *grp;
-
if (kvg->file != f.file)
continue;
- grp = kvm_vfio_file_iommu_group(kvg->file);
- if (WARN_ON_ONCE(!grp)) {
- ret = -EIO;
- goto err_fdput;
+ if (!kvg->iommu_group) {
+ kvg->iommu_group = kvm_vfio_file_iommu_group(kvg->file);
+ if (WARN_ON_ONCE(!kvg->iommu_group)) {
+ ret = -EIO;
+ goto err_fdput;
+ }
}
ret = kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd,
- grp);
+ kvg->iommu_group);
break;
}
git.samba.org / sfrench / cifs-2.6.git / commitdiff