diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 855503e9e1fd110bec741e95704d3ec378ed9aef..405f30cf62da4332e6b620116741ded41caacba2 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -266,6 +266,14 @@ static int validate_group(struct perf_event *event) return 0; } +static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs, + struct perf_event *event) +{ + return perf_ibs == &perf_ibs_op && + (ibs_caps & IBS_CAPS_OPLDLAT) && + (event->attr.config1 & 0xFFF); +} + static int perf_ibs_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -324,6 +332,17 @@ static int perf_ibs_init(struct perf_event *event) return -EINVAL; } + if (perf_ibs_ldlat_event(perf_ibs, event)) { + u64 ldlat = event->attr.config1 & 0xFFF; + + if (ldlat < 128 || ldlat > 2048) + return -EINVAL; + ldlat >>= 7; + + config |= (ldlat - 1) << 59; + config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN; + } + /* * If we modify hwc->sample_period, we also need to update * hwc->last_period and hwc->period_left. @@ -602,7 +621,10 @@ PMU_FORMAT_ATTR(rand_en, "config:57"); PMU_FORMAT_ATTR(cnt_ctl, "config:19"); PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); +PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11"); PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); +PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); +PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) @@ -610,6 +632,18 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; } +static umode_t +ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0; +} + +static umode_t +ibs_op_dtlb_pgsize_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return ibs_caps & IBS_CAPS_OPDTLBPGSIZE ? attr->mode : 0; +} + static struct attribute *rand_en_attrs[] = { &format_attr_rand_en.attr, NULL, @@ -625,6 +659,16 @@ static struct attribute *zen4_ibs_extensions_attrs[] = { NULL, }; +static struct attribute *ibs_op_ldlat_cap_attrs[] = { + &ibs_op_ldlat_cap.attr.attr, + NULL, +}; + +static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = { + &ibs_op_dtlb_pgsize_cap.attr.attr, + NULL, +}; + static struct attribute_group group_rand_en = { .name = "format", .attrs = rand_en_attrs, @@ -642,6 +686,18 @@ static struct attribute_group group_zen4_ibs_extensions = { .is_visible = zen4_ibs_extensions_is_visible, }; +static struct attribute_group group_ibs_op_ldlat_cap = { + .name = "caps", + .attrs = ibs_op_ldlat_cap_attrs, + .is_visible = ibs_op_ldlat_is_visible, +}; + +static struct attribute_group group_ibs_op_dtlb_pgsize_cap = { + .name = "caps", + .attrs = ibs_op_dtlb_pgsize_cap_attrs, + .is_visible = ibs_op_dtlb_pgsize_is_visible, +}; + static const struct attribute_group *fetch_attr_groups[] = { &group_rand_en, &empty_caps_group, @@ -670,6 +726,11 @@ static struct attribute *op_l3missonly_attrs[] = { NULL, }; +static struct attribute *ibs_op_ldlat_format_attrs[] = { + &ibs_op_ldlat_format.attr.attr, + NULL, +}; + static struct attribute_group group_cnt_ctl = { .name = "format", .attrs = cnt_ctl_attrs, @@ -682,10 +743,19 @@ static struct attribute_group group_op_l3missonly = { .is_visible = zen4_ibs_extensions_is_visible, }; +static struct attribute_group group_ibs_op_ldlat_format = { + .name = "format", + .attrs = ibs_op_ldlat_format_attrs, + .is_visible = ibs_op_ldlat_is_visible, +}; + static const struct attribute_group *op_attr_update[] = { &group_cnt_ctl, &group_op_l3missonly, &group_zen4_ibs_extensions, + &group_ibs_op_ldlat_cap, + &group_ibs_op_ldlat_format, + &group_ibs_op_dtlb_pgsize_cap, NULL, }; @@ -936,6 +1006,10 @@ static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, if (!op_data3->dc_lin_addr_valid) return; + if ((ibs_caps & IBS_CAPS_OPDTLBPGSIZE) && + !op_data3->dc_phy_addr_valid) + return; + if (!op_data3->dc_l1tlb_miss) { data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT; return; @@ -1040,15 +1114,25 @@ static void perf_ibs_parse_ld_st_data(__u64 sample_type, } } -static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type, +static bool perf_ibs_is_mem_sample_type(struct perf_ibs *perf_ibs, + struct perf_event *event) +{ + u64 sample_type = event->attr.sample_type; + + return perf_ibs == &perf_ibs_op && + sample_type & (PERF_SAMPLE_DATA_SRC | + PERF_SAMPLE_WEIGHT_TYPE | + PERF_SAMPLE_ADDR | + PERF_SAMPLE_PHYS_ADDR); +} + +static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, + struct perf_event *event, int check_rip) { - if (sample_type & PERF_SAMPLE_RAW || - (perf_ibs == &perf_ibs_op && - (sample_type & PERF_SAMPLE_DATA_SRC || - sample_type & PERF_SAMPLE_WEIGHT_TYPE || - sample_type & PERF_SAMPLE_ADDR || - sample_type & PERF_SAMPLE_PHYS_ADDR))) + if (event->attr.sample_type & PERF_SAMPLE_RAW || + perf_ibs_is_mem_sample_type(perf_ibs, event) || + perf_ibs_ldlat_event(perf_ibs, event)) return perf_ibs->offset_max; else if (check_rip) return 3; @@ -1103,7 +1187,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) offset = 1; check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); - offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip); + offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip); do { rdmsrl(msr + offset, *buf++); @@ -1112,6 +1196,22 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) perf_ibs->offset_max, offset + 1); } while (offset < offset_max); + + if (perf_ibs_ldlat_event(perf_ibs, event)) { + union ibs_op_data3 op_data3; + + op_data3.val = ibs_data.regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; + /* + * Opening event is errored out if load latency threshold is + * outside of [128, 2048] range. Since the event has reached + * interrupt handler, we can safely assume the threshold is + * within [128, 2048] range. + */ + if (!op_data3.ld_op || !op_data3.dc_miss || + op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) + goto out; + } + /* * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately * depending on their availability. diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd-ibs.h index cb2a5e113daa75d50ae8dd1ed338bd1c77db9273..77f3a589a99ac58fdd0e41291adcf1e48a44e2c2 100644 --- a/arch/x86/include/asm/amd-ibs.h +++ b/arch/x86/include/asm/amd-ibs.h @@ -64,7 +64,8 @@ union ibs_op_ctl { opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */ reserved0:5, /* 27-31: reserved */ opcurcnt:27, /* 32-58: periodic op counter current count */ - reserved1:5; /* 59-63: reserved */ + ldlat_thrsh:4, /* 59-62: Load Latency threshold */ + ldlat_en:1; /* 63: Load Latency enabled */ }; }; diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 338b8a7042d26691726eaae3bfd75d497850c4a8..e671cce1b4179491a49e7bd163803966af87709a 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -540,6 +540,8 @@ struct pebs_cntr_header { #define IBS_CAPS_FETCHCTLEXTD (1U<<9) #define IBS_CAPS_OPDATA4 (1U<<10) #define IBS_CAPS_ZEN4 (1U<<11) +#define IBS_CAPS_OPLDLAT (1U<<12) +#define IBS_CAPS_OPDTLBPGSIZE (1U<<19) #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \ | IBS_CAPS_FETCHSAM \ @@ -565,6 +567,8 @@ struct pebs_cntr_header { * The lower 7 bits of the current count are random bits * preloaded by hardware and ignored in software */ +#define IBS_OP_LDLAT_EN (1ULL<<63) +#define IBS_OP_LDLAT_THRSH (0xFULL<<59) #define IBS_OP_CUR_CNT (0xFFF80ULL<<32) #define IBS_OP_CUR_CNT_RAND (0x0007FULL<<32) #define IBS_OP_CUR_CNT_EXT_MASK (0x7FULL<<52) diff --git a/tools/arch/x86/include/asm/amd-ibs.h b/tools/arch/x86/include/asm/amd-ibs.h index 93807b437e4deb85b2d96e61df2bfd0b0262e711..cb1740bc3da2df521a2282e900e7bc127dbe550d 100644 --- a/tools/arch/x86/include/asm/amd-ibs.h +++ b/tools/arch/x86/include/asm/amd-ibs.h @@ -64,7 +64,8 @@ union ibs_op_ctl { opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */ reserved0:5, /* 27-31: reserved */ opcurcnt:27, /* 32-58: periodic op counter current count */ - reserved1:5; /* 59-63: reserved */ + ldlat_thrsh:4, /* 59-62: Load Latency threshold */ + ldlat_en:1; /* 63: Load Latency enabled */ }; }; diff --git a/tools/perf/Documentation/perf-amd-ibs.txt b/tools/perf/Documentation/perf-amd-ibs.txt new file mode 100644 index 0000000000000000000000000000000000000000..55f80beae0375a72e58fd4bdb571b29b018ac45a --- /dev/null +++ b/tools/perf/Documentation/perf-amd-ibs.txt @@ -0,0 +1,198 @@ +perf-amd-ibs(1) +=============== + +NAME +---- +perf-amd-ibs - Support for AMD Instruction-Based Sampling (IBS) with perf tool + +SYNOPSIS +-------- +[verse] +'perf record' -e ibs_op// +'perf record' -e ibs_fetch// + +DESCRIPTION +----------- + +Instruction-Based Sampling (IBS) provides precise Instruction Pointer (IP) +profiling support on AMD platforms. IBS has two independent components: IBS +Op and IBS Fetch. IBS Op sampling provides information about instruction +execution (micro-op execution to be precise) with details like d-cache +hit/miss, d-TLB hit/miss, cache miss latency, load/store data source, branch +behavior etc. IBS Fetch sampling provides information about instruction fetch +with details like i-cache hit/miss, i-TLB hit/miss, fetch latency etc. IBS is +per-smt-thread i.e. each SMT hardware thread contains standalone IBS units. + +Both, IBS Op and IBS Fetch, are exposed as PMUs by Linux and can be exploited +using the Linux perf utility. The following files will be created at boot time +if IBS is supported by the hardware and kernel. + + /sys/bus/event_source/devices/ibs_op/ + /sys/bus/event_source/devices/ibs_fetch/ + +IBS Op PMU supports two events: cycles and micro ops. IBS Fetch PMU supports +one event: fetch ops. + +IBS PMUs do not have user/kernel filtering capability and thus it requires +CAP_SYS_ADMIN or CAP_PERFMON privilege. + +IBS VS. REGULAR CORE PMU +------------------------ + +IBS gives samples with precise IP, i.e. the IP recorded with IBS sample has +no skid. Whereas the IP recorded by regular core PMU will have some skid +(sample was generated at IP X but perf would record it at IP X+n). Hence, +regular core PMU might not help for profiling with instruction level +precision. Further, IBS provides additional information about the sample in +question. On the other hand, regular core PMU has it's own advantages like +plethora of events, counting mode (less interference), up to 6 parallel +counters, event grouping support, filtering capabilities etc. + +Three regular core PMU events are internally forwarded to IBS Op PMU when +precise_ip attribute is set: + + -e cpu-cycles:p becomes -e ibs_op// + -e r076:p becomes -e ibs_op// + -e r0C1:p becomes -e ibs_op/cnt_ctl=1/ + +EXAMPLES +-------- + +IBS Op PMU +~~~~~~~~~~ + +System-wide profile, cycles event, sampling period: 100000 + + # perf record -e ibs_op// -c 100000 -a + +Per-cpu profile (cpu10), cycles event, sampling period: 100000 + + # perf record -e ibs_op// -c 100000 -C 10 + +Per-cpu profile (cpu10), cycles event, sampling freq: 1000 + + # perf record -e ibs_op// -F 1000 -C 10 + +System-wide profile, uOps event, sampling period: 100000 + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -a + +Same command, but also capture IBS register raw dump along with perf sample: + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -a --raw-samples + +System-wide profile, uOps event, sampling period: 100000, L3MissOnly (Zen4 onward) + + # perf record -e ibs_op/cnt_ctl=1,l3missonly=1/ -c 100000 -a + +System-wide profile, cycles event, sampling period: 100000, LdLat filtering (Zen5 +onward) + + # perf record -e ibs_op/ldlat=128/ -c 100000 -a + + Supported load latency threshold values are 128 to 2048 (both inclusive). + Latency value which is a multiple of 128 incurs a little less profiling + overhead compared to other values. + +Per process(upstream v6.2 onward), uOps event, sampling period: 100000 + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -p 1234 + +Per process(upstream v6.2 onward), uOps event, sampling period: 100000 + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -- ls + +To analyse recorded profile in aggregate mode + + # perf report + /* Select a line and press 'a' to drill down at instruction level. */ + +To go over each sample + + # perf script + +Raw dump of IBS registers when profiled with --raw-samples + + # perf report -D + /* Look for PERF_RECORD_SAMPLE */ + + Example register raw dump: + + ibs_op_ctl: 000002c30006186a MaxCnt 100000 L3MissOnly 0 En 1 + Val 1 CntCtl 0=cycles CurCnt 707 + IbsOpRip: ffffffff8204aea7 + ibs_op_data: 0000010002550001 CompToRetCtr 1 TagToRetCtr 597 + BrnRet 0 RipInvalid 0 BrnFuse 0 Microcode 1 + ibs_op_data2: 0000000000000013 RmtNode 1 DataSrc 3=DRAM + ibs_op_data3: 0000000031960092 LdOp 0 StOp 1 DcL1TlbMiss 0 + DcL2TlbMiss 0 DcL1TlbHit2M 1 DcL1TlbHit1G 0 DcL2TlbHit2M 0 + DcMiss 1 DcMisAcc 0 DcWcMemAcc 0 DcUcMemAcc 0 DcLockedOp 0 + DcMissNoMabAlloc 0 DcLinAddrValid 1 DcPhyAddrValid 1 + DcL2TlbHit1G 0 L2Miss 1 SwPf 0 OpMemWidth 32 bytes + OpDcMissOpenMemReqs 12 DcMissLat 0 TlbRefillLat 0 + IbsDCLinAd: ff110008a5398920 + IbsDCPhysAd: 00000008a5398920 + +IBS applied in a real world usecase + + ~90% regression was observed in tbench with specific scheduler hint + which was counter intuitive. IBS profile of good and bad run captured + using perf helped in identifying exact cause of the problem: + + https://lore.kernel.org/r/20220921063638.2489-1-kprateek.nayak@amd.com + +IBS Fetch PMU +~~~~~~~~~~~~~ + +Similar commands can be used with Fetch PMU as well. + +System-wide profile, fetch ops event, sampling period: 100000 + + # perf record -e ibs_fetch// -c 100000 -a + +System-wide profile, fetch ops event, sampling period: 100000, Random enable + + # perf record -e ibs_fetch/rand_en=1/ -c 100000 -a + + Random enable adds small degree of variability to sample period. This + helps in cases like long running loops where PMU is tagging the same + instruction over and over because of fixed sample period. + +etc. + +PERF MEM AND PERF C2C +--------------------- + +perf mem is a memory access profiler tool and perf c2c is a shared data +cacheline analyser tool. Both of them internally uses IBS Op PMU on AMD. +Below is a simple example of the perf mem tool. + + # perf mem record -c 100000 -- make + # perf mem report + +A normal perf mem report output will provide detailed memory access profile. +However, it can also be aggregated based on output fields. For example: + + # perf mem report -F mem,sample,snoop + Samples: 3M of event 'ibs_op//', Event count (approx.): 23524876 + Memory access Samples Snoop + N/A 1903343 N/A + L1 hit 1056754 N/A + L2 hit 75231 N/A + L3 hit 9496 HitM + L3 hit 2270 N/A + RAM hit 8710 N/A + Remote node, same socket RAM hit 3241 N/A + Remote core, same node Any cache hit 1572 HitM + Remote core, same node Any cache hit 514 N/A + Remote node, same socket Any cache hit 1216 HitM + Remote node, same socket Any cache hit 350 N/A + Uncached hit 18 N/A + +Please refer to their man page for more detail. + +SEE ALSO +-------- + +linkperf:perf-record[1], linkperf:perf-script[1], linkperf:perf-report[1], +linkperf:perf-mem[1], linkperf:perf-c2c[1] diff --git a/tools/perf/Documentation/perf-c2c.txt b/tools/perf/Documentation/perf-c2c.txt index 192ab0415ee9e855ff61c74f88d15632ee3364b2..c9c3dc18bb9d34c64fcc4ec48ce3f2ac6f1dbb85 100644 --- a/tools/perf/Documentation/perf-c2c.txt +++ b/tools/perf/Documentation/perf-c2c.txt @@ -54,8 +54,15 @@ RECORD OPTIONS -l:: --ldlat:: - Configure mem-loads latency. Supported on Intel and Arm64 processors - only. Ignored on other archs. + Configure mem-loads latency. Supported on Intel, Arm64 and some AMD + processors. Ignored on other archs. + + On supported AMD processors: + - /sys/bus/event_source/devices/ibs_op/caps/ldlat file contains '1'. + - Supported latency values are 128 to 2048 (both inclusive). + - Latency value which is a multiple of 128 incurs a little less profiling + overhead compared to other values. + - Load latency filtering is disabled by default. -k:: --all-kernel:: diff --git a/tools/perf/Documentation/perf-mem.txt b/tools/perf/Documentation/perf-mem.txt index 19862572e3f2b64dcfc40f0fbf175baadfc284b4..117b90b91e1f5c7809b8d13c59062fd5d2e24544 100644 --- a/tools/perf/Documentation/perf-mem.txt +++ b/tools/perf/Documentation/perf-mem.txt @@ -28,6 +28,8 @@ and kernel support is required. See linkperf:perf-arm-spe[1] for a setup guide. Due to the statistical nature of SPE sampling, not every memory operation will be sampled. +On AMD this use IBS Op PMU to sample load-store operations. + OPTIONS ------- ...:: @@ -90,8 +92,15 @@ RECORD OPTIONS Be more verbose (show counter open errors, etc) --ldlat :: - Specify desired latency for loads event. Supported on Intel and Arm64 - processors only. Ignored on other archs. + Specify desired latency for loads event. Supported on Intel, Arm64 and + some AMD processors. Ignored on other archs. + + On supported AMD processors: + - /sys/bus/event_source/devices/ibs_op/caps/ldlat file contains '1'. + - Supported latency values are 128 to 2048 (both inclusive). + - Latency value which is a multiple of 128 incurs a little less profiling + overhead compared to other values. + - Load latency filtering is disabled by default. In addition, for report all perf report options are valid, and for record all perf record options. diff --git a/tools/perf/Documentation/perf.txt b/tools/perf/Documentation/perf.txt index ba3df49c169d329223f6d03fb6cab122e737b757..fa20f54e347bbe4d06bd79dd91d817586ba14b7c 100644 --- a/tools/perf/Documentation/perf.txt +++ b/tools/perf/Documentation/perf.txt @@ -77,7 +77,8 @@ linkperf:perf-stat[1], linkperf:perf-top[1], linkperf:perf-record[1], linkperf:perf-report[1], linkperf:perf-list[1] -linkperf:perf-annotate[1],linkperf:perf-archive[1],linkperf:perf-arm-spe[1], +linkperf:perf-amd-ibs[1], linkperf:perf-annotate[1], +linkperf:perf-archive[1], linkperf:perf-arm-spe[1], linkperf:perf-bench[1], linkperf:perf-buildid-cache[1], linkperf:perf-buildid-list[1], linkperf:perf-c2c[1], linkperf:perf-config[1], linkperf:perf-data[1], linkperf:perf-diff[1], diff --git a/tools/perf/arch/arm/util/pmu.c b/tools/perf/arch/arm/util/pmu.c index 7f3af3b97f3bac2a35faddc2a0a0ad180198a618..8b7cb68ba1a8a24ec86593570756304aa3fdfc43 100644 --- a/tools/perf/arch/arm/util/pmu.c +++ b/tools/perf/arch/arm/util/pmu.c @@ -13,6 +13,7 @@ #include "hisi-ptt.h" #include "../../../util/pmu.h" #include "../../../util/cs-etm.h" +#include "../../arm64/util/mem-events.h" void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) { @@ -26,6 +27,8 @@ void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) pmu->selectable = true; pmu->is_uncore = false; pmu->perf_event_attr_init_default = arm_spe_pmu_default_config; + if (!strcmp(pmu->name, "arm_spe_0")) + pmu->mem_events = perf_mem_events_arm; } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) { pmu->selectable = true; #endif diff --git a/tools/perf/arch/arm64/util/mem-events.c b/tools/perf/arch/arm64/util/mem-events.c index 3bcc5c7035c21429eb1f55ae8891ac642249f6e2..9f8da7937255cc9b14c3c58a1119b40bd0c76f6b 100644 --- a/tools/perf/arch/arm64/util/mem-events.c +++ b/tools/perf/arch/arm64/util/mem-events.c @@ -1,37 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -#include "map_symbol.h" +#include "util/map_symbol.h" +#include "util/mem-events.h" #include "mem-events.h" -#define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s } +#define E(t, n, s, l, a) { .tag = t, .name = n, .event_name = s, .ldlat = l, .aux_event = a } -static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { - E("spe-load", "arm_spe_0/ts_enable=1,pa_enable=1,load_filter=1,store_filter=0,min_latency=%u/", "arm_spe_0"), - E("spe-store", "arm_spe_0/ts_enable=1,pa_enable=1,load_filter=0,store_filter=1/", "arm_spe_0"), - E("spe-ldst", "arm_spe_0/ts_enable=1,pa_enable=1,load_filter=1,store_filter=1,min_latency=%u/", "arm_spe_0"), +struct perf_mem_event perf_mem_events_arm[PERF_MEM_EVENTS__MAX] = { + E("spe-load", "%s/ts_enable=1,pa_enable=1,load_filter=1,store_filter=0,min_latency=%u/", NULL, true, 0), + E("spe-store", "%s/ts_enable=1,pa_enable=1,load_filter=0,store_filter=1/", NULL, false, 0), + E("spe-ldst", "%s/ts_enable=1,pa_enable=1,load_filter=1,store_filter=1,min_latency=%u/", NULL, true, 0), }; - -static char mem_ev_name[100]; - -struct perf_mem_event *perf_mem_events__ptr(int i) -{ - if (i >= PERF_MEM_EVENTS__MAX) - return NULL; - - return &perf_mem_events[i]; -} - -const char *perf_mem_events__name(int i, const char *pmu_name __maybe_unused) -{ - struct perf_mem_event *e = perf_mem_events__ptr(i); - - if (i >= PERF_MEM_EVENTS__MAX) - return NULL; - - if (i == PERF_MEM_EVENTS__LOAD || i == PERF_MEM_EVENTS__LOAD_STORE) - scnprintf(mem_ev_name, sizeof(mem_ev_name), - e->name, perf_mem_events__loads_ldlat); - else /* PERF_MEM_EVENTS__STORE */ - scnprintf(mem_ev_name, sizeof(mem_ev_name), e->name); - - return mem_ev_name; -} diff --git a/tools/perf/arch/arm64/util/mem-events.h b/tools/perf/arch/arm64/util/mem-events.h new file mode 100644 index 0000000000000000000000000000000000000000..5fc50be4be3802fd647c50524f107d0f59a750c9 --- /dev/null +++ b/tools/perf/arch/arm64/util/mem-events.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ARM64_MEM_EVENTS_H +#define _ARM64_MEM_EVENTS_H + +extern struct perf_mem_event perf_mem_events_arm[PERF_MEM_EVENTS__MAX]; + +#endif /* _ARM64_MEM_EVENTS_H */ diff --git a/tools/perf/arch/powerpc/util/Build b/tools/perf/arch/powerpc/util/Build index 9889245c555c4cfb9730997d2ae5b1cdfbf1b3f9..1d323f3a3322bc394a22380c0cef41cd57905f6e 100644 --- a/tools/perf/arch/powerpc/util/Build +++ b/tools/perf/arch/powerpc/util/Build @@ -2,6 +2,7 @@ perf-y += header.o perf-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o perf-y += perf_regs.o perf-y += mem-events.o +perf-y += pmu.o perf-y += sym-handling.o perf-y += evsel.o perf-y += event.o diff --git a/tools/perf/arch/powerpc/util/mem-events.c b/tools/perf/arch/powerpc/util/mem-events.c index 78b986e5268d95ec7f80a9b2a91a0596041a4608..765d4a054b0a42362f34af60ede150e0eacdca28 100644 --- a/tools/perf/arch/powerpc/util/mem-events.c +++ b/tools/perf/arch/powerpc/util/mem-events.c @@ -1,12 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -#include "map_symbol.h" +#include "util/map_symbol.h" +#include "util/mem-events.h" #include "mem-events.h" -/* PowerPC does not support 'ldlat' parameter. */ -const char *perf_mem_events__name(int i, const char *pmu_name __maybe_unused) -{ - if (i == PERF_MEM_EVENTS__LOAD) - return "cpu/mem-loads/"; +#define E(t, n, s, l, a) { .tag = t, .name = n, .event_name = s, .ldlat = l, .aux_event = a } - return "cpu/mem-stores/"; -} +struct perf_mem_event perf_mem_events_power[PERF_MEM_EVENTS__MAX] = { + E("ldlat-loads", "%s/mem-loads/", "mem-loads", false, 0), + E("ldlat-stores", "%s/mem-stores/", "mem-stores", false, 0), + E(NULL, NULL, NULL, false, 0), +}; diff --git a/tools/perf/arch/powerpc/util/mem-events.h b/tools/perf/arch/powerpc/util/mem-events.h new file mode 100644 index 0000000000000000000000000000000000000000..6acc3d1b687318669d9834aa9d08788301d28e93 --- /dev/null +++ b/tools/perf/arch/powerpc/util/mem-events.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _POWER_MEM_EVENTS_H +#define _POWER_MEM_EVENTS_H + +extern struct perf_mem_event perf_mem_events_power[PERF_MEM_EVENTS__MAX]; + +#endif /* _POWER_MEM_EVENTS_H */ diff --git a/tools/perf/arch/powerpc/util/pmu.c b/tools/perf/arch/powerpc/util/pmu.c new file mode 100644 index 0000000000000000000000000000000000000000..554675deef7ba2890da2d413369d56f3c695462a --- /dev/null +++ b/tools/perf/arch/powerpc/util/pmu.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include "../../../util/pmu.h" +#include "mem-events.h" + +void perf_pmu__arch_init(struct perf_pmu *pmu) +{ + if (pmu->is_core) + pmu->mem_events = perf_mem_events_power; +} diff --git a/tools/perf/arch/x86/util/mem-events.c b/tools/perf/arch/x86/util/mem-events.c index f8d9aecbf2f28763b5361d48b03dba54022affa3..aee029421eaf534df147a211aad219ae91ab074a 100644 --- a/tools/perf/arch/x86/util/mem-events.c +++ b/tools/perf/arch/x86/util/mem-events.c @@ -7,38 +7,34 @@ #include "linux/string.h" #include "env.h" -static char mem_loads_name[100]; -static bool mem_loads_name__init; -static char mem_stores_name[100]; - #define MEM_LOADS_AUX 0x8203 -#define MEM_LOADS_AUX_NAME "{%s/mem-loads-aux/,%s/mem-loads,ldlat=%u/}:P" -#define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s } +#define E(t, n, s, l, a) { .tag = t, .name = n, .event_name = s, .ldlat = l, .aux_event = a } -static struct perf_mem_event perf_mem_events_intel[PERF_MEM_EVENTS__MAX] = { - E("ldlat-loads", "%s/mem-loads,ldlat=%u/P", "%s/events/mem-loads"), - E("ldlat-stores", "%s/mem-stores/P", "%s/events/mem-stores"), - E(NULL, NULL, NULL), +struct perf_mem_event perf_mem_events_intel[PERF_MEM_EVENTS__MAX] = { + E("ldlat-loads", "%s/mem-loads,ldlat=%u/P", "mem-loads", true, 0), + E("ldlat-stores", "%s/mem-stores/P", "mem-stores", false, 0), + E(NULL, NULL, NULL, false, 0), }; -static struct perf_mem_event perf_mem_events_amd[PERF_MEM_EVENTS__MAX] = { - E(NULL, NULL, NULL), - E(NULL, NULL, NULL), - E("mem-ldst", "ibs_op//", "ibs_op"), +struct perf_mem_event perf_mem_events_intel_aux[PERF_MEM_EVENTS__MAX] = { + E("ldlat-loads", "{%s/mem-loads-aux/,%s/mem-loads,ldlat=%u/}:P", "mem-loads", true, MEM_LOADS_AUX), + E("ldlat-stores", "%s/mem-stores/P", "mem-stores", false, 0), + E(NULL, NULL, NULL, false, 0), }; -struct perf_mem_event *perf_mem_events__ptr(int i) -{ - if (i >= PERF_MEM_EVENTS__MAX) - return NULL; - - if (x86__is_amd_cpu() || x86__is_hygon_cpu()) - return &perf_mem_events_amd[i]; +struct perf_mem_event perf_mem_events_amd[PERF_MEM_EVENTS__MAX] = { + E(NULL, NULL, NULL, false, 0), + E(NULL, NULL, NULL, false, 0), + E("mem-ldst", "%s//", NULL, false, 0), +}; - return &perf_mem_events_intel[i]; -} +struct perf_mem_event perf_mem_events_amd_ldlat[PERF_MEM_EVENTS__MAX] = { + E(NULL, NULL, NULL, false, 0), + E(NULL, NULL, NULL, false, 0), + E("mem-ldst", "%s/ldlat=%u/", NULL, true, 0), +}; bool is_mem_loads_aux_event(struct evsel *leader) { struct perf_pmu *pmu = perf_pmus__find("cpu"); @@ -51,43 +47,3 @@ bool is_mem_loads_aux_event(struct evsel *leader) return leader->core.attr.config == MEM_LOADS_AUX; } - -const char *perf_mem_events__name(int i, const char *pmu_name) -{ - struct perf_mem_event *e = perf_mem_events__ptr(i); - - if (!e) - return NULL; - - if (i == PERF_MEM_EVENTS__LOAD) { - if (mem_loads_name__init && !pmu_name) - return mem_loads_name; - - if (!pmu_name) { - mem_loads_name__init = true; - pmu_name = "cpu"; - } - - if (perf_pmus__have_event(pmu_name, "mem-loads-aux")) { - scnprintf(mem_loads_name, sizeof(mem_loads_name), - MEM_LOADS_AUX_NAME, pmu_name, pmu_name, - perf_mem_events__loads_ldlat); - } else { - scnprintf(mem_loads_name, sizeof(mem_loads_name), - e->name, pmu_name, - perf_mem_events__loads_ldlat); - } - return mem_loads_name; - } - - if (i == PERF_MEM_EVENTS__STORE) { - if (!pmu_name) - pmu_name = "cpu"; - - scnprintf(mem_stores_name, sizeof(mem_stores_name), - e->name, pmu_name); - return mem_stores_name; - } - - return e->name; -} diff --git a/tools/perf/arch/x86/util/mem-events.h b/tools/perf/arch/x86/util/mem-events.h new file mode 100644 index 0000000000000000000000000000000000000000..11e09a256f5bb084b8c272f8830f512037616bc4 --- /dev/null +++ b/tools/perf/arch/x86/util/mem-events.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_MEM_EVENTS_H +#define _X86_MEM_EVENTS_H + +extern struct perf_mem_event perf_mem_events_intel[PERF_MEM_EVENTS__MAX]; +extern struct perf_mem_event perf_mem_events_intel_aux[PERF_MEM_EVENTS__MAX]; + +extern struct perf_mem_event perf_mem_events_amd[PERF_MEM_EVENTS__MAX]; +extern struct perf_mem_event perf_mem_events_amd_ldlat[PERF_MEM_EVENTS__MAX]; + +#endif /* _X86_MEM_EVENTS_H */ diff --git a/tools/perf/arch/x86/util/pmu.c b/tools/perf/arch/x86/util/pmu.c index bd36c1c885b67aa043b396e2fbf784b765bfc735..c5c7a93b635a1a93b4e021ac95b2ea58f3f66f8a 100644 --- a/tools/perf/arch/x86/util/pmu.c +++ b/tools/perf/arch/x86/util/pmu.c @@ -15,10 +15,13 @@ #include "../../../util/pmu.h" #include "../../../util/fncache.h" #include "../../../util/pmus.h" +#include "mem-events.h" #include "env.h" -void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) +void perf_pmu__arch_init(struct perf_pmu *pmu) { + struct perf_pmu_caps *ldlat_cap; + #ifdef HAVE_AUXTRACE_SUPPORT if (!strcmp(pmu->name, INTEL_PT_PMU_NAME)) { pmu->auxtrace = true; @@ -30,6 +33,28 @@ void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) pmu->selectable = true; } #endif + + if (x86__is_amd_cpu()) { + if (strcmp(pmu->name, "ibs_op")) + return; + + pmu->mem_events = perf_mem_events_amd; + + if (!perf_pmu__caps_parse(pmu)) + return; + + ldlat_cap = perf_pmu__get_cap(pmu, "ldlat"); + if (!ldlat_cap || strcmp(ldlat_cap->value, "1")) + return; + + perf_mem_events__loads_ldlat = 0; + pmu->mem_events = perf_mem_events_amd_ldlat; + } else if (pmu->is_core) { + if (perf_pmu__have_event(pmu, "mem-loads-aux")) + pmu->mem_events = perf_mem_events_intel_aux; + else + pmu->mem_events = perf_mem_events_intel; + } } int perf_pmus__num_mem_pmus(void) diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index ef7ed53a4b4e2dc5da83ff6a458266d2acb9089f..5e136b4786843fae1e531397a30dcbbbb2ac2ad5 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -3211,12 +3211,19 @@ static int parse_record_events(const struct option *opt, const char *str, int unset __maybe_unused) { bool *event_set = (bool *) opt->value; + struct perf_pmu *pmu; + + pmu = perf_mem_events_find_pmu(); + if (!pmu) { + pr_err("failed: there is no PMU that supports perf c2c\n"); + exit(-1); + } if (!strcmp(str, "list")) { - perf_mem_events__list(); + perf_pmu__mem_events_list(pmu); exit(0); } - if (perf_mem_events__parse(str)) + if (perf_pmu__mem_events_parse(pmu, str)) exit(-1); *event_set = true; @@ -3241,6 +3248,7 @@ static int perf_c2c__record(int argc, const char **argv) bool all_user = false, all_kernel = false; bool event_set = false; struct perf_mem_event *e; + struct perf_pmu *pmu; struct option options[] = { OPT_CALLBACK('e', "event", &event_set, "event", "event selector. Use 'perf c2c record -e list' to list available events", @@ -3252,7 +3260,13 @@ static int perf_c2c__record(int argc, const char **argv) OPT_END() }; - if (perf_mem_events__init()) { + pmu = perf_mem_events_find_pmu(); + if (!pmu) { + pr_err("failed: no PMU supports the memory events\n"); + return -1; + } + + if (perf_pmu__mem_events_init(pmu)) { pr_err("failed: memory events not supported\n"); return -1; } @@ -3276,7 +3290,7 @@ static int perf_c2c__record(int argc, const char **argv) rec_argv[i++] = "record"; if (!event_set) { - e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD_STORE); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD_STORE); /* * The load and store operations are required, use the event * PERF_MEM_EVENTS__LOAD_STORE if it is supported. @@ -3285,15 +3299,15 @@ static int perf_c2c__record(int argc, const char **argv) e->record = true; rec_argv[i++] = "-W"; } else { - e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD); e->record = true; - e = perf_mem_events__ptr(PERF_MEM_EVENTS__STORE); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__STORE); e->record = true; } } - e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD); if (e->record) rec_argv[i++] = "-W"; diff --git a/tools/perf/builtin-mem.c b/tools/perf/builtin-mem.c index 286105be91cec9516fce1657a95dc6c9cd044032..a65d3981ff57dc5ddfa9de3551a8270204fa9e60 100644 --- a/tools/perf/builtin-mem.c +++ b/tools/perf/builtin-mem.c @@ -43,12 +43,19 @@ static int parse_record_events(const struct option *opt, const char *str, int unset __maybe_unused) { struct perf_mem *mem = *(struct perf_mem **)opt->value; + struct perf_pmu *pmu; + + pmu = perf_mem_events_find_pmu(); + if (!pmu) { + pr_err("failed: there is no PMU that supports perf mem\n"); + exit(-1); + } if (!strcmp(str, "list")) { - perf_mem_events__list(); + perf_pmu__mem_events_list(pmu); exit(0); } - if (perf_mem_events__parse(str)) + if (perf_pmu__mem_events_parse(pmu, str)) exit(-1); mem->operation = 0; @@ -72,6 +79,7 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) int ret; bool all_user = false, all_kernel = false; struct perf_mem_event *e; + struct perf_pmu *pmu; struct option options[] = { OPT_CALLBACK('e', "event", &mem, "event", "event selector. use 'perf mem record -e list' to list available events", @@ -84,7 +92,13 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) OPT_END() }; - if (perf_mem_events__init()) { + pmu = perf_mem_events_find_pmu(); + if (!pmu) { + pr_err("failed: no PMU supports the memory events\n"); + return -1; + } + + if (perf_pmu__mem_events_init(pmu)) { pr_err("failed: memory events not supported\n"); return -1; } @@ -113,7 +127,7 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) rec_argv[i++] = "record"; - e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD_STORE); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD_STORE); /* * The load and store operations are required, use the event @@ -126,17 +140,17 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) rec_argv[i++] = "-W"; } else { if (mem->operation & MEM_OPERATION_LOAD) { - e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD); e->record = true; } if (mem->operation & MEM_OPERATION_STORE) { - e = perf_mem_events__ptr(PERF_MEM_EVENTS__STORE); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__STORE); e->record = true; } } - e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD); + e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD); if (e->record) rec_argv[i++] = "-W"; diff --git a/tools/perf/tests/shell/test_data_symbol.sh b/tools/perf/tests/shell/test_data_symbol.sh index 3dfa91832aa87f89b8f0ef0c1ba51df6d130d2d2..b7d5ddc6b41ba80ecf2760bfd90414113f71a74d 100755 --- a/tools/perf/tests/shell/test_data_symbol.sh +++ b/tools/perf/tests/shell/test_data_symbol.sh @@ -55,11 +55,34 @@ trap cleanup_files exit term int echo "Recording workload..." -# perf mem/c2c internally uses IBS PMU on AMD CPU which doesn't support -# user/kernel filtering and per-process monitoring, spin program on -# specific CPU and test in per-CPU mode. is_amd=$(grep -E -c 'vendor_id.*AuthenticAMD' /proc/cpuinfo) if (($is_amd >= 1)); then + mem_events="$(perf mem record -v -e list 2>&1)" + if ! [[ "$mem_events" =~ ^mem\-ldst.*ibs_op/(.*)/.*available ]]; then + echo "ERROR: mem-ldst event is not matching" + exit 1 + fi + + # --ldlat on AMD: + # o Zen4 and earlier uarch does not support ldlat + # o Even on supported platforms, it's disabled (--ldlat=0) by default. + ldlat=${BASH_REMATCH[1]} + if [[ -n $ldlat ]]; then + if ! [[ "$ldlat" =~ ldlat=0 ]]; then + echo "ERROR: ldlat not initialized to 0?" + exit 1 + fi + + mem_events="$(perf mem record -v --ldlat=150 -e list 2>&1)" + if ! [[ "$mem_events" =~ ^mem-ldst.*ibs_op/ldlat=150/.*available ]]; then + echo "ERROR: --ldlat not honored?" + exit 1 + fi + fi + + # perf mem/c2c internally uses IBS PMU on AMD CPU which doesn't + # support user/kernel filtering and per-process monitoring on older + # kernels, spin program on specific CPU and test in per-CPU mode. perf mem record -vvv -o ${PERF_DATA} -C 0 -- taskset -c 0 $TEST_PROGRAM 2>"${ERR_FILE}" & else perf mem record -vvv --all-user -o ${PERF_DATA} -- $TEST_PROGRAM 2>"${ERR_FILE}" & diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index 9d0ce88e90e450c72804f0c5c3aacec5c5caa47d..022c9eb3950909226ea273a564137aa41460ed37 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -19,6 +19,8 @@ static u32 cpu_family, cpu_model, ibs_fetch_type, ibs_op_type; static bool zen4_ibs_extensions; +static bool ldlat_cap; +static bool dtlb_pgsize_cap; static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) { @@ -78,14 +80,20 @@ static void pr_ic_ibs_extd_ctl(union ic_ibs_extd_ctl reg) static void pr_ibs_op_ctl(union ibs_op_ctl reg) { char l3_miss_only[sizeof(" L3MissOnly _")] = ""; + char ldlat[sizeof(" LdLatThrsh __ LdLatEn _")] = ""; if (zen4_ibs_extensions) snprintf(l3_miss_only, sizeof(l3_miss_only), " L3MissOnly %d", reg.l3_miss_only); - printf("ibs_op_ctl:\t%016llx MaxCnt %9d%s En %d Val %d CntCtl %d=%s CurCnt %9d\n", + if (ldlat_cap) { + snprintf(ldlat, sizeof(ldlat), " LdLatThrsh %2d LdLatEn %d", + reg.ldlat_thrsh, reg.ldlat_en); + } + + printf("ibs_op_ctl:\t%016llx MaxCnt %9d%s En %d Val %d CntCtl %d=%s CurCnt %9d%s\n", reg.val, ((reg.opmaxcnt_ext << 16) | reg.opmaxcnt) << 4, l3_miss_only, reg.op_en, reg.op_val, reg.cnt_ctl, - reg.cnt_ctl ? "uOps" : "cycles", reg.opcurcnt); + reg.cnt_ctl ? "uOps" : "cycles", reg.opcurcnt, ldlat); } static void pr_ibs_op_data(union ibs_op_data reg) @@ -154,9 +162,20 @@ static void pr_ibs_op_data2(union ibs_op_data2 reg) static void pr_ibs_op_data3(union ibs_op_data3 reg) { - char l2_miss_str[sizeof(" L2Miss _")] = ""; - char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + static const char * const dc_page_sizes[] = { + " 4K", + " 2M", + " 1G", + " ??", + }; char op_dc_miss_open_mem_reqs_str[sizeof(" OpDcMissOpenMemReqs __")] = ""; + char dc_l1_l2tlb_miss_str[sizeof(" DcL1TlbMiss _ DcL2TlbMiss _")] = ""; + char dc_l1tlb_hit_str[sizeof(" DcL1TlbHit2M _ DcL1TlbHit1G _")] = ""; + char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + char dc_l2tlb_hit_2m_str[sizeof(" DcL2TlbHit2M _")] = ""; + char dc_l2tlb_hit_1g_str[sizeof(" DcL2TlbHit1G _")] = ""; + char dc_page_size_str[sizeof(" DcPageSize ____")] = ""; + char l2_miss_str[sizeof(" L2Miss _")] = ""; /* * Erratum #1293 @@ -172,16 +191,40 @@ static void pr_ibs_op_data3(union ibs_op_data3 reg) snprintf(op_mem_width_str, sizeof(op_mem_width_str), " OpMemWidth %2d bytes", 1 << (reg.op_mem_width - 1)); - printf("ibs_op_data3:\t%016llx LdOp %d StOp %d DcL1TlbMiss %d DcL2TlbMiss %d " - "DcL1TlbHit2M %d DcL1TlbHit1G %d DcL2TlbHit2M %d DcMiss %d DcMisAcc %d " - "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d DcLinAddrValid %d " - "DcPhyAddrValid %d DcL2TlbHit1G %d%s SwPf %d%s%s DcMissLat %5d TlbRefillLat %5d\n", - reg.val, reg.ld_op, reg.st_op, reg.dc_l1tlb_miss, reg.dc_l2tlb_miss, - reg.dc_l1tlb_hit_2m, reg.dc_l1tlb_hit_1g, reg.dc_l2tlb_hit_2m, reg.dc_miss, - reg.dc_mis_acc, reg.dc_wc_mem_acc, reg.dc_uc_mem_acc, reg.dc_locked_op, - reg.dc_miss_no_mab_alloc, reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, - reg.dc_l2_tlb_hit_1g, l2_miss_str, reg.sw_pf, op_mem_width_str, - op_dc_miss_open_mem_reqs_str, reg.dc_miss_lat, reg.tlb_refill_lat); + if (dtlb_pgsize_cap) { + if (reg.dc_phy_addr_valid) { + int idx = (reg.dc_l1tlb_hit_1g << 1) | reg.dc_l1tlb_hit_2m; + + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_page_size_str, sizeof(dc_page_size_str), + " DcPageSize %4s", dc_page_sizes[idx]); + } + } else { + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_l1tlb_hit_str, sizeof(dc_l1tlb_hit_str), + " DcL1TlbHit2M %d DcL1TlbHit1G %d", + reg.dc_l1tlb_hit_2m, reg.dc_l1tlb_hit_1g); + snprintf(dc_l2tlb_hit_2m_str, sizeof(dc_l2tlb_hit_2m_str), + " DcL2TlbHit2M %d", reg.dc_l2tlb_hit_2m); + snprintf(dc_l2tlb_hit_1g_str, sizeof(dc_l2tlb_hit_1g_str), + " DcL2TlbHit1G %d", reg.dc_l2_tlb_hit_1g); + } + + printf("ibs_op_data3:\t%016llx LdOp %d StOp %d%s%s%s DcMiss %d DcMisAcc %d " + "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d " + "DcLinAddrValid %d DcPhyAddrValid %d%s%s SwPf %d%s%s " + "DcMissLat %5d TlbRefillLat %5d\n", + reg.val, reg.ld_op, reg.st_op, dc_l1_l2tlb_miss_str, + dtlb_pgsize_cap ? dc_page_size_str : dc_l1tlb_hit_str, + dc_l2tlb_hit_2m_str, reg.dc_miss, reg.dc_mis_acc, reg.dc_wc_mem_acc, + reg.dc_uc_mem_acc, reg.dc_locked_op, reg.dc_miss_no_mab_alloc, + reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, dc_l2tlb_hit_1g_str, + l2_miss_str, reg.sw_pf, op_mem_width_str, op_dc_miss_open_mem_reqs_str, + reg.dc_miss_lat, reg.tlb_refill_lat); } /* @@ -331,6 +374,12 @@ bool evlist__has_amd_ibs(struct evlist *evlist) if (perf_env__find_pmu_cap(env, "ibs_op", "zen4_ibs_extensions")) zen4_ibs_extensions = 1; + if (perf_env__find_pmu_cap(env, "ibs_op", "ldlat")) + ldlat_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize")) + dtlb_pgsize_cap = 1; + if (ibs_fetch_type || ibs_op_type) { if (!cpu_family) parse_cpuid(env); diff --git a/tools/perf/util/mem-events.c b/tools/perf/util/mem-events.c index 3a2e3687878c1862c64d0f723496a76ceb2f8229..32890848bb3dfc4f682329695b3fdd19b34cb1a6 100644 --- a/tools/perf/util/mem-events.c +++ b/tools/perf/util/mem-events.c @@ -17,43 +17,94 @@ unsigned int perf_mem_events__loads_ldlat = 30; -#define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s } +#define E(t, n, s, l, a) { .tag = t, .name = n, .event_name = s, .ldlat = l, .aux_event = a } -static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { - E("ldlat-loads", "cpu/mem-loads,ldlat=%u/P", "cpu/events/mem-loads"), - E("ldlat-stores", "cpu/mem-stores/P", "cpu/events/mem-stores"), - E(NULL, NULL, NULL), +struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { + E("ldlat-loads", "%s/mem-loads,ldlat=%u/P", "mem-loads", true, 0), + E("ldlat-stores", "%s/mem-stores/P", "mem-stores", false, 0), + E(NULL, NULL, NULL, false, 0), }; #undef E static char mem_loads_name[100]; -static bool mem_loads_name__init; +static char mem_stores_name[100]; -struct perf_mem_event * __weak perf_mem_events__ptr(int i) +struct perf_mem_event *perf_pmu__mem_events_ptr(struct perf_pmu *pmu, int i) { - if (i >= PERF_MEM_EVENTS__MAX) + if (i >= PERF_MEM_EVENTS__MAX || !pmu) return NULL; - return &perf_mem_events[i]; + return &pmu->mem_events[i]; } -const char * __weak perf_mem_events__name(int i, const char *pmu_name __maybe_unused) +static struct perf_pmu *perf_pmus__scan_mem(struct perf_pmu *pmu) { - struct perf_mem_event *e = perf_mem_events__ptr(i); + while ((pmu = perf_pmus__scan(pmu)) != NULL) { + if (pmu->mem_events) + return pmu; + } + return NULL; +} + +struct perf_pmu *perf_mem_events_find_pmu(void) +{ + /* + * The current perf mem doesn't support per-PMU configuration. + * The exact same configuration is applied to all the + * mem_events supported PMUs. + * Return the first mem_events supported PMU. + * + * Notes: The only case which may support multiple mem_events + * supported PMUs is Intel hybrid. The exact same mem_events + * is shared among the PMUs. Only configure the first PMU + * is good enough as well. + */ + return perf_pmus__scan_mem(NULL); +} + +static const char *perf_pmu__mem_events_name(int i, struct perf_pmu *pmu) +{ + struct perf_mem_event *e; + + if (i >= PERF_MEM_EVENTS__MAX || !pmu) + return NULL; + e = &pmu->mem_events[i]; if (!e) return NULL; - if (i == PERF_MEM_EVENTS__LOAD) { - if (!mem_loads_name__init) { - mem_loads_name__init = true; - scnprintf(mem_loads_name, sizeof(mem_loads_name), - e->name, perf_mem_events__loads_ldlat); + if (i == PERF_MEM_EVENTS__LOAD || i == PERF_MEM_EVENTS__LOAD_STORE) { + if (e->ldlat) { + if (!e->aux_event) { + /* ARM and Most of Intel */ + scnprintf(mem_loads_name, sizeof(mem_loads_name), + e->name, pmu->name, + perf_mem_events__loads_ldlat); + } else { + /* Intel with mem-loads-aux event */ + scnprintf(mem_loads_name, sizeof(mem_loads_name), + e->name, pmu->name, pmu->name, + perf_mem_events__loads_ldlat); + } + } else { + if (!e->aux_event) { + /* AMD and POWER */ + scnprintf(mem_loads_name, sizeof(mem_loads_name), + e->name, pmu->name); + } else + return NULL; } + return mem_loads_name; } - return e->name; + if (i == PERF_MEM_EVENTS__STORE) { + scnprintf(mem_stores_name, sizeof(mem_stores_name), + e->name, pmu->name); + return mem_stores_name; + } + + return NULL; } __weak bool is_mem_loads_aux_event(struct evsel *leader __maybe_unused) @@ -61,7 +112,7 @@ __weak bool is_mem_loads_aux_event(struct evsel *leader __maybe_unused) return false; } -int perf_mem_events__parse(const char *str) +int perf_pmu__mem_events_parse(struct perf_pmu *pmu, const char *str) { char *tok, *saveptr = NULL; bool found = false; @@ -79,7 +130,7 @@ int perf_mem_events__parse(const char *str) while (tok) { for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { - struct perf_mem_event *e = perf_mem_events__ptr(j); + struct perf_mem_event *e = perf_pmu__mem_events_ptr(pmu, j); if (!e->tag) continue; @@ -100,19 +151,21 @@ int perf_mem_events__parse(const char *str) return -1; } -static bool perf_mem_event__supported(const char *mnt, struct perf_pmu *pmu, +static bool perf_pmu__mem_events_supported(const char *mnt, struct perf_pmu *pmu, struct perf_mem_event *e) { - char sysfs_name[100]; char path[PATH_MAX]; struct stat st; - scnprintf(sysfs_name, sizeof(sysfs_name), e->sysfs_name, pmu->name); - scnprintf(path, PATH_MAX, "%s/devices/%s", mnt, sysfs_name); + if (!e->event_name) + return true; + + scnprintf(path, PATH_MAX, "%s/devices/%s/events/%s", mnt, pmu->name, e->event_name); + return !stat(path, &st); } -int perf_mem_events__init(void) +int perf_pmu__mem_events_init(struct perf_pmu *pmu) { const char *mnt = sysfs__mount(); bool found = false; @@ -122,8 +175,7 @@ int perf_mem_events__init(void) return -ENOENT; for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { - struct perf_mem_event *e = perf_mem_events__ptr(j); - struct perf_pmu *pmu = NULL; + struct perf_mem_event *e = perf_pmu__mem_events_ptr(pmu, j); /* * If the event entry isn't valid, skip initialization @@ -132,86 +184,59 @@ int perf_mem_events__init(void) if (!e->tag) continue; - /* - * Scan all PMUs not just core ones, since perf mem/c2c on - * platforms like AMD uses IBS OP PMU which is independent - * of core PMU. - */ - while ((pmu = perf_pmus__scan(pmu)) != NULL) { - e->supported |= perf_mem_event__supported(mnt, pmu, e); - if (e->supported) { - found = true; - break; - } - } + e->supported |= perf_pmu__mem_events_supported(mnt, pmu, e); + if (e->supported) + found = true; } return found ? 0 : -ENOENT; } -void perf_mem_events__list(void) +void perf_pmu__mem_events_list(struct perf_pmu *pmu) { int j; for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { - struct perf_mem_event *e = perf_mem_events__ptr(j); + struct perf_mem_event *e = perf_pmu__mem_events_ptr(pmu, j); fprintf(stderr, "%-*s%-*s%s", e->tag ? 13 : 0, e->tag ? : "", e->tag && verbose > 0 ? 25 : 0, - e->tag && verbose > 0 ? perf_mem_events__name(j, NULL) : "", + e->tag && verbose > 0 ? perf_pmu__mem_events_name(j, pmu) : "", e->supported ? ": available\n" : ""); } } -static void perf_mem_events__print_unsupport_hybrid(struct perf_mem_event *e, - int idx) -{ - const char *mnt = sysfs__mount(); - struct perf_pmu *pmu = NULL; - - while ((pmu = perf_pmus__scan(pmu)) != NULL) { - if (!perf_mem_event__supported(mnt, pmu, e)) { - pr_err("failed: event '%s' not supported\n", - perf_mem_events__name(idx, pmu->name)); - } - } -} - int perf_mem_events__record_args(const char **rec_argv, int *argv_nr, char **rec_tmp, int *tmp_nr) { const char *mnt = sysfs__mount(); + struct perf_pmu *pmu = NULL; int i = *argv_nr, k = 0; struct perf_mem_event *e; - for (int j = 0; j < PERF_MEM_EVENTS__MAX; j++) { - e = perf_mem_events__ptr(j); - if (!e->record) - continue; - if (perf_pmus__num_mem_pmus() == 1) { - if (!e->supported) { - pr_err("failed: event '%s' not supported\n", - perf_mem_events__name(j, NULL)); - return -1; - } + while ((pmu = perf_pmus__scan_mem(pmu)) != NULL) { + for (int j = 0; j < PERF_MEM_EVENTS__MAX; j++) { + e = perf_pmu__mem_events_ptr(pmu, j); - rec_argv[i++] = "-e"; - rec_argv[i++] = perf_mem_events__name(j, NULL); - } else { - struct perf_pmu *pmu = NULL; + if (!e->record) + continue; if (!e->supported) { - perf_mem_events__print_unsupport_hybrid(e, j); + pr_err("failed: event '%s' not supported\n", + perf_pmu__mem_events_name(j, pmu)); return -1; } - while ((pmu = perf_pmus__scan(pmu)) != NULL) { - const char *s = perf_mem_events__name(j, pmu->name); + if (perf_pmus__num_mem_pmus() == 1) { + rec_argv[i++] = "-e"; + rec_argv[i++] = perf_pmu__mem_events_name(j, pmu); + } else { + const char *s = perf_pmu__mem_events_name(j, pmu); - if (!perf_mem_event__supported(mnt, pmu, e)) + if (!perf_pmu__mem_events_supported(mnt, pmu, e)) continue; rec_argv[i++] = "-e"; diff --git a/tools/perf/util/mem-events.h b/tools/perf/util/mem-events.h index b40ad6ea93fcdb70d0111b20abb8cbab475ec120..f817a507b106ae8653c7109f8a94b89eae960d3b 100644 --- a/tools/perf/util/mem-events.h +++ b/tools/perf/util/mem-events.h @@ -14,9 +14,11 @@ struct perf_mem_event { bool record; bool supported; + bool ldlat; + u32 aux_event; const char *tag; const char *name; - const char *sysfs_name; + const char *event_name; }; struct mem_info { @@ -34,15 +36,16 @@ enum { }; extern unsigned int perf_mem_events__loads_ldlat; +extern struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX]; -int perf_mem_events__parse(const char *str); -int perf_mem_events__init(void); +int perf_pmu__mem_events_parse(struct perf_pmu *pmu, const char *str); +int perf_pmu__mem_events_init(struct perf_pmu *pmu); -const char *perf_mem_events__name(int i, const char *pmu_name); -struct perf_mem_event *perf_mem_events__ptr(int i); +struct perf_mem_event *perf_pmu__mem_events_ptr(struct perf_pmu *pmu, int i); +struct perf_pmu *perf_mem_events_find_pmu(void); bool is_mem_loads_aux_event(struct evsel *leader); -void perf_mem_events__list(void); +void perf_pmu__mem_events_list(struct perf_pmu *pmu); int perf_mem_events__record_args(const char **rec_argv, int *argv_nr, char **rec_tmp, int *tmp_nr); diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 5edfeac95c444c5ae6e9667d4bf15516c85df202..15eac86f15d13bdb85490e1e32348bd54929a745 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -1032,8 +1032,10 @@ static int pmu_max_precise(int dirfd, struct perf_pmu *pmu) } void __weak -perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) +perf_pmu__arch_init(struct perf_pmu *pmu) { + if (pmu->is_core) + pmu->mem_events = perf_mem_events; } struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char *name) @@ -1939,6 +1941,17 @@ static void perf_pmu__del_caps(struct perf_pmu *pmu) } } +struct perf_pmu_caps *perf_pmu__get_cap(struct perf_pmu *pmu, const char *name) +{ + struct perf_pmu_caps *caps; + + list_for_each_entry(caps, &pmu->caps, list) { + if (!strcmp(caps->name, name)) + return caps; + } + return NULL; +} + /* * Reading/parsing the given pmu capabilities, which should be located at: * /sys/bus/event_source/devices//caps as sysfs group attributes. diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 1765c35080d4db4d08d83bb19a74eb7f04dc802e..9e85d58ad224988e02f968e4db9335f8413e821c 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -10,6 +10,8 @@ #include #include "parse-events.h" #include "pmu-events/pmu-events.h" +#include "map_symbol.h" +#include "mem-events.h" struct evsel_config_term; struct perf_cpu_map; @@ -169,6 +171,11 @@ struct perf_pmu { */ bool exclude_guest; } missing_features; + + /** + * @mem_events: List of the supported mem events + */ + struct perf_mem_event *mem_events; }; /** @perf_pmu__fake: A special global PMU used for testing. */ @@ -254,6 +261,8 @@ bool pmu_uncore_identifier_match(const char *compat, const char *id); int perf_pmu__convert_scale(const char *scale, char **end, double *sval); +struct perf_pmu_caps *perf_pmu__get_cap(struct perf_pmu *pmu, const char *name); + int perf_pmu__caps_parse(struct perf_pmu *pmu); void perf_pmu__warn_invalid_config(struct perf_pmu *pmu, __u64 config,