From 849ebc905bf9c994ccfcec379a51ff51479f1215 Mon Sep 17 00:00:00 2001 From: Sean Khan Date: Mon, 8 Apr 2024 23:11:37 -0400 Subject: [PATCH] qualcommax: enhance smp_affinity (log, uci, syntax) 1.) Changed logic of `set_affinity` to now use physical cores rather than knowing the bitmask. Rather than having to know a bitmask, users can provide a numerical instance of one or more CPU cores (numbered 0-63). This is done via function `cpus_to_bitmask`. Functions Added: a.) bitmask_to_cpus - Takes a bitmask of CPUs and returns a list of CPU numbers. (i.e. `bitmask_to_cpus "f"` -> 0,1,2,3) b.) cpus_to_bitmask - Takes a comma/space or range list of CPUs and returns a bitmask. Example: `cpus_to_bitmask "2,3"` -> c `cpus_to_bitmask "0,1,2,3"` -> f `cpus_to_bitmask "1,3"` -> a With or without quotes `cpus_to_bitmask "1 3"` -> a `cpus_to_bitmask 1 3` -> a 2.) Added UCI options: enable - to enable/disable the script from running. [default 1 (on)] enable_log - to enable/disable logging output to `logger`.[default 1 (on)] Log output: ``` Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(87) reo2host-destination-ring1 to CPU 0 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(88) reo2host-destination-ring2 to CPU 1 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(89) reo2host-destination-ring3 to CPU 2 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(90) reo2host-destination-ring4 to CPU 3 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(79) wbm2host-tx-completions-ring1 to CPU 1 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(83) wbm2host-tx-completions-ring2 to CPU 2 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(85) wbm2host-tx-completions-ring3 to CPU 3 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(73) ppdu-end-interrupts-mac1 to CPU 1 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(77) ppdu-end-interrupts-mac2 to CPU 2 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(75) ppdu-end-interrupts-mac3 to CPU 3 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(32) edma_txcmpl to CPU 3 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(33) edma_rxfill to CPU 3 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(35) edma_rxdesc to CPU 3 Mon Apr 8 23:00:01 2024 user.notice smp_affinity: Pinning IRQ(36) edma_misc to CPU 3 ``` Output of `/proc/interrupts`: ``` 69: 0 0 0 0 GIC-0 209 Edge rxdma2host-destination-ring-mac1 70: 0 0 0 0 GIC-0 211 Edge rxdma2host-destination-ring-mac3 71: 0 0 0 0 GIC-0 210 Edge rxdma2host-destination-ring-mac2 72: 2435 0 0 0 GIC-0 321 Edge reo2host-status 73: 268427 8011 0 0 GIC-0 261 Edge ppdu-end-interrupts-mac1 74: 2 0 0 0 GIC-0 255 Edge rxdma2host-monitor-status-ring-mac1 75: 176169 0 4 10035 GIC-0 263 Edge ppdu-end-interrupts-mac3 76: 2 0 0 0 GIC-0 260 Edge rxdma2host-monitor-status-ring-mac3 77: 0 0 0 0 GIC-0 262 Edge ppdu-end-interrupts-mac2 78: 0 0 0 0 GIC-0 256 Edge rxdma2host-monitor-status-ring-mac2 79: 3428 3123 0 0 GIC-0 189 Edge wbm2host-tx-completions-ring1 80: 0 0 0 0 GIC-0 323 Edge reo2ost-exception 81: 178 0 0 0 GIC-0 322 Edge wbm2host-rx-release 82: 0 0 0 0 GIC-0 212 Edge host2rxdma-host-buf-ring-mac1 83: 6524 0 13712 0 GIC-0 190 Edge wbm2host-tx-completions-ring2 84: 4 0 0 0 GIC-0 235 Edge host2rxdma-host-buf-ring-mac3 85: 560 0 0 1979 GIC-0 191 Edge wbm2host-tx-completions-ring3 86: 0 0 0 0 GIC-0 215 Edge host2rxdma-host-buf-ring-mac2 87: 4520 0 0 0 GIC-0 267 Edge reo2host-destination-ring1 88: 2231 2811 0 0 GIC-0 268 Edge reo2host-destination-ring2 89: 2180 0 2512 0 GIC-0 271 Edge reo2host-destination-ring3 90: 1990 0 0 2321 GIC-0 320 Edge reo2host-destination-ring4 ``` 3.) Added `uci-defaults` script `15_smp_affinity` to configure defaults options on first boot. Signed-off-by: Sean Khan --- .../base-files/etc/init.d/smp_affinity | 132 +++++++++++++++--- .../etc/uci-defaults/15_smp_affinity | 11 ++ 2 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 target/linux/qualcommax/ipq807x/base-files/etc/uci-defaults/15_smp_affinity diff --git a/target/linux/qualcommax/ipq807x/base-files/etc/init.d/smp_affinity b/target/linux/qualcommax/ipq807x/base-files/etc/init.d/smp_affinity index 05d13fff63..ecf88aea09 100755 --- a/target/linux/qualcommax/ipq807x/base-files/etc/init.d/smp_affinity +++ b/target/linux/qualcommax/ipq807x/base-files/etc/init.d/smp_affinity @@ -1,36 +1,122 @@ #!/bin/sh /etc/rc.common +###################################################################### +# vim: set ft=bash +# shellcheck disable=2155,3019,3043,3057,3060 +###################################################################### START=93 +PROG=smp_affinity + +log_msg() { + + local irq_name="$1" affinity="$2" irq="$3" + + msg="$(printf "Pinning IRQ($irq) %-24s to CPU ${affinity}\n" "$irq_name")" + + logger -t "$PROG" "$msg" +} + +###################################################################### +### Takes a comma, space separated, or range list of CPU numbers and +## returns a bitmask of CPUs. +## cpus_to_bitmask "0,1,2,3" -> f +## cpus_to_bitmask "0 1 2 3" -> f +## cpus_to_bitmask "0-3" -> f +## cpus_to_bitmask "3" -> 8 +####################################################################### + +cpus_to_bitmask() { + + local bitmask=0 + # shellcheck disable=2048 + for range in ${*//,/ }; do + start="${range%-*}" + end="${range#*-}" + if [ -z "$end" ]; then + bitmask="$((bitmask | 1 << start))" + else + bitmask="$((bitmask | (2 ** (end - start + 1) - 1) << start))" + fi + done + printf '%x' $bitmask +} + +###################################################################### +### Takes a bitmask of CPUs and returns a space separated list of +## CPU numbers. +## bitmask_to_cpus f -> 0 1 2 3 +###################################################################### + +bitmask_to_cpus() { + + [ "${1:0:2}" != "0x" ] && set -- "0x$1" + local bitmask="$(printf '%d' "$1")" + + local cpus="" + for i in $(seq 0 63); do + if [ $((bitmask & 1)) -ne 0 ]; then + cpus="$cpus $i" + fi + bitmask=$((bitmask >> 1)) + done + echo "${cpus# }" +} + +###################################################################### +### Sets the affinity of the IRQs with the given name to the given CPU. +## 1st argument: IRQ name ("reo2host-destination-ring1") (req) +## 2nd argument: CPU number (req) +###################################################################### + +set_affinity() { + + local irq_name="$1" affinity="$2" bitmask irq + awk -v irq_name="$1" '$0 ~ irq_name { print substr($1, 1, length($1)-1); exit }' /proc/interrupts \ + | while read -r irq; do + $enable_log && { + log_msg "$irq_name" "$affinity" "$irq" + } + bitmask=$(cpus_to_bitmask "$affinity") && echo "$bitmask" > "/proc/irq/$irq/smp_affinity" + done +} + enable_affinity_ipq807x() { - set_affinity() { - irq=$(awk "/$1/{ print substr(\$1, 1, length(\$1)-1); exit }" /proc/interrupts) - [ -n "$irq" ] && echo $2 > /proc/irq/$irq/smp_affinity - } - # assign 4 rx interrupts to each core - set_affinity 'reo2host-destination-ring1' 1 - set_affinity 'reo2host-destination-ring2' 2 - set_affinity 'reo2host-destination-ring3' 4 - set_affinity 'reo2host-destination-ring4' 8 + # assign 4 rx interrupts to each core + set_affinity 'reo2host-destination-ring1' 0 + set_affinity 'reo2host-destination-ring2' 1 + set_affinity 'reo2host-destination-ring3' 2 + set_affinity 'reo2host-destination-ring4' 3 - # assign 3 tcl completions to last 3 CPUs - set_affinity 'wbm2host-tx-completions-ring1' 2 - set_affinity 'wbm2host-tx-completions-ring2' 4 - set_affinity 'wbm2host-tx-completions-ring3' 8 + # assign 3 tcl completions to last 3 CPUs + set_affinity 'wbm2host-tx-completions-ring1' 1 + set_affinity 'wbm2host-tx-completions-ring2' 2 + set_affinity 'wbm2host-tx-completions-ring3' 3 - # assign 3 ppdu mac interrupts to last 3 cores - set_affinity 'ppdu-end-interrupts-mac1' 2 - set_affinity 'ppdu-end-interrupts-mac2' 4 - set_affinity 'ppdu-end-interrupts-mac3' 8 + # assign 3 ppdu mac interrupts to last 3 cores + set_affinity 'ppdu-end-interrupts-mac1' 1 + set_affinity 'ppdu-end-interrupts-mac2' 2 + set_affinity 'ppdu-end-interrupts-mac3' 3 - # assign lan/wan to core 4 - set_affinity 'edma_txcmpl' 8 - set_affinity 'edma_rxfill' 8 - set_affinity 'edma_rxdesc' 8 - set_affinity 'edma_misc' 8 + # assign 4 lan/wan to core 4 + set_affinity 'edma_txcmpl' 3 + set_affinity 'edma_rxfill' 3 + set_affinity 'edma_rxdesc' 3 + set_affinity 'edma_misc' 3 } boot() { - enable_affinity_ipq807x + + local enable + + config_load smp_affinity + + config_get_bool enable "general" enable 1 + config_get_bool enable_log "general" enable_log 1 + + [ "$enable" -eq 1 ] && enable=true || enable=false + [ "$enable_log" -eq 1 ] && enable_log=true || enable_log=false + + $enable && enable_affinity_ipq807x } diff --git a/target/linux/qualcommax/ipq807x/base-files/etc/uci-defaults/15_smp_affinity b/target/linux/qualcommax/ipq807x/base-files/etc/uci-defaults/15_smp_affinity new file mode 100644 index 0000000000..3ec0adc44c --- /dev/null +++ b/target/linux/qualcommax/ipq807x/base-files/etc/uci-defaults/15_smp_affinity @@ -0,0 +1,11 @@ +#!/bin/sh + +uci -q get smp_affinity && exit 0 +touch /etc/config/smp_affinity + +uci -q batch << EOF + set smp_affinity.general=smp_affinity + set smp_affinity.general.enable='1' + set smp_affinity.general.enable_log='1' + commit smp_affinity +EOF