1# SPDX-License-Identifier: CDDL-1.0 2# 3# This file and its contents are supplied under the terms of the 4# Common Development and Distribution License ("CDDL"), version 1.0. 5# You may only use this file in accordance with the terms of version 6# 1.0 of the CDDL. 7# 8# A full copy of the text of the CDDL should have accompanied this 9# source. A copy of the CDDL is also available via the Internet at 10# https://opensource.org/license/CDDL-1.0. 11# 12 13# 14# Copyright (c) 2025, Klara, Inc. 15# 16 17# 18# This file provides the following helpers to read kstats from tests. 19# 20# kstat [-g] <stat> 21# kstat_pool [-g] <pool> <stat> 22# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 23# 24# `kstat` and `kstat_pool` return the value of of the given <stat>, either 25# a global or pool-specific state. 26# 27# $ kstat dbgmsg 28# timestamp message 29# 1736848201 spa_history.c:304:spa_history_log_sync(): txg 14734896 ... 30# 1736848201 spa_history.c:330:spa_history_log_sync(): ioctl ... 31# ... 32# 33# $ kstat_pool garden state 34# ONLINE 35# 36# To get a single stat within a group or collection, separate the name with 37# '.' characters. 38# 39# $ kstat dbufstats.cache_target_bytes 40# 3215780693 41# 42# $ kstat_pool crayon iostats.arc_read_bytes 43# 253671670784 44# 45# -g is "group" mode. If the kstat is a group or collection, all stats in that 46# group are returned, one stat per line, key and value separated by a space. 47# 48# $ kstat -g dbufstats 49# cache_count 1792 50# cache_size_bytes 87720376 51# cache_size_bytes_max 305187768 52# cache_target_bytes 97668555 53# ... 54# 55# $ kstat_pool -g crayon iostats 56# trim_extents_written 0 57# trim_bytes_written 0 58# trim_extents_skipped 0 59# trim_bytes_skipped 0 60# ... 61# 62# `kstat_dataset` accesses the per-dataset group kstat. The dataset can be 63# specified by name: 64# 65# $ kstat_dataset crayon/home/robn nunlinks 66# 2628514 67# 68# or, with the -N switch, as <pool>/<objsetID>: 69# 70# $ kstat_dataset -N crayon/7 writes 71# 125135 72# 73 74#################### 75# Public interface 76 77# 78# kstat [-g] <stat> 79# 80function kstat 81{ 82 typeset -i want_group=0 83 84 OPTIND=1 85 while getopts "g" opt ; do 86 case $opt in 87 'g') want_group=1 ;; 88 *) log_fail "kstat: invalid option '$opt'" ;; 89 esac 90 done 91 shift $(expr $OPTIND - 1) 92 93 typeset stat=$1 94 95 $_kstat_os 'global' '' "$stat" $want_group 96} 97 98# 99# kstat_pool [-g] <pool> <stat> 100# 101function kstat_pool 102{ 103 typeset -i want_group=0 104 105 OPTIND=1 106 while getopts "g" opt ; do 107 case $opt in 108 'g') want_group=1 ;; 109 *) log_fail "kstat_pool: invalid option '$opt'" ;; 110 esac 111 done 112 shift $(expr $OPTIND - 1) 113 114 typeset pool=$1 115 typeset stat=$2 116 117 $_kstat_os 'pool' "$pool" "$stat" $want_group 118} 119 120# 121# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 122# 123function kstat_dataset 124{ 125 typeset -i opt_objsetid=0 126 127 OPTIND=1 128 while getopts "N" opt ; do 129 case $opt in 130 'N') opt_objsetid=1 ;; 131 *) log_fail "kstat_dataset: invalid option '$opt'" ;; 132 esac 133 done 134 shift $(expr $OPTIND - 1) 135 136 typeset dsarg=$1 137 typeset stat=$2 138 139 if [[ $opt_objsetid == 0 ]] ; then 140 typeset pool="${dsarg%%/*}" # clear first / -> end 141 typeset objsetid=$($_resolve_dsname_os "$pool" "$dsarg") 142 if [[ -z "$objsetid" ]] ; then 143 log_fail "kstat_dataset: dataset not found: $dsarg" 144 fi 145 dsarg="$pool/$objsetid" 146 fi 147 148 $_kstat_os 'dataset' "$dsarg" "$stat" 0 149} 150 151#################### 152# Platform-specific interface 153 154# 155# Implementation notes 156# 157# There's not a lot of uniformity between platforms, so I've written to a rough 158# imagined model that seems to fit the majority of OpenZFS kstats. 159# 160# The main platform entry points look like this: 161# 162# _kstat_freebsd <scope> <object> <stat> <want_group> 163# _kstat_linux <scope> <object> <stat> <want_group> 164# 165# - scope: one of 'global', 'pool', 'dataset'. The "kind" of object the kstat 166# is attached to. 167# - object: name of the scoped object 168# global: empty string 169# pool: pool name 170# dataset: <pool>/<objsetId> pair 171# - stat: kstat name to get 172# - want_group: 0 to get the single value for the kstat, 1 to treat the kstat 173# as a group and get all the stat names+values under it. group 174# kstats cannot have values, and stat kstats cannot have 175# children (by definition) 176# 177# Stat values can have multiple lines, so be prepared for those. 178# 179# These functions either succeed and produce the requested output, or call 180# log_fail. They should never output empty, or 0, or anything else. 181# 182# Output: 183# 184# - want_group=0: the single stat value, followed by newline 185# - want_group=1: One stat per line, <name><SP><value><newline> 186# 187 188# 189# To support kstat_dataset(), platforms also need to provide a dataset 190# name->object id resolver function. 191# 192# _resolve_dsname_freebsd <pool> <dsname> 193# _resolve_dsname_linux <pool> <dsname> 194# 195# - pool: pool name. always the first part of the dataset name 196# - dsname: dataset name, in the standard <pool>/<some>/<dataset> format. 197# 198# Output is <objsetID>. objsetID is a decimal integer, > 0 199# 200 201#################### 202# FreeBSD 203 204# 205# All kstats are accessed through sysctl. We model "groups" as interior nodes 206# in the stat tree, which are normally opaque. Because sysctl has no filtering 207# options, and requesting any node produces all nodes below it, we have to 208# always get the name and value, and then consider the output to understand 209# if we got a group or a single stat, and post-process accordingly. 210# 211# Scopes are mostly mapped directly to known locations in the tree, but there 212# are a handful of stats that are out of position, so we need to adjust. 213# 214 215# 216# _kstat_freebsd <scope> <object> <stat> <want_group> 217# 218function _kstat_freebsd 219{ 220 typeset scope=$1 221 typeset obj=$2 222 typeset stat=$3 223 typeset -i want_group=$4 224 225 typeset oid="" 226 case "$scope" in 227 global) 228 oid="kstat.zfs.misc.$stat" 229 ;; 230 pool) 231 # For reasons unknown, the "multihost", "txgs" and "reads" 232 # pool-specific kstats are directly under kstat.zfs.<pool>, 233 # rather than kstat.zfs.<pool>.misc like the other pool kstats. 234 # Adjust for that here. 235 case "$stat" in 236 multihost|txgs|reads) 237 oid="kstat.zfs.$obj.$stat" 238 ;; 239 *) 240 oid="kstat.zfs.$obj.misc.$stat" 241 ;; 242 esac 243 ;; 244 dataset) 245 typeset pool="" 246 typeset -i objsetid=0 247 _split_pool_objsetid $obj pool objsetid 248 oid=$(printf 'kstat.zfs.%s.dataset.objset-0x%x.%s' \ 249 $pool $objsetid $stat) 250 ;; 251 esac 252 253 # Calling sysctl on a "group" node will return everything under that 254 # node, so we have to inspect the first line to make sure we are 255 # getting back what we expect. For a single value, the key will have 256 # the name we requested, while for a group, the key will not have the 257 # name (group nodes are "opaque", not returned by sysctl by default. 258 259 # Multi-line values already end in a newline, and sysctl adds another 260 # one after every value, so hold each line back and drop the empty 261 # last one, to match what the same kstat looks like on Linux. 262 263 if [[ $want_group == 0 ]] ; then 264 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 265 NR == 1 && $0 !~ oidre { exit 1 } 266 NR == 1 { prev = substr($0, length(oid)+2) ; next } 267 { print prev ; prev = $0 } 268 END { if (prev != "") print prev } 269 ' 270 else 271 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 272 NR == 1 && $0 ~ oidre { exit 2 } 273 { 274 sub("^" oid "\.", "") 275 sub("=", " ") 276 print 277 } 278 ' 279 fi 280 281 typeset -i err=$? 282 case $err in 283 0) return ;; 284 1) log_fail "kstat: can't get value for group kstat: $oid" ;; 285 2) log_fail "kstat: not a group kstat: $oid" ;; 286 esac 287 288 log_fail "kstat: unknown error: $oid" 289} 290 291# 292# _resolve_dsname_freebsd <pool> <dsname> 293# 294function _resolve_dsname_freebsd 295{ 296 # we're searching for: 297 # 298 # kstat.zfs.shed.dataset.objset-0x8087.dataset_name: shed/poudriere 299 # 300 # We split on '.', then get the hex objsetid from field 5. 301 # 302 # We convert hex to decimal in the shell because there isn't a _simple_ 303 # portable way to do it in awk and this code is already too intense to 304 # do it a complicated way. 305 typeset pool=$1 306 typeset dsname=$2 307 sysctl -e kstat.zfs.$pool | \ 308 awk -F '.' -v dsnamere="=$dsname$" ' 309 /\.objset-0x[0-9a-f]+\.dataset_name=/ && $6 ~ dsnamere { 310 print substr($5, 8) 311 exit 312 } 313 ' | xargs printf %d 314} 315 316#################### 317# Linux 318 319# 320# kstats all live under /proc/spl/kstat/zfs. They have a flat structure: global 321# at top-level, pool in a directory, and dataset in a objset- file inside the 322# pool dir. 323# 324# Groups are challenge. A single stat can be the entire text of a file, or 325# a single line that must be extracted from a "group" file. The only way to 326# recognise a group from the outside is to look for its header. This naturally 327# breaks if a raw file had a matching header, or if a group file chooses to 328# hid its header. Fortunately OpenZFS does none of these things at the moment. 329# 330 331# 332# _kstat_linux <scope> <object> <stat> <want_group> 333# 334function _kstat_linux 335{ 336 typeset scope=$1 337 typeset obj=$2 338 typeset stat=$3 339 typeset -i want_group=$4 340 341 typeset singlestat="" 342 343 if [[ $scope == 'dataset' ]] ; then 344 typeset pool="" 345 typeset -i objsetid=0 346 _split_pool_objsetid $obj pool objsetid 347 stat=$(printf 'objset-0x%x.%s' $objsetid $stat) 348 obj=$pool 349 scope='pool' 350 fi 351 352 typeset path="" 353 if [[ $scope == 'global' ]] ; then 354 path="/proc/spl/kstat/zfs/$stat" 355 else 356 path="/proc/spl/kstat/zfs/$obj/$stat" 357 fi 358 359 if [[ ! -e "$path" && $want_group -eq 0 ]] ; then 360 # This single stat doesn't have its own file, but the wanted 361 # stat could be in a group kstat file, which we now need to 362 # find. To do this, we split a single stat name into two parts: 363 # the file that would contain the stat, and the key within that 364 # file to match on. This works by converting all bar the last 365 # '.' separator to '/', then splitting on the remaining '.' 366 # separator. If there are no '.' separators, the second arg 367 # returned will be empty. 368 # 369 # foo -> (foo) 370 # foo.bar -> (foo, bar) 371 # foo.bar.baz -> (foo/bar, baz) 372 # foo.bar.baz.quux -> (foo/bar/baz, quux) 373 # 374 # This is how we will target single stats within a larger NAMED 375 # kstat file, eg dbufstats.cache_target_bytes. 376 typeset -a split=($(echo "$stat" | \ 377 sed -E 's/^(.+)\.([^\.]+)$/\1 \2/ ; s/\./\//g')) 378 typeset statfile=${split[0]} 379 singlestat=${split[1]:-""} 380 381 if [[ $scope == 'global' ]] ; then 382 path="/proc/spl/kstat/zfs/$statfile" 383 else 384 path="/proc/spl/kstat/zfs/$obj/$statfile" 385 fi 386 fi 387 if [[ ! -r "$path" ]] ; then 388 log_fail "kstat: can't read $path" 389 fi 390 391 if [[ $want_group == 1 ]] ; then 392 # "group" (NAMED) kstats on Linux start: 393 # 394 # $ cat /proc/spl/kstat/zfs/crayon/iostats 395 # 70 1 0x01 26 7072 8577844978 661416318663496 396 # name type data 397 # trim_extents_written 4 0 398 # trim_bytes_written 4 0 399 # 400 # The second value on the first row is the ks_type. Group 401 # mode only works for type 1, KSTAT_TYPE_NAMED. So we check 402 # for that, and eject if it's the wrong type. Otherwise, we 403 # skip the header row and process the values. 404 awk ' 405 NR == 1 && ! /^[0-9]+ 1 / { exit 2 } 406 NR < 3 { next } 407 { print $1 " " $NF } 408 ' "$path" 409 elif [[ -n $singlestat ]] ; then 410 # single stat. must be a single line within a group stat, so 411 # we look for the header again as above. 412 awk -v singlestat="$singlestat" \ 413 -v singlestatre="^$singlestat " ' 414 NR == 1 && /^[0-9]+ [^1] / { exit 2 } 415 NR < 3 { next } 416 $0 ~ singlestatre { print $NF ; exit 0 } 417 ENDFILE { exit 3 } 418 ' "$path" 419 else 420 # raw stat. dump contents, exclude group stats 421 awk ' 422 NR == 1 && /^[0-9]+ 1 / { exit 1 } 423 { print } 424 ' "$path" 425 fi 426 427 typeset -i err=$? 428 case $err in 429 0) return ;; 430 1) log_fail "kstat: can't get value for group kstat: $path" ;; 431 2) log_fail "kstat: not a group kstat: $path" ;; 432 3) log_fail "kstat: stat not found in group: $path $singlestat" ;; 433 esac 434 435 log_fail "kstat: unknown error: $path" 436} 437 438# 439# _resolve_dsname_linux <pool> <dsname> 440# 441function _resolve_dsname_linux 442{ 443 # We look inside all: 444 # 445 # /proc/spl/kstat/zfs/crayon/objset-0x113 446 # 447 # and check the dataset_name field inside. If we get a match, we split 448 # the filename on /, then extract the hex objsetid. 449 # 450 # We convert hex to decimal in the shell because there isn't a _simple_ 451 # portable way to do it in awk and this code is already too intense to 452 # do it a complicated way. 453 typeset pool=$1 454 typeset dsname=$2 455 awk -v dsname="$dsname" ' 456 $1 == "dataset_name" && $3 == dsname { 457 split(FILENAME, a, "/") 458 print substr(a[7], 8) 459 exit 460 } 461 ' /proc/spl/kstat/zfs/$pool/objset-0x* | xargs printf %d 462} 463 464#################### 465 466# 467# _split_pool_objsetid <obj> <*pool> <*objsetid> 468# 469# Splits pool/objsetId string in <obj> and fills <pool> and <objsetid>. 470# 471function _split_pool_objsetid 472{ 473 typeset obj=$1 474 typeset -n pool=$2 475 typeset -n objsetid=$3 476 477 pool="${obj%%/*}" # clear first / -> end 478 typeset osidarg="${obj#*/}" # clear start -> first / 479 480 # ensure objsetid arg does not contain a /. we're about to convert it, 481 # but ksh will treat it as an expression, and a / will give a 482 # divide-by-zero 483 if [[ "${osidarg%%/*}" != "$osidarg" ]] ; then 484 log_fail "kstat: invalid objsetid: $osidarg" 485 fi 486 487 typeset -i id=$osidarg 488 if [[ $id -le 0 ]] ; then 489 log_fail "kstat: invalid objsetid: $osidarg" 490 fi 491 objsetid=$id 492} 493 494#################### 495 496# 497# Per-platform function selection. 498# 499# To avoid needing platform check throughout, we store the names of the 500# platform functions and call through them. 501# 502if is_freebsd ; then 503 _kstat_os='_kstat_freebsd' 504 _resolve_dsname_os='_resolve_dsname_freebsd' 505elif is_linux ; then 506 _kstat_os='_kstat_linux' 507 _resolve_dsname_os='_resolve_dsname_linux' 508else 509 _kstat_os='_kstat_unknown_platform_implement_me' 510 _resolve_dsname_os='_resolve_dsname_unknown_platform_implement_me' 511fi 512 513