1# SPDX-License-Identifier: CDDL-1.0 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or https://opensource.org/licenses/CDDL-1.0. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright (c) 2025, Klara, Inc. 25# 26 27# 28# This file provides the following helpers to read kstats from tests. 29# 30# kstat [-g] <stat> 31# kstat_pool [-g] <pool> <stat> 32# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 33# 34# `kstat` and `kstat_pool` return the value of of the given <stat>, either 35# a global or pool-specific state. 36# 37# $ kstat dbgmsg 38# timestamp message 39# 1736848201 spa_history.c:304:spa_history_log_sync(): txg 14734896 ... 40# 1736848201 spa_history.c:330:spa_history_log_sync(): ioctl ... 41# ... 42# 43# $ kstat_pool garden state 44# ONLINE 45# 46# To get a single stat within a group or collection, separate the name with 47# '.' characters. 48# 49# $ kstat dbufstats.cache_target_bytes 50# 3215780693 51# 52# $ kstat_pool crayon iostats.arc_read_bytes 53# 253671670784 54# 55# -g is "group" mode. If the kstat is a group or collection, all stats in that 56# group are returned, one stat per line, key and value separated by a space. 57# 58# $ kstat -g dbufstats 59# cache_count 1792 60# cache_size_bytes 87720376 61# cache_size_bytes_max 305187768 62# cache_target_bytes 97668555 63# ... 64# 65# $ kstat_pool -g crayon iostats 66# trim_extents_written 0 67# trim_bytes_written 0 68# trim_extents_skipped 0 69# trim_bytes_skipped 0 70# ... 71# 72# `kstat_dataset` accesses the per-dataset group kstat. The dataset can be 73# specified by name: 74# 75# $ kstat_dataset crayon/home/robn nunlinks 76# 2628514 77# 78# or, with the -N switch, as <pool>/<objsetID>: 79# 80# $ kstat_dataset -N crayon/7 writes 81# 125135 82# 83 84#################### 85# Public interface 86 87# 88# kstat [-g] <stat> 89# 90function kstat 91{ 92 typeset -i want_group=0 93 94 OPTIND=1 95 while getopts "g" opt ; do 96 case $opt in 97 'g') want_group=1 ;; 98 *) log_fail "kstat: invalid option '$opt'" ;; 99 esac 100 done 101 shift $(expr $OPTIND - 1) 102 103 typeset stat=$1 104 105 $_kstat_os 'global' '' "$stat" $want_group 106} 107 108# 109# kstat_pool [-g] <pool> <stat> 110# 111function kstat_pool 112{ 113 typeset -i want_group=0 114 115 OPTIND=1 116 while getopts "g" opt ; do 117 case $opt in 118 'g') want_group=1 ;; 119 *) log_fail "kstat_pool: invalid option '$opt'" ;; 120 esac 121 done 122 shift $(expr $OPTIND - 1) 123 124 typeset pool=$1 125 typeset stat=$2 126 127 $_kstat_os 'pool' "$pool" "$stat" $want_group 128} 129 130# 131# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 132# 133function kstat_dataset 134{ 135 typeset -i opt_objsetid=0 136 137 OPTIND=1 138 while getopts "N" opt ; do 139 case $opt in 140 'N') opt_objsetid=1 ;; 141 *) log_fail "kstat_dataset: invalid option '$opt'" ;; 142 esac 143 done 144 shift $(expr $OPTIND - 1) 145 146 typeset dsarg=$1 147 typeset stat=$2 148 149 if [[ $opt_objsetid == 0 ]] ; then 150 typeset pool="${dsarg%%/*}" # clear first / -> end 151 typeset objsetid=$($_resolve_dsname_os "$pool" "$dsarg") 152 if [[ -z "$objsetid" ]] ; then 153 log_fail "kstat_dataset: dataset not found: $dsarg" 154 fi 155 dsarg="$pool/$objsetid" 156 fi 157 158 $_kstat_os 'dataset' "$dsarg" "$stat" 0 159} 160 161#################### 162# Platform-specific interface 163 164# 165# Implementation notes 166# 167# There's not a lot of uniformity between platforms, so I've written to a rough 168# imagined model that seems to fit the majority of OpenZFS kstats. 169# 170# The main platform entry points look like this: 171# 172# _kstat_freebsd <scope> <object> <stat> <want_group> 173# _kstat_linux <scope> <object> <stat> <want_group> 174# 175# - scope: one of 'global', 'pool', 'dataset'. The "kind" of object the kstat 176# is attached to. 177# - object: name of the scoped object 178# global: empty string 179# pool: pool name 180# dataset: <pool>/<objsetId> pair 181# - stat: kstat name to get 182# - want_group: 0 to get the single value for the kstat, 1 to treat the kstat 183# as a group and get all the stat names+values under it. group 184# kstats cannot have values, and stat kstats cannot have 185# children (by definition) 186# 187# Stat values can have multiple lines, so be prepared for those. 188# 189# These functions either succeed and produce the requested output, or call 190# log_fail. They should never output empty, or 0, or anything else. 191# 192# Output: 193# 194# - want_group=0: the single stat value, followed by newline 195# - want_group=1: One stat per line, <name><SP><value><newline> 196# 197 198# 199# To support kstat_dataset(), platforms also need to provide a dataset 200# name->object id resolver function. 201# 202# _resolve_dsname_freebsd <pool> <dsname> 203# _resolve_dsname_linux <pool> <dsname> 204# 205# - pool: pool name. always the first part of the dataset name 206# - dsname: dataset name, in the standard <pool>/<some>/<dataset> format. 207# 208# Output is <objsetID>. objsetID is a decimal integer, > 0 209# 210 211#################### 212# FreeBSD 213 214# 215# All kstats are accessed through sysctl. We model "groups" as interior nodes 216# in the stat tree, which are normally opaque. Because sysctl has no filtering 217# options, and requesting any node produces all nodes below it, we have to 218# always get the name and value, and then consider the output to understand 219# if we got a group or a single stat, and post-process accordingly. 220# 221# Scopes are mostly mapped directly to known locations in the tree, but there 222# are a handful of stats that are out of position, so we need to adjust. 223# 224 225# 226# _kstat_freebsd <scope> <object> <stat> <want_group> 227# 228function _kstat_freebsd 229{ 230 typeset scope=$1 231 typeset obj=$2 232 typeset stat=$3 233 typeset -i want_group=$4 234 235 typeset oid="" 236 case "$scope" in 237 global) 238 oid="kstat.zfs.misc.$stat" 239 ;; 240 pool) 241 # For reasons unknown, the "multihost", "txgs" and "reads" 242 # pool-specific kstats are directly under kstat.zfs.<pool>, 243 # rather than kstat.zfs.<pool>.misc like the other pool kstats. 244 # Adjust for that here. 245 case "$stat" in 246 multihost|txgs|reads) 247 oid="kstat.zfs.$obj.$stat" 248 ;; 249 *) 250 oid="kstat.zfs.$obj.misc.$stat" 251 ;; 252 esac 253 ;; 254 dataset) 255 typeset pool="" 256 typeset -i objsetid=0 257 _split_pool_objsetid $obj pool objsetid 258 oid=$(printf 'kstat.zfs.%s.dataset.objset-0x%x.%s' \ 259 $pool $objsetid $stat) 260 ;; 261 esac 262 263 # Calling sysctl on a "group" node will return everything under that 264 # node, so we have to inspect the first line to make sure we are 265 # getting back what we expect. For a single value, the key will have 266 # the name we requested, while for a group, the key will not have the 267 # name (group nodes are "opaque", not returned by sysctl by default. 268 269 if [[ $want_group == 0 ]] ; then 270 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 271 NR == 1 && $0 !~ oidre { exit 1 } 272 NR == 1 { print substr($0, length(oid)+2) ; next } 273 { print } 274 ' 275 else 276 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 277 NR == 1 && $0 ~ oidre { exit 2 } 278 { 279 sub("^" oid "\.", "") 280 sub("=", " ") 281 print 282 } 283 ' 284 fi 285 286 typeset -i err=$? 287 case $err in 288 0) return ;; 289 1) log_fail "kstat: can't get value for group kstat: $oid" ;; 290 2) log_fail "kstat: not a group kstat: $oid" ;; 291 esac 292 293 log_fail "kstat: unknown error: $oid" 294} 295 296# 297# _resolve_dsname_freebsd <pool> <dsname> 298# 299function _resolve_dsname_freebsd 300{ 301 # we're searching for: 302 # 303 # kstat.zfs.shed.dataset.objset-0x8087.dataset_name: shed/poudriere 304 # 305 # We split on '.', then get the hex objsetid from field 5. 306 # 307 # We convert hex to decimal in the shell because there isn't a _simple_ 308 # portable way to do it in awk and this code is already too intense to 309 # do it a complicated way. 310 typeset pool=$1 311 typeset dsname=$2 312 sysctl -e kstat.zfs.$pool | \ 313 awk -F '.' -v dsnamere="=$dsname$" ' 314 /\.objset-0x[0-9a-f]+\.dataset_name=/ && $6 ~ dsnamere { 315 print substr($5, 8) 316 exit 317 } 318 ' | xargs printf %d 319} 320 321#################### 322# Linux 323 324# 325# kstats all live under /proc/spl/kstat/zfs. They have a flat structure: global 326# at top-level, pool in a directory, and dataset in a objset- file inside the 327# pool dir. 328# 329# Groups are challenge. A single stat can be the entire text of a file, or 330# a single line that must be extracted from a "group" file. The only way to 331# recognise a group from the outside is to look for its header. This naturally 332# breaks if a raw file had a matching header, or if a group file chooses to 333# hid its header. Fortunately OpenZFS does none of these things at the moment. 334# 335 336# 337# _kstat_linux <scope> <object> <stat> <want_group> 338# 339function _kstat_linux 340{ 341 typeset scope=$1 342 typeset obj=$2 343 typeset stat=$3 344 typeset -i want_group=$4 345 346 typeset singlestat="" 347 348 if [[ $scope == 'dataset' ]] ; then 349 typeset pool="" 350 typeset -i objsetid=0 351 _split_pool_objsetid $obj pool objsetid 352 stat=$(printf 'objset-0x%x.%s' $objsetid $stat) 353 obj=$pool 354 scope='pool' 355 fi 356 357 typeset path="" 358 if [[ $scope == 'global' ]] ; then 359 path="/proc/spl/kstat/zfs/$stat" 360 else 361 path="/proc/spl/kstat/zfs/$obj/$stat" 362 fi 363 364 if [[ ! -e "$path" && $want_group -eq 0 ]] ; then 365 # This single stat doesn't have its own file, but the wanted 366 # stat could be in a group kstat file, which we now need to 367 # find. To do this, we split a single stat name into two parts: 368 # the file that would contain the stat, and the key within that 369 # file to match on. This works by converting all bar the last 370 # '.' separator to '/', then splitting on the remaining '.' 371 # separator. If there are no '.' separators, the second arg 372 # returned will be empty. 373 # 374 # foo -> (foo) 375 # foo.bar -> (foo, bar) 376 # foo.bar.baz -> (foo/bar, baz) 377 # foo.bar.baz.quux -> (foo/bar/baz, quux) 378 # 379 # This is how we will target single stats within a larger NAMED 380 # kstat file, eg dbufstats.cache_target_bytes. 381 typeset -a split=($(echo "$stat" | \ 382 sed -E 's/^(.+)\.([^\.]+)$/\1 \2/ ; s/\./\//g')) 383 typeset statfile=${split[0]} 384 singlestat=${split[1]:-""} 385 386 if [[ $scope == 'global' ]] ; then 387 path="/proc/spl/kstat/zfs/$statfile" 388 else 389 path="/proc/spl/kstat/zfs/$obj/$statfile" 390 fi 391 fi 392 if [[ ! -r "$path" ]] ; then 393 log_fail "kstat: can't read $path" 394 fi 395 396 if [[ $want_group == 1 ]] ; then 397 # "group" (NAMED) kstats on Linux start: 398 # 399 # $ cat /proc/spl/kstat/zfs/crayon/iostats 400 # 70 1 0x01 26 7072 8577844978 661416318663496 401 # name type data 402 # trim_extents_written 4 0 403 # trim_bytes_written 4 0 404 # 405 # The second value on the first row is the ks_type. Group 406 # mode only works for type 1, KSTAT_TYPE_NAMED. So we check 407 # for that, and eject if it's the wrong type. Otherwise, we 408 # skip the header row and process the values. 409 awk ' 410 NR == 1 && ! /^[0-9]+ 1 / { exit 2 } 411 NR < 3 { next } 412 { print $1 " " $NF } 413 ' "$path" 414 elif [[ -n $singlestat ]] ; then 415 # single stat. must be a single line within a group stat, so 416 # we look for the header again as above. 417 awk -v singlestat="$singlestat" \ 418 -v singlestatre="^$singlestat " ' 419 NR == 1 && /^[0-9]+ [^1] / { exit 2 } 420 NR < 3 { next } 421 $0 ~ singlestatre { print $NF ; exit 0 } 422 ENDFILE { exit 3 } 423 ' "$path" 424 else 425 # raw stat. dump contents, exclude group stats 426 awk ' 427 NR == 1 && /^[0-9]+ 1 / { exit 1 } 428 { print } 429 ' "$path" 430 fi 431 432 typeset -i err=$? 433 case $err in 434 0) return ;; 435 1) log_fail "kstat: can't get value for group kstat: $path" ;; 436 2) log_fail "kstat: not a group kstat: $path" ;; 437 3) log_fail "kstat: stat not found in group: $path $singlestat" ;; 438 esac 439 440 log_fail "kstat: unknown error: $path" 441} 442 443# 444# _resolve_dsname_linux <pool> <dsname> 445# 446function _resolve_dsname_linux 447{ 448 # We look inside all: 449 # 450 # /proc/spl/kstat/zfs/crayon/objset-0x113 451 # 452 # and check the dataset_name field inside. If we get a match, we split 453 # the filename on /, then extract the hex objsetid. 454 # 455 # We convert hex to decimal in the shell because there isn't a _simple_ 456 # portable way to do it in awk and this code is already too intense to 457 # do it a complicated way. 458 typeset pool=$1 459 typeset dsname=$2 460 awk -v dsname="$dsname" ' 461 $1 == "dataset_name" && $3 == dsname { 462 split(FILENAME, a, "/") 463 print substr(a[7], 8) 464 exit 465 } 466 ' /proc/spl/kstat/zfs/$pool/objset-0x* | xargs printf %d 467} 468 469#################### 470 471# 472# _split_pool_objsetid <obj> <*pool> <*objsetid> 473# 474# Splits pool/objsetId string in <obj> and fills <pool> and <objsetid>. 475# 476function _split_pool_objsetid 477{ 478 typeset obj=$1 479 typeset -n pool=$2 480 typeset -n objsetid=$3 481 482 pool="${obj%%/*}" # clear first / -> end 483 typeset osidarg="${obj#*/}" # clear start -> first / 484 485 # ensure objsetid arg does not contain a /. we're about to convert it, 486 # but ksh will treat it as an expression, and a / will give a 487 # divide-by-zero 488 if [[ "${osidarg%%/*}" != "$osidarg" ]] ; then 489 log_fail "kstat: invalid objsetid: $osidarg" 490 fi 491 492 typeset -i id=$osidarg 493 if [[ $id -le 0 ]] ; then 494 log_fail "kstat: invalid objsetid: $osidarg" 495 fi 496 objsetid=$id 497} 498 499#################### 500 501# 502# Per-platform function selection. 503# 504# To avoid needing platform check throughout, we store the names of the 505# platform functions and call through them. 506# 507if is_freebsd ; then 508 _kstat_os='_kstat_freebsd' 509 _resolve_dsname_os='_resolve_dsname_freebsd' 510elif is_linux ; then 511 _kstat_os='_kstat_linux' 512 _resolve_dsname_os='_resolve_dsname_linux' 513else 514 _kstat_os='_kstat_unknown_platform_implement_me' 515 _resolve_dsname_os='_resolve_dsname_unknown_platform_implement_me' 516fi 517 518