1# SPDX-License-Identifier: CDDL-1.0 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or https://opensource.org/licenses/CDDL-1.0. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright (c) 2025, Klara, Inc. 25# Copyright 2025 Edgecast Cloud LLC. 26# 27 28# 29# This file provides the following helpers to read kstats from tests. 30# 31# kstat [-g] <stat> 32# kstat_pool [-g] <pool> <stat> 33# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 34# 35# `kstat` and `kstat_pool` return the value of the given <stat>, either 36# a global or pool-specific state. 37# 38# $ kstat dbgmsg 39# timestamp message 40# 1736848201 spa_history.c:304:spa_history_log_sync(): txg 14734896 ... 41# 1736848201 spa_history.c:330:spa_history_log_sync(): ioctl ... 42# ... 43# 44# $ kstat_pool garden state 45# ONLINE 46# 47# To get a single stat within a group or collection, separate the name with 48# '.' characters. 49# 50# $ kstat dbufstats.cache_target_bytes 51# 3215780693 52# 53# $ kstat_pool crayon iostats.arc_read_bytes 54# 253671670784 55# 56# -g is "group" mode. If the kstat is a group or collection, all stats in that 57# group are returned, one stat per line, key and value separated by a space. 58# 59# $ kstat -g dbufstats 60# cache_count 1792 61# cache_size_bytes 87720376 62# cache_size_bytes_max 305187768 63# cache_target_bytes 97668555 64# ... 65# 66# $ kstat_pool -g crayon iostats 67# trim_extents_written 0 68# trim_bytes_written 0 69# trim_extents_skipped 0 70# trim_bytes_skipped 0 71# ... 72# 73# `kstat_dataset` accesses the per-dataset group kstat. The dataset can be 74# specified by name: 75# 76# $ kstat_dataset crayon/home/robn nunlinks 77# 2628514 78# 79# or, with the -N switch, as <pool>/<objsetID>: 80# 81# $ kstat_dataset -N crayon/7 writes 82# 125135 83# 84 85#################### 86# Public interface 87 88# 89# kstat [-g] <stat> 90# 91function kstat 92{ 93 typeset -i want_group=0 94 95 OPTIND=1 96 while getopts "g" opt ; do 97 case $opt in 98 'g') want_group=1 ;; 99 *) log_fail "kstat: invalid option '$opt'" ;; 100 esac 101 done 102 shift $(expr $OPTIND - 1) 103 104 typeset stat=$1 105 106 $_kstat_os 'global' '' "$stat" $want_group 107} 108 109# 110# kstat_pool [-g] <pool> <stat> 111# 112function kstat_pool 113{ 114 typeset -i want_group=0 115 116 OPTIND=1 117 while getopts "g" opt ; do 118 case $opt in 119 'g') want_group=1 ;; 120 *) log_fail "kstat_pool: invalid option '$opt'" ;; 121 esac 122 done 123 shift $(expr $OPTIND - 1) 124 125 typeset pool=$1 126 typeset stat=$2 127 128 $_kstat_os 'pool' "$pool" "$stat" $want_group 129} 130 131# 132# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 133# 134function kstat_dataset 135{ 136 typeset -i opt_objsetid=0 137 138 OPTIND=1 139 while getopts "N" opt ; do 140 case $opt in 141 'N') opt_objsetid=1 ;; 142 *) log_fail "kstat_dataset: invalid option '$opt'" ;; 143 esac 144 done 145 shift $(expr $OPTIND - 1) 146 147 typeset dsarg=$1 148 typeset stat=$2 149 150 if [[ $opt_objsetid == 0 ]] ; then 151 typeset pool="${dsarg%%/*}" # clear first / -> end 152 typeset objsetid=$($_resolve_dsname_os "$pool" "$dsarg") 153 if [[ -z "$objsetid" ]] ; then 154 log_fail "kstat_dataset: dataset not found: $dsarg" 155 fi 156 dsarg="$pool/$objsetid" 157 fi 158 159 $_kstat_os 'dataset' "$dsarg" "$stat" 0 160} 161 162#################### 163# Platform-specific interface 164 165# 166# Implementation notes 167# 168# There's not a lot of uniformity between platforms, so I've written to a rough 169# imagined model that seems to fit the majority of OpenZFS kstats. 170# 171# The main platform entry points look like this: 172# 173# _kstat_freebsd <scope> <object> <stat> <want_group> 174# _kstat_illumos <scope> <object> <stat> <want_group> 175# _kstat_linux <scope> <object> <stat> <want_group> 176# 177# - scope: one of 'global', 'pool', 'dataset'. The "kind" of object the kstat 178# is attached to. 179# - object: name of the scoped object 180# global: empty string 181# pool: pool name 182# dataset: <pool>/<objsetId> pair 183# - stat: kstat name to get 184# - want_group: 0 to get the single value for the kstat, 1 to treat the kstat 185# as a group and get all the stat names+values under it. group 186# kstats cannot have values, and stat kstats cannot have 187# children (by definition) 188# 189# Stat values can have multiple lines, so be prepared for those. 190# 191# These functions either succeed and produce the requested output, or call 192# log_fail. They should never output empty, or 0, or anything else. 193# 194# Output: 195# 196# - want_group=0: the single stat value, followed by newline 197# - want_group=1: One stat per line, <name><SP><value><newline> 198# 199 200# 201# To support kstat_dataset(), platforms also need to provide a dataset 202# name->object id resolver function. 203# 204# _resolve_dsname_freebsd <pool> <dsname> 205# _resolve_dsname_illumos <pool> <dsname> 206# _resolve_dsname_linux <pool> <dsname> 207# 208# - pool: pool name. always the first part of the dataset name 209# - dsname: dataset name, in the standard <pool>/<some>/<dataset> format. 210# 211# Output is <objsetID>. objsetID is a decimal integer, > 0 212# 213 214#################### 215# FreeBSD 216 217# 218# All kstats are accessed through sysctl. We model "groups" as interior nodes 219# in the stat tree, which are normally opaque. Because sysctl has no filtering 220# options, and requesting any node produces all nodes below it, we have to 221# always get the name and value, and then consider the output to understand 222# if we got a group or a single stat, and post-process accordingly. 223# 224# Scopes are mostly mapped directly to known locations in the tree, but there 225# are a handful of stats that are out of position, so we need to adjust. 226# 227 228# 229# _kstat_freebsd <scope> <object> <stat> <want_group> 230# 231function _kstat_freebsd 232{ 233 typeset scope=$1 234 typeset obj=$2 235 typeset stat=$3 236 typeset -i want_group=$4 237 238 typeset oid="" 239 case "$scope" in 240 global) 241 oid="kstat.zfs.misc.$stat" 242 ;; 243 pool) 244 # For reasons unknown, the "multihost", "txgs" and "reads" 245 # pool-specific kstats are directly under kstat.zfs.<pool>, 246 # rather than kstat.zfs.<pool>.misc like the other pool kstats. 247 # Adjust for that here. 248 case "$stat" in 249 multihost|txgs|reads) 250 oid="kstat.zfs.$obj.$stat" 251 ;; 252 *) 253 oid="kstat.zfs.$obj.misc.$stat" 254 ;; 255 esac 256 ;; 257 dataset) 258 typeset pool="" 259 typeset -i objsetid=0 260 _split_pool_objsetid $obj pool objsetid 261 oid=$(printf 'kstat.zfs.%s.dataset.objset-0x%x.%s' \ 262 $pool $objsetid $stat) 263 ;; 264 esac 265 266 # Calling sysctl on a "group" node will return everything under that 267 # node, so we have to inspect the first line to make sure we are 268 # getting back what we expect. For a single value, the key will have 269 # the name we requested, while for a group, the key will not have the 270 # name (group nodes are "opaque", not returned by sysctl by default. 271 272 if [[ $want_group == 0 ]] ; then 273 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 274 NR == 1 && $0 !~ oidre { exit 1 } 275 NR == 1 { print substr($0, length(oid)+2) ; next } 276 { print } 277 ' 278 else 279 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 280 NR == 1 && $0 ~ oidre { exit 2 } 281 { 282 sub("^" oid "\.", "") 283 sub("=", " ") 284 print 285 } 286 ' 287 fi 288 289 typeset -i err=$? 290 case $err in 291 0) return ;; 292 1) log_fail "kstat: can't get value for group kstat: $oid" ;; 293 2) log_fail "kstat: not a group kstat: $oid" ;; 294 esac 295 296 log_fail "kstat: unknown error: $oid" 297} 298 299# 300# _resolve_dsname_freebsd <pool> <dsname> 301# 302function _resolve_dsname_freebsd 303{ 304 # we're searching for: 305 # 306 # kstat.zfs.shed.dataset.objset-0x8087.dataset_name: shed/poudriere 307 # 308 # We split on '.', then get the hex objsetid from field 5. 309 # 310 # We convert hex to decimal in the shell because there isn't a _simple_ 311 # portable way to do it in awk and this code is already too intense to 312 # do it a complicated way. 313 typeset pool=$1 314 typeset dsname=$2 315 sysctl -e kstat.zfs.$pool | \ 316 awk -F '.' -v dsnamere="=$dsname$" ' 317 /\.objset-0x[0-9a-f]+\.dataset_name=/ && $6 ~ dsnamere { 318 print substr($5, 8) 319 exit 320 } 321 ' | xargs printf %d 322} 323 324#################### 325# Linux 326 327# 328# kstats all live under /proc/spl/kstat/zfs. They have a flat structure: global 329# at top-level, pool in a directory, and dataset in a objset- file inside the 330# pool dir. 331# 332# Groups are challenge. A single stat can be the entire text of a file, or 333# a single line that must be extracted from a "group" file. The only way to 334# recognise a group from the outside is to look for its header. This naturally 335# breaks if a raw file had a matching header, or if a group file chooses to 336# hid its header. Fortunately OpenZFS does none of these things at the moment. 337# 338 339# 340# _kstat_linux <scope> <object> <stat> <want_group> 341# 342function _kstat_linux 343{ 344 typeset scope=$1 345 typeset obj=$2 346 typeset stat=$3 347 typeset -i want_group=$4 348 349 typeset singlestat="" 350 351 if [[ $scope == 'dataset' ]] ; then 352 typeset pool="" 353 typeset -i objsetid=0 354 _split_pool_objsetid $obj pool objsetid 355 stat=$(printf 'objset-0x%x.%s' $objsetid $stat) 356 obj=$pool 357 scope='pool' 358 fi 359 360 typeset path="" 361 if [[ $scope == 'global' ]] ; then 362 path="/proc/spl/kstat/zfs/$stat" 363 else 364 path="/proc/spl/kstat/zfs/$obj/$stat" 365 fi 366 367 if [[ ! -e "$path" && $want_group -eq 0 ]] ; then 368 # This single stat doesn't have its own file, but the wanted 369 # stat could be in a group kstat file, which we now need to 370 # find. To do this, we split a single stat name into two parts: 371 # the file that would contain the stat, and the key within that 372 # file to match on. This works by converting all bar the last 373 # '.' separator to '/', then splitting on the remaining '.' 374 # separator. If there are no '.' separators, the second arg 375 # returned will be empty. 376 # 377 # foo -> (foo) 378 # foo.bar -> (foo, bar) 379 # foo.bar.baz -> (foo/bar, baz) 380 # foo.bar.baz.quux -> (foo/bar/baz, quux) 381 # 382 # This is how we will target single stats within a larger NAMED 383 # kstat file, eg dbufstats.cache_target_bytes. 384 typeset -a split=($(echo "$stat" | \ 385 sed -E 's/^(.+)\.([^\.]+)$/\1 \2/ ; s/\./\//g')) 386 typeset statfile=${split[0]} 387 singlestat=${split[1]:-""} 388 389 if [[ $scope == 'global' ]] ; then 390 path="/proc/spl/kstat/zfs/$statfile" 391 else 392 path="/proc/spl/kstat/zfs/$obj/$statfile" 393 fi 394 fi 395 if [[ ! -r "$path" ]] ; then 396 log_fail "kstat: can't read $path" 397 fi 398 399 if [[ $want_group == 1 ]] ; then 400 # "group" (NAMED) kstats on Linux start: 401 # 402 # $ cat /proc/spl/kstat/zfs/crayon/iostats 403 # 70 1 0x01 26 7072 8577844978 661416318663496 404 # name type data 405 # trim_extents_written 4 0 406 # trim_bytes_written 4 0 407 # 408 # The second value on the first row is the ks_type. Group 409 # mode only works for type 1, KSTAT_TYPE_NAMED. So we check 410 # for that, and eject if it's the wrong type. Otherwise, we 411 # skip the header row and process the values. 412 awk ' 413 NR == 1 && ! /^[0-9]+ 1 / { exit 2 } 414 NR < 3 { next } 415 { print $1 " " $NF } 416 ' "$path" 417 elif [[ -n $singlestat ]] ; then 418 # single stat. must be a single line within a group stat, so 419 # we look for the header again as above. 420 awk -v singlestat="$singlestat" \ 421 -v singlestatre="^$singlestat " ' 422 NR == 1 && /^[0-9]+ [^1] / { exit 2 } 423 NR < 3 { next } 424 $0 ~ singlestatre { print $NF ; exit 0 } 425 ENDFILE { exit 3 } 426 ' "$path" 427 else 428 # raw stat. dump contents, exclude group stats 429 awk ' 430 NR == 1 && /^[0-9]+ 1 / { exit 1 } 431 { print } 432 ' "$path" 433 fi 434 435 typeset -i err=$? 436 case $err in 437 0) return ;; 438 1) log_fail "kstat: can't get value for group kstat: $path" ;; 439 2) log_fail "kstat: not a group kstat: $path" ;; 440 3) log_fail "kstat: stat not found in group: $path $singlestat" ;; 441 esac 442 443 log_fail "kstat: unknown error: $path" 444} 445 446# 447# _resolve_dsname_linux <pool> <dsname> 448# 449function _resolve_dsname_linux 450{ 451 # We look inside all: 452 # 453 # /proc/spl/kstat/zfs/crayon/objset-0x113 454 # 455 # and check the dataset_name field inside. If we get a match, we split 456 # the filename on /, then extract the hex objsetid. 457 # 458 # We convert hex to decimal in the shell because there isn't a _simple_ 459 # portable way to do it in awk and this code is already too intense to 460 # do it a complicated way. 461 typeset pool=$1 462 typeset dsname=$2 463 awk -v dsname="$dsname" ' 464 $1 == "dataset_name" && $3 == dsname { 465 split(FILENAME, a, "/") 466 print substr(a[7], 8) 467 exit 468 } 469 ' /proc/spl/kstat/zfs/$pool/objset-0x* | xargs printf %d 470} 471 472#################### 473 474# 475# _split_pool_objsetid <obj> <*pool> <*objsetid> 476# 477# Splits pool/objsetId string in <obj> and fills <pool> and <objsetid>. 478# 479function _split_pool_objsetid 480{ 481 typeset obj=$1 482 typeset -n pool=$2 483 typeset -n objsetid=$3 484 485 pool="${obj%%/*}" # clear first / -> end 486 typeset osidarg="${obj#*/}" # clear start -> first / 487 488 # ensure objsetid arg does not contain a /. we're about to convert it, 489 # but ksh will treat it as an expression, and a / will give a 490 # divide-by-zero 491 if [[ "${osidarg%%/*}" != "$osidarg" ]] ; then 492 log_fail "kstat: invalid objsetid: $osidarg" 493 fi 494 495 typeset -i id=$osidarg 496 if [[ $id -le 0 ]] ; then 497 log_fail "kstat: invalid objsetid: $osidarg" 498 fi 499 objsetid=$id 500} 501 502#################### 503# illumos 504 505# 506# _kstat_illumos <scope> <object> <stat> <want_group> 507# 508function _kstat_illumos 509{ 510 typeset scope=$1 511 typeset obj=$2 512 typeset stat=${3/\./:} 513 typeset -i want_group=$4 514 515 typeset oid="" 516 case "$scope" in 517 global) 518 oid="zfs::$stat" 519 ;; 520 pool) 521 oid="zfs::$obj:$stat" 522 ;; 523 dataset) 524 log_fail "kstat: dataset kstats are not implemented" 525 ;; 526 esac 527 command kstat -pV $oid 528} 529 530# 531# _resolve_dsname_illumos <pool> <dsname> 532# 533function _resolve_dsname_illumos 534{ 535 typeset pool=$1 536 typeset dsname=$2 537 538 # we do not have per dataset kstats 539} 540 541#################### 542 543# 544# Per-platform function selection. 545# 546# To avoid needing platform check throughout, we store the names of the 547# platform functions and call through them. 548# 549if is_freebsd ; then 550 _kstat_os='_kstat_freebsd' 551 _resolve_dsname_os='_resolve_dsname_freebsd' 552elif is_linux ; then 553 _kstat_os='_kstat_linux' 554 _resolve_dsname_os='_resolve_dsname_linux' 555elif is_illumos ; then 556 _kstat_os='_kstat_illumos' 557 _resolve_dsname_os='_resolve_dsname_illumos' 558else 559 _kstat_os='_kstat_unknown_platform_implement_me' 560 _resolve_dsname_os='_resolve_dsname_unknown_platform_implement_me' 561fi 562