common/cpr/cpr_dump.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 * Fill in and write out the cpr state file
 *	1. Allocate and write headers, ELF and cpr dump header
 *	2. Allocate bitmaps according to phys_install
 *	3. Tag kernel pages into corresponding bitmap
 *	4. Write bitmaps to state file
 *	5. Write actual physical page data to state file
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/vm.h>
#include <sys/memlist.h>
#include <sys/kmem.h>
#include <sys/vnode.h>
#include <sys/fs/ufs_inode.h>
#include <sys/errno.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <vm/page.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/hat.h>
#include <sys/cpr.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/panic.h>
#include <sys/thread.h>

/* Local defines and variables */
#define	BTOb(bytes)	((bytes) << 3)		/* Bytes to bits, log2(NBBY) */
#define	bTOB(bits)	((bits) >> 3)		/* bits to Bytes, log2(NBBY) */

static uint_t cpr_pages_tobe_dumped;
static uint_t cpr_regular_pgs_dumped;

static int cpr_dump_regular_pages(vnode_t *);
static int cpr_count_upages(int, bitfunc_t);
static int cpr_compress_and_write(vnode_t *, uint_t, pfn_t, pgcnt_t);
int cpr_flush_write(vnode_t *);

int cpr_contig_pages(vnode_t *, int);

void cpr_clear_bitmaps();

extern size_t cpr_get_devsize(dev_t);
extern int i_cpr_dump_setup(vnode_t *);
extern int i_cpr_blockzero(char *, char **, int *, vnode_t *);
extern int cpr_test_mode;

ctrm_t cpr_term;

char *cpr_buf, *cpr_buf_end;
int cpr_buf_blocks;		/* size of cpr_buf in blocks */
size_t cpr_buf_size;		/* size of cpr_buf in bytes */
size_t cpr_bitmap_size;
int cpr_nbitmaps;

char *cpr_pagedata;		/* page buffer for compression / tmp copy */
size_t cpr_pagedata_size;	/* page buffer size in bytes */

static char *cpr_wptr;		/* keep track of where to write to next */
static int cpr_file_bn;		/* cpr state-file block offset */
static int cpr_disk_writes_ok;
static size_t cpr_dev_space = 0;

char cpr_pagecopy[CPR_MAXCONTIG * MMU_PAGESIZE];

/*
 * On some platforms bcopy may modify the thread structure
 * during bcopy (eg, to prevent cpu migration).  If the
 * range we are currently writing out includes our own
 * thread structure then it will be snapshotted by bcopy
 * including those modified members - and the updates made
 * on exit from bcopy will no longer be seen when we later
 * restore the mid-bcopy kthread_t.  So if the range we
 * need to copy overlaps with our thread structure we will
 * use a simple byte copy.
 */
void
cprbcopy(void *from, void *to, size_t bytes)
{
	extern int curthreadremapped;
	caddr_t kthrend;

	kthrend = (caddr_t)curthread + sizeof (kthread_t) - 1;
	if (curthreadremapped || (kthrend >= (caddr_t)from &&
	    kthrend < (caddr_t)from + bytes + sizeof (kthread_t) - 1)) {
		caddr_t src = from, dst = to;

		while (bytes-- > 0)
			*dst++ = *src++;
	} else {
		bcopy(from, to, bytes);
	}
}

/*
 * Allocate pages for buffers used in writing out the statefile
 */
static int
cpr_alloc_bufs(void)
{
	char *allocerr = "Unable to allocate memory for cpr buffer";
	size_t size;

	/*
	 * set the cpr write buffer size to at least the historic
	 * size (128k) or large enough to store the both the early
	 * set of statefile structures (well under 0x800) plus the
	 * bitmaps, and roundup to the next pagesize.
	 */
	size = PAGE_ROUNDUP(dbtob(4) + cpr_bitmap_size);
	cpr_buf_size = MAX(size, CPRBUFSZ);
	cpr_buf_blocks = btodb(cpr_buf_size);
	cpr_buf = kmem_alloc(cpr_buf_size, KM_NOSLEEP);
	if (cpr_buf == NULL) {
		cpr_err(CE_WARN, allocerr);
		return (ENOMEM);
	}
	cpr_buf_end = cpr_buf + cpr_buf_size;

	cpr_pagedata_size = mmu_ptob(CPR_MAXCONTIG + 1);
	cpr_pagedata = kmem_alloc(cpr_pagedata_size, KM_NOSLEEP);
	if (cpr_pagedata == NULL) {
		kmem_free(cpr_buf, cpr_buf_size);
		cpr_buf = NULL;
		cpr_err(CE_WARN, allocerr);
		return (ENOMEM);
	}

	return (0);
}


/*
 * Set bitmap size in bytes based on phys_install.
 */
void
cpr_set_bitmap_size(void)
{
	struct memlist *pmem;
	size_t size = 0;

	memlist_read_lock();
	for (pmem = phys_install; pmem; pmem = pmem->next)
		size += pmem->size;
	memlist_read_unlock();
	cpr_bitmap_size = BITMAP_BYTES(size);
}


/*
 * CPR dump header contains the following information:
 *	1. header magic -- unique to cpr state file
 *	2. kernel return pc & ppn for resume
 *	3. current thread info
 *	4. debug level and test mode
 *	5. number of bitmaps allocated
 *	6. number of page records
 */
static int
cpr_write_header(vnode_t *vp)
{
	extern ushort_t cpr_mach_type;
	struct cpr_dump_desc cdump;
	pgcnt_t bitmap_pages;
	pgcnt_t kpages, vpages, upages;

	cdump.cdd_magic = (uint_t)CPR_DUMP_MAGIC;
	cdump.cdd_version = CPR_VERSION;
	cdump.cdd_machine = cpr_mach_type;
	cdump.cdd_debug = cpr_debug;
	cdump.cdd_test_mode = cpr_test_mode;
	cdump.cdd_bitmaprec = cpr_nbitmaps;

	cpr_clear_bitmaps();

	/*
	 * Remember how many pages we plan to save to statefile.
	 * This information will be used for sanity checks.
	 * Untag those pages that will not be saved to statefile.
	 */
	kpages = cpr_count_kpages(REGULAR_BITMAP, cpr_setbit);
	vpages = cpr_count_volatile_pages(REGULAR_BITMAP, cpr_clrbit);
	upages = cpr_count_upages(REGULAR_BITMAP, cpr_setbit);
	cdump.cdd_dumppgsize = kpages - vpages + upages;
	cpr_pages_tobe_dumped = cdump.cdd_dumppgsize;
	CPR_DEBUG(CPR_DEBUG7,
	    "\ncpr_write_header: kpages %ld - vpages %ld + upages %ld = %d\n",
	    kpages, vpages, upages, cdump.cdd_dumppgsize);

	/*
	 * Some pages contain volatile data (cpr_buf and storage area for
	 * sensitive kpages), which are no longer needed after the statefile
	 * is dumped to disk.  We have already untagged them from regular
	 * bitmaps.  Now tag them into the volatile bitmaps.  The pages in
	 * volatile bitmaps will be claimed during resume, and the resumed
	 * kernel will free them.
	 */
	(void) cpr_count_volatile_pages(VOLATILE_BITMAP, cpr_setbit);

	bitmap_pages = mmu_btopr(cpr_bitmap_size);

	/*
	 * Export accurate statefile size for statefile allocation retry.
	 * statefile_size = all the headers + total pages +
	 * number of pages used by the bitmaps.
	 * Roundup will be done in the file allocation code.
	 */
	STAT->cs_nocomp_statefsz = sizeof (cdd_t) + sizeof (cmd_t) +
		(sizeof (cbd_t) * cdump.cdd_bitmaprec) +
		(sizeof (cpd_t) * cdump.cdd_dumppgsize) +
		mmu_ptob(cdump.cdd_dumppgsize + bitmap_pages);

	/*
	 * If the estimated statefile is not big enough,
	 * go retry now to save un-necessary operations.
	 */
	if (!(CPR->c_flags & C_COMPRESSING) &&
		(STAT->cs_nocomp_statefsz > STAT->cs_est_statefsz)) {
		if (cpr_debug & (CPR_DEBUG1 | CPR_DEBUG7))
		    prom_printf("cpr_write_header: STAT->cs_nocomp_statefsz > "
			"STAT->cs_est_statefsz\n");
		return (ENOSPC);
	}

	/* now write cpr dump descriptor */
	return (cpr_write(vp, (caddr_t)&cdump, sizeof (cdd_t)));
}


/*
 * CPR dump tail record contains the following information:
 *	1. header magic -- unique to cpr state file
 *	2. all misc info that needs to be passed to cprboot or resumed kernel
 */
static int
cpr_write_terminator(vnode_t *vp)
{
	cpr_term.magic = (uint_t)CPR_TERM_MAGIC;
	cpr_term.va = (cpr_ptr)&cpr_term;
	cpr_term.pfn = (cpr_ext)va_to_pfn(&cpr_term);

	/* count the last one (flush) */
	cpr_term.real_statef_size = STAT->cs_real_statefsz +
		btod(cpr_wptr - cpr_buf) * DEV_BSIZE;

	CPR_DEBUG(CPR_DEBUG9, "cpr_dump: Real Statefile Size: %ld\n",
		STAT->cs_real_statefsz);

	cpr_tod_get(&cpr_term.tm_shutdown);

	return (cpr_write(vp, (caddr_t)&cpr_term, sizeof (cpr_term)));
}

/*
 * Write bitmap descriptor array, followed by merged bitmaps.
 */
static int
cpr_write_bitmap(vnode_t *vp)
{
	char *rmap, *vmap, *dst, *tail;
	size_t size, bytes;
	cbd_t *dp;
	int err;

	dp = CPR->c_bmda;
	if (err = cpr_write(vp, (caddr_t)dp, cpr_nbitmaps * sizeof (*dp)))
		return (err);

	/*
	 * merge regular and volatile bitmaps into tmp space
	 * and write to disk
	 */
	for (; dp->cbd_size; dp++) {
		rmap = (char *)dp->cbd_reg_bitmap;
		vmap = (char *)dp->cbd_vlt_bitmap;
		for (size = dp->cbd_size; size; size -= bytes) {
			bytes = min(size, sizeof (cpr_pagecopy));
			tail = &cpr_pagecopy[bytes];
			for (dst = cpr_pagecopy; dst < tail; dst++)
				*dst = *rmap++ | *vmap++;
			if (err = cpr_write(vp, cpr_pagecopy, bytes))
				break;
		}
	}

	return (err);
}


static int
cpr_write_statefile(vnode_t *vp)
{
	uint_t error = 0;
	extern	int	i_cpr_check_pgs_dumped();
	void flush_windows(void);
	pgcnt_t spages;
	char *str;

	flush_windows();

	/*
	 * to get an accurate view of kas, we need to untag sensitive
	 * pages *before* dumping them because the disk driver makes
	 * allocations and changes kas along the way.  The remaining
	 * pages referenced in the bitmaps are dumped out later as
	 * regular kpages.
	 */
	str = "cpr_write_statefile:";
	spages = i_cpr_count_sensitive_kpages(REGULAR_BITMAP, cpr_clrbit);
	CPR_DEBUG(CPR_DEBUG7, "%s untag %ld sens pages\n", str, spages);

	/*
	 * now it's OK to call a driver that makes allocations
	 */
	cpr_disk_writes_ok = 1;

	/*
	 * now write out the clean sensitive kpages
	 * according to the sensitive descriptors
	 */
	error = i_cpr_dump_sensitive_kpages(vp);
	if (error) {
		CPR_DEBUG(CPR_DEBUG7,
		    "%s cpr_dump_sensitive_kpages() failed!\n", str);
		return (error);
	}

	/*
	 * cpr_dump_regular_pages() counts cpr_regular_pgs_dumped
	 */
	error = cpr_dump_regular_pages(vp);
	if (error) {
		CPR_DEBUG(CPR_DEBUG7,
		    "%s cpr_dump_regular_pages() failed!\n", str);
		return (error);
	}

	/*
	 * sanity check to verify the right number of pages were dumped
	 */
	error = i_cpr_check_pgs_dumped(cpr_pages_tobe_dumped,
	    cpr_regular_pgs_dumped);

	if (error) {
		prom_printf("\n%s page count mismatch!\n", str);
#ifdef DEBUG
		if (cpr_test_mode)
			debug_enter(NULL);
#endif
	}

	return (error);
}


/*
 * creates the CPR state file, the following sections are
 * written out in sequence:
 *    - writes the cpr dump header
 *    - writes the memory usage bitmaps
 *    - writes the platform dependent info
 *    - writes the remaining user pages
 *    - writes the kernel pages
 */
int
cpr_dump(vnode_t *vp)
{
	int error;

	if (cpr_buf == NULL) {
		ASSERT(cpr_pagedata == NULL);
		if (error = cpr_alloc_bufs())
			return (error);
	}
	/* point to top of internal buffer */
	cpr_wptr = cpr_buf;

	/* initialize global variables used by the write operation */
	cpr_file_bn = cpr_statefile_offset();
	cpr_dev_space = 0;

	/* allocate bitmaps */
	if (CPR->c_bmda == NULL) {
		if (error = i_cpr_alloc_bitmaps()) {
			cpr_err(CE_WARN, "cannot allocate bitmaps");
			return (error);
		}
	}

	if (error = i_cpr_prom_pages(CPR_PROM_SAVE))
		return (error);

	if (error = i_cpr_dump_setup(vp))
		return (error);

	/*
	 * set internal cross checking; we dont want to call
	 * a disk driver that makes allocations until after
	 * sensitive pages are saved
	 */
	cpr_disk_writes_ok = 0;

	/*
	 * 1253112: heap corruption due to memory allocation when dumpping
	 *	    statefile.
	 * Theoretically on Sun4u only the kernel data nucleus, kvalloc and
	 * kvseg segments can be contaminated should memory allocations happen
	 * during sddump, which is not supposed to happen after the system
	 * is quiesced. Let's call the kernel pages that tend to be affected
	 * 'sensitive kpages' here. To avoid saving inconsistent pages, we
	 * will allocate some storage space to save the clean sensitive pages
	 * aside before statefile dumping takes place. Since there may not be
	 * much memory left at this stage, the sensitive pages will be
	 * compressed before they are saved into the storage area.
	 */
	if (error = i_cpr_save_sensitive_kpages()) {
		CPR_DEBUG(CPR_DEBUG7,
		    "cpr_dump: save_sensitive_kpages failed!\n");
		return (error);
	}

	/*
	 * since all cpr allocations are done (space for sensitive kpages,
	 * bitmaps, cpr_buf), kas is stable, and now we can accurately
	 * count regular and sensitive kpages.
	 */
	if (error = cpr_write_header(vp)) {
		CPR_DEBUG(CPR_DEBUG7,
		    "cpr_dump: cpr_write_header() failed!\n");
		return (error);
	}

	if (error = i_cpr_write_machdep(vp))
		return (error);

	if (error = i_cpr_blockzero(cpr_buf, &cpr_wptr, NULL, NULL))
		return (error);

	if (error = cpr_write_bitmap(vp))
		return (error);

	if (error = cpr_write_statefile(vp)) {
		CPR_DEBUG(CPR_DEBUG7,
		    "cpr_dump: cpr_write_statefile() failed!\n");
		return (error);
	}

	if (error = cpr_write_terminator(vp))
		return (error);

	if (error = cpr_flush_write(vp))
		return (error);

	if (error = i_cpr_blockzero(cpr_buf, &cpr_wptr, &cpr_file_bn, vp))
		return (error);

	return (0);
}


/*
 * cpr_xwalk() is called many 100x with a range within kvseg or kvseg_reloc;
 * a page-count from each range is accumulated at arg->pages.
 */
static void
cpr_xwalk(void *arg, void *base, size_t size)
{
	struct cpr_walkinfo *cwip = arg;

	cwip->pages += cpr_count_pages(base, size,
	    cwip->mapflag, cwip->bitfunc, DBG_DONTSHOWRANGE);
	cwip->size += size;
	cwip->ranges++;
}

/*
 * cpr_walk() is called many 100x with a range within kvseg or kvseg_reloc;
 * a page-count from each range is accumulated at arg->pages.
 */
static void
cpr_walk(void *arg, void *base, size_t size)
{
	caddr_t addr = base;
	caddr_t addr_end = addr + size;

	/*
	 * If we are about to start walking the range of addresses we
	 * carved out of the kernel heap for the large page heap walk
	 * heap_lp_arena to find what segments are actually populated
	 */
	if (SEGKMEM_USE_LARGEPAGES &&
	    addr == heap_lp_base && addr_end == heap_lp_end &&
	    vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
		vmem_walk(heap_lp_arena, VMEM_ALLOC, cpr_xwalk, arg);
	} else {
		cpr_xwalk(arg, base, size);
	}
}


/*
 * faster scan of kvseg using vmem_walk() to visit
 * allocated ranges.
 */
pgcnt_t
cpr_scan_kvseg(int mapflag, bitfunc_t bitfunc, struct seg *seg)
{
	struct cpr_walkinfo cwinfo;

	bzero(&cwinfo, sizeof (cwinfo));
	cwinfo.mapflag = mapflag;
	cwinfo.bitfunc = bitfunc;

	vmem_walk(heap_arena, VMEM_ALLOC, cpr_walk, &cwinfo);

	if (cpr_debug & CPR_DEBUG7) {
		prom_printf("walked %d sub-ranges, total pages %ld\n",
		    cwinfo.ranges, mmu_btop(cwinfo.size));
		cpr_show_range(seg->s_base, seg->s_size,
		    mapflag, bitfunc, cwinfo.pages);
	}

	return (cwinfo.pages);
}


/*
 * cpr_walk_kpm() is called for every used area within the large
 * segkpm virtual address window. A page-count is accumulated at
 * arg->pages.
 */
static void
cpr_walk_kpm(void *arg, void *base, size_t size)
{
	struct cpr_walkinfo *cwip = arg;

	cwip->pages += cpr_count_pages(base, size,
	    cwip->mapflag, cwip->bitfunc, DBG_DONTSHOWRANGE);
	cwip->size += size;
	cwip->ranges++;
}


/*
 * faster scan of segkpm using hat_kpm_walk() to visit only used ranges.
 */
/*ARGSUSED*/
static pgcnt_t
cpr_scan_segkpm(int mapflag, bitfunc_t bitfunc, struct seg *seg)
{
	struct cpr_walkinfo cwinfo;

	if (kpm_enable == 0)
		return (0);

	bzero(&cwinfo, sizeof (cwinfo));
	cwinfo.mapflag = mapflag;
	cwinfo.bitfunc = bitfunc;
	hat_kpm_walk(cpr_walk_kpm, &cwinfo);

	if (cpr_debug & CPR_DEBUG7) {
		prom_printf("walked %d sub-ranges, total pages %ld\n",
		    cwinfo.ranges, mmu_btop(cwinfo.size));
		cpr_show_range(segkpm->s_base, segkpm->s_size,
		    mapflag, bitfunc, cwinfo.pages);
	}

	return (cwinfo.pages);
}


/*
 * Sparsely filled kernel segments are registered in kseg_table for
 * easier lookup. See also block comment for cpr_count_seg_pages.
 */

#define	KSEG_SEG_ADDR	0	/* address of struct seg */
#define	KSEG_PTR_ADDR	1	/* address of pointer to struct seg */

typedef struct {
	struct seg **st_seg;		/* segment pointer or segment address */
	pgcnt_t	(*st_fcn)(int, bitfunc_t, struct seg *); /* function to call */
	int	st_addrtype;		/* address type in st_seg */
} ksegtbl_entry_t;

ksegtbl_entry_t kseg_table[] = {
	{(struct seg **)&kvseg,		cpr_scan_kvseg,		KSEG_SEG_ADDR},
	{&segkpm,			cpr_scan_segkpm,	KSEG_PTR_ADDR},
	{NULL,				0,			0}
};


/*
 * Compare seg with each entry in kseg_table; when there is a match
 * return the entry pointer, otherwise return NULL.
 */
static ksegtbl_entry_t *
cpr_sparse_seg_check(struct seg *seg)
{
	ksegtbl_entry_t *ste = &kseg_table[0];
	struct seg *tseg;

	for (; ste->st_seg; ste++) {
		tseg = (ste->st_addrtype == KSEG_PTR_ADDR) ?
				*ste->st_seg : (struct seg *)ste->st_seg;
		if (seg == tseg)
			return (ste);
	}

	return ((ksegtbl_entry_t *)NULL);
}


/*
 * Count pages within each kernel segment; call cpr_sparse_seg_check()
 * to find out whether a sparsely filled segment needs special
 * treatment (e.g. kvseg).
 * Todo: A "SEGOP_CPR" like SEGOP_DUMP should be introduced, the cpr
 *       module shouldn't need to know segment details like if it is
 *       sparsely filled or not (makes kseg_table obsolete).
 */
pgcnt_t
cpr_count_seg_pages(int mapflag, bitfunc_t bitfunc)
{
	struct seg *segp;
	pgcnt_t pages;
	ksegtbl_entry_t *ste;

	pages = 0;
	for (segp = AS_SEGFIRST(&kas); segp; segp = AS_SEGNEXT(&kas, segp)) {
		if (ste = cpr_sparse_seg_check(segp)) {
			pages += (ste->st_fcn)(mapflag, bitfunc, segp);
		} else {
			pages += cpr_count_pages(segp->s_base,
			    segp->s_size, mapflag, bitfunc, DBG_SHOWRANGE);
		}
	}

	return (pages);
}


/*
 * count kernel pages within kas and any special ranges
 */
pgcnt_t
cpr_count_kpages(int mapflag, bitfunc_t bitfunc)
{
	pgcnt_t kas_cnt;

	/*
	 * Some pages need to be taken care of differently.
	 * eg: panicbuf pages of sun4m are not in kas but they need
	 * to be saved.  On sun4u, the physical pages of panicbuf are
	 * allocated via prom_retain().
	 */
	kas_cnt = i_cpr_count_special_kpages(mapflag, bitfunc);
	kas_cnt += cpr_count_seg_pages(mapflag, bitfunc);

	CPR_DEBUG(CPR_DEBUG9, "cpr_count_kpages: kas_cnt=%ld\n", kas_cnt);
	CPR_DEBUG(CPR_DEBUG7, "\ncpr_count_kpages: %ld pages, 0x%lx bytes\n",
		kas_cnt, mmu_ptob(kas_cnt));
	return (kas_cnt);
}


/*
 * Set a bit corresponding to the arg phys page number;
 * returns 0 when the ppn is valid and the corresponding
 * map bit was clear, otherwise returns 1.
 */
int
cpr_setbit(pfn_t ppn, int mapflag)
{
	char *bitmap;
	cbd_t *dp;
	pfn_t rel;
	int clr;

	for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
		if (PPN_IN_RANGE(ppn, dp)) {
			bitmap = DESC_TO_MAP(dp, mapflag);
			rel = ppn - dp->cbd_spfn;
			if ((clr = isclr(bitmap, rel)) != 0)
				setbit(bitmap, rel);
			return (clr == 0);
		}
	}

	return (1);
}


/*
 * Clear a bit corresponding to the arg phys page number.
 */
int
cpr_clrbit(pfn_t ppn, int mapflag)
{
	char *bitmap;
	cbd_t *dp;
	pfn_t rel;
	int set;

	for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
		if (PPN_IN_RANGE(ppn, dp)) {
			bitmap = DESC_TO_MAP(dp, mapflag);
			rel = ppn - dp->cbd_spfn;
			if ((set = isset(bitmap, rel)) != 0)
				clrbit(bitmap, rel);
			return (set == 0);
		}
	}

	return (1);
}


/* ARGSUSED */
int
cpr_nobit(pfn_t ppn, int mapflag)
{
	return (0);
}


/*
 * Lookup a bit corresponding to the arg phys page number.
 */
int
cpr_isset(pfn_t ppn, int mapflag)
{
	char *bitmap;
	cbd_t *dp;
	pfn_t rel;

	for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
		if (PPN_IN_RANGE(ppn, dp)) {
			bitmap = DESC_TO_MAP(dp, mapflag);
			rel = ppn - dp->cbd_spfn;
			return (isset(bitmap, rel));
		}
	}

	return (0);
}


/*
 * Go thru all pages and pick up any page not caught during the invalidation
 * stage. This is also used to save pages with cow lock or phys page lock held
 * (none zero p_lckcnt or p_cowcnt)
 */
static	int
cpr_count_upages(int mapflag, bitfunc_t bitfunc)
{
	page_t *pp, *page0;
	pgcnt_t dcnt = 0, tcnt = 0;
	pfn_t pfn;

	page0 = pp = page_first();

	do {
#if defined(__sparc)
		extern struct vnode prom_ppages;
		if (pp->p_vnode == NULL || PP_ISKAS(pp) ||
		    pp->p_vnode == &prom_ppages ||
			PP_ISFREE(pp) && PP_ISAGED(pp))
#else
		if (pp->p_vnode == NULL || PP_ISKAS(pp) ||
		    PP_ISFREE(pp) && PP_ISAGED(pp))
#endif /* __sparc */
			continue;

		pfn = page_pptonum(pp);
		if (pf_is_memory(pfn)) {
			tcnt++;
			if ((*bitfunc)(pfn, mapflag) == 0)
				dcnt++; /* dirty count */
		}
	} while ((pp = page_next(pp)) != page0);

	STAT->cs_upage2statef = dcnt;
	CPR_DEBUG(CPR_DEBUG9, "cpr_count_upages: dirty=%ld total=%ld\n",
		dcnt, tcnt);
	CPR_DEBUG(CPR_DEBUG7, "cpr_count_upages: %ld pages, 0x%lx bytes\n",
		dcnt, mmu_ptob(dcnt));
	return (dcnt);
}


/*
 * try compressing pages based on cflag,
 * and for DEBUG kernels, verify uncompressed data checksum;
 *
 * this routine replaces common code from
 * i_cpr_compress_and_save() and cpr_compress_and_write()
 */
char *
cpr_compress_pages(cpd_t *dp, pgcnt_t pages, int cflag)
{
	size_t nbytes, clen, len;
	uint32_t test_sum;
	char *datap;

	nbytes = mmu_ptob(pages);

	/*
	 * set length to the original uncompressed data size;
	 * always init cpd_flag to zero
	 */
	dp->cpd_length = nbytes;
	dp->cpd_flag = 0;

#ifdef	DEBUG
	/*
	 * Make a copy of the uncompressed data so we can checksum it.
	 * Compress that copy so the checksum works at the other end
	 */
	cprbcopy(CPR->c_mapping_area, cpr_pagecopy, nbytes);
	dp->cpd_usum = checksum32(cpr_pagecopy, nbytes);
	dp->cpd_flag |= CPD_USUM;
	datap = cpr_pagecopy;
#else
	datap = CPR->c_mapping_area;
	dp->cpd_usum = 0;
#endif

	/*
	 * try compressing the raw data to cpr_pagedata;
	 * if there was a size reduction: record the new length,
	 * flag the compression, and point to the compressed data.
	 */
	dp->cpd_csum = 0;
	if (cflag) {
		clen = compress(datap, cpr_pagedata, nbytes);
		if (clen < nbytes) {
			dp->cpd_flag |= CPD_COMPRESS;
			dp->cpd_length = clen;
			datap = cpr_pagedata;
#ifdef	DEBUG
			dp->cpd_csum = checksum32(datap, clen);
			dp->cpd_flag |= CPD_CSUM;

			/*
			 * decompress the data back to a scratch area
			 * and compare the new checksum with the original
			 * checksum to verify the compression.
			 */
			bzero(cpr_pagecopy, sizeof (cpr_pagecopy));
			len = decompress(datap, cpr_pagecopy,
			    clen, sizeof (cpr_pagecopy));
			test_sum = checksum32(cpr_pagecopy, len);
			ASSERT(test_sum == dp->cpd_usum);
#endif
		}
	}

	return (datap);
}


/*
 * 1. Prepare cpr page descriptor and write it to file
 * 2. Compress page data and write it out
 */
static int
cpr_compress_and_write(vnode_t *vp, uint_t va, pfn_t pfn, pgcnt_t npg)
{
	int error = 0;
	char *datap;
	cpd_t cpd;	/* cpr page descriptor */
	extern void i_cpr_mapin(caddr_t, uint_t, pfn_t);
	extern void i_cpr_mapout(caddr_t, uint_t);

	i_cpr_mapin(CPR->c_mapping_area, npg, pfn);

	CPR_DEBUG(CPR_DEBUG3, "mapped-in %ld pages, vaddr 0x%p, pfn 0x%lx\n",
		npg, CPR->c_mapping_area, pfn);

	/*
	 * Fill cpr page descriptor.
	 */
	cpd.cpd_magic = (uint_t)CPR_PAGE_MAGIC;
	cpd.cpd_pfn = pfn;
	cpd.cpd_pages = npg;

	STAT->cs_dumped_statefsz += mmu_ptob(npg);

	datap = cpr_compress_pages(&cpd, npg, CPR->c_flags & C_COMPRESSING);

	/* Write cpr page descriptor */
	error = cpr_write(vp, (caddr_t)&cpd, sizeof (cpd_t));

	/* Write compressed page data */
	error = cpr_write(vp, (caddr_t)datap, cpd.cpd_length);

	/*
	 * Unmap the pages for tlb and vac flushing
	 */
	i_cpr_mapout(CPR->c_mapping_area, npg);

	if (error) {
		CPR_DEBUG(CPR_DEBUG1,
		    "cpr_compress_and_write: vp 0x%p va 0x%x ", vp, va);
		CPR_DEBUG(CPR_DEBUG1, "pfn 0x%lx blk %d err %d\n",
		    pfn, cpr_file_bn, error);
	} else {
		cpr_regular_pgs_dumped += npg;
	}

	return (error);
}


int
cpr_write(vnode_t *vp, caddr_t buffer, size_t size)
{
	caddr_t	fromp = buffer;
	size_t bytes, wbytes;
	int error;

	if (cpr_dev_space == 0) {
		if (vp->v_type == VBLK) {
			cpr_dev_space = cpr_get_devsize(vp->v_rdev);
			ASSERT(cpr_dev_space);
		} else
			cpr_dev_space = 1;	/* not used in this case */
	}

	/*
	 * break the write into multiple part if request is large,
	 * calculate count up to buf page boundary, then write it out.
	 * repeat until done.
	 */
	while (size) {
		bytes = MIN(size, cpr_buf_end - cpr_wptr);
		cprbcopy(fromp, cpr_wptr, bytes);
		cpr_wptr += bytes;
		fromp += bytes;
		size -= bytes;
		if (cpr_wptr < cpr_buf_end)
			return (0);	/* buffer not full yet */
		ASSERT(cpr_wptr == cpr_buf_end);

		wbytes = dbtob(cpr_file_bn + cpr_buf_blocks);
		if (vp->v_type == VBLK) {
			if (wbytes > cpr_dev_space)
				return (ENOSPC);
		} else {
			if (wbytes > VTOI(vp)->i_size)
				return (ENOSPC);
		}

		CPR_DEBUG(CPR_DEBUG3,
		    "cpr_write: frmp=%p wptr=%p cnt=%lx...",
		    fromp, cpr_wptr, bytes);
		/*
		 * cross check, this should not happen!
		 */
		if (cpr_disk_writes_ok == 0) {
			prom_printf("cpr_write: disk write too early!\n");
			return (EINVAL);
		}

		do_polled_io = 1;
		error = VOP_DUMP(vp, cpr_buf, cpr_file_bn, cpr_buf_blocks);
		do_polled_io = 0;
		CPR_DEBUG(CPR_DEBUG3, "done\n");

		STAT->cs_real_statefsz += cpr_buf_size;

		if (error) {
			cpr_err(CE_WARN, "cpr_write error %d", error);
			return (error);
		}
		cpr_file_bn += cpr_buf_blocks;	/* Increment block count */
		cpr_wptr = cpr_buf;		/* back to top of buffer */
	}
	return (0);
}


int
cpr_flush_write(vnode_t *vp)
{
	int	nblk;
	int	error;

	/*
	 * Calculate remaining blocks in buffer, rounded up to nearest
	 * disk block
	 */
	nblk = btod(cpr_wptr - cpr_buf);

	do_polled_io = 1;
	error = VOP_DUMP(vp, (caddr_t)cpr_buf, cpr_file_bn, nblk);
	do_polled_io = 0;

	cpr_file_bn += nblk;
	if (error)
		CPR_DEBUG(CPR_DEBUG2, "cpr_flush_write: error (%d)\n",
		    error);
	return (error);
}

void
cpr_clear_bitmaps(void)
{
	cbd_t *dp;

	for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
		bzero((void *)dp->cbd_reg_bitmap,
		    (size_t)dp->cbd_size * 2);
	}
	CPR_DEBUG(CPR_DEBUG7, "\ncleared reg and vlt bitmaps\n");
}

int
cpr_contig_pages(vnode_t *vp, int flag)
{
	int chunks = 0, error = 0;
	pgcnt_t i, j, totbit;
	pfn_t spfn;
	cbd_t *dp;
	uint_t	spin_cnt = 0;
	extern	int i_cpr_compress_and_save();

	for (dp = CPR->c_bmda; dp->cbd_size; dp++) {
		spfn = dp->cbd_spfn;
		totbit = BTOb(dp->cbd_size);
		i = 0; /* Beginning of bitmap */
		j = 0;
		while (i < totbit) {
			while ((j < CPR_MAXCONTIG) && ((j + i) < totbit)) {
				if (isset((char *)dp->cbd_reg_bitmap, j+i))
					j++;
				else /* not contiguous anymore */
					break;
			}

			if (j) {
				chunks++;
				if (flag == SAVE_TO_STORAGE) {
					error = i_cpr_compress_and_save(
					    chunks, spfn + i, j);
					if (error)
						return (error);
				} else if (flag == WRITE_TO_STATEFILE) {
					error = cpr_compress_and_write(vp, 0,
					    spfn + i, j);
					if (error)
						return (error);
					else {
						spin_cnt++;
						if ((spin_cnt & 0x5F) == 1)
							cpr_spinning_bar();
					}
				}
			}

			i += j;
			if (j != CPR_MAXCONTIG) {
				/* Stopped on a non-tagged page */
				i++;
			}

			j = 0;
		}
	}

	if (flag == STORAGE_DESC_ALLOC)
		return (chunks);
	else
		return (0);
}


void
cpr_show_range(caddr_t vaddr, size_t size,
    int mapflag, bitfunc_t bitfunc, pgcnt_t count)
{
	char *action, *bname;

	bname = (mapflag == REGULAR_BITMAP) ? "regular" : "volatile";
	if (bitfunc == cpr_setbit)
		action = "tag";
	else if (bitfunc == cpr_clrbit)
		action = "untag";
	else
		action = "none";
	prom_printf("range (0x%p, 0x%p), %s bitmap, %s %ld\n",
	    vaddr, vaddr + size, bname, action, count);
}


pgcnt_t
cpr_count_pages(caddr_t sva, size_t size,
    int mapflag, bitfunc_t bitfunc, int showrange)
{
	caddr_t	va, eva;
	pfn_t pfn;
	pgcnt_t count = 0;

	eva = sva + PAGE_ROUNDUP(size);
	for (va = sva; va < eva; va += MMU_PAGESIZE) {
		pfn = va_to_pfn(va);
		if (pfn != PFN_INVALID && pf_is_memory(pfn)) {
			if ((*bitfunc)(pfn, mapflag) == 0)
				count++;
		}
	}

	if ((cpr_debug & CPR_DEBUG7) && showrange == DBG_SHOWRANGE)
		cpr_show_range(sva, size, mapflag, bitfunc, count);

	return (count);
}


pgcnt_t
cpr_count_volatile_pages(int mapflag, bitfunc_t bitfunc)
{
	pgcnt_t count = 0;

	if (cpr_buf) {
		count += cpr_count_pages(cpr_buf, cpr_buf_size,
		    mapflag, bitfunc, DBG_SHOWRANGE);
	}
	if (cpr_pagedata) {
		count += cpr_count_pages(cpr_pagedata, cpr_pagedata_size,
		    mapflag, bitfunc, DBG_SHOWRANGE);
	}
	count += i_cpr_count_storage_pages(mapflag, bitfunc);

	CPR_DEBUG(CPR_DEBUG7, "cpr_count_vpages: %ld pages, 0x%lx bytes\n",
	    count, mmu_ptob(count));
	return (count);
}


static int
cpr_dump_regular_pages(vnode_t *vp)
{
	int error;

	cpr_regular_pgs_dumped = 0;
	error = cpr_contig_pages(vp, WRITE_TO_STATEFILE);
	if (!error)
		CPR_DEBUG(CPR_DEBUG7, "cpr_dump_regular_pages() done.\n");
	return (error);
}