File Manager

003 File Manager

Current Path: /usr/src/sys/contrib/openzfs/module/zfs

usr / src / sys / contrib / openzfs / module / zfs /

📁 ..
📄 Makefile.in(4.57 KB)
📄 THIRDPARTYLICENSE.cityhash(1.03 KB)
📄 THIRDPARTYLICENSE.cityhash.descrip(39 B)
📄 abd.c(31.72 KB)
📄 aggsum.c(8.46 KB)
📄 arc.c(330.91 KB)
📄 blkptr.c(4.26 KB)
📄 bplist.c(2.38 KB)
📄 bpobj.c(27.82 KB)
📄 bptree.c(8.2 KB)
📄 bqueue.c(4.7 KB)
📄 btree.c(65.19 KB)
📄 dataset_kstats.c(6.32 KB)
📄 dbuf.c(141.84 KB)
📄 dbuf_stats.c(6.05 KB)
📄 ddt.c(28.01 KB)
📄 ddt_zap.c(4.38 KB)
📄 dmu.c(60.44 KB)
📄 dmu_diff.c(6.25 KB)
📄 dmu_object.c(14.73 KB)
📄 dmu_objset.c(79.38 KB)
📄 dmu_recv.c(98.69 KB)
📄 dmu_redact.c(37.09 KB)
📄 dmu_send.c(95.51 KB)
📄 dmu_traverse.c(21.98 KB)
📄 dmu_tx.c(40.13 KB)
📄 dmu_zfetch.c(13.56 KB)
📄 dnode.c(71.32 KB)
📄 dnode_sync.c(24.93 KB)
📄 dsl_bookmark.c(51.22 KB)
📄 dsl_crypt.c(78.06 KB)
📄 dsl_dataset.c(141.61 KB)
📄 dsl_deadlist.c(27.94 KB)
📄 dsl_deleg.c(19.92 KB)
📄 dsl_destroy.c(36.73 KB)
📄 dsl_dir.c(65.45 KB)
📄 dsl_pool.c(42.92 KB)
📄 dsl_prop.c(32.88 KB)
📄 dsl_scan.c(134.47 KB)
📄 dsl_synctask.c(7.94 KB)
📄 dsl_userhold.c(18.26 KB)
📄 edonr_zfs.c(3.21 KB)
📄 fm.c(39.75 KB)
📄 gzip.c(2.57 KB)
📄 hkdf.c(4.5 KB)
📄 lz4.c(26.46 KB)
📄 lzjb.c(3.85 KB)
📄 metaslab.c(190.62 KB)
📄 mmp.c(24.93 KB)
📄 multilist.c(12.21 KB)
📄 objlist.c(2.58 KB)
📄 pathname.c(2.52 KB)
📄 range_tree.c(24.9 KB)
📄 refcount.c(7.82 KB)
📄 rrwlock.c(10.82 KB)
📄 sa.c(59.23 KB)
📄 sha256.c(2.89 KB)
📄 skein_zfs.c(2.92 KB)
📄 spa.c(272.81 KB)
📄 spa_boot.c(1.28 KB)
📄 spa_checkpoint.c(21.54 KB)
📄 spa_config.c(16.97 KB)
📄 spa_errlog.c(11.35 KB)
📄 spa_history.c(17.75 KB)
📄 spa_log_spacemap.c(47.13 KB)
📄 spa_misc.c(75.4 KB)
📄 spa_stats.c(27.08 KB)
📄 space_map.c(31.45 KB)
📄 space_reftree.c(4.21 KB)
📄 txg.c(27.49 KB)
📄 uberblock.c(2.17 KB)
📄 unique.c(2.53 KB)
📄 vdev.c(143.91 KB)
📄 vdev_cache.c(11.71 KB)
📄 vdev_draid.c(96.86 KB)
📄 vdev_draid_rand.c(1.16 KB)
📄 vdev_indirect.c(60.83 KB)
📄 vdev_indirect_births.c(5.88 KB)
📄 vdev_indirect_mapping.c(18.06 KB)
📄 vdev_initialize.c(22.38 KB)
📄 vdev_label.c(57.52 KB)
📄 vdev_mirror.c(26.76 KB)
📄 vdev_missing.c(3.74 KB)
📄 vdev_queue.c(37.61 KB)
📄 vdev_raidz.c(76.09 KB)
📄 vdev_raidz_math.c(17.18 KB)
📄 vdev_raidz_math_aarch64_neon.c(112.13 KB)
📄 vdev_raidz_math_aarch64_neon_common.h(23.64 KB)
📄 vdev_raidz_math_aarch64_neonx2.c(5.39 KB)
📄 vdev_raidz_math_avx2.c(10.64 KB)
📄 vdev_raidz_math_avx512bw.c(10.93 KB)
📄 vdev_raidz_math_avx512f.c(17.9 KB)
📄 vdev_raidz_math_impl.h(34.78 KB)
📄 vdev_raidz_math_powerpc_altivec.c(213.46 KB)
📄 vdev_raidz_math_powerpc_altivec_common.h(22.62 KB)
📄 vdev_raidz_math_scalar.c(9.54 KB)
📄 vdev_raidz_math_sse2.c(25.84 KB)
📄 vdev_raidz_math_ssse3.c(118 KB)
📄 vdev_rebuild.c(34.65 KB)
📄 vdev_removal.c(72.08 KB)
📄 vdev_root.c(4.28 KB)
📄 vdev_trim.c(50.92 KB)
📄 zap.c(35.35 KB)
📄 zap_leaf.c(22.75 KB)
📄 zap_micro.c(42.18 KB)
📄 zcp.c(42.29 KB)
📄 zcp_get.c(21.01 KB)
📄 zcp_global.c(1.9 KB)
📄 zcp_iter.c(18.21 KB)
📄 zcp_set.c(2.37 KB)
📄 zcp_synctask.c(13.69 KB)
📄 zfeature.c(17.6 KB)
📄 zfs_byteswap.c(5.64 KB)
📄 zfs_fm.c(40.58 KB)
📄 zfs_fuid.c(19.69 KB)
📄 zfs_ioctl.c(195.48 KB)
📄 zfs_log.c(20.6 KB)
📄 zfs_onexit.c(5.09 KB)
📄 zfs_quota.c(12.93 KB)
📄 zfs_ratelimit.c(2.35 KB)
📄 zfs_replay.c(26.11 KB)
📄 zfs_rlock.c(20.38 KB)
📄 zfs_sa.c(13.09 KB)
📄 zfs_vnops.c(22 KB)
📄 zil.c(111.32 KB)
📄 zio.c(144.72 KB)
📄 zio_checksum.c(17.28 KB)
📄 zio_compress.c(5.92 KB)
📄 zio_inject.c(25.79 KB)
📄 zle.c(2.52 KB)
📄 zrlock.c(4.45 KB)
📄 zthr.c(16.41 KB)
📄 zvol.c(42.96 KB)

Editing: aggsum.c

/*
 * CDDL HEADER START
 *
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2017, 2018 by Delphix. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/aggsum.h>

/*
 * Aggregate-sum counters are a form of fanned-out counter, used when atomic
 * instructions on a single field cause enough CPU cache line contention to
 * slow system performance. Due to their increased overhead and the expense
 * involved with precisely reading from them, they should only be used in cases
 * where the write rate (increment/decrement) is much higher than the read rate
 * (get value).
 *
 * Aggregate sum counters are comprised of two basic parts, the core and the
 * buckets. The core counter contains a lock for the entire counter, as well
 * as the current upper and lower bounds on the value of the counter. The
 * aggsum_bucket structure contains a per-bucket lock to protect the contents of
 * the bucket, the current amount that this bucket has changed from the global
 * counter (called the delta), and the amount of increment and decrement we have
 * "borrowed" from the core counter.
 *
 * The basic operation of an aggsum is simple. Threads that wish to modify the
 * counter will modify one bucket's counter (determined by their current CPU, to
 * help minimize lock and cache contention). If the bucket already has
 * sufficient capacity borrowed from the core structure to handle their request,
 * they simply modify the delta and return.  If the bucket does not, we clear
 * the bucket's current state (to prevent the borrowed amounts from getting too
 * large), and borrow more from the core counter. Borrowing is done by adding to
 * the upper bound (or subtracting from the lower bound) of the core counter,
 * and setting the borrow value for the bucket to the amount added (or
 * subtracted).  Clearing the bucket is the opposite; we add the current delta
 * to both the lower and upper bounds of the core counter, subtract the borrowed
 * incremental from the upper bound, and add the borrowed decrement from the
 * lower bound.  Note that only borrowing and clearing require access to the
 * core counter; since all other operations access CPU-local resources,
 * performance can be much higher than a traditional counter.
 *
 * Threads that wish to read from the counter have a slightly more challenging
 * task. It is fast to determine the upper and lower bounds of the aggum; this
 * does not require grabbing any locks. This suffices for cases where an
 * approximation of the aggsum's value is acceptable. However, if one needs to
 * know whether some specific value is above or below the current value in the
 * aggsum, they invoke aggsum_compare(). This function operates by repeatedly
 * comparing the target value to the upper and lower bounds of the aggsum, and
 * then clearing a bucket. This proceeds until the target is outside of the
 * upper and lower bounds and we return a response, or the last bucket has been
 * cleared and we know that the target is equal to the aggsum's value. Finally,
 * the most expensive operation is determining the precise value of the aggsum.
 * To do this, we clear every bucket and then return the upper bound (which must
 * be equal to the lower bound). What makes aggsum_compare() and aggsum_value()
 * expensive is clearing buckets. This involves grabbing the global lock
 * (serializing against themselves and borrow operations), grabbing a bucket's
 * lock (preventing threads on those CPUs from modifying their delta), and
 * zeroing out the borrowed value (forcing that thread to borrow on its next
 * request, which will also be expensive).  This is what makes aggsums well
 * suited for write-many read-rarely operations.
 *
 * Note that the aggsums do not expand if more CPUs are hot-added. In that
 * case, we will have less fanout than boot_ncpus, but we don't want to always
 * reserve the RAM necessary to create the extra slots for additional CPUs up
 * front, and dynamically adding them is a complex task.
 */

/*
 * We will borrow aggsum_borrow_multiplier times the current request, so we will
 * have to get the as_lock approximately every aggsum_borrow_multiplier calls to
 * aggsum_delta().
 */
static uint_t aggsum_borrow_multiplier = 10;

void
aggsum_init(aggsum_t *as, uint64_t value)
{
	bzero(as, sizeof (*as));
	as->as_lower_bound = as->as_upper_bound = value;
	mutex_init(&as->as_lock, NULL, MUTEX_DEFAULT, NULL);
	as->as_numbuckets = boot_ncpus;
	as->as_buckets = kmem_zalloc(boot_ncpus * sizeof (aggsum_bucket_t),
	    KM_SLEEP);
	for (int i = 0; i < as->as_numbuckets; i++) {
		mutex_init(&as->as_buckets[i].asc_lock,
		    NULL, MUTEX_DEFAULT, NULL);
	}
}

void
aggsum_fini(aggsum_t *as)
{
	for (int i = 0; i < as->as_numbuckets; i++)
		mutex_destroy(&as->as_buckets[i].asc_lock);
	kmem_free(as->as_buckets, as->as_numbuckets * sizeof (aggsum_bucket_t));
	mutex_destroy(&as->as_lock);
}

int64_t
aggsum_lower_bound(aggsum_t *as)
{
	return (as->as_lower_bound);
}

int64_t
aggsum_upper_bound(aggsum_t *as)
{
	return (as->as_upper_bound);
}

static void
aggsum_flush_bucket(aggsum_t *as, struct aggsum_bucket *asb)
{
	ASSERT(MUTEX_HELD(&as->as_lock));
	ASSERT(MUTEX_HELD(&asb->asc_lock));

/*
	 * We use atomic instructions for this because we read the upper and
	 * lower bounds without the lock, so we need stores to be atomic.
	 */
	atomic_add_64((volatile uint64_t *)&as->as_lower_bound,
	    asb->asc_delta + asb->asc_borrowed);
	atomic_add_64((volatile uint64_t *)&as->as_upper_bound,
	    asb->asc_delta - asb->asc_borrowed);
	asb->asc_delta = 0;
	asb->asc_borrowed = 0;
}

uint64_t
aggsum_value(aggsum_t *as)
{
	int64_t rv;

mutex_enter(&as->as_lock);
	if (as->as_lower_bound == as->as_upper_bound) {
		rv = as->as_lower_bound;
		for (int i = 0; i < as->as_numbuckets; i++) {
			ASSERT0(as->as_buckets[i].asc_delta);
			ASSERT0(as->as_buckets[i].asc_borrowed);
		}
		mutex_exit(&as->as_lock);
		return (rv);
	}
	for (int i = 0; i < as->as_numbuckets; i++) {
		struct aggsum_bucket *asb = &as->as_buckets[i];
		mutex_enter(&asb->asc_lock);
		aggsum_flush_bucket(as, asb);
		mutex_exit(&asb->asc_lock);
	}
	VERIFY3U(as->as_lower_bound, ==, as->as_upper_bound);
	rv = as->as_lower_bound;
	mutex_exit(&as->as_lock);

return (rv);
}

void
aggsum_add(aggsum_t *as, int64_t delta)
{
	struct aggsum_bucket *asb;
	int64_t borrow;

asb = &as->as_buckets[CPU_SEQID_UNSTABLE % as->as_numbuckets];

/* Try fast path if we already borrowed enough before. */
	mutex_enter(&asb->asc_lock);
	if (asb->asc_delta + delta <= (int64_t)asb->asc_borrowed &&
	    asb->asc_delta + delta >= -(int64_t)asb->asc_borrowed) {
		asb->asc_delta += delta;
		mutex_exit(&asb->asc_lock);
		return;
	}
	mutex_exit(&asb->asc_lock);

/*
	 * We haven't borrowed enough.  Take the global lock and borrow
	 * considering what is requested now and what we borrowed before.
	 */
	borrow = (delta < 0 ? -delta : delta) * aggsum_borrow_multiplier;
	mutex_enter(&as->as_lock);
	mutex_enter(&asb->asc_lock);
	delta += asb->asc_delta;
	asb->asc_delta = 0;
	if (borrow >= asb->asc_borrowed)
		borrow -= asb->asc_borrowed;
	else
		borrow = (borrow - (int64_t)asb->asc_borrowed) / 4;
	asb->asc_borrowed += borrow;
	atomic_add_64((volatile uint64_t *)&as->as_lower_bound,
	    delta - borrow);
	atomic_add_64((volatile uint64_t *)&as->as_upper_bound,
	    delta + borrow);
	mutex_exit(&asb->asc_lock);
	mutex_exit(&as->as_lock);
}

/*
 * Compare the aggsum value to target efficiently. Returns -1 if the value
 * represented by the aggsum is less than target, 1 if it's greater, and 0 if
 * they are equal.
 */
int
aggsum_compare(aggsum_t *as, uint64_t target)
{
	if (as->as_upper_bound < target)
		return (-1);
	if (as->as_lower_bound > target)
		return (1);
	mutex_enter(&as->as_lock);
	for (int i = 0; i < as->as_numbuckets; i++) {
		struct aggsum_bucket *asb = &as->as_buckets[i];
		mutex_enter(&asb->asc_lock);
		aggsum_flush_bucket(as, asb);
		mutex_exit(&asb->asc_lock);
		if (as->as_upper_bound < target) {
			mutex_exit(&as->as_lock);
			return (-1);
		}
		if (as->as_lower_bound > target) {
			mutex_exit(&as->as_lock);
			return (1);
		}
	}
	VERIFY3U(as->as_lower_bound, ==, as->as_upper_bound);
	ASSERT3U(as->as_lower_bound, ==, target);
	mutex_exit(&as->as_lock);
	return (0);
}

003 File Manager

Editing: aggsum.c

Upload File

Create Folder