File Manager

003 File Manager

Current Path: /usr/src/sys/contrib/openzfs/module/zfs

usr / src / sys / contrib / openzfs / module / zfs /

📁 ..
📄 Makefile.in(4.57 KB)
📄 THIRDPARTYLICENSE.cityhash(1.03 KB)
📄 THIRDPARTYLICENSE.cityhash.descrip(39 B)
📄 abd.c(31.72 KB)
📄 aggsum.c(8.46 KB)
📄 arc.c(330.91 KB)
📄 blkptr.c(4.26 KB)
📄 bplist.c(2.38 KB)
📄 bpobj.c(27.82 KB)
📄 bptree.c(8.2 KB)
📄 bqueue.c(4.7 KB)
📄 btree.c(65.19 KB)
📄 dataset_kstats.c(6.32 KB)
📄 dbuf.c(141.84 KB)
📄 dbuf_stats.c(6.05 KB)
📄 ddt.c(28.01 KB)
📄 ddt_zap.c(4.38 KB)
📄 dmu.c(60.44 KB)
📄 dmu_diff.c(6.25 KB)
📄 dmu_object.c(14.73 KB)
📄 dmu_objset.c(79.38 KB)
📄 dmu_recv.c(98.69 KB)
📄 dmu_redact.c(37.09 KB)
📄 dmu_send.c(95.51 KB)
📄 dmu_traverse.c(21.98 KB)
📄 dmu_tx.c(40.13 KB)
📄 dmu_zfetch.c(13.56 KB)
📄 dnode.c(71.32 KB)
📄 dnode_sync.c(24.93 KB)
📄 dsl_bookmark.c(51.22 KB)
📄 dsl_crypt.c(78.06 KB)
📄 dsl_dataset.c(141.61 KB)
📄 dsl_deadlist.c(27.94 KB)
📄 dsl_deleg.c(19.92 KB)
📄 dsl_destroy.c(36.73 KB)
📄 dsl_dir.c(65.45 KB)
📄 dsl_pool.c(42.92 KB)
📄 dsl_prop.c(32.88 KB)
📄 dsl_scan.c(134.47 KB)
📄 dsl_synctask.c(7.94 KB)
📄 dsl_userhold.c(18.26 KB)
📄 edonr_zfs.c(3.21 KB)
📄 fm.c(39.75 KB)
📄 gzip.c(2.57 KB)
📄 hkdf.c(4.5 KB)
📄 lz4.c(26.46 KB)
📄 lzjb.c(3.85 KB)
📄 metaslab.c(190.62 KB)
📄 mmp.c(24.93 KB)
📄 multilist.c(12.21 KB)
📄 objlist.c(2.58 KB)
📄 pathname.c(2.52 KB)
📄 range_tree.c(24.9 KB)
📄 refcount.c(7.82 KB)
📄 rrwlock.c(10.82 KB)
📄 sa.c(59.23 KB)
📄 sha256.c(2.89 KB)
📄 skein_zfs.c(2.92 KB)
📄 spa.c(272.81 KB)
📄 spa_boot.c(1.28 KB)
📄 spa_checkpoint.c(21.54 KB)
📄 spa_config.c(16.97 KB)
📄 spa_errlog.c(11.35 KB)
📄 spa_history.c(17.75 KB)
📄 spa_log_spacemap.c(47.13 KB)
📄 spa_misc.c(75.4 KB)
📄 spa_stats.c(27.08 KB)
📄 space_map.c(31.45 KB)
📄 space_reftree.c(4.21 KB)
📄 txg.c(27.49 KB)
📄 uberblock.c(2.17 KB)
📄 unique.c(2.53 KB)
📄 vdev.c(143.91 KB)
📄 vdev_cache.c(11.71 KB)
📄 vdev_draid.c(96.86 KB)
📄 vdev_draid_rand.c(1.16 KB)
📄 vdev_indirect.c(60.83 KB)
📄 vdev_indirect_births.c(5.88 KB)
📄 vdev_indirect_mapping.c(18.06 KB)
📄 vdev_initialize.c(22.38 KB)
📄 vdev_label.c(57.52 KB)
📄 vdev_mirror.c(26.76 KB)
📄 vdev_missing.c(3.74 KB)
📄 vdev_queue.c(37.61 KB)
📄 vdev_raidz.c(76.09 KB)
📄 vdev_raidz_math.c(17.18 KB)
📄 vdev_raidz_math_aarch64_neon.c(112.13 KB)
📄 vdev_raidz_math_aarch64_neon_common.h(23.64 KB)
📄 vdev_raidz_math_aarch64_neonx2.c(5.39 KB)
📄 vdev_raidz_math_avx2.c(10.64 KB)
📄 vdev_raidz_math_avx512bw.c(10.93 KB)
📄 vdev_raidz_math_avx512f.c(17.9 KB)
📄 vdev_raidz_math_impl.h(34.78 KB)
📄 vdev_raidz_math_powerpc_altivec.c(213.46 KB)
📄 vdev_raidz_math_powerpc_altivec_common.h(22.62 KB)
📄 vdev_raidz_math_scalar.c(9.54 KB)
📄 vdev_raidz_math_sse2.c(25.84 KB)
📄 vdev_raidz_math_ssse3.c(118 KB)
📄 vdev_rebuild.c(34.65 KB)
📄 vdev_removal.c(72.08 KB)
📄 vdev_root.c(4.28 KB)
📄 vdev_trim.c(50.92 KB)
📄 zap.c(35.35 KB)
📄 zap_leaf.c(22.75 KB)
📄 zap_micro.c(42.18 KB)
📄 zcp.c(42.29 KB)
📄 zcp_get.c(21.01 KB)
📄 zcp_global.c(1.9 KB)
📄 zcp_iter.c(18.21 KB)
📄 zcp_set.c(2.37 KB)
📄 zcp_synctask.c(13.69 KB)
📄 zfeature.c(17.6 KB)
📄 zfs_byteswap.c(5.64 KB)
📄 zfs_fm.c(40.58 KB)
📄 zfs_fuid.c(19.69 KB)
📄 zfs_ioctl.c(195.48 KB)
📄 zfs_log.c(20.6 KB)
📄 zfs_onexit.c(5.09 KB)
📄 zfs_quota.c(12.93 KB)
📄 zfs_ratelimit.c(2.35 KB)
📄 zfs_replay.c(26.11 KB)
📄 zfs_rlock.c(20.38 KB)
📄 zfs_sa.c(13.09 KB)
📄 zfs_vnops.c(22 KB)
📄 zil.c(111.32 KB)
📄 zio.c(144.72 KB)
📄 zio_checksum.c(17.28 KB)
📄 zio_compress.c(5.92 KB)
📄 zio_inject.c(25.79 KB)
📄 zle.c(2.52 KB)
📄 zrlock.c(4.45 KB)
📄 zthr.c(16.41 KB)
📄 zvol.c(42.96 KB)

Editing: rrwlock.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright (c) 2012 by Delphix. All rights reserved.
 */

#include <sys/rrwlock.h>
#include <sys/trace_zfs.h>

/*
 * This file contains the implementation of a re-entrant read
 * reader/writer lock (aka "rrwlock").
 *
 * This is a normal reader/writer lock with the additional feature
 * of allowing threads who have already obtained a read lock to
 * re-enter another read lock (re-entrant read) - even if there are
 * waiting writers.
 *
 * Callers who have not obtained a read lock give waiting writers priority.
 *
 * The rrwlock_t lock does not allow re-entrant writers, nor does it
 * allow a re-entrant mix of reads and writes (that is, it does not
 * allow a caller who has already obtained a read lock to be able to
 * then grab a write lock without first dropping all read locks, and
 * vice versa).
 *
 * The rrwlock_t uses tsd (thread specific data) to keep a list of
 * nodes (rrw_node_t), where each node keeps track of which specific
 * lock (rrw_node_t::rn_rrl) the thread has grabbed.  Since re-entering
 * should be rare, a thread that grabs multiple reads on the same rrwlock_t
 * will store multiple rrw_node_ts of the same 'rrn_rrl'. Nodes on the
 * tsd list can represent a different rrwlock_t.  This allows a thread
 * to enter multiple and unique rrwlock_ts for read locks at the same time.
 *
 * Since using tsd exposes some overhead, the rrwlock_t only needs to
 * keep tsd data when writers are waiting.  If no writers are waiting, then
 * a reader just bumps the anonymous read count (rr_anon_rcount) - no tsd
 * is needed.  Once a writer attempts to grab the lock, readers then
 * keep tsd data and bump the linked readers count (rr_linked_rcount).
 *
 * If there are waiting writers and there are anonymous readers, then a
 * reader doesn't know if it is a re-entrant lock. But since it may be one,
 * we allow the read to proceed (otherwise it could deadlock).  Since once
 * waiting writers are active, readers no longer bump the anonymous count,
 * the anonymous readers will eventually flush themselves out.  At this point,
 * readers will be able to tell if they are a re-entrant lock (have a
 * rrw_node_t entry for the lock) or not. If they are a re-entrant lock, then
 * we must let the proceed.  If they are not, then the reader blocks for the
 * waiting writers.  Hence, we do not starve writers.
 */

/* global key for TSD */
uint_t rrw_tsd_key;

typedef struct rrw_node {
	struct rrw_node *rn_next;
	rrwlock_t *rn_rrl;
	void *rn_tag;
} rrw_node_t;

static rrw_node_t *
rrn_find(rrwlock_t *rrl)
{
	rrw_node_t *rn;

if (zfs_refcount_count(&rrl->rr_linked_rcount) == 0)
		return (NULL);

for (rn = tsd_get(rrw_tsd_key); rn != NULL; rn = rn->rn_next) {
		if (rn->rn_rrl == rrl)
			return (rn);
	}
	return (NULL);
}

/*
 * Add a node to the head of the singly linked list.
 */
static void
rrn_add(rrwlock_t *rrl, void *tag)
{
	rrw_node_t *rn;

rn = kmem_alloc(sizeof (*rn), KM_SLEEP);
	rn->rn_rrl = rrl;
	rn->rn_next = tsd_get(rrw_tsd_key);
	rn->rn_tag = tag;
	VERIFY(tsd_set(rrw_tsd_key, rn) == 0);
}

/*
 * If a node is found for 'rrl', then remove the node from this
 * thread's list and return TRUE; otherwise return FALSE.
 */
static boolean_t
rrn_find_and_remove(rrwlock_t *rrl, void *tag)
{
	rrw_node_t *rn;
	rrw_node_t *prev = NULL;

if (zfs_refcount_count(&rrl->rr_linked_rcount) == 0)
		return (B_FALSE);

for (rn = tsd_get(rrw_tsd_key); rn != NULL; rn = rn->rn_next) {
		if (rn->rn_rrl == rrl && rn->rn_tag == tag) {
			if (prev)
				prev->rn_next = rn->rn_next;
			else
				VERIFY(tsd_set(rrw_tsd_key, rn->rn_next) == 0);
			kmem_free(rn, sizeof (*rn));
			return (B_TRUE);
		}
		prev = rn;
	}
	return (B_FALSE);
}

void
rrw_init(rrwlock_t *rrl, boolean_t track_all)
{
	mutex_init(&rrl->rr_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&rrl->rr_cv, NULL, CV_DEFAULT, NULL);
	rrl->rr_writer = NULL;
	zfs_refcount_create(&rrl->rr_anon_rcount);
	zfs_refcount_create(&rrl->rr_linked_rcount);
	rrl->rr_writer_wanted = B_FALSE;
	rrl->rr_track_all = track_all;
}

void
rrw_destroy(rrwlock_t *rrl)
{
	mutex_destroy(&rrl->rr_lock);
	cv_destroy(&rrl->rr_cv);
	ASSERT(rrl->rr_writer == NULL);
	zfs_refcount_destroy(&rrl->rr_anon_rcount);
	zfs_refcount_destroy(&rrl->rr_linked_rcount);
}

static void
rrw_enter_read_impl(rrwlock_t *rrl, boolean_t prio, void *tag)
{
	mutex_enter(&rrl->rr_lock);
#if !defined(ZFS_DEBUG) && defined(_KERNEL)
	if (rrl->rr_writer == NULL && !rrl->rr_writer_wanted &&
	    !rrl->rr_track_all) {
		rrl->rr_anon_rcount.rc_count++;
		mutex_exit(&rrl->rr_lock);
		return;
	}
	DTRACE_PROBE(zfs__rrwfastpath__rdmiss);
#endif
	ASSERT(rrl->rr_writer != curthread);
	ASSERT(zfs_refcount_count(&rrl->rr_anon_rcount) >= 0);

while (rrl->rr_writer != NULL || (rrl->rr_writer_wanted &&
	    zfs_refcount_is_zero(&rrl->rr_anon_rcount) && !prio &&
	    rrn_find(rrl) == NULL))
		cv_wait(&rrl->rr_cv, &rrl->rr_lock);

if (rrl->rr_writer_wanted || rrl->rr_track_all) {
		/* may or may not be a re-entrant enter */
		rrn_add(rrl, tag);
		(void) zfs_refcount_add(&rrl->rr_linked_rcount, tag);
	} else {
		(void) zfs_refcount_add(&rrl->rr_anon_rcount, tag);
	}
	ASSERT(rrl->rr_writer == NULL);
	mutex_exit(&rrl->rr_lock);
}

void
rrw_enter_read(rrwlock_t *rrl, void *tag)
{
	rrw_enter_read_impl(rrl, B_FALSE, tag);
}

/*
 * take a read lock even if there are pending write lock requests. if we want
 * to take a lock reentrantly, but from different threads (that have a
 * relationship to each other), the normal detection mechanism to overrule
 * the pending writer does not work, so we have to give an explicit hint here.
 */
void
rrw_enter_read_prio(rrwlock_t *rrl, void *tag)
{
	rrw_enter_read_impl(rrl, B_TRUE, tag);
}

void
rrw_enter_write(rrwlock_t *rrl)
{
	mutex_enter(&rrl->rr_lock);
	ASSERT(rrl->rr_writer != curthread);

while (zfs_refcount_count(&rrl->rr_anon_rcount) > 0 ||
	    zfs_refcount_count(&rrl->rr_linked_rcount) > 0 ||
	    rrl->rr_writer != NULL) {
		rrl->rr_writer_wanted = B_TRUE;
		cv_wait(&rrl->rr_cv, &rrl->rr_lock);
	}
	rrl->rr_writer_wanted = B_FALSE;
	rrl->rr_writer = curthread;
	mutex_exit(&rrl->rr_lock);
}

void
rrw_enter(rrwlock_t *rrl, krw_t rw, void *tag)
{
	if (rw == RW_READER)
		rrw_enter_read(rrl, tag);
	else
		rrw_enter_write(rrl);
}

void
rrw_exit(rrwlock_t *rrl, void *tag)
{
	mutex_enter(&rrl->rr_lock);
#if !defined(ZFS_DEBUG) && defined(_KERNEL)
	if (!rrl->rr_writer && rrl->rr_linked_rcount.rc_count == 0) {
		rrl->rr_anon_rcount.rc_count--;
		if (rrl->rr_anon_rcount.rc_count == 0)
			cv_broadcast(&rrl->rr_cv);
		mutex_exit(&rrl->rr_lock);
		return;
	}
	DTRACE_PROBE(zfs__rrwfastpath__exitmiss);
#endif
	ASSERT(!zfs_refcount_is_zero(&rrl->rr_anon_rcount) ||
	    !zfs_refcount_is_zero(&rrl->rr_linked_rcount) ||
	    rrl->rr_writer != NULL);

if (rrl->rr_writer == NULL) {
		int64_t count;
		if (rrn_find_and_remove(rrl, tag)) {
			count = zfs_refcount_remove(
			    &rrl->rr_linked_rcount, tag);
		} else {
			ASSERT(!rrl->rr_track_all);
			count = zfs_refcount_remove(&rrl->rr_anon_rcount, tag);
		}
		if (count == 0)
			cv_broadcast(&rrl->rr_cv);
	} else {
		ASSERT(rrl->rr_writer == curthread);
		ASSERT(zfs_refcount_is_zero(&rrl->rr_anon_rcount) &&
		    zfs_refcount_is_zero(&rrl->rr_linked_rcount));
		rrl->rr_writer = NULL;
		cv_broadcast(&rrl->rr_cv);
	}
	mutex_exit(&rrl->rr_lock);
}

/*
 * If the lock was created with track_all, rrw_held(RW_READER) will return
 * B_TRUE iff the current thread has the lock for reader.  Otherwise it may
 * return B_TRUE if any thread has the lock for reader.
 */
boolean_t
rrw_held(rrwlock_t *rrl, krw_t rw)
{
	boolean_t held;

mutex_enter(&rrl->rr_lock);
	if (rw == RW_WRITER) {
		held = (rrl->rr_writer == curthread);
	} else {
		held = (!zfs_refcount_is_zero(&rrl->rr_anon_rcount) ||
		    rrn_find(rrl) != NULL);
	}
	mutex_exit(&rrl->rr_lock);

return (held);
}

void
rrw_tsd_destroy(void *arg)
{
	rrw_node_t *rn = arg;
	if (rn != NULL) {
		panic("thread %p terminating with rrw lock %p held",
		    (void *)curthread, (void *)rn->rn_rrl);
	}
}

/*
 * A reader-mostly lock implementation, tuning above reader-writer locks
 * for hightly parallel read acquisitions, while pessimizing writes.
 *
 * The idea is to split single busy lock into array of locks, so that
 * each reader can lock only one of them for read, depending on result
 * of simple hash function.  That proportionally reduces lock congestion.
 * Writer at the same time has to sequentially acquire write on all the locks.
 * That makes write acquisition proportionally slower, but in places where
 * it is used (filesystem unmount) performance is not critical.
 *
 * All the functions below are direct wrappers around functions above.
 */
void
rrm_init(rrmlock_t *rrl, boolean_t track_all)
{
	int i;

for (i = 0; i < RRM_NUM_LOCKS; i++)
		rrw_init(&rrl->locks[i], track_all);
}

void
rrm_destroy(rrmlock_t *rrl)
{
	int i;

for (i = 0; i < RRM_NUM_LOCKS; i++)
		rrw_destroy(&rrl->locks[i]);
}

void
rrm_enter(rrmlock_t *rrl, krw_t rw, void *tag)
{
	if (rw == RW_READER)
		rrm_enter_read(rrl, tag);
	else
		rrm_enter_write(rrl);
}

/*
 * This maps the current thread to a specific lock.  Note that the lock
 * must be released by the same thread that acquired it.  We do this
 * mapping by taking the thread pointer mod a prime number.  We examine
 * only the low 32 bits of the thread pointer, because 32-bit division
 * is faster than 64-bit division, and the high 32 bits have little
 * entropy anyway.
 */
#define	RRM_TD_LOCK()	(((uint32_t)(uintptr_t)(curthread)) % RRM_NUM_LOCKS)

void
rrm_enter_read(rrmlock_t *rrl, void *tag)
{
	rrw_enter_read(&rrl->locks[RRM_TD_LOCK()], tag);
}

void
rrm_enter_write(rrmlock_t *rrl)
{
	int i;

for (i = 0; i < RRM_NUM_LOCKS; i++)
		rrw_enter_write(&rrl->locks[i]);
}

void
rrm_exit(rrmlock_t *rrl, void *tag)
{
	int i;

if (rrl->locks[0].rr_writer == curthread) {
		for (i = 0; i < RRM_NUM_LOCKS; i++)
			rrw_exit(&rrl->locks[i], tag);
	} else {
		rrw_exit(&rrl->locks[RRM_TD_LOCK()], tag);
	}
}

boolean_t
rrm_held(rrmlock_t *rrl, krw_t rw)
{
	if (rw == RW_WRITER) {
		return (rrw_held(&rrl->locks[0], rw));
	} else {
		return (rrw_held(&rrl->locks[RRM_TD_LOCK()], rw));
	}
}

003 File Manager

Editing: rrwlock.c

Upload File

Create Folder