File Manager

003 File Manager

Current Path: /usr/src/sys/contrib/openzfs/module/zfs

usr / src / sys / contrib / openzfs / module / zfs /

📁 ..
📄 Makefile.in(4.57 KB)
📄 THIRDPARTYLICENSE.cityhash(1.03 KB)
📄 THIRDPARTYLICENSE.cityhash.descrip(39 B)
📄 abd.c(31.72 KB)
📄 aggsum.c(8.46 KB)
📄 arc.c(330.91 KB)
📄 blkptr.c(4.26 KB)
📄 bplist.c(2.38 KB)
📄 bpobj.c(27.82 KB)
📄 bptree.c(8.2 KB)
📄 bqueue.c(4.7 KB)
📄 btree.c(65.19 KB)
📄 dataset_kstats.c(6.32 KB)
📄 dbuf.c(141.84 KB)
📄 dbuf_stats.c(6.05 KB)
📄 ddt.c(28.01 KB)
📄 ddt_zap.c(4.38 KB)
📄 dmu.c(60.44 KB)
📄 dmu_diff.c(6.25 KB)
📄 dmu_object.c(14.73 KB)
📄 dmu_objset.c(79.38 KB)
📄 dmu_recv.c(98.69 KB)
📄 dmu_redact.c(37.09 KB)
📄 dmu_send.c(95.51 KB)
📄 dmu_traverse.c(21.98 KB)
📄 dmu_tx.c(40.13 KB)
📄 dmu_zfetch.c(13.56 KB)
📄 dnode.c(71.32 KB)
📄 dnode_sync.c(24.93 KB)
📄 dsl_bookmark.c(51.22 KB)
📄 dsl_crypt.c(78.06 KB)
📄 dsl_dataset.c(141.61 KB)
📄 dsl_deadlist.c(27.94 KB)
📄 dsl_deleg.c(19.92 KB)
📄 dsl_destroy.c(36.73 KB)
📄 dsl_dir.c(65.45 KB)
📄 dsl_pool.c(42.92 KB)
📄 dsl_prop.c(32.88 KB)
📄 dsl_scan.c(134.47 KB)
📄 dsl_synctask.c(7.94 KB)
📄 dsl_userhold.c(18.26 KB)
📄 edonr_zfs.c(3.21 KB)
📄 fm.c(39.75 KB)
📄 gzip.c(2.57 KB)
📄 hkdf.c(4.5 KB)
📄 lz4.c(26.46 KB)
📄 lzjb.c(3.85 KB)
📄 metaslab.c(190.62 KB)
📄 mmp.c(24.93 KB)
📄 multilist.c(12.21 KB)
📄 objlist.c(2.58 KB)
📄 pathname.c(2.52 KB)
📄 range_tree.c(24.9 KB)
📄 refcount.c(7.82 KB)
📄 rrwlock.c(10.82 KB)
📄 sa.c(59.23 KB)
📄 sha256.c(2.89 KB)
📄 skein_zfs.c(2.92 KB)
📄 spa.c(272.81 KB)
📄 spa_boot.c(1.28 KB)
📄 spa_checkpoint.c(21.54 KB)
📄 spa_config.c(16.97 KB)
📄 spa_errlog.c(11.35 KB)
📄 spa_history.c(17.75 KB)
📄 spa_log_spacemap.c(47.13 KB)
📄 spa_misc.c(75.4 KB)
📄 spa_stats.c(27.08 KB)
📄 space_map.c(31.45 KB)
📄 space_reftree.c(4.21 KB)
📄 txg.c(27.49 KB)
📄 uberblock.c(2.17 KB)
📄 unique.c(2.53 KB)
📄 vdev.c(143.91 KB)
📄 vdev_cache.c(11.71 KB)
📄 vdev_draid.c(96.86 KB)
📄 vdev_draid_rand.c(1.16 KB)
📄 vdev_indirect.c(60.83 KB)
📄 vdev_indirect_births.c(5.88 KB)
📄 vdev_indirect_mapping.c(18.06 KB)
📄 vdev_initialize.c(22.38 KB)
📄 vdev_label.c(57.52 KB)
📄 vdev_mirror.c(26.76 KB)
📄 vdev_missing.c(3.74 KB)
📄 vdev_queue.c(37.61 KB)
📄 vdev_raidz.c(76.09 KB)
📄 vdev_raidz_math.c(17.18 KB)
📄 vdev_raidz_math_aarch64_neon.c(112.13 KB)
📄 vdev_raidz_math_aarch64_neon_common.h(23.64 KB)
📄 vdev_raidz_math_aarch64_neonx2.c(5.39 KB)
📄 vdev_raidz_math_avx2.c(10.64 KB)
📄 vdev_raidz_math_avx512bw.c(10.93 KB)
📄 vdev_raidz_math_avx512f.c(17.9 KB)
📄 vdev_raidz_math_impl.h(34.78 KB)
📄 vdev_raidz_math_powerpc_altivec.c(213.46 KB)
📄 vdev_raidz_math_powerpc_altivec_common.h(22.62 KB)
📄 vdev_raidz_math_scalar.c(9.54 KB)
📄 vdev_raidz_math_sse2.c(25.84 KB)
📄 vdev_raidz_math_ssse3.c(118 KB)
📄 vdev_rebuild.c(34.65 KB)
📄 vdev_removal.c(72.08 KB)
📄 vdev_root.c(4.28 KB)
📄 vdev_trim.c(50.92 KB)
📄 zap.c(35.35 KB)
📄 zap_leaf.c(22.75 KB)
📄 zap_micro.c(42.18 KB)
📄 zcp.c(42.29 KB)
📄 zcp_get.c(21.01 KB)
📄 zcp_global.c(1.9 KB)
📄 zcp_iter.c(18.21 KB)
📄 zcp_set.c(2.37 KB)
📄 zcp_synctask.c(13.69 KB)
📄 zfeature.c(17.6 KB)
📄 zfs_byteswap.c(5.64 KB)
📄 zfs_fm.c(40.58 KB)
📄 zfs_fuid.c(19.69 KB)
📄 zfs_ioctl.c(195.48 KB)
📄 zfs_log.c(20.6 KB)
📄 zfs_onexit.c(5.09 KB)
📄 zfs_quota.c(12.93 KB)
📄 zfs_ratelimit.c(2.35 KB)
📄 zfs_replay.c(26.11 KB)
📄 zfs_rlock.c(20.38 KB)
📄 zfs_sa.c(13.09 KB)
📄 zfs_vnops.c(22 KB)
📄 zil.c(111.32 KB)
📄 zio.c(144.72 KB)
📄 zio_checksum.c(17.28 KB)
📄 zio_compress.c(5.92 KB)
📄 zio_inject.c(25.79 KB)
📄 zle.c(2.52 KB)
📄 zrlock.c(4.45 KB)
📄 zthr.c(16.41 KB)
📄 zvol.c(42.96 KB)

Editing: vdev_raidz_math_avx2.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
 */
#include <sys/isa_defs.h>

#if defined(__x86_64) && defined(HAVE_AVX2)

#include <sys/types.h>
#include <sys/simd.h>

#ifdef __linux__
#define	__asm __asm__ __volatile__
#endif

#define	_REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
#define	REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)

#define	VR0_(REG, ...) "ymm"#REG
#define	VR1_(_1, REG, ...) "ymm"#REG
#define	VR2_(_1, _2, REG, ...) "ymm"#REG
#define	VR3_(_1, _2, _3, REG, ...) "ymm"#REG
#define	VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
#define	VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
#define	VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
#define	VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG

#define	VR0(r...) VR0_(r)
#define	VR1(r...) VR1_(r)
#define	VR2(r...) VR2_(r, 1)
#define	VR3(r...) VR3_(r, 1, 2)
#define	VR4(r...) VR4_(r, 1, 2)
#define	VR5(r...) VR5_(r, 1, 2, 3)
#define	VR6(r...) VR6_(r, 1, 2, 3, 4)
#define	VR7(r...) VR7_(r, 1, 2, 3, 4, 5)

#define	R_01(REG1, REG2, ...) REG1, REG2
#define	_R_23(_0, _1, REG2, REG3, ...) REG2, REG3
#define	R_23(REG...) _R_23(REG, 1, 2, 3)

#define	ZFS_ASM_BUG()	ASSERT(0)

extern const uint8_t gf_clmul_mod_lt[4*256][16];

#define	ELEM_SIZE 32

typedef struct v {
	uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
} v_t;

#define	XOR_ACC(src, r...)						\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
		    "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n"	\
		    "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n"	\
		    : : [SRC] "r" (src));				\
		break;							\
	case 2:								\
		__asm(							\
		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
		    : : [SRC] "r" (src));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	XOR(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		    "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n"	\
		    "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n"	\
		    "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n"	\
		    "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r));		\
		break;							\
	case 4:								\
		__asm(							\
		    "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n"	\
		    "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r));		\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	ZERO(r...)	XOR(r, r)

#define	COPY(r...) 							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		    "vmovdqa %" VR0(r) ", %" VR4(r) "\n"		\
		    "vmovdqa %" VR1(r) ", %" VR5(r) "\n"		\
		    "vmovdqa %" VR2(r) ", %" VR6(r) "\n"		\
		    "vmovdqa %" VR3(r) ", %" VR7(r));			\
		break;							\
	case 4:								\
		__asm(							\
		    "vmovdqa %" VR0(r) ", %" VR2(r) "\n"		\
		    "vmovdqa %" VR1(r) ", %" VR3(r));			\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	LOAD(src, r...) 						\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
		    "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n"		\
		    "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n"		\
		    : : [SRC] "r" (src));				\
		break;							\
	case 2:								\
		__asm(							\
		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
		    : : [SRC] "r" (src));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	STORE(dst, r...)   						\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
		    "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n"		\
		    "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n"		\
		    : : [DST] "r" (dst));				\
		break;							\
	case 2:								\
		__asm(							\
		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
		    : : [DST] "r" (dst));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	FLUSH()								\
{									\
	__asm("vzeroupper");						\
}

#define	MUL2_SETUP() 							\
{   									\
	__asm("vmovq %0,   %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d));	\
	__asm("vpbroadcastq %xmm14, %ymm14");				\
	__asm("vpxor        %ymm15, %ymm15 ,%ymm15");			\
}

#define	_MUL2(r...) 							\
{									\
	switch	(REG_CNT(r)) {						\
	case 2:								\
		__asm(							\
		    "vpcmpgtb %" VR0(r)", %ymm15,     %ymm12\n"		\
		    "vpcmpgtb %" VR1(r)", %ymm15,     %ymm13\n"		\
		    "vpaddb   %" VR0(r)", %" VR0(r)", %" VR0(r) "\n"	\
		    "vpaddb   %" VR1(r)", %" VR1(r)", %" VR1(r) "\n"	\
		    "vpand    %ymm14,     %ymm12,     %ymm12\n"		\
		    "vpand    %ymm14,     %ymm13,     %ymm13\n"		\
		    "vpxor    %ymm12,     %" VR0(r)", %" VR0(r) "\n"	\
		    "vpxor    %ymm13,     %" VR1(r)", %" VR1(r));	\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL2(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
	    _MUL2(R_01(r));						\
	    _MUL2(R_23(r));						\
	    break;							\
	case 2:								\
	    _MUL2(r);							\
	    break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL4(r...)							\
{									\
	MUL2(r);							\
	MUL2(r);							\
}

#define	_0f		"ymm15"
#define	_as		"ymm14"
#define	_bs		"ymm13"
#define	_ltmod		"ymm12"
#define	_ltmul		"ymm11"
#define	_ta		"ymm10"
#define	_tb		"ymm15"

static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;

#define	_MULx2(c, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 2:								\
		__asm(							\
		    "vpbroadcastb (%[mask]), %%" _0f "\n"		\
		    /* upper bits */					\
		    "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n"	\
		    "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n"	\
									\
		    "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n"		\
		    "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n"		\
		    "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n"	\
		    "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n"	\
		    "vpand %%" _0f ", %%" _as ", %%" _as "\n"		\
		    "vpand %%" _0f ", %%" _bs ", %%" _bs "\n"		\
									\
		    "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n"	\
		    "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n"	\
		    "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n"	\
		    "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n"	\
		    /* lower bits */					\
		    "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n"	\
		    "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n"	\
									\
		    "vpxor %%" _ta ", %%" _as ", %%" _as "\n"		\
		    "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n"		\
									\
		    "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n"	\
		    "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n"	\
		    "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
		    "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
									\
		    "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n"	\
		    "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n"	\
		    "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n"	\
		    "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n"	\
		    : : [mask] "r" (&_mul_mask),			\
		    [lt] "r" (gf_clmul_mod_lt[4*(c)]));			\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL(c, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		_MULx2(c, R_01(r));					\
		_MULx2(c, R_23(r));					\
		break;							\
	case 2:								\
		_MULx2(c, R_01(r));					\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	raidz_math_begin()	kfpu_begin()
#define	raidz_math_end()						\
{									\
	FLUSH();							\
	kfpu_end();							\
}

#define	SYN_STRIDE		4

#define	ZERO_STRIDE		4
#define	ZERO_DEFINE()		{}
#define	ZERO_D			0, 1, 2, 3

#define	COPY_STRIDE		4
#define	COPY_DEFINE()		{}
#define	COPY_D			0, 1, 2, 3

#define	ADD_STRIDE		4
#define	ADD_DEFINE()		{}
#define	ADD_D 			0, 1, 2, 3

#define	MUL_STRIDE		4
#define	MUL_DEFINE() 		{}
#define	MUL_D			0, 1, 2, 3

#define	GEN_P_STRIDE		4
#define	GEN_P_DEFINE()		{}
#define	GEN_P_P			0, 1, 2, 3

#define	GEN_PQ_STRIDE		4
#define	GEN_PQ_DEFINE() 	{}
#define	GEN_PQ_D		0, 1, 2, 3
#define	GEN_PQ_C		4, 5, 6, 7

#define	GEN_PQR_STRIDE		4
#define	GEN_PQR_DEFINE() 	{}
#define	GEN_PQR_D		0, 1, 2, 3
#define	GEN_PQR_C		4, 5, 6, 7

#define	SYN_Q_DEFINE()		{}
#define	SYN_Q_D			0, 1, 2, 3
#define	SYN_Q_X			4, 5, 6, 7

#define	SYN_R_DEFINE()		{}
#define	SYN_R_D			0, 1, 2, 3
#define	SYN_R_X			4, 5, 6, 7

#define	SYN_PQ_DEFINE() 	{}
#define	SYN_PQ_D		0, 1, 2, 3
#define	SYN_PQ_X		4, 5, 6, 7

#define	REC_PQ_STRIDE		2
#define	REC_PQ_DEFINE() 	{}
#define	REC_PQ_X		0, 1
#define	REC_PQ_Y		2, 3
#define	REC_PQ_T		4, 5

#define	SYN_PR_DEFINE() 	{}
#define	SYN_PR_D		0, 1, 2, 3
#define	SYN_PR_X		4, 5, 6, 7

#define	REC_PR_STRIDE		2
#define	REC_PR_DEFINE() 	{}
#define	REC_PR_X		0, 1
#define	REC_PR_Y		2, 3
#define	REC_PR_T		4, 5

#define	SYN_QR_DEFINE() 	{}
#define	SYN_QR_D		0, 1, 2, 3
#define	SYN_QR_X		4, 5, 6, 7

#define	REC_QR_STRIDE		2
#define	REC_QR_DEFINE() 	{}
#define	REC_QR_X		0, 1
#define	REC_QR_Y		2, 3
#define	REC_QR_T		4, 5

#define	SYN_PQR_DEFINE() 	{}
#define	SYN_PQR_D		0, 1, 2, 3
#define	SYN_PQR_X		4, 5, 6, 7

#define	REC_PQR_STRIDE		2
#define	REC_PQR_DEFINE() 	{}
#define	REC_PQR_X		0, 1
#define	REC_PQR_Y		2, 3
#define	REC_PQR_Z		4, 5
#define	REC_PQR_XS		6, 7
#define	REC_PQR_YS		8, 9

#include <sys/vdev_raidz_impl.h>
#include "vdev_raidz_math_impl.h"

DEFINE_GEN_METHODS(avx2);
DEFINE_REC_METHODS(avx2);

static boolean_t
raidz_will_avx2_work(void)
{
	return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
}

const raidz_impl_ops_t vdev_raidz_avx2_impl = {
	.init = NULL,
	.fini = NULL,
	.gen = RAIDZ_GEN_METHODS(avx2),
	.rec = RAIDZ_REC_METHODS(avx2),
	.is_supported = &raidz_will_avx2_work,
	.name = "avx2"
};

#endif /* defined(__x86_64) && defined(HAVE_AVX2) */

003 File Manager

Editing: vdev_raidz_math_avx2.c

Upload File

Create Folder