003 File Manager
Current Path:
/usr/src/sys/contrib/openzfs/module/zfs
usr
/
src
/
sys
/
contrib
/
openzfs
/
module
/
zfs
/
📁
..
📄
Makefile.in
(4.57 KB)
📄
THIRDPARTYLICENSE.cityhash
(1.03 KB)
📄
THIRDPARTYLICENSE.cityhash.descrip
(39 B)
📄
abd.c
(31.72 KB)
📄
aggsum.c
(8.46 KB)
📄
arc.c
(330.91 KB)
📄
blkptr.c
(4.26 KB)
📄
bplist.c
(2.38 KB)
📄
bpobj.c
(27.82 KB)
📄
bptree.c
(8.2 KB)
📄
bqueue.c
(4.7 KB)
📄
btree.c
(65.19 KB)
📄
dataset_kstats.c
(6.32 KB)
📄
dbuf.c
(141.84 KB)
📄
dbuf_stats.c
(6.05 KB)
📄
ddt.c
(28.01 KB)
📄
ddt_zap.c
(4.38 KB)
📄
dmu.c
(60.44 KB)
📄
dmu_diff.c
(6.25 KB)
📄
dmu_object.c
(14.73 KB)
📄
dmu_objset.c
(79.38 KB)
📄
dmu_recv.c
(98.69 KB)
📄
dmu_redact.c
(37.09 KB)
📄
dmu_send.c
(95.51 KB)
📄
dmu_traverse.c
(21.98 KB)
📄
dmu_tx.c
(40.13 KB)
📄
dmu_zfetch.c
(13.56 KB)
📄
dnode.c
(71.32 KB)
📄
dnode_sync.c
(24.93 KB)
📄
dsl_bookmark.c
(51.22 KB)
📄
dsl_crypt.c
(78.06 KB)
📄
dsl_dataset.c
(141.61 KB)
📄
dsl_deadlist.c
(27.94 KB)
📄
dsl_deleg.c
(19.92 KB)
📄
dsl_destroy.c
(36.73 KB)
📄
dsl_dir.c
(65.45 KB)
📄
dsl_pool.c
(42.92 KB)
📄
dsl_prop.c
(32.88 KB)
📄
dsl_scan.c
(134.47 KB)
📄
dsl_synctask.c
(7.94 KB)
📄
dsl_userhold.c
(18.26 KB)
📄
edonr_zfs.c
(3.21 KB)
📄
fm.c
(39.75 KB)
📄
gzip.c
(2.57 KB)
📄
hkdf.c
(4.5 KB)
📄
lz4.c
(26.46 KB)
📄
lzjb.c
(3.85 KB)
📄
metaslab.c
(190.62 KB)
📄
mmp.c
(24.93 KB)
📄
multilist.c
(12.21 KB)
📄
objlist.c
(2.58 KB)
📄
pathname.c
(2.52 KB)
📄
range_tree.c
(24.9 KB)
📄
refcount.c
(7.82 KB)
📄
rrwlock.c
(10.82 KB)
📄
sa.c
(59.23 KB)
📄
sha256.c
(2.89 KB)
📄
skein_zfs.c
(2.92 KB)
📄
spa.c
(272.81 KB)
📄
spa_boot.c
(1.28 KB)
📄
spa_checkpoint.c
(21.54 KB)
📄
spa_config.c
(16.97 KB)
📄
spa_errlog.c
(11.35 KB)
📄
spa_history.c
(17.75 KB)
📄
spa_log_spacemap.c
(47.13 KB)
📄
spa_misc.c
(75.4 KB)
📄
spa_stats.c
(27.08 KB)
📄
space_map.c
(31.45 KB)
📄
space_reftree.c
(4.21 KB)
📄
txg.c
(27.49 KB)
📄
uberblock.c
(2.17 KB)
📄
unique.c
(2.53 KB)
📄
vdev.c
(143.91 KB)
📄
vdev_cache.c
(11.71 KB)
📄
vdev_draid.c
(96.86 KB)
📄
vdev_draid_rand.c
(1.16 KB)
📄
vdev_indirect.c
(60.83 KB)
📄
vdev_indirect_births.c
(5.88 KB)
📄
vdev_indirect_mapping.c
(18.06 KB)
📄
vdev_initialize.c
(22.38 KB)
📄
vdev_label.c
(57.52 KB)
📄
vdev_mirror.c
(26.76 KB)
📄
vdev_missing.c
(3.74 KB)
📄
vdev_queue.c
(37.61 KB)
📄
vdev_raidz.c
(76.09 KB)
📄
vdev_raidz_math.c
(17.18 KB)
📄
vdev_raidz_math_aarch64_neon.c
(112.13 KB)
📄
vdev_raidz_math_aarch64_neon_common.h
(23.64 KB)
📄
vdev_raidz_math_aarch64_neonx2.c
(5.39 KB)
📄
vdev_raidz_math_avx2.c
(10.64 KB)
📄
vdev_raidz_math_avx512bw.c
(10.93 KB)
📄
vdev_raidz_math_avx512f.c
(17.9 KB)
📄
vdev_raidz_math_impl.h
(34.78 KB)
📄
vdev_raidz_math_powerpc_altivec.c
(213.46 KB)
📄
vdev_raidz_math_powerpc_altivec_common.h
(22.62 KB)
📄
vdev_raidz_math_scalar.c
(9.54 KB)
📄
vdev_raidz_math_sse2.c
(25.84 KB)
📄
vdev_raidz_math_ssse3.c
(118 KB)
📄
vdev_rebuild.c
(34.65 KB)
📄
vdev_removal.c
(72.08 KB)
📄
vdev_root.c
(4.28 KB)
📄
vdev_trim.c
(50.92 KB)
📄
zap.c
(35.35 KB)
📄
zap_leaf.c
(22.75 KB)
📄
zap_micro.c
(42.18 KB)
📄
zcp.c
(42.29 KB)
📄
zcp_get.c
(21.01 KB)
📄
zcp_global.c
(1.9 KB)
📄
zcp_iter.c
(18.21 KB)
📄
zcp_set.c
(2.37 KB)
📄
zcp_synctask.c
(13.69 KB)
📄
zfeature.c
(17.6 KB)
📄
zfs_byteswap.c
(5.64 KB)
📄
zfs_fm.c
(40.58 KB)
📄
zfs_fuid.c
(19.69 KB)
📄
zfs_ioctl.c
(195.48 KB)
📄
zfs_log.c
(20.6 KB)
📄
zfs_onexit.c
(5.09 KB)
📄
zfs_quota.c
(12.93 KB)
📄
zfs_ratelimit.c
(2.35 KB)
📄
zfs_replay.c
(26.11 KB)
📄
zfs_rlock.c
(20.38 KB)
📄
zfs_sa.c
(13.09 KB)
📄
zfs_vnops.c
(22 KB)
📄
zil.c
(111.32 KB)
📄
zio.c
(144.72 KB)
📄
zio_checksum.c
(17.28 KB)
📄
zio_compress.c
(5.92 KB)
📄
zio_inject.c
(25.79 KB)
📄
zle.c
(2.52 KB)
📄
zrlock.c
(4.45 KB)
📄
zthr.c
(16.41 KB)
📄
zvol.c
(42.96 KB)
Editing: vdev_raidz_math_aarch64_neon_common.h
/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Romain Dolbeau. All rights reserved. */ #include <sys/types.h> #include <sys/simd.h> #ifdef __linux__ #define __asm __asm__ __volatile__ #endif #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "%[w"#REG"]" #define VR1_(_1, REG, ...) "%[w"#REG"]" #define VR2_(_1, _2, REG, ...) "%[w"#REG"]" #define VR3_(_1, _2, _3, REG, ...) "%[w"#REG"]" #define VR4_(_1, _2, _3, _4, REG, ...) "%[w"#REG"]" #define VR5_(_1, _2, _3, _4, _5, REG, ...) "%[w"#REG"]" #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "%[w"#REG"]" #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "%[w"#REG"]" /* * Here we need registers not used otherwise. * They will be used in unused ASM for the case * with more registers than required... but GCC * will still need to make sure the constraints * are correct, and duplicate constraints are illegal * ... and we use the "register" number as a name */ #define VR0(r...) VR0_(r) #define VR1(r...) VR1_(r) #define VR2(r...) VR2_(r, 36) #define VR3(r...) VR3_(r, 36, 35) #define VR4(r...) VR4_(r, 36, 35, 34, 33) #define VR5(r...) VR5_(r, 36, 35, 34, 33, 32) #define VR6(r...) VR6_(r, 36, 35, 34, 33, 32, 31) #define VR7(r...) VR7_(r, 36, 35, 34, 33, 32, 31, 30) #define VR(X) "%[w"#X"]" #define RVR0_(REG, ...) [w##REG] "w" (w##REG) #define RVR1_(_1, REG, ...) [w##REG] "w" (w##REG) #define RVR2_(_1, _2, REG, ...) [w##REG] "w" (w##REG) #define RVR3_(_1, _2, _3, REG, ...) [w##REG] "w" (w##REG) #define RVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "w" (w##REG) #define RVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "w" (w##REG) #define RVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "w" (w##REG) #define RVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "w" (w##REG) #define RVR0(r...) RVR0_(r) #define RVR1(r...) RVR1_(r) #define RVR2(r...) RVR2_(r, 36) #define RVR3(r...) RVR3_(r, 36, 35) #define RVR4(r...) RVR4_(r, 36, 35, 34, 33) #define RVR5(r...) RVR5_(r, 36, 35, 34, 33, 32) #define RVR6(r...) RVR6_(r, 36, 35, 34, 33, 32, 31) #define RVR7(r...) RVR7_(r, 36, 35, 34, 33, 32, 31, 30) #define RVR(X) [w##X] "w" (w##X) #define WVR0_(REG, ...) [w##REG] "=w" (w##REG) #define WVR1_(_1, REG, ...) [w##REG] "=w" (w##REG) #define WVR2_(_1, _2, REG, ...) [w##REG] "=w" (w##REG) #define WVR3_(_1, _2, _3, REG, ...) [w##REG] "=w" (w##REG) #define WVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "=w" (w##REG) #define WVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "=w" (w##REG) #define WVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "=w" (w##REG) #define WVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "=w" (w##REG) #define WVR0(r...) WVR0_(r) #define WVR1(r...) WVR1_(r) #define WVR2(r...) WVR2_(r, 36) #define WVR3(r...) WVR3_(r, 36, 35) #define WVR4(r...) WVR4_(r, 36, 35, 34, 33) #define WVR5(r...) WVR5_(r, 36, 35, 34, 33, 32) #define WVR6(r...) WVR6_(r, 36, 35, 34, 33, 32, 31) #define WVR7(r...) WVR7_(r, 36, 35, 34, 33, 32, 31, 30) #define WVR(X) [w##X] "=w" (w##X) #define UVR0_(REG, ...) [w##REG] "+&w" (w##REG) #define UVR1_(_1, REG, ...) [w##REG] "+&w" (w##REG) #define UVR2_(_1, _2, REG, ...) [w##REG] "+&w" (w##REG) #define UVR3_(_1, _2, _3, REG, ...) [w##REG] "+&w" (w##REG) #define UVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "+&w" (w##REG) #define UVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "+&w" (w##REG) #define UVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "+&w" (w##REG) #define UVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "+&w" (w##REG) #define UVR0(r...) UVR0_(r) #define UVR1(r...) UVR1_(r) #define UVR2(r...) UVR2_(r, 36) #define UVR3(r...) UVR3_(r, 36, 35) #define UVR4(r...) UVR4_(r, 36, 35, 34, 33) #define UVR5(r...) UVR5_(r, 36, 35, 34, 33, 32) #define UVR6(r...) UVR6_(r, 36, 35, 34, 33, 32, 31) #define UVR7(r...) UVR7_(r, 36, 35, 34, 33, 32, 31, 30) #define UVR(X) [w##X] "+&w" (w##X) #define R_01(REG1, REG2, ...) REG1, REG2 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) #define ZFS_ASM_BUG() ASSERT(0) #define OFFSET(ptr, val) (((unsigned char *)(ptr))+val) extern const uint8_t gf_clmul_mod_lt[4*256][16]; #define ELEM_SIZE 16 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "ld1 { v21.4s },%[SRC0]\n" \ "ld1 { v20.4s },%[SRC1]\n" \ "ld1 { v19.4s },%[SRC2]\n" \ "ld1 { v18.4s },%[SRC3]\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b,v19.16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b,v18.16b\n" \ "ld1 { v21.4s },%[SRC4]\n" \ "ld1 { v20.4s },%[SRC5]\n" \ "ld1 { v19.4s },%[SRC6]\n" \ "ld1 { v18.4s },%[SRC7]\n" \ "eor " VR4(r) ".16b," VR4(r) ".16b,v21.16b\n" \ "eor " VR5(r) ".16b," VR5(r) ".16b,v20.16b\n" \ "eor " VR6(r) ".16b," VR6(r) ".16b,v19.16b\n" \ "eor " VR7(r) ".16b," VR7(r) ".16b,v18.16b\n" \ : UVR0(r), UVR1(r), UVR2(r), UVR3(r), \ UVR4(r), UVR5(r), UVR6(r), UVR7(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48))), \ [SRC4] "Q" (*(OFFSET(src, 64))), \ [SRC5] "Q" (*(OFFSET(src, 80))), \ [SRC6] "Q" (*(OFFSET(src, 96))), \ [SRC7] "Q" (*(OFFSET(src, 112))) \ : "v18", "v19", "v20", "v21"); \ break; \ case 4: \ __asm( \ "ld1 { v21.4s },%[SRC0]\n" \ "ld1 { v20.4s },%[SRC1]\n" \ "ld1 { v19.4s },%[SRC2]\n" \ "ld1 { v18.4s },%[SRC3]\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b,v19.16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b,v18.16b\n" \ : UVR0(r), UVR1(r), UVR2(r), UVR3(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48))) \ : "v18", "v19", "v20", "v21"); \ break; \ case 2: \ __asm( \ "ld1 { v21.4s },%[SRC0]\n" \ "ld1 { v20.4s },%[SRC1]\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n" \ : UVR0(r), UVR1(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))) \ : "v20", "v21"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "eor " VR4(r) ".16b," VR4(r) ".16b," VR0(r) ".16b\n" \ "eor " VR5(r) ".16b," VR5(r) ".16b," VR1(r) ".16b\n" \ "eor " VR6(r) ".16b," VR6(r) ".16b," VR2(r) ".16b\n" \ "eor " VR7(r) ".16b," VR7(r) ".16b," VR3(r) ".16b\n" \ : UVR4(r), UVR5(r), UVR6(r), UVR7(r) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r)); \ break; \ case 4: \ __asm( \ "eor " VR2(r) ".16b," VR2(r) ".16b," VR0(r) ".16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b," VR1(r) ".16b\n" \ : UVR2(r), UVR3(r) \ : RVR0(r), RVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define ZERO(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b," VR2(r) ".16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b," VR3(r) ".16b\n" \ "eor " VR4(r) ".16b," VR4(r) ".16b," VR4(r) ".16b\n" \ "eor " VR5(r) ".16b," VR5(r) ".16b," VR5(r) ".16b\n" \ "eor " VR6(r) ".16b," VR6(r) ".16b," VR6(r) ".16b\n" \ "eor " VR7(r) ".16b," VR7(r) ".16b," VR7(r) ".16b\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r), \ WVR4(r), WVR5(r), WVR6(r), WVR7(r)); \ break; \ case 4: \ __asm( \ "eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b," VR2(r) ".16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b," VR3(r) ".16b\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r)); \ break; \ case 2: \ __asm( \ "eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n" \ : WVR0(r), WVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "mov " VR4(r) ".16b," VR0(r) ".16b\n" \ "mov " VR5(r) ".16b," VR1(r) ".16b\n" \ "mov " VR6(r) ".16b," VR2(r) ".16b\n" \ "mov " VR7(r) ".16b," VR3(r) ".16b\n" \ : WVR4(r), WVR5(r), WVR6(r), WVR7(r) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r)); \ break; \ case 4: \ __asm( \ "mov " VR2(r) ".16b," VR0(r) ".16b\n" \ "mov " VR3(r) ".16b," VR1(r) ".16b\n" \ : WVR2(r), WVR3(r) \ : RVR0(r), RVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "ld1 { " VR0(r) ".4s },%[SRC0]\n" \ "ld1 { " VR1(r) ".4s },%[SRC1]\n" \ "ld1 { " VR2(r) ".4s },%[SRC2]\n" \ "ld1 { " VR3(r) ".4s },%[SRC3]\n" \ "ld1 { " VR4(r) ".4s },%[SRC4]\n" \ "ld1 { " VR5(r) ".4s },%[SRC5]\n" \ "ld1 { " VR6(r) ".4s },%[SRC6]\n" \ "ld1 { " VR7(r) ".4s },%[SRC7]\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r), \ WVR4(r), WVR5(r), WVR6(r), WVR7(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48))), \ [SRC4] "Q" (*(OFFSET(src, 64))), \ [SRC5] "Q" (*(OFFSET(src, 80))), \ [SRC6] "Q" (*(OFFSET(src, 96))), \ [SRC7] "Q" (*(OFFSET(src, 112)))); \ break; \ case 4: \ __asm( \ "ld1 { " VR0(r) ".4s },%[SRC0]\n" \ "ld1 { " VR1(r) ".4s },%[SRC1]\n" \ "ld1 { " VR2(r) ".4s },%[SRC2]\n" \ "ld1 { " VR3(r) ".4s },%[SRC3]\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48)))); \ break; \ case 2: \ __asm( \ "ld1 { " VR0(r) ".4s },%[SRC0]\n" \ "ld1 { " VR1(r) ".4s },%[SRC1]\n" \ : WVR0(r), WVR1(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16)))); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "st1 { " VR0(r) ".4s },%[DST0]\n" \ "st1 { " VR1(r) ".4s },%[DST1]\n" \ "st1 { " VR2(r) ".4s },%[DST2]\n" \ "st1 { " VR3(r) ".4s },%[DST3]\n" \ "st1 { " VR4(r) ".4s },%[DST4]\n" \ "st1 { " VR5(r) ".4s },%[DST5]\n" \ "st1 { " VR6(r) ".4s },%[DST6]\n" \ "st1 { " VR7(r) ".4s },%[DST7]\n" \ : [DST0] "=Q" (*(OFFSET(dst, 0))), \ [DST1] "=Q" (*(OFFSET(dst, 16))), \ [DST2] "=Q" (*(OFFSET(dst, 32))), \ [DST3] "=Q" (*(OFFSET(dst, 48))), \ [DST4] "=Q" (*(OFFSET(dst, 64))), \ [DST5] "=Q" (*(OFFSET(dst, 80))), \ [DST6] "=Q" (*(OFFSET(dst, 96))), \ [DST7] "=Q" (*(OFFSET(dst, 112))) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r), \ RVR4(r), RVR5(r), RVR6(r), RVR7(r)); \ break; \ case 4: \ __asm( \ "st1 { " VR0(r) ".4s },%[DST0]\n" \ "st1 { " VR1(r) ".4s },%[DST1]\n" \ "st1 { " VR2(r) ".4s },%[DST2]\n" \ "st1 { " VR3(r) ".4s },%[DST3]\n" \ : [DST0] "=Q" (*(OFFSET(dst, 0))), \ [DST1] "=Q" (*(OFFSET(dst, 16))), \ [DST2] "=Q" (*(OFFSET(dst, 32))), \ [DST3] "=Q" (*(OFFSET(dst, 48))) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r)); \ break; \ case 2: \ __asm( \ "st1 { " VR0(r) ".4s },%[DST0]\n" \ "st1 { " VR1(r) ".4s },%[DST1]\n" \ : [DST0] "=Q" (*(OFFSET(dst, 0))), \ [DST1] "=Q" (*(OFFSET(dst, 16))) \ : RVR0(r), RVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } /* * Unfortunately cannot use the macro, because GCC * will try to use the macro name and not value * later on... * Kept as a reference to what a numbered variable is */ #define _00 "v17" #define _1d "v16" #define _temp0 "v19" #define _temp1 "v18" #define MUL2_SETUP() \ { \ __asm( \ "eor " VR(17) ".16b," VR(17) ".16b," VR(17) ".16b\n" \ "movi " VR(16) ".16b,#0x1d\n" \ : WVR(16), WVR(17)); \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n" \ "cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n" \ "cmgt v21.16b," VR(17) ".16b," VR2(r) ".16b\n" \ "cmgt v20.16b," VR(17) ".16b," VR3(r) ".16b\n" \ "and v19.16b,v19.16b," VR(16) ".16b\n" \ "and v18.16b,v18.16b," VR(16) ".16b\n" \ "and v21.16b,v21.16b," VR(16) ".16b\n" \ "and v20.16b,v20.16b," VR(16) ".16b\n" \ "shl " VR0(r) ".16b," VR0(r) ".16b,#1\n" \ "shl " VR1(r) ".16b," VR1(r) ".16b,#1\n" \ "shl " VR2(r) ".16b," VR2(r) ".16b,#1\n" \ "shl " VR3(r) ".16b," VR3(r) ".16b,#1\n" \ "eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n" \ "eor " VR2(r) ".16b,v21.16b," VR2(r) ".16b\n" \ "eor " VR3(r) ".16b,v20.16b," VR3(r) ".16b\n" \ : UVR0(r), UVR1(r), UVR2(r), UVR3(r) \ : RVR(17), RVR(16) \ : "v18", "v19", "v20", "v21"); \ break; \ case 2: \ __asm( \ "cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n" \ "cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n" \ "and v19.16b,v19.16b," VR(16) ".16b\n" \ "and v18.16b,v18.16b," VR(16) ".16b\n" \ "shl " VR0(r) ".16b," VR0(r) ".16b,#1\n" \ "shl " VR1(r) ".16b," VR1(r) ".16b,#1\n" \ "eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n" \ : UVR0(r), UVR1(r) \ : RVR(17), RVR(16) \ : "v18", "v19"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } /* * Unfortunately cannot use the macro, because GCC * will try to use the macro name and not value * later on... * Kept as a reference to what a register is * (here we're using actual registers for the * clobbered ones) */ #define _0f "v15" #define _a_save "v14" #define _b_save "v13" #define _lt_mod_a "v12" #define _lt_clmul_a "v11" #define _lt_mod_b "v10" #define _lt_clmul_b "v15" #define _MULx2(c, r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ /* lts for upper part */ \ "movi v15.16b,#0x0f\n" \ "ld1 { v10.4s },%[lt0]\n" \ "ld1 { v11.4s },%[lt1]\n" \ /* upper part */ \ "and v14.16b," VR0(r) ".16b,v15.16b\n" \ "and v13.16b," VR1(r) ".16b,v15.16b\n" \ "ushr " VR0(r) ".16b," VR0(r) ".16b,#4\n" \ "ushr " VR1(r) ".16b," VR1(r) ".16b,#4\n" \ \ "tbl v12.16b,{v10.16b}," VR0(r) ".16b\n" \ "tbl v10.16b,{v10.16b}," VR1(r) ".16b\n" \ "tbl v15.16b,{v11.16b}," VR0(r) ".16b\n" \ "tbl v11.16b,{v11.16b}," VR1(r) ".16b\n" \ \ "eor " VR0(r) ".16b,v15.16b,v12.16b\n" \ "eor " VR1(r) ".16b,v11.16b,v10.16b\n" \ /* lts for lower part */ \ "ld1 { v10.4s },%[lt2]\n" \ "ld1 { v15.4s },%[lt3]\n" \ /* lower part */ \ "tbl v12.16b,{v10.16b},v14.16b\n" \ "tbl v10.16b,{v10.16b},v13.16b\n" \ "tbl v11.16b,{v15.16b},v14.16b\n" \ "tbl v15.16b,{v15.16b},v13.16b\n" \ \ "eor " VR0(r) ".16b," VR0(r) ".16b,v12.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v10.16b\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v11.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v15.16b\n" \ : UVR0(r), UVR1(r) \ : [lt0] "Q" ((gf_clmul_mod_lt[4*(c)+0][0])), \ [lt1] "Q" ((gf_clmul_mod_lt[4*(c)+1][0])), \ [lt2] "Q" ((gf_clmul_mod_lt[4*(c)+2][0])), \ [lt3] "Q" ((gf_clmul_mod_lt[4*(c)+3][0])) \ : "v10", "v11", "v12", "v13", "v14", "v15"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MULx2(c, R_23(r)); \ _MULx2(c, R_01(r)); \ break; \ case 2: \ _MULx2(c, R_01(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() /* Overkill... */ #if defined(_KERNEL) #define GEN_X_DEFINE_0_3() \ register unsigned char w0 asm("v0") __attribute__((vector_size(16))); \ register unsigned char w1 asm("v1") __attribute__((vector_size(16))); \ register unsigned char w2 asm("v2") __attribute__((vector_size(16))); \ register unsigned char w3 asm("v3") __attribute__((vector_size(16))); #define GEN_X_DEFINE_4_5() \ register unsigned char w4 asm("v4") __attribute__((vector_size(16))); \ register unsigned char w5 asm("v5") __attribute__((vector_size(16))); #define GEN_X_DEFINE_6_7() \ register unsigned char w6 asm("v6") __attribute__((vector_size(16))); \ register unsigned char w7 asm("v7") __attribute__((vector_size(16))); #define GEN_X_DEFINE_8_9() \ register unsigned char w8 asm("v8") __attribute__((vector_size(16))); \ register unsigned char w9 asm("v9") __attribute__((vector_size(16))); #define GEN_X_DEFINE_10_11() \ register unsigned char w10 asm("v10") __attribute__((vector_size(16))); \ register unsigned char w11 asm("v11") __attribute__((vector_size(16))); #define GEN_X_DEFINE_12_15() \ register unsigned char w12 asm("v12") __attribute__((vector_size(16))); \ register unsigned char w13 asm("v13") __attribute__((vector_size(16))); \ register unsigned char w14 asm("v14") __attribute__((vector_size(16))); \ register unsigned char w15 asm("v15") __attribute__((vector_size(16))); #define GEN_X_DEFINE_16() \ register unsigned char w16 asm("v16") __attribute__((vector_size(16))); #define GEN_X_DEFINE_17() \ register unsigned char w17 asm("v17") __attribute__((vector_size(16))); #define GEN_X_DEFINE_18_21() \ register unsigned char w18 asm("v18") __attribute__((vector_size(16))); \ register unsigned char w19 asm("v19") __attribute__((vector_size(16))); \ register unsigned char w20 asm("v20") __attribute__((vector_size(16))); \ register unsigned char w21 asm("v21") __attribute__((vector_size(16))); #define GEN_X_DEFINE_22_23() \ register unsigned char w22 asm("v22") __attribute__((vector_size(16))); \ register unsigned char w23 asm("v23") __attribute__((vector_size(16))); #define GEN_X_DEFINE_24_27() \ register unsigned char w24 asm("v24") __attribute__((vector_size(16))); \ register unsigned char w25 asm("v25") __attribute__((vector_size(16))); \ register unsigned char w26 asm("v26") __attribute__((vector_size(16))); \ register unsigned char w27 asm("v27") __attribute__((vector_size(16))); #define GEN_X_DEFINE_28_30() \ register unsigned char w28 asm("v28") __attribute__((vector_size(16))); \ register unsigned char w29 asm("v29") __attribute__((vector_size(16))); \ register unsigned char w30 asm("v30") __attribute__((vector_size(16))); #define GEN_X_DEFINE_31() \ register unsigned char w31 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_32() \ register unsigned char w32 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_33_36() \ register unsigned char w33 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w34 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w35 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w36 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_37_38() \ register unsigned char w37 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w38 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_ALL() \ GEN_X_DEFINE_0_3() \ GEN_X_DEFINE_4_5() \ GEN_X_DEFINE_6_7() \ GEN_X_DEFINE_8_9() \ GEN_X_DEFINE_10_11() \ GEN_X_DEFINE_12_15() \ GEN_X_DEFINE_16() \ GEN_X_DEFINE_17() \ GEN_X_DEFINE_18_21() \ GEN_X_DEFINE_22_23() \ GEN_X_DEFINE_24_27() \ GEN_X_DEFINE_28_30() \ GEN_X_DEFINE_31() \ GEN_X_DEFINE_32() \ GEN_X_DEFINE_33_36() \ GEN_X_DEFINE_37_38() #else #define GEN_X_DEFINE_0_3() \ unsigned char w0 __attribute__((vector_size(16))); \ unsigned char w1 __attribute__((vector_size(16))); \ unsigned char w2 __attribute__((vector_size(16))); \ unsigned char w3 __attribute__((vector_size(16))); #define GEN_X_DEFINE_4_5() \ unsigned char w4 __attribute__((vector_size(16))); \ unsigned char w5 __attribute__((vector_size(16))); #define GEN_X_DEFINE_6_7() \ unsigned char w6 __attribute__((vector_size(16))); \ unsigned char w7 __attribute__((vector_size(16))); #define GEN_X_DEFINE_8_9() \ unsigned char w8 __attribute__((vector_size(16))); \ unsigned char w9 __attribute__((vector_size(16))); #define GEN_X_DEFINE_10_11() \ unsigned char w10 __attribute__((vector_size(16))); \ unsigned char w11 __attribute__((vector_size(16))); #define GEN_X_DEFINE_12_15() \ unsigned char w12 __attribute__((vector_size(16))); \ unsigned char w13 __attribute__((vector_size(16))); \ unsigned char w14 __attribute__((vector_size(16))); \ unsigned char w15 __attribute__((vector_size(16))); #define GEN_X_DEFINE_16() \ unsigned char w16 __attribute__((vector_size(16))); #define GEN_X_DEFINE_17() \ unsigned char w17 __attribute__((vector_size(16))); #define GEN_X_DEFINE_18_21() \ unsigned char w18 __attribute__((vector_size(16))); \ unsigned char w19 __attribute__((vector_size(16))); \ unsigned char w20 __attribute__((vector_size(16))); \ unsigned char w21 __attribute__((vector_size(16))); #define GEN_X_DEFINE_22_23() \ unsigned char w22 __attribute__((vector_size(16))); \ unsigned char w23 __attribute__((vector_size(16))); #define GEN_X_DEFINE_24_27() \ unsigned char w24 __attribute__((vector_size(16))); \ unsigned char w25 __attribute__((vector_size(16))); \ unsigned char w26 __attribute__((vector_size(16))); \ unsigned char w27 __attribute__((vector_size(16))); #define GEN_X_DEFINE_28_30() \ unsigned char w28 __attribute__((vector_size(16))); \ unsigned char w29 __attribute__((vector_size(16))); \ unsigned char w30 __attribute__((vector_size(16))); #define GEN_X_DEFINE_31() \ unsigned char w31 __attribute__((vector_size(16))); #define GEN_X_DEFINE_32() \ unsigned char w32 __attribute__((vector_size(16))); #define GEN_X_DEFINE_33_36() \ unsigned char w33 __attribute__((vector_size(16))); \ unsigned char w34 __attribute__((vector_size(16))); \ unsigned char w35 __attribute__((vector_size(16))); \ unsigned char w36 __attribute__((vector_size(16))); #define GEN_X_DEFINE_37_38() \ unsigned char w37 __attribute__((vector_size(16))); \ unsigned char w38 __attribute__((vector_size(16))); #define GEN_X_DEFINE_ALL() \ GEN_X_DEFINE_0_3() \ GEN_X_DEFINE_4_5() \ GEN_X_DEFINE_6_7() \ GEN_X_DEFINE_8_9() \ GEN_X_DEFINE_10_11() \ GEN_X_DEFINE_12_15() \ GEN_X_DEFINE_16() \ GEN_X_DEFINE_17() \ GEN_X_DEFINE_18_21() \ GEN_X_DEFINE_22_23() \ GEN_X_DEFINE_24_27() \ GEN_X_DEFINE_28_30() \ GEN_X_DEFINE_31() \ GEN_X_DEFINE_32() \ GEN_X_DEFINE_33_36() \ GEN_X_DEFINE_37_38() #endif
Upload File
Create Folder