003 File Manager
Current Path:
/usr/src/sys/kern
usr
/
src
/
sys
/
kern
/
📁
..
📄
Make.tags.inc
(2.13 KB)
📄
Makefile
(302 B)
📄
bus_if.m
(26.31 KB)
📄
capabilities.conf
(13.67 KB)
📄
clock_if.m
(1.7 KB)
📄
cpufreq_if.m
(2.27 KB)
📄
device_if.m
(10.41 KB)
📄
firmw.S
(2.15 KB)
📄
genassym.sh
(1.11 KB)
📄
genoffset.c
(1.68 KB)
📄
genoffset.sh
(3.58 KB)
📄
imgact_aout.c
(9.45 KB)
📄
imgact_binmisc.c
(18.64 KB)
📄
imgact_elf.c
(76.32 KB)
📄
imgact_elf32.c
(1.47 KB)
📄
imgact_elf64.c
(1.47 KB)
📄
imgact_shell.c
(8.41 KB)
📄
init_main.c
(24.31 KB)
📄
init_sysent.c
(95.3 KB)
📄
kern_acct.c
(19.03 KB)
📄
kern_alq.c
(24.97 KB)
📄
kern_clock.c
(21.12 KB)
📄
kern_clocksource.c
(23.34 KB)
📄
kern_condvar.c
(11.28 KB)
📄
kern_conf.c
(36.14 KB)
📄
kern_cons.c
(15.75 KB)
📄
kern_context.c
(3.59 KB)
📄
kern_cpu.c
(30.77 KB)
📄
kern_cpuset.c
(59.78 KB)
📄
kern_ctf.c
(8.73 KB)
📄
kern_descrip.c
(112.87 KB)
📄
kern_dtrace.c
(2.94 KB)
📄
kern_dump.c
(8.51 KB)
📄
kern_environment.c
(22.75 KB)
📄
kern_et.c
(7.1 KB)
📄
kern_event.c
(62.49 KB)
📄
kern_exec.c
(46.67 KB)
📄
kern_exit.c
(34.61 KB)
📄
kern_fail.c
(29.32 KB)
📄
kern_ffclock.c
(12.66 KB)
📄
kern_fork.c
(28.29 KB)
📄
kern_hhook.c
(13.58 KB)
📄
kern_idle.c
(2.74 KB)
📄
kern_intr.c
(40.44 KB)
📄
kern_jail.c
(112.67 KB)
📄
kern_kcov.c
(15.32 KB)
📄
kern_khelp.c
(9.45 KB)
📄
kern_kthread.c
(11.8 KB)
📄
kern_ktr.c
(11.93 KB)
📄
kern_ktrace.c
(31.41 KB)
📄
kern_linker.c
(54.3 KB)
📄
kern_lock.c
(46.99 KB)
📄
kern_lockf.c
(64.46 KB)
📄
kern_lockstat.c
(3.8 KB)
📄
kern_loginclass.c
(6.69 KB)
📄
kern_malloc.c
(37.09 KB)
📄
kern_mbuf.c
(43.16 KB)
📄
kern_mib.c
(24.26 KB)
📄
kern_module.c
(11.05 KB)
📄
kern_mtxpool.c
(5.82 KB)
📄
kern_mutex.c
(33.62 KB)
📄
kern_ntptime.c
(32.49 KB)
📄
kern_osd.c
(12.37 KB)
📄
kern_physio.c
(5.74 KB)
📄
kern_pmc.c
(8.89 KB)
📄
kern_poll.c
(15.86 KB)
📄
kern_priv.c
(9.14 KB)
📄
kern_proc.c
(80.01 KB)
📄
kern_procctl.c
(19.48 KB)
📄
kern_prot.c
(57.94 KB)
📄
kern_racct.c
(34.01 KB)
📄
kern_rangelock.c
(8.67 KB)
📄
kern_rctl.c
(53.87 KB)
📄
kern_resource.c
(36.66 KB)
📄
kern_rmlock.c
(28.27 KB)
📄
kern_rwlock.c
(40.72 KB)
📄
kern_sdt.c
(2.05 KB)
📄
kern_sema.c
(4.85 KB)
📄
kern_sendfile.c
(33.97 KB)
📄
kern_sharedpage.c
(10.37 KB)
📄
kern_shutdown.c
(43.34 KB)
📄
kern_sig.c
(101.89 KB)
📄
kern_switch.c
(13.85 KB)
📄
kern_sx.c
(40.27 KB)
📄
kern_synch.c
(18.17 KB)
📄
kern_syscalls.c
(6.74 KB)
📄
kern_sysctl.c
(67.24 KB)
📄
kern_tc.c
(55.73 KB)
📄
kern_thr.c
(14.14 KB)
📄
kern_thread.c
(41.75 KB)
📄
kern_time.c
(40.89 KB)
📄
kern_timeout.c
(43.08 KB)
📄
kern_tslog.c
(3.44 KB)
📄
kern_ubsan.c
(50.74 KB)
📄
kern_umtx.c
(107.14 KB)
📄
kern_uuid.c
(11.68 KB)
📄
kern_xxx.c
(10.44 KB)
📄
ksched.c
(6.56 KB)
📄
link_elf.c
(47.99 KB)
📄
link_elf_obj.c
(44.41 KB)
📄
linker_if.m
(3.96 KB)
📄
makesyscalls.sh
(23.57 KB)
📄
md4c.c
(7.89 KB)
📄
md5c.c
(9.56 KB)
📄
msi_if.m
(2.48 KB)
📄
p1003_1b.c
(8.84 KB)
📄
pic_if.m
(3.9 KB)
📄
posix4_mib.c
(5.59 KB)
📄
sched_4bsd.c
(45.03 KB)
📄
sched_ule.c
(82.65 KB)
📄
serdev_if.m
(3.49 KB)
📄
stack_protector.c
(613 B)
📄
subr_acl_nfs4.c
(37.42 KB)
📄
subr_acl_posix1e.c
(17.71 KB)
📄
subr_atomic64.c
(3.97 KB)
📄
subr_autoconf.c
(7.7 KB)
📄
subr_blist.c
(31.88 KB)
📄
subr_boot.c
(5.8 KB)
📄
subr_bufring.c
(2.21 KB)
📄
subr_bus.c
(145.4 KB)
📄
subr_bus_dma.c
(19.67 KB)
📄
subr_busdma_bufalloc.c
(5.24 KB)
📄
subr_capability.c
(11.93 KB)
📄
subr_clock.c
(10.61 KB)
📄
subr_compressor.c
(13.11 KB)
📄
subr_counter.c
(4.44 KB)
📄
subr_coverage.c
(6.17 KB)
📄
subr_csan.c
(25.39 KB)
📄
subr_devmap.c
(9.8 KB)
📄
subr_devstat.c
(16.21 KB)
📄
subr_disk.c
(8.54 KB)
📄
subr_dummy_vdso_tc.c
(1.7 KB)
📄
subr_early.c
(2.26 KB)
📄
subr_epoch.c
(25.02 KB)
📄
subr_eventhandler.c
(9.17 KB)
📄
subr_fattime.c
(9.98 KB)
📄
subr_filter.c
(12.2 KB)
📄
subr_firmware.c
(13.88 KB)
📄
subr_gtaskqueue.c
(20.19 KB)
📄
subr_hash.c
(4.8 KB)
📄
subr_hints.c
(12.87 KB)
📄
subr_intr.c
(40.61 KB)
📄
subr_kdb.c
(16.13 KB)
📄
subr_kobj.c
(7.1 KB)
📄
subr_lock.c
(18.81 KB)
📄
subr_log.c
(7.64 KB)
📄
subr_mchain.c
(11.06 KB)
📄
subr_module.c
(12.98 KB)
📄
subr_msgbuf.c
(10.6 KB)
📄
subr_param.c
(10.93 KB)
📄
subr_pcpu.c
(10.18 KB)
📄
subr_pctrie.c
(20.99 KB)
📄
subr_physmem.c
(11.52 KB)
📄
subr_pidctrl.c
(5.43 KB)
📄
subr_power.c
(3.13 KB)
📄
subr_prf.c
(27.42 KB)
📄
subr_prng.c
(3.36 KB)
📄
subr_prof.c
(15.43 KB)
📄
subr_rangeset.c
(8.5 KB)
📄
subr_rman.c
(27.61 KB)
📄
subr_rtc.c
(11.42 KB)
📄
subr_sbuf.c
(20.53 KB)
📄
subr_scanf.c
(15.59 KB)
📄
subr_sfbuf.c
(6.17 KB)
📄
subr_sglist.c
(22.83 KB)
📄
subr_sleepqueue.c
(39.43 KB)
📄
subr_smp.c
(31.62 KB)
📄
subr_smr.c
(20.17 KB)
📄
subr_stack.c
(6.47 KB)
📄
subr_stats.c
(103.01 KB)
📄
subr_syscall.c
(7.98 KB)
📄
subr_taskqueue.c
(21.1 KB)
📄
subr_terminal.c
(15.52 KB)
📄
subr_trap.c
(10.87 KB)
📄
subr_turnstile.c
(35.58 KB)
📄
subr_uio.c
(11.38 KB)
📄
subr_unit.c
(22.97 KB)
📄
subr_vmem.c
(43.25 KB)
📄
subr_witness.c
(84.59 KB)
📄
sys_capability.c
(15.06 KB)
📄
sys_eventfd.c
(8.42 KB)
📄
sys_generic.c
(44.22 KB)
📄
sys_getrandom.c
(4.21 KB)
📄
sys_pipe.c
(45.14 KB)
📄
sys_procdesc.c
(14.57 KB)
📄
sys_process.c
(30.73 KB)
📄
sys_socket.c
(20.11 KB)
📄
syscalls.c
(22.73 KB)
📄
syscalls.master
(60.26 KB)
📄
systrace_args.c
(178.49 KB)
📄
sysv_ipc.c
(6.53 KB)
📄
sysv_msg.c
(48.65 KB)
📄
sysv_sem.c
(49.85 KB)
📄
sysv_shm.c
(43.93 KB)
📄
tty.c
(55.14 KB)
📄
tty_compat.c
(11.46 KB)
📄
tty_info.c
(9.93 KB)
📄
tty_inq.c
(12.22 KB)
📄
tty_outq.c
(8.74 KB)
📄
tty_pts.c
(19.74 KB)
📄
tty_tty.c
(2.83 KB)
📄
tty_ttydisc.c
(28.6 KB)
📄
uipc_accf.c
(8.07 KB)
📄
uipc_debug.c
(12.42 KB)
📄
uipc_domain.c
(13.13 KB)
📄
uipc_ktls.c
(55.7 KB)
📄
uipc_mbuf.c
(52.45 KB)
📄
uipc_mbuf2.c
(12.64 KB)
📄
uipc_mbufhash.c
(4.9 KB)
📄
uipc_mqueue.c
(64.64 KB)
📄
uipc_sem.c
(25.18 KB)
📄
uipc_shm.c
(50.47 KB)
📄
uipc_sockbuf.c
(42.9 KB)
📄
uipc_socket.c
(110.61 KB)
📄
uipc_syscalls.c
(35.94 KB)
📄
uipc_usrreq.c
(75.11 KB)
📄
vfs_acl.c
(14.5 KB)
📄
vfs_aio.c
(76.32 KB)
📄
vfs_bio.c
(145.39 KB)
📄
vfs_cache.c
(143.09 KB)
📄
vfs_cluster.c
(28.36 KB)
📄
vfs_default.c
(33.16 KB)
📄
vfs_export.c
(14.55 KB)
📄
vfs_extattr.c
(17.91 KB)
📄
vfs_hash.c
(6 KB)
📄
vfs_init.c
(15.86 KB)
📄
vfs_lookup.c
(45.48 KB)
📄
vfs_mount.c
(62.58 KB)
📄
vfs_mountroot.c
(26.23 KB)
📄
vfs_subr.c
(167.52 KB)
📄
vfs_syscalls.c
(106.86 KB)
📄
vfs_vnops.c
(86.28 KB)
📄
vnode_if.src
(13.66 KB)
Editing: subr_gtaskqueue.c
/*- * Copyright (c) 2000 Doug Rabson * Copyright (c) 2014 Jeff Roberson * Copyright (c) 2016 Matthew Macy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/bus.h> #include <sys/cpuset.h> #include <sys/kernel.h> #include <sys/kthread.h> #include <sys/libkern.h> #include <sys/limits.h> #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> #include <sys/epoch.h> #include <sys/sched.h> #include <sys/smp.h> #include <sys/gtaskqueue.h> #include <sys/unistd.h> #include <machine/stdarg.h> static MALLOC_DEFINE(M_GTASKQUEUE, "gtaskqueue", "Group Task Queues"); static void gtaskqueue_thread_enqueue(void *); static void gtaskqueue_thread_loop(void *arg); static int task_is_running(struct gtaskqueue *queue, struct gtask *gtask); static void gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask); TASKQGROUP_DEFINE(softirq, mp_ncpus, 1); struct gtaskqueue_busy { struct gtask *tb_running; u_int tb_seq; LIST_ENTRY(gtaskqueue_busy) tb_link; }; typedef void (*gtaskqueue_enqueue_fn)(void *context); struct gtaskqueue { STAILQ_HEAD(, gtask) tq_queue; LIST_HEAD(, gtaskqueue_busy) tq_active; u_int tq_seq; int tq_callouts; struct mtx_padalign tq_mutex; gtaskqueue_enqueue_fn tq_enqueue; void *tq_context; char *tq_name; struct thread **tq_threads; int tq_tcount; int tq_spin; int tq_flags; taskqueue_callback_fn tq_callbacks[TASKQUEUE_NUM_CALLBACKS]; void *tq_cb_contexts[TASKQUEUE_NUM_CALLBACKS]; }; #define TQ_FLAGS_ACTIVE (1 << 0) #define TQ_FLAGS_BLOCKED (1 << 1) #define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) #define DT_CALLOUT_ARMED (1 << 0) #define TQ_LOCK(tq) \ do { \ if ((tq)->tq_spin) \ mtx_lock_spin(&(tq)->tq_mutex); \ else \ mtx_lock(&(tq)->tq_mutex); \ } while (0) #define TQ_ASSERT_LOCKED(tq) mtx_assert(&(tq)->tq_mutex, MA_OWNED) #define TQ_UNLOCK(tq) \ do { \ if ((tq)->tq_spin) \ mtx_unlock_spin(&(tq)->tq_mutex); \ else \ mtx_unlock(&(tq)->tq_mutex); \ } while (0) #define TQ_ASSERT_UNLOCKED(tq) mtx_assert(&(tq)->tq_mutex, MA_NOTOWNED) #ifdef INVARIANTS static void gtask_dump(struct gtask *gtask) { printf("gtask: %p ta_flags=%x ta_priority=%d ta_func=%p ta_context=%p\n", gtask, gtask->ta_flags, gtask->ta_priority, gtask->ta_func, gtask->ta_context); } #endif static __inline int TQ_SLEEP(struct gtaskqueue *tq, void *p, const char *wm) { if (tq->tq_spin) return (msleep_spin(p, (struct mtx *)&tq->tq_mutex, wm, 0)); return (msleep(p, &tq->tq_mutex, 0, wm, 0)); } static struct gtaskqueue * _gtaskqueue_create(const char *name, int mflags, taskqueue_enqueue_fn enqueue, void *context, int mtxflags, const char *mtxname __unused) { struct gtaskqueue *queue; char *tq_name; tq_name = malloc(TASKQUEUE_NAMELEN, M_GTASKQUEUE, mflags | M_ZERO); if (!tq_name) return (NULL); snprintf(tq_name, TASKQUEUE_NAMELEN, "%s", (name) ? name : "taskqueue"); queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO); if (!queue) { free(tq_name, M_GTASKQUEUE); return (NULL); } STAILQ_INIT(&queue->tq_queue); LIST_INIT(&queue->tq_active); queue->tq_enqueue = enqueue; queue->tq_context = context; queue->tq_name = tq_name; queue->tq_spin = (mtxflags & MTX_SPIN) != 0; queue->tq_flags |= TQ_FLAGS_ACTIVE; if (enqueue == gtaskqueue_thread_enqueue) queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE; mtx_init(&queue->tq_mutex, tq_name, NULL, mtxflags); return (queue); } /* * Signal a taskqueue thread to terminate. */ static void gtaskqueue_terminate(struct thread **pp, struct gtaskqueue *tq) { while (tq->tq_tcount > 0 || tq->tq_callouts > 0) { wakeup(tq); TQ_SLEEP(tq, pp, "gtq_destroy"); } } static void __unused gtaskqueue_free(struct gtaskqueue *queue) { TQ_LOCK(queue); queue->tq_flags &= ~TQ_FLAGS_ACTIVE; gtaskqueue_terminate(queue->tq_threads, queue); KASSERT(LIST_EMPTY(&queue->tq_active), ("Tasks still running?")); KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks")); mtx_destroy(&queue->tq_mutex); free(queue->tq_threads, M_GTASKQUEUE); free(queue->tq_name, M_GTASKQUEUE); free(queue, M_GTASKQUEUE); } /* * Wait for all to complete, then prevent it from being enqueued */ void grouptask_block(struct grouptask *grouptask) { struct gtaskqueue *queue = grouptask->gt_taskqueue; struct gtask *gtask = &grouptask->gt_task; #ifdef INVARIANTS if (queue == NULL) { gtask_dump(gtask); panic("queue == NULL"); } #endif TQ_LOCK(queue); gtask->ta_flags |= TASK_NOENQUEUE; gtaskqueue_drain_locked(queue, gtask); TQ_UNLOCK(queue); } void grouptask_unblock(struct grouptask *grouptask) { struct gtaskqueue *queue = grouptask->gt_taskqueue; struct gtask *gtask = &grouptask->gt_task; #ifdef INVARIANTS if (queue == NULL) { gtask_dump(gtask); panic("queue == NULL"); } #endif TQ_LOCK(queue); gtask->ta_flags &= ~TASK_NOENQUEUE; TQ_UNLOCK(queue); } int grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask) { #ifdef INVARIANTS if (queue == NULL) { gtask_dump(gtask); panic("queue == NULL"); } #endif TQ_LOCK(queue); if (gtask->ta_flags & TASK_ENQUEUED) { TQ_UNLOCK(queue); return (0); } if (gtask->ta_flags & TASK_NOENQUEUE) { TQ_UNLOCK(queue); return (EAGAIN); } STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link); gtask->ta_flags |= TASK_ENQUEUED; TQ_UNLOCK(queue); if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) queue->tq_enqueue(queue->tq_context); return (0); } static void gtaskqueue_task_nop_fn(void *context) { } /* * Block until all currently queued tasks in this taskqueue * have begun execution. Tasks queued during execution of * this function are ignored. */ static void gtaskqueue_drain_tq_queue(struct gtaskqueue *queue) { struct gtask t_barrier; if (STAILQ_EMPTY(&queue->tq_queue)) return; /* * Enqueue our barrier after all current tasks, but with * the highest priority so that newly queued tasks cannot * pass it. Because of the high priority, we can not use * taskqueue_enqueue_locked directly (which drops the lock * anyway) so just insert it at tail while we have the * queue lock. */ GTASK_INIT(&t_barrier, 0, USHRT_MAX, gtaskqueue_task_nop_fn, &t_barrier); STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link); t_barrier.ta_flags |= TASK_ENQUEUED; /* * Once the barrier has executed, all previously queued tasks * have completed or are currently executing. */ while (t_barrier.ta_flags & TASK_ENQUEUED) TQ_SLEEP(queue, &t_barrier, "gtq_qdrain"); } /* * Block until all currently executing tasks for this taskqueue * complete. Tasks that begin execution during the execution * of this function are ignored. */ static void gtaskqueue_drain_tq_active(struct gtaskqueue *queue) { struct gtaskqueue_busy *tb; u_int seq; if (LIST_EMPTY(&queue->tq_active)) return; /* Block taskq_terminate().*/ queue->tq_callouts++; /* Wait for any active task with sequence from the past. */ seq = queue->tq_seq; restart: LIST_FOREACH(tb, &queue->tq_active, tb_link) { if ((int)(tb->tb_seq - seq) <= 0) { TQ_SLEEP(queue, tb->tb_running, "gtq_adrain"); goto restart; } } /* Release taskqueue_terminate(). */ queue->tq_callouts--; if ((queue->tq_flags & TQ_FLAGS_ACTIVE) == 0) wakeup_one(queue->tq_threads); } void gtaskqueue_block(struct gtaskqueue *queue) { TQ_LOCK(queue); queue->tq_flags |= TQ_FLAGS_BLOCKED; TQ_UNLOCK(queue); } void gtaskqueue_unblock(struct gtaskqueue *queue) { TQ_LOCK(queue); queue->tq_flags &= ~TQ_FLAGS_BLOCKED; if (!STAILQ_EMPTY(&queue->tq_queue)) queue->tq_enqueue(queue->tq_context); TQ_UNLOCK(queue); } static void gtaskqueue_run_locked(struct gtaskqueue *queue) { struct epoch_tracker et; struct gtaskqueue_busy tb; struct gtask *gtask; bool in_net_epoch; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); in_net_epoch = false; while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); gtask->ta_flags &= ~TASK_ENQUEUED; tb.tb_running = gtask; tb.tb_seq = ++queue->tq_seq; TQ_UNLOCK(queue); KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL")); if (!in_net_epoch && TASK_IS_NET(gtask)) { in_net_epoch = true; NET_EPOCH_ENTER(et); } else if (in_net_epoch && !TASK_IS_NET(gtask)) { NET_EPOCH_EXIT(et); in_net_epoch = false; } gtask->ta_func(gtask->ta_context); TQ_LOCK(queue); wakeup(gtask); } if (in_net_epoch) NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } static int task_is_running(struct gtaskqueue *queue, struct gtask *gtask) { struct gtaskqueue_busy *tb; TQ_ASSERT_LOCKED(queue); LIST_FOREACH(tb, &queue->tq_active, tb_link) { if (tb->tb_running == gtask) return (1); } return (0); } static int gtaskqueue_cancel_locked(struct gtaskqueue *queue, struct gtask *gtask) { if (gtask->ta_flags & TASK_ENQUEUED) STAILQ_REMOVE(&queue->tq_queue, gtask, gtask, ta_link); gtask->ta_flags &= ~TASK_ENQUEUED; return (task_is_running(queue, gtask) ? EBUSY : 0); } int gtaskqueue_cancel(struct gtaskqueue *queue, struct gtask *gtask) { int error; TQ_LOCK(queue); error = gtaskqueue_cancel_locked(queue, gtask); TQ_UNLOCK(queue); return (error); } static void gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask) { while ((gtask->ta_flags & TASK_ENQUEUED) || task_is_running(queue, gtask)) TQ_SLEEP(queue, gtask, "gtq_drain"); } void gtaskqueue_drain(struct gtaskqueue *queue, struct gtask *gtask) { if (!queue->tq_spin) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); TQ_LOCK(queue); gtaskqueue_drain_locked(queue, gtask); TQ_UNLOCK(queue); } void gtaskqueue_drain_all(struct gtaskqueue *queue) { if (!queue->tq_spin) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); TQ_LOCK(queue); gtaskqueue_drain_tq_queue(queue); gtaskqueue_drain_tq_active(queue); TQ_UNLOCK(queue); } static int _gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, cpuset_t *mask, const char *name, va_list ap) { char ktname[MAXCOMLEN + 1]; struct thread *td; struct gtaskqueue *tq; int i, error; if (count <= 0) return (EINVAL); vsnprintf(ktname, sizeof(ktname), name, ap); tq = *tqp; tq->tq_threads = malloc(sizeof(struct thread *) * count, M_GTASKQUEUE, M_NOWAIT | M_ZERO); if (tq->tq_threads == NULL) { printf("%s: no memory for %s threads\n", __func__, ktname); return (ENOMEM); } for (i = 0; i < count; i++) { if (count == 1) error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, &tq->tq_threads[i], RFSTOPPED, 0, "%s", ktname); else error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, &tq->tq_threads[i], RFSTOPPED, 0, "%s_%d", ktname, i); if (error) { /* should be ok to continue, taskqueue_free will dtrt */ printf("%s: kthread_add(%s): error %d", __func__, ktname, error); tq->tq_threads[i] = NULL; /* paranoid */ } else tq->tq_tcount++; } for (i = 0; i < count; i++) { if (tq->tq_threads[i] == NULL) continue; td = tq->tq_threads[i]; if (mask) { error = cpuset_setthread(td->td_tid, mask); /* * Failing to pin is rarely an actual fatal error; * it'll just affect performance. */ if (error) printf("%s: curthread=%llu: can't pin; " "error=%d\n", __func__, (unsigned long long) td->td_tid, error); } thread_lock(td); sched_prio(td, pri); sched_add(td, SRQ_BORING); } return (0); } static int gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, const char *name, ...) { va_list ap; int error; va_start(ap, name); error = _gtaskqueue_start_threads(tqp, count, pri, NULL, name, ap); va_end(ap); return (error); } static inline void gtaskqueue_run_callback(struct gtaskqueue *tq, enum taskqueue_callback_type cb_type) { taskqueue_callback_fn tq_callback; TQ_ASSERT_UNLOCKED(tq); tq_callback = tq->tq_callbacks[cb_type]; if (tq_callback != NULL) tq_callback(tq->tq_cb_contexts[cb_type]); } static void gtaskqueue_thread_loop(void *arg) { struct gtaskqueue **tqp, *tq; tqp = arg; tq = *tqp; gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_INIT); TQ_LOCK(tq); while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { /* XXX ? */ gtaskqueue_run_locked(tq); /* * Because taskqueue_run() can drop tq_mutex, we need to * check if the TQ_FLAGS_ACTIVE flag wasn't removed in the * meantime, which means we missed a wakeup. */ if ((tq->tq_flags & TQ_FLAGS_ACTIVE) == 0) break; TQ_SLEEP(tq, tq, "-"); } gtaskqueue_run_locked(tq); /* * This thread is on its way out, so just drop the lock temporarily * in order to call the shutdown callback. This allows the callback * to look at the taskqueue, even just before it dies. */ TQ_UNLOCK(tq); gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN); TQ_LOCK(tq); /* rendezvous with thread that asked us to terminate */ tq->tq_tcount--; wakeup_one(tq->tq_threads); TQ_UNLOCK(tq); kthread_exit(); } static void gtaskqueue_thread_enqueue(void *context) { struct gtaskqueue **tqp, *tq; tqp = context; tq = *tqp; wakeup_any(tq); } static struct gtaskqueue * gtaskqueue_create_fast(const char *name, int mflags, taskqueue_enqueue_fn enqueue, void *context) { return _gtaskqueue_create(name, mflags, enqueue, context, MTX_SPIN, "fast_taskqueue"); } struct taskqgroup_cpu { LIST_HEAD(, grouptask) tgc_tasks; struct gtaskqueue *tgc_taskq; int tgc_cnt; int tgc_cpu; }; struct taskqgroup { struct taskqgroup_cpu tqg_queue[MAXCPU]; struct mtx tqg_lock; const char * tqg_name; int tqg_cnt; }; struct taskq_bind_task { struct gtask bt_task; int bt_cpuid; }; static void taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu) { struct taskqgroup_cpu *qcpu; qcpu = &qgroup->tqg_queue[idx]; LIST_INIT(&qcpu->tgc_tasks); qcpu->tgc_taskq = gtaskqueue_create_fast(NULL, M_WAITOK, taskqueue_thread_enqueue, &qcpu->tgc_taskq); gtaskqueue_start_threads(&qcpu->tgc_taskq, 1, PI_SOFT, "%s_%d", qgroup->tqg_name, idx); qcpu->tgc_cpu = cpu; } /* * Find the taskq with least # of tasks that doesn't currently have any * other queues from the uniq identifier. */ static int taskqgroup_find(struct taskqgroup *qgroup, void *uniq) { struct grouptask *n; int i, idx, mincnt; int strict; mtx_assert(&qgroup->tqg_lock, MA_OWNED); KASSERT(qgroup->tqg_cnt != 0, ("qgroup %s has no queues", qgroup->tqg_name)); /* * Two passes: first scan for a queue with the least tasks that * does not already service this uniq id. If that fails simply find * the queue with the least total tasks. */ for (idx = -1, mincnt = INT_MAX, strict = 1; mincnt == INT_MAX; strict = 0) { for (i = 0; i < qgroup->tqg_cnt; i++) { if (qgroup->tqg_queue[i].tgc_cnt > mincnt) continue; if (strict) { LIST_FOREACH(n, &qgroup->tqg_queue[i].tgc_tasks, gt_list) if (n->gt_uniq == uniq) break; if (n != NULL) continue; } mincnt = qgroup->tqg_queue[i].tgc_cnt; idx = i; } } if (idx == -1) panic("%s: failed to pick a qid.", __func__); return (idx); } void taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, void *uniq, device_t dev, struct resource *irq, const char *name) { int cpu, qid, error; KASSERT(qgroup->tqg_cnt > 0, ("qgroup %s has no queues", qgroup->tqg_name)); gtask->gt_uniq = uniq; snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); gtask->gt_dev = dev; gtask->gt_irq = irq; gtask->gt_cpu = -1; mtx_lock(&qgroup->tqg_lock); qid = taskqgroup_find(qgroup, uniq); qgroup->tqg_queue[qid].tgc_cnt++; LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; if (dev != NULL && irq != NULL) { cpu = qgroup->tqg_queue[qid].tgc_cpu; gtask->gt_cpu = cpu; mtx_unlock(&qgroup->tqg_lock); error = bus_bind_intr(dev, irq, cpu); if (error) printf("%s: binding interrupt failed for %s: %d\n", __func__, gtask->gt_name, error); } else mtx_unlock(&qgroup->tqg_lock); } int taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, void *uniq, int cpu, device_t dev, struct resource *irq, const char *name) { int i, qid, error; gtask->gt_uniq = uniq; snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); gtask->gt_dev = dev; gtask->gt_irq = irq; gtask->gt_cpu = cpu; mtx_lock(&qgroup->tqg_lock); for (i = 0, qid = -1; i < qgroup->tqg_cnt; i++) if (qgroup->tqg_queue[i].tgc_cpu == cpu) { qid = i; break; } if (qid == -1) { mtx_unlock(&qgroup->tqg_lock); printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); return (EINVAL); } qgroup->tqg_queue[qid].tgc_cnt++; LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; cpu = qgroup->tqg_queue[qid].tgc_cpu; mtx_unlock(&qgroup->tqg_lock); if (dev != NULL && irq != NULL) { error = bus_bind_intr(dev, irq, cpu); if (error) printf("%s: binding interrupt failed for %s: %d\n", __func__, gtask->gt_name, error); } return (0); } void taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) { int i; grouptask_block(gtask); mtx_lock(&qgroup->tqg_lock); for (i = 0; i < qgroup->tqg_cnt; i++) if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue) break; if (i == qgroup->tqg_cnt) panic("%s: task %s not in group", __func__, gtask->gt_name); qgroup->tqg_queue[i].tgc_cnt--; LIST_REMOVE(gtask, gt_list); mtx_unlock(&qgroup->tqg_lock); gtask->gt_taskqueue = NULL; gtask->gt_task.ta_flags &= ~TASK_NOENQUEUE; } static void taskqgroup_binder(void *ctx) { struct taskq_bind_task *gtask; cpuset_t mask; int error; gtask = ctx; CPU_ZERO(&mask); CPU_SET(gtask->bt_cpuid, &mask); error = cpuset_setthread(curthread->td_tid, &mask); thread_lock(curthread); sched_bind(curthread, gtask->bt_cpuid); thread_unlock(curthread); if (error) printf("%s: binding curthread failed: %d\n", __func__, error); free(gtask, M_DEVBUF); } void taskqgroup_bind(struct taskqgroup *qgroup) { struct taskq_bind_task *gtask; int i; /* * Bind taskqueue threads to specific CPUs, if they have been assigned * one. */ if (qgroup->tqg_cnt == 1) return; for (i = 0; i < qgroup->tqg_cnt; i++) { gtask = malloc(sizeof(*gtask), M_DEVBUF, M_WAITOK); GTASK_INIT(>ask->bt_task, 0, 0, taskqgroup_binder, gtask); gtask->bt_cpuid = qgroup->tqg_queue[i].tgc_cpu; grouptaskqueue_enqueue(qgroup->tqg_queue[i].tgc_taskq, >ask->bt_task); } } struct taskqgroup * taskqgroup_create(const char *name, int cnt, int stride) { struct taskqgroup *qgroup; int cpu, i, j; qgroup = malloc(sizeof(*qgroup), M_GTASKQUEUE, M_WAITOK | M_ZERO); mtx_init(&qgroup->tqg_lock, "taskqgroup", NULL, MTX_DEF); qgroup->tqg_name = name; qgroup->tqg_cnt = cnt; for (cpu = i = 0; i < cnt; i++) { taskqgroup_cpu_create(qgroup, i, cpu); for (j = 0; j < stride; j++) cpu = CPU_NEXT(cpu); } return (qgroup); } void taskqgroup_destroy(struct taskqgroup *qgroup) { }
Upload File
Create Folder