003 File Manager
Current Path:
/usr/src/sys/geom/bde
usr
/
src
/
sys
/
geom
/
bde
/
📁
..
📄
g_bde.c
(7.72 KB)
📄
g_bde.h
(6.44 KB)
📄
g_bde_crypt.c
(10.06 KB)
📄
g_bde_lock.c
(12.25 KB)
📄
g_bde_work.c
(20.12 KB)
Editing: g_bde_work.c
/*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2002 Poul-Henning Kamp * Copyright (c) 2002 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Poul-Henning Kamp * and NAI Labs, the Security Research Division of Network Associates, Inc. * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the * DARPA CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This source file contains the state-engine which makes things happen in the * right order. * * Outline: * 1) g_bde_start1() * Break the struct bio into multiple work packets one per zone. * 2) g_bde_start2() * Setup the necessary sector buffers and start those read operations * which we can start at this time and put the item on the work-list. * 3) g_bde_worker() * Scan the work-list for items which are ready for crypto processing * and call the matching crypto function in g_bde_crypt.c and schedule * any writes needed. Read operations finish here by releasing the * sector buffers and delivering the original bio request. * 4) g_bde_write_done() * Release sector buffers and deliver the original bio request. * * Because of the C-scope rules, the functions are almost perfectly in the * opposite order in this source file. * * XXX: A switch to the hardware assisted crypto in src/sys/opencrypto will add * XXX: additional states to this state-engine. Since no hardware available * XXX: at this time has AES support, implementing this has been postponed * XXX: until such time as it would result in a benefit. */ #include <sys/param.h> #include <sys/bio.h> #include <sys/lock.h> #include <sys/mutex.h> #include <sys/queue.h> #include <sys/malloc.h> #include <sys/systm.h> #include <sys/kernel.h> #include <sys/sysctl.h> #include <sys/proc.h> #include <sys/kthread.h> #include <crypto/rijndael/rijndael-api-fst.h> #include <crypto/sha2/sha512.h> #include <geom/geom.h> #include <geom/bde/g_bde.h> /* * FIXME: This used to call malloc_last_fail which in practice was almost * guaranteed to return time_uptime even in face of severe memory shortage. * As GBDE is the only consumer the kludge below was added to facilitate the * removal with minimial changes. The code should be fixed to respond to memory * pressure (e.g., by using lowmem eventhandler) instead. */ static int g_bde_malloc_last_fail(void) { return (time_uptime); } static void g_bde_delete_sector(struct g_bde_softc *wp, struct g_bde_sector *sp); static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len); static void g_bde_release_keysector(struct g_bde_work *wp); static struct g_bde_sector *g_bde_get_keysector(struct g_bde_work *wp); static int g_bde_start_read(struct g_bde_sector *sp); static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction); /* * Work item allocation. * * C++ would call these constructors and destructors. */ static u_int g_bde_nwork; SYSCTL_UINT(_debug, OID_AUTO, gbde_nwork, CTLFLAG_RD, &g_bde_nwork, 0, ""); static MALLOC_DEFINE(M_GBDE, "gbde", "GBDE data structures"); static struct g_bde_work * g_bde_new_work(struct g_bde_softc *sc) { struct g_bde_work *wp; wp = malloc(sizeof *wp, M_GBDE, M_NOWAIT | M_ZERO); if (wp == NULL) return (wp); wp->state = SETUP; wp->softc = sc; g_bde_nwork++; sc->nwork++; TAILQ_INSERT_TAIL(&sc->worklist, wp, list); return (wp); } static void g_bde_delete_work(struct g_bde_work *wp) { struct g_bde_softc *sc; sc = wp->softc; g_bde_nwork--; sc->nwork--; TAILQ_REMOVE(&sc->worklist, wp, list); free(wp, M_GBDE); } /* * Sector buffer allocation * * These two functions allocate and free back variable sized sector buffers */ static u_int g_bde_nsect; SYSCTL_UINT(_debug, OID_AUTO, gbde_nsect, CTLFLAG_RD, &g_bde_nsect, 0, ""); static void g_bde_delete_sector(struct g_bde_softc *sc, struct g_bde_sector *sp) { g_bde_nsect--; sc->nsect--; if (sp->malloc) free(sp->data, M_GBDE); free(sp, M_GBDE); } static struct g_bde_sector * g_bde_new_sector(struct g_bde_work *wp, u_int len) { struct g_bde_sector *sp; sp = malloc(sizeof *sp, M_GBDE, M_NOWAIT | M_ZERO); if (sp == NULL) return (sp); if (len > 0) { sp->data = malloc(len, M_GBDE, M_NOWAIT | M_ZERO); if (sp->data == NULL) { free(sp, M_GBDE); return (NULL); } sp->malloc = 1; } g_bde_nsect++; wp->softc->nsect++; sp->size = len; sp->softc = wp->softc; sp->ref = 1; sp->owner = wp; sp->offset = wp->so; sp->state = JUNK; return (sp); } /* * Skey sector cache. * * Nothing prevents two separate I/O requests from addressing the same zone * and thereby needing the same skey sector. We therefore need to sequence * I/O operations to the skey sectors. A certain amount of caching is also * desirable, although the extent of benefit from this is not at this point * determined. * * XXX: GEOM may be able to grow a generic caching facility at some point * XXX: to support such needs. */ static u_int g_bde_ncache; SYSCTL_UINT(_debug, OID_AUTO, gbde_ncache, CTLFLAG_RD, &g_bde_ncache, 0, ""); static void g_bde_purge_one_sector(struct g_bde_softc *sc, struct g_bde_sector *sp) { g_trace(G_T_TOPOLOGY, "g_bde_purge_one_sector(%p, %p)", sc, sp); if (sp->ref != 0) return; TAILQ_REMOVE(&sc->freelist, sp, list); g_bde_ncache--; sc->ncache--; bzero(sp->data, sp->size); g_bde_delete_sector(sc, sp); } static struct g_bde_sector * g_bde_get_keysector(struct g_bde_work *wp) { struct g_bde_sector *sp; struct g_bde_softc *sc; off_t offset; offset = wp->kso; g_trace(G_T_TOPOLOGY, "g_bde_get_keysector(%p, %jd)", wp, (intmax_t)offset); sc = wp->softc; if (g_bde_malloc_last_fail() < g_bde_ncache) g_bde_purge_sector(sc, -1); sp = TAILQ_FIRST(&sc->freelist); if (sp != NULL && sp->ref == 0 && sp->used + 300 < time_uptime) g_bde_purge_one_sector(sc, sp); TAILQ_FOREACH(sp, &sc->freelist, list) { if (sp->offset == offset) break; } if (sp != NULL) { sp->ref++; KASSERT(sp->offset == offset, ("wrong offset")); KASSERT(sp->softc == wp->softc, ("wrong softc")); if (sp->ref == 1) sp->owner = wp; } else { if (g_bde_malloc_last_fail() < g_bde_ncache) { TAILQ_FOREACH(sp, &sc->freelist, list) if (sp->ref == 0) break; } if (sp == NULL && !TAILQ_EMPTY(&sc->freelist)) sp = TAILQ_FIRST(&sc->freelist); if (sp != NULL && sp->ref > 0) sp = NULL; if (sp == NULL) { sp = g_bde_new_sector(wp, sc->sectorsize); if (sp != NULL) { g_bde_ncache++; sc->ncache++; TAILQ_INSERT_TAIL(&sc->freelist, sp, list); sp->malloc = 2; } } if (sp != NULL) { sp->offset = offset; sp->softc = wp->softc; sp->ref = 1; sp->owner = wp; sp->state = JUNK; sp->error = 0; } } if (sp != NULL) { TAILQ_REMOVE(&sc->freelist, sp, list); TAILQ_INSERT_TAIL(&sc->freelist, sp, list); sp->used = time_uptime; } wp->ksp = sp; return(sp); } static void g_bde_release_keysector(struct g_bde_work *wp) { struct g_bde_softc *sc; struct g_bde_work *wp2; struct g_bde_sector *sp; sp = wp->ksp; g_trace(G_T_TOPOLOGY, "g_bde_release_keysector(%p)", sp); KASSERT(sp->malloc == 2, ("Wrong sector released")); sc = sp->softc; KASSERT(sc != NULL, ("NULL sp->softc")); KASSERT(wp == sp->owner, ("Releasing, not owner")); sp->owner = NULL; wp->ksp = NULL; sp->ref--; if (sp->ref > 0) { TAILQ_REMOVE(&sc->freelist, sp, list); TAILQ_INSERT_TAIL(&sc->freelist, sp, list); TAILQ_FOREACH(wp2, &sc->worklist, list) { if (wp2->ksp == sp) { KASSERT(wp2 != wp, ("Self-reowning")); sp->owner = wp2; wakeup(sp->softc); break; } } KASSERT(wp2 != NULL, ("Failed to pick up owner for %p\n", sp)); } else if (sp->error != 0) { sp->offset = ~0; sp->error = 0; sp->state = JUNK; } TAILQ_REMOVE(&sc->freelist, sp, list); TAILQ_INSERT_HEAD(&sc->freelist, sp, list); } static void g_bde_purge_sector(struct g_bde_softc *sc, int fraction) { struct g_bde_sector *sp; int n; g_trace(G_T_TOPOLOGY, "g_bde_purge_sector(%p)", sc); if (fraction > 0) n = sc->ncache / fraction + 1; else n = g_bde_ncache - g_bde_malloc_last_fail(); if (n < 0) return; if (n > sc->ncache) n = sc->ncache; while(n--) { TAILQ_FOREACH(sp, &sc->freelist, list) { if (sp->ref != 0) continue; TAILQ_REMOVE(&sc->freelist, sp, list); g_bde_ncache--; sc->ncache--; bzero(sp->data, sp->size); g_bde_delete_sector(sc, sp); break; } } } static struct g_bde_sector * g_bde_read_keysector(struct g_bde_softc *sc, struct g_bde_work *wp) { struct g_bde_sector *sp; g_trace(G_T_TOPOLOGY, "g_bde_read_keysector(%p)", wp); sp = g_bde_get_keysector(wp); if (sp == NULL) { g_bde_purge_sector(sc, -1); sp = g_bde_get_keysector(wp); } if (sp == NULL) return (sp); if (sp->owner != wp) return (sp); if (sp->state == VALID) return (sp); if (g_bde_start_read(sp) == 0) return (sp); g_bde_release_keysector(wp); return (NULL); } /* * Contribute to the completion of the original bio request. * * We have no simple way to tell how many bits the original bio request has * been segmented into, so the easiest way to determine when we can deliver * it is to keep track of the number of bytes we have completed. We keep * track of any errors underway and latch onto the first one. * * We always report "nothing done" in case of error, because random bits here * and there may be completed and returning a number of completed bytes does * not convey any useful information about which bytes they were. If some * piece of broken code somewhere interprets this to mean that nothing has * changed on the underlying media they deserve the lossage headed for them. * * A single mutex per g_bde instance is used to prevent contention. */ static void g_bde_contribute(struct bio *bp, off_t bytes, int error) { g_trace(G_T_TOPOLOGY, "g_bde_contribute bp %p bytes %jd error %d", bp, (intmax_t)bytes, error); if (bp->bio_error == 0) bp->bio_error = error; bp->bio_completed += bytes; KASSERT(bp->bio_completed <= bp->bio_length, ("Too large contribution")); if (bp->bio_completed == bp->bio_length) { if (bp->bio_error != 0) bp->bio_completed = 0; g_io_deliver(bp, bp->bio_error); } } /* * This is the common case "we're done with this work package" function */ static void g_bde_work_done(struct g_bde_work *wp, int error) { g_bde_contribute(wp->bp, wp->length, error); if (wp->sp != NULL) g_bde_delete_sector(wp->softc, wp->sp); if (wp->ksp != NULL) g_bde_release_keysector(wp); g_bde_delete_work(wp); } /* * A write operation has finished. When we have all expected cows in the * barn close the door and call it a day. */ static void g_bde_write_done(struct bio *bp) { struct g_bde_sector *sp; struct g_bde_work *wp; struct g_bde_softc *sc; sp = bp->bio_caller1; sc = bp->bio_caller2; mtx_lock(&sc->worklist_mutex); KASSERT(sp != NULL, ("NULL sp")); KASSERT(sc != NULL, ("NULL sc")); KASSERT(sp->owner != NULL, ("NULL sp->owner")); g_trace(G_T_TOPOLOGY, "g_bde_write_done(%p)", sp); if (bp->bio_error == 0 && bp->bio_completed != sp->size) bp->bio_error = EIO; sp->error = bp->bio_error; g_destroy_bio(bp); wp = sp->owner; if (wp->error == 0) wp->error = sp->error; if (wp->bp->bio_cmd == BIO_DELETE) { KASSERT(sp == wp->sp, ("trashed delete op")); g_bde_work_done(wp, wp->error); mtx_unlock(&sc->worklist_mutex); return; } KASSERT(wp->bp->bio_cmd == BIO_WRITE, ("Confused in g_bde_write_done()")); KASSERT(sp == wp->sp || sp == wp->ksp, ("trashed write op")); if (wp->sp == sp) { g_bde_delete_sector(sc, wp->sp); wp->sp = NULL; } else { sp->state = VALID; } if (wp->sp == NULL && wp->ksp != NULL && wp->ksp->state == VALID) g_bde_work_done(wp, wp->error); mtx_unlock(&sc->worklist_mutex); return; } /* * Send a write request for the given sector down the pipeline. */ static int g_bde_start_write(struct g_bde_sector *sp) { struct bio *bp; struct g_bde_softc *sc; g_trace(G_T_TOPOLOGY, "g_bde_start_write(%p)", sp); sc = sp->softc; KASSERT(sc != NULL, ("NULL sc in g_bde_start_write")); KASSERT(sp->owner != NULL, ("NULL sp->owner in g_bde_start_write")); bp = g_new_bio(); if (bp == NULL) return (ENOMEM); bp->bio_cmd = BIO_WRITE; bp->bio_offset = sp->offset; bp->bio_data = sp->data; bp->bio_length = sp->size; bp->bio_done = g_bde_write_done; bp->bio_caller1 = sp; bp->bio_caller2 = sc; sp->state = IO; g_io_request(bp, sc->consumer); return(0); } /* * A read operation has finished. Mark the sector no longer iobusy and * wake up the worker thread and let it do its thing. */ static void g_bde_read_done(struct bio *bp) { struct g_bde_sector *sp; struct g_bde_softc *sc; sp = bp->bio_caller1; g_trace(G_T_TOPOLOGY, "g_bde_read_done(%p)", sp); sc = bp->bio_caller2; mtx_lock(&sc->worklist_mutex); if (bp->bio_error == 0 && bp->bio_completed != sp->size) bp->bio_error = EIO; sp->error = bp->bio_error; if (sp->error == 0) sp->state = VALID; else sp->state = JUNK; wakeup(sc); g_destroy_bio(bp); mtx_unlock(&sc->worklist_mutex); } /* * Send a read request for the given sector down the pipeline. */ static int g_bde_start_read(struct g_bde_sector *sp) { struct bio *bp; struct g_bde_softc *sc; g_trace(G_T_TOPOLOGY, "g_bde_start_read(%p)", sp); sc = sp->softc; KASSERT(sc != NULL, ("Null softc in sp %p", sp)); bp = g_new_bio(); if (bp == NULL) return (ENOMEM); bp->bio_cmd = BIO_READ; bp->bio_offset = sp->offset; bp->bio_data = sp->data; bp->bio_length = sp->size; bp->bio_done = g_bde_read_done; bp->bio_caller1 = sp; bp->bio_caller2 = sc; sp->state = IO; g_io_request(bp, sc->consumer); return(0); } /* * The worker thread. * * The up/down path of GEOM is not allowed to sleep or do any major work * so we use this thread to do the actual crypto operations and to push * the state engine onwards. * * XXX: if we switch to the src/sys/opencrypt hardware assisted encryption * XXX: using a thread here is probably not needed. */ void g_bde_worker(void *arg) { struct g_bde_softc *sc; struct g_bde_work *wp, *twp; struct g_geom *gp; int restart, error; gp = arg; sc = gp->softc; mtx_lock(&sc->worklist_mutex); for (;;) { restart = 0; g_trace(G_T_TOPOLOGY, "g_bde_worker scan"); TAILQ_FOREACH_SAFE(wp, &sc->worklist, list, twp) { KASSERT(wp != NULL, ("NULL wp")); KASSERT(wp->softc != NULL, ("NULL wp->softc")); if (wp->state != WAIT) continue; /* Not interesting here */ KASSERT(wp->bp != NULL, ("NULL wp->bp")); KASSERT(wp->sp != NULL, ("NULL wp->sp")); if (wp->ksp != NULL) { if (wp->ksp->owner != wp) continue; if (wp->ksp->state == IO) continue; KASSERT(wp->ksp->state == VALID, ("Illegal sector state (%d)", wp->ksp->state)); } if (wp->bp->bio_cmd == BIO_READ && wp->sp->state == IO) continue; if (wp->ksp != NULL && wp->ksp->error != 0) { g_bde_work_done(wp, wp->ksp->error); continue; } switch(wp->bp->bio_cmd) { case BIO_READ: if (wp->ksp == NULL) { KASSERT(wp->error != 0, ("BIO_READ, no ksp and no error")); g_bde_work_done(wp, wp->error); break; } if (wp->sp->error != 0) { g_bde_work_done(wp, wp->sp->error); break; } mtx_unlock(&sc->worklist_mutex); g_bde_crypt_read(wp); mtx_lock(&sc->worklist_mutex); restart++; g_bde_work_done(wp, wp->sp->error); break; case BIO_WRITE: wp->state = FINISH; KASSERT(wp->sp->owner == wp, ("Write not owner sp")); KASSERT(wp->ksp->owner == wp, ("Write not owner ksp")); mtx_unlock(&sc->worklist_mutex); g_bde_crypt_write(wp); mtx_lock(&sc->worklist_mutex); restart++; error = g_bde_start_write(wp->sp); if (error) { g_bde_work_done(wp, error); break; } error = g_bde_start_write(wp->ksp); if (wp->error != 0) wp->error = error; break; case BIO_DELETE: wp->state = FINISH; mtx_unlock(&sc->worklist_mutex); g_bde_crypt_delete(wp); mtx_lock(&sc->worklist_mutex); restart++; g_bde_start_write(wp->sp); break; } if (restart) break; } if (!restart) { /* * We don't look for our death-warrant until we are * idle. Shouldn't make a difference in practice. */ if (sc->dead) break; g_trace(G_T_TOPOLOGY, "g_bde_worker sleep"); error = msleep(sc, &sc->worklist_mutex, PRIBIO, "-", hz); if (error == EWOULDBLOCK) { /* * Lose our skey cache in an orderly fashion. * The exact rate can be tuned to be less * aggressive if this is desirable. 10% per * second means that the cache is gone in a * few minutes. */ g_bde_purge_sector(sc, 10); } } } g_trace(G_T_TOPOLOGY, "g_bde_worker die"); g_bde_purge_sector(sc, 1); KASSERT(sc->nwork == 0, ("Dead but %d work remaining", sc->nwork)); KASSERT(sc->ncache == 0, ("Dead but %d cache remaining", sc->ncache)); KASSERT(sc->nsect == 0, ("Dead but %d sect remaining", sc->nsect)); mtx_unlock(&sc->worklist_mutex); sc->dead = 2; wakeup(sc); kproc_exit(0); } /* * g_bde_start1 has chopped the incoming request up so all the requests * we see here are inside a single zone. Map the data and key locations * grab the buffers we need and fire off the first volley of read requests. */ static void g_bde_start2(struct g_bde_work *wp) { struct g_bde_softc *sc; KASSERT(wp != NULL, ("NULL wp in g_bde_start2")); KASSERT(wp->softc != NULL, ("NULL wp->softc")); g_trace(G_T_TOPOLOGY, "g_bde_start2(%p)", wp); sc = wp->softc; switch (wp->bp->bio_cmd) { case BIO_READ: wp->sp = g_bde_new_sector(wp, 0); if (wp->sp == NULL) { g_bde_work_done(wp, ENOMEM); return; } wp->sp->size = wp->length; wp->sp->data = wp->data; if (g_bde_start_read(wp->sp) != 0) { g_bde_work_done(wp, ENOMEM); return; } g_bde_read_keysector(sc, wp); if (wp->ksp == NULL) wp->error = ENOMEM; break; case BIO_DELETE: wp->sp = g_bde_new_sector(wp, wp->length); if (wp->sp == NULL) { g_bde_work_done(wp, ENOMEM); return; } break; case BIO_WRITE: wp->sp = g_bde_new_sector(wp, wp->length); if (wp->sp == NULL) { g_bde_work_done(wp, ENOMEM); return; } g_bde_read_keysector(sc, wp); if (wp->ksp == NULL) { g_bde_work_done(wp, ENOMEM); return; } break; default: KASSERT(0 == 1, ("Wrong bio_cmd %d in g_bde_start2", wp->bp->bio_cmd)); } wp->state = WAIT; wakeup(sc); } /* * Create a sequence of work structures, and have g_bde_map_sector() determine * how long they each can be. Feed them to g_bde_start2(). */ void g_bde_start1(struct bio *bp) { struct g_bde_softc *sc; struct g_bde_work *wp; off_t done; sc = bp->bio_to->geom->softc; bp->bio_driver1 = sc; mtx_lock(&sc->worklist_mutex); for(done = 0; done < bp->bio_length; ) { wp = g_bde_new_work(sc); if (wp != NULL) { wp->bp = bp; wp->offset = bp->bio_offset + done; wp->data = bp->bio_data + done; wp->length = bp->bio_length - done; g_bde_map_sector(wp); done += wp->length; g_bde_start2(wp); } if (wp == NULL || bp->bio_error != 0) { g_bde_contribute(bp, bp->bio_length - done, ENOMEM); break; } } mtx_unlock(&sc->worklist_mutex); return; }
Upload File
Create Folder