// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2019 Oracle. All Rights Reserved. * Author: Darrick J. Wong <darrick.wong@oracle.com> */ #include "xfs.h" #include "xfs_fs.h" #include "xfs_shared.h" #include "xfs_format.h" #include "xfs_log_format.h" #include "xfs_trans_resv.h" #include "xfs_mount.h" #include "xfs_inode.h" #include "xfs_btree.h" #include "xfs_ialloc.h" #include "xfs_ialloc_btree.h" #include "xfs_iwalk.h" #include "xfs_error.h" #include "xfs_trace.h" #include "xfs_icache.h" #include "xfs_health.h" #include "xfs_trans.h" #include "xfs_pwork.h" /* * Walking Inodes in the Filesystem * ================================ * * This iterator function walks a subset of filesystem inodes in increasing * order from @startino until there are no more inodes. For each allocated * inode it finds, it calls a walk function with the relevant inode number and * a pointer to caller-provided data. The walk function can return the usual * negative error code to stop the iteration; 0 to continue the iteration; or * -ECANCELED to stop the iteration. This return value is returned to the * caller. * * Internally, we allow the walk function to do anything, which means that we * cannot maintain the inobt cursor or our lock on the AGI buffer. We * therefore cache the inobt records in kernel memory and only call the walk * function when our memory buffer is full. @nr_recs is the number of records * that we've cached, and @sz_recs is the size of our cache. * * It is the responsibility of the walk function to ensure it accesses * allocated inodes, as the inobt records may be stale by the time they are * acted upon. */ struct xfs_iwalk_ag { /* parallel work control data; will be null if single threaded */ struct xfs_pwork pwork; struct xfs_mount *mp; struct xfs_trans *tp; /* Where do we start the traversal? */ xfs_ino_t startino; /* Array of inobt records we cache. */ struct xfs_inobt_rec_incore *recs; /* Number of entries allocated for the @recs array. */ unsigned int sz_recs; /* Number of entries in the @recs array that are in use. */ unsigned int nr_recs; /* Inode walk function and data pointer. */ xfs_iwalk_fn iwalk_fn; xfs_inobt_walk_fn inobt_walk_fn; void *data; /* * Make it look like the inodes up to startino are free so that * bulkstat can start its inode iteration at the correct place without * needing to special case everywhere. */ unsigned int trim_start:1; /* Skip empty inobt records? */ unsigned int skip_empty:1; }; /* * Loop over all clusters in a chunk for a given incore inode allocation btree * record. Do a readahead if there are any allocated inodes in that cluster. */ STATIC void xfs_iwalk_ichunk_ra( struct xfs_mount *mp, xfs_agnumber_t agno, struct xfs_inobt_rec_incore *irec) { struct xfs_ino_geometry *igeo = M_IGEO(mp); xfs_agblock_t agbno; struct blk_plug plug; int i; /* inode chunk index */ agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino); blk_start_plug(&plug); for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) { xfs_inofree_t imask; imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster); if (imask & ~irec->ir_free) { xfs_btree_reada_bufs(mp, agno, agbno, igeo->blocks_per_cluster, &xfs_inode_buf_ops); } agbno += igeo->blocks_per_cluster; } blk_finish_plug(&plug); } /* * Set the bits in @irec's free mask that correspond to the inodes before * @agino so that we skip them. This is how we restart an inode walk that was * interrupted in the middle of an inode record. */ STATIC void xfs_iwalk_adjust_start( xfs_agino_t agino, /* starting inode of chunk */ struct xfs_inobt_rec_incore *irec) /* btree record */ { int idx; /* index into inode chunk */ int i; idx = agino - irec->ir_startino; /* * We got a right chunk with some left inodes allocated at it. Grab * the chunk record. Mark all the uninteresting inodes free because * they're before our start point. */ for (i = 0; i < idx; i++) { if (XFS_INOBT_MASK(i) & ~irec->ir_free) irec->ir_freecount++; } irec->ir_free |= xfs_inobt_maskn(0, idx); } /* Allocate memory for a walk. */ STATIC int xfs_iwalk_alloc( struct xfs_iwalk_ag *iwag) { size_t size; ASSERT(iwag->recs == NULL); iwag->nr_recs = 0; /* Allocate a prefetch buffer for inobt records. */ size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore); iwag->recs = kmem_alloc(size, KM_MAYFAIL); if (iwag->recs == NULL) return -ENOMEM; return 0; } /* Free memory we allocated for a walk. */ STATIC void xfs_iwalk_free( struct xfs_iwalk_ag *iwag) { kmem_free(iwag->recs); iwag->recs = NULL; } /* For each inuse inode in each cached inobt record, call our function. */ STATIC int xfs_iwalk_ag_recs( struct xfs_iwalk_ag *iwag) { struct xfs_mount *mp = iwag->mp; struct xfs_trans *tp = iwag->tp; xfs_ino_t ino; unsigned int i, j; xfs_agnumber_t agno; int error; agno = XFS_INO_TO_AGNO(mp, iwag->startino); for (i = 0; i < iwag->nr_recs; i++) { struct xfs_inobt_rec_incore *irec = &iwag->recs[i]; trace_xfs_iwalk_ag_rec(mp, agno, irec); if (xfs_pwork_want_abort(&iwag->pwork)) return 0; if (iwag->inobt_walk_fn) { error = iwag->inobt_walk_fn(mp, tp, agno, irec, iwag->data); if (error) return error; } if (!iwag->iwalk_fn) continue; for (j = 0; j < XFS_INODES_PER_CHUNK; j++) { if (xfs_pwork_want_abort(&iwag->pwork)) return 0; /* Skip if this inode is free */ if (XFS_INOBT_MASK(j) & irec->ir_free) continue; /* Otherwise call our function. */ ino = XFS_AGINO_TO_INO(mp, agno, irec->ir_startino + j); error = iwag->iwalk_fn(mp, tp, ino, iwag->data); if (error) return error; } } return 0; } /* Delete cursor and let go of AGI. */ static inline void xfs_iwalk_del_inobt( struct xfs_trans *tp, struct xfs_btree_cur **curpp, struct xfs_buf **agi_bpp, int error) { if (*curpp) { xfs_btree_del_cursor(*curpp, error); *curpp = NULL; } if (*agi_bpp) { xfs_trans_brelse(tp, *agi_bpp); *agi_bpp = NULL; } } /* * Set ourselves up for walking inobt records starting from a given point in * the filesystem. * * If caller passed in a nonzero start inode number, load the record from the * inobt and make the record look like all the inodes before agino are free so * that we skip them, and then move the cursor to the next inobt record. This * is how we support starting an iwalk in the middle of an inode chunk. * * If the caller passed in a start number of zero, move the cursor to the first * inobt record. * * The caller is responsible for cleaning up the cursor and buffer pointer * regardless of the error status. */ STATIC int xfs_iwalk_ag_start( struct xfs_iwalk_ag *iwag, xfs_agnumber_t agno, xfs_agino_t agino, struct xfs_btree_cur **curpp, struct xfs_buf **agi_bpp, int *has_more) { struct xfs_mount *mp = iwag->mp; struct xfs_trans *tp = iwag->tp; struct xfs_inobt_rec_incore *irec; int error; /* Set up a fresh cursor and empty the inobt cache. */ iwag->nr_recs = 0; error = xfs_inobt_cur(mp, tp, agno, XFS_BTNUM_INO, curpp, agi_bpp); if (error) return error; /* Starting at the beginning of the AG? That's easy! */ if (agino == 0) return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more); /* * Otherwise, we have to grab the inobt record where we left off, stuff * the record into our cache, and then see if there are more records. * We require a lookup cache of at least two elements so that the * caller doesn't have to deal with tearing down the cursor to walk the * records. */ error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more); if (error) return error; /* * If the LE lookup at @agino yields no records, jump ahead to the * inobt cursor increment to see if there are more records to process. */ if (!*has_more) goto out_advance; /* Get the record, should always work */ irec = &iwag->recs[iwag->nr_recs]; error = xfs_inobt_get_rec(*curpp, irec, has_more); if (error) return error; if (XFS_IS_CORRUPT(mp, *has_more != 1)) return -EFSCORRUPTED; /* * If the LE lookup yielded an inobt record before the cursor position, * skip it and see if there's another one after it. */ if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino) goto out_advance; /* * If agino fell in the middle of the inode record, make it look like * the inodes up to agino are free so that we don't return them again. */ if (iwag->trim_start) xfs_iwalk_adjust_start(agino, irec); /* * The prefetch calculation is supposed to give us a large enough inobt * record cache that grab_ichunk can stage a partial first record and * the loop body can cache a record without having to check for cache * space until after it reads an inobt record. */ iwag->nr_recs++; ASSERT(iwag->nr_recs < iwag->sz_recs); out_advance: return xfs_btree_increment(*curpp, 0, has_more); } /* * The inobt record cache is full, so preserve the inobt cursor state and * run callbacks on the cached inobt records. When we're done, restore the * cursor state to wherever the cursor would have been had the cache not been * full (and therefore we could've just incremented the cursor) if *@has_more * is true. On exit, *@has_more will indicate whether or not the caller should * try for more inode records. */ STATIC int xfs_iwalk_run_callbacks( struct xfs_iwalk_ag *iwag, xfs_agnumber_t agno, struct xfs_btree_cur **curpp, struct xfs_buf **agi_bpp, int *has_more) { struct xfs_mount *mp = iwag->mp; struct xfs_trans *tp = iwag->tp; struct xfs_inobt_rec_incore *irec; xfs_agino_t restart; int error; ASSERT(iwag->nr_recs > 0); /* Delete cursor but remember the last record we cached... */ xfs_iwalk_del_inobt(tp, curpp, agi_bpp, 0); irec = &iwag->recs[iwag->nr_recs - 1]; restart = irec->ir_startino + XFS_INODES_PER_CHUNK - 1; error = xfs_iwalk_ag_recs(iwag); if (error) return error; /* ...empty the cache... */ iwag->nr_recs = 0; if (!has_more) return 0; /* ...and recreate the cursor just past where we left off. */ error = xfs_inobt_cur(mp, tp, agno, XFS_BTNUM_INO, curpp, agi_bpp); if (error) return error; return xfs_inobt_lookup(*curpp, restart, XFS_LOOKUP_GE, has_more); } /* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */ STATIC int xfs_iwalk_ag( struct xfs_iwalk_ag *iwag) { struct xfs_mount *mp = iwag->mp; struct xfs_trans *tp = iwag->tp; struct xfs_buf *agi_bp = NULL; struct xfs_btree_cur *cur = NULL; xfs_agnumber_t agno; xfs_agino_t agino; int has_more; int error = 0; /* Set up our cursor at the right place in the inode btree. */ agno = XFS_INO_TO_AGNO(mp, iwag->startino); agino = XFS_INO_TO_AGINO(mp, iwag->startino); error = xfs_iwalk_ag_start(iwag, agno, agino, &cur, &agi_bp, &has_more); while (!error && has_more) { struct xfs_inobt_rec_incore *irec; cond_resched(); if (xfs_pwork_want_abort(&iwag->pwork)) goto out; /* Fetch the inobt record. */ irec = &iwag->recs[iwag->nr_recs]; error = xfs_inobt_get_rec(cur, irec, &has_more); if (error || !has_more) break; /* No allocated inodes in this chunk; skip it. */ if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) { error = xfs_btree_increment(cur, 0, &has_more); if (error) break; continue; } /* * Start readahead for this inode chunk in anticipation of * walking the inodes. */ if (iwag->iwalk_fn) xfs_iwalk_ichunk_ra(mp, agno, irec); /* * If there's space in the buffer for more records, increment * the btree cursor and grab more. */ if (++iwag->nr_recs < iwag->sz_recs) { error = xfs_btree_increment(cur, 0, &has_more); if (error || !has_more) break; continue; } /* * Otherwise, we need to save cursor state and run the callback * function on the cached records. The run_callbacks function * is supposed to return a cursor pointing to the record where * we would be if we had been able to increment like above. */ ASSERT(has_more); error = xfs_iwalk_run_callbacks(iwag, agno, &cur, &agi_bp, &has_more); } if (iwag->nr_recs == 0 || error) goto out; /* Walk the unprocessed records in the cache. */ error = xfs_iwalk_run_callbacks(iwag, agno, &cur, &agi_bp, &has_more); out: xfs_iwalk_del_inobt(tp, &cur, &agi_bp, error); return error; } /* * We experimentally determined that the reduction in ioctl call overhead * diminishes when userspace asks for more than 2048 inodes, so we'll cap * prefetch at this point. */ #define IWALK_MAX_INODE_PREFETCH (2048U) /* * Given the number of inodes to prefetch, set the number of inobt records that * we cache in memory, which controls the number of inodes we try to read * ahead. Set the maximum if @inodes == 0. */ static inline unsigned int xfs_iwalk_prefetch( unsigned int inodes) { unsigned int inobt_records; /* * If the caller didn't tell us the number of inodes they wanted, * assume the maximum prefetch possible for best performance. * Otherwise, cap prefetch at that maximum so that we don't start an * absurd amount of prefetch. */ if (inodes == 0) inodes = IWALK_MAX_INODE_PREFETCH; inodes = min(inodes, IWALK_MAX_INODE_PREFETCH); /* Round the inode count up to a full chunk. */ inodes = round_up(inodes, XFS_INODES_PER_CHUNK); /* * In order to convert the number of inodes to prefetch into an * estimate of the number of inobt records to cache, we require a * conversion factor that reflects our expectations of the average * loading factor of an inode chunk. Based on data gathered, most * (but not all) filesystems manage to keep the inode chunks totally * full, so we'll underestimate slightly so that our readahead will * still deliver the performance we want on aging filesystems: * * inobt = inodes / (INODES_PER_CHUNK * (4 / 5)); * * The funny math is to avoid integer division. */ inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK); /* * Allocate enough space to prefetch at least two inobt records so that * we can cache both the record where the iwalk started and the next * record. This simplifies the AG inode walk loop setup code. */ return max(inobt_records, 2U); } /* * Walk all inodes in the filesystem starting from @startino. The @iwalk_fn * will be called for each allocated inode, being passed the inode's number and * @data. @max_prefetch controls how many inobt records' worth of inodes we * try to readahead. */ int xfs_iwalk( struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t startino, unsigned int flags, xfs_iwalk_fn iwalk_fn, unsigned int inode_records, void *data) { struct xfs_iwalk_ag iwag = { .mp = mp, .tp = tp, .iwalk_fn = iwalk_fn, .data = data, .startino = startino, .sz_recs = xfs_iwalk_prefetch(inode_records), .trim_start = 1, .skip_empty = 1, .pwork = XFS_PWORK_SINGLE_THREADED, }; xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino); int error; ASSERT(agno < mp->m_sb.sb_agcount); ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL)); error = xfs_iwalk_alloc(&iwag); if (error) return error; for (; agno < mp->m_sb.sb_agcount; agno++) { error = xfs_iwalk_ag(&iwag); if (error) break; iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0); if (flags & XFS_INOBT_WALK_SAME_AG) break; } xfs_iwalk_free(&iwag); return error; } /* Run per-thread iwalk work. */ static int xfs_iwalk_ag_work( struct xfs_mount *mp, struct xfs_pwork *pwork) { struct xfs_iwalk_ag *iwag; int error = 0; iwag = container_of(pwork, struct xfs_iwalk_ag, pwork); if (xfs_pwork_want_abort(pwork)) goto out; error = xfs_iwalk_alloc(iwag); if (error) goto out; error = xfs_iwalk_ag(iwag); xfs_iwalk_free(iwag); out: kmem_free(iwag); return error; } /* * Walk all the inodes in the filesystem using multiple threads to process each * AG. */ int xfs_iwalk_threaded( struct xfs_mount *mp, xfs_ino_t startino, unsigned int flags, xfs_iwalk_fn iwalk_fn, unsigned int inode_records, bool polled, void *data) { struct xfs_pwork_ctl pctl; xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino); unsigned int nr_threads; int error; ASSERT(agno < mp->m_sb.sb_agcount); ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL)); nr_threads = xfs_pwork_guess_datadev_parallelism(mp); error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk", nr_threads); if (error) return error; for (; agno < mp->m_sb.sb_agcount; agno++) { struct xfs_iwalk_ag *iwag; if (xfs_pwork_ctl_want_abort(&pctl)) break; iwag = kmem_zalloc(sizeof(struct xfs_iwalk_ag), 0); iwag->mp = mp; iwag->iwalk_fn = iwalk_fn; iwag->data = data; iwag->startino = startino; iwag->sz_recs = xfs_iwalk_prefetch(inode_records); xfs_pwork_queue(&pctl, &iwag->pwork); startino = XFS_AGINO_TO_INO(mp, agno + 1, 0); if (flags & XFS_INOBT_WALK_SAME_AG) break; } if (polled) xfs_pwork_poll(&pctl); return xfs_pwork_destroy(&pctl); } /* * Allow callers to cache up to a page's worth of inobt records. This reflects * the existing inumbers prefetching behavior. Since the inobt walk does not * itself do anything with the inobt records, we can set a fairly high limit * here. */ #define MAX_INOBT_WALK_PREFETCH \ (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore)) /* * Given the number of records that the user wanted, set the number of inobt * records that we buffer in memory. Set the maximum if @inobt_records == 0. */ static inline unsigned int xfs_inobt_walk_prefetch( unsigned int inobt_records) { /* * If the caller didn't tell us the number of inobt records they * wanted, assume the maximum prefetch possible for best performance. */ if (inobt_records == 0) inobt_records = MAX_INOBT_WALK_PREFETCH; /* * Allocate enough space to prefetch at least two inobt records so that * we can cache both the record where the iwalk started and the next * record. This simplifies the AG inode walk loop setup code. */ inobt_records = max(inobt_records, 2U); /* * Cap prefetch at that maximum so that we don't use an absurd amount * of memory. */ return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH); } /* * Walk all inode btree records in the filesystem starting from @startino. The * @inobt_walk_fn will be called for each btree record, being passed the incore * record and @data. @max_prefetch controls how many inobt records we try to * cache ahead of time. */ int xfs_inobt_walk( struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t startino, unsigned int flags, xfs_inobt_walk_fn inobt_walk_fn, unsigned int inobt_records, void *data) { struct xfs_iwalk_ag iwag = { .mp = mp, .tp = tp, .inobt_walk_fn = inobt_walk_fn, .data = data, .startino = startino, .sz_recs = xfs_inobt_walk_prefetch(inobt_records), .pwork = XFS_PWORK_SINGLE_THREADED, }; xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino); int error; ASSERT(agno < mp->m_sb.sb_agcount); ASSERT(!(flags & ~XFS_INOBT_WALK_FLAGS_ALL)); error = xfs_iwalk_alloc(&iwag); if (error) return error; for (; agno < mp->m_sb.sb_agcount; agno++) { error = xfs_iwalk_ag(&iwag); if (error) break; iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0); if (flags & XFS_INOBT_WALK_SAME_AG) break; } xfs_iwalk_free(&iwag); return error; }