include/linux/folio_queue.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /* Queue of folios definitions
   3  *
   4  * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * See:
   8  *
   9  *      Documentation/core-api/folio_queue.rst
  10  *
  11  * for a description of the API.
  12  */
  13
  14 #ifndef _LINUX_FOLIO_QUEUE_H
  15 #define _LINUX_FOLIO_QUEUE_H
  16
  17 #include <linux/pagevec.h>
  18 #include <linux/mm.h>
  19
  20 /*
  21  * Segment in a queue of running buffers.  Each segment can hold a number of
  22  * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
  23  * iterator.  The possibility exists of inserting non-folio elements into the
  24  * queue (such as gaps).
  25  *
  26  * Explicit prev and next pointers are used instead of a list_head to make it
  27  * easier to add segments to tail and remove them from the head without the
  28  * need for a lock.
  29  */
  30 struct folio_queue {
  31         struct folio_batch      vec;            /* Folios in the queue segment */
  32         u8                      orders[PAGEVEC_SIZE]; /* Order of each folio */
  33         struct folio_queue      *next;          /* Next queue segment or NULL */
  34         struct folio_queue      *prev;          /* Previous queue segment of NULL */
  35         unsigned long           marks;          /* 1-bit mark per folio */
  36         unsigned long           marks2;         /* Second 1-bit mark per folio */
  37 #if PAGEVEC_SIZE > BITS_PER_LONG
  38 #error marks is not big enough
  39 #endif
  40         unsigned int            rreq_id;
  41         unsigned int            debug_id;
  42 };
  43
  44 /**
  45  * folioq_init - Initialise a folio queue segment
  46  * @folioq: The segment to initialise
  47  * @rreq_id: The request identifier to use in tracelines.
  48  *
  49  * Initialise a folio queue segment and set an identifier to be used in traces.
  50  *
  51  * Note that the folio pointers are left uninitialised.
  52  */
  53 static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
  54 {
  55         folio_batch_init(&folioq->vec);
  56         folioq->next = NULL;
  57         folioq->prev = NULL;
  58         folioq->marks = 0;
  59         folioq->marks2 = 0;
  60         folioq->rreq_id = rreq_id;
  61         folioq->debug_id = 0;
  62 }
  63
  64 /**
  65  * folioq_nr_slots: Query the capacity of a folio queue segment
  66  * @folioq: The segment to query
  67  *
  68  * Query the number of folios that a particular folio queue segment might hold.
  69  * [!] NOTE: This must not be assumed to be the same for every segment!
  70  */
  71 static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
  72 {
  73         return PAGEVEC_SIZE;
  74 }
  75
  76 /**
  77  * folioq_count: Query the occupancy of a folio queue segment
  78  * @folioq: The segment to query
  79  *
  80  * Query the number of folios that have been added to a folio queue segment.
  81  * Note that this is not decreased as folios are removed from a segment.
  82  */
  83 static inline unsigned int folioq_count(struct folio_queue *folioq)
  84 {
  85         return folio_batch_count(&folioq->vec);
  86 }
  87
  88 /**
  89  * folioq_full: Query if a folio queue segment is full
  90  * @folioq: The segment to query
  91  *
  92  * Query if a folio queue segment is fully occupied.  Note that this does not
  93  * change if folios are removed from a segment.
  94  */
  95 static inline bool folioq_full(struct folio_queue *folioq)
  96 {
  97         //return !folio_batch_space(&folioq->vec);
  98         return folioq_count(folioq) >= folioq_nr_slots(folioq);
  99 }
 100
 101 /**
 102  * folioq_is_marked: Check first folio mark in a folio queue segment
 103  * @folioq: The segment to query
 104  * @slot: The slot number of the folio to query
 105  *
 106  * Determine if the first mark is set for the folio in the specified slot in a
 107  * folio queue segment.
 108  */
 109 static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
 110 {
 111         return test_bit(slot, &folioq->marks);
 112 }
 113
 114 /**
 115  * folioq_mark: Set the first mark on a folio in a folio queue segment
 116  * @folioq: The segment to modify
 117  * @slot: The slot number of the folio to modify
 118  *
 119  * Set the first mark for the folio in the specified slot in a folio queue
 120  * segment.
 121  */
 122 static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
 123 {
 124         set_bit(slot, &folioq->marks);
 125 }
 126
 127 /**
 128  * folioq_unmark: Clear the first mark on a folio in a folio queue segment
 129  * @folioq: The segment to modify
 130  * @slot: The slot number of the folio to modify
 131  *
 132  * Clear the first mark for the folio in the specified slot in a folio queue
 133  * segment.
 134  */
 135 static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
 136 {
 137         clear_bit(slot, &folioq->marks);
 138 }
 139
 140 /**
 141  * folioq_is_marked2: Check second folio mark in a folio queue segment
 142  * @folioq: The segment to query
 143  * @slot: The slot number of the folio to query
 144  *
 145  * Determine if the second mark is set for the folio in the specified slot in a
 146  * folio queue segment.
 147  */
 148 static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
 149 {
 150         return test_bit(slot, &folioq->marks2);
 151 }
 152
 153 /**
 154  * folioq_mark2: Set the second mark on a folio in a folio queue segment
 155  * @folioq: The segment to modify
 156  * @slot: The slot number of the folio to modify
 157  *
 158  * Set the second mark for the folio in the specified slot in a folio queue
 159  * segment.
 160  */
 161 static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
 162 {
 163         set_bit(slot, &folioq->marks2);
 164 }
 165
 166 /**
 167  * folioq_unmark2: Clear the second mark on a folio in a folio queue segment
 168  * @folioq: The segment to modify
 169  * @slot: The slot number of the folio to modify
 170  *
 171  * Clear the second mark for the folio in the specified slot in a folio queue
 172  * segment.
 173  */
 174 static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
 175 {
 176         clear_bit(slot, &folioq->marks2);
 177 }
 178
 179 /**
 180  * folioq_append: Add a folio to a folio queue segment
 181  * @folioq: The segment to add to
 182  * @folio: The folio to add
 183  *
 184  * Add a folio to the tail of the sequence in a folio queue segment, increasing
 185  * the occupancy count and returning the slot number for the folio just added.
 186  * The folio size is extracted and stored in the queue and the marks are left
 187  * unmodified.
 188  *
 189  * Note that it's left up to the caller to check that the segment capacity will
 190  * not be exceeded and to extend the queue.
 191  */
 192 static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
 193 {
 194         unsigned int slot = folioq->vec.nr++;
 195
 196         folioq->vec.folios[slot] = folio;
 197         folioq->orders[slot] = folio_order(folio);
 198         return slot;
 199 }
 200
 201 /**
 202  * folioq_append_mark: Add a folio to a folio queue segment
 203  * @folioq: The segment to add to
 204  * @folio: The folio to add
 205  *
 206  * Add a folio to the tail of the sequence in a folio queue segment, increasing
 207  * the occupancy count and returning the slot number for the folio just added.
 208  * The folio size is extracted and stored in the queue, the first mark is set
 209  * and and the second and third marks are left unmodified.
 210  *
 211  * Note that it's left up to the caller to check that the segment capacity will
 212  * not be exceeded and to extend the queue.
 213  */
 214 static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
 215 {
 216         unsigned int slot = folioq->vec.nr++;
 217
 218         folioq->vec.folios[slot] = folio;
 219         folioq->orders[slot] = folio_order(folio);
 220         folioq_mark(folioq, slot);
 221         return slot;
 222 }
 223
 224 /**
 225  * folioq_folio: Get a folio from a folio queue segment
 226  * @folioq: The segment to access
 227  * @slot: The folio slot to access
 228  *
 229  * Retrieve the folio in the specified slot from a folio queue segment.  Note
 230  * that no bounds check is made and if the slot hasn't been added into yet, the
 231  * pointer will be undefined.  If the slot has been cleared, NULL will be
 232  * returned.
 233  */
 234 static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
 235 {
 236         return folioq->vec.folios[slot];
 237 }
 238
 239 /**
 240  * folioq_folio_order: Get the order of a folio from a folio queue segment
 241  * @folioq: The segment to access
 242  * @slot: The folio slot to access
 243  *
 244  * Retrieve the order of the folio in the specified slot from a folio queue
 245  * segment.  Note that no bounds check is made and if the slot hasn't been
 246  * added into yet, the order returned will be 0.
 247  */
 248 static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
 249 {
 250         return folioq->orders[slot];
 251 }
 252
 253 /**
 254  * folioq_folio_size: Get the size of a folio from a folio queue segment
 255  * @folioq: The segment to access
 256  * @slot: The folio slot to access
 257  *
 258  * Retrieve the size of the folio in the specified slot from a folio queue
 259  * segment.  Note that no bounds check is made and if the slot hasn't been
 260  * added into yet, the size returned will be PAGE_SIZE.
 261  */
 262 static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
 263 {
 264         return PAGE_SIZE << folioq_folio_order(folioq, slot);
 265 }
 266
 267 /**
 268  * folioq_clear: Clear a folio from a folio queue segment
 269  * @folioq: The segment to clear
 270  * @slot: The folio slot to clear
 271  *
 272  * Clear a folio from a sequence in a folio queue segment and clear its marks.
 273  * The occupancy count is left unchanged.
 274  */
 275 static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
 276 {
 277         folioq->vec.folios[slot] = NULL;
 278         folioq_unmark(folioq, slot);
 279         folioq_unmark2(folioq, slot);
 280 }
 281
 282 #endif /* _LINUX_FOLIO_QUEUE_H */