Branch data Line data Source code
1 : : /* SPDX-License-Identifier: GPL-2.0-or-later */
2 : : /*
3 : : * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4 : : */
5 : : #ifndef LINUX_DMAENGINE_H
6 : : #define LINUX_DMAENGINE_H
7 : :
8 : : #include <linux/device.h>
9 : : #include <linux/err.h>
10 : : #include <linux/uio.h>
11 : : #include <linux/bug.h>
12 : : #include <linux/scatterlist.h>
13 : : #include <linux/bitmap.h>
14 : : #include <linux/types.h>
15 : : #include <asm/page.h>
16 : :
17 : : /**
18 : : * typedef dma_cookie_t - an opaque DMA cookie
19 : : *
20 : : * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
21 : : */
22 : : typedef s32 dma_cookie_t;
23 : : #define DMA_MIN_COOKIE 1
24 : :
25 : : static inline int dma_submit_error(dma_cookie_t cookie)
26 : : {
27 : : return cookie < 0 ? cookie : 0;
28 : : }
29 : :
30 : : /**
31 : : * enum dma_status - DMA transaction status
32 : : * @DMA_COMPLETE: transaction completed
33 : : * @DMA_IN_PROGRESS: transaction not yet processed
34 : : * @DMA_PAUSED: transaction is paused
35 : : * @DMA_ERROR: transaction failed
36 : : */
37 : : enum dma_status {
38 : : DMA_COMPLETE,
39 : : DMA_IN_PROGRESS,
40 : : DMA_PAUSED,
41 : : DMA_ERROR,
42 : : };
43 : :
44 : : /**
45 : : * enum dma_transaction_type - DMA transaction types/indexes
46 : : *
47 : : * Note: The DMA_ASYNC_TX capability is not to be set by drivers. It is
48 : : * automatically set as dma devices are registered.
49 : : */
50 : : enum dma_transaction_type {
51 : : DMA_MEMCPY,
52 : : DMA_XOR,
53 : : DMA_PQ,
54 : : DMA_XOR_VAL,
55 : : DMA_PQ_VAL,
56 : : DMA_MEMSET,
57 : : DMA_MEMSET_SG,
58 : : DMA_INTERRUPT,
59 : : DMA_PRIVATE,
60 : : DMA_ASYNC_TX,
61 : : DMA_SLAVE,
62 : : DMA_CYCLIC,
63 : : DMA_INTERLEAVE,
64 : : /* last transaction type for creation of the capabilities mask */
65 : : DMA_TX_TYPE_END,
66 : : };
67 : :
68 : : /**
69 : : * enum dma_transfer_direction - dma transfer mode and direction indicator
70 : : * @DMA_MEM_TO_MEM: Async/Memcpy mode
71 : : * @DMA_MEM_TO_DEV: Slave mode & From Memory to Device
72 : : * @DMA_DEV_TO_MEM: Slave mode & From Device to Memory
73 : : * @DMA_DEV_TO_DEV: Slave mode & From Device to Device
74 : : */
75 : : enum dma_transfer_direction {
76 : : DMA_MEM_TO_MEM,
77 : : DMA_MEM_TO_DEV,
78 : : DMA_DEV_TO_MEM,
79 : : DMA_DEV_TO_DEV,
80 : : DMA_TRANS_NONE,
81 : : };
82 : :
83 : : /**
84 : : * Interleaved Transfer Request
85 : : * ----------------------------
86 : : * A chunk is collection of contiguous bytes to be transfered.
87 : : * The gap(in bytes) between two chunks is called inter-chunk-gap(ICG).
88 : : * ICGs may or maynot change between chunks.
89 : : * A FRAME is the smallest series of contiguous {chunk,icg} pairs,
90 : : * that when repeated an integral number of times, specifies the transfer.
91 : : * A transfer template is specification of a Frame, the number of times
92 : : * it is to be repeated and other per-transfer attributes.
93 : : *
94 : : * Practically, a client driver would have ready a template for each
95 : : * type of transfer it is going to need during its lifetime and
96 : : * set only 'src_start' and 'dst_start' before submitting the requests.
97 : : *
98 : : *
99 : : * | Frame-1 | Frame-2 | ~ | Frame-'numf' |
100 : : * |====....==.===...=...|====....==.===...=...| ~ |====....==.===...=...|
101 : : *
102 : : * == Chunk size
103 : : * ... ICG
104 : : */
105 : :
106 : : /**
107 : : * struct data_chunk - Element of scatter-gather list that makes a frame.
108 : : * @size: Number of bytes to read from source.
109 : : * size_dst := fn(op, size_src), so doesn't mean much for destination.
110 : : * @icg: Number of bytes to jump after last src/dst address of this
111 : : * chunk and before first src/dst address for next chunk.
112 : : * Ignored for dst(assumed 0), if dst_inc is true and dst_sgl is false.
113 : : * Ignored for src(assumed 0), if src_inc is true and src_sgl is false.
114 : : * @dst_icg: Number of bytes to jump after last dst address of this
115 : : * chunk and before the first dst address for next chunk.
116 : : * Ignored if dst_inc is true and dst_sgl is false.
117 : : * @src_icg: Number of bytes to jump after last src address of this
118 : : * chunk and before the first src address for next chunk.
119 : : * Ignored if src_inc is true and src_sgl is false.
120 : : */
121 : : struct data_chunk {
122 : : size_t size;
123 : : size_t icg;
124 : : size_t dst_icg;
125 : : size_t src_icg;
126 : : };
127 : :
128 : : /**
129 : : * struct dma_interleaved_template - Template to convey DMAC the transfer pattern
130 : : * and attributes.
131 : : * @src_start: Bus address of source for the first chunk.
132 : : * @dst_start: Bus address of destination for the first chunk.
133 : : * @dir: Specifies the type of Source and Destination.
134 : : * @src_inc: If the source address increments after reading from it.
135 : : * @dst_inc: If the destination address increments after writing to it.
136 : : * @src_sgl: If the 'icg' of sgl[] applies to Source (scattered read).
137 : : * Otherwise, source is read contiguously (icg ignored).
138 : : * Ignored if src_inc is false.
139 : : * @dst_sgl: If the 'icg' of sgl[] applies to Destination (scattered write).
140 : : * Otherwise, destination is filled contiguously (icg ignored).
141 : : * Ignored if dst_inc is false.
142 : : * @numf: Number of frames in this template.
143 : : * @frame_size: Number of chunks in a frame i.e, size of sgl[].
144 : : * @sgl: Array of {chunk,icg} pairs that make up a frame.
145 : : */
146 : : struct dma_interleaved_template {
147 : : dma_addr_t src_start;
148 : : dma_addr_t dst_start;
149 : : enum dma_transfer_direction dir;
150 : : bool src_inc;
151 : : bool dst_inc;
152 : : bool src_sgl;
153 : : bool dst_sgl;
154 : : size_t numf;
155 : : size_t frame_size;
156 : : struct data_chunk sgl[0];
157 : : };
158 : :
159 : : /**
160 : : * enum dma_ctrl_flags - DMA flags to augment operation preparation,
161 : : * control completion, and communicate status.
162 : : * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
163 : : * this transaction
164 : : * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
165 : : * acknowledges receipt, i.e. has has a chance to establish any dependency
166 : : * chains
167 : : * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q
168 : : * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P
169 : : * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as
170 : : * sources that were the result of a previous operation, in the case of a PQ
171 : : * operation it continues the calculation with new sources
172 : : * @DMA_PREP_FENCE - tell the driver that subsequent operations depend
173 : : * on the result of this operation
174 : : * @DMA_CTRL_REUSE: client can reuse the descriptor and submit again till
175 : : * cleared or freed
176 : : * @DMA_PREP_CMD: tell the driver that the data passed to DMA API is command
177 : : * data and the descriptor should be in different format from normal
178 : : * data descriptors.
179 : : */
180 : : enum dma_ctrl_flags {
181 : : DMA_PREP_INTERRUPT = (1 << 0),
182 : : DMA_CTRL_ACK = (1 << 1),
183 : : DMA_PREP_PQ_DISABLE_P = (1 << 2),
184 : : DMA_PREP_PQ_DISABLE_Q = (1 << 3),
185 : : DMA_PREP_CONTINUE = (1 << 4),
186 : : DMA_PREP_FENCE = (1 << 5),
187 : : DMA_CTRL_REUSE = (1 << 6),
188 : : DMA_PREP_CMD = (1 << 7),
189 : : };
190 : :
191 : : /**
192 : : * enum sum_check_bits - bit position of pq_check_flags
193 : : */
194 : : enum sum_check_bits {
195 : : SUM_CHECK_P = 0,
196 : : SUM_CHECK_Q = 1,
197 : : };
198 : :
199 : : /**
200 : : * enum pq_check_flags - result of async_{xor,pq}_zero_sum operations
201 : : * @SUM_CHECK_P_RESULT - 1 if xor zero sum error, 0 otherwise
202 : : * @SUM_CHECK_Q_RESULT - 1 if reed-solomon zero sum error, 0 otherwise
203 : : */
204 : : enum sum_check_flags {
205 : : SUM_CHECK_P_RESULT = (1 << SUM_CHECK_P),
206 : : SUM_CHECK_Q_RESULT = (1 << SUM_CHECK_Q),
207 : : };
208 : :
209 : :
210 : : /**
211 : : * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
212 : : * See linux/cpumask.h
213 : : */
214 : : typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
215 : :
216 : : /**
217 : : * struct dma_chan_percpu - the per-CPU part of struct dma_chan
218 : : * @memcpy_count: transaction counter
219 : : * @bytes_transferred: byte counter
220 : : */
221 : :
222 : : /**
223 : : * enum dma_desc_metadata_mode - per descriptor metadata mode types supported
224 : : * @DESC_METADATA_CLIENT - the metadata buffer is allocated/provided by the
225 : : * client driver and it is attached (via the dmaengine_desc_attach_metadata()
226 : : * helper) to the descriptor.
227 : : *
228 : : * Client drivers interested to use this mode can follow:
229 : : * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
230 : : * 1. prepare the descriptor (dmaengine_prep_*)
231 : : * construct the metadata in the client's buffer
232 : : * 2. use dmaengine_desc_attach_metadata() to attach the buffer to the
233 : : * descriptor
234 : : * 3. submit the transfer
235 : : * - DMA_DEV_TO_MEM:
236 : : * 1. prepare the descriptor (dmaengine_prep_*)
237 : : * 2. use dmaengine_desc_attach_metadata() to attach the buffer to the
238 : : * descriptor
239 : : * 3. submit the transfer
240 : : * 4. when the transfer is completed, the metadata should be available in the
241 : : * attached buffer
242 : : *
243 : : * @DESC_METADATA_ENGINE - the metadata buffer is allocated/managed by the DMA
244 : : * driver. The client driver can ask for the pointer, maximum size and the
245 : : * currently used size of the metadata and can directly update or read it.
246 : : * dmaengine_desc_get_metadata_ptr() and dmaengine_desc_set_metadata_len() is
247 : : * provided as helper functions.
248 : : *
249 : : * Note: the metadata area for the descriptor is no longer valid after the
250 : : * transfer has been completed (valid up to the point when the completion
251 : : * callback returns if used).
252 : : *
253 : : * Client drivers interested to use this mode can follow:
254 : : * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
255 : : * 1. prepare the descriptor (dmaengine_prep_*)
256 : : * 2. use dmaengine_desc_get_metadata_ptr() to get the pointer to the engine's
257 : : * metadata area
258 : : * 3. update the metadata at the pointer
259 : : * 4. use dmaengine_desc_set_metadata_len() to tell the DMA engine the amount
260 : : * of data the client has placed into the metadata buffer
261 : : * 5. submit the transfer
262 : : * - DMA_DEV_TO_MEM:
263 : : * 1. prepare the descriptor (dmaengine_prep_*)
264 : : * 2. submit the transfer
265 : : * 3. on transfer completion, use dmaengine_desc_get_metadata_ptr() to get the
266 : : * pointer to the engine's metadata area
267 : : * 4. Read out the metadata from the pointer
268 : : *
269 : : * Note: the two mode is not compatible and clients must use one mode for a
270 : : * descriptor.
271 : : */
272 : : enum dma_desc_metadata_mode {
273 : : DESC_METADATA_NONE = 0,
274 : : DESC_METADATA_CLIENT = BIT(0),
275 : : DESC_METADATA_ENGINE = BIT(1),
276 : : };
277 : :
278 : : struct dma_chan_percpu {
279 : : /* stats */
280 : : unsigned long memcpy_count;
281 : : unsigned long bytes_transferred;
282 : : };
283 : :
284 : : /**
285 : : * struct dma_router - DMA router structure
286 : : * @dev: pointer to the DMA router device
287 : : * @route_free: function to be called when the route can be disconnected
288 : : */
289 : : struct dma_router {
290 : : struct device *dev;
291 : : void (*route_free)(struct device *dev, void *route_data);
292 : : };
293 : :
294 : : /**
295 : : * struct dma_chan - devices supply DMA channels, clients use them
296 : : * @device: ptr to the dma device who supplies this channel, always !%NULL
297 : : * @slave: ptr to the device using this channel
298 : : * @cookie: last cookie value returned to client
299 : : * @completed_cookie: last completed cookie for this channel
300 : : * @chan_id: channel ID for sysfs
301 : : * @dev: class device for sysfs
302 : : * @name: backlink name for sysfs
303 : : * @device_node: used to add this to the device chan list
304 : : * @local: per-cpu pointer to a struct dma_chan_percpu
305 : : * @client_count: how many clients are using this channel
306 : : * @table_count: number of appearances in the mem-to-mem allocation table
307 : : * @router: pointer to the DMA router structure
308 : : * @route_data: channel specific data for the router
309 : : * @private: private data for certain client-channel associations
310 : : */
311 : : struct dma_chan {
312 : : struct dma_device *device;
313 : : struct device *slave;
314 : : dma_cookie_t cookie;
315 : : dma_cookie_t completed_cookie;
316 : :
317 : : /* sysfs */
318 : : int chan_id;
319 : : struct dma_chan_dev *dev;
320 : : const char *name;
321 : :
322 : : struct list_head device_node;
323 : : struct dma_chan_percpu __percpu *local;
324 : : int client_count;
325 : : int table_count;
326 : :
327 : : /* DMA router */
328 : : struct dma_router *router;
329 : : void *route_data;
330 : :
331 : : void *private;
332 : : };
333 : :
334 : : /**
335 : : * struct dma_chan_dev - relate sysfs device node to backing channel device
336 : : * @chan: driver channel device
337 : : * @device: sysfs device
338 : : * @dev_id: parent dma_device dev_id
339 : : * @idr_ref: reference count to gate release of dma_device dev_id
340 : : */
341 : : struct dma_chan_dev {
342 : : struct dma_chan *chan;
343 : : struct device device;
344 : : int dev_id;
345 : : atomic_t *idr_ref;
346 : : };
347 : :
348 : : /**
349 : : * enum dma_slave_buswidth - defines bus width of the DMA slave
350 : : * device, source or target buses
351 : : */
352 : : enum dma_slave_buswidth {
353 : : DMA_SLAVE_BUSWIDTH_UNDEFINED = 0,
354 : : DMA_SLAVE_BUSWIDTH_1_BYTE = 1,
355 : : DMA_SLAVE_BUSWIDTH_2_BYTES = 2,
356 : : DMA_SLAVE_BUSWIDTH_3_BYTES = 3,
357 : : DMA_SLAVE_BUSWIDTH_4_BYTES = 4,
358 : : DMA_SLAVE_BUSWIDTH_8_BYTES = 8,
359 : : DMA_SLAVE_BUSWIDTH_16_BYTES = 16,
360 : : DMA_SLAVE_BUSWIDTH_32_BYTES = 32,
361 : : DMA_SLAVE_BUSWIDTH_64_BYTES = 64,
362 : : };
363 : :
364 : : /**
365 : : * struct dma_slave_config - dma slave channel runtime config
366 : : * @direction: whether the data shall go in or out on this slave
367 : : * channel, right now. DMA_MEM_TO_DEV and DMA_DEV_TO_MEM are
368 : : * legal values. DEPRECATED, drivers should use the direction argument
369 : : * to the device_prep_slave_sg and device_prep_dma_cyclic functions or
370 : : * the dir field in the dma_interleaved_template structure.
371 : : * @src_addr: this is the physical address where DMA slave data
372 : : * should be read (RX), if the source is memory this argument is
373 : : * ignored.
374 : : * @dst_addr: this is the physical address where DMA slave data
375 : : * should be written (TX), if the source is memory this argument
376 : : * is ignored.
377 : : * @src_addr_width: this is the width in bytes of the source (RX)
378 : : * register where DMA data shall be read. If the source
379 : : * is memory this may be ignored depending on architecture.
380 : : * Legal values: 1, 2, 3, 4, 8, 16, 32, 64.
381 : : * @dst_addr_width: same as src_addr_width but for destination
382 : : * target (TX) mutatis mutandis.
383 : : * @src_maxburst: the maximum number of words (note: words, as in
384 : : * units of the src_addr_width member, not bytes) that can be sent
385 : : * in one burst to the device. Typically something like half the
386 : : * FIFO depth on I/O peripherals so you don't overflow it. This
387 : : * may or may not be applicable on memory sources.
388 : : * @dst_maxburst: same as src_maxburst but for destination target
389 : : * mutatis mutandis.
390 : : * @src_port_window_size: The length of the register area in words the data need
391 : : * to be accessed on the device side. It is only used for devices which is using
392 : : * an area instead of a single register to receive the data. Typically the DMA
393 : : * loops in this area in order to transfer the data.
394 : : * @dst_port_window_size: same as src_port_window_size but for the destination
395 : : * port.
396 : : * @device_fc: Flow Controller Settings. Only valid for slave channels. Fill
397 : : * with 'true' if peripheral should be flow controller. Direction will be
398 : : * selected at Runtime.
399 : : * @slave_id: Slave requester id. Only valid for slave channels. The dma
400 : : * slave peripheral will have unique id as dma requester which need to be
401 : : * pass as slave config.
402 : : *
403 : : * This struct is passed in as configuration data to a DMA engine
404 : : * in order to set up a certain channel for DMA transport at runtime.
405 : : * The DMA device/engine has to provide support for an additional
406 : : * callback in the dma_device structure, device_config and this struct
407 : : * will then be passed in as an argument to the function.
408 : : *
409 : : * The rationale for adding configuration information to this struct is as
410 : : * follows: if it is likely that more than one DMA slave controllers in
411 : : * the world will support the configuration option, then make it generic.
412 : : * If not: if it is fixed so that it be sent in static from the platform
413 : : * data, then prefer to do that.
414 : : */
415 : : struct dma_slave_config {
416 : : enum dma_transfer_direction direction;
417 : : phys_addr_t src_addr;
418 : : phys_addr_t dst_addr;
419 : : enum dma_slave_buswidth src_addr_width;
420 : : enum dma_slave_buswidth dst_addr_width;
421 : : u32 src_maxburst;
422 : : u32 dst_maxburst;
423 : : u32 src_port_window_size;
424 : : u32 dst_port_window_size;
425 : : bool device_fc;
426 : : unsigned int slave_id;
427 : : };
428 : :
429 : : /**
430 : : * enum dma_residue_granularity - Granularity of the reported transfer residue
431 : : * @DMA_RESIDUE_GRANULARITY_DESCRIPTOR: Residue reporting is not support. The
432 : : * DMA channel is only able to tell whether a descriptor has been completed or
433 : : * not, which means residue reporting is not supported by this channel. The
434 : : * residue field of the dma_tx_state field will always be 0.
435 : : * @DMA_RESIDUE_GRANULARITY_SEGMENT: Residue is updated after each successfully
436 : : * completed segment of the transfer (For cyclic transfers this is after each
437 : : * period). This is typically implemented by having the hardware generate an
438 : : * interrupt after each transferred segment and then the drivers updates the
439 : : * outstanding residue by the size of the segment. Another possibility is if
440 : : * the hardware supports scatter-gather and the segment descriptor has a field
441 : : * which gets set after the segment has been completed. The driver then counts
442 : : * the number of segments without the flag set to compute the residue.
443 : : * @DMA_RESIDUE_GRANULARITY_BURST: Residue is updated after each transferred
444 : : * burst. This is typically only supported if the hardware has a progress
445 : : * register of some sort (E.g. a register with the current read/write address
446 : : * or a register with the amount of bursts/beats/bytes that have been
447 : : * transferred or still need to be transferred).
448 : : */
449 : : enum dma_residue_granularity {
450 : : DMA_RESIDUE_GRANULARITY_DESCRIPTOR = 0,
451 : : DMA_RESIDUE_GRANULARITY_SEGMENT = 1,
452 : : DMA_RESIDUE_GRANULARITY_BURST = 2,
453 : : };
454 : :
455 : : /**
456 : : * struct dma_slave_caps - expose capabilities of a slave channel only
457 : : * @src_addr_widths: bit mask of src addr widths the channel supports.
458 : : * Width is specified in bytes, e.g. for a channel supporting
459 : : * a width of 4 the mask should have BIT(4) set.
460 : : * @dst_addr_widths: bit mask of dst addr widths the channel supports
461 : : * @directions: bit mask of slave directions the channel supports.
462 : : * Since the enum dma_transfer_direction is not defined as bit flag for
463 : : * each type, the dma controller should set BIT(<TYPE>) and same
464 : : * should be checked by controller as well
465 : : * @max_burst: max burst capability per-transfer
466 : : * @cmd_pause: true, if pause is supported (i.e. for reading residue or
467 : : * for resume later)
468 : : * @cmd_resume: true, if resume is supported
469 : : * @cmd_terminate: true, if terminate cmd is supported
470 : : * @residue_granularity: granularity of the reported transfer residue
471 : : * @descriptor_reuse: if a descriptor can be reused by client and
472 : : * resubmitted multiple times
473 : : */
474 : : struct dma_slave_caps {
475 : : u32 src_addr_widths;
476 : : u32 dst_addr_widths;
477 : : u32 directions;
478 : : u32 max_burst;
479 : : bool cmd_pause;
480 : : bool cmd_resume;
481 : : bool cmd_terminate;
482 : : enum dma_residue_granularity residue_granularity;
483 : : bool descriptor_reuse;
484 : : };
485 : :
486 : : static inline const char *dma_chan_name(struct dma_chan *chan)
487 : : {
488 : : return dev_name(&chan->dev->device);
489 : : }
490 : :
491 : : void dma_chan_cleanup(struct kref *kref);
492 : :
493 : : /**
494 : : * typedef dma_filter_fn - callback filter for dma_request_channel
495 : : * @chan: channel to be reviewed
496 : : * @filter_param: opaque parameter passed through dma_request_channel
497 : : *
498 : : * When this optional parameter is specified in a call to dma_request_channel a
499 : : * suitable channel is passed to this routine for further dispositioning before
500 : : * being returned. Where 'suitable' indicates a non-busy channel that
501 : : * satisfies the given capability mask. It returns 'true' to indicate that the
502 : : * channel is suitable.
503 : : */
504 : : typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
505 : :
506 : : typedef void (*dma_async_tx_callback)(void *dma_async_param);
507 : :
508 : : enum dmaengine_tx_result {
509 : : DMA_TRANS_NOERROR = 0, /* SUCCESS */
510 : : DMA_TRANS_READ_FAILED, /* Source DMA read failed */
511 : : DMA_TRANS_WRITE_FAILED, /* Destination DMA write failed */
512 : : DMA_TRANS_ABORTED, /* Op never submitted / aborted */
513 : : };
514 : :
515 : : struct dmaengine_result {
516 : : enum dmaengine_tx_result result;
517 : : u32 residue;
518 : : };
519 : :
520 : : typedef void (*dma_async_tx_callback_result)(void *dma_async_param,
521 : : const struct dmaengine_result *result);
522 : :
523 : : struct dmaengine_unmap_data {
524 : : #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
525 : : u16 map_cnt;
526 : : #else
527 : : u8 map_cnt;
528 : : #endif
529 : : u8 to_cnt;
530 : : u8 from_cnt;
531 : : u8 bidi_cnt;
532 : : struct device *dev;
533 : : struct kref kref;
534 : : size_t len;
535 : : dma_addr_t addr[0];
536 : : };
537 : :
538 : : struct dma_async_tx_descriptor;
539 : :
540 : : struct dma_descriptor_metadata_ops {
541 : : int (*attach)(struct dma_async_tx_descriptor *desc, void *data,
542 : : size_t len);
543 : :
544 : : void *(*get_ptr)(struct dma_async_tx_descriptor *desc,
545 : : size_t *payload_len, size_t *max_len);
546 : : int (*set_len)(struct dma_async_tx_descriptor *desc,
547 : : size_t payload_len);
548 : : };
549 : :
550 : : /**
551 : : * struct dma_async_tx_descriptor - async transaction descriptor
552 : : * ---dma generic offload fields---
553 : : * @cookie: tracking cookie for this transaction, set to -EBUSY if
554 : : * this tx is sitting on a dependency list
555 : : * @flags: flags to augment operation preparation, control completion, and
556 : : * communicate status
557 : : * @phys: physical address of the descriptor
558 : : * @chan: target channel for this operation
559 : : * @tx_submit: accept the descriptor, assign ordered cookie and mark the
560 : : * descriptor pending. To be pushed on .issue_pending() call
561 : : * @callback: routine to call after this operation is complete
562 : : * @callback_param: general parameter to pass to the callback routine
563 : : * @desc_metadata_mode: core managed metadata mode to protect mixed use of
564 : : * DESC_METADATA_CLIENT or DESC_METADATA_ENGINE. Otherwise
565 : : * DESC_METADATA_NONE
566 : : * @metadata_ops: DMA driver provided metadata mode ops, need to be set by the
567 : : * DMA driver if metadata mode is supported with the descriptor
568 : : * ---async_tx api specific fields---
569 : : * @next: at completion submit this descriptor
570 : : * @parent: pointer to the next level up in the dependency chain
571 : : * @lock: protect the parent and next pointers
572 : : */
573 : : struct dma_async_tx_descriptor {
574 : : dma_cookie_t cookie;
575 : : enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
576 : : dma_addr_t phys;
577 : : struct dma_chan *chan;
578 : : dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
579 : : int (*desc_free)(struct dma_async_tx_descriptor *tx);
580 : : dma_async_tx_callback callback;
581 : : dma_async_tx_callback_result callback_result;
582 : : void *callback_param;
583 : : struct dmaengine_unmap_data *unmap;
584 : : enum dma_desc_metadata_mode desc_metadata_mode;
585 : : struct dma_descriptor_metadata_ops *metadata_ops;
586 : : #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
587 : : struct dma_async_tx_descriptor *next;
588 : : struct dma_async_tx_descriptor *parent;
589 : : spinlock_t lock;
590 : : #endif
591 : : };
592 : :
593 : : #ifdef CONFIG_DMA_ENGINE
594 : : static inline void dma_set_unmap(struct dma_async_tx_descriptor *tx,
595 : : struct dmaengine_unmap_data *unmap)
596 : : {
597 : : kref_get(&unmap->kref);
598 : : tx->unmap = unmap;
599 : : }
600 : :
601 : : struct dmaengine_unmap_data *
602 : : dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags);
603 : : void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap);
604 : : #else
605 : : static inline void dma_set_unmap(struct dma_async_tx_descriptor *tx,
606 : : struct dmaengine_unmap_data *unmap)
607 : : {
608 : : }
609 : : static inline struct dmaengine_unmap_data *
610 : : dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
611 : : {
612 : : return NULL;
613 : : }
614 : : static inline void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
615 : : {
616 : : }
617 : : #endif
618 : :
619 : : static inline void dma_descriptor_unmap(struct dma_async_tx_descriptor *tx)
620 : : {
621 : : if (tx->unmap) {
622 : : dmaengine_unmap_put(tx->unmap);
623 : : tx->unmap = NULL;
624 : : }
625 : : }
626 : :
627 : : #ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
628 : : static inline void txd_lock(struct dma_async_tx_descriptor *txd)
629 : : {
630 : : }
631 : : static inline void txd_unlock(struct dma_async_tx_descriptor *txd)
632 : : {
633 : : }
634 : : static inline void txd_chain(struct dma_async_tx_descriptor *txd, struct dma_async_tx_descriptor *next)
635 : : {
636 : : BUG();
637 : : }
638 : : static inline void txd_clear_parent(struct dma_async_tx_descriptor *txd)
639 : : {
640 : : }
641 : : static inline void txd_clear_next(struct dma_async_tx_descriptor *txd)
642 : : {
643 : : }
644 : 0 : static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descriptor *txd)
645 : : {
646 : 0 : return NULL;
647 : : }
648 : : static inline struct dma_async_tx_descriptor *txd_parent(struct dma_async_tx_descriptor *txd)
649 : : {
650 : : return NULL;
651 : : }
652 : :
653 : : #else
654 : : static inline void txd_lock(struct dma_async_tx_descriptor *txd)
655 : : {
656 : : spin_lock_bh(&txd->lock);
657 : : }
658 : : static inline void txd_unlock(struct dma_async_tx_descriptor *txd)
659 : : {
660 : : spin_unlock_bh(&txd->lock);
661 : : }
662 : : static inline void txd_chain(struct dma_async_tx_descriptor *txd, struct dma_async_tx_descriptor *next)
663 : : {
664 : : txd->next = next;
665 : : next->parent = txd;
666 : : }
667 : : static inline void txd_clear_parent(struct dma_async_tx_descriptor *txd)
668 : : {
669 : : txd->parent = NULL;
670 : : }
671 : : static inline void txd_clear_next(struct dma_async_tx_descriptor *txd)
672 : : {
673 : : txd->next = NULL;
674 : : }
675 : : static inline struct dma_async_tx_descriptor *txd_parent(struct dma_async_tx_descriptor *txd)
676 : : {
677 : : return txd->parent;
678 : : }
679 : : static inline struct dma_async_tx_descriptor *txd_next(struct dma_async_tx_descriptor *txd)
680 : : {
681 : : return txd->next;
682 : : }
683 : : #endif
684 : :
685 : : /**
686 : : * struct dma_tx_state - filled in to report the status of
687 : : * a transfer.
688 : : * @last: last completed DMA cookie
689 : : * @used: last issued DMA cookie (i.e. the one in progress)
690 : : * @residue: the remaining number of bytes left to transmit
691 : : * on the selected transfer for states DMA_IN_PROGRESS and
692 : : * DMA_PAUSED if this is implemented in the driver, else 0
693 : : * @in_flight_bytes: amount of data in bytes cached by the DMA.
694 : : */
695 : : struct dma_tx_state {
696 : : dma_cookie_t last;
697 : : dma_cookie_t used;
698 : : u32 residue;
699 : : u32 in_flight_bytes;
700 : : };
701 : :
702 : : /**
703 : : * enum dmaengine_alignment - defines alignment of the DMA async tx
704 : : * buffers
705 : : */
706 : : enum dmaengine_alignment {
707 : : DMAENGINE_ALIGN_1_BYTE = 0,
708 : : DMAENGINE_ALIGN_2_BYTES = 1,
709 : : DMAENGINE_ALIGN_4_BYTES = 2,
710 : : DMAENGINE_ALIGN_8_BYTES = 3,
711 : : DMAENGINE_ALIGN_16_BYTES = 4,
712 : : DMAENGINE_ALIGN_32_BYTES = 5,
713 : : DMAENGINE_ALIGN_64_BYTES = 6,
714 : : };
715 : :
716 : : /**
717 : : * struct dma_slave_map - associates slave device and it's slave channel with
718 : : * parameter to be used by a filter function
719 : : * @devname: name of the device
720 : : * @slave: slave channel name
721 : : * @param: opaque parameter to pass to struct dma_filter.fn
722 : : */
723 : : struct dma_slave_map {
724 : : const char *devname;
725 : : const char *slave;
726 : : void *param;
727 : : };
728 : :
729 : : /**
730 : : * struct dma_filter - information for slave device/channel to filter_fn/param
731 : : * mapping
732 : : * @fn: filter function callback
733 : : * @mapcnt: number of slave device/channel in the map
734 : : * @map: array of channel to filter mapping data
735 : : */
736 : : struct dma_filter {
737 : : dma_filter_fn fn;
738 : : int mapcnt;
739 : : const struct dma_slave_map *map;
740 : : };
741 : :
742 : : /**
743 : : * struct dma_device - info on the entity supplying DMA services
744 : : * @chancnt: how many DMA channels are supported
745 : : * @privatecnt: how many DMA channels are requested by dma_request_channel
746 : : * @channels: the list of struct dma_chan
747 : : * @global_node: list_head for global dma_device_list
748 : : * @filter: information for device/slave to filter function/param mapping
749 : : * @cap_mask: one or more dma_capability flags
750 : : * @desc_metadata_modes: supported metadata modes by the DMA device
751 : : * @max_xor: maximum number of xor sources, 0 if no capability
752 : : * @max_pq: maximum number of PQ sources and PQ-continue capability
753 : : * @copy_align: alignment shift for memcpy operations
754 : : * @xor_align: alignment shift for xor operations
755 : : * @pq_align: alignment shift for pq operations
756 : : * @fill_align: alignment shift for memset operations
757 : : * @dev_id: unique device ID
758 : : * @dev: struct device reference for dma mapping api
759 : : * @owner: owner module (automatically set based on the provided dev)
760 : : * @src_addr_widths: bit mask of src addr widths the device supports
761 : : * Width is specified in bytes, e.g. for a device supporting
762 : : * a width of 4 the mask should have BIT(4) set.
763 : : * @dst_addr_widths: bit mask of dst addr widths the device supports
764 : : * @directions: bit mask of slave directions the device supports.
765 : : * Since the enum dma_transfer_direction is not defined as bit flag for
766 : : * each type, the dma controller should set BIT(<TYPE>) and same
767 : : * should be checked by controller as well
768 : : * @max_burst: max burst capability per-transfer
769 : : * @residue_granularity: granularity of the transfer residue reported
770 : : * by tx_status
771 : : * @device_alloc_chan_resources: allocate resources and return the
772 : : * number of allocated descriptors
773 : : * @device_free_chan_resources: release DMA channel's resources
774 : : * @device_prep_dma_memcpy: prepares a memcpy operation
775 : : * @device_prep_dma_xor: prepares a xor operation
776 : : * @device_prep_dma_xor_val: prepares a xor validation operation
777 : : * @device_prep_dma_pq: prepares a pq operation
778 : : * @device_prep_dma_pq_val: prepares a pqzero_sum operation
779 : : * @device_prep_dma_memset: prepares a memset operation
780 : : * @device_prep_dma_memset_sg: prepares a memset operation over a scatter list
781 : : * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
782 : : * @device_prep_slave_sg: prepares a slave dma operation
783 : : * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio.
784 : : * The function takes a buffer of size buf_len. The callback function will
785 : : * be called after period_len bytes have been transferred.
786 : : * @device_prep_interleaved_dma: Transfer expression in a generic way.
787 : : * @device_prep_dma_imm_data: DMA's 8 byte immediate data to the dst address
788 : : * @device_config: Pushes a new configuration to a channel, return 0 or an error
789 : : * code
790 : : * @device_pause: Pauses any transfer happening on a channel. Returns
791 : : * 0 or an error code
792 : : * @device_resume: Resumes any transfer on a channel previously
793 : : * paused. Returns 0 or an error code
794 : : * @device_terminate_all: Aborts all transfers on a channel. Returns 0
795 : : * or an error code
796 : : * @device_synchronize: Synchronizes the termination of a transfers to the
797 : : * current context.
798 : : * @device_tx_status: poll for transaction completion, the optional
799 : : * txstate parameter can be supplied with a pointer to get a
800 : : * struct with auxiliary transfer status information, otherwise the call
801 : : * will just return a simple status code
802 : : * @device_issue_pending: push pending transactions to hardware
803 : : * @descriptor_reuse: a submitted transfer can be resubmitted after completion
804 : : * @device_release: called sometime atfer dma_async_device_unregister() is
805 : : * called and there are no further references to this structure. This
806 : : * must be implemented to free resources however many existing drivers
807 : : * do not and are therefore not safe to unbind while in use.
808 : : *
809 : : */
810 : : struct dma_device {
811 : : struct kref ref;
812 : : unsigned int chancnt;
813 : : unsigned int privatecnt;
814 : : struct list_head channels;
815 : : struct list_head global_node;
816 : : struct dma_filter filter;
817 : : dma_cap_mask_t cap_mask;
818 : : enum dma_desc_metadata_mode desc_metadata_modes;
819 : : unsigned short max_xor;
820 : : unsigned short max_pq;
821 : : enum dmaengine_alignment copy_align;
822 : : enum dmaengine_alignment xor_align;
823 : : enum dmaengine_alignment pq_align;
824 : : enum dmaengine_alignment fill_align;
825 : : #define DMA_HAS_PQ_CONTINUE (1 << 15)
826 : :
827 : : int dev_id;
828 : : struct device *dev;
829 : : struct module *owner;
830 : :
831 : : u32 src_addr_widths;
832 : : u32 dst_addr_widths;
833 : : u32 directions;
834 : : u32 max_burst;
835 : : bool descriptor_reuse;
836 : : enum dma_residue_granularity residue_granularity;
837 : :
838 : : int (*device_alloc_chan_resources)(struct dma_chan *chan);
839 : : void (*device_free_chan_resources)(struct dma_chan *chan);
840 : :
841 : : struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
842 : : struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
843 : : size_t len, unsigned long flags);
844 : : struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
845 : : struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
846 : : unsigned int src_cnt, size_t len, unsigned long flags);
847 : : struct dma_async_tx_descriptor *(*device_prep_dma_xor_val)(
848 : : struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
849 : : size_t len, enum sum_check_flags *result, unsigned long flags);
850 : : struct dma_async_tx_descriptor *(*device_prep_dma_pq)(
851 : : struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
852 : : unsigned int src_cnt, const unsigned char *scf,
853 : : size_t len, unsigned long flags);
854 : : struct dma_async_tx_descriptor *(*device_prep_dma_pq_val)(
855 : : struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
856 : : unsigned int src_cnt, const unsigned char *scf, size_t len,
857 : : enum sum_check_flags *pqres, unsigned long flags);
858 : : struct dma_async_tx_descriptor *(*device_prep_dma_memset)(
859 : : struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
860 : : unsigned long flags);
861 : : struct dma_async_tx_descriptor *(*device_prep_dma_memset_sg)(
862 : : struct dma_chan *chan, struct scatterlist *sg,
863 : : unsigned int nents, int value, unsigned long flags);
864 : : struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
865 : : struct dma_chan *chan, unsigned long flags);
866 : :
867 : : struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
868 : : struct dma_chan *chan, struct scatterlist *sgl,
869 : : unsigned int sg_len, enum dma_transfer_direction direction,
870 : : unsigned long flags, void *context);
871 : : struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)(
872 : : struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
873 : : size_t period_len, enum dma_transfer_direction direction,
874 : : unsigned long flags);
875 : : struct dma_async_tx_descriptor *(*device_prep_interleaved_dma)(
876 : : struct dma_chan *chan, struct dma_interleaved_template *xt,
877 : : unsigned long flags);
878 : : struct dma_async_tx_descriptor *(*device_prep_dma_imm_data)(
879 : : struct dma_chan *chan, dma_addr_t dst, u64 data,
880 : : unsigned long flags);
881 : :
882 : : int (*device_config)(struct dma_chan *chan,
883 : : struct dma_slave_config *config);
884 : : int (*device_pause)(struct dma_chan *chan);
885 : : int (*device_resume)(struct dma_chan *chan);
886 : : int (*device_terminate_all)(struct dma_chan *chan);
887 : : void (*device_synchronize)(struct dma_chan *chan);
888 : :
889 : : enum dma_status (*device_tx_status)(struct dma_chan *chan,
890 : : dma_cookie_t cookie,
891 : : struct dma_tx_state *txstate);
892 : : void (*device_issue_pending)(struct dma_chan *chan);
893 : : void (*device_release)(struct dma_device *dev);
894 : : };
895 : :
896 : 0 : static inline int dmaengine_slave_config(struct dma_chan *chan,
897 : : struct dma_slave_config *config)
898 : : {
899 [ # # # # ]: 0 : if (chan->device->device_config)
900 : 0 : return chan->device->device_config(chan, config);
901 : :
902 : : return -ENOSYS;
903 : : }
904 : :
905 : 0 : static inline bool is_slave_direction(enum dma_transfer_direction direction)
906 : : {
907 [ # # ]: 0 : return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM);
908 : : }
909 : :
910 : 0 : static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
911 : : struct dma_chan *chan, dma_addr_t buf, size_t len,
912 : : enum dma_transfer_direction dir, unsigned long flags)
913 : : {
914 : 0 : struct scatterlist sg;
915 : 0 : sg_init_table(&sg, 1);
916 : 0 : sg_dma_address(&sg) = buf;
917 : 0 : sg_dma_len(&sg) = len;
918 : :
919 [ # # # # : 0 : if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
# # ]
920 : : return NULL;
921 : :
922 : 0 : return chan->device->device_prep_slave_sg(chan, &sg, 1,
923 : : dir, flags, NULL);
924 : : }
925 : :
926 : : static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
927 : : struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
928 : : enum dma_transfer_direction dir, unsigned long flags)
929 : : {
930 : : if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
931 : : return NULL;
932 : :
933 : : return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
934 : : dir, flags, NULL);
935 : : }
936 : :
937 : : #ifdef CONFIG_RAPIDIO_DMA_ENGINE
938 : : struct rio_dma_ext;
939 : : static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg(
940 : : struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
941 : : enum dma_transfer_direction dir, unsigned long flags,
942 : : struct rio_dma_ext *rio_ext)
943 : : {
944 : : if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
945 : : return NULL;
946 : :
947 : : return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
948 : : dir, flags, rio_ext);
949 : : }
950 : : #endif
951 : :
952 : : static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_cyclic(
953 : : struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
954 : : size_t period_len, enum dma_transfer_direction dir,
955 : : unsigned long flags)
956 : : {
957 : : if (!chan || !chan->device || !chan->device->device_prep_dma_cyclic)
958 : : return NULL;
959 : :
960 : : return chan->device->device_prep_dma_cyclic(chan, buf_addr, buf_len,
961 : : period_len, dir, flags);
962 : : }
963 : :
964 : : static inline struct dma_async_tx_descriptor *dmaengine_prep_interleaved_dma(
965 : : struct dma_chan *chan, struct dma_interleaved_template *xt,
966 : : unsigned long flags)
967 : : {
968 : : if (!chan || !chan->device || !chan->device->device_prep_interleaved_dma)
969 : : return NULL;
970 : :
971 : : return chan->device->device_prep_interleaved_dma(chan, xt, flags);
972 : : }
973 : :
974 : : static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memset(
975 : : struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
976 : : unsigned long flags)
977 : : {
978 : : if (!chan || !chan->device || !chan->device->device_prep_dma_memset)
979 : : return NULL;
980 : :
981 : : return chan->device->device_prep_dma_memset(chan, dest, value,
982 : : len, flags);
983 : : }
984 : :
985 : : static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy(
986 : : struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
987 : : size_t len, unsigned long flags)
988 : : {
989 : : if (!chan || !chan->device || !chan->device->device_prep_dma_memcpy)
990 : : return NULL;
991 : :
992 : : return chan->device->device_prep_dma_memcpy(chan, dest, src,
993 : : len, flags);
994 : : }
995 : :
996 : 0 : static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan,
997 : : enum dma_desc_metadata_mode mode)
998 : : {
999 [ # # # # : 0 : if (!chan)
# # ]
1000 : : return false;
1001 : :
1002 [ # # # # : 0 : return !!(chan->device->desc_metadata_modes & mode);
# # ]
1003 : : }
1004 : :
1005 : : #ifdef CONFIG_DMA_ENGINE
1006 : : int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
1007 : : void *data, size_t len);
1008 : : void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
1009 : : size_t *payload_len, size_t *max_len);
1010 : : int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
1011 : : size_t payload_len);
1012 : : #else /* CONFIG_DMA_ENGINE */
1013 : : static inline int dmaengine_desc_attach_metadata(
1014 : : struct dma_async_tx_descriptor *desc, void *data, size_t len)
1015 : : {
1016 : : return -EINVAL;
1017 : : }
1018 : : static inline void *dmaengine_desc_get_metadata_ptr(
1019 : : struct dma_async_tx_descriptor *desc, size_t *payload_len,
1020 : : size_t *max_len)
1021 : : {
1022 : : return NULL;
1023 : : }
1024 : : static inline int dmaengine_desc_set_metadata_len(
1025 : : struct dma_async_tx_descriptor *desc, size_t payload_len)
1026 : : {
1027 : : return -EINVAL;
1028 : : }
1029 : : #endif /* CONFIG_DMA_ENGINE */
1030 : :
1031 : : /**
1032 : : * dmaengine_terminate_all() - Terminate all active DMA transfers
1033 : : * @chan: The channel for which to terminate the transfers
1034 : : *
1035 : : * This function is DEPRECATED use either dmaengine_terminate_sync() or
1036 : : * dmaengine_terminate_async() instead.
1037 : : */
1038 : : static inline int dmaengine_terminate_all(struct dma_chan *chan)
1039 : : {
1040 : : if (chan->device->device_terminate_all)
1041 : : return chan->device->device_terminate_all(chan);
1042 : :
1043 : : return -ENOSYS;
1044 : : }
1045 : :
1046 : : /**
1047 : : * dmaengine_terminate_async() - Terminate all active DMA transfers
1048 : : * @chan: The channel for which to terminate the transfers
1049 : : *
1050 : : * Calling this function will terminate all active and pending descriptors
1051 : : * that have previously been submitted to the channel. It is not guaranteed
1052 : : * though that the transfer for the active descriptor has stopped when the
1053 : : * function returns. Furthermore it is possible the complete callback of a
1054 : : * submitted transfer is still running when this function returns.
1055 : : *
1056 : : * dmaengine_synchronize() needs to be called before it is safe to free
1057 : : * any memory that is accessed by previously submitted descriptors or before
1058 : : * freeing any resources accessed from within the completion callback of any
1059 : : * perviously submitted descriptors.
1060 : : *
1061 : : * This function can be called from atomic context as well as from within a
1062 : : * complete callback of a descriptor submitted on the same channel.
1063 : : *
1064 : : * If none of the two conditions above apply consider using
1065 : : * dmaengine_terminate_sync() instead.
1066 : : */
1067 : 0 : static inline int dmaengine_terminate_async(struct dma_chan *chan)
1068 : : {
1069 [ # # ]: 0 : if (chan->device->device_terminate_all)
1070 : 0 : return chan->device->device_terminate_all(chan);
1071 : :
1072 : : return -EINVAL;
1073 : : }
1074 : :
1075 : : /**
1076 : : * dmaengine_synchronize() - Synchronize DMA channel termination
1077 : : * @chan: The channel to synchronize
1078 : : *
1079 : : * Synchronizes to the DMA channel termination to the current context. When this
1080 : : * function returns it is guaranteed that all transfers for previously issued
1081 : : * descriptors have stopped and and it is safe to free the memory assoicated
1082 : : * with them. Furthermore it is guaranteed that all complete callback functions
1083 : : * for a previously submitted descriptor have finished running and it is safe to
1084 : : * free resources accessed from within the complete callbacks.
1085 : : *
1086 : : * The behavior of this function is undefined if dma_async_issue_pending() has
1087 : : * been called between dmaengine_terminate_async() and this function.
1088 : : *
1089 : : * This function must only be called from non-atomic context and must not be
1090 : : * called from within a complete callback of a descriptor submitted on the same
1091 : : * channel.
1092 : : */
1093 : 0 : static inline void dmaengine_synchronize(struct dma_chan *chan)
1094 : : {
1095 : 0 : might_sleep();
1096 : :
1097 [ # # ]: 0 : if (chan->device->device_synchronize)
1098 : 0 : chan->device->device_synchronize(chan);
1099 : : }
1100 : :
1101 : : /**
1102 : : * dmaengine_terminate_sync() - Terminate all active DMA transfers
1103 : : * @chan: The channel for which to terminate the transfers
1104 : : *
1105 : : * Calling this function will terminate all active and pending transfers
1106 : : * that have previously been submitted to the channel. It is similar to
1107 : : * dmaengine_terminate_async() but guarantees that the DMA transfer has actually
1108 : : * stopped and that all complete callbacks have finished running when the
1109 : : * function returns.
1110 : : *
1111 : : * This function must only be called from non-atomic context and must not be
1112 : : * called from within a complete callback of a descriptor submitted on the same
1113 : : * channel.
1114 : : */
1115 : 0 : static inline int dmaengine_terminate_sync(struct dma_chan *chan)
1116 : : {
1117 : 0 : int ret;
1118 : :
1119 [ # # ]: 0 : ret = dmaengine_terminate_async(chan);
1120 [ # # ]: 0 : if (ret)
1121 : 0 : return ret;
1122 : :
1123 : 0 : dmaengine_synchronize(chan);
1124 : :
1125 : : return 0;
1126 : : }
1127 : :
1128 : 0 : static inline int dmaengine_pause(struct dma_chan *chan)
1129 : : {
1130 [ # # ]: 0 : if (chan->device->device_pause)
1131 : 0 : return chan->device->device_pause(chan);
1132 : :
1133 : : return -ENOSYS;
1134 : : }
1135 : :
1136 : : static inline int dmaengine_resume(struct dma_chan *chan)
1137 : : {
1138 : : if (chan->device->device_resume)
1139 : : return chan->device->device_resume(chan);
1140 : :
1141 : : return -ENOSYS;
1142 : : }
1143 : :
1144 : 0 : static inline enum dma_status dmaengine_tx_status(struct dma_chan *chan,
1145 : : dma_cookie_t cookie, struct dma_tx_state *state)
1146 : : {
1147 : 0 : return chan->device->device_tx_status(chan, cookie, state);
1148 : : }
1149 : :
1150 : 0 : static inline dma_cookie_t dmaengine_submit(struct dma_async_tx_descriptor *desc)
1151 : : {
1152 : 0 : return desc->tx_submit(desc);
1153 : : }
1154 : :
1155 : : static inline bool dmaengine_check_align(enum dmaengine_alignment align,
1156 : : size_t off1, size_t off2, size_t len)
1157 : : {
1158 : : size_t mask;
1159 : :
1160 : : if (!align)
1161 : : return true;
1162 : : mask = (1 << align) - 1;
1163 : : if (mask & (off1 | off2 | len))
1164 : : return false;
1165 : : return true;
1166 : : }
1167 : :
1168 : : static inline bool is_dma_copy_aligned(struct dma_device *dev, size_t off1,
1169 : : size_t off2, size_t len)
1170 : : {
1171 : : return dmaengine_check_align(dev->copy_align, off1, off2, len);
1172 : : }
1173 : :
1174 : : static inline bool is_dma_xor_aligned(struct dma_device *dev, size_t off1,
1175 : : size_t off2, size_t len)
1176 : : {
1177 : : return dmaengine_check_align(dev->xor_align, off1, off2, len);
1178 : : }
1179 : :
1180 : : static inline bool is_dma_pq_aligned(struct dma_device *dev, size_t off1,
1181 : : size_t off2, size_t len)
1182 : : {
1183 : : return dmaengine_check_align(dev->pq_align, off1, off2, len);
1184 : : }
1185 : :
1186 : : static inline bool is_dma_fill_aligned(struct dma_device *dev, size_t off1,
1187 : : size_t off2, size_t len)
1188 : : {
1189 : : return dmaengine_check_align(dev->fill_align, off1, off2, len);
1190 : : }
1191 : :
1192 : : static inline void
1193 : : dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue)
1194 : : {
1195 : : dma->max_pq = maxpq;
1196 : : if (has_pq_continue)
1197 : : dma->max_pq |= DMA_HAS_PQ_CONTINUE;
1198 : : }
1199 : :
1200 : : static inline bool dmaf_continue(enum dma_ctrl_flags flags)
1201 : : {
1202 : : return (flags & DMA_PREP_CONTINUE) == DMA_PREP_CONTINUE;
1203 : : }
1204 : :
1205 : : static inline bool dmaf_p_disabled_continue(enum dma_ctrl_flags flags)
1206 : : {
1207 : : enum dma_ctrl_flags mask = DMA_PREP_CONTINUE | DMA_PREP_PQ_DISABLE_P;
1208 : :
1209 : : return (flags & mask) == mask;
1210 : : }
1211 : :
1212 : : static inline bool dma_dev_has_pq_continue(struct dma_device *dma)
1213 : : {
1214 : : return (dma->max_pq & DMA_HAS_PQ_CONTINUE) == DMA_HAS_PQ_CONTINUE;
1215 : : }
1216 : :
1217 : : static inline unsigned short dma_dev_to_maxpq(struct dma_device *dma)
1218 : : {
1219 : : return dma->max_pq & ~DMA_HAS_PQ_CONTINUE;
1220 : : }
1221 : :
1222 : : /* dma_maxpq - reduce maxpq in the face of continued operations
1223 : : * @dma - dma device with PQ capability
1224 : : * @flags - to check if DMA_PREP_CONTINUE and DMA_PREP_PQ_DISABLE_P are set
1225 : : *
1226 : : * When an engine does not support native continuation we need 3 extra
1227 : : * source slots to reuse P and Q with the following coefficients:
1228 : : * 1/ {00} * P : remove P from Q', but use it as a source for P'
1229 : : * 2/ {01} * Q : use Q to continue Q' calculation
1230 : : * 3/ {00} * Q : subtract Q from P' to cancel (2)
1231 : : *
1232 : : * In the case where P is disabled we only need 1 extra source:
1233 : : * 1/ {01} * Q : use Q to continue Q' calculation
1234 : : */
1235 : : static inline int dma_maxpq(struct dma_device *dma, enum dma_ctrl_flags flags)
1236 : : {
1237 : : if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags))
1238 : : return dma_dev_to_maxpq(dma);
1239 : : else if (dmaf_p_disabled_continue(flags))
1240 : : return dma_dev_to_maxpq(dma) - 1;
1241 : : else if (dmaf_continue(flags))
1242 : : return dma_dev_to_maxpq(dma) - 3;
1243 : : BUG();
1244 : : }
1245 : :
1246 : : static inline size_t dmaengine_get_icg(bool inc, bool sgl, size_t icg,
1247 : : size_t dir_icg)
1248 : : {
1249 : : if (inc) {
1250 : : if (dir_icg)
1251 : : return dir_icg;
1252 : : else if (sgl)
1253 : : return icg;
1254 : : }
1255 : :
1256 : : return 0;
1257 : : }
1258 : :
1259 : : static inline size_t dmaengine_get_dst_icg(struct dma_interleaved_template *xt,
1260 : : struct data_chunk *chunk)
1261 : : {
1262 : : return dmaengine_get_icg(xt->dst_inc, xt->dst_sgl,
1263 : : chunk->icg, chunk->dst_icg);
1264 : : }
1265 : :
1266 : : static inline size_t dmaengine_get_src_icg(struct dma_interleaved_template *xt,
1267 : : struct data_chunk *chunk)
1268 : : {
1269 : : return dmaengine_get_icg(xt->src_inc, xt->src_sgl,
1270 : : chunk->icg, chunk->src_icg);
1271 : : }
1272 : :
1273 : : /* --- public DMA engine API --- */
1274 : :
1275 : : #ifdef CONFIG_DMA_ENGINE
1276 : : void dmaengine_get(void);
1277 : : void dmaengine_put(void);
1278 : : #else
1279 : : static inline void dmaengine_get(void)
1280 : : {
1281 : : }
1282 : : static inline void dmaengine_put(void)
1283 : : {
1284 : : }
1285 : : #endif
1286 : :
1287 : : #ifdef CONFIG_ASYNC_TX_DMA
1288 : : #define async_dmaengine_get() dmaengine_get()
1289 : : #define async_dmaengine_put() dmaengine_put()
1290 : : #ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
1291 : : #define async_dma_find_channel(type) dma_find_channel(DMA_ASYNC_TX)
1292 : : #else
1293 : : #define async_dma_find_channel(type) dma_find_channel(type)
1294 : : #endif /* CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH */
1295 : : #else
1296 : : static inline void async_dmaengine_get(void)
1297 : : {
1298 : : }
1299 : : static inline void async_dmaengine_put(void)
1300 : : {
1301 : : }
1302 : : static inline struct dma_chan *
1303 : : async_dma_find_channel(enum dma_transaction_type type)
1304 : : {
1305 : : return NULL;
1306 : : }
1307 : : #endif /* CONFIG_ASYNC_TX_DMA */
1308 : : void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
1309 : : struct dma_chan *chan);
1310 : :
1311 : 0 : static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
1312 : : {
1313 : 0 : tx->flags |= DMA_CTRL_ACK;
1314 : : }
1315 : :
1316 : : static inline void async_tx_clear_ack(struct dma_async_tx_descriptor *tx)
1317 : : {
1318 : : tx->flags &= ~DMA_CTRL_ACK;
1319 : : }
1320 : :
1321 : : static inline bool async_tx_test_ack(struct dma_async_tx_descriptor *tx)
1322 : : {
1323 : : return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;
1324 : : }
1325 : :
1326 : : #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
1327 : : static inline void
1328 : 0 : __dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
1329 : : {
1330 : 0 : set_bit(tx_type, dstp->bits);
1331 : : }
1332 : :
1333 : : #define dma_cap_clear(tx, mask) __dma_cap_clear((tx), &(mask))
1334 : : static inline void
1335 : 0 : __dma_cap_clear(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
1336 : : {
1337 : 0 : clear_bit(tx_type, dstp->bits);
1338 : 0 : }
1339 : :
1340 : : #define dma_cap_zero(mask) __dma_cap_zero(&(mask))
1341 : 0 : static inline void __dma_cap_zero(dma_cap_mask_t *dstp)
1342 : : {
1343 : 0 : bitmap_zero(dstp->bits, DMA_TX_TYPE_END);
1344 : : }
1345 : :
1346 : : #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
1347 : : static inline int
1348 : 0 : __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
1349 : : {
1350 : 0 : return test_bit(tx_type, srcp->bits);
1351 : : }
1352 : :
1353 : : #define for_each_dma_cap_mask(cap, mask) \
1354 : : for_each_set_bit(cap, mask.bits, DMA_TX_TYPE_END)
1355 : :
1356 : : /**
1357 : : * dma_async_issue_pending - flush pending transactions to HW
1358 : : * @chan: target DMA channel
1359 : : *
1360 : : * This allows drivers to push copies to HW in batches,
1361 : : * reducing MMIO writes where possible.
1362 : : */
1363 : 0 : static inline void dma_async_issue_pending(struct dma_chan *chan)
1364 : : {
1365 : 0 : chan->device->device_issue_pending(chan);
1366 : 0 : }
1367 : :
1368 : : /**
1369 : : * dma_async_is_tx_complete - poll for transaction completion
1370 : : * @chan: DMA channel
1371 : : * @cookie: transaction identifier to check status of
1372 : : * @last: returns last completed cookie, can be NULL
1373 : : * @used: returns last issued cookie, can be NULL
1374 : : *
1375 : : * If @last and @used are passed in, upon return they reflect the driver
1376 : : * internal state and can be used with dma_async_is_complete() to check
1377 : : * the status of multiple cookies without re-checking hardware state.
1378 : : */
1379 : 0 : static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
1380 : : dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
1381 : : {
1382 : 0 : struct dma_tx_state state;
1383 : 0 : enum dma_status status;
1384 : :
1385 : 0 : status = chan->device->device_tx_status(chan, cookie, &state);
1386 : 0 : if (last)
1387 : : *last = state.last;
1388 : 0 : if (used)
1389 : : *used = state.used;
1390 [ # # ]: 0 : return status;
1391 : : }
1392 : :
1393 : : /**
1394 : : * dma_async_is_complete - test a cookie against chan state
1395 : : * @cookie: transaction identifier to test status of
1396 : : * @last_complete: last know completed transaction
1397 : : * @last_used: last cookie value handed out
1398 : : *
1399 : : * dma_async_is_complete() is used in dma_async_is_tx_complete()
1400 : : * the test logic is separated for lightweight testing of multiple cookies
1401 : : */
1402 : 0 : static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
1403 : : dma_cookie_t last_complete, dma_cookie_t last_used)
1404 : : {
1405 [ # # # # ]: 0 : if (last_complete <= last_used) {
1406 [ # # # # ]: 0 : if ((cookie <= last_complete) || (cookie > last_used))
1407 : : return DMA_COMPLETE;
1408 : : } else {
1409 [ # # # # ]: 0 : if ((cookie <= last_complete) && (cookie > last_used))
1410 : : return DMA_COMPLETE;
1411 : : }
1412 : : return DMA_IN_PROGRESS;
1413 : : }
1414 : :
1415 : : static inline void
1416 : : dma_set_tx_state(struct dma_tx_state *st, dma_cookie_t last, dma_cookie_t used, u32 residue)
1417 : : {
1418 : : if (st) {
1419 : : st->last = last;
1420 : : st->used = used;
1421 : : st->residue = residue;
1422 : : }
1423 : : }
1424 : :
1425 : : #ifdef CONFIG_DMA_ENGINE
1426 : : struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
1427 : : enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
1428 : : enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx);
1429 : : void dma_issue_pending_all(void);
1430 : : struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
1431 : : dma_filter_fn fn, void *fn_param,
1432 : : struct device_node *np);
1433 : : struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name);
1434 : :
1435 : : struct dma_chan *dma_request_chan(struct device *dev, const char *name);
1436 : : struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
1437 : :
1438 : : void dma_release_channel(struct dma_chan *chan);
1439 : : int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps);
1440 : : #else
1441 : : static inline struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
1442 : : {
1443 : : return NULL;
1444 : : }
1445 : : static inline enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
1446 : : {
1447 : : return DMA_COMPLETE;
1448 : : }
1449 : : static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
1450 : : {
1451 : : return DMA_COMPLETE;
1452 : : }
1453 : : static inline void dma_issue_pending_all(void)
1454 : : {
1455 : : }
1456 : : static inline struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
1457 : : dma_filter_fn fn,
1458 : : void *fn_param,
1459 : : struct device_node *np)
1460 : : {
1461 : : return NULL;
1462 : : }
1463 : : static inline struct dma_chan *dma_request_slave_channel(struct device *dev,
1464 : : const char *name)
1465 : : {
1466 : : return NULL;
1467 : : }
1468 : : static inline struct dma_chan *dma_request_chan(struct device *dev,
1469 : : const char *name)
1470 : : {
1471 : : return ERR_PTR(-ENODEV);
1472 : : }
1473 : : static inline struct dma_chan *dma_request_chan_by_mask(
1474 : : const dma_cap_mask_t *mask)
1475 : : {
1476 : : return ERR_PTR(-ENODEV);
1477 : : }
1478 : : static inline void dma_release_channel(struct dma_chan *chan)
1479 : : {
1480 : : }
1481 : : static inline int dma_get_slave_caps(struct dma_chan *chan,
1482 : : struct dma_slave_caps *caps)
1483 : : {
1484 : : return -ENXIO;
1485 : : }
1486 : : #endif
1487 : :
1488 : : #define dma_request_slave_channel_reason(dev, name) dma_request_chan(dev, name)
1489 : :
1490 : : static inline int dmaengine_desc_set_reuse(struct dma_async_tx_descriptor *tx)
1491 : : {
1492 : : struct dma_slave_caps caps;
1493 : : int ret;
1494 : :
1495 : : ret = dma_get_slave_caps(tx->chan, &caps);
1496 : : if (ret)
1497 : : return ret;
1498 : :
1499 : : if (caps.descriptor_reuse) {
1500 : : tx->flags |= DMA_CTRL_REUSE;
1501 : : return 0;
1502 : : } else {
1503 : : return -EPERM;
1504 : : }
1505 : : }
1506 : :
1507 : 0 : static inline void dmaengine_desc_clear_reuse(struct dma_async_tx_descriptor *tx)
1508 : : {
1509 : 0 : tx->flags &= ~DMA_CTRL_REUSE;
1510 : : }
1511 : :
1512 : 0 : static inline bool dmaengine_desc_test_reuse(struct dma_async_tx_descriptor *tx)
1513 : : {
1514 [ # # ]: 0 : return (tx->flags & DMA_CTRL_REUSE) == DMA_CTRL_REUSE;
1515 : : }
1516 : :
1517 : : static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc)
1518 : : {
1519 : : /* this is supported for reusable desc, so check that */
1520 : : if (dmaengine_desc_test_reuse(desc))
1521 : : return desc->desc_free(desc);
1522 : : else
1523 : : return -EPERM;
1524 : : }
1525 : :
1526 : : /* --- DMA device --- */
1527 : :
1528 : : int dma_async_device_register(struct dma_device *device);
1529 : : int dmaenginem_async_device_register(struct dma_device *device);
1530 : : void dma_async_device_unregister(struct dma_device *device);
1531 : : int dma_async_device_channel_register(struct dma_device *device,
1532 : : struct dma_chan *chan);
1533 : : void dma_async_device_channel_unregister(struct dma_device *device,
1534 : : struct dma_chan *chan);
1535 : : void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
1536 : : #define dma_request_channel(mask, x, y) \
1537 : : __dma_request_channel(&(mask), x, y, NULL)
1538 : :
1539 : : static inline struct dma_chan
1540 : 0 : *dma_request_slave_channel_compat(const dma_cap_mask_t mask,
1541 : : dma_filter_fn fn, void *fn_param,
1542 : : struct device *dev, const char *name)
1543 : : {
1544 : 0 : struct dma_chan *chan;
1545 : :
1546 : 0 : chan = dma_request_slave_channel(dev, name);
1547 [ # # ]: 0 : if (chan)
1548 : : return chan;
1549 : :
1550 [ # # ]: 0 : if (!fn || !fn_param)
1551 : : return NULL;
1552 : :
1553 : 0 : return __dma_request_channel(&mask, fn, fn_param, NULL);
1554 : : }
1555 : :
1556 : : static inline char *
1557 : : dmaengine_get_direction_text(enum dma_transfer_direction dir)
1558 : : {
1559 : : switch (dir) {
1560 : : case DMA_DEV_TO_MEM:
1561 : : return "DEV_TO_MEM";
1562 : : case DMA_MEM_TO_DEV:
1563 : : return "MEM_TO_DEV";
1564 : : case DMA_MEM_TO_MEM:
1565 : : return "MEM_TO_MEM";
1566 : : case DMA_DEV_TO_DEV:
1567 : : return "DEV_TO_DEV";
1568 : : default:
1569 : : break;
1570 : : }
1571 : :
1572 : : return "invalid";
1573 : : }
1574 : : #endif /* DMAENGINE_H */
|
