]> rtime.felk.cvut.cz Git - zynq/linux.git/blob - drivers/staging/apf/xilinx-dma-apf.c
staging: apf: Correct sg list length assignment in apf dma
[zynq/linux.git] / drivers / staging / apf / xilinx-dma-apf.c
1 /*
2  * Xilinx AXI DMA Engine support
3  *
4  * Copyright (C) 2010 Xilinx, Inc. All rights reserved.
5  *
6  * Description:
7  * This driver supports Xilinx AXI DMA engine:
8  *  . Axi DMA engine, it does transfers between memory and device. It can be
9  *    configured to have one channel or two channels. If configured as two
10  *    channels, one is for transmit to device and another is for receive from
11  *    device.
12  *
13  * This is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  */
19
20 #include <linux/init.h>
21 #include <linux/module.h>
22 #include <linux/platform_device.h>
23 #include <linux/interrupt.h>
24 #include <linux/dmapool.h>
25 #include <linux/slab.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/pagemap.h>
28 #include <linux/device.h>
29 #include <linux/types.h>
30 #include <linux/pm.h>
31 #include <linux/fs.h>
32 #include <linux/gfp.h>
33 #include <linux/string.h>
34 #include <linux/uaccess.h>
35 #include <asm/cacheflush.h>
36 #include <linux/sched.h>
37 #include <linux/dma-buf.h>
38
39 #include <linux/of.h>
40 #include <linux/irq.h>
41 #include <linux/of_irq.h>
42
43 #include "xilinx-dma-apf.h"
44
45 #include "xlnk.h"
46
47 static DEFINE_MUTEX(dma_list_mutex);
48 static LIST_HEAD(dma_device_list);
49 /* IO accessors */
50 #define DMA_OUT_64(addr, val)   (writeq(val, addr))
51 #define DMA_OUT(addr, val)      (iowrite32(val, addr))
52 #define DMA_IN(addr)            (ioread32(addr))
53
54 #define GET_LOW(x) ((u32)((x) & 0xFFFFFFFF))
55 #define GET_HI(x) ((u32)((x) / 0x100000000))
56
57 static int unpin_user_pages(struct scatterlist *sglist, unsigned int cnt);
58 /* Driver functions */
59 static void xdma_clean_bd(struct xdma_desc_hw *bd)
60 {
61         bd->src_addr = 0x0;
62         bd->control = 0x0;
63         bd->status = 0x0;
64         bd->app[0] = 0x0;
65         bd->app[1] = 0x0;
66         bd->app[2] = 0x0;
67         bd->app[3] = 0x0;
68         bd->app[4] = 0x0;
69         bd->dmahead = 0x0;
70         bd->sw_flag = 0x0;
71 }
72
73 static int dma_is_running(struct xdma_chan *chan)
74 {
75         return !(DMA_IN(&chan->regs->sr) & XDMA_SR_HALTED_MASK) &&
76                 (DMA_IN(&chan->regs->cr) & XDMA_CR_RUNSTOP_MASK);
77 }
78
79 static int dma_is_idle(struct xdma_chan *chan)
80 {
81         return DMA_IN(&chan->regs->sr) & XDMA_SR_IDLE_MASK;
82 }
83
84 static void dma_halt(struct xdma_chan *chan)
85 {
86         DMA_OUT(&chan->regs->cr,
87                 (DMA_IN(&chan->regs->cr)  & ~XDMA_CR_RUNSTOP_MASK));
88 }
89
90 static void dma_start(struct xdma_chan *chan)
91 {
92         DMA_OUT(&chan->regs->cr,
93                 (DMA_IN(&chan->regs->cr) | XDMA_CR_RUNSTOP_MASK));
94 }
95
96 static int dma_init(struct xdma_chan *chan)
97 {
98         int loop = XDMA_RESET_LOOP;
99
100         DMA_OUT(&chan->regs->cr,
101                 (DMA_IN(&chan->regs->cr) | XDMA_CR_RESET_MASK));
102
103         /* Wait for the hardware to finish reset
104          */
105         while (loop) {
106                 if (!(DMA_IN(&chan->regs->cr) & XDMA_CR_RESET_MASK))
107                         break;
108
109                 loop -= 1;
110         }
111
112         if (!loop)
113                 return 1;
114
115         return 0;
116 }
117
118 static int xdma_alloc_chan_descriptors(struct xdma_chan *chan)
119 {
120         int i;
121         u8 *ptr;
122
123         /*
124          * We need the descriptor to be aligned to 64bytes
125          * for meeting Xilinx DMA specification requirement.
126          */
127         ptr = (u8 *)dma_alloc_coherent(chan->dev,
128                                 (sizeof(struct xdma_desc_hw) * XDMA_MAX_BD_CNT),
129                                 &chan->bd_phys_addr,
130                                 GFP_KERNEL);
131
132         if (!ptr) {
133                 dev_err(chan->dev,
134                         "unable to allocate channel %d descriptor pool\n",
135                         chan->id);
136                 return -ENOMEM;
137         }
138
139         memset(ptr, 0, (sizeof(struct xdma_desc_hw) * XDMA_MAX_BD_CNT));
140         chan->bd_cur = 0;
141         chan->bd_tail = 0;
142         chan->bd_used = 0;
143         chan->bd_chain_size = sizeof(struct xdma_desc_hw) * XDMA_MAX_BD_CNT;
144
145         /*
146          * Pre allocate all the channels.
147          */
148         for (i = 0; i < XDMA_MAX_BD_CNT; i++) {
149                 chan->bds[i] = (struct xdma_desc_hw *)
150                                 (ptr + (sizeof(struct xdma_desc_hw) * i));
151                 chan->bds[i]->next_desc = chan->bd_phys_addr +
152                                         (sizeof(struct xdma_desc_hw) *
153                                                 ((i + 1) % XDMA_MAX_BD_CNT));
154         }
155
156         /* there is at least one descriptor free to be allocated */
157         return 0;
158 }
159
160 static void xdma_free_chan_resources(struct xdma_chan *chan)
161 {
162         dev_dbg(chan->dev, "Free all channel resources.\n");
163         dma_free_coherent(chan->dev, (sizeof(struct xdma_desc_hw) *
164                         XDMA_MAX_BD_CNT), chan->bds[0], chan->bd_phys_addr);
165 }
166
167 static void xilinx_chan_desc_reinit(struct xdma_chan *chan)
168 {
169         struct xdma_desc_hw *desc;
170         unsigned int start, end;
171         unsigned long flags;
172
173         spin_lock_irqsave(&chan->lock, flags);
174         start = 0;
175         end = XDMA_MAX_BD_CNT;
176
177         while (start < end) {
178                 desc = chan->bds[start];
179                 xdma_clean_bd(desc);
180                 start++;
181         }
182         /* Re-initialize bd_cur and bd_tail values */
183         chan->bd_cur = 0;
184         chan->bd_tail = 0;
185         chan->bd_used = 0;
186         spin_unlock_irqrestore(&chan->lock, flags);
187 }
188
189 static void xilinx_chan_desc_cleanup(struct xdma_chan *chan)
190 {
191         struct xdma_head *dmahead;
192         struct xdma_desc_hw *desc;
193         struct completion *cmp;
194         unsigned long flags;
195
196         spin_lock_irqsave(&chan->lock, flags);
197 #define XDMA_BD_STS_RXEOF_MASK 0x04000000
198         desc = chan->bds[chan->bd_cur];
199         while (desc->status & XDMA_BD_STS_ALL_MASK) {
200                 if ((desc->status & XDMA_BD_STS_RXEOF_MASK) &&
201                     !(desc->dmahead)) {
202                         pr_info("ERROR: premature EOF on DMA\n");
203                         dma_init(chan); /* reset the dma HW */
204                         while (!(desc->dmahead)) {
205                                 xdma_clean_bd(desc);
206                                 chan->bd_used--;
207                                 chan->bd_cur++;
208                                 if (chan->bd_cur >= XDMA_MAX_BD_CNT)
209                                         chan->bd_cur = 0;
210                                 desc = chan->bds[chan->bd_cur];
211                         }
212                 }
213                 if (desc->dmahead) {
214                         if ((desc->sw_flag & XDMA_BD_SF_POLL_MODE_MASK))
215                                 if (!(desc->sw_flag & XDMA_BD_SF_SW_DONE_MASK))
216                                         break;
217
218                         dmahead = (struct xdma_head *)desc->dmahead;
219                         cmp = (struct completion *)&dmahead->cmp;
220                         if (dmahead->nappwords_o)
221                                 memcpy(dmahead->appwords_o, desc->app,
222                                        dmahead->nappwords_o * sizeof(u32));
223
224                         if (chan->poll_mode)
225                                 cmp->done = 1;
226                         else
227                                 complete(cmp);
228                 }
229                 xdma_clean_bd(desc);
230                 chan->bd_used--;
231                 chan->bd_cur++;
232                 if (chan->bd_cur >= XDMA_MAX_BD_CNT)
233                         chan->bd_cur = 0;
234                 desc = chan->bds[chan->bd_cur];
235         }
236         spin_unlock_irqrestore(&chan->lock, flags);
237 }
238
239 static void xdma_err_tasklet(unsigned long data)
240 {
241         struct xdma_chan *chan = (struct xdma_chan *)data;
242
243         if (chan->err) {
244                 /* If reset failed, need to hard reset
245                  * Channel is no longer functional
246                  */
247                 if (!dma_init(chan))
248                         chan->err = 0;
249                 else
250                         dev_err(chan->dev, "DMA channel reset failed, please reset system\n");
251         }
252
253         /* Barrier to assert descriptor init is reaches memory */
254         rmb();
255         xilinx_chan_desc_cleanup(chan);
256
257         xilinx_chan_desc_reinit(chan);
258 }
259
260 static void xdma_tasklet(unsigned long data)
261 {
262         struct xdma_chan *chan = (struct xdma_chan *)data;
263
264         xilinx_chan_desc_cleanup(chan);
265 }
266
267 static void dump_cur_bd(struct xdma_chan *chan)
268 {
269         u32 index;
270
271         index = (((u32)DMA_IN(&chan->regs->cdr)) - chan->bd_phys_addr) /
272                         sizeof(struct xdma_desc_hw);
273
274         dev_err(chan->dev, "cur bd @ %08x\n",   (u32)DMA_IN(&chan->regs->cdr));
275         dev_err(chan->dev, "  buf  = %p\n",
276                 (void *)chan->bds[index]->src_addr);
277         dev_err(chan->dev, "  ctrl = 0x%08x\n", chan->bds[index]->control);
278         dev_err(chan->dev, "  sts  = 0x%08x\n", chan->bds[index]->status);
279         dev_err(chan->dev, "  next = %p\n",
280                 (void *)chan->bds[index]->next_desc);
281 }
282
283 static irqreturn_t xdma_rx_intr_handler(int irq, void *data)
284 {
285         struct xdma_chan *chan = data;
286         u32 stat;
287
288         stat = DMA_IN(&chan->regs->sr);
289
290         if (!(stat & XDMA_XR_IRQ_ALL_MASK))
291                 return IRQ_NONE;
292
293         /* Ack the interrupts */
294         DMA_OUT(&chan->regs->sr, (stat & XDMA_XR_IRQ_ALL_MASK));
295
296         if (stat & XDMA_XR_IRQ_ERROR_MASK) {
297                 dev_err(chan->dev, "Channel %s has errors %x, cdr %x tdr %x\n",
298                         chan->name, (unsigned int)stat,
299                         (unsigned int)DMA_IN(&chan->regs->cdr),
300                         (unsigned int)DMA_IN(&chan->regs->tdr));
301
302                 dump_cur_bd(chan);
303
304                 chan->err = 1;
305                 tasklet_schedule(&chan->dma_err_tasklet);
306         }
307
308         if (!(chan->poll_mode) && ((stat & XDMA_XR_IRQ_DELAY_MASK) ||
309                                    (stat & XDMA_XR_IRQ_IOC_MASK)))
310                 tasklet_schedule(&chan->tasklet);
311
312         return IRQ_HANDLED;
313 }
314
315 static irqreturn_t xdma_tx_intr_handler(int irq, void *data)
316 {
317         struct xdma_chan *chan = data;
318         u32 stat;
319
320         stat = DMA_IN(&chan->regs->sr);
321
322         if (!(stat & XDMA_XR_IRQ_ALL_MASK))
323                 return IRQ_NONE;
324
325         /* Ack the interrupts */
326         DMA_OUT(&chan->regs->sr, (stat & XDMA_XR_IRQ_ALL_MASK));
327
328         if (stat & XDMA_XR_IRQ_ERROR_MASK) {
329                 dev_err(chan->dev, "Channel %s has errors %x, cdr %x tdr %x\n",
330                         chan->name, (unsigned int)stat,
331                         (unsigned int)DMA_IN(&chan->regs->cdr),
332                         (unsigned int)DMA_IN(&chan->regs->tdr));
333
334                 dump_cur_bd(chan);
335
336                 chan->err = 1;
337                 tasklet_schedule(&chan->dma_err_tasklet);
338         }
339
340         if (!(chan->poll_mode) && ((stat & XDMA_XR_IRQ_DELAY_MASK) ||
341                                    (stat & XDMA_XR_IRQ_IOC_MASK)))
342                 tasklet_schedule(&chan->tasklet);
343
344         return IRQ_HANDLED;
345 }
346
347 static void xdma_start_transfer(struct xdma_chan *chan,
348                                 int start_index,
349                                 int end_index)
350 {
351         xlnk_intptr_type cur_phys;
352         xlnk_intptr_type tail_phys;
353         u32 regval;
354
355         if (chan->err)
356                 return;
357
358         cur_phys = chan->bd_phys_addr + (start_index *
359                                         sizeof(struct xdma_desc_hw));
360         tail_phys = chan->bd_phys_addr + (end_index *
361                                         sizeof(struct xdma_desc_hw));
362         /* If hardware is busy, move the tail & return */
363         if (dma_is_running(chan) || dma_is_idle(chan)) {
364 #if XLNK_SYS_BIT_WIDTH == 32
365                 DMA_OUT(&chan->regs->tdr, tail_phys);
366 #else
367                 DMA_OUT_64(&chan->regs->tdr, tail_phys);
368 #endif
369                 return;
370         }
371
372 #if XLNK_SYS_BIT_WIDTH == 32
373         DMA_OUT(&chan->regs->cdr, cur_phys);
374 #else
375         DMA_OUT_64(&chan->regs->cdr, cur_phys);
376 #endif
377
378         dma_start(chan);
379
380         /* Enable interrupts */
381         regval = DMA_IN(&chan->regs->cr);
382         regval |= (chan->poll_mode ? XDMA_XR_IRQ_ERROR_MASK
383                                         : XDMA_XR_IRQ_ALL_MASK);
384         DMA_OUT(&chan->regs->cr, regval);
385
386         /* Update tail ptr register and start the transfer */
387 #if XLNK_SYS_BIT_WIDTH == 32
388         DMA_OUT(&chan->regs->tdr, tail_phys);
389 #else
390         DMA_OUT_64(&chan->regs->tdr, tail_phys);
391 #endif
392 }
393
394 static int xdma_setup_hw_desc(struct xdma_chan *chan,
395                               struct xdma_head *dmahead,
396                               struct scatterlist *sgl,
397                               unsigned int sg_len,
398                               enum dma_data_direction direction,
399                               unsigned int nappwords_i,
400                               u32 *appwords_i)
401 {
402         struct xdma_desc_hw *bd = NULL;
403         size_t copy;
404         struct scatterlist *sg;
405         size_t sg_used;
406         dma_addr_t dma_src;
407         int i, start_index = -1, end_index1 = 0, end_index2 = -1;
408         int status;
409         unsigned long flags;
410         unsigned int bd_used_saved;
411
412         if (!chan) {
413                 pr_err("Requested transfer on invalid channel\n");
414                 return -ENODEV;
415         }
416
417         /* if we almost run out of bd, try to recycle some */
418         if ((chan->poll_mode) && (chan->bd_used >= XDMA_BD_CLEANUP_THRESHOLD))
419                 xilinx_chan_desc_cleanup(chan);
420
421         spin_lock_irqsave(&chan->lock, flags);
422
423         bd_used_saved = chan->bd_used;
424         /*
425          * Build transactions using information in the scatter gather list
426          */
427         for_each_sg(sgl, sg, sg_len, i) {
428                 sg_used = 0;
429
430                 /* Loop until the entire scatterlist entry is used */
431                 while (sg_used < sg_dma_len(sg)) {
432                         /* Allocate the link descriptor from DMA pool */
433                         bd = chan->bds[chan->bd_tail];
434                         if ((bd->control) & (XDMA_BD_STS_ACTUAL_LEN_MASK)) {
435                                 end_index2 = chan->bd_tail;
436                                 status = -ENOMEM;
437                                 /* If first was not set, then we failed to
438                                  * allocate the very first descriptor,
439                                  * and we're done
440                                  */
441                                 if (start_index == -1)
442                                         goto out_unlock;
443                                 else
444                                         goto out_clean;
445                         }
446                         /*
447                          * Calculate the maximum number of bytes to transfer,
448                          * making sure it is less than the DMA controller limit
449                          */
450                         copy = min((size_t)(sg_dma_len(sg) - sg_used),
451                                    (size_t)chan->max_len);
452                         /*
453                          * Only the src address for DMA
454                          */
455                         dma_src = sg_dma_address(sg) + sg_used;
456                         bd->src_addr = dma_src;
457
458                         /* Fill in the descriptor */
459                         bd->control = copy;
460
461                         /*
462                          * If this is not the first descriptor, chain the
463                          * current descriptor after the previous descriptor
464                          *
465                          * For the first DMA_TO_DEVICE transfer, set SOP
466                          */
467                         if (start_index == -1) {
468                                 start_index = chan->bd_tail;
469
470                                 if (nappwords_i)
471                                         memcpy(bd->app, appwords_i,
472                                                nappwords_i * sizeof(u32));
473
474                                 if (direction == DMA_TO_DEVICE)
475                                         bd->control |= XDMA_BD_SOP;
476                         }
477
478                         sg_used += copy;
479                         end_index2 = chan->bd_tail;
480                         chan->bd_tail++;
481                         chan->bd_used++;
482                         if (chan->bd_tail >= XDMA_MAX_BD_CNT) {
483                                 end_index1 = XDMA_MAX_BD_CNT;
484                                 chan->bd_tail = 0;
485                         }
486                 }
487         }
488
489         if (start_index == -1) {
490                 status = -EINVAL;
491                 goto out_unlock;
492         }
493
494         bd->dmahead = (xlnk_intptr_type)dmahead;
495         bd->sw_flag = chan->poll_mode ? XDMA_BD_SF_POLL_MODE_MASK : 0;
496         dmahead->last_bd_index = end_index2;
497
498         if (direction == DMA_TO_DEVICE)
499                 bd->control |= XDMA_BD_EOP;
500
501         /* Barrier to assert control word write commits */
502         wmb();
503
504         xdma_start_transfer(chan, start_index, end_index2);
505
506         spin_unlock_irqrestore(&chan->lock, flags);
507         return 0;
508
509 out_clean:
510         if (!end_index1) {
511                 for (i = start_index; i < end_index2; i++)
512                         xdma_clean_bd(chan->bds[i]);
513         } else {
514                 /* clean till the end of bd list first, and then 2nd end */
515                 for (i = start_index; i < end_index1; i++)
516                         xdma_clean_bd(chan->bds[i]);
517
518                 end_index1 = 0;
519                 for (i = end_index1; i < end_index2; i++)
520                         xdma_clean_bd(chan->bds[i]);
521         }
522         /* Move the bd_tail back */
523         chan->bd_tail = start_index;
524         chan->bd_used = bd_used_saved;
525
526 out_unlock:
527         spin_unlock_irqrestore(&chan->lock, flags);
528
529         return status;
530 }
531
532 /*
533  *  create minimal length scatter gather list for physically contiguous buffer
534  *  that starts at phy_buf and has length phy_buf_len bytes
535  */
536 static unsigned int phy_buf_to_sgl(xlnk_intptr_type phy_buf,
537                                    unsigned int phy_buf_len,
538                                    struct scatterlist *sgl)
539 {
540         unsigned int sgl_cnt = 0;
541         struct scatterlist *sgl_head;
542         unsigned int dma_len;
543         unsigned int num_bd;
544
545         if (!phy_buf || !phy_buf_len) {
546                 pr_err("phy_buf is NULL or phy_buf_len = 0\n");
547                 return sgl_cnt;
548         }
549
550         num_bd = (phy_buf_len + (XDMA_MAX_TRANS_LEN - 1))
551                 / XDMA_MAX_TRANS_LEN;
552         sgl_head = sgl;
553         sg_init_table(sgl, num_bd);
554
555         while (phy_buf_len > 0) {
556                 xlnk_intptr_type page_id = phy_buf >> PAGE_SHIFT;
557                 unsigned int offset = phy_buf - (page_id << PAGE_SHIFT);
558
559                 sgl_cnt++;
560                 if (sgl_cnt > XDMA_MAX_BD_CNT)
561                         return 0;
562
563                 dma_len = (phy_buf_len > XDMA_MAX_TRANS_LEN) ?
564                                 XDMA_MAX_TRANS_LEN : phy_buf_len;
565
566                 sg_set_page(sgl_head, pfn_to_page(page_id), dma_len, offset);
567                 sg_dma_address(sgl_head) = (dma_addr_t)phy_buf;
568                 sg_dma_len(sgl_head) = dma_len;
569                 sgl_head = sg_next(sgl_head);
570
571                 phy_buf += dma_len;
572                 phy_buf_len -= dma_len;
573         }
574
575         return sgl_cnt;
576 }
577
578 /*  merge sg list, sgl, with length sgl_len, to sgl_merged, to save dma bds */
579 static unsigned int sgl_merge(struct scatterlist *sgl,
580                               unsigned int sgl_len,
581                               struct scatterlist *sgl_merged)
582 {
583         struct scatterlist *sghead, *sgend, *sgnext, *sg_merged_head;
584         unsigned int sg_visited_cnt = 0, sg_merged_num = 0;
585         unsigned int dma_len = 0;
586
587         sg_init_table(sgl_merged, sgl_len);
588         sg_merged_head = sgl_merged;
589         sghead = sgl;
590
591         while (sghead && (sg_visited_cnt < sgl_len)) {
592                 dma_len = sg_dma_len(sghead);
593                 sgend = sghead;
594                 sg_visited_cnt++;
595                 sgnext = sg_next(sgend);
596
597                 while (sgnext && (sg_visited_cnt < sgl_len)) {
598                         if ((sg_dma_address(sgend) + sg_dma_len(sgend)) !=
599                                 sg_dma_address(sgnext))
600                                 break;
601
602                         if (dma_len + sg_dma_len(sgnext) >= XDMA_MAX_TRANS_LEN)
603                                 break;
604
605                         sgend = sgnext;
606                         dma_len += sg_dma_len(sgend);
607                         sg_visited_cnt++;
608                         sgnext = sg_next(sgnext);
609                 }
610
611                 sg_merged_num++;
612                 if (sg_merged_num > XDMA_MAX_BD_CNT)
613                         return 0;
614
615                 memcpy(sg_merged_head, sghead, sizeof(struct scatterlist));
616
617                 sg_dma_len(sg_merged_head) = dma_len;
618
619                 sg_merged_head = sg_next(sg_merged_head);
620                 sghead = sg_next(sgend);
621         }
622
623         return sg_merged_num;
624 }
625
626 static int pin_user_pages(xlnk_intptr_type uaddr,
627                           unsigned int ulen,
628                           int write,
629                           struct scatterlist **scatterpp,
630                           unsigned int *cntp,
631                           unsigned int user_flags)
632 {
633         int status;
634         struct mm_struct *mm = current->mm;
635         unsigned int first_page;
636         unsigned int last_page;
637         unsigned int num_pages;
638         struct scatterlist *sglist;
639         struct page **mapped_pages;
640
641         unsigned int pgidx;
642         unsigned int pglen;
643         unsigned int pgoff;
644         unsigned int sublen;
645
646         first_page = uaddr / PAGE_SIZE;
647         last_page = (uaddr + ulen - 1) / PAGE_SIZE;
648         num_pages = last_page - first_page + 1;
649         mapped_pages = vmalloc(sizeof(*mapped_pages) * num_pages);
650         if (!mapped_pages)
651                 return -ENOMEM;
652
653         down_read(&mm->mmap_sem);
654         status = get_user_pages(uaddr, num_pages,
655                                 (write ? FOLL_WRITE : 0) | FOLL_FORCE,
656                                 mapped_pages, NULL);
657         up_read(&mm->mmap_sem);
658
659         if (status == num_pages) {
660                 sglist = kcalloc(num_pages,
661                                  sizeof(struct scatterlist),
662                                  GFP_KERNEL);
663                 if (!sglist) {
664                         pr_err("%s: kcalloc failed to create sg list\n",
665                                __func__);
666                         vfree(mapped_pages);
667                         return -ENOMEM;
668                 }
669                 sg_init_table(sglist, num_pages);
670                 sublen = 0;
671                 for (pgidx = 0; pgidx < status; pgidx++) {
672                         if (pgidx == 0 && num_pages != 1) {
673                                 pgoff = uaddr & (~PAGE_MASK);
674                                 pglen = PAGE_SIZE - pgoff;
675                         } else if (pgidx == 0 && num_pages == 1) {
676                                 pgoff = uaddr & (~PAGE_MASK);
677                                 pglen = ulen;
678                         } else if (pgidx == num_pages - 1) {
679                                 pgoff = 0;
680                                 pglen = ulen - sublen;
681                         } else {
682                                 pgoff = 0;
683                                 pglen = PAGE_SIZE;
684                         }
685
686                         sublen += pglen;
687
688                         sg_set_page(&sglist[pgidx],
689                                     mapped_pages[pgidx],
690                                     pglen, pgoff);
691
692                         sg_dma_len(&sglist[pgidx]) = pglen;
693                 }
694
695                 *scatterpp = sglist;
696                 *cntp = num_pages;
697
698                 vfree(mapped_pages);
699                 return 0;
700         }
701         pr_err("Failed to pin user pages\n");
702         for (pgidx = 0; pgidx < status; pgidx++)
703                 put_page(mapped_pages[pgidx]);
704         vfree(mapped_pages);
705         return -ENOMEM;
706 }
707
708 static int unpin_user_pages(struct scatterlist *sglist, unsigned int cnt)
709 {
710         struct page *pg;
711         unsigned int i;
712
713         if (!sglist)
714                 return 0;
715
716         for (i = 0; i < cnt; i++) {
717                 pg = sg_page(sglist + i);
718                 if (pg)
719                         put_page(pg);
720         }
721
722         kfree(sglist);
723         return 0;
724 }
725
726 struct xdma_chan *xdma_request_channel(char *name)
727 {
728         int i;
729         struct xdma_device *device, *tmp;
730
731         list_for_each_entry_safe(device, tmp, &dma_device_list, node) {
732                 for (i = 0; i < device->channel_count; i++) {
733                         if (!strcmp(device->chan[i]->name, name))
734                                 return device->chan[i];
735                 }
736         }
737         return NULL;
738 }
739 EXPORT_SYMBOL(xdma_request_channel);
740
741 void xdma_release_channel(struct xdma_chan *chan)
742 { }
743 EXPORT_SYMBOL(xdma_release_channel);
744
745 void xdma_release_all_channels(void)
746 {
747         int i;
748         struct xdma_device *device, *tmp;
749
750         list_for_each_entry_safe(device, tmp, &dma_device_list, node) {
751                 for (i = 0; i < device->channel_count; i++) {
752                         if (device->chan[i]->client_count) {
753                                 dma_halt(device->chan[i]);
754                                 xilinx_chan_desc_reinit(device->chan[i]);
755                                 pr_info("%s: chan %s freed\n",
756                                         __func__,
757                                         device->chan[i]->name);
758                         }
759                 }
760         }
761 }
762 EXPORT_SYMBOL(xdma_release_all_channels);
763
764 static void xdma_release(struct device *dev)
765 {
766 }
767
768 int xdma_submit(struct xdma_chan *chan,
769                 xlnk_intptr_type userbuf,
770                 void *kaddr,
771                 unsigned int size,
772                 unsigned int nappwords_i,
773                 u32 *appwords_i,
774                 unsigned int nappwords_o,
775                 unsigned int user_flags,
776                 struct xdma_head **dmaheadpp,
777                 struct xlnk_dmabuf_reg *dp)
778 {
779         struct xdma_head *dmahead;
780         struct scatterlist *pagelist = NULL;
781         struct scatterlist *sglist = NULL;
782         unsigned int pagecnt = 0;
783         unsigned int sgcnt = 0;
784         enum dma_data_direction dmadir;
785         int status;
786         unsigned long attrs = 0;
787
788         dmahead = kzalloc(sizeof(*dmahead), GFP_KERNEL);
789         if (!dmahead)
790                 return -ENOMEM;
791
792         dmahead->chan = chan;
793         dmahead->userbuf = userbuf;
794         dmahead->size = size;
795         dmahead->dmadir = chan->direction;
796         dmahead->userflag = user_flags;
797         dmahead->dmabuf = dp;
798         dmadir = chan->direction;
799
800         if (!(user_flags & CF_FLAG_CACHE_FLUSH_INVALIDATE))
801                 attrs |= DMA_ATTR_SKIP_CPU_SYNC;
802
803         if (dp) {
804                 int i;
805                 struct scatterlist *sg;
806                 unsigned int remaining_size = size;
807
808                 if (IS_ERR_OR_NULL(dp->dbuf_sg_table)) {
809                         pr_err("%s dmabuf not mapped: %p\n",
810                                __func__, dp->dbuf_sg_table);
811                         return -EINVAL;
812                 }
813                 if (dp->dbuf_sg_table->nents == 0) {
814                         pr_err("%s: cannot map a scatterlist with 0 entries\n",
815                                __func__);
816                         return -EINVAL;
817                 }
818                 sglist = kmalloc_array(dp->dbuf_sg_table->nents,
819                                        sizeof(*sglist),
820                                        GFP_KERNEL);
821                 if (!sglist)
822                         return -ENOMEM;
823
824                 sg_init_table(sglist, dp->dbuf_sg_table->nents);
825                 sgcnt = 0;
826                 for_each_sg(dp->dbuf_sg_table->sgl,
827                             sg,
828                             dp->dbuf_sg_table->nents,
829                             i) {
830                         sg_set_page(sglist + i,
831                                     sg_page(sg),
832                                     sg_dma_len(sg),
833                                     sg->offset);
834                         sg_dma_address(sglist + i) = sg_dma_address(sg);
835                         if (remaining_size == 0) {
836                                 sg_dma_len(sglist + i) = 0;
837                         } else if (sg_dma_len(sg) > remaining_size) {
838                                 sg_dma_len(sglist + i) = remaining_size;
839                                 sgcnt++;
840                         } else {
841                                 sg_dma_len(sglist + i) = sg_dma_len(sg);
842                                 remaining_size -= sg_dma_len(sg);
843                                 sgcnt++;
844                         }
845                 }
846                 dmahead->userbuf = (xlnk_intptr_type)sglist->dma_address;
847                 pagelist = NULL;
848                 pagecnt = 0;
849         } else if (user_flags & CF_FLAG_PHYSICALLY_CONTIGUOUS) {
850                 size_t elem_cnt;
851
852                 elem_cnt = DIV_ROUND_UP(size, XDMA_MAX_TRANS_LEN);
853                 sglist = kmalloc_array(elem_cnt, sizeof(*sglist), GFP_KERNEL);
854                 sgcnt = phy_buf_to_sgl(userbuf, size, sglist);
855                 if (!sgcnt)
856                         return -ENOMEM;
857
858                 status = get_dma_ops(chan->dev)->map_sg(chan->dev,
859                                                         sglist,
860                                                         sgcnt,
861                                                         dmadir,
862                                                         attrs);
863
864                 if (!status) {
865                         pr_err("sg contiguous mapping failed\n");
866                         return -ENOMEM;
867                 }
868                 pagelist = NULL;
869                 pagecnt = 0;
870         } else {
871                 status = pin_user_pages(userbuf,
872                                         size,
873                                         dmadir != DMA_TO_DEVICE,
874                                         &pagelist,
875                                         &pagecnt,
876                                         user_flags);
877                 if (status < 0) {
878                         pr_err("pin_user_pages failed\n");
879                         return status;
880                 }
881
882                 status = get_dma_ops(chan->dev)->map_sg(chan->dev,
883                                                         pagelist,
884                                                         pagecnt,
885                                                         dmadir,
886                                                         attrs);
887                 if (!status) {
888                         pr_err("dma_map_sg failed\n");
889                         unpin_user_pages(pagelist, pagecnt);
890                         return -ENOMEM;
891                 }
892
893                 sglist = kmalloc_array(pagecnt, sizeof(*sglist), GFP_KERNEL);
894                 if (sglist)
895                         sgcnt = sgl_merge(pagelist, pagecnt, sglist);
896                 if (!sgcnt) {
897                         get_dma_ops(chan->dev)->unmap_sg(chan->dev,
898                                                          pagelist,
899                                                          pagecnt,
900                                                          dmadir,
901                                                          attrs);
902                         unpin_user_pages(pagelist, pagecnt);
903                         kfree(sglist);
904                         return -ENOMEM;
905                 }
906         }
907         dmahead->sglist = sglist;
908         dmahead->sgcnt = sgcnt;
909         dmahead->pagelist = pagelist;
910         dmahead->pagecnt = pagecnt;
911
912         /* skipping config */
913         init_completion(&dmahead->cmp);
914
915         if (nappwords_i > XDMA_MAX_APPWORDS)
916                 nappwords_i = XDMA_MAX_APPWORDS;
917
918         if (nappwords_o > XDMA_MAX_APPWORDS)
919                 nappwords_o = XDMA_MAX_APPWORDS;
920
921         dmahead->nappwords_o = nappwords_o;
922
923         status = xdma_setup_hw_desc(chan, dmahead, sglist, sgcnt,
924                                     dmadir, nappwords_i, appwords_i);
925         if (status) {
926                 pr_err("setup hw desc failed\n");
927                 if (dmahead->pagelist) {
928                         get_dma_ops(chan->dev)->unmap_sg(chan->dev,
929                                                          pagelist,
930                                                          pagecnt,
931                                                          dmadir,
932                                                          attrs);
933                         unpin_user_pages(pagelist, pagecnt);
934                 } else if (!dp) {
935                         get_dma_ops(chan->dev)->unmap_sg(chan->dev,
936                                                          sglist,
937                                                          sgcnt,
938                                                          dmadir,
939                                                          attrs);
940                 }
941                 kfree(dmahead->sglist);
942                 return -ENOMEM;
943         }
944
945         *dmaheadpp = dmahead;
946         return 0;
947 }
948 EXPORT_SYMBOL(xdma_submit);
949
950 int xdma_wait(struct xdma_head *dmahead,
951               unsigned int user_flags,
952               unsigned int *operating_flags)
953 {
954         struct xdma_chan *chan = dmahead->chan;
955         unsigned long attrs = 0;
956
957         if (chan->poll_mode) {
958                 xilinx_chan_desc_cleanup(chan);
959                 *operating_flags |= XDMA_FLAGS_WAIT_COMPLETE;
960         } else {
961                 if (*operating_flags & XDMA_FLAGS_TRYWAIT) {
962                         if (!try_wait_for_completion(&dmahead->cmp))
963                                 return 0;
964                         *operating_flags |= XDMA_FLAGS_WAIT_COMPLETE;
965                 } else {
966                         wait_for_completion(&dmahead->cmp);
967                         *operating_flags |= XDMA_FLAGS_WAIT_COMPLETE;
968                 }
969         }
970
971         if (!dmahead->dmabuf) {
972                 if (!(user_flags & CF_FLAG_CACHE_FLUSH_INVALIDATE))
973                         attrs |= DMA_ATTR_SKIP_CPU_SYNC;
974
975                 if (user_flags & CF_FLAG_PHYSICALLY_CONTIGUOUS) {
976                         get_dma_ops(chan->dev)->unmap_sg(chan->dev,
977                                                          dmahead->sglist,
978                                                          dmahead->sgcnt,
979                                                          dmahead->dmadir,
980                                                          attrs);
981                 } else {
982                         get_dma_ops(chan->dev)->unmap_sg(chan->dev,
983                                                          dmahead->pagelist,
984                                                          dmahead->pagecnt,
985                                                          dmahead->dmadir,
986                                                          attrs);
987                         unpin_user_pages(dmahead->pagelist, dmahead->pagecnt);
988                 }
989         }
990         kfree(dmahead->sglist);
991
992         return 0;
993 }
994 EXPORT_SYMBOL(xdma_wait);
995
996 int xdma_getconfig(struct xdma_chan *chan,
997                    unsigned char *irq_thresh,
998                    unsigned char *irq_delay)
999 {
1000         *irq_thresh = (DMA_IN(&chan->regs->cr) >> XDMA_COALESCE_SHIFT) & 0xff;
1001         *irq_delay = (DMA_IN(&chan->regs->cr) >> XDMA_DELAY_SHIFT) & 0xff;
1002         return 0;
1003 }
1004 EXPORT_SYMBOL(xdma_getconfig);
1005
1006 int xdma_setconfig(struct xdma_chan *chan,
1007                    unsigned char irq_thresh,
1008                    unsigned char irq_delay)
1009 {
1010         unsigned long val;
1011
1012         if (dma_is_running(chan))
1013                 return -EBUSY;
1014
1015         val = DMA_IN(&chan->regs->cr);
1016         val &= ~((0xff << XDMA_COALESCE_SHIFT) |
1017                                 (0xff << XDMA_DELAY_SHIFT));
1018         val |= ((irq_thresh << XDMA_COALESCE_SHIFT) |
1019                                 (irq_delay << XDMA_DELAY_SHIFT));
1020
1021         DMA_OUT(&chan->regs->cr, val);
1022         return 0;
1023 }
1024 EXPORT_SYMBOL(xdma_setconfig);
1025
1026 static const struct of_device_id gic_match[] = {
1027         { .compatible = "arm,cortex-a9-gic", },
1028         { .compatible = "arm,cortex-a15-gic", },
1029         { },
1030 };
1031
1032 static struct device_node *gic_node;
1033
1034 unsigned int xlate_irq(unsigned int hwirq)
1035 {
1036         struct of_phandle_args irq_data;
1037         unsigned int irq;
1038
1039         if (!gic_node)
1040                 gic_node = of_find_matching_node(NULL, gic_match);
1041
1042         if (WARN_ON(!gic_node))
1043                 return hwirq;
1044
1045         irq_data.np = gic_node;
1046         irq_data.args_count = 3;
1047         irq_data.args[0] = 0;
1048 #if XLNK_SYS_BIT_WIDTH == 32
1049         irq_data.args[1] = hwirq - 32; /* GIC SPI offset */
1050 #else
1051         irq_data.args[1] = hwirq;
1052 #endif
1053         irq_data.args[2] = IRQ_TYPE_LEVEL_HIGH;
1054
1055         irq = irq_create_of_mapping(&irq_data);
1056         if (WARN_ON(!irq))
1057                 irq = hwirq;
1058
1059         pr_info("%s: hwirq %d, irq %d\n", __func__, hwirq, irq);
1060
1061         return irq;
1062 }
1063
1064 /* Brute-force probing for xilinx DMA
1065  */
1066 static int xdma_probe(struct platform_device *pdev)
1067 {
1068         struct xdma_device *xdev;
1069         struct resource *res;
1070         int err, i, j;
1071         struct xdma_chan *chan;
1072         struct xdma_device_config *dma_config;
1073         int dma_chan_dir;
1074         int dma_chan_reg_offset;
1075
1076         pr_info("%s: probe dma %p, nres %d, id %d\n", __func__,
1077                 &pdev->dev, pdev->num_resources, pdev->id);
1078
1079         xdev = devm_kzalloc(&pdev->dev, sizeof(struct xdma_device), GFP_KERNEL);
1080         if (!xdev)
1081                 return -ENOMEM;
1082         xdev->dev = &pdev->dev;
1083
1084         /* Set this as configurable once HPC works */
1085         arch_setup_dma_ops(&pdev->dev, 0, 0, NULL, false);
1086         dma_set_mask(&pdev->dev, 0xFFFFFFFFFFFFFFFFull);
1087
1088         dma_config = (struct xdma_device_config *)xdev->dev->platform_data;
1089         if (dma_config->channel_count < 1 || dma_config->channel_count > 2)
1090                 return -EFAULT;
1091
1092         /* Get the memory resource */
1093         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1094         xdev->regs = devm_ioremap_resource(&pdev->dev, res);
1095         if (!xdev->regs) {
1096                 dev_err(&pdev->dev, "unable to iomap registers\n");
1097                 return -EFAULT;
1098         }
1099
1100         dev_info(&pdev->dev, "AXIDMA device %d physical base address=%pa\n",
1101                  pdev->id, &res->start);
1102         dev_info(&pdev->dev, "AXIDMA device %d remapped to %pa\n",
1103                  pdev->id, &xdev->regs);
1104
1105         /* Allocate the channels */
1106
1107         dev_info(&pdev->dev, "has %d channel(s)\n", dma_config->channel_count);
1108         for (i = 0; i < dma_config->channel_count; i++) {
1109                 chan = devm_kzalloc(&pdev->dev, sizeof(*chan), GFP_KERNEL);
1110                 if (!chan)
1111                         return -ENOMEM;
1112
1113                 dma_chan_dir = strcmp(dma_config->channel_config[i].type,
1114                                       "axi-dma-mm2s-channel") ?
1115                                         DMA_FROM_DEVICE :
1116                                         DMA_TO_DEVICE;
1117                 dma_chan_reg_offset = (dma_chan_dir == DMA_TO_DEVICE) ?
1118                                         0 :
1119                                         0x30;
1120
1121                 /* Initialize channel parameters */
1122                 chan->id = i;
1123                 chan->regs = xdev->regs + dma_chan_reg_offset;
1124                 /* chan->regs = xdev->regs; */
1125                 chan->dev = xdev->dev;
1126                 chan->max_len = XDMA_MAX_TRANS_LEN;
1127                 chan->direction = dma_chan_dir;
1128                 sprintf(chan->name, "%s:%d", dma_config->name, chan->id);
1129                 pr_info("  chan %d name: %s\n", chan->id, chan->name);
1130                 pr_info("  chan %d direction: %s\n", chan->id,
1131                         dma_chan_dir == DMA_FROM_DEVICE ?
1132                                 "FROM_DEVICE" : "TO_DEVICE");
1133
1134                 spin_lock_init(&chan->lock);
1135                 tasklet_init(&chan->tasklet,
1136                              xdma_tasklet,
1137                              (unsigned long)chan);
1138                 tasklet_init(&chan->dma_err_tasklet,
1139                              xdma_err_tasklet,
1140                              (unsigned long)chan);
1141
1142                 xdev->chan[chan->id] = chan;
1143
1144                 /* The IRQ resource */
1145                 chan->irq = xlate_irq(dma_config->channel_config[i].irq);
1146                 if (chan->irq <= 0) {
1147                         pr_err("get_resource for IRQ for dev %d failed\n",
1148                                pdev->id);
1149                         return -ENODEV;
1150                 }
1151
1152                 err = devm_request_irq(&pdev->dev,
1153                                        chan->irq,
1154                                        dma_chan_dir == DMA_TO_DEVICE ?
1155                                         xdma_tx_intr_handler :
1156                                         xdma_rx_intr_handler,
1157                                        IRQF_SHARED,
1158                                        pdev->name,
1159                                        chan);
1160                 if (err) {
1161                         dev_err(&pdev->dev, "unable to request IRQ\n");
1162                         return err;
1163                 }
1164                 pr_info("  chan%d irq: %d\n", chan->id, chan->irq);
1165
1166                 chan->poll_mode = dma_config->channel_config[i].poll_mode;
1167                 pr_info("  chan%d poll mode: %s\n",
1168                         chan->id,
1169                         chan->poll_mode ? "on" : "off");
1170
1171                 /* Allocate channel BD's */
1172                 err = xdma_alloc_chan_descriptors(xdev->chan[chan->id]);
1173                 if (err) {
1174                         dev_err(&pdev->dev, "unable to allocate BD's\n");
1175                         return -ENOMEM;
1176                 }
1177                 pr_info("  chan%d bd ring @ 0x%p (size: 0x%x bytes)\n",
1178                         chan->id,
1179                         (void *)chan->bd_phys_addr,
1180                         chan->bd_chain_size);
1181
1182                 err = dma_init(xdev->chan[chan->id]);
1183                 if (err) {
1184                         dev_err(&pdev->dev, "DMA init failed\n");
1185                         /* FIXME Check this - unregister all chan resources */
1186                         for (j = 0; j <= i; j++)
1187                                 xdma_free_chan_resources(xdev->chan[j]);
1188                         return -EIO;
1189                 }
1190         }
1191         xdev->channel_count = dma_config->channel_count;
1192         pdev->dev.release = xdma_release;
1193         /* Add the DMA device to the global list */
1194         mutex_lock(&dma_list_mutex);
1195         list_add_tail(&xdev->node, &dma_device_list);
1196         mutex_unlock(&dma_list_mutex);
1197
1198         platform_set_drvdata(pdev, xdev);
1199
1200         return 0;
1201 }
1202
1203 static int xdma_remove(struct platform_device *pdev)
1204 {
1205         int i;
1206         struct xdma_device *xdev = platform_get_drvdata(pdev);
1207
1208         /* Remove the DMA device from the global list */
1209         mutex_lock(&dma_list_mutex);
1210         list_del(&xdev->node);
1211         mutex_unlock(&dma_list_mutex);
1212
1213         for (i = 0; i < XDMA_MAX_CHANS_PER_DEVICE; i++) {
1214                 if (xdev->chan[i])
1215                         xdma_free_chan_resources(xdev->chan[i]);
1216         }
1217
1218         return 0;
1219 }
1220
1221 static struct platform_driver xdma_driver = {
1222         .probe = xdma_probe,
1223         .remove = xdma_remove,
1224         .driver = {
1225                 .name = "xilinx-axidma",
1226         },
1227 };
1228
1229 module_platform_driver(xdma_driver);
1230
1231 MODULE_DESCRIPTION("Xilinx DMA driver");
1232 MODULE_LICENSE("GPL");