17
17
#include "ipe_private.h"
18
#include "ipe_registers.h"
21
20
#define WR(addr, value) { *(uint32_t*)(ctx->base_addr + addr) = value; }
22
21
#define RD(addr, value) { value = *(uint32_t*)(ctx->base_addr + addr); }
25
pcilib_dma_context_t *dma_ipe_init(pcilib_t *pcilib, pcilib_dma_modification_t type, void *arg) {
24
pcilib_dma_context_t *dma_ipe_init(pcilib_t *pcilib, const char *model, const void *arg) {
28
pcilib_model_description_t *model_info = pcilib_get_model_description(pcilib);
27
const pcilib_model_description_t *model_info = pcilib_get_model_description(pcilib);
30
29
ipe_dma_t *ctx = malloc(sizeof(ipe_dma_t));
33
32
memset(ctx, 0, sizeof(ipe_dma_t));
33
ctx->dmactx.pcilib = pcilib;
35
34
// ctx->mode64 = 1;
37
37
memset(ctx->engine, 0, 2 * sizeof(pcilib_dma_engine_description_t));
38
38
ctx->engine[0].addr = 0;
39
39
ctx->engine[0].type = PCILIB_DMA_TYPE_PACKET;
41
41
ctx->engine[0].addr_bits = 32;
42
42
pcilib_set_dma_engine_description(pcilib, 0, &ctx->engine[0]);
43
43
pcilib_set_dma_engine_description(pcilib, 1, NULL);
45
pcilib_register_bank_t dma_bank = pcilib_find_bank_by_addr(pcilib, PCILIB_REGISTER_BANK_DMA);
46
pcilib_register_bank_t dma_bank = pcilib_find_register_bank_by_addr(pcilib, PCILIB_REGISTER_BANK_DMA);
48
if (dma_bank == PCILIB_REGISTER_BANK_INVALID) {
49
err = pcilib_add_register_banks(ctx->pcilib, 0, ipe_dma_register_banks);
52
pcilib_error("Error (%i) adding DMA register bank");
57
dma_bank = pcilib_find_bank_by_addr(pcilib, PCILIB_REGISTER_BANK_DMA);
46
59
if (dma_bank == PCILIB_REGISTER_BANK_INVALID) {
48
61
pcilib_error("DMA Register Bank could not be found");
52
65
ctx->dma_bank = model_info->banks + dma_bank;
53
66
ctx->base_addr = pcilib_resolve_register_address(pcilib, ctx->dma_bank->bar, ctx->dma_bank->read_addr);
55
69
err = pcilib_add_registers(ctx->pcilib, 0, ipe_dma_registers);
58
72
pcilib_error("Error adding DMA registers");
63
78
return (pcilib_dma_context_t*)ctx;
100
115
if (ctx->pages) return 0;
102
117
kflags = PCILIB_KMEM_FLAG_REUSE|PCILIB_KMEM_FLAG_EXCLUSIVE|PCILIB_KMEM_FLAG_HARDWARE|(ctx->preserve?PCILIB_KMEM_FLAG_PERSISTENT:0);
103
pcilib_kmem_handle_t *desc = pcilib_alloc_kernel_memory(ctx->pcilib, PCILIB_KMEM_TYPE_CONSISTENT, 1, IPEDMA_DESCRIPTOR_SIZE, IPEDMA_DESCRIPTOR_ALIGNMENT, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_RING, 0x00), kflags);
104
pcilib_kmem_handle_t *pages = pcilib_alloc_kernel_memory(ctx->pcilib, PCILIB_KMEM_TYPE_DMA_C2S_PAGE, IPEDMA_DMA_PAGES, 0, 0, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_PAGES, 0x00), kflags);
118
pcilib_kmem_handle_t *desc = pcilib_alloc_kernel_memory(ctx->dmactx.pcilib, PCILIB_KMEM_TYPE_CONSISTENT, 1, IPEDMA_DESCRIPTOR_SIZE, IPEDMA_DESCRIPTOR_ALIGNMENT, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_RING, 0x00), kflags);
119
pcilib_kmem_handle_t *pages = pcilib_alloc_kernel_memory(ctx->dmactx.pcilib, PCILIB_KMEM_TYPE_DMA_C2S_PAGE, IPEDMA_DMA_PAGES, 0, 0, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_PAGES, 0x00), kflags);
106
121
if (!desc||!pages) {
107
if (pages) pcilib_free_kernel_memory(ctx->pcilib, pages, 0);
108
if (desc) pcilib_free_kernel_memory(ctx->pcilib, desc, 0);
122
if (pages) pcilib_free_kernel_memory(ctx->dmactx.pcilib, pages, 0);
123
if (desc) pcilib_free_kernel_memory(ctx->dmactx.pcilib, desc, 0);
109
124
return PCILIB_ERROR_MEMORY;
111
reuse_desc = pcilib_kmem_is_reused(ctx->pcilib, desc);
112
reuse_pages = pcilib_kmem_is_reused(ctx->pcilib, pages);
126
reuse_desc = pcilib_kmem_is_reused(ctx->dmactx.pcilib, desc);
127
reuse_pages = pcilib_kmem_is_reused(ctx->dmactx.pcilib, pages);
114
129
if (reuse_desc == reuse_pages) {
115
130
if (reuse_desc & PCILIB_KMEM_REUSE_PARTIAL) pcilib_warning("Inconsistent DMA buffers are found (only part of required buffers is available), reinitializing...");
129
144
} else pcilib_warning("Inconsistent DMA buffers (modes of ring and page buffers does not match), reinitializing....");
131
desc_va = pcilib_kmem_get_ua(ctx->pcilib, desc);
146
desc_va = pcilib_kmem_get_ua(ctx->dmactx.pcilib, desc);
132
147
if (ctx->mode64) last_written_addr_ptr = desc_va + 3 * sizeof(uint32_t);
133
148
else last_written_addr_ptr = desc_va + 4 * sizeof(uint32_t);
187
202
// Setting current read position and configuring progress register
188
203
WR(IPEDMA_REG_LAST_READ, IPEDMA_DMA_PAGES);
189
WR(IPEDMA_REG_UPDATE_ADDR, pcilib_kmem_get_block_ba(ctx->pcilib, desc, 0));
204
WR(IPEDMA_REG_UPDATE_ADDR, pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, desc, 0));
191
206
// Instructing DMA engine that writting should start from the first DMA page
192
*last_written_addr_ptr = 0;//htonl(pcilib_kmem_get_block_ba(ctx->pcilib, pages, IPEDMA_DMA_PAGES - 1));
207
*last_written_addr_ptr = 0;//htonl(pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, pages, IPEDMA_DMA_PAGES - 1));
195
210
for (i = 0; i < IPEDMA_DMA_PAGES; i++) {
196
uintptr_t bus_addr_check, bus_addr = pcilib_kmem_get_block_ba(ctx->pcilib, pages, i);
211
uintptr_t bus_addr_check, bus_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, pages, i);
197
212
WR(IPEDMA_REG_PAGE_ADDR, bus_addr);
198
213
if (bus_addr%4096) printf("Bad address %lu: %lx\n", i, bus_addr);
219
234
#endif /* IPEDMA_BUG_DMARD */
222
// ctx->last_read_addr = htonl(pcilib_kmem_get_block_ba(ctx->pcilib, pages, ctx->last_read));
223
ctx->last_read_addr = pcilib_kmem_get_block_ba(ctx->pcilib, pages, ctx->last_read);
237
// ctx->last_read_addr = htonl(pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, pages, ctx->last_read));
238
ctx->last_read_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, pages, ctx->last_read);
226
241
ctx->desc = desc;
227
242
ctx->pages = pages;
228
ctx->page_size = pcilib_kmem_get_block_size(ctx->pcilib, pages, 0);;
243
ctx->page_size = pcilib_kmem_get_block_size(ctx->dmactx.pcilib, pages, 0);;
229
244
ctx->ring_size = IPEDMA_DMA_PAGES;
272
pcilib_free_kernel_memory(ctx->pcilib, ctx->desc, kflags);
287
pcilib_free_kernel_memory(ctx->dmactx.pcilib, ctx->desc, kflags);
273
288
ctx->desc = NULL;
276
291
if (ctx->pages) {
277
pcilib_free_kernel_memory(ctx->pcilib, ctx->pages, kflags);
292
pcilib_free_kernel_memory(ctx->dmactx.pcilib, ctx->pages, kflags);
278
293
ctx->pages = NULL;
287
302
ipe_dma_t *ctx = (ipe_dma_t*)vctx;
289
void *desc_va = (void*)pcilib_kmem_get_ua(ctx->pcilib, ctx->desc);
304
void *desc_va = (void*)pcilib_kmem_get_ua(ctx->dmactx.pcilib, ctx->desc);
290
305
uint32_t *last_written_addr_ptr;
291
306
uint32_t last_written_addr;
308
323
// Find where the ring head is actually are
309
324
for (i = 0; i < ctx->ring_size; i++) {
310
uintptr_t bus_addr = pcilib_kmem_get_block_ba(ctx->pcilib, ctx->pages, i);
325
uintptr_t bus_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, i);
312
327
if (bus_addr == last_written_addr) {
313
328
status->ring_head = bus_addr;
373
388
err = dma_ipe_start(vctx, dma, PCILIB_DMA_FLAGS_DEFAULT);
374
389
if (err) return err;
376
desc_va = (void*)pcilib_kmem_get_ua(ctx->pcilib, ctx->desc);
391
desc_va = (void*)pcilib_kmem_get_ua(ctx->dmactx.pcilib, ctx->desc);
378
393
if (ctx->mode64) last_written_addr_ptr = desc_va + 3 * sizeof(uint32_t);
379
394
else last_written_addr_ptr = desc_va + 4 * sizeof(uint32_t);
428
443
#endif /* IPEDMA_DEBUG */
430
445
#ifdef IPEDMA_DETECT_PACKETS
431
if ((*empty_detected_ptr)&&(pcilib_kmem_get_block_ba(ctx->pcilib, ctx->pages, cur_read) == (*last_written_addr_ptr))) packet_flags = PCILIB_DMA_FLAG_EOP;
446
if ((*empty_detected_ptr)&&(pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, cur_read) == (*last_written_addr_ptr))) packet_flags = PCILIB_DMA_FLAG_EOP;
432
447
else packet_flags = 0;
433
448
#endif /* IPEDMA_DETECT_PACKETS */
435
pcilib_kmem_sync_block(ctx->pcilib, ctx->pages, PCILIB_KMEM_SYNC_FROMDEVICE, cur_read);
436
void *buf = pcilib_kmem_get_block_ua(ctx->pcilib, ctx->pages, cur_read);
450
pcilib_kmem_sync_block(ctx->dmactx.pcilib, ctx->pages, PCILIB_KMEM_SYNC_FROMDEVICE, cur_read);
451
void *buf = pcilib_kmem_get_block_ua(ctx->dmactx.pcilib, ctx->pages, cur_read);
437
452
ret = cb(cbattr, packet_flags, ctx->page_size, buf);
438
453
if (ret < 0) return -ret;
440
455
// DS: Fixme, it looks like we can avoid calling this for the sake of performance
441
// pcilib_kmem_sync_block(ctx->pcilib, ctx->pages, PCILIB_KMEM_SYNC_TODEVICE, cur_read);
456
// pcilib_kmem_sync_block(ctx->dmactx.pcilib, ctx->pages, PCILIB_KMEM_SYNC_TODEVICE, cur_read);
443
458
// Numbered from 1
444
459
WR(IPEDMA_REG_LAST_READ, cur_read + 1);
446
461
ctx->last_read = cur_read;
447
// ctx->last_read_addr = htonl(pcilib_kmem_get_block_ba(ctx->pcilib, ctx->pages, cur_read));
448
ctx->last_read_addr = pcilib_kmem_get_block_ba(ctx->pcilib, ctx->pages, cur_read);
462
// ctx->last_read_addr = htonl(pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, cur_read));
463
ctx->last_read_addr = pcilib_kmem_get_block_ba(ctx->dmactx.pcilib, ctx->pages, cur_read);
450
465
#ifdef IPEDMA_BUG_DMARD
451
466
FILE *f = fopen("/tmp/pcitool_lastread", "w");
482
497
WR(IPEDMA_REG_CONTROL, 0x0);
484
err = pcilib_skip_dma(ctx->pcilib, 0);
499
err = pcilib_skip_dma(ctx->dmactx.pcilib, 0);
486
501
pcilib_error("Can't start benchmark, devices continuously writes unexpected data using DMA engine");
500
515
WR(IPEDMA_REG_CONTROL, 0x1);
502
517
for (bytes = 0; bytes < size; bytes += rbytes) {
503
err = pcilib_read_dma(ctx->pcilib, 0, addr, size - bytes, buf + bytes, &rbytes);
518
err = pcilib_read_dma(ctx->dmactx.pcilib, 0, addr, size - bytes, buf + bytes, &rbytes);
505
520
pcilib_error("Can't read data from DMA, error %i", err);
514
529
gettimeofday(&cur, NULL);
515
530
us += ((cur.tv_sec - start.tv_sec)*1000000 + (cur.tv_usec - start.tv_usec));
517
err = pcilib_skip_dma(ctx->pcilib, 0);
532
err = pcilib_skip_dma(ctx->dmactx.pcilib, 0);
519
534
pcilib_error("Can't start iteration, devices continuously writes unexpected data using DMA engine");