Resolution of the user-space BAR addresses

7 files changed, 76 insertions, 27 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cd889f0..33be2fb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -49,7 +49,7 @@ if (NOT DISABLE_PYTHON)
     
     execute_process (COMMAND ${PYTHON_EXECUTABLE} -c "import site; print site.PREFIXES[-1]" OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE)
     file (TO_CMAKE_PATH "${PYTHON_PREFIX}" PYTHON_PREFIX)
-    execute_process (COMMAND ${PYTHON_EXECUTABLE} -c "import site; print site.getsitepackages()[-1]" OUTPUT_VARIABLE PYTHON_SITE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process (COMMAND ${PYTHON_EXECUTABLE} -c "import site; print site.getsitepackages()[0]" OUTPUT_VARIABLE PYTHON_SITE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
     file (TO_CMAKE_PATH "${PYTHON_SITE_DIR}" PYTHON_SITE_DIR)
 
     string (REGEX REPLACE "^${PYTHON_PREFIX}/" "${CMAKE_INSTALL_PREFIX}/" PYTHON_SITE_DIR "${PYTHON_SITE_DIR}")
diff --git a/driver/Makefile b/driver/Makefile
index 0a860bf..8d8ada8 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -4,7 +4,8 @@ obj-m := pciDriver.o
 pciDriver-objs := base.o int.o umem.o kmem.o sysfs.o ioctl.o compat.o rdma.o
 
 KERNELDIR ?= /lib/modules/$(shell uname -r)/build
-INSTALLDIR ?= /lib/modules/$(shell uname -r)/kernel/extra
+INSTALLDIR ?= /lib/modules/$(shell uname -r)/extra
+MAININSTALLDIR ?= /lib/modules/$(shell uname -r)/kernel/extra
 PWD := $(shell pwd)
 
 EXTRA_CFLAGS += -I$(M)/..
@@ -53,13 +54,18 @@ install:
 	@mkdir -p $(INSTALLDIR)
 	@echo "INSTALL $(INSTALLDIR)/pciDriver.ko"
 	@install -m 755 pciDriver.ko $(INSTALLDIR)
-	@echo "INSTALL /usr/include/pciDriver/driver/pciDriver.h"
-	@mkdir -p /usr/include/pciDriver/driver
-	@install -m 644 pciDriver.h /usr/include/pciDriver/driver
+	@echo "INSTALL $(INSTALLDIR)/pciDriver.symvers"
+	@install -m 644 Module.symvers $(INSTALLDIR)/pciDriver.symvers
+#	@echo "INSTALL /usr/include/pciDriver/driver/pciDriver.h"
+#	@mkdir -p /usr/include/pciDriver/driver
+#	@install -m 644 pciDriver.h /usr/include/pciDriver/driver
 
 uninstall:
 	@echo "UNINSTALL $(INSTALLDIR)/pciDriver.ko"
 	@rm -f $(INSTALLDIR)/pciDriver.ko
+	@rm -f $(MAININSTALLDIR)/pciDriver.ko
+	@rm -f $(INSTALLDIR)/pciDriver.symvers
+	@rm -f $(MAININSTALLDIR)/pciDriver.symvers
 	@echo "UNINSTALL /usr/include/pciDriver/driver/pciDriver.h"
 	@rm -rf /usr/include/pciDriver/driver
 
diff --git a/driver/base.c b/driver/base.c
index 4e55dda..dfd82c6 100644
--- a/driver/base.c
+++ b/driver/base.c
@@ -211,7 +211,7 @@ MODULE_LICENSE("GPL v2");
 static struct class_compat *pcidriver_class;
 
 #ifdef PCIDRIVER_DUMMY_DEVICE
-pcidriver_privdata_t *pcidriver_privdata = NULL;
+pcidriver_privdata_t *pcidriver_dummydata = NULL;
 #endif /* PCIDRIVER_DUMMY_DEVICE */
 
 /**
@@ -225,6 +225,8 @@ static int __init pcidriver_init(void)
 
 	/* Initialize the device count */
 	atomic_set(&pcidriver_deviceCount, 0);
+	
+	memset(pcidriver_privdata, 0, sizeof(pcidriver_privdata));
 
 	/* Allocate character device region dynamically */
 	if ((err = alloc_chrdev_region(&pcidriver_devt, MINORNR, MAXDEVICES, NODENAME)) != 0) {
@@ -377,6 +379,8 @@ static int __devinit pcidriver_probe(struct pci_dev *pdev, const struct pci_devi
 		err = -ENOMEM;
 		goto probe_nomem;
 	}
+	
+	privdata->devid = devid;
 
 	INIT_LIST_HEAD(&(privdata->kmem_list));
 	spin_lock_init(&(privdata->kmemlist_lock));
@@ -387,7 +391,7 @@ static int __devinit pcidriver_probe(struct pci_dev *pdev, const struct pci_devi
 	atomic_set(&privdata->umem_count, 0);
 
 #ifdef PCIDRIVER_DUMMY_DEVICE
-	pcidriver_privdata = privdata;
+	pcidriver_dummydata = privdata;
 #else /* PCIDRIVER_DUMMY_DEVICE */
 	pci_set_drvdata(pdev, privdata);
 	privdata->pdev = pdev;
@@ -443,6 +447,8 @@ static int __devinit pcidriver_probe(struct pci_dev *pdev, const struct pci_devi
 		goto probe_cdevadd_fail;
 	}
 
+        pcidriver_privdata[devid] = privdata;
+
 	return 0;
 
 probe_device_create_fail:
@@ -473,13 +479,16 @@ static void __devexit pcidriver_remove(struct pci_dev *pdev)
 	pcidriver_privdata_t *privdata;
 
 #ifdef PCIDRIVER_DUMMY_DEVICE
-	privdata = pcidriver_privdata;
-	pcidriver_privdata = NULL;
+	privdata = pcidriver_dummydata;
+	pcidriver_dummydata = NULL;
 #else /* PCIDRIVER_DUMMY_DEVICE */
 	/* Get private data from the device */
 	privdata = pci_get_drvdata(pdev);
 #endif /* PCIDRIVER_DUMMY_DEVICE */
 
+            // Theoretically we should lock here and when using...
+        pcidriver_privdata[privdata->devid] = NULL;
+
 	/* Removing sysfs attributes from class device */
 	#define sysfs_attr(name) do { \
 			class_device_remove_file(sysfs_attr_def_pointer, &sysfs_attr_def_name(name)); \
@@ -729,3 +738,14 @@ int pcidriver_mmap_pci(pcidriver_privdata_t *privdata, struct vm_area_struct *vm
 	return 0;	/* success */
 #endif /* PCIDRIVER_DUMMY_DEVICE */
 }
+
+pcidriver_privdata_t *pcidriver_get_privdata(int devid) {
+    if (devid >= MAXDEVICES)
+        return NULL;
+
+    return pcidriver_privdata[devid];
+}
+
+void pcidriver_put_privdata(pcidriver_privdata_t *privdata) {
+
+}
diff --git a/driver/base.h b/driver/base.h
index 5b977fa..dce865f 100644
--- a/driver/base.h
+++ b/driver/base.h
@@ -69,6 +69,9 @@ static dev_t pcidriver_devt;
 /* Number of devices allocated */
 static atomic_t pcidriver_deviceCount;
 
+/* Private data for probed devices */
+static pcidriver_privdata_t* pcidriver_privdata[MAXDEVICES];
+
 /* Sysfs attributes */
 static DEVICE_ATTR(mmap_mode, 0664, pcidriver_show_mmap_mode, pcidriver_store_mmap_mode);
 static DEVICE_ATTR(mmap_area, 0664, pcidriver_show_mmap_area, pcidriver_store_mmap_area);
diff --git a/driver/common.h b/driver/common.h
index 6bc1329..a83fd5e 100644
--- a/driver/common.h
+++ b/driver/common.h
@@ -36,30 +36,28 @@ typedef struct {
 	struct list_head list;
 	unsigned int nr_pages;		/* number of pages for this user memeory area */
 	struct page **pages;		/* list of pointers to the pages */
-	unsigned int nents;			/* actual entries in the scatter/gatter list (NOT nents for the map function, but the result) */
+	unsigned int nents;		/* actual entries in the scatter/gatter list (NOT nents for the map function, but the result) */
 	struct scatterlist *sg;		/* list of sg entries */
 	struct class_device_attribute sysfs_attr;	/* initialized when adding the entry */
 } pcidriver_umem_entry_t;
 
 /* Hold the driver private data */
 typedef struct  {
-	dev_t devno;						/* device number (major and minor) */
+        int devid;                                      /* the device id */
+	dev_t devno;					/* device number (major and minor) */
 	struct pci_dev *pdev;				/* PCI device */
-	struct class_device *class_dev;		/* Class device */
-	struct cdev cdev;					/* char device struct */
-	int mmap_mode;						/* current mmap mode */
-	int mmap_area;						/* current PCI mmap area */
+	struct class_device *class_dev;                 /* Class device */
+	struct cdev cdev;				/* char device struct */
+	int mmap_mode;					/* current mmap mode */
+	int mmap_area;					/* current PCI mmap area */
 
 #ifdef ENABLE_IRQ
-	int irq_enabled;					/* Non-zero if IRQ is enabled */
-	int irq_count;						/* Just an IRQ counter */
-
-	wait_queue_head_t irq_queues[ PCIDRIVER_INT_MAXSOURCES ];
-										/* One queue per interrupt source */
-	atomic_t irq_outstanding[ PCIDRIVER_INT_MAXSOURCES ];
-										/* Outstanding interrupts per queue */
-	volatile unsigned int *bars_kmapped[6];		/* PCI BARs mmapped in kernel space */
+	int irq_enabled;				/* Non-zero if IRQ is enabled */
+	int irq_count;					/* Just an IRQ counter */
 
+	wait_queue_head_t irq_queues[ PCIDRIVER_INT_MAXSOURCES ];       /* One queue per interrupt source */
+	atomic_t irq_outstanding[ PCIDRIVER_INT_MAXSOURCES ];           /* Outstanding interrupts per queue */
+	volatile unsigned int *bars_kmapped[6];		                /* PCI BARs mmapped in kernel space */
 #endif
 	
 	spinlock_t kmemlist_lock;			/* Spinlock to lock kmem list operations */
@@ -81,6 +79,10 @@ typedef struct  {
 void pcidriver_module_get(pcidriver_privdata_t *privdata);
 void pcidriver_module_put(pcidriver_privdata_t *privdata);
 
+pcidriver_privdata_t *pcidriver_get_privdata(int devid);
+void pcidriver_put_privdata(pcidriver_privdata_t *privdata);
+
+
 /*************************************************************************/
 /* Some nice defines that make code more readable */
 /* This is to print nice info in the log */
diff --git a/driver/rdma.c b/driver/rdma.c
index 22a4a5e..78c6d69 100644
--- a/driver/rdma.c
+++ b/driver/rdma.c
@@ -7,7 +7,12 @@
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/hugetlb.h>
+#include <linux/cdev.h>
 
+#include "config.h"
+#include "compat.h"
+#include "pciDriver.h"
+#include "common.h"
 #include "rdma.h"
 
 static unsigned long pcidriver_follow_pte(struct mm_struct *mm, unsigned long address)
@@ -42,12 +47,28 @@ static unsigned long pcidriver_follow_pte(struct mm_struct *mm, unsigned long ad
 }
 
 unsigned long pcidriver_resolve_bar(unsigned long address) {
+    int dev, bar;
     unsigned long pfn;
 
     address = (address >> PAGE_SHIFT) << PAGE_SHIFT;
     pfn = pcidriver_follow_pte(current->mm, address);
 
-    return pfn;
+    for (dev = 0; dev < MAXDEVICES; dev++)
+    {
+        pcidriver_privdata_t *privdata =  pcidriver_get_privdata(dev);
+        if (!privdata) continue;
+
+        for (bar = 0; bar < 6; bar++)
+        {
+            unsigned long start = pci_resource_start(privdata->pdev, bar);
+            unsigned long end = start + pci_resource_len(privdata->pdev, bar);
+            if ((pfn >= start)&&(pfn < end))
+                return pfn;
+        }
+        pcidriver_put_privdata(privdata);
+    }
+
+    return 0;
 }
 
 EXPORT_SYMBOL(pcidriver_resolve_bar);
diff --git a/driver/rdma.h b/driver/rdma.h
index 4aeda78..cfe9c83 100644
--- a/driver/rdma.h
+++ b/driver/rdma.h
@@ -1,9 +1,6 @@
 #ifndef _PCIDRIVER_RDMA_H
 #define _PCIDRIVER_RDMA_H
 
-#include <linux/mm.h>
-#include <linux/pagemap.h>
-
 extern unsigned long pcidriver_resolve_bar(unsigned long address);
 
 #endif /* _PCIDRIVER_RDMA_H */