From 5cdb425d6843c100189611c2edb37b03efc1dc5a Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Tue, 14 Aug 2018 21:58:29 +0200 Subject: Make it compatible with CentOS 7.5 --- bin-centos/.gitignore | 2 + bin-fedora/.gitignore | 2 + config.sh | 17 ++- run-client.sh | 3 +- run-server.sh | 4 +- src/CMakeLists.txt | 12 +- src/ConfigReader/ConfigReader.h | 4 - src/DetectorModule/DetectorModule.cpp | 4 - src/ReceiverThreads.vma/ReceiverThreads.cpp | 166 ---------------------------- src/ReceiverThreads.vma/ReceiverThreads.h | 34 ------ src/UDPClient/UDPClient.cpp | 1 + src/UDPServer/UDPServer.cpp | 20 +++- src/cmake/FindLibConfig.cmake | 22 ++-- src/main_client.cpp | 18 +-- src/main_server.cpp | 23 +--- 15 files changed, 64 insertions(+), 268 deletions(-) create mode 100644 bin-centos/.gitignore create mode 100644 bin-fedora/.gitignore delete mode 100644 src/ReceiverThreads.vma/ReceiverThreads.cpp delete mode 100644 src/ReceiverThreads.vma/ReceiverThreads.h diff --git a/bin-centos/.gitignore b/bin-centos/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/bin-centos/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/bin-fedora/.gitignore b/bin-fedora/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/bin-fedora/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/config.sh b/config.sh index b43906c..c1c656b 100644 --- a/config.sh +++ b/config.sh @@ -1,2 +1,15 @@ -ods_path=/mnt/ands/ods/bin/ -vma_path=/mnt/ands +uname -r | grep el7 &> /dev/null +el7=$(($? == 0)) + +ods_path=/mnt/ands/ods/bin-fedora/ +vma_path=/mnt/ands/ +vma_lib_path=/mnt/ands/lib64-fedora/ + +[ $el7 -eq 1 ] && ods_path="${ods_path/fedora/centos}" +[ $el7 -eq 1 ] && vma_lib_path="${vma_lib_path/fedora/centos}" + +# Standard library +vma_lib=${vma_lib_path}/libvma.so.8.6.10 + +# With Mellanox OFED extensions (./configure --enable-socketxtreme) +#vma_lib=${vma_lib_path}/mlx/libvma.so.8.6.10 diff --git a/run-client.sh b/run-client.sh index 7ed9967..14f3b09 100755 --- a/run-client.sh +++ b/run-client.sh @@ -21,10 +21,9 @@ ip link set $int mtu $mtu if [ $mtu -le 1500 ]; then sed -i '' -e 's/numberOfProjectionsPerPacket.*/numberOfProjectionsPerPacket = 40/' config.cfg +# run "$ip" $first_port $num_ports 140000 run "$ip" $first_port $num_ports 120000 elif [ $mtu -eq 9000 ]; then sed -i '' -e 's/numberOfProjectionsPerPacket.*/numberOfProjectionsPerPacket = 250/' config.cfg run "$ip" $first_port $num_ports 20000 fi - - diff --git a/run-server.sh b/run-server.sh index 99330c2..8b04859 100755 --- a/run-server.sh +++ b/run-server.sh @@ -3,7 +3,9 @@ . config.sh function run { - VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=800000 LD_PRELOAD=$vma_path/lib64/libvma.so.8.6.10 $ods_path/bin/onlineDetectorSimulatorServer "$@" +# VMA_TRACELEVEL=DEBUG VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=800000 LD_PRELOAD=$vma_lib $ods_path/bin/onlineDetectorSimulatorServer "$@" + VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=800000 LD_PRELOAD=$vma_lib $ods_path/bin/onlineDetectorSimulatorServer "$@" +# $ods_path/bin/onlineDetectorSimulatorServer "$@" } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d4ee49d..92c4ca3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 2.8) project(OnlineDetectorSimulator) #define some speciel output paths for libraries and executables @@ -11,13 +11,16 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(Boost_USE_MULTITHREADED ON) -SET(BOOST_MIN_VERSION "1.58.0") -add_definitions(-DBOOST_LOG_DYN_LINK=1) +SET(BOOST_MIN_VERSION "1.53.0") + +add_definitions(-std=c++11) #find required packages find_package(LibConfig REQUIRED) -find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS log filesystem) +find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS filesystem) find_package(OpenMP) +set(LIBVMA_INCLUDE_DIR "/mnt/ands/include") + #add openmp if(OPENMP_FOUND) @@ -29,6 +32,7 @@ endif() include_directories( ${LIBCONFIGPP_INCLUDE_DIR} ${BOOST_INCLUDE_DIRS} + ${LIBVMA_INCLUDE_DIR} ) set(LINK_LIBRARIES ${LINK_LIBRARIES} diff --git a/src/ConfigReader/ConfigReader.h b/src/ConfigReader/ConfigReader.h index 2819f41..f654a5a 100644 --- a/src/ConfigReader/ConfigReader.h +++ b/src/ConfigReader/ConfigReader.h @@ -11,8 +11,6 @@ #define CONFIGREADER_H #pragma once -#include - #include #include @@ -26,7 +24,6 @@ public: template bool lookupValue(const std::string& identifier, T& value) { bool ret = cfg.lookupValue(identifier.c_str(), value); - BOOST_LOG_TRIVIAL(debug) << "Configuration value " << identifier << ": " << value; return ret; } @@ -35,7 +32,6 @@ public: libconfig::Setting& s = cfg.lookup(identifier.c_str()); if(s.getLength() > index){ value = s[index]; - BOOST_LOG_TRIVIAL(debug) << "Configuration value " << identifier << "[" << index << "]: " << value; return true; } return false; diff --git a/src/DetectorModule/DetectorModule.cpp b/src/DetectorModule/DetectorModule.cpp index 6767eb9..ff517fe 100644 --- a/src/DetectorModule/DetectorModule.cpp +++ b/src/DetectorModule/DetectorModule.cpp @@ -10,8 +10,6 @@ #include "DetectorModule.h" #include "../ConfigReader/ConfigReader.h" -#include - #include #include @@ -64,7 +62,6 @@ DetectorModule::DetectorModule(const int detectorID, const std::string& address, } auto DetectorModule::send(int packets = 1) -> void{ - BOOST_LOG_TRIVIAL(debug) << "Detectormodule " << detectorID_ << " :sending udp packet with index " << index_ << "."; int numberOfParts = numberOfProjections_/numberOfProjectionsPerPacket_; // sendBuffer_[0] = (sizeof(std::size_t)) & 0xff; // sendBuffer_[1] = (sizeof(std::size_t) >> 8) & 0xff; @@ -131,7 +128,6 @@ auto DetectorModule::readInput() -> void { path_.append("/"); //open file const std::string filePath = path_ + fileName_ + std::to_string(detectorID_+1) + fileEnding_; - BOOST_LOG_TRIVIAL(debug) << "DetectorModule: Path = " << filePath; std::ifstream input(filePath, std::ios::in | std::ios::binary); if(input){ //allocate memory in vector diff --git a/src/ReceiverThreads.vma/ReceiverThreads.cpp b/src/ReceiverThreads.vma/ReceiverThreads.cpp deleted file mode 100644 index 650a840..0000000 --- a/src/ReceiverThreads.vma/ReceiverThreads.cpp +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2016 - * - * ReceiverThreads.cpp - * - * Created on: 21.07.2016 - * Author: Tobias Frust - */ - -#include "ReceiverThreads.h" -#include "../UDPServer/UDPServer.h" - -#include - -ReceiverThreads::ReceiverThreads(const std::string& address, const int timeIntervall, const int numberOfDetectorModules) - : timeIntervall_{timeIntervall}, numberOfDetectorModules_{numberOfDetectorModules}, address_{address}, loss_{0} { - - for(auto i = 0; i < numberOfDetectorModules; i++){ - receiverModules_.emplace_back(&ReceiverThreads::receiverThread, this, 4000+i); - } - - for(auto i = 0; i < numberOfDetectorModules; i++){ - receiverModules_[i].join(); - } - -} - -auto ReceiverThreads::receiverThread(const int port) -> void { - int max_packets = 100; - int max_packet_size = 65535; - - - struct vma_ap_t *vma = vma_get_api(); - if (!vma) throw "Can't get LibVMA API" - - struct vma_packet_desc_t vma_packet; - struct vma_buff_t *vma_buf; - int vma_buf_offset; - - int vma_ring_fd; - vma->get_socket_rings_fds(fd?, &vma_ring_fd, 1); - if (vma_ring_fd < 0) throw "Can't get ring fds"; - - while (true) { - // free vma packets - - - struct vma_completion_t vma_comps[max_packets]; - int n_packets = vma->socketxtreme_poll(vma_ring_fd, vma_comps, max_packets, 0); - if (!n_packets) continue; - - for (int i = 0; i < n_packets; i++) { - switch (vma_comps[i].events) { - case VMA_SOCKETXTREME_PACKET: - break; - case EPOLLERR: - case EPOLLRDHUP: - printf("Polling error event=0x%lx user_data=%ld\n", vma_comps[i].events, vma_comps[i].user_data); - throw "Polling error"; - default: - printf("Unsupported event=0x%lx user_data=%ld\n", vma_comps[i].events, vma_comps[i].user_data); - throw "Polling error"; - } - - int packet_len = vma_comps[i].packet.total_len; - int n_buffs = vma_comps[i].packet.num_bufs; - - vma_comps.packet.buff_lst - - vma_comps[i].packet.buff_lst->payload -/* -<------><------><------><------><------>conn = conns_in[vma_comps.user_data] vma_comps.packet.total_len; -<------><------><------><------><------>conn->vma_packet.num_bufs = vma_comps.packet.num_bufs; -<------><------><------><------><------>conn->vma_packet.total_len = vma_comps.packet.total_len; -<------><------><------><------><------>conn->vma_packet.buff_lst = vma_comps.packet.buff_lst; -<------><------><------><------><------>conn->vma_buf = conn->vma_packet.buff_lst; -<------><------><------><------><------>conn->vma_buf_offset = 0; -*/ -/* - ret = _min((_config.msg_size - conn->msg_len), (conn->vma_buf->len - conn->vma_buf_offset)); - memcpy(((uint8_t *)msg_hdr) + conn->msg_len, - ((uint8_t *)conn->vma_buf->payload) + conn->vma_buf_offset, - ret); -*/ - _vma_api->socketxtreme_free_vma_packets(&conn->vma_packet, 1); - } - - -auto ReceiverThreads::receiverThread(const int port) -> void { - int max_packets = 100; - int max_packet_size = 65535; - - UDPServer server = UDPServer(address_, port); - std::vector> buffers; - - std::size_t rcv_index = 0; - std::size_t rcv_packets = 0; - std::size_t rcv_size = 0; - - std::size_t lastIndex{0}; - std::size_t loss = 0; - - struct mmsghdr msg[max_packets]; - struct iovec msgvec[max_packets]; - - buffers.resize(max_packets); - - memset(msg, 0, sizeof(msg)); - memset(msgvec, 0, sizeof(msgvec)); - for (int i = 0; i < max_packets; i++) { - buffers[i].resize(max_packet_size); - - msgvec[i].iov_base = buffers[i].data(); - msgvec[i].iov_len = buffers[i].size(); - msg[i].msg_hdr.msg_iov = &msgvec[i]; - msg[i].msg_hdr.msg_iovlen = 1; - } - - - BOOST_LOG_TRIVIAL(info) << "Address: " << address_ << " port: " << port << " timeout: " << timeIntervall_; - - double coef = 1000. * 1000. * 1000. / 1024. / 1024. / 1024.; - auto ts_last = std::chrono::high_resolution_clock::now(); - while(true){ - int packets = server.mrecv(max_packets, msg, 1); //timeIntervall_); - - if (packets >= 0) { - for (int i = 0; i < packets; i++) { - int bytes = msg[i].msg_len; - unsigned short *buf = reinterpret_cast(msgvec[i].iov_base); - - rcv_packets++; - rcv_size += bytes; - -// BOOST_LOG_TRIVIAL(debug) << "Received " << bytes << " Bytes."; - std::size_t index =*((std::size_t *)buf); - int diff = index - lastIndex - 1; - if(diff > 0){ - loss += diff; - BOOST_LOG_TRIVIAL(debug) << "Packet loss or wrong order! new: " << index << " old: " << lastIndex; - } - -/* if (port == 4000) { - printf("%i:%i:%i:%i,", index, diff, loss, i); - }*/ - - lastIndex = index; - } - } - - auto ts = std::chrono::high_resolution_clock::now(); - std::chrono::nanoseconds d = ts - ts_last; - if (d.count() >= 1000000000) { - printf("Lost %.2lf%, Received: %i (%zu bytes, %.3lf GBit/s) in %.3lf ms\n", loss / (double)(lastIndex - rcv_index)*100.0, rcv_packets, rcv_size, 8. * rcv_size * coef / d.count() , 1. * d.count() / 1000000); - rcv_packets = 0; - rcv_size = 0; - rcv_index = lastIndex; - loss = 0; - ts_last = ts; - } - } - - BOOST_LOG_TRIVIAL(info) << "Lost " << loss << " from " << lastIndex << " packets; (" << loss/(double)lastIndex*100.0 << "%)"; - loss_ += loss; -} - diff --git a/src/ReceiverThreads.vma/ReceiverThreads.h b/src/ReceiverThreads.vma/ReceiverThreads.h deleted file mode 100644 index 3cc986c..0000000 --- a/src/ReceiverThreads.vma/ReceiverThreads.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2016 - * - * ReceiverThreads.h - * - * Created on: 21.07.2016 - * Author: Tobias Frust - */ - -#ifndef RECEIVERTHREADS_H_ -#define RECEIVERTHREADS_H_ - -#include -#include - -class ReceiverThreads { -public: - ReceiverThreads(const std::string& address, const int timeIntervall, const int numberOfDetectorModules); - - auto run() -> void; -private: - auto receiverThread(const int port) -> void; - - std::vector receiverModules_; - - std::size_t loss_; - - int timeIntervall_; - int numberOfDetectorModules_; - - std::string address_; -}; - -#endif /* RECEIVERTHREADS_H_ */ diff --git a/src/UDPClient/UDPClient.cpp b/src/UDPClient/UDPClient.cpp index b9d55d0..803b9f1 100644 --- a/src/UDPClient/UDPClient.cpp +++ b/src/UDPClient/UDPClient.cpp @@ -40,6 +40,7 @@ #include "UDPClient.h" +#include #include #include diff --git a/src/UDPServer/UDPServer.cpp b/src/UDPServer/UDPServer.cpp index 42166b4..d2d1988 100644 --- a/src/UDPServer/UDPServer.cpp +++ b/src/UDPServer/UDPServer.cpp @@ -46,6 +46,7 @@ #include #include +#include #ifndef SOCK_CLOEXEC #define SOCK_CLOEXEC 0 @@ -55,6 +56,7 @@ UDPServer::UDPServer(const std::string& addr, int port) : f_port(port) , f_addr(addr) { + char errno_str[16]; char decimal_port[16]; snprintf(decimal_port, sizeof(decimal_port), "%d", f_port); decimal_port[sizeof(decimal_port) / sizeof(decimal_port[0]) - 1] = '\0'; @@ -69,11 +71,27 @@ UDPServer::UDPServer(const std::string& addr, int port) throw udp_client_server_runtime_error(("invalid address or port for UDP socket: \"" + addr + ":" + decimal_port + "\"").c_str()); } f_socket = socket(f_addrinfo->ai_family, SOCK_DGRAM | SOCK_CLOEXEC, IPPROTO_UDP); + // First call fails with LibVMA for some reason? + if (f_socket == -1) + f_socket = socket(f_addrinfo->ai_family, SOCK_DGRAM | SOCK_CLOEXEC, IPPROTO_UDP); if(f_socket == -1) { + sprintf(errno_str, "%d", errno); freeaddrinfo(f_addrinfo); - throw udp_client_server_runtime_error(("could not create UDP socket for: \"" + addr + ":" + decimal_port + "\"").c_str()); + throw udp_client_server_runtime_error(("could not create UDP socket for: \"" + addr + ":" + decimal_port + "\", errno: " + errno_str).c_str()); } + +/* + const char *strdev = "enp5s0d1"; + r = setsockopt(f_socket, SOL_SOCKET, SO_BINDTODEVICE, (void *)strdev, strlen(strdev)); + if(r != 0) + { + freeaddrinfo(f_addrinfo); + close(f_socket); + throw udp_client_server_runtime_error(("could not associate socket : \"" + addr + ":" + decimal_port + "\" with device \"" + strdev + "\"").c_str()); + } +*/ + r = bind(f_socket, f_addrinfo->ai_addr, f_addrinfo->ai_addrlen); if(r != 0) { diff --git a/src/cmake/FindLibConfig.cmake b/src/cmake/FindLibConfig.cmake index 436d2a5..a888412 100644 --- a/src/cmake/FindLibConfig.cmake +++ b/src/cmake/FindLibConfig.cmake @@ -31,22 +31,22 @@ FIND_LIBRARY(LIBCONFIGPP_LIBRARY config++ /usr/lib ) -FIND_LIBRARY(LIBCONFIG_STATIC_LIBRARY "libconfig${CMAKE_STATIC_LIBRARY_SUFFIX}" - /usr/local/lib - /usr/lib -) +#FIND_LIBRARY(LIBCONFIG_STATIC_LIBRARY "libconfig${CMAKE_STATIC_LIBRARY_SUFFIX}" +# /usr/local/lib +# /usr/lib +#) -FIND_LIBRARY(LIBCONFIGPP_STATIC_LIBRARY "libconfig++${CMAKE_STATIC_LIBRARY_SUFFIX}" - /usr/local/lib - /usr/lib -) +#FIND_LIBRARY(LIBCONFIGPP_STATIC_LIBRARY "libconfig++${CMAKE_STATIC_LIBRARY_SUFFIX}" +# /usr/local/lib +# /usr/lib +#) IF(LIBCONFIG_INCLUDE_DIR) IF(LIBCONFIG_LIBRARY) SET(LIBCONFIG_FOUND TRUE) SET(LIBCONFIG_LIBRARIES ${LIBCONFIG_LIBRARY}) - SET(LIBCONFIG_STATIC_LIBRARY_PATH ${LIBCONFIG_STATIC_LIBRARY}) +# SET(LIBCONFIG_STATIC_LIBRARY_PATH ${LIBCONFIG_STATIC_LIBRARY}) ENDIF(LIBCONFIG_LIBRARY) ENDIF(LIBCONFIG_INCLUDE_DIR) @@ -54,7 +54,7 @@ IF(LIBCONFIGPP_INCLUDE_DIR) IF(LIBCONFIGPP_LIBRARY) SET(LIBCONFIGPP_FOUND TRUE) SET(LIBCONFIGPP_LIBRARIES ${LIBCONFIGPP_LIBRARY}) - SET(LIBCONFIGPP_STATIC_LIBRARY_PATH ${LIBCONFIGPP_STATIC_LIBRARY}) +# SET(LIBCONFIGPP_STATIC_LIBRARY_PATH ${LIBCONFIGPP_STATIC_LIBRARY}) ENDIF(LIBCONFIGPP_LIBRARY) ENDIF(LIBCONFIGPP_INCLUDE_DIR) @@ -62,7 +62,7 @@ IF (LIBCONFIG_FOUND) IF (NOT LibConfig_FIND_QUIETLY) MESSAGE(STATUS "Found LibConfig++: ${LIBCONFIGPP_LIBRARIES}" ) MESSAGE(STATUS "Found LibConfig: ${LIBCONFIG_LIBRARIES}") - MESSAGE(STATUS "static LibConfig path: ${LIBCONFIG_STATIC_LIBRARY_PATH}") +# MESSAGE(STATUS "static LibConfig path: ${LIBCONFIG_STATIC_LIBRARY_PATH}") ENDIF (NOT LibConfig_FIND_QUIETLY) ELSE (LIBCONFIG_FOUND) IF (LibConfig_FIND_REQUIRED) diff --git a/src/main_client.cpp b/src/main_client.cpp index fb0c1e3..28c472c 100644 --- a/src/main_client.cpp +++ b/src/main_client.cpp @@ -2,28 +2,14 @@ #include "DetectorModule/DetectorModule.h" #include "Detector/Detector.h" -#include -#include -#include - #include #include -void initLog() { -/* -#ifndef NDEBUG - boost::log::core::get()->set_filter(boost::log::trivial::severity >= boost::log::trivial::debug); -#else - boost::log::core::get()->set_filter(boost::log::trivial::severity >= boost::log::trivial::info); -#endif -*/ - boost::log::core::get()->set_filter(boost::log::trivial::severity >= boost::log::trivial::info); -} int main (int argc, char *argv[]){ if(argc < 5){ - BOOST_LOG_TRIVIAL(error) << "Program usage: ./onlineDetectorSimulatorClient
"; + std::cerr << "Program usage: ./onlineDetectorSimulatorClient
"; return 0; } @@ -36,8 +22,6 @@ int main (int argc, char *argv[]){ double timegap = 1./(double)imagesPerSec; unsigned int intervall = timegap*1000*1000; - initLog(); - std::cout << "Sending UDP packages: " << std::endl; auto configPath = std::string { "config.cfg" }; diff --git a/src/main_server.cpp b/src/main_server.cpp index b2f9425..7c6cee4 100644 --- a/src/main_server.cpp +++ b/src/main_server.cpp @@ -1,25 +1,10 @@ #include "UDPServer/UDPServer.h" #include "ReceiverThreads/ReceiverThreads.h" -#include -#include -#include - #include #include #include -void initLog() { -/* -#ifndef NDEBUG - boost::log::core::get()->set_filter(boost::log::trivial::severity >= boost::log::trivial::debug); -#else - boost::log::core::get()->set_filter(boost::log::trivial::severity >= boost::log::trivial::info); -#endif -*/ - boost::log::core::get()->set_filter(boost::log::trivial::severity >= boost::log::trivial::info); -} - void start(std::function func){ std::thread([func]() { while (true) @@ -31,10 +16,8 @@ void start(std::function func){ int main (int argc, char *argv[]){ - initLog(); - if(argc < 4){ - BOOST_LOG_TRIVIAL(error) << "Program usage: ./onlineDetectorSimulatorServer
"; + std::cerr << "Program usage: ./onlineDetectorSimulatorServer
"; return 0; } @@ -66,12 +49,8 @@ int main (int argc, char *argv[]){ // std::size_t index = *((std::size_t *)buf.data()); // if(index%1000 == 99) printf("%lu\n", index); // -// if(lastIndex != (index-1)) -// BOOST_LOG_TRIVIAL(warning) << "Packet loss or wrong order!"; -// // lastIndex = index; // -// BOOST_LOG_TRIVIAL(debug) << "Server: Received " << bytes << " Bytes with Index " << index; // } return 0; -- cgit v1.2.1