Merging upstream changes in develop tree since 1.3.2

 - no fork during DMA, it can lead to corrupt data
 - Bug fix: vedl_wait_exception() can return without an exception.
 - Bug fix: veorun consumes 100% CPU time. VEO must send "Blocking
   Sysem Call" message to VEOS on BLOCK.
This patch will be reverted once re-syncing with upstream but
currently fixes some important issues.

Makefile.am

@@ -14,5 +14,5 @@ edit = $(SED) -e 's|@PACKAGE[@]|@PACKAGE@|g' \
               -e 's|@VEORUN_BIN[@]|@VEORUN_BIN@|g' \
               -e 's|@VEOS_SOCKET[@]|@VEOS_SOCKET@|g'
 
-veo.spec: $(srcdir)/veo.spec.in
+veo.spec: $(srcdir)/veo.spec.in Makefile
 	$(edit) $< > $@

doc/GettingStarted.md

@@ -83,7 +83,7 @@ int main()
   struct veo_args *argp = veo_args_alloc();
   uint64_t id = veo_call_async(ctx, sym, argp);
   uint64_t retval;
-  veo_wait_result(ctx, id, &retval);
+  veo_call_wait_result(ctx, id, &retval);
   veo_args_free(argp);
   veo_context_close(ctx);
   return 0;
@@ -106,13 +106,13 @@ To execute a VE function with VEO:
  See the next chapter "Various Arguments for a VE function" in detail.
 5. Call a VE function by veo_call_async() with an address of a function
  and a VEO arguments object. A request ID is returned.
-6. Wait for the completion and get the return value by veo_wait_result().
+6. Wait for the completion and get the return value by veo_call_wait_result().
 
 ### Compile VH Main Program
 Compile source code on VH side as shown below.
 
 ~~~
-$ gcc -o hello hello.c -I/opt/nec/ve/veos/include -L/opt/nec/ve/veo/lib64 \
+$ gcc -o hello hello.c -I/opt/nec/ve/veos/include -L/opt/nec/ve/veos/lib64 \
    -Wl,-rpath=/opt/nec/ve/veos/lib64 -lveo
 ~~~
 

src/libveo/ProcHandle.cpp

@@ -30,7 +30,7 @@ extern "C" {
 #undef new
 
 /* symbols required but undefined in libvepseudo */
-__thread veos_handle *g_handle;
+extern __thread veos_handle *g_handle;
 extern struct tid_info global_tid_info[VEOS_MAX_VE_THREADS];
 
 int init_stack_veo(veos_handle*, int, char**, char**, struct ve_start_ve_req_cmd *);
@@ -106,6 +106,50 @@ void pseudo_abort()
 #endif
 
 namespace veo {
+/**
+ * @brief Initializes rwlock on DMA and fork
+ *
+ * This function initiazes a lock which will be used to synchronize
+ * request related to DMA(reading data from VE memory) and creating new
+ * process(fork/vfork). Execution of both requests parallely leads to
+ * memory curruption.
+ *
+ * @return abort on failure.
+ */
+void init_rwlock_to_sync_dma_fork()
+{
+  // copied from pseudo_process.c
+  int ret = -1;
+  pthread_rwlockattr_t sync_fork_dma_attr;
+
+  ret = pthread_rwlockattr_init(&sync_fork_dma_attr);
+  if (ret) {
+    PSEUDO_ERROR("Failed to initialize attribute %s", strerror(ret));
+    fprintf(stderr, "VE process setup failed\n");
+    pseudo_abort();
+  }
+
+  ret = pthread_rwlockattr_setkind_np(&sync_fork_dma_attr,
+          PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+  if (ret) {
+    PSEUDO_ERROR("Failed to set rwlock attribute: %s", strerror(ret));
+    fprintf(stderr, "VE process setup failed\n");
+    pseudo_abort();
+  }
+
+  ret = pthread_rwlock_init(&sync_fork_dma, &sync_fork_dma_attr);
+  if (ret) {
+    PSEUDO_ERROR("Failed to init rwlock %s", strerror(ret));
+    fprintf(stderr, "VE process setup failed\n");
+    pseudo_abort();
+  }
+
+  ret = pthread_rwlockattr_destroy(&sync_fork_dma_attr);
+  if (ret) {
+    PSEUDO_ERROR("Failed to destroy rwlock attribute: %s", strerror(ret));
+  }
+}
+
 /**
  * @brief create a VE process and initialize a thread context
  *
@@ -144,6 +188,13 @@ int spawn_helper(ThreadContext *ctx, veos_handle *oshandle, const char *binname)
 
   // Set global TID array for main thread.
   global_tid_info[0].vefd = oshandle->ve_handle->vefd;
+  global_tid_info[0].veos_hndl = oshandle;
+  tid_counter = 0;
+  global_tid_info[0].tid_val = getpid();// main thread
+  global_tid_info[0].flag = 0;
+  global_tid_info[0].mutex = PTHREAD_MUTEX_INITIALIZER;
+  global_tid_info[0].cond = PTHREAD_COND_INITIALIZER;
+  init_rwlock_to_sync_dma_fork();
   // Initialize the syscall argument area.
   rv = init_lhm_shm_area(oshandle);
   if (rv < 0) {
@@ -267,6 +318,16 @@ ProcHandle::ProcHandle(const char *ossock, const char *vedev,
     throw VEOException("Failed to receive data from VE");
   }
   VEO_ASSERT(this->funcs.version == VEORUN_VERSION);
+#define DEBUG_PRINT_HELPER(ctx, data, member) \
+  VEO_DEBUG(ctx, #member " = %#lx", data.member)
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, version);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, load_library);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, alloc_buff);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, free_buff);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, find_sym);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, create_thread);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, call_func);
+  DEBUG_PRINT_HELPER(this->main_thread.get(), this->funcs, exit);
   // create worker
   CallArgs args_create_thread;
   this->main_thread->_doCall(this->funcs.create_thread, args_create_thread);

src/libveo/ThreadContext.cpp

@@ -115,9 +115,19 @@ int ThreadContext::handleSingleException(uint64_t &exs, SyscallFilter filter)
   int ret;
   int break_flag = 0;
   VEO_TRACE(this, "%s()", __func__);
-  ret = vedl_wait_exception(this->os_handle->ve_handle, &exs);
-  if (ret != 0) {
-    throw VEOException("vedl_wait_exception failed", errno);
+  constexpr uint64_t VEO_EXCEPTION_MASK = ~0xffUL;
+  for (;;) {
+    ret = vedl_wait_exception(this->os_handle->ve_handle, &exs);
+    if (ret != 0) {
+      throw VEOException("vedl_wait_exception failed", errno);
+    }
+    if (!(exs & VEO_EXCEPTION_MASK)) { // no exceptions
+      VEO_DEBUG(this, "No exception; exs = %lx", exs);
+      // vedl_wait_exception() can return when no exceptions are raised.
+      // Retry in such case.
+      continue;
+    } else
+      break;
   }
   VEO_TRACE(this, "exs = 0x%016lx", exs);
   if (exs & EXS_MONC) {
@@ -172,6 +182,7 @@ bool ThreadContext::defaultFilter(int sysnum, int *break_flag)
     return true;
   }
   if (internal::is_veo_block(this->os_handle->ve_handle, sysnum)) {
+    block_syscall_req_ve_os(this->os_handle);// notify VEOS of BLOCKED state.
     *break_flag = VEO_HANDLER_STATUS_BLOCK_REQUESTED;
     this->state = VEO_STATE_BLOCKED;
     return true;

src/libveo/libveo.map

@@ -38,6 +38,7 @@ VERSION_1 {
     global_tid_info;
     load_elf;
     tid_counter_mutex;
+    sync_fork_dma;
     ve_info;
     ve_page_info;
     ve_proc_sigmask;

src/libveo/vars.c

@@ -10,9 +10,11 @@
 #include "vemva_mgmt.h"
 #include "loader.h"
 __thread sigset_t ve_proc_sigmask;
+__thread veos_handle *g_handle;
 __thread struct _ve_page_info ve_page_info;
 pthread_mutex_t tid_counter_mutex = PTHREAD_MUTEX_INITIALIZER;
 struct tid_info global_tid_info[VEOS_MAX_VE_THREADS];
+pthread_rwlock_t sync_fork_dma;
 struct ve_address_space_info ve_info;
 uint64_t default_page_size;
 struct ve_load_data load_elf;