Fix crash when SHM not available

Enhance debug logging of SHM related functions

common/os_calls.c

@@ -3168,12 +3168,12 @@ g_text2bool(const char *s)
 }
 
 /*****************************************************************************/
-/* returns pointer or nil on error */
+/* returns pointer or -1 on error */
 void * APP_CC
 g_shmat(int shmid)
 {
 #if defined(_WIN32)
-    return 0;
+    return -1;
 #else
      return shmat(shmid, 0, 0);
 #endif

xup/xup.c

@@ -31,6 +31,7 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <netdb.h>
+#include <errno.h>
 
 /******************************************************************************/
 /**
@@ -1232,6 +1233,11 @@ process_server_paint_rect_shmem(struct mod *mod, struct stream *s)
         {
             mod->screen_shmem_id = shmem_id;
             mod->screen_shmem_pixels = g_shmat(mod->screen_shmem_id);
+            if (mod->screen_shmem_pixels == (void *)-1)
+            {
+		mod->screen_shmem_pixels = 0;
+                log_message(LOG_LEVEL_ERROR, "g_shmat() returned error, errno: %d\nIs X11rdp running on the same host as xrdp?\n", errno);
+            }
         }
         if (mod->screen_shmem_pixels != 0)
         {
@@ -1341,6 +1347,11 @@ process_server_paint_rect_shmem_ex(struct mod *amod, struct stream *s)
         {
             amod->screen_shmem_id = shmem_id;
             amod->screen_shmem_pixels = g_shmat(amod->screen_shmem_id);
+            if (amod->screen_shmem_pixels == (void *)-1)
+            {
+		amod->screen_shmem_pixels = 0;
+                log_message(LOG_LEVEL_ERROR, "g_shmat() returned error, errno: %d\nIs X11rdp running on the same host as xrdp?\n", errno);
+            }
         }
         if (amod->screen_shmem_pixels != 0)
         {