Bug 699661: Avoid sharing pointers between pdf14 compositors

If a copdevice is triggered when the pdf14 compositor is the device, we make
a copy of the device, then throw an error because, by default we're only allowed
to copy the device prototype - then freeing it calls the finalize, which frees
several pointers shared with the parent.

Make a pdf14 specific finish_copydevice() which NULLs the relevant pointers,
before, possibly, throwing the same error as the default method.

This also highlighted a problem with reopening the X11 devices, where a custom
error handler could be replaced with itself, meaning it also called itself,
and infifite recursion resulted.

Keep a note of if the handler replacement has been done, and don't do it a
second time.

2 files changed, 24 insertions, 5 deletions

diff --git a/base/gdevp14.c b/base/gdevp14.c
index d9f8e794c..eb9cc23bc 100644
--- a/base/gdevp14.c
+++ b/base/gdevp14.c
@@ -178,6 +178,7 @@ static	dev_proc_fill_mask(pdf14_fill_mask);
 static  dev_proc_stroke_path(pdf14_stroke_path);
 static  dev_proc_begin_typed_image(pdf14_begin_typed_image);
 static  dev_proc_text_begin(pdf14_text_begin);
+static  dev_proc_finish_copydevice(pdf14_finish_copydevice);
 static  dev_proc_create_compositor(pdf14_create_compositor);
 static  dev_proc_create_compositor(pdf14_forward_create_compositor);
 static  dev_proc_begin_transparency_group(pdf14_begin_transparency_group);
@@ -245,7 +246,7 @@ static	const gx_color_map_procs *
         pdf14_create_compositor,        /* create_compositor */\
         NULL,                           /* get_hardware_params */\
         pdf14_text_begin,               /* text_begin */\
-        NULL,                           /* finish_copydevice */\
+        pdf14_finish_copydevice,        /* finish_copydevice */\
         pdf14_begin_transparency_group,\
         pdf14_end_transparency_group,\
         pdf14_begin_transparency_mask,\
@@ -3935,6 +3936,19 @@ pdf14_text_begin(gx_device * dev, gs_gstate * pgs,
     return code;
 }
 
+static  int
+pdf14_finish_copydevice(gx_device *new_dev, const gx_device *from_dev)
+{
+    pdf14_device *pdev = (pdf14_device*)new_dev;
+
+    pdev->ctx = NULL;
+    pdev->trans_group_parent_cmap_procs = NULL;
+    pdev->smaskcolor = NULL;
+
+    /* Only allow copying the prototype. */
+    return (from_dev->memory ? gs_note_error(gs_error_rangecheck) : 0);
+}
+
 /*
  * Implement copy_mono by filling lots of small rectangles.
  */
@@ -8093,6 +8107,7 @@ c_pdf14trans_clist_read_update(gs_composite_t *	pcte, gx_device	* cdev,
                        before reopening the device */
                     if (p14dev->ctx != NULL) {
                         pdf14_ctx_free(p14dev->ctx);
+                        p14dev->ctx = NULL;
                     }
                     dev_proc(tdev, open_device) (tdev);
                 }
diff --git a/devices/gdevxini.c b/devices/gdevxini.c
index 8511eac80..23b8c35a6 100644
--- a/devices/gdevxini.c
+++ b/devices/gdevxini.c
@@ -59,7 +59,8 @@ static struct xv_ {
     Boolean alloc_error;
     XErrorHandler orighandler;
     XErrorHandler oldhandler;
-} x_error_handler;
+    Boolean set;
+} x_error_handler = {0};
 
 static int
 x_catch_alloc(Display * dpy, XErrorEvent * err)
@@ -74,7 +75,8 @@ x_catch_alloc(Display * dpy, XErrorEvent * err)
 int
 x_catch_free_colors(Display * dpy, XErrorEvent * err)
 {
-    if (err->request_code == X_FreeColors)
+    if (err->request_code == X_FreeColors ||
+        x_error_handler.orighandler == x_catch_free_colors)
         return 0;
     return x_error_handler.orighandler(dpy, err);
 }
@@ -274,8 +276,10 @@ gdev_x_open(gx_device_X * xdev)
         return_error(gs_error_ioerror);
     }
     /* Buggy X servers may cause a Bad Access on XFreeColors. */
-    x_error_handler.orighandler = XSetErrorHandler(x_catch_free_colors);
-
+    if (!x_error_handler.set) {
+        x_error_handler.orighandler = XSetErrorHandler(x_catch_free_colors);
+        x_error_handler.set = True;
+    }
     /* Get X Resources.  Use the toolkit for this. */
     XtToolkitInitialize();
     app_con = XtCreateApplicationContext();