Implement Browser.GetImage on Linux (#427)

README.md

@@ -374,6 +374,7 @@ Additional information for v31.2 release:
   * [GetFrameCount](api/Browser.md#getframecount)
   * [GetFrameIdentifiers](api/Browser.md#getframeidentifiers)
   * [GetFrameNames](api/Browser.md#getframenames)
+  * [GetImage](api/Browser.md#getimage)
   * [GetJavascriptBindings](api/Browser.md#getjavascriptbindings)
   * [GetMainFrame](api/Browser.md#getmainframe)
   * [GetNSTextInputContext](api/Browser.md#getnstextinputcontext)

api/API-index.md

@@ -60,6 +60,7 @@
   * [GetFrameCount](Browser.md#getframecount)
   * [GetFrameIdentifiers](Browser.md#getframeidentifiers)
   * [GetFrameNames](Browser.md#getframenames)
+  * [GetImage](Browser.md#getimage)
   * [GetJavascriptBindings](Browser.md#getjavascriptbindings)
   * [GetMainFrame](Browser.md#getmainframe)
   * [GetNSTextInputContext](Browser.md#getnstextinputcontext)

api/Browser.md

@@ -39,6 +39,7 @@ Table of contents:
   * [GetFrameCount](#getframecount)
   * [GetFrameIdentifiers](#getframeidentifiers)
   * [GetFrameNames](#getframenames)
+  * [GetImage](#getimage)
   * [GetJavascriptBindings](#getjavascriptbindings)
   * [GetMainFrame](#getmainframe)
   * [GetNSTextInputContext](#getnstextinputcontext)
@@ -420,6 +421,29 @@ Returns the identifiers of all existing frames.
 Returns the names of all existing frames. This list does not include the main frame.
 
 
+### GetImage
+
+| | |
+| --- | --- |
+| __Return__ | tuple(bytes buffer, int width, int height) |
+
+Currently works only on Linux (Issue [#427](../../../issues/427)).
+
+Get browser contents as image. Only screen visible contents are returned.
+
+Returns an RGB buffer which can be converted to an image
+using PIL library with such code:
+
+```py
+from PIL import Image, ImageFile
+buffer_len = (width * 3 + 3) & -4
+image = Image.frombytes("RGB", (width, height), data,
+                         "raw", "RGB", buffer_len, 1)
+ImageFile.MAXBLOCK = width * height
+image.save("image.png", "PNG")
+```
+
+
 ### GetJavascriptBindings
 
 | | |

src/browser.pyx

@@ -274,6 +274,35 @@ cdef class PyBrowser:
     cpdef JavascriptBindings GetJavascriptBindings(self):
         return self.javascriptBindings
 
+    cdef bytes b(self, int x):
+        return struct.pack(b'<B', x)
+
+    cpdef object GetImage(self):
+        IF UNAME_SYSNAME == "Linux":
+            cdef XImage* image
+            image = x11.CefBrowser_GetImage(self.cefBrowser)
+            if not image:
+                return None
+            cdef int width = image.width
+            cdef int height = image.height
+            cdef list pixels = [b'0'] * (3 * width * height)
+            cdef int x, y, blue, green, red, offset
+            cdef unsigned long pixel
+            for x in range(width):
+                for y in range(height):
+                    pixel = XGetPixel(image, x, y)
+                    blue = pixel & 255
+                    green = (pixel & 65280) >> 8
+                    red = (pixel & 16711680) >> 16
+                    offset = (x + width * y) * 3
+                    pixels[offset:offset+3] = self.b(red), self.b(green),\
+                                              self.b(blue)
+            XDestroyImage(image)
+            return b''.join(pixels), width, height
+        ELSE:
+            NonCriticalError("GetImage not implemented on this platform")
+            return None
+
     # --------------
     # CEF API.
     # --------------

src/cef_v59..v66_changes.txt

@@ -60,6 +60,13 @@ internal/cef_types.h
   - + Remove: UR_FLAG_ALLOW_CACHED_CREDENTIALS
 
 
+TODO
+----
+1. Compare src/handler/dialog_handler_gtk.cpp (and .h) with upstream
+   cefclient files
+2. In subprocess/print_handler_gtk.cpp use GetWindow implementation
+   from x11.cpp
+
 NEW FEATURES
 ------------
 

src/cefpython.pyx

@@ -132,6 +132,8 @@ import json
 import datetime
 # noinspection PyUnresolvedReferences
 import random
+# noinspection PyUnresolvedReferences
+import struct
 
 if sys.version_info.major == 2:
     # noinspection PyUnresolvedReferences

src/client_handler/dialog_handler_gtk.cpp

@@ -138,49 +138,10 @@ void AddFilters(GtkFileChooser* chooser,
   }
 }
 
-GtkWindow* GetWindow(CefRefPtr<CefBrowser> browser) {
-  // -- REWRITTEN FOR CEF PYTHON USE CASE --
-  // X11 window handle
-  ::Window xwindow = browser->GetHost()->GetWindowHandle();
-  // X11 display
-  ::Display* xdisplay = cef_get_xdisplay();
-  // GDK display
-  GdkDisplay* gdk_display = NULL;
-  if (xdisplay) {
-    // See if we can find GDK display using X11 display
-    gdk_display = gdk_x11_lookup_xdisplay(xdisplay);
-  }
-  if (!gdk_display) {
-    // If not then get the default display
-    gdk_display = gdk_display_get_default();
-  }
-  if (!gdk_display) {
-    // The tkinter_.py and hello_world.py examples do not use GTK
-    // internally, so GTK wasn't yet initialized and must do it
-    // now, so that display is available. Also must install X11
-    // error handlers to avoid 'BadWindow' errors.
-    LOG(INFO) << "[Browser process] Initialize GTK";
-    gtk_init(0, NULL);
-    InstallX11ErrorHandlers();
-    // Now the display is available
-    gdk_display = gdk_display_get_default();
-  }
-  // In kivy_.py example getting error message:
-  // > Can't create GtkPlug as child of non-GtkSocket
-  // However dialog handler works just fine.
-  GtkWidget* widget = gtk_plug_new_for_display(gdk_display, xwindow);
-  // Getting top level widget doesn't seem to be required.
-  // OFF: GtkWidget* toplevel = gtk_widget_get_toplevel(widget);
-  GtkWindow* window = GTK_WINDOW(widget);
-  if (!window) {
-    LOG(ERROR) << "No GtkWindow for browser";
-  }
-  return window;
-}
-
 }  // namespace
 
 
+
 ClientDialogHandlerGtk::ClientDialogHandlerGtk()
     : gtk_dialog_(NULL) {
 }
@@ -238,7 +199,7 @@ bool ClientDialogHandlerGtk::OnFileDialog(
     }
   }
 
-  GtkWindow* window = GetWindow(browser);
+  GtkWindow* window = CefBrowser_GetGtkWindow(browser);
   if (!window)
     return false;
 
@@ -378,7 +339,7 @@ bool ClientDialogHandlerGtk::OnJSDialog(
     // title += CefFormatUrlForSecurityDisplay(origin_url).ToString();
   }
 
-  GtkWindow* window = GetWindow(browser);
+  GtkWindow* window = CefBrowser_GetGtkWindow(browser);
   if (!window)
     return false;
 

src/client_handler/x11.cpp

@@ -46,3 +46,65 @@ void SetX11WindowTitle(CefRefPtr<CefBrowser> browser, char* title) {
     ::Display* xdisplay = cef_get_xdisplay();
     XStoreName(xdisplay, xwindow, title);
 }
+
+GtkWindow* CefBrowser_GetGtkWindow(CefRefPtr<CefBrowser> browser) {
+  // -- REWRITTEN FOR CEF PYTHON USE CASE --
+  // X11 window handle
+  ::Window xwindow = browser->GetHost()->GetWindowHandle();
+  // X11 display
+  ::Display* xdisplay = cef_get_xdisplay();
+  // GDK display
+  GdkDisplay* gdk_display = NULL;
+  if (xdisplay) {
+    // See if we can find GDK display using X11 display
+    gdk_display = gdk_x11_lookup_xdisplay(xdisplay);
+  }
+  if (!gdk_display) {
+    // If not then get the default display
+    gdk_display = gdk_display_get_default();
+  }
+  if (!gdk_display) {
+    // The tkinter_.py and hello_world.py examples do not use GTK
+    // internally, so GTK wasn't yet initialized and must do it
+    // now, so that display is available. Also must install X11
+    // error handlers to avoid 'BadWindow' errors.
+    LOG(INFO) << "[Browser process] Initialize GTK";
+    gtk_init(0, NULL);
+    InstallX11ErrorHandlers();
+    // Now the display is available
+    gdk_display = gdk_display_get_default();
+  }
+  // In kivy_.py example getting error message:
+  // > Can't create GtkPlug as child of non-GtkSocket
+  // However dialog handler works just fine.
+  GtkWidget* widget = gtk_plug_new_for_display(gdk_display, xwindow);
+  // Getting top level widget doesn't seem to be required.
+  // OFF: GtkWidget* toplevel = gtk_widget_get_toplevel(widget);
+  GtkWindow* window = GTK_WINDOW(widget);
+  if (!window) {
+    LOG(ERROR) << "No GtkWindow for browser";
+  }
+  return window;
+}
+
+XImage* CefBrowser_GetImage(CefRefPtr<CefBrowser> browser) {
+    ::Display* display = cef_get_xdisplay();
+    if (!display) {
+        LOG(ERROR) << "XOpenDisplay failed in CefBrowser_GetImage";
+        return NULL;
+    }
+    ::Window browser_window = browser->GetHost()->GetWindowHandle();
+    XWindowAttributes attrs;
+    if (!XGetWindowAttributes(display, browser_window, &attrs)) {
+        LOG(ERROR) << "XGetWindowAttributes failed in CefBrowser_GetImage";
+        return NULL;
+    }
+    XImage* image = XGetImage(display, browser_window,
+                              0, 0, attrs.width, attrs.height,
+                              AllPlanes, ZPixmap);
+    if (!image) {
+        LOG(ERROR) << "XGetImage failed in CefBrowser_GetImage";
+        return NULL;
+    }
+    return image;
+}

src/client_handler/x11.h

@@ -5,9 +5,15 @@
 #pragma once
 
 #include <X11/Xlib.h>
+#include <gtk/gtk.h>
+#include <gdk/gdkx.h>
+
 #include "include/cef_browser.h"
 
 void InstallX11ErrorHandlers();
 void SetX11WindowBounds(CefRefPtr<CefBrowser> browser,
                         int x, int y, int width, int height);
 void SetX11WindowTitle(CefRefPtr<CefBrowser> browser, char* title);
+
+GtkWindow* CefBrowser_GetGtkWindow(CefRefPtr<CefBrowser> browser);
+XImage* CefBrowser_GetImage(CefRefPtr<CefBrowser> browser);

src/extern/linux.pxd

@@ -7,3 +7,26 @@ cdef extern from "gtk/gtk.h" nogil:
     ctypedef void* GtkWidget
     cdef GtkWidget* gtk_plug_new(GdkNativeWindow socket_id)
     cdef void gtk_widget_show(GtkWidget* widget)
+
+    ctypedef char* XPointer
+    ctypedef struct XImage:
+        int width
+        int height
+        int xoffset             # number of pixels offset in X direction
+        int format              # XYBitmap, XYPixmap, ZPixmap
+        char *data              # pointer to image data
+        int byte_order          # data byte order, LSBFirst, MSBFirst
+        int bitmap_unit         # quant. of scanline 8, 16, 32
+        int bitmap_bit_order    # LSBFirst, MSBFirst
+        int bitmap_pad          # 8, 16, 32 either XY or ZPixmap
+        int depth               # depth of image
+        int bytes_per_line      # accelerator to next scanline
+        int bits_per_pixel      # bits per pixel (ZPixmap)
+        unsigned long red_mask  # bits in z arrangement
+        unsigned long green_mask
+        unsigned long blue_mask
+        XPointer *obdata
+        void *funcs
+    void XDestroyImage(XImage *ximage)
+    unsigned long XGetPixel(XImage* image, int x, int y)
+

src/extern/x11.pxd

@@ -5,9 +5,11 @@
 from cef_ptr cimport CefRefPtr
 # noinspection PyUnresolvedReferences
 from cef_browser cimport CefBrowser
+from linux cimport XImage
 
 cdef extern from "client_handler/x11.h" nogil:
     void InstallX11ErrorHandlers()
     void SetX11WindowBounds(CefRefPtr[CefBrowser] browser,
                             int x, int y, int width, int height)
     void SetX11WindowTitle(CefRefPtr[CefBrowser] browser, char* title)
+    XImage* CefBrowser_GetImage(CefRefPtr[CefBrowser] browser)

tools/cython_setup.py

@@ -360,6 +360,9 @@ def get_libraries():
             "gobject-2.0",
             "glib-2.0",
             "gtk-x11-2.0",
+            "gdk-x11-2.0",
+            # "gdk_pixbuf-2.0",
+            # "gdk_pixbuf_xlib-2.0",
             # CEF and CEF Python libraries
             "cef_dll_wrapper",
             "cefpythonapp",