Support for transparency in images

The ECMA48 sixel parser now recognizes transparency (the missing
pixels kind, not alpha blending).

When using the Swing backend, if jexer.Swing.imagesOverText is true
then text glyphs will be rendered underneath image cells.  This is
defaulted to false because of the hideous performance cost at the
moment, especially against sixel animations in terminal windows.

ECMA48 backend is still unaware of transparency.  I would need to
define a font with fallback and render images-over-text to that, which
is slightly more than I want to handle at the moment.  (Though I do
use a similar method for VT100 double-width / double-height inside the
terminal window.)

Jexer PNG images in terminal windows also handle 24-bit +
transparency.  One can use the 'imgls' script in the examples
directory to try this out.

README.md

@@ -136,7 +136,16 @@ The color wheel with that palette is shown below:
 
 ![Sixel Color Wheel](/screenshots/sixel_color_wheel.png?raw=true "Sixel Color Wheel")
 
+There is experimental support, only on the Swing backend, for
+rendering the text underneath images (jexer.Swing.imagesOverText).
+This is currently very not optimized -- cells are rendered below
+images irregardless if they are actually fully covered by pixels --
+but it is very cool looking.  (And if you like this, then you need to
+go check out [notcurses](https://github.com/dankamongmen/notcurses)
+poste haste.)  This is most visible in terminal windows with sixel and
+PNG images.
 
+![Hello notcurses! 🤘](/screenshots/for_nick.png?raw=true "Hello notcurses! 🤘🙂")
 
 Terminal Support
 ----------------

src/jexer/backend/LogicalScreen.java

@@ -353,7 +353,7 @@ public class LogicalScreen implements Screen {
         }
 
         if ((X >= 0) && (X < width) && (Y >= 0) && (Y < height)) {
-            logical[X][Y].setTo(attr);
+            logical[X][Y].setAttr(attr, true);
 
             // If this happens to be the cursor position, make the position
             // dirty.

src/jexer/backend/SwingTerminal.java

@@ -313,6 +313,12 @@ public class SwingTerminal extends LogicalScreen
      */
     private boolean mouse3 = false;
 
+    /**
+     * If true, draw text glyphs underneath images on cells.  This is
+     * expensive.
+     */
+    private boolean imagesOverText = false;
+
     // ------------------------------------------------------------------------
     // Constructors -----------------------------------------------------------
     // ------------------------------------------------------------------------
@@ -677,6 +683,13 @@ public class SwingTerminal extends LogicalScreen
 
         setMouseStyle(System.getProperty("jexer.Swing.mouseStyle", "default"));
 
+        if (System.getProperty("jexer.Swing.imagesOverText",
+                "false").equals("true")) {
+            imagesOverText = true;
+        } else {
+            imagesOverText = false;
+        }
+
         // Set custom colors
         setCustomSystemColors();
     }
@@ -1323,7 +1336,6 @@ public class SwingTerminal extends LogicalScreen
             " " + cell);
         */
 
-        // Draw the background rectangle, then the foreground character.
         assert (cell.isImage());
 
         BufferedImage image = cell.getImage();
@@ -1580,6 +1592,10 @@ public class SwingTerminal extends LogicalScreen
                         || (swing.getFrame() == null)) {
 
                         if (lCell.isImage()) {
+                            if (imagesOverText) {
+                                // Draw the glyph underneath the image.
+                                drawGlyph(gr, lCell, xPixel, yPixel);
+                            }
                             drawImage(gr, lCell, xPixel, yPixel);
                         } else {
                             drawGlyph(gr, lCell, xPixel, yPixel);
@@ -1654,6 +1670,10 @@ public class SwingTerminal extends LogicalScreen
                             || (lCell.isBlink())
                         ) {
                             if (lCell.isImage()) {
+                                if (imagesOverText) {
+                                    // Draw the glyph underneath the image.
+                                    drawGlyph(gr, lCell, xPixel, yPixel);
+                                }
                                 drawImage(gr, lCell, xPixel, yPixel);
                             } else {
                                 drawGlyph(gr, lCell, xPixel, yPixel);

src/jexer/bits/Cell.java

@@ -388,13 +388,13 @@ public class Cell extends CellAttributes {
                 return false;
             }
             // Either both objects have their image inverted, or neither do.
-            // Now if the images are identical the cells are the same
-            // visually.
             if ((imageHashCode == that.imageHashCode)
                 && (background.equals(that.background))
             ) {
-                return true;
+                // Fall through to the attributes check below.
+                // ...
             } else {
+                // The cells are not the same visually.
                 return false;
             }
         }
@@ -470,6 +470,19 @@ public class Cell extends CellAttributes {
         super.setTo(that);
     }
 
+    /**
+     * Set my field attr values to that's field.
+     *
+     * @param that a CellAttributes instance
+     * @param keepImage if true, retain the image data
+     */
+    public void setAttr(final CellAttributes that, final boolean keepImage) {
+        if (!keepImage) {
+            image = null;
+        }
+        super.setTo(that);
+    }
+
     /**
      * Make human-readable description of this Cell.
      *

src/jexer/tterminal/ECMA48.java

@@ -7224,6 +7224,16 @@ public class ECMA48 implements Runnable {
             // 0x71 goes to DCS_SIXEL
             if (ch == 0x71) {
                 sixelParseBuffer.setLength(0);
+                // Params contains the sixel introducer string, include it
+                // and the trailing 'q'.
+                for (Integer ps: csiParams) {
+                    sixelParseBuffer.append(ps.toString());
+                    sixelParseBuffer.append(';');
+                }
+                if (sixelParseBuffer.length() > 0) {
+                    sixelParseBuffer.setLength(sixelParseBuffer.length() - 1);
+                    sixelParseBuffer.append('q');
+                }
                 scanState = ScanState.DCS_SIXEL;
             } else if ((ch >= 0x40) && (ch <= 0x7E)) {
                 // 0x40-7E goes to DCS_PASSTHROUGH
@@ -7480,7 +7490,8 @@ public class ECMA48 implements Runnable {
             + "'");
         */
 
-        Sixel sixel = new Sixel(sixelParseBuffer.toString(), sixelPalette);
+        Sixel sixel = new Sixel(sixelParseBuffer.toString(), sixelPalette,
+            jexer.backend.SwingTerminal.attrToBackgroundColor(currentState.attr));
         BufferedImage image = sixel.getImage();
 
         // System.err.println("parseSixel(): image " + image);
@@ -7497,7 +7508,12 @@ public class ECMA48 implements Runnable {
             return;
         }
 
-        imageToCells(image, true);
+        boolean maybeTransparent = false;
+        if (System.getProperty("jexer.Swing.imagesOverText",
+                "false").equals("true")) {
+            maybeTransparent = true;
+        }
+        imageToCells(image, true, maybeTransparent);
     }
 
     /**
@@ -7564,7 +7580,7 @@ public class ECMA48 implements Runnable {
             }
         }
 
-        imageToCells(image, scroll);
+        imageToCells(image, scroll, false);
     }
 
     /**
@@ -7582,6 +7598,7 @@ public class ECMA48 implements Runnable {
         int imageHeight = 0;
         boolean scroll = false;
         BufferedImage image = null;
+        boolean maybeTransparent = false;
         try {
             byte [] bytes = StringUtils.fromBase64(data.getBytes());
 
@@ -7599,6 +7616,7 @@ public class ECMA48 implements Runnable {
                     // File does not have PNG header, bail out.
                     return;
                 }
+                maybeTransparent = true;
                 break;
 
             case 2:
@@ -7639,7 +7657,7 @@ public class ECMA48 implements Runnable {
             return;
         }
 
-        imageToCells(image, scroll);
+        imageToCells(image, scroll, maybeTransparent);
     }
 
     /**
@@ -7647,8 +7665,12 @@ public class ECMA48 implements Runnable {
      *
      * @param image the image to display
      * @param scroll if true, scroll the image and move the cursor
+     * @param maybeTransparent if true, this image format might have
+     * transparency
      */
-    private void imageToCells(final BufferedImage image, final boolean scroll) {
+    private void imageToCells(BufferedImage image, final boolean scroll,
+        final boolean maybeTransparent) {
+
         assert (image != null);
 
         /*
@@ -7667,26 +7689,50 @@ public class ECMA48 implements Runnable {
          *
          * 2. Set (x, y) cell image data.
          *
-         * 3. For the right and bottom edges:
+         * 3. For the right and bottom edges (not yet done):
          *
          *   a. Render the text to pixels using Terminus font.
          *
          *   b. Blit the image on top of the text, using alpha channel.
          */
+
+        // If the backend supports transparent images, then we will not
+        // draw the black underneath the cells.
+        boolean transparent = false;
+        if (System.getProperty("jexer.Swing.imagesOverText",
+                "false").equals("true")) {
+            transparent = true;
+        }
+
         int cellColumns = image.getWidth() / textWidth;
-        if (cellColumns * textWidth < image.getWidth()) {
+        while (cellColumns * textWidth < image.getWidth()) {
             cellColumns++;
         }
         int cellRows = image.getHeight() / textHeight;
-        if (cellRows * textHeight < image.getHeight()) {
+        while (cellRows * textHeight < image.getHeight()) {
             cellRows++;
         }
 
+        if (!transparent && maybeTransparent) {
+            // Re-render the image against a black background, so that alpha
+            // in the image does not lead to bleed-through artifacts.
+            BufferedImage newImage;
+            newImage = new BufferedImage(cellColumns * textWidth,
+                cellRows * textHeight, BufferedImage.TYPE_INT_ARGB);
+
+            java.awt.Graphics gr = newImage.getGraphics();
+            gr.setColor(java.awt.Color.BLACK);
+            gr.fillRect(0, 0, newImage.getWidth(), newImage.getHeight());
+            gr.drawImage(image, 0, 0, null, null);
+            gr.dispose();
+            image = newImage;
+        }
+
         // Break the image up into an array of cells.
         Cell [][] cells = new Cell[cellColumns][cellRows];
-
         for (int x = 0; x < cellColumns; x++) {
             for (int y = 0; y < cellRows; y++) {
+                Cell cell = new Cell();
 
                 int width = textWidth;
                 if ((x + 1) * textWidth > image.getWidth()) {
@@ -7696,25 +7742,27 @@ public class ECMA48 implements Runnable {
                 if ((y + 1) * textHeight > image.getHeight()) {
                     height = image.getHeight() - (y * textHeight);
                 }
-
+                if ((width != textWidth) || (height != textHeight)) {
-                Cell cell = new Cell();
+                    // Copy the smaller-than-text-cell-size image to a
-
+                    // full-text-cell-size.
-                // Always re-render the image against a black background, so
+                    BufferedImage newImage;
-                // that alpha in the image does not lead to bleed-through
+                    newImage = new BufferedImage(textWidth, textHeight,
-                // artifacts.
+                        BufferedImage.TYPE_INT_ARGB);
-                BufferedImage newImage;
+                    java.awt.Graphics gr = newImage.getGraphics();
-                newImage = new BufferedImage(textWidth, textHeight,
+                    gr.setColor(java.awt.Color.BLACK);
-                    BufferedImage.TYPE_INT_ARGB);
+                    if (!transparent) {
-
+                        gr.fillRect(0, 0, newImage.getWidth(),
-                java.awt.Graphics gr = newImage.getGraphics();
+                            newImage.getHeight());
-                gr.setColor(java.awt.Color.BLACK);
+                    }
-                gr.fillRect(0, 0, textWidth, textHeight);
+                    gr.drawImage(image.getSubimage(x * textWidth,
-                gr.drawImage(image.getSubimage(x * textWidth,
+                            y * textHeight, width, height),
-                        y * textHeight, width, height),
+                        0, 0, null, null);
-                    0, 0, null, null);
+                    gr.dispose();
-                gr.dispose();
+                    cell.setImage(newImage);
-                cell.setImage(newImage);
+                } else {
-
+                    cell.setImage(image.getSubimage(x * textWidth,
+                            y * textHeight, width, height));
+                }
                 cells[x][y] = cell;
             }
         }
@@ -7722,14 +7770,16 @@ public class ECMA48 implements Runnable {
         int x0 = currentState.cursorX;
         int y0 = currentState.cursorY;
         for (int y = 0; y < cellRows; y++) {
+            DisplayLine line = display.get(currentState.cursorY);
+
             for (int x = 0; x < cellColumns; x++) {
                 assert (currentState.cursorX <= rightMargin);
 
-                // A real sixel terminal would render the text of the current
+                // Keep the character data from the old cell, putting the
-                // cell first, then image over it (accounting for blank
+                // image data over it.
-                // pixels).  We do not support that.  A cell is either text,
+                Cell oldCell = line.charAt(currentState.cursorX);
-                // or image, but not a mix of image-over-text.
+                cells[x][y].setChar(oldCell.getChar());
-                DisplayLine line = display.get(currentState.cursorY);
+                cells[x][y].setAttr(oldCell, true);
                 line.replace(currentState.cursorX, cells[x][y]);
 
                 // If at the end of the visible screen, stop.

src/jexer/tterminal/Sixel.java

@@ -46,6 +46,7 @@ public class Sixel {
      * Parser character scan states.
      */
     private enum ScanState {
+        INIT,
         GROUND,
         RASTER,
         COLOR,
@@ -87,7 +88,7 @@ public class Sixel {
     /**
      * Current scanning state.
      */
-    private ScanState scanState = ScanState.GROUND;
+    private ScanState scanState = ScanState.INIT;
 
     /**
      * Parameters being collected.
@@ -154,11 +155,21 @@ public class Sixel {
      */
     private Color color = Color.BLACK;
 
+    /**
+     * The background color.
+     */
+    private Color background = Color.BLACK;
+
     /**
      * If set, abort processing this image.
      */
     private boolean abort = false;
 
+    /**
+     * If set, color index 0 will be set to transparent.
+     */
+    private boolean transparent = false;
+
     // ------------------------------------------------------------------------
     // Constructors -----------------------------------------------------------
     // ------------------------------------------------------------------------
@@ -168,14 +179,18 @@ public class Sixel {
      *
      * @param buffer the sixel data to parse
      * @param palette palette to use, or null for a private palette
+     * @param the background color to use
      */
-    public Sixel(final String buffer, final HashMap<Integer, Color> palette) {
+    public Sixel(final String buffer, final HashMap<Integer, Color> palette,
+        Color background) {
+
         this.buffer = buffer;
         if (palette == null) {
             this.palette = new HashMap<Integer, Color>();
         } else {
             this.palette = palette;
         }
+        this.background = background;
     }
 
     // ------------------------------------------------------------------------
@@ -200,8 +215,8 @@ public class Sixel {
 
         if ((width > 0) && (height > 0) && (image != null)) {
             /*
-            System.err.println(String.format("%d %d %d %d", width, y + 1,
+            System.err.println(String.format("getImage() %d %d %d %d %d %d",
-                    rasterWidth, rasterHeight));
+                    width, height, x, y, rasterWidth, rasterHeight));
             */
 
             if ((rasterWidth > width) || (rasterHeight > y + 1)) {
@@ -225,18 +240,23 @@ public class Sixel {
         BufferedImage newImage = new BufferedImage(newWidth, newHeight,
             BufferedImage.TYPE_INT_ARGB);
 
-        if (image == null) {
-            image = newImage;
-            return;
-        }
-
         if (DEBUG) {
-            System.err.println("resizeImage(); old " + image.getWidth() + "x" +
+            System.err.println("resizeImage(); old " +
-                image.getHeight() + " new " + newWidth + "x" + newHeight);
+                (image != null ? image.getWidth() : "null ") + "x " +
+                (image != null ? image.getHeight() : "null ") + "y " +
+                "new " + newWidth + "x " + newHeight + "y " +
+                "transparency: " + transparent);
         }
 
         Graphics2D gr = newImage.createGraphics();
-        gr.drawImage(image, 0, 0, image.getWidth(), image.getHeight(), null);
+        if (!transparent) {
+            gr.setColor(background);
+            gr.fillRect(0, 0, newWidth, newHeight);
+        }
+        if (image != null) {
+            gr.drawImage(image, 0, 0, image.getWidth(), image.getHeight(),
+                null);
+        }
         gr.dispose();
         image = newImage;
     }
@@ -421,6 +441,44 @@ public class Sixel {
         }
     }
 
+    /**
+     * Parse the initializer.
+     */
+    private void parseInit() {
+        int p1 = getParam(0, 0);        // Pixel aspect ratio (ignored)
+        int p2 = getParam(1, 0);        // Background color option
+        int p3 = getParam(2, 0);        // Horizontal grid size (ignored)
+
+        if (DEBUG) {
+            System.err.println("parseInit() " + p1 + " " + p2 + " " + p3);
+        }
+
+        switch (p2) {
+        case 1:
+            /*
+             * Pixels that are not specified with a color will be
+             * transparent.
+             *
+             * The only backend that can currently display transparent images
+             * is the Swing backend.  If that backend is not enabled, then do
+             * not support transparency here because it would lead to screen
+             * artifacts.
+             */
+            if (System.getProperty("jexer.Swing.imagesOverText",
+                    "false").equals("true")) {
+                transparent = true;
+            } else {
+                transparent = false;
+            }
+            break;
+        default:
+            // Pixels that are not specified with a color will be the current
+            // background color.
+            transparent = false;
+            break;
+        }
+    }
+
     /**
      * Parse the raster attributes.
      */
@@ -430,6 +488,11 @@ public class Sixel {
         int pah = getParam(2, 0);  // Horizontal width
         int pav = getParam(3, 0);  // Vertical height
 
+        if (DEBUG) {
+            System.err.println("parseRaster() " + pan + " " + pad + " " +
+                pah + " " + pav);
+        }
+
         if ((pan == pad) && (pah > 0) && (pav > 0)) {
             rasterWidth = pah;
             rasterHeight = pav;
@@ -453,11 +516,22 @@ public class Sixel {
         // DEBUG
         // System.err.printf("Sixel.consume() %c STATE = %s\n", ch, scanState);
 
+        if ((ch == 'q') && (scanState == ScanState.INIT)) {
+            // This is the normal happy path with the introducer string.
+            parseInit();
+            toGround();
+            return;
+        }
+
         // Between decimal 63 (inclusive) and 127 (exclusive) --> pixels
         if ((ch >= 63) && (ch < 127)) {
             if (scanState == ScanState.COLOR) {
                 setPalette();
             }
+            if (scanState == ScanState.INIT) {
+                parseInit();
+                toGround();
+            }
             if (scanState == ScanState.RASTER) {
                 parseRaster();
                 toGround();
@@ -473,6 +547,10 @@ public class Sixel {
                 setPalette();
                 toGround();
             }
+            if (scanState == ScanState.INIT) {
+                parseInit();
+                toGround();
+            }
             if (scanState == ScanState.RASTER) {
                 parseRaster();
                 toGround();
@@ -487,6 +565,10 @@ public class Sixel {
                 setPalette();
                 toGround();
             }
+            if (scanState == ScanState.INIT) {
+                parseInit();
+                toGround();
+            }
             if (scanState == ScanState.RASTER) {
                 parseRaster();
                 toGround();
@@ -501,6 +583,10 @@ public class Sixel {
                 setPalette();
                 toGround();
             }
+            if (scanState == ScanState.INIT) {
+                parseInit();
+                toGround();
+            }
             if (scanState == ScanState.RASTER) {
                 parseRaster();
                 toGround();
@@ -523,6 +609,10 @@ public class Sixel {
                 setPalette();
                 toGround();
             }
+            if (scanState == ScanState.INIT) {
+                parseInit();
+                toGround();
+            }
             if (scanState == ScanState.RASTER) {
                 parseRaster();
                 toGround();
@@ -536,6 +626,10 @@ public class Sixel {
                 setPalette();
                 toGround();
             }
+            if (scanState == ScanState.INIT) {
+                parseInit();
+                toGround();
+            }
             scanState = ScanState.RASTER;
             return;
         }
@@ -549,6 +643,19 @@ public class Sixel {
             }
             return;
 
+        case INIT:
+            // 30-39, 3B --> param
+            if ((ch >= '0') && (ch <= '9')) {
+                params[paramsI] *= 10;
+                params[paramsI] += (ch - '0');
+            }
+            if (ch == ';') {
+                if (paramsI < params.length - 1) {
+                    paramsI++;
+                }
+            }
+            return;
+
         case RASTER:
             // 30-39, 3B --> param
             if ((ch >= '0') && (ch <= '9')) {