Aztec encode with ECI for non-default character sets (#1330)

* Aztec encoder: add ECI codes according to character set

Added redundant methods to avoid modifying existing tests.

* fix testAztecWriter

- ISO-8859-1 cannot actually encode Euro symbol ('€'); this test case only
  passed before because the Decoder wasn't actually doing the bytes→String
  decode, but simply round-tripping an unknown byte.
- Add extra test cases for implicit ISO-8859-1 (without ECI code), explicit
  ISO-8559-1 (with ECI code), and Shift_JIS

* remove unnecessary conversion between String and byte[] in Aztec EncoderTest and DetectorTest

* Aztec DecoderTest: use constants for charsets

* Aztec Code: remove unnecessary conversion between Charset and Charset.name() strings

* PDF417, QR, DataMatrix: remove unnecessary conversion between Charset and Charset.name() strings

Includes replacing StringUtils.guessEncoding() with .guessCharset(), to return
Charset rather than String.

This change makes the tacit assumption that Shift_JIS charset *will* be
available.  There are existing comments suggesting that it might not always
be available… but the existing *tests* assume it will be.

4 files changed, 58 insertions, 46 deletions

diff --git a/core/src/test/java/com/google/zxing/aztec/detector/DetectorTest.java b/core/src/test/java/com/google/zxing/aztec/detector/DetectorTest.java
index 11963c691..6d3afa673 100644
--- a/core/src/test/java/com/google/zxing/aztec/detector/DetectorTest.java
+++ b/core/src/test/java/com/google/zxing/aztec/detector/DetectorTest.java
@@ -27,7 +27,6 @@ import com.google.zxing.common.DecoderResult;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -62,7 +61,7 @@ public final class DetectorTest extends Assert {
 
   // Test that we can tolerate errors in the parameter locator bits
   private static void testErrorInParameterLocator(String data) throws Exception {
-    AztecCode aztec = Encoder.encode(data.getBytes(StandardCharsets.ISO_8859_1), 25, Encoder.DEFAULT_AZTEC_LAYERS);
+    AztecCode aztec = Encoder.encode(data, 25, Encoder.DEFAULT_AZTEC_LAYERS);
     Random random = new Random(aztec.getMatrix().hashCode());   // pseudo-random, but deterministic
     int layers = aztec.getLayers();
     boolean compact = aztec.isCompact();
diff --git a/core/src/test/java/com/google/zxing/aztec/encoder/EncoderTest.java b/core/src/test/java/com/google/zxing/aztec/encoder/EncoderTest.java
index c0e5adae8..13b323abd 100644
--- a/core/src/test/java/com/google/zxing/aztec/encoder/EncoderTest.java
+++ b/core/src/test/java/com/google/zxing/aztec/encoder/EncoderTest.java
@@ -44,6 +44,12 @@ import java.util.regex.Pattern;
  */
 public final class EncoderTest extends Assert {
 
+  private static final Charset ISO_8859_1 = StandardCharsets.ISO_8859_1;
+  private static final Charset UTF_8 = StandardCharsets.UTF_8;
+  private static final Charset SHIFT_JIS = Charset.forName("Shift_JIS");
+  private static final Charset ISO_8859_15 = Charset.forName("ISO-8859-15");
+  private static final Charset WINDOWS_1252 = Charset.forName("Windows-1252");
+
   private static final Pattern DOTX = Pattern.compile("[^.X]");
   private static final Pattern SPACES = Pattern.compile("\\s+");
   private static final ResultPoint[] NO_POINTS = new ResultPoint[0];
@@ -128,17 +134,20 @@ public final class EncoderTest extends Assert {
 
   @Test
   public void testAztecWriter() throws Exception {
-    testWriter("\u20AC 1 sample data.", "ISO-8859-1", 25, true, 2);
-    testWriter("\u20AC 1 sample data.", "ISO-8859-15", 25, true, 2);
-    testWriter("\u20AC 1 sample data.", "UTF-8", 25, true, 2);
-    testWriter("\u20AC 1 sample data.", "UTF-8", 100, true, 3);
-    testWriter("\u20AC 1 sample data.", "UTF-8", 300, true, 4);
-    testWriter("\u20AC 1 sample data.", "UTF-8", 500, false, 5);
+    testWriter("Espa\u00F1ol", null, 25, true, 1);                   // Without ECI (implicit ISO-8859-1)
+    testWriter("Espa\u00F1ol", ISO_8859_1, 25, true, 1);             // Explicit ISO-8859-1
+    testWriter("\u20AC 1 sample data.", WINDOWS_1252, 25, true, 2);  // Standard ISO-8859-1 cannot encode Euro symbol; Windows-1252 superset can
+    testWriter("\u20AC 1 sample data.", ISO_8859_15, 25, true, 2);
+    testWriter("\u20AC 1 sample data.", UTF_8, 25, true, 2);
+    testWriter("\u20AC 1 sample data.", UTF_8, 100, true, 3);
+    testWriter("\u20AC 1 sample data.", UTF_8, 300, true, 4);
+    testWriter("\u20AC 1 sample data.", UTF_8, 500, false, 5);
+    testWriter("The capital of Japan is named \u6771\u4EAC.", SHIFT_JIS, 25, true, 3);
     // Test AztecWriter defaults
     String data = "In ut magna vel mauris malesuada";
     AztecWriter writer = new AztecWriter();
     BitMatrix matrix = writer.encode(data, BarcodeFormat.AZTEC, 0, 0);
-    AztecCode aztec = Encoder.encode(data.getBytes(StandardCharsets.ISO_8859_1),
+    AztecCode aztec = Encoder.encode(data,
         Encoder.DEFAULT_EC_PERCENT, Encoder.DEFAULT_AZTEC_LAYERS);
     BitMatrix expectedMatrix = aztec.getMatrix();
     assertEquals(matrix, expectedMatrix);
@@ -418,7 +427,7 @@ public final class EncoderTest extends Assert {
 
   @Test
   public void testUserSpecifiedLayers() throws FormatException {
-    byte[] alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes(StandardCharsets.ISO_8859_1);
+    String alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
     AztecCode aztec = Encoder.encode(alphabet, 25, -2);
     assertEquals(2, aztec.getLayers());
     assertTrue(aztec.isCompact());
@@ -449,22 +458,21 @@ public final class EncoderTest extends Assert {
     String alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
     // encodes as 26 * 5 * 4 = 520 bits of data
     String alphabet4 = alphabet + alphabet + alphabet + alphabet;
-    byte[] data = alphabet4.getBytes(StandardCharsets.ISO_8859_1);
     try {
-      Encoder.encode(data, 0, -4);
+      Encoder.encode(alphabet4, 0, -4);
       fail("Encode should have failed.  Text can't fit in 1-layer compact");
     } catch (IllegalArgumentException expected) {
       // continue
     }
 
     // If we just try to encode it normally, it will go to a non-compact 4 layer
-    AztecCode aztecCode = Encoder.encode(data, 0, Encoder.DEFAULT_AZTEC_LAYERS);
+    AztecCode aztecCode = Encoder.encode(alphabet4, 0, Encoder.DEFAULT_AZTEC_LAYERS);
     assertFalse(aztecCode.isCompact());
     assertEquals(4, aztecCode.getLayers());
 
     // But shortening the string to 100 bytes (500 bits of data), compact works fine, even if we
     // include more error checking.
-    aztecCode = Encoder.encode(alphabet4.substring(0, 100).getBytes(StandardCharsets.ISO_8859_1), 10, Encoder.DEFAULT_AZTEC_LAYERS);
+    aztecCode = Encoder.encode(alphabet4.substring(0, 100), 10, Encoder.DEFAULT_AZTEC_LAYERS);
     assertTrue(aztecCode.isCompact());
     assertEquals(4, aztecCode.getLayers());
   }
@@ -472,7 +480,7 @@ public final class EncoderTest extends Assert {
   // Helper routines
 
   private static void testEncode(String data, boolean compact, int layers, String expected) throws FormatException {
-    AztecCode aztec = Encoder.encode(data.getBytes(StandardCharsets.ISO_8859_1), 33, Encoder.DEFAULT_AZTEC_LAYERS);
+    AztecCode aztec = Encoder.encode(data, 33, Encoder.DEFAULT_AZTEC_LAYERS);
     assertEquals("Unexpected symbol format (compact)", compact, aztec.isCompact());
     assertEquals("Unexpected nr. of layers", layers, aztec.getLayers());
     BitMatrix matrix = aztec.getMatrix();
@@ -480,7 +488,7 @@ public final class EncoderTest extends Assert {
   }
 
   private static void testEncodeDecode(String data, boolean compact, int layers) throws Exception {
-    AztecCode aztec = Encoder.encode(data.getBytes(StandardCharsets.ISO_8859_1), 25, Encoder.DEFAULT_AZTEC_LAYERS);
+    AztecCode aztec = Encoder.encode(data, 25, Encoder.DEFAULT_AZTEC_LAYERS);
     assertEquals("Unexpected symbol format (compact)", compact, aztec.isCompact());
     assertEquals("Unexpected nr. of layers", layers, aztec.getLayers());
     BitMatrix matrix = aztec.getMatrix();
@@ -500,20 +508,20 @@ public final class EncoderTest extends Assert {
   }
 
   private static void testWriter(String data,
-                                 String charset,
+                                 Charset charset,
                                  int eccPercent,
                                  boolean compact,
                                  int layers) throws FormatException {
-    // 1. Perform an encode-decode round-trip because it can be lossy.
-    // 2. Aztec Decoder currently always decodes the data with a LATIN-1 charset:
-    String expectedData = new String(data.getBytes(Charset.forName(charset)), StandardCharsets.ISO_8859_1);
+    // Perform an encode-decode round-trip because it can be lossy.
     Map<EncodeHintType,Object> hints = new EnumMap<>(EncodeHintType.class);
-    hints.put(EncodeHintType.CHARACTER_SET, charset);
+    if (null != charset) {
+        hints.put(EncodeHintType.CHARACTER_SET, charset.name());
+    }
     hints.put(EncodeHintType.ERROR_CORRECTION, eccPercent);
     AztecWriter writer = new AztecWriter();
     BitMatrix matrix = writer.encode(data, BarcodeFormat.AZTEC, 0, 0, hints);
-    AztecCode aztec = Encoder.encode(data.getBytes(Charset.forName(charset)), eccPercent,
-        Encoder.DEFAULT_AZTEC_LAYERS);
+    AztecCode aztec = Encoder.encode(data, eccPercent,
+        Encoder.DEFAULT_AZTEC_LAYERS, charset);
     assertEquals("Unexpected symbol format (compact)", compact, aztec.isCompact());
     assertEquals("Unexpected nr. of layers", layers, aztec.getLayers());
     BitMatrix matrix2 = aztec.getMatrix();
@@ -521,7 +529,7 @@ public final class EncoderTest extends Assert {
     AztecDetectorResult r =
         new AztecDetectorResult(matrix, NO_POINTS, aztec.isCompact(), aztec.getCodeWords(), aztec.getLayers());
     DecoderResult res = new Decoder().decode(r);
-    assertEquals(expectedData, res.getText());
+    assertEquals(data, res.getText());
     // Check error correction by introducing up to eccPercent/2 errors
     int ecWords = aztec.getCodeWords() * eccPercent / 100 / 2;
     Random random = getPseudoRandom();
@@ -537,7 +545,7 @@ public final class EncoderTest extends Assert {
     }
     r = new AztecDetectorResult(matrix, NO_POINTS, aztec.isCompact(), aztec.getCodeWords(), aztec.getLayers());
     res = new Decoder().decode(r);
-    assertEquals(expectedData, res.getText());
+    assertEquals(data, res.getText());
   }
 
   private static Random getPseudoRandom() {
diff --git a/core/src/test/java/com/google/zxing/common/StringUtilsTestCase.java b/core/src/test/java/com/google/zxing/common/StringUtilsTestCase.java
index 578843b1e..6440209b9 100644
--- a/core/src/test/java/com/google/zxing/common/StringUtilsTestCase.java
+++ b/core/src/test/java/com/google/zxing/common/StringUtilsTestCase.java
@@ -19,6 +19,7 @@ package com.google.zxing.common;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.charset.Charset;
 
 /**
@@ -28,34 +29,42 @@ public final class StringUtilsTestCase extends Assert {
 
   @Test
   public void testShortShiftJIS1() {
-    // 金魚
-    doTest(new byte[] { (byte) 0x8b, (byte) 0xe0, (byte) 0x8b, (byte) 0x9b, }, "SJIS");
+    // 金魚
+    doTest(new byte[] { (byte) 0x8b, (byte) 0xe0, (byte) 0x8b, (byte) 0x9b, }, StringUtils.SHIFT_JIS_CHARSET, "SJIS");
   }
 
   @Test
   public void testShortISO885911() {
-    // båd
-    doTest(new byte[] { (byte) 0x62, (byte) 0xe5, (byte) 0x64, }, "ISO-8859-1");
+    // båd
+    doTest(new byte[] { (byte) 0x62, (byte) 0xe5, (byte) 0x64, }, StandardCharsets.ISO_8859_1, "ISO8859_1");
+  }
+
+  @Test
+  public void testShortUTF81() {
+    // Español
+    doTest(new byte[] { (byte) 0x45, (byte) 0x73, (byte) 0x70, (byte) 0x61, (byte) 0xc3,
+                        (byte) 0xb1, (byte) 0x6f, (byte) 0x6c },
+           StandardCharsets.UTF_8, "UTF8");
   }
 
   @Test
   public void testMixedShiftJIS1() {
-    // Hello Èáë!
+    // Hello 金!
     doTest(new byte[] { (byte) 0x48, (byte) 0x65, (byte) 0x6c, (byte) 0x6c, (byte) 0x6f,
                         (byte) 0x20, (byte) 0x8b, (byte) 0xe0, (byte) 0x21, },
-           "SJIS");
+           StringUtils.SHIFT_JIS_CHARSET, "SJIS");
   }
 
-  private static void doTest(byte[] bytes, String charsetName) {
-    Charset charset = Charset.forName(charsetName);
-    String guessedName = StringUtils.guessEncoding(bytes, null);
-    Charset guessedEncoding = Charset.forName(guessedName);
-    assertEquals(charset, guessedEncoding);
+  private static void doTest(byte[] bytes, Charset charset, String encoding) {
+    Charset guessedCharset = StringUtils.guessCharset(bytes, null);
+    String guessedEncoding = StringUtils.guessEncoding(bytes, null);
+    assertEquals(charset, guessedCharset);
+    assertEquals(encoding, guessedEncoding);
   }
 
   /**
    * Utility for printing out a string in given encoding as a Java statement, since it's better
-   * to write that into the Java source file rather than risk character encoding issues in the 
+   * to write that into the Java source file rather than risk character encoding issues in the
    * source file itself.
    *
    * @param args command line arguments
diff --git a/core/src/test/java/com/google/zxing/qrcode/encoder/EncoderTestCase.java b/core/src/test/java/com/google/zxing/qrcode/encoder/EncoderTestCase.java
index ebf00e261..b8e2bd660 100644
--- a/core/src/test/java/com/google/zxing/qrcode/encoder/EncoderTestCase.java
+++ b/core/src/test/java/com/google/zxing/qrcode/encoder/EncoderTestCase.java
@@ -19,6 +19,7 @@ package com.google.zxing.qrcode.encoder;
 import com.google.zxing.EncodeHintType;
 import com.google.zxing.WriterException;
 import com.google.zxing.common.BitArray;
+import com.google.zxing.common.StringUtils;
 import com.google.zxing.qrcode.decoder.ErrorCorrectionLevel;
 import com.google.zxing.qrcode.decoder.Mode;
 import com.google.zxing.qrcode.decoder.Version;
@@ -26,7 +27,6 @@ import com.google.zxing.qrcode.decoder.Version;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.io.UnsupportedEncodingException;
 import java.util.EnumMap;
 import java.util.Map;
 
@@ -127,7 +127,7 @@ public final class EncoderTestCase extends Assert {
           ">>\n";
     assertEquals(expected, qrCode.toString());
   }
-  
+
   @Test
   public void testEncodeWithVersion() throws WriterException {
     Map<EncodeHintType, Object> hints = new EnumMap<>(EncodeHintType.class);
@@ -135,7 +135,7 @@ public final class EncoderTestCase extends Assert {
     QRCode qrCode = Encoder.encode("ABCDEF", ErrorCorrectionLevel.H, hints);
     assertTrue(qrCode.toString().contains(" version: 7\n"));
   }
-  
+
   @Test(expected = WriterException.class)
   public void testEncodeWithVersionTooSmall() throws WriterException {
     Map<EncodeHintType, Object> hints = new EnumMap<>(EncodeHintType.class);
@@ -742,12 +742,8 @@ public final class EncoderTestCase extends Assert {
     assertEquals(expected, qrCode.toString());
   }
 
-  private static String shiftJISString(byte[] bytes) throws WriterException {
-    try {
-      return new String(bytes, "Shift_JIS");
-    } catch (UnsupportedEncodingException uee) {
-      throw new WriterException(uee.toString());
-    }
+  private static String shiftJISString(byte[] bytes) {
+    return new String(bytes, StringUtils.SHIFT_JIS_CHARSET);
   }
 
 }