Merge pull request #1727 from open-keychain/mime-fallback

try to guess if data is text or not

2 files changed, 281 insertions, 4 deletions

diff --git a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java
index 4277649fb..fba8be061 100644
--- a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java
+++ b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java
@@ -20,6 +20,7 @@ package org.sufficientlysecure.keychain.pgp;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
 import java.io.PrintStream;
 import java.security.Security;
 import java.util.ArrayList;
@@ -30,18 +31,20 @@ import android.content.ContentValues;
 import android.net.Uri;
 
 import junit.framework.Assert;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.openintents.openpgp.OpenPgpMetadata;
 import org.robolectric.RobolectricGradleTestRunner;
 import org.robolectric.RuntimeEnvironment;
 import org.robolectric.annotation.Config;
 import org.robolectric.shadows.ShadowLog;
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.sufficientlysecure.keychain.WorkaroundBuildConfig;
 import org.sufficientlysecure.keychain.operations.InputDataOperation;
 import org.sufficientlysecure.keychain.operations.results.InputDataResult;
+import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType;
 import org.sufficientlysecure.keychain.provider.ProviderHelper;
 import org.sufficientlysecure.keychain.provider.TemporaryFileProvider;
 import org.sufficientlysecure.keychain.service.InputDataParcel;
@@ -80,20 +83,20 @@ public class InputDataOperationTest {
     }
 
     @Test
-    public void testMimeDecoding () throws Exception {
+    public void testMimeDecoding() throws Exception {
 
         String mimeMail =
             "Content-Type: multipart/mixed; boundary=\"=-26BafqxfXmhVNMbYdoIi\"\n" +
             "\n" +
             "--=-26BafqxfXmhVNMbYdoIi\n" +
-            "Content-Type: text/plain\n" +
+            "Content-Type: text/plain; charset=utf-8\n" +
             "Content-Transfer-Encoding: quoted-printable\n" +
             "Content-Disposition: attachment; filename=data.txt\n" +
             "\n" +
             "message part 1\n" +
             "\n" +
             "--=-26BafqxfXmhVNMbYdoIi\n" +
-            "Content-Type: text/testvalue\n" +
+            "Content-Type: text/testvalue; charset=iso-8859-1\n" +
             "Content-Description: Dummy content description\n" +
             "\n" +
             "message part 2.1\n" +
@@ -156,7 +159,137 @@ public class InputDataOperationTest {
         Assert.assertEquals("second part must have expected content",
                 "message part 2.1\nmessage part 2.2\n", new String(outStream2.toByteArray()));
 
+        OpenPgpMetadata metadata = result.mMetadata.get(0);
+        Assert.assertEquals("text/plain", metadata.getMimeType());
+        Assert.assertEquals("utf-8", metadata.getCharset());
+
+        metadata = result.mMetadata.get(1);
+        Assert.assertEquals("text/testvalue", metadata.getMimeType());
+        Assert.assertEquals("iso-8859-1", metadata.getCharset());
+    }
+
+    @Test
+    public void testMimeDecodingExplicitFaultyCharset() throws Exception {
+
+        String mimeContent = "Content-Type: text/plain; charset=utf-8\n" +
+                        "\n" +
+                        "message with binary data in it\n";
+
+        byte[] data = mimeContent.getBytes();
+        data[60] = (byte) 0xc3;
+        data[61] = (byte) 0x28;
+
+        InputDataResult result = runSimpleDataInputOperation(data);
+
+        // must be successful, no verification, have two output URIs
+        Assert.assertTrue(result.success());
+        Assert.assertNull(result.mDecryptVerifyResult);
+
+        OpenPgpMetadata metadata = result.mMetadata.get(0);
+        Assert.assertEquals("text/plain", metadata.getMimeType());
+
+        Assert.assertEquals("charset should be set since it was explicitly specified",
+                "utf-8", metadata.getCharset());
+        Assert.assertTrue("faulty charset should have been detected",
+                result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_FAULTY));
+    }
+
+    @Test
+    public void testMimeDecodingImplicitFaultyCharset() throws Exception {
+
+        String mimeContent = "Content-Type: text/plain\n" +
+                "\n" +
+                "message with binary data in it\n";
+
+        byte[] data = mimeContent.getBytes();
+        data[45] = (byte) 0xc3;
+        data[46] = (byte) 0x28;
+
+        InputDataResult result = runSimpleDataInputOperation(data);
+
+        // must be successful, no verification, have two output URIs
+        Assert.assertTrue(result.success());
+        Assert.assertNull(result.mDecryptVerifyResult);
+
+        OpenPgpMetadata metadata = result.mMetadata.get(0);
+        Assert.assertEquals("text/plain", metadata.getMimeType());
+
+        Assert.assertNull("charset was bad so it should not be set", metadata.getCharset());
+        Assert.assertTrue("faulty charset should have been detected",
+                result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN));
+    }
+
+    @Test
+    public void testMimeDecodingImplicitGuessedCharset() throws Exception {
+
+        String mimeContent = "Content-Type: text/plain\n" +
+                "\n" +
+                "proper, utf-8 encoded message ☭\n";
+
+        InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes());
+
+        // must be successful, no verification, have two output URIs
+        Assert.assertTrue(result.success());
+        Assert.assertNull(result.mDecryptVerifyResult);
+
+        OpenPgpMetadata metadata = result.mMetadata.get(0);
+        Assert.assertEquals("text/plain", metadata.getMimeType());
 
+        Assert.assertEquals("charset should be set since it was guessed and not faulty",
+                "utf-8", metadata.getCharset());
+        Assert.assertTrue("charset should have been guessed",
+                result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS));
+    }
+
+    @Test
+    public void testMimeDecodingOctetStreamGuessedCharset() throws Exception {
+
+        String mimeContent = "Content-Type: application/octet-stream\n" +
+                "\n" +
+                "proper, utf-8 encoded message ☭\n";
+
+        InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes());
+
+        // must be successful, no verification, have two output URIs
+        Assert.assertTrue(result.success());
+        Assert.assertNull(result.mDecryptVerifyResult);
+
+        OpenPgpMetadata metadata = result.mMetadata.get(0);
+        Assert.assertEquals("text/plain", metadata.getMimeType());
+
+        Assert.assertEquals("charset should be set since it was guessed and not faulty",
+                "utf-8", metadata.getCharset());
+        Assert.assertTrue("charset should have been guessed",
+                result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS));
+    }
+
+    private InputDataResult runSimpleDataInputOperation(byte[] mimeContentBytes) throws FileNotFoundException {
+        ByteArrayOutputStream outStream1 = new ByteArrayOutputStream();
+        ByteArrayOutputStream outStream2 = new ByteArrayOutputStream();
+        ContentResolver mockResolver = mock(ContentResolver.class);
+
+        // fake openOutputStream first and second
+        when(mockResolver.openOutputStream(any(Uri.class), eq("w")))
+                .thenReturn(outStream1, outStream2);
+
+        // fake openInputStream
+        Uri fakeInputUri = Uri.parse("content://fake/1");
+        when(mockResolver.openInputStream(fakeInputUri)).thenReturn(
+                new ByteArrayInputStream(mimeContentBytes));
+
+        Uri fakeOutputUri1 = Uri.parse("content://fake/out/1");
+        when(mockResolver.insert(eq(TemporaryFileProvider.CONTENT_URI), any(ContentValues.class)))
+                .thenReturn(fakeOutputUri1);
+
+        // application which returns mockresolver
+        Application spyApplication = spy(RuntimeEnvironment.application);
+        when(spyApplication.getContentResolver()).thenReturn(mockResolver);
+
+        InputDataOperation op = new InputDataOperation(spyApplication,
+                new ProviderHelper(RuntimeEnvironment.application), null);
+
+        InputDataParcel input = new InputDataParcel(fakeInputUri, null);
+        return op.execute(input, new CryptoInputParcel());
     }
 
 }
diff --git a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java
new file mode 100644
index 000000000..a87c1595e
--- /dev/null
+++ b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java
@@ -0,0 +1,144 @@
+package org.sufficientlysecure.keychain.util;
+
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricGradleTestRunner;
+import org.robolectric.annotation.Config;
+import org.sufficientlysecure.keychain.WorkaroundBuildConfig;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(RobolectricGradleTestRunner.class)
+@Config(constants = WorkaroundBuildConfig.class, sdk = 21, manifest = "src/main/AndroidManifest.xml")
+public class CharsetVerifierTest {
+
+    @Test
+    public void testTypeImagePngAlwaysBinary() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("utf-8");
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "image/png", null);
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+
+        assertTrue("image/png should be marked as definitely binary", charsetVerifier.isDefinitelyBinary());
+        assertFalse("image/png should never be marked as, even if it is", charsetVerifier.isProbablyText());
+        assertNull("charset should be null", charsetVerifier.getCharset());
+    }
+
+    @Test
+    public void testUtf8SpecifiedButFaulty() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("utf-8");
+        bytes[4] = (byte) 0xc3;
+        bytes[5] = (byte) 0x28;
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/something", "utf-8");
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+
+        assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary());
+        assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText());
+        assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty());
+        assertFalse("charset was specified and should not be marked as guessed", charsetVerifier.isCharsetGuessed());
+        assertEquals("mimetype should be preserved", "text/something", charsetVerifier.getGuessedMimeType());
+        assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getCharset());
+        assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getMaybeFaultyCharset());
+    }
+
+    @Test
+    public void testUtf8GuessedAndFaulty() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("utf-8");
+        bytes[4] = (byte) 0xc3;
+        bytes[5] = (byte) 0x28;
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null);
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+
+        assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary());
+        assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText());
+        assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty());
+        assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed());
+        assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset());
+        assertEquals("mimetype should be set to text", "text/plain", charsetVerifier.getGuessedMimeType());
+        assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset());
+    }
+
+    @Test
+    public void testGuessedEncoding() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("utf-8");
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "application/octet-stream", null);
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+
+        assertFalse("application/octet-stream with text content is not definitely binary", charsetVerifier.isDefinitelyBinary());
+        assertTrue("application/octet-stream with text content should be probably text", charsetVerifier.isProbablyText());
+        assertFalse("detected charset should not be faulty", charsetVerifier.isCharsetFaulty());
+        assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed());
+        assertEquals("mimetype should be set to text", "text/plain", charsetVerifier.getGuessedMimeType());
+        assertEquals("guessed charset is utf-8", "utf-8", charsetVerifier.getCharset());
+    }
+
+    @Test
+    public void testWindows1252Faulty() throws Exception {
+        byte[] bytes = "bla bluh  ☭".getBytes("windows-1252");
+        bytes[2] = (byte) 0x9d;
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252");
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+
+        assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary());
+        assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText());
+        assertTrue("charset contained faulty characters", charsetVerifier.isCharsetFaulty());
+        assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed());
+        assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset());
+    }
+
+    @Test
+    public void testWindows1252Good() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("windows-1252");
+        // this is ‡ in windows-1252
+        bytes[2] = (byte) 0x87;
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252");
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+
+        assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary());
+        assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText());
+        assertFalse("charset contained no faulty characters", charsetVerifier.isCharsetFaulty());
+        assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed());
+        assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset());
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testReadAfterGetterShouldCrash() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("utf-8");
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null);
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+        charsetVerifier.isCharsetFaulty();
+
+        charsetVerifier.readBytesFromBuffer(0, bytes.length);
+    }
+
+
+    @Test
+    public void testStaggeredInput() throws Exception {
+        byte[] bytes = "bla bluh ☭".getBytes("utf-8");
+        bytes[4] = (byte) 0xc3;
+        bytes[5] = (byte) 0x28;
+
+        CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null);
+        for (int i = 0; i < bytes.length; i++) {
+            charsetVerifier.readBytesFromBuffer(i, i+1);
+        }
+
+        assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary());
+        assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText());
+        assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty());
+        assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed());
+        assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset());
+        assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset());
+    }
+
+}