From bdae99c0847556dd8103f172fc1836eb83ae4c4a Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Fri, 12 Feb 2016 17:08:09 +0100 Subject: mime: try to decode with given charset or utf-8 while file is read --- .../keychain/operations/InputDataOperation.java | 86 ++++++++++++- .../operations/results/OperationResult.java | 4 + OpenKeychain/src/main/res/values/strings.xml | 4 + .../keychain/pgp/InputDataOperationTest.java | 141 ++++++++++++++++++++- 4 files changed, 225 insertions(+), 10 deletions(-) (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java index bb8d6ad73..80f9d6368 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java @@ -23,6 +23,13 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; import android.content.ClipDescription; @@ -67,10 +74,15 @@ import org.sufficientlysecure.keychain.service.input.CryptoInputParcel; */ public class InputDataOperation extends BaseOperation { - final private byte[] buf = new byte[256]; + private final byte[] buf = new byte[256]; + private final ByteBuffer bufWrap; + private final CharBuffer dummyOutput; public InputDataOperation(Context context, ProviderHelper providerHelper, Progressable progressable) { super(context, providerHelper, progressable); + + bufWrap = ByteBuffer.wrap(buf); + dummyOutput = CharBuffer.allocate(256); } Uri mSignedDataUri; @@ -326,20 +338,82 @@ public class InputDataOperation extends BaseOperation { throw new IOException("Error getting file for writing!"); } + boolean isPossibleTextMimeType = ClipDescription.compareMimeTypes(mimeType, "application/octet-stream") + || ClipDescription.compareMimeTypes(mimeType, "application/x-download") + || ClipDescription.compareMimeTypes(mimeType, "text/*"); + + // If this data looks like text, we pipe the incoming data into a charset + // decoder, to see if the data is legal for the assumed charset. + String charset; + boolean charsetIsFaulty; + boolean charsetIsGuessed; + CharsetDecoder charsetDecoder = null; + if (isPossibleTextMimeType) { + charset = bd.getCharset(); + // the charset defaults to us-ascii, but we want to default to utf-8 + if (charset == null || "us-ascii".equals(charset)) { + charset = "utf-8"; + charsetIsGuessed = true; + } else { + charsetIsGuessed = false; + } + + try { + charsetDecoder = Charset.forName(charset).newDecoder(); + charsetDecoder.onMalformedInput(CodingErrorAction.REPORT); + charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); + charsetDecoder.reset(); + charsetIsFaulty = false; + } catch (UnsupportedCharsetException e) { + charsetIsFaulty = true; + } + } else { + charsetIsFaulty = true; + charsetIsGuessed = false; + charset = null; + } + int totalLength = 0; do { totalLength += len; out.write(buf, 0, len); + + if (isPossibleTextMimeType && !charsetIsFaulty) { + bufWrap.rewind(); + bufWrap.limit(len); + dummyOutput.rewind(); + CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, false); + if (result.isError()) { + charsetIsFaulty = true; + } + } } while ((len = is.read(buf)) > 0); - log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength)); + if (!charsetIsFaulty) { + bufWrap.rewind(); + bufWrap.limit(0); + dummyOutput.rewind(); + CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, true); + if (result.isError()) { + charsetIsFaulty = true; + } + } - String charset = bd.getCharset(); - // the charset defaults to us-ascii, but we want to default to utf-8 - if ("us-ascii".equals(charset)) { - charset = "utf-8"; + if (isPossibleTextMimeType) { + if (charsetIsFaulty && charsetIsGuessed) { + log.add(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN, 3, charset); + charset = null; + } else if (charsetIsFaulty) { + log.add(LogType.MSG_DATA_MIME_CHARSET_FAULTY, 3, charset); + } else if (charsetIsGuessed) { + log.add(LogType.MSG_DATA_MIME_CHARSET_GUESS, 3, charset); + } else { + log.add(LogType.MSG_DATA_MIME_CHARSET, 3, charset); + } } + log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength)); + OpenPgpMetadata metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength, charset); out.close(); diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java index f9c2db8e8..ec2fddbd0 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java @@ -860,6 +860,10 @@ public abstract class OperationResult implements Parcelable { MSG_DATA_MIME_FROM_EXTENSION (LogLevel.DEBUG, R.string.msg_data_mime_from_extension), MSG_DATA_MIME_FILENAME (LogLevel.DEBUG, R.string.msg_data_mime_filename), MSG_DATA_MIME_LENGTH (LogLevel.DEBUG, R.string.msg_data_mime_length), + MSG_DATA_MIME_CHARSET (LogLevel.DEBUG, R.string.msg_data_mime_charset), + MSG_DATA_MIME_CHARSET_FAULTY (LogLevel.WARN, R.string.msg_data_mime_charset_faulty), + MSG_DATA_MIME_CHARSET_GUESS (LogLevel.DEBUG, R.string.msg_data_mime_charset_guess), + MSG_DATA_MIME_CHARSET_UNKNOWN (LogLevel.DEBUG, R.string.msg_data_mime_charset_unknown), MSG_DATA_MIME (LogLevel.DEBUG, R.string.msg_data_mime), MSG_DATA_MIME_OK (LogLevel.INFO, R.string.msg_data_mime_ok), MSG_DATA_MIME_NONE (LogLevel.DEBUG, R.string.msg_data_mime_none), diff --git a/OpenKeychain/src/main/res/values/strings.xml b/OpenKeychain/src/main/res/values/strings.xml index f32eb0269..7d66c06a1 100644 --- a/OpenKeychain/src/main/res/values/strings.xml +++ b/OpenKeychain/src/main/res/values/strings.xml @@ -1401,6 +1401,10 @@ "Filename: '%s'" "Guessing MIME type from extension" "Content-Length: %s" + "Charset is '%s'" + "Charset is '%s', but decoding failed!" + "Charset appears to be '%s'" + "Charset is unknown, or data is not text." "Parsing MIME data structure" "Finished parsing" "No MIME structure found" diff --git a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java index 4277649fb..0464bf508 100644 --- a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java +++ b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java @@ -20,6 +20,7 @@ package org.sufficientlysecure.keychain.pgp; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.PrintStream; import java.security.Security; import java.util.ArrayList; @@ -30,18 +31,20 @@ import android.content.ContentValues; import android.net.Uri; import junit.framework.Assert; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; +import org.openintents.openpgp.OpenPgpMetadata; import org.robolectric.RobolectricGradleTestRunner; import org.robolectric.RuntimeEnvironment; import org.robolectric.annotation.Config; import org.robolectric.shadows.ShadowLog; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.sufficientlysecure.keychain.WorkaroundBuildConfig; import org.sufficientlysecure.keychain.operations.InputDataOperation; import org.sufficientlysecure.keychain.operations.results.InputDataResult; +import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType; import org.sufficientlysecure.keychain.provider.ProviderHelper; import org.sufficientlysecure.keychain.provider.TemporaryFileProvider; import org.sufficientlysecure.keychain.service.InputDataParcel; @@ -80,20 +83,20 @@ public class InputDataOperationTest { } @Test - public void testMimeDecoding () throws Exception { + public void testMimeDecoding() throws Exception { String mimeMail = "Content-Type: multipart/mixed; boundary=\"=-26BafqxfXmhVNMbYdoIi\"\n" + "\n" + "--=-26BafqxfXmhVNMbYdoIi\n" + - "Content-Type: text/plain\n" + + "Content-Type: text/plain; charset=utf-8\n" + "Content-Transfer-Encoding: quoted-printable\n" + "Content-Disposition: attachment; filename=data.txt\n" + "\n" + "message part 1\n" + "\n" + "--=-26BafqxfXmhVNMbYdoIi\n" + - "Content-Type: text/testvalue\n" + + "Content-Type: text/testvalue; charset=iso-8859-1\n" + "Content-Description: Dummy content description\n" + "\n" + "message part 2.1\n" + @@ -156,7 +159,137 @@ public class InputDataOperationTest { Assert.assertEquals("second part must have expected content", "message part 2.1\nmessage part 2.2\n", new String(outStream2.toByteArray())); + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + Assert.assertEquals("utf-8", metadata.getCharset()); + + metadata = result.mMetadata.get(1); + Assert.assertEquals("text/testvalue", metadata.getMimeType()); + Assert.assertEquals("iso-8859-1", metadata.getCharset()); + } + + @Test + public void testMimeDecodingExplicitFaultyCharset() throws Exception { + + String mimeContent = "Content-Type: text/plain; charset=utf-8\n" + + "\n" + + "message with binary data in it\n"; + + byte[] data = mimeContent.getBytes(); + data[60] = (byte) 0xc3; + data[61] = (byte) 0x28; + + InputDataResult result = runSimpleDataInputOperation(data); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + + Assert.assertEquals("charset should be set since it was explicitly specified", + "utf-8", metadata.getCharset()); + Assert.assertTrue("faulty charset should have been detected", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_FAULTY)); + } + + @Test + public void testMimeDecodingImplicitFaultyCharset() throws Exception { + + String mimeContent = "Content-Type: text/plain\n" + + "\n" + + "message with binary data in it\n"; + + byte[] data = mimeContent.getBytes(); + data[45] = (byte) 0xc3; + data[46] = (byte) 0x28; + + InputDataResult result = runSimpleDataInputOperation(data); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + + Assert.assertNull("charset was bad so it should not be set", metadata.getCharset()); + Assert.assertTrue("faulty charset should have been detected", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN)); + } + + @Test + public void testMimeDecodingImplicitGuessedCharset() throws Exception { + + String mimeContent = "Content-Type: text/plain\n" + + "\n" + + "proper, utf-8 encoded message ☭\n"; + + InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes()); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("text/plain", metadata.getMimeType()); + Assert.assertEquals("charset should be set since it was guessed and not faulty", + "utf-8", metadata.getCharset()); + Assert.assertTrue("charset should have been guessed", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS)); + } + + @Test + public void testMimeDecodingOctetStreamGuessedCharset() throws Exception { + + String mimeContent = "Content-Type: application/octet-stream\n" + + "\n" + + "proper, utf-8 encoded message ☭\n"; + + InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes()); + + // must be successful, no verification, have two output URIs + Assert.assertTrue(result.success()); + Assert.assertNull(result.mDecryptVerifyResult); + + OpenPgpMetadata metadata = result.mMetadata.get(0); + Assert.assertEquals("application/octet-stream", metadata.getMimeType()); + + Assert.assertEquals("charset should be set since it was guessed and not faulty", + "utf-8", metadata.getCharset()); + Assert.assertTrue("charset should have been guessed", + result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS)); + } + + private InputDataResult runSimpleDataInputOperation(byte[] mimeContentBytes) throws FileNotFoundException { + ByteArrayOutputStream outStream1 = new ByteArrayOutputStream(); + ByteArrayOutputStream outStream2 = new ByteArrayOutputStream(); + ContentResolver mockResolver = mock(ContentResolver.class); + + // fake openOutputStream first and second + when(mockResolver.openOutputStream(any(Uri.class), eq("w"))) + .thenReturn(outStream1, outStream2); + + // fake openInputStream + Uri fakeInputUri = Uri.parse("content://fake/1"); + when(mockResolver.openInputStream(fakeInputUri)).thenReturn( + new ByteArrayInputStream(mimeContentBytes)); + + Uri fakeOutputUri1 = Uri.parse("content://fake/out/1"); + when(mockResolver.insert(eq(TemporaryFileProvider.CONTENT_URI), any(ContentValues.class))) + .thenReturn(fakeOutputUri1); + + // application which returns mockresolver + Application spyApplication = spy(RuntimeEnvironment.application); + when(spyApplication.getContentResolver()).thenReturn(mockResolver); + + InputDataOperation op = new InputDataOperation(spyApplication, + new ProviderHelper(RuntimeEnvironment.application), null); + + InputDataParcel input = new InputDataParcel(fakeInputUri, null); + return op.execute(input, new CryptoInputParcel()); } } -- cgit v1.2.3 From daf243082c6cd7fb7f518bfbf0acf9acafaa27d1 Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Mon, 22 Feb 2016 21:12:36 +0100 Subject: externalize CharsetVerifier, add looksLikeText to OpenPgpMetadata object --- .../keychain/operations/CharsetVerifier.java | 122 +++++++++++++++++++++ .../keychain/operations/InputDataOperation.java | 94 +++------------- .../keychain/pgp/PgpDecryptVerifyOperation.java | 19 ++-- .../keychain/pgp/InputDataOperationTest.java | 4 + 4 files changed, 152 insertions(+), 87 deletions(-) create mode 100644 OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java new file mode 100644 index 000000000..5d63ced22 --- /dev/null +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java @@ -0,0 +1,122 @@ +package org.sufficientlysecure.keychain.operations; + + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; + +import android.content.ClipDescription; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + + +public class CharsetVerifier { + + private final ByteBuffer bufWrap; + private final CharBuffer dummyOutput; + + private final CharsetDecoder charsetDecoder; + + private boolean isFinished; + private boolean isFaulty; + private boolean isGuessed; + private boolean isPossibleTextMimeType; + private boolean isTextMimeType; + private String charset; + + public CharsetVerifier(@NonNull byte[] buf, String mimeType, @Nullable String charset) { + + isPossibleTextMimeType = ClipDescription.compareMimeTypes(mimeType, "application/octet-stream") + || ClipDescription.compareMimeTypes(mimeType, "application/x-download") + || ClipDescription.compareMimeTypes(mimeType, "text/*"); + if (!isPossibleTextMimeType) { + charsetDecoder = null; + bufWrap = null; + dummyOutput = null; + return; + } + isTextMimeType = ClipDescription.compareMimeTypes(mimeType, "text/*"); + + bufWrap = ByteBuffer.wrap(buf); + dummyOutput = CharBuffer.allocate(buf.length); + + // the charset defaults to us-ascii, but we want to default to utf-8 + if (charset == null || "us-ascii".equals(charset)) { + charset = "utf-8"; + isGuessed = true; + } else { + isGuessed = false; + } + this.charset = charset; + + charsetDecoder = Charset.forName(charset).newDecoder(); + charsetDecoder.onMalformedInput(CodingErrorAction.REPORT); + charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); + charsetDecoder.reset(); + } + + public void write(int pos, int len) { + if (isFinished) { + throw new IllegalStateException("cannot write again after reading charset status!"); + } + if (isFaulty || bufWrap == null) { + return; + } + bufWrap.rewind(); + bufWrap.position(pos); + bufWrap.limit(len); + dummyOutput.rewind(); + CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, false); + if (result.isError()) { + isFaulty = true; + } + } + + private void finishIfNecessary() { + if (isFinished || isFaulty || bufWrap == null) { + return; + } + isFinished = true; + bufWrap.rewind(); + bufWrap.limit(0); + dummyOutput.rewind(); + CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, true); + if (result.isError()) { + isFaulty = true; + } + } + + public boolean isCharsetFaulty() { + finishIfNecessary(); + return isFaulty; + } + + public boolean isCharsetGuessed() { + finishIfNecessary(); + return isGuessed; + } + + public String getCharset() { + finishIfNecessary(); + if (!isPossibleTextMimeType || (isGuessed && isFaulty)) { + return null; + } + return charset; + } + + public String getMaybeFaultyCharset() { + return charset; + } + + public boolean isDefinitelyBinary() { + finishIfNecessary(); + return !isTextMimeType && (!isPossibleTextMimeType || (isGuessed && isFaulty)); + } + + public boolean isProbablyText() { + return isTextMimeType || isPossibleTextMimeType && (!isGuessed || !isFaulty); + } +} diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java index 80f9d6368..ff9377581 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java @@ -23,13 +23,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; import android.content.ClipDescription; @@ -75,14 +68,9 @@ import org.sufficientlysecure.keychain.service.input.CryptoInputParcel; public class InputDataOperation extends BaseOperation { private final byte[] buf = new byte[256]; - private final ByteBuffer bufWrap; - private final CharBuffer dummyOutput; public InputDataOperation(Context context, ProviderHelper providerHelper, Progressable progressable) { super(context, providerHelper, progressable); - - bufWrap = ByteBuffer.wrap(buf); - dummyOutput = CharBuffer.allocate(256); } Uri mSignedDataUri; @@ -338,83 +326,37 @@ public class InputDataOperation extends BaseOperation { throw new IOException("Error getting file for writing!"); } - boolean isPossibleTextMimeType = ClipDescription.compareMimeTypes(mimeType, "application/octet-stream") - || ClipDescription.compareMimeTypes(mimeType, "application/x-download") - || ClipDescription.compareMimeTypes(mimeType, "text/*"); - // If this data looks like text, we pipe the incoming data into a charset // decoder, to see if the data is legal for the assumed charset. - String charset; - boolean charsetIsFaulty; - boolean charsetIsGuessed; - CharsetDecoder charsetDecoder = null; - if (isPossibleTextMimeType) { - charset = bd.getCharset(); - // the charset defaults to us-ascii, but we want to default to utf-8 - if (charset == null || "us-ascii".equals(charset)) { - charset = "utf-8"; - charsetIsGuessed = true; - } else { - charsetIsGuessed = false; - } - - try { - charsetDecoder = Charset.forName(charset).newDecoder(); - charsetDecoder.onMalformedInput(CodingErrorAction.REPORT); - charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); - charsetDecoder.reset(); - charsetIsFaulty = false; - } catch (UnsupportedCharsetException e) { - charsetIsFaulty = true; - } - } else { - charsetIsFaulty = true; - charsetIsGuessed = false; - charset = null; - } + String charset = bd.getCharset(); + CharsetVerifier charsetVerifier = new CharsetVerifier(buf, mimeType, charset); int totalLength = 0; do { totalLength += len; out.write(buf, 0, len); - - if (isPossibleTextMimeType && !charsetIsFaulty) { - bufWrap.rewind(); - bufWrap.limit(len); - dummyOutput.rewind(); - CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, false); - if (result.isError()) { - charsetIsFaulty = true; - } - } + charsetVerifier.write(0, len); } while ((len = is.read(buf)) > 0); - if (!charsetIsFaulty) { - bufWrap.rewind(); - bufWrap.limit(0); - dummyOutput.rewind(); - CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, true); - if (result.isError()) { - charsetIsFaulty = true; - } - } + log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength)); - if (isPossibleTextMimeType) { - if (charsetIsFaulty && charsetIsGuessed) { - log.add(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN, 3, charset); - charset = null; - } else if (charsetIsFaulty) { - log.add(LogType.MSG_DATA_MIME_CHARSET_FAULTY, 3, charset); - } else if (charsetIsGuessed) { - log.add(LogType.MSG_DATA_MIME_CHARSET_GUESS, 3, charset); + OpenPgpMetadata metadata; + if (charsetVerifier.isDefinitelyBinary()) { + metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength); + } else { + if (charsetVerifier.isCharsetFaulty() && charsetVerifier.isCharsetGuessed()) { + log.add(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN, 3, charsetVerifier.getMaybeFaultyCharset()); + } else if (charsetVerifier.isCharsetFaulty()) { + log.add(LogType.MSG_DATA_MIME_CHARSET_FAULTY, 3, charsetVerifier.getCharset()); + } else if (charsetVerifier.isCharsetGuessed()) { + log.add(LogType.MSG_DATA_MIME_CHARSET_GUESS, 3, charsetVerifier.getCharset()); } else { - log.add(LogType.MSG_DATA_MIME_CHARSET, 3, charset); + log.add(LogType.MSG_DATA_MIME_CHARSET, 3, charsetVerifier.getCharset()); } - } - - log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength)); - OpenPgpMetadata metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength, charset); + metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength, + charsetVerifier.getCharset(), charsetVerifier.isProbablyText()); + } out.close(); outputUris.add(uri); diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java index c4755c7c5..59ba8df5f 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java @@ -377,9 +377,11 @@ public class PgpDecryptVerifyOperation extends BaseOperation Date: Mon, 22 Feb 2016 21:59:57 +0100 Subject: decryptVerify: use CharsetVerifier to guess whether data is binary or not --- .../keychain/pgp/PgpDecryptVerifyOperation.java | 29 ++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java index 59ba8df5f..feff78726 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java @@ -35,8 +35,6 @@ import android.support.annotation.NonNull; import android.text.TextUtils; import android.webkit.MimeTypeMap; -import org.openintents.openpgp.OpenPgpDecryptionResult; -import org.openintents.openpgp.OpenPgpMetadata; import org.bouncycastle.bcpg.ArmoredInputStream; import org.bouncycastle.openpgp.PGPCompressedData; import org.bouncycastle.openpgp.PGPDataValidationException; @@ -56,10 +54,13 @@ import org.bouncycastle.openpgp.operator.jcajce.CachingDataDecryptorFactory; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; import org.bouncycastle.openpgp.operator.jcajce.JcePBEDataDecryptorFactoryBuilder; import org.bouncycastle.util.encoders.DecoderException; +import org.openintents.openpgp.OpenPgpDecryptionResult; +import org.openintents.openpgp.OpenPgpMetadata; import org.sufficientlysecure.keychain.Constants; import org.sufficientlysecure.keychain.Constants.key; import org.sufficientlysecure.keychain.R; import org.sufficientlysecure.keychain.operations.BaseOperation; +import org.sufficientlysecure.keychain.operations.CharsetVerifier; import org.sufficientlysecure.keychain.operations.results.DecryptVerifyResult; import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType; import org.sufficientlysecure.keychain.operations.results.OperationResult.OperationLog; @@ -377,11 +378,9 @@ public class PgpDecryptVerifyOperation extends BaseOperation 0) { // Log.d(Constants.TAG, "read bytes: " + length); if (out != null) { @@ -449,6 +449,8 @@ public class PgpDecryptVerifyOperation extends BaseOperation firstBytes.length ? firstBytes.length : length); @@ -491,8 +493,15 @@ public class PgpDecryptVerifyOperation extends BaseOperation Date: Mon, 22 Feb 2016 22:22:18 +0100 Subject: decrypt: always show "View in OpenKeychain" as intent chooser option iff isLikelyText --- .../sufficientlysecure/keychain/ui/DecryptListFragment.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptListFragment.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptListFragment.java index 9419cf8ce..9ffd95ae9 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptListFragment.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptListFragment.java @@ -591,6 +591,18 @@ public class DecryptListFragment Intent chooserIntent = Intent.createChooser(intent, getString(R.string.intent_show)); chooserIntent.setFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION); + + if (!share && metadata.isLooksLikeText()) { + LabeledIntent internalIntent = new LabeledIntent( + new Intent(intent) + .setClass(activity, DisplayTextActivity.class) + .putExtra(DisplayTextActivity.EXTRA_RESULT, result.mDecryptVerifyResult) + .putExtra(DisplayTextActivity.EXTRA_METADATA, metadata), + BuildConfig.APPLICATION_ID, R.string.view_internal, R.mipmap.ic_launcher); + chooserIntent.putExtra(Intent.EXTRA_INITIAL_INTENTS, + new Parcelable[] { internalIntent }); + } + startActivity(chooserIntent); } -- cgit v1.2.3 From 4df63ccdeb8bd26f507c88980b360bdc367faa0f Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Mon, 22 Feb 2016 22:22:45 +0100 Subject: displaytext: add "View Log" option, move "Copy to clipboard" into overflow menu --- .../sufficientlysecure/keychain/ui/DecryptFragment.java | 11 +++++++++++ .../keychain/ui/DisplayTextFragment.java | 6 +++++- OpenKeychain/src/main/res/menu/decrypt_menu.xml | 15 ++++++++++----- OpenKeychain/src/main/res/values/strings.xml | 2 +- 4 files changed, 27 insertions(+), 7 deletions(-) (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptFragment.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptFragment.java index 351b62ba7..bc3470b0a 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptFragment.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptFragment.java @@ -19,6 +19,7 @@ package org.sufficientlysecure.keychain.ui; import java.util.ArrayList; +import android.app.Activity; import android.content.Intent; import android.database.Cursor; import android.net.Uri; @@ -472,6 +473,16 @@ public abstract class DecryptFragment extends Fragment implements LoaderManager. protected abstract void onVerifyLoaded(boolean hideErrorOverlay); + public void startDisplayLogActivity() { + Activity activity = getActivity(); + if (activity == null) { + return; + } + Intent intent = new Intent(activity, LogDisplayActivity.class); + intent.putExtra(LogDisplayFragment.EXTRA_RESULT, mDecryptVerifyResult); + activity.startActivity(intent); + } + @Override public void onActivityResult(int requestCode, int resultCode, Intent data) { if (mImportOpHelper != null) { diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DisplayTextFragment.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DisplayTextFragment.java index 1060714f0..97f723168 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DisplayTextFragment.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DisplayTextFragment.java @@ -103,7 +103,7 @@ public class DisplayTextFragment extends DecryptFragment { Bundle args = getArguments(); String plaintext = args.getString(ARG_PLAINTEXT); - DecryptVerifyResult result = args.getParcelable(ARG_DECRYPT_VERIFY_RESULT); + DecryptVerifyResult result = args.getParcelable(ARG_DECRYPT_VERIFY_RESULT); // display signature result in activity mText.setText(plaintext); @@ -137,6 +137,10 @@ public class DisplayTextFragment extends DecryptFragment { copyToClipboard(mText.getText().toString()); break; } + case R.id.decrypt_view_log: { + startDisplayLogActivity(); + break; + } default: { return super.onOptionsItemSelected(item); } diff --git a/OpenKeychain/src/main/res/menu/decrypt_menu.xml b/OpenKeychain/src/main/res/menu/decrypt_menu.xml index 0b81ea1db..4e31fb6a6 100644 --- a/OpenKeychain/src/main/res/menu/decrypt_menu.xml +++ b/OpenKeychain/src/main/res/menu/decrypt_menu.xml @@ -2,16 +2,21 @@ + + + app:showAsAction="never" /> + android:id="@+id/decrypt_view_log" + android:title="@string/btn_view_log" + app:showAsAction="never" /> diff --git a/OpenKeychain/src/main/res/values/strings.xml b/OpenKeychain/src/main/res/values/strings.xml index 7d66c06a1..9920d8057 100644 --- a/OpenKeychain/src/main/res/values/strings.xml +++ b/OpenKeychain/src/main/res/values/strings.xml @@ -95,7 +95,7 @@ "Add file(s)" "Share" "Open with…" - "Copy decrypted text" + "Copy to clipboard" "Read from clipboard" "Select input file" "Encrypt files" -- cgit v1.2.3 From a0c90f0ad57b66d6e7e0957526748b2e4a239063 Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Mon, 22 Feb 2016 23:38:02 +0100 Subject: documentation and cleanup --- .../keychain/operations/CharsetVerifier.java | 24 ++++++++++++++++++++-- .../keychain/operations/InputDataOperation.java | 2 +- .../keychain/pgp/PgpDecryptVerifyOperation.java | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java index 5d63ced22..c563beeac 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java @@ -12,7 +12,14 @@ import android.content.ClipDescription; import android.support.annotation.NonNull; import android.support.annotation.Nullable; - +/** This class can be used to guess whether a stream of data is encoded in a given + * charset or not. + * + * An object of this class must be initialized with a byte[] buffer, which should + * be filled with data, then processed with {@link #readBytesFromBuffer}. This can + * be done any number of times. Once all data has been read, a final status can be + * read using the getter methods. + */ public class CharsetVerifier { private final ByteBuffer bufWrap; @@ -58,7 +65,7 @@ public class CharsetVerifier { charsetDecoder.reset(); } - public void write(int pos, int len) { + public void readBytesFromBuffer(int pos, int len) { if (isFinished) { throw new IllegalStateException("cannot write again after reading charset status!"); } @@ -111,12 +118,25 @@ public class CharsetVerifier { return charset; } + /** Returns true if the data which was read is definitely binary. + * + * This can happen when either the supplied mimeType indicated a non-ambiguous + * binary data type, or if we guessed a charset but got errors while decoding. + */ public boolean isDefinitelyBinary() { finishIfNecessary(); return !isTextMimeType && (!isPossibleTextMimeType || (isGuessed && isFaulty)); } + /** Returns true iff the data which was read is probably (or + * definitely) text. + * + * The corner case where isDefinitelyBinary returns false but isProbablyText + * returns true is where the charset was provided by the data (so is not + * guessed) but is still faulty. + */ public boolean isProbablyText() { + finishIfNecessary(); return isTextMimeType || isPossibleTextMimeType && (!isGuessed || !isFaulty); } } diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java index ff9377581..74d94d83e 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java @@ -335,7 +335,7 @@ public class InputDataOperation extends BaseOperation { do { totalLength += len; out.write(buf, 0, len); - charsetVerifier.write(0, len); + charsetVerifier.readBytesFromBuffer(0, len); } while ((len = is.read(buf)) > 0); log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength)); diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java index feff78726..90c1b1242 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java @@ -449,7 +449,7 @@ public class PgpDecryptVerifyOperation extends BaseOperation Date: Tue, 23 Feb 2016 14:14:12 +0100 Subject: move CharsetVerifier to utils package --- .../keychain/operations/CharsetVerifier.java | 142 --------------------- .../keychain/operations/InputDataOperation.java | 1 + .../keychain/pgp/PgpDecryptVerifyOperation.java | 2 +- .../keychain/util/CharsetVerifier.java | 142 +++++++++++++++++++++ 4 files changed, 144 insertions(+), 143 deletions(-) delete mode 100644 OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java create mode 100644 OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/CharsetVerifier.java (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java deleted file mode 100644 index c563beeac..000000000 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/CharsetVerifier.java +++ /dev/null @@ -1,142 +0,0 @@ -package org.sufficientlysecure.keychain.operations; - - -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; -import java.nio.charset.CodingErrorAction; - -import android.content.ClipDescription; -import android.support.annotation.NonNull; -import android.support.annotation.Nullable; - -/** This class can be used to guess whether a stream of data is encoded in a given - * charset or not. - * - * An object of this class must be initialized with a byte[] buffer, which should - * be filled with data, then processed with {@link #readBytesFromBuffer}. This can - * be done any number of times. Once all data has been read, a final status can be - * read using the getter methods. - */ -public class CharsetVerifier { - - private final ByteBuffer bufWrap; - private final CharBuffer dummyOutput; - - private final CharsetDecoder charsetDecoder; - - private boolean isFinished; - private boolean isFaulty; - private boolean isGuessed; - private boolean isPossibleTextMimeType; - private boolean isTextMimeType; - private String charset; - - public CharsetVerifier(@NonNull byte[] buf, String mimeType, @Nullable String charset) { - - isPossibleTextMimeType = ClipDescription.compareMimeTypes(mimeType, "application/octet-stream") - || ClipDescription.compareMimeTypes(mimeType, "application/x-download") - || ClipDescription.compareMimeTypes(mimeType, "text/*"); - if (!isPossibleTextMimeType) { - charsetDecoder = null; - bufWrap = null; - dummyOutput = null; - return; - } - isTextMimeType = ClipDescription.compareMimeTypes(mimeType, "text/*"); - - bufWrap = ByteBuffer.wrap(buf); - dummyOutput = CharBuffer.allocate(buf.length); - - // the charset defaults to us-ascii, but we want to default to utf-8 - if (charset == null || "us-ascii".equals(charset)) { - charset = "utf-8"; - isGuessed = true; - } else { - isGuessed = false; - } - this.charset = charset; - - charsetDecoder = Charset.forName(charset).newDecoder(); - charsetDecoder.onMalformedInput(CodingErrorAction.REPORT); - charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); - charsetDecoder.reset(); - } - - public void readBytesFromBuffer(int pos, int len) { - if (isFinished) { - throw new IllegalStateException("cannot write again after reading charset status!"); - } - if (isFaulty || bufWrap == null) { - return; - } - bufWrap.rewind(); - bufWrap.position(pos); - bufWrap.limit(len); - dummyOutput.rewind(); - CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, false); - if (result.isError()) { - isFaulty = true; - } - } - - private void finishIfNecessary() { - if (isFinished || isFaulty || bufWrap == null) { - return; - } - isFinished = true; - bufWrap.rewind(); - bufWrap.limit(0); - dummyOutput.rewind(); - CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, true); - if (result.isError()) { - isFaulty = true; - } - } - - public boolean isCharsetFaulty() { - finishIfNecessary(); - return isFaulty; - } - - public boolean isCharsetGuessed() { - finishIfNecessary(); - return isGuessed; - } - - public String getCharset() { - finishIfNecessary(); - if (!isPossibleTextMimeType || (isGuessed && isFaulty)) { - return null; - } - return charset; - } - - public String getMaybeFaultyCharset() { - return charset; - } - - /** Returns true if the data which was read is definitely binary. - * - * This can happen when either the supplied mimeType indicated a non-ambiguous - * binary data type, or if we guessed a charset but got errors while decoding. - */ - public boolean isDefinitelyBinary() { - finishIfNecessary(); - return !isTextMimeType && (!isPossibleTextMimeType || (isGuessed && isFaulty)); - } - - /** Returns true iff the data which was read is probably (or - * definitely) text. - * - * The corner case where isDefinitelyBinary returns false but isProbablyText - * returns true is where the charset was provided by the data (so is not - * guessed) but is still faulty. - */ - public boolean isProbablyText() { - finishIfNecessary(); - return isTextMimeType || isPossibleTextMimeType && (!isGuessed || !isFaulty); - } -} diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java index 74d94d83e..0de878232 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java @@ -53,6 +53,7 @@ import org.sufficientlysecure.keychain.provider.ProviderHelper; import org.sufficientlysecure.keychain.provider.TemporaryFileProvider; import org.sufficientlysecure.keychain.service.InputDataParcel; import org.sufficientlysecure.keychain.service.input.CryptoInputParcel; +import org.sufficientlysecure.keychain.util.CharsetVerifier; /** This operation deals with input data, trying to determine its type as it goes. diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java index 90c1b1242..b0d39d88f 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java @@ -60,7 +60,7 @@ import org.sufficientlysecure.keychain.Constants; import org.sufficientlysecure.keychain.Constants.key; import org.sufficientlysecure.keychain.R; import org.sufficientlysecure.keychain.operations.BaseOperation; -import org.sufficientlysecure.keychain.operations.CharsetVerifier; +import org.sufficientlysecure.keychain.util.CharsetVerifier; import org.sufficientlysecure.keychain.operations.results.DecryptVerifyResult; import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType; import org.sufficientlysecure.keychain.operations.results.OperationResult.OperationLog; diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/CharsetVerifier.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/CharsetVerifier.java new file mode 100644 index 000000000..c1d11cc26 --- /dev/null +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/CharsetVerifier.java @@ -0,0 +1,142 @@ +package org.sufficientlysecure.keychain.util; + + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; + +import android.content.ClipDescription; +import android.support.annotation.NonNull; +import android.support.annotation.Nullable; + +/** This class can be used to guess whether a stream of data is encoded in a given + * charset or not. + * + * An object of this class must be initialized with a byte[] buffer, which should + * be filled with data, then processed with {@link #readBytesFromBuffer}. This can + * be done any number of times. Once all data has been read, a final status can be + * read using the getter methods. + */ +public class CharsetVerifier { + + private final ByteBuffer bufWrap; + private final CharBuffer dummyOutput; + + private final CharsetDecoder charsetDecoder; + + private boolean isFinished; + private boolean isFaulty; + private boolean isGuessed; + private boolean isPossibleTextMimeType; + private boolean isTextMimeType; + private String charset; + + public CharsetVerifier(@NonNull byte[] buf, String mimeType, @Nullable String charset) { + + isPossibleTextMimeType = ClipDescription.compareMimeTypes(mimeType, "application/octet-stream") + || ClipDescription.compareMimeTypes(mimeType, "application/x-download") + || ClipDescription.compareMimeTypes(mimeType, "text/*"); + if (!isPossibleTextMimeType) { + charsetDecoder = null; + bufWrap = null; + dummyOutput = null; + return; + } + isTextMimeType = ClipDescription.compareMimeTypes(mimeType, "text/*"); + + bufWrap = ByteBuffer.wrap(buf); + dummyOutput = CharBuffer.allocate(buf.length); + + // the charset defaults to us-ascii, but we want to default to utf-8 + if (charset == null || "us-ascii".equals(charset)) { + charset = "utf-8"; + isGuessed = true; + } else { + isGuessed = false; + } + this.charset = charset; + + charsetDecoder = Charset.forName(charset).newDecoder(); + charsetDecoder.onMalformedInput(CodingErrorAction.REPORT); + charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); + charsetDecoder.reset(); + } + + public void readBytesFromBuffer(int pos, int len) { + if (isFinished) { + throw new IllegalStateException("cannot write again after reading charset status!"); + } + if (isFaulty || bufWrap == null) { + return; + } + bufWrap.rewind(); + bufWrap.position(pos); + bufWrap.limit(len); + dummyOutput.rewind(); + CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, false); + if (result.isError()) { + isFaulty = true; + } + } + + private void finishIfNecessary() { + if (isFinished || isFaulty || bufWrap == null) { + return; + } + isFinished = true; + bufWrap.rewind(); + bufWrap.limit(0); + dummyOutput.rewind(); + CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, true); + if (result.isError()) { + isFaulty = true; + } + } + + public boolean isCharsetFaulty() { + finishIfNecessary(); + return isFaulty; + } + + public boolean isCharsetGuessed() { + finishIfNecessary(); + return isGuessed; + } + + public String getCharset() { + finishIfNecessary(); + if (!isPossibleTextMimeType || (isGuessed && isFaulty)) { + return null; + } + return charset; + } + + public String getMaybeFaultyCharset() { + return charset; + } + + /** Returns true if the data which was read is definitely binary. + * + * This can happen when either the supplied mimeType indicated a non-ambiguous + * binary data type, or if we guessed a charset but got errors while decoding. + */ + public boolean isDefinitelyBinary() { + finishIfNecessary(); + return !isTextMimeType && (!isPossibleTextMimeType || (isGuessed && isFaulty)); + } + + /** Returns true iff the data which was read is probably (or + * definitely) text. + * + * The corner case where isDefinitelyBinary returns false but isProbablyText + * returns true is where the charset was provided by the data (so is not + * guessed) but is still faulty. + */ + public boolean isProbablyText() { + finishIfNecessary(); + return isTextMimeType || isPossibleTextMimeType && (!isGuessed || !isFaulty); + } +} -- cgit v1.2.3 From 8714a5eac42449dc3584a6bc14a7404c6a4686b5 Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Tue, 23 Feb 2016 14:15:47 +0100 Subject: add test for CharsetVerifier --- .../keychain/util/CharsetVerifierTest.java | 141 +++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java new file mode 100644 index 000000000..8ad6a021f --- /dev/null +++ b/OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java @@ -0,0 +1,141 @@ +package org.sufficientlysecure.keychain.util; + + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricGradleTestRunner; +import org.robolectric.annotation.Config; +import org.sufficientlysecure.keychain.WorkaroundBuildConfig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(RobolectricGradleTestRunner.class) +@Config(constants = WorkaroundBuildConfig.class, sdk = 21, manifest = "src/main/AndroidManifest.xml") +public class CharsetVerifierTest { + + @Test + public void testTypeImagePngAlwaysBinary() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "image/png", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertTrue("image/png should be marked as definitely binary", charsetVerifier.isDefinitelyBinary()); + assertFalse("image/png should never be marked as, even if it is", charsetVerifier.isProbablyText()); + assertNull("charset should be null", charsetVerifier.getCharset()); + } + + @Test + public void testUtf8SpecifiedButFaulty() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + bytes[4] = (byte) 0xc3; + bytes[5] = (byte) 0x28; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "utf-8"); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText()); + assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty()); + assertFalse("charset was specified and should not be marked as guessed", charsetVerifier.isCharsetGuessed()); + assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getCharset()); + assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getMaybeFaultyCharset()); + } + + @Test + public void testUtf8GuessedAndFaulty() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + bytes[4] = (byte) 0xc3; + bytes[5] = (byte) 0x28; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText()); + assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty()); + assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed()); + assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset()); + assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset()); + } + + @Test + public void testGuessedEncoding() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "application/octet-stream", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("application/octet-stream with text content is not definitely binary", charsetVerifier.isDefinitelyBinary()); + assertTrue("application/octet-stream with text content should be probably text", charsetVerifier.isProbablyText()); + assertFalse("detected charset should not be faulty", charsetVerifier.isCharsetFaulty()); + assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed()); + assertEquals("guessed charset is utf-8", "utf-8", charsetVerifier.getCharset()); + } + + @Test + public void testWindows1252Faulty() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("windows-1252"); + bytes[2] = (byte) 0x9d; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252"); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText()); + assertTrue("charset contained faulty characters", charsetVerifier.isCharsetFaulty()); + assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed()); + assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset()); + } + + @Test + public void testWindows1252Good() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("windows-1252"); + // this is ‡ in windows-1252 + bytes[2] = (byte) 0x87; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252"); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + + assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText()); + assertFalse("charset contained no faulty characters", charsetVerifier.isCharsetFaulty()); + assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed()); + assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset()); + } + + @Test(expected = IllegalStateException.class) + public void testReadAfterGetterShouldCrash() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null); + charsetVerifier.readBytesFromBuffer(0, bytes.length); + charsetVerifier.isCharsetFaulty(); + + charsetVerifier.readBytesFromBuffer(0, bytes.length); + } + + + @Test + public void testStaggeredInput() throws Exception { + byte[] bytes = "bla bluh ☭".getBytes("utf-8"); + bytes[4] = (byte) 0xc3; + bytes[5] = (byte) 0x28; + + CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null); + for (int i = 0; i < bytes.length; i++) { + charsetVerifier.readBytesFromBuffer(i, i+1); + } + + assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary()); + assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText()); + assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty()); + assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed()); + assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset()); + assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset()); + } + +} -- cgit v1.2.3 From 03e695c6509ee6ada0ad6a0a21181277ba298c34 Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Tue, 23 Feb 2016 14:55:23 +0100 Subject: ditch isLikelyText flag, set mimeType to text/plain in that case --- .../keychain/operations/InputDataOperation.java | 4 ++-- .../keychain/pgp/PgpDecryptVerifyOperation.java | 18 +++++++----------- .../keychain/ui/DecryptListFragment.java | 2 +- .../keychain/util/CharsetVerifier.java | 22 +++++++++++++++++----- .../keychain/pgp/InputDataOperationTest.java | 6 +----- .../keychain/util/CharsetVerifierTest.java | 5 ++++- 6 files changed, 32 insertions(+), 25 deletions(-) (limited to 'OpenKeychain') diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java index 0de878232..43fc11b84 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java @@ -355,8 +355,8 @@ public class InputDataOperation extends BaseOperation { log.add(LogType.MSG_DATA_MIME_CHARSET, 3, charsetVerifier.getCharset()); } - metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength, - charsetVerifier.getCharset(), charsetVerifier.isProbablyText()); + metadata = new OpenPgpMetadata(mFilename, charsetVerifier.getGuessedMimeType(), 0L, totalLength, + charsetVerifier.getCharset()); } out.close(); diff --git a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java index b0d39d88f..e15139a7f 100644 --- a/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java +++ b/OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java @@ -417,7 +417,7 @@ public class PgpDecryptVerifyOperation extends BaseOperation