add RSA OAEP SHA2 vectors via custom vector creation (#2829)

* add RSA OAEP SHA2 vectors via custom vector creation

* update the docs

* retab

* remove sha2 oaep support (it will be added again in another PR)

* add header to each vector file

* Add Java 8 Bouncy Castle based verification of RSA-OAEP SHA-2 test vectors (#4)

* Update Java verifier to skip past test vector file header if present (#5)

* Update Java verifier to skip past test vector file header if present

* Fix tabs vs. space whitespace

* generate sha1 oaep permutations as well + verify support in java

* address review feedback

4 files changed, 600 insertions, 0 deletions

diff --git a/docs/development/custom-vectors/rsa-oaep-sha2.rst b/docs/development/custom-vectors/rsa-oaep-sha2.rst
new file mode 100644
index 00000000..36f256d7
--- /dev/null
+++ b/docs/development/custom-vectors/rsa-oaep-sha2.rst
@@ -0,0 +1,56 @@
+RSA OAEP SHA2 vector creation
+=============================
+
+This page documents the code that was used to generate the RSA OAEP SHA2
+test vectors as well as code used to verify them against another
+implementation.
+
+
+Creation
+--------
+
+``cryptography`` was modified to allow the use of SHA2 in OAEP encryption. Then
+the following python script was run to generate the vector files.
+
+.. literalinclude:: /development/custom-vectors/rsa-oaep-sha2/generate_rsa_oaep_sha2.py
+
+Download link: :download:`generate_rsa_oaep_sha2.py
+</development/custom-vectors/rsa-oaep-sha2/generate_rsa_oaep_sha2.py>`
+
+
+Verification
+------------
+
+A Java 8 program was written using `Bouncy Castle`_ to load and verify the test
+vectors.
+
+
+.. literalinclude:: /development/custom-vectors/rsa-oaep-sha2/VerifyRSAOAEPSHA2.java
+
+Download link: :download:`VerifyRSAOAEPSHA2.java
+</development/custom-vectors/rsa-oaep-sha2/VerifyRSAOAEPSHA2.java>`
+
+Using the Verifier
+------------------
+
+Download and install the `Java 8 SDK`_. Initial verification was performed
+using ``jdk-8u77-macosx-x64.dmg``.
+
+Download the latest `Bouncy Castle`_ JAR.  Initial verification was performed
+using ``bcprov-jdk15on-154.jar``.
+
+Set the ``-classpath`` to include the Bouncy Castle jar and the path to
+``VerifyRSAOAEPSHA2.java`` and compile the program.
+
+.. code-block:: console
+
+    $ javac -classpath ~/Downloads/bcprov-jdk15on-154.jar:./ VerifyRSAOAEPSHA2.java
+
+Finally, run the program with the path to the SHA-2 vectors:
+
+.. code-block:: console
+
+    $ java -classpath ~/Downloads/bcprov-jdk15on-154.jar:./ VerifyRSAOAEPSHA2
+
+.. _`Bouncy Castle`: https://www.bouncycastle.org/
+.. _`Java 8 SDK`: https://www.oracle.com/technetwork/java/javase/downloads/index.html
diff --git a/docs/development/custom-vectors/rsa-oaep-sha2/VerifyRSAOAEPSHA2.java b/docs/development/custom-vectors/rsa-oaep-sha2/VerifyRSAOAEPSHA2.java
new file mode 100644
index 00000000..e1bfd3d1
--- /dev/null
+++ b/docs/development/custom-vectors/rsa-oaep-sha2/VerifyRSAOAEPSHA2.java
@@ -0,0 +1,416 @@
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.AlgorithmParameters;
+import java.security.GeneralSecurityException;
+import java.security.KeyFactory;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.security.spec.AlgorithmParameterSpec;
+import java.security.spec.MGF1ParameterSpec;
+import java.security.spec.RSAPrivateKeySpec;
+import java.util.Arrays;
+
+import javax.crypto.Cipher;
+import javax.crypto.spec.OAEPParameterSpec;
+import javax.crypto.spec.PSource;
+import javax.xml.bind.DatatypeConverter;
+
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+
+class TestVectorData {
+    public BigInteger pub_key_modulus;
+    public BigInteger pub_key_exponent;
+    public BigInteger priv_key_public_exponent;
+    public BigInteger priv_key_modulus;
+    public BigInteger priv_key_exponent;
+    public BigInteger priv_key_prime_1;
+    public BigInteger priv_key_prime_2;
+    public BigInteger priv_key_prime_exponent_1;
+    public BigInteger priv_key_prime_exponent_2;
+    public BigInteger priv_key_coefficient;
+    public byte[] plaintext;
+    public byte[] ciphertext;
+}
+
+class TestVectorLoader {
+    private static final String FILE_HEADER = "# RSA OAEP SHA2 vectors built";
+    private static final String EXAMPLE_HEADER = "# =====";
+    private static final String EXAMPLE = "# Example";
+    private static final String PUBLIC_KEY = "# Public key";
+    private static final String PUB_MODULUS = "# Modulus:";
+    private static final String PUB_EXPONENT = "# Exponent:";
+    private static final String PRIVATE_KEY = "# Private key";
+    private static final String PRIV_MODULUS = "# Modulus:";
+    private static final String PRIV_PUBLIC_EXPONENT = "# Public exponent:";
+    private static final String PRIV_EXPONENT = "# Exponent:";
+    private static final String PRIV_PRIME_1 = "# Prime 1:";
+    private static final String PRIV_PRIME_2 = "# Prime 2:";
+    private static final String PRIV_PRIME_EXPONENT_1 = "# Prime exponent 1:";
+    private static final String PRIV_PRIME_EXPONENT_2 = "# Prime exponent 2:";
+    private static final String PRIV_COEFFICIENT = "# Coefficient:";
+    private static final String OAEP_EXAMPLE_HEADER = "# OAEP Example";
+    private static final String MESSAGE = "# Message:";
+    private static final String ENCRYPTION = "# Encryption:";
+
+    private BufferedReader m_reader = null;
+    private FileReader m_file_reader = null;
+    private TestVectorData m_data = null;
+
+    TestVectorLoader() {
+
+    }
+
+    protected void finalize() {
+        close();
+    }
+
+    public void open(String path) throws IOException {
+        close();
+        m_file_reader = new FileReader(path);
+        m_reader = new BufferedReader(m_file_reader);
+        m_data = new TestVectorData();
+    }
+
+    public void close() {
+        try {
+            if (m_reader != null) {
+                m_reader.close();
+                m_reader = null;
+            }
+            if (m_file_reader != null) {
+                m_file_reader.close();
+                m_file_reader = null;
+            }
+            m_data = null;
+        } catch (IOException e) {
+            System.out.println("Exception closing files");
+            e.printStackTrace();
+        }
+    }
+
+    public TestVectorData loadNextTest() throws IOException {
+        if (m_file_reader == null || m_reader == null || m_data == null) {
+            throw new IOException("A test vector file must be opened first");
+        }
+
+        String line = m_reader.readLine();
+
+        if (line == null) {
+            // end of file
+            return null;
+        }
+
+        if (line.startsWith(FILE_HEADER)) {
+            // start of file
+            skipFileHeader(m_reader);
+            line = m_reader.readLine();
+        }
+
+        if (line.startsWith(OAEP_EXAMPLE_HEADER)) {
+            // Next example, keep existing keys and load next message
+            loadMessage(m_reader, m_data);
+            return m_data;
+        }
+
+        // otherwise it's a new example
+        if (!line.startsWith(EXAMPLE_HEADER)) {
+            throw new IOException("Test Header Missing");
+        }
+        startNewTest(m_reader);
+        m_data = new TestVectorData();
+
+        line = m_reader.readLine();
+        if (!line.startsWith(PUBLIC_KEY))
+            throw new IOException("Public Key Missing");
+        loadPublicKey(m_reader, m_data);
+
+        line = m_reader.readLine();
+        if (!line.startsWith(PRIVATE_KEY))
+            throw new IOException("Private Key Missing");
+        loadPrivateKey(m_reader, m_data);
+
+        line = m_reader.readLine();
+        if (!line.startsWith(OAEP_EXAMPLE_HEADER))
+            throw new IOException("Message Missing");
+        loadMessage(m_reader, m_data);
+
+        return m_data;
+    }
+
+    private byte[] unhexlify(String line) {
+        byte[] bytes = DatatypeConverter.parseHexBinary(line);
+        return bytes;
+    }
+
+    private BigInteger readBigInteger(BufferedReader br) throws IOException {
+        return new BigInteger(br.readLine(), 16);
+    }
+
+    private void skipFileHeader(BufferedReader br) throws IOException {
+        br.readLine(); // # # Derived from the NIST OAEP SHA1 vectors
+        br.readLine(); // # # Verified against the Bouncy Castle OAEP SHA2 implementation
+        br.readLine(); // #
+    }
+
+    private void startNewTest(BufferedReader br) throws IOException {
+        String line = br.readLine();
+        if (!line.startsWith(EXAMPLE))
+            throw new IOException("Example Header Missing");
+    }
+
+    private void loadPublicKey(BufferedReader br, TestVectorData data) throws IOException {
+        String line = br.readLine();
+        if (!line.startsWith(PUB_MODULUS))
+            throw new IOException("Public Key Modulus Missing");
+        data.pub_key_modulus = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PUB_EXPONENT))
+            throw new IOException("Public Key Exponent Missing");
+        data.pub_key_exponent = readBigInteger(br);
+    }
+
+    private void loadPrivateKey(BufferedReader br, TestVectorData data) throws IOException {
+        String line = br.readLine();
+        if (!line.startsWith(PRIV_MODULUS))
+            throw new IOException("Private Key Modulus Missing");
+        data.priv_key_modulus = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_PUBLIC_EXPONENT))
+            throw new IOException("Private Key Public Exponent Missing");
+        data.priv_key_public_exponent = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_EXPONENT))
+            throw new IOException("Private Key Exponent Missing");
+        data.priv_key_exponent = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_PRIME_1))
+            throw new IOException("Private Key Prime 1 Missing");
+        data.priv_key_prime_1 = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_PRIME_2))
+            throw new IOException("Private Key Prime 2 Missing");
+        data.priv_key_prime_2 = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_PRIME_EXPONENT_1))
+            throw new IOException("Private Key Prime Exponent 1 Missing");
+        data.priv_key_prime_exponent_1 = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_PRIME_EXPONENT_2))
+            throw new IOException("Private Key Prime Exponent 2 Missing");
+        data.priv_key_prime_exponent_2 = readBigInteger(br);
+
+        line = br.readLine();
+        if (!line.startsWith(PRIV_COEFFICIENT))
+            throw new IOException("Private Key Coefficient Missing");
+        data.priv_key_coefficient = readBigInteger(br);
+    }
+
+    private void loadMessage(BufferedReader br, TestVectorData data) throws IOException {
+        String line = br.readLine();
+        if (!line.startsWith(MESSAGE))
+            throw new IOException("Plaintext Missing");
+        data.plaintext = unhexlify(br.readLine());
+
+        line = br.readLine();
+        if (!line.startsWith(ENCRYPTION))
+            throw new IOException("Ciphertext Missing");
+        data.ciphertext = unhexlify(br.readLine());
+    }
+
+}
+
+public class VerifyRSAOAEPSHA2 {
+
+    public enum SHAHash {
+        SHA1, SHA224, SHA256, SHA384, SHA512
+    }
+
+    private SHAHash m_mgf1_hash;
+    private SHAHash m_alg_hash;
+    private Cipher m_cipher;
+    private PrivateKey m_private_key;
+    private AlgorithmParameters m_algo_param;
+
+    VerifyRSAOAEPSHA2(SHAHash mgf1_hash, SHAHash alg_hash, TestVectorData test_data) throws Exception {
+
+        m_mgf1_hash = mgf1_hash;
+        m_alg_hash = alg_hash;
+
+        MGF1ParameterSpec mgf1_spec = getMGF1ParameterSpec(m_mgf1_hash);
+        AlgorithmParameterSpec algo_param_spec = getAlgorithmParameterSpec(m_alg_hash, mgf1_spec);
+
+        m_algo_param = AlgorithmParameters.getInstance("OAEP");
+        m_algo_param.init(algo_param_spec);
+
+        m_private_key = loadPrivateKey(test_data);
+
+        m_cipher = getCipher(m_alg_hash);
+    }
+
+    private Cipher getCipher(SHAHash alg_hash) throws GeneralSecurityException {
+        Cipher cipher = null;
+
+        switch (alg_hash) {
+
+        case SHA1:
+            cipher = Cipher.getInstance("RSA/ECB/OAEPwithSHA1andMGF1Padding", "BC");
+            break;
+
+        case SHA224:
+            cipher = Cipher.getInstance("RSA/ECB/OAEPwithSHA-224andMGF1Padding", "BC");
+            break;
+
+        case SHA256:
+            cipher = Cipher.getInstance("RSA/ECB/OAEPwithSHA-256andMGF1Padding", "BC");
+            break;
+
+        case SHA384:
+            cipher = Cipher.getInstance("RSA/ECB/OAEPwithSHA-384andMGF1Padding", "BC");
+            break;
+
+        case SHA512:
+            cipher = Cipher.getInstance("RSA/ECB/OAEPwithSHA-512andMGF1Padding", "BC");
+            break;
+        }
+
+        return cipher;
+    }
+
+    private MGF1ParameterSpec getMGF1ParameterSpec(SHAHash mgf1_hash) {
+        MGF1ParameterSpec mgf1 = null;
+
+        switch (mgf1_hash) {
+
+        case SHA1:
+            mgf1 = MGF1ParameterSpec.SHA1;
+            break;
+        case SHA224:
+            mgf1 = MGF1ParameterSpec.SHA224;
+            break;
+
+        case SHA256:
+            mgf1 = MGF1ParameterSpec.SHA256;
+            break;
+
+        case SHA384:
+            mgf1 = MGF1ParameterSpec.SHA384;
+            break;
+
+        case SHA512:
+            mgf1 = MGF1ParameterSpec.SHA512;
+            break;
+        }
+
+        return mgf1;
+    }
+
+    private AlgorithmParameterSpec getAlgorithmParameterSpec(SHAHash alg_hash, MGF1ParameterSpec mgf1_spec) {
+
+        OAEPParameterSpec oaep_spec = null;
+
+        switch (alg_hash) {
+
+        case SHA1:
+            oaep_spec = new OAEPParameterSpec("SHA1", "MGF1", mgf1_spec, PSource.PSpecified.DEFAULT);
+            break;
+
+        case SHA224:
+            oaep_spec = new OAEPParameterSpec("SHA-224", "MGF1", mgf1_spec, PSource.PSpecified.DEFAULT);
+            break;
+
+        case SHA256:
+            oaep_spec = new OAEPParameterSpec("SHA-256", "MGF1", mgf1_spec, PSource.PSpecified.DEFAULT);
+            break;
+
+        case SHA384:
+            oaep_spec = new OAEPParameterSpec("SHA-384", "MGF1", mgf1_spec, PSource.PSpecified.DEFAULT);
+            break;
+
+        case SHA512:
+            oaep_spec = new OAEPParameterSpec("SHA-512", "MGF1", mgf1_spec, PSource.PSpecified.DEFAULT);
+            break;
+        }
+
+        return oaep_spec;
+    }
+
+    private PrivateKey loadPrivateKey(TestVectorData test_data) throws Exception {
+        KeyFactory kf = KeyFactory.getInstance("RSA");
+
+        RSAPrivateKeySpec keySpec = new RSAPrivateKeySpec(test_data.priv_key_modulus, test_data.priv_key_exponent);
+
+        return kf.generatePrivate(keySpec);
+    }
+
+    public void testDecrypt(byte[] plaintext, byte[] ciphertext) throws Exception {
+        System.out.println("Verifying OAEP with mgf1_hash: " + m_mgf1_hash + " alg_hash: " + m_alg_hash + " - "
+                + ciphertext.length + " bytes ciphertext - "
+                + plaintext.length + " bytes plaintext");
+
+        m_cipher.init(Cipher.DECRYPT_MODE, m_private_key, m_algo_param);
+        byte[] java_plaintext = m_cipher.doFinal(ciphertext);
+
+        if (Arrays.equals(java_plaintext, plaintext) == false) {
+            throw new Exception("Verification failure - plaintext does not match after decryption.");
+        }
+    }
+
+    public static void main(String[] args) {
+        Security.addProvider(new BouncyCastleProvider());
+
+        // assume current directory if no path given on command line
+        String vector_path = "./vectors/cryptography_vectors/asymmetric/RSA/oaep-custom";
+
+        if (args.length > 0) {
+            vector_path = args[0];
+        }
+
+        System.out.println("Vector file path: " + vector_path);
+
+        try {
+            // loop over each combination of hash loading the vector file
+            // to verify for each
+            for (SHAHash mgf1_hash : SHAHash.values()) {
+                for (SHAHash alg_hash : SHAHash.values()) {
+                    if (mgf1_hash.name().toLowerCase().equals("sha1") &&
+                        alg_hash.name().toLowerCase().equals("sha1")) {
+                        continue;
+                    }
+                    String filename = "oaep-" + mgf1_hash.name().toLowerCase() +
+                                          "-" + alg_hash.name().toLowerCase() + ".txt";
+
+                    System.out.println("Loading " + filename + "...");
+
+                    TestVectorLoader loader = new TestVectorLoader();
+                    loader.open(vector_path + filename);
+
+                    TestVectorData test_data;
+
+                    // load each test in the file and verify
+                    while ((test_data = loader.loadNextTest()) != null) {
+                        VerifyRSAOAEPSHA2 verify = new VerifyRSAOAEPSHA2(mgf1_hash, alg_hash, test_data);
+                        verify.testDecrypt(test_data.plaintext, test_data.ciphertext);
+                    }
+
+                    System.out.println("Verifying " + filename + " completed successfully.");
+                }
+            }
+
+            System.out.println("All verification completed successfully");
+
+        } catch (Exception e) {
+            // if any exception is thrown the verification has failed
+            e.printStackTrace();
+            System.out.println("Verification Failed!");
+        }
+    }
+}
diff --git a/docs/development/custom-vectors/rsa-oaep-sha2/generate_rsa_oaep_sha2.py b/docs/development/custom-vectors/rsa-oaep-sha2/generate_rsa_oaep_sha2.py
new file mode 100644
index 00000000..5a48e0ce
--- /dev/null
+++ b/docs/development/custom-vectors/rsa-oaep-sha2/generate_rsa_oaep_sha2.py
@@ -0,0 +1,127 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import, division, print_function
+
+import binascii
+import itertools
+import os
+
+from cryptography.hazmat.backends.openssl.backend import backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import padding, rsa
+
+from tests.utils import load_pkcs1_vectors, load_vectors_from_file
+
+
+def build_vectors(mgf1alg, hashalg, filename):
+    vectors = load_vectors_from_file(filename, load_pkcs1_vectors)
+
+    output = []
+    for vector in vectors:
+        # RSA keys for this must be long enough to accommodate the length of
+        # the underlying hash function. This means we can't use the keys from
+        # the sha1 test vectors for sha512 tests because 1024-bit keys are too
+        # small. Instead we parse the vectors for the test cases, then
+        # generate our own 2048-bit keys for each.
+        private, _ = vector
+        skey = rsa.generate_private_key(65537, 2048, backend)
+        pn = skey.private_numbers()
+        examples = private["examples"]
+        output.append(b"# =============================================")
+        output.append(b"# Example")
+        output.append(b"# Public key")
+        output.append(b"# Modulus:")
+        output.append(format(pn.public_numbers.n, "x"))
+        output.append(b"# Exponent:")
+        output.append(format(pn.public_numbers.e, "x"))
+        output.append(b"# Private key")
+        output.append(b"# Modulus:")
+        output.append(format(pn.public_numbers.n, "x"))
+        output.append(b"# Public exponent:")
+        output.append(format(pn.public_numbers.e, "x"))
+        output.append(b"# Exponent:")
+        output.append(format(pn.d, "x"))
+        output.append(b"# Prime 1:")
+        output.append(format(pn.p, "x"))
+        output.append(b"# Prime 2:")
+        output.append(format(pn.q, "x"))
+        output.append(b"# Prime exponent 1:")
+        output.append(format(pn.dmp1, "x"))
+        output.append(b"# Prime exponent 2:")
+        output.append(format(pn.dmq1, "x"))
+        output.append(b"# Coefficient:")
+        output.append(format(pn.iqmp, "x"))
+        pkey = skey.public_key()
+        vectorkey = rsa.RSAPrivateNumbers(
+            p=private["p"],
+            q=private["q"],
+            d=private["private_exponent"],
+            dmp1=private["dmp1"],
+            dmq1=private["dmq1"],
+            iqmp=private["iqmp"],
+            public_numbers=rsa.RSAPublicNumbers(
+                e=private["public_exponent"],
+                n=private["modulus"]
+            )
+        ).private_key(backend)
+        count = 1
+
+        for example in examples:
+            message = vectorkey.decrypt(
+                binascii.unhexlify(example["encryption"]),
+                padding.OAEP(
+                    mgf=padding.MGF1(algorithm=hashes.SHA1()),
+                    algorithm=hashes.SHA1(),
+                    label=None
+                )
+            )
+            assert message == binascii.unhexlify(example["message"])
+            ct = pkey.encrypt(
+                message,
+                padding.OAEP(
+                    mgf=padding.MGF1(algorithm=mgf1alg),
+                    algorithm=hashalg,
+                    label=None
+                )
+            )
+            output.append(
+                b"# OAEP Example {0} alg={1} mgf1={2}".format(
+                    count, hashalg.name, mgf1alg.name
+                )
+            )
+            count += 1
+            output.append(b"# Message:")
+            output.append(example["message"])
+            output.append(b"# Encryption:")
+            output.append(binascii.hexlify(ct))
+
+    return b"\n".join(output)
+
+
+def write_file(data, filename):
+    with open(filename, "w") as f:
+        f.write(data)
+
+oaep_path = os.path.join(
+    "asymmetric", "RSA", "pkcs-1v2-1d2-vec", "oaep-vect.txt"
+)
+hashalgs = [
+    hashes.SHA1(),
+    hashes.SHA224(),
+    hashes.SHA256(),
+    hashes.SHA384(),
+    hashes.SHA512(),
+]
+for hashtuple in itertools.product(hashalgs, hashalgs):
+    if (
+        isinstance(hashtuple[0], hashes.SHA1) and
+        isinstance(hashtuple[1], hashes.SHA1)
+    ):
+        continue
+
+    write_file(
+        build_vectors(hashtuple[0], hashtuple[1], oaep_path),
+        "oaep-{0}-{1}.txt".format(hashtuple[0].name, hashtuple[1].name)
+    )
diff --git a/docs/development/test-vectors.rst b/docs/development/test-vectors.rst
index 61717cfc..4f0ece5c 100644
--- a/docs/development/test-vectors.rst
+++ b/docs/development/test-vectors.rst
@@ -44,6 +44,7 @@ Custom asymmetric vectors
     :maxdepth: 1
 
     custom-vectors/secp256k1
+    custom-vectors/rsa-oaep-sha2
 
 * ``asymmetric/PEM_Serialization/ec_private_key.pem`` and
   ``asymmetric/DER_Serialization/ec_private_key.der`` - Contains an Elliptic