ltq-deu: fix temp size exceed in hmac algorithms

Exceeding the temp array size was not checked and instead storage not
allocated by the driver was used/overwritten which in most cases
resulted in reboots. This patch implements processing the input to the
hash algorithm in tempsize chunks.
The _hmac_final methods were changed to _hmac_final_impl adding a
parameter that indicates intermediate or final processing. The started
variable was added to the context to indicate, if there is an
intermediate result in the context. For sha1_hmac the variable to store
the intermediate hash was added to the context too.
In order to avoid md5_hmac_final_impl being recursively called if the
padding of the input and the resulting last transform during the hmac
algorighms final processing causes the temp array to overflow and to
make sure that there is at least one block in the temp array when the
_hmac_final for final processing is called, the check for exceeding
the temp array in _hmac_transform was moved before copying the block
and incrementing dbn. dbn needs to be at least 1 at final processing
time to let the hash engine apply the opad operation.
To make the hash engine not apply the hmac algorithms final opad
operation, for intermediate processing the dbn in the control register
is set to a higher value than number of dbns are actually processed.

Signed-off-by: Daniel Kestrel <kestrel1974@t-online.de>

2 files changed, 146 insertions, 71 deletions

diff --git a/package/kernel/lantiq/ltq-deu/src/ifxmips_md5_hmac.c b/package/kernel/lantiq/ltq-deu/src/ifxmips_md5_hmac.c
index 092b0581cc..ea808fdc5b 100644
--- a/package/kernel/lantiq/ltq-deu/src/ifxmips_md5_hmac.c
+++ b/package/kernel/lantiq/ltq-deu/src/ifxmips_md5_hmac.c
@@ -79,6 +79,7 @@ struct md5_hmac_ctx {
     u32 block[MD5_BLOCK_WORDS];
     u64 byte_count;
     u32 dbn;
+    int started;
     unsigned int keylen;
     struct shash_desc *desc;
     u32 (*temp)[MD5_BLOCK_WORDS];
@@ -86,6 +87,8 @@ struct md5_hmac_ctx {
 
 extern int disable_deudma;
 
+static int md5_hmac_final_impl(struct shash_desc *desc, u8 *out, bool hash_final);
+
 /*! \fn static void md5_hmac_transform(struct crypto_tfm *tfm, u32 const *in)
  *  \ingroup IFX_MD5_HMAC_FUNCTIONS
  *  \brief save input block to context   
@@ -96,14 +99,14 @@ static void md5_hmac_transform(struct shash_desc *desc, u32 const *in)
 {
     struct md5_hmac_ctx *mctx = crypto_shash_ctx(desc->tfm);
 
-    memcpy(&mctx->temp[mctx->dbn], in, 64); //dbn workaround
-    mctx->dbn += 1;
-    
-    if ( (mctx->dbn<<4) > MD5_HMAC_DBN_TEMP_SIZE )
+    if ( ((mctx->dbn<<4)+1) > MD5_HMAC_DBN_TEMP_SIZE )
     {
-        printk("MD5_HMAC_DBN_TEMP_SIZE exceeded\n");
+        //printk("MD5_HMAC_DBN_TEMP_SIZE exceeded\n");
+        md5_hmac_final_impl(desc, (u8 *)mctx->hash, false);
     }
 
+    memcpy(&mctx->temp[mctx->dbn], in, 64); //dbn workaround
+    mctx->dbn += 1;
 }
 
 /*! \fn int md5_hmac_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen)
@@ -184,6 +187,7 @@ static int md5_hmac_init(struct shash_desc *desc)
     
 
     mctx->dbn = 0; //dbn workaround
+    mctx->started = 0;
 
     return 0;
 }
@@ -226,14 +230,26 @@ static int md5_hmac_update(struct shash_desc *desc, const u8 *data, unsigned int
     return 0;    
 }
 
-/*! \fn void md5_hmac_final(struct crypto_tfm *tfm, u8 *out)
+/*! \fn static int md5_hmac_final(struct crypto_tfm *tfm, u8 *out)
  *  \ingroup IFX_MD5_HMAC_FUNCTIONS
- *  \brief compute final md5 hmac value   
+ *  \brief call md5_hmac_final_impl with hash_final true   
  *  \param tfm linux crypto algo transform  
  *  \param out final md5 hmac output value  
 */                                 
 static int md5_hmac_final(struct shash_desc *desc, u8 *out)
 {
+    return md5_hmac_final_impl(desc, out, true);
+}
+
+/*! \fn static int md5_hmac_final_impl(struct crypto_tfm *tfm, u8 *out, bool hash_final)
+ *  \ingroup IFX_MD5_HMAC_FUNCTIONS
+ *  \brief compute final or intermediate md5 hmac value   
+ *  \param tfm linux crypto algo transform  
+ *  \param out final md5 hmac output value  
+ *  \param in finalize or intermediate processing  
+*/                                 
+static int md5_hmac_final_impl(struct shash_desc *desc, u8 *out, bool hash_final)
+{
     struct md5_hmac_ctx *mctx = crypto_shash_ctx(desc->tfm);
     const unsigned int offset = mctx->byte_count & 0x3f;
     char *p = (char *)mctx->block + offset;
@@ -244,21 +260,22 @@ static int md5_hmac_final(struct shash_desc *desc, u8 *out)
     int dbn;
     u32 *in = mctx->temp[0];
 
+    if (hash_final) {
+        *p++ = 0x80;
+        if (padding < 0) {
+            memset(p, 0x00, padding + sizeof (u64));
+            md5_hmac_transform(desc, mctx->block);
+            p = (char *)mctx->block;
+            padding = 56;
+        }
+
+        memset(p, 0, padding);
+        mctx->block[14] = le32_to_cpu((mctx->byte_count + 64) << 3); // need to add 512 bit of the IPAD operation 
+        mctx->block[15] = 0x00000000;
 
-    *p++ = 0x80;
-    if (padding < 0) {
-        memset(p, 0x00, padding + sizeof (u64));
         md5_hmac_transform(desc, mctx->block);
-        p = (char *)mctx->block;
-        padding = 56;
     }
 
-    memset(p, 0, padding);
-    mctx->block[14] = le32_to_cpu((mctx->byte_count + 64) << 3); // need to add 512 bit of the IPAD operation 
-    mctx->block[15] = 0x00000000;
-
-    md5_hmac_transform(desc, mctx->block);
-
     CRTCL_SECT_HASH_START;
 
     MD5_HASH_INIT;
@@ -266,7 +283,11 @@ static int md5_hmac_final(struct shash_desc *desc, u8 *out)
     md5_hmac_setkey_hw(mctx->key, mctx->keylen);
 
     //printk("\ndbn = %d\n", mctx->dbn); 
-    hashs->DBN = mctx->dbn;
+    if (hash_final) {
+       hashs->DBN = mctx->dbn;
+    } else {
+       hashs->DBN = mctx->dbn + 5;
+    }
     asm("sync");
     
     *IFX_HASH_CON = 0x0703002D; //khs, go, init, ndc, endi, kyue, hmen, md5 	
@@ -276,6 +297,15 @@ static int md5_hmac_final(struct shash_desc *desc, u8 *out)
         // this will not take long
     }
 
+    if (mctx->started) {
+        hashs->D1R = *((u32 *) mctx->hash + 0);
+        hashs->D2R = *((u32 *) mctx->hash + 1);
+        hashs->D3R = *((u32 *) mctx->hash + 2);
+        hashs->D4R = *((u32 *) mctx->hash + 3);
+    } else {
+        mctx->started = 1;
+    }
+
     for (dbn = 0; dbn < mctx->dbn; dbn++)
     {
         for (i = 0; i < 16; i++) {
@@ -295,9 +325,11 @@ static int md5_hmac_final(struct shash_desc *desc, u8 *out)
 
 
 #if 1
-    //wait for digest ready
-    while (! hashs->controlr.DGRY) {
-        // this will not take long
+    if (hash_final) {
+        //wait for digest ready
+        while (! hashs->controlr.DGRY) {
+            // this will not take long
+        }
     }
 #endif
 
@@ -306,12 +338,15 @@ static int md5_hmac_final(struct shash_desc *desc, u8 *out)
     *((u32 *) out + 2) = hashs->D3R;
     *((u32 *) out + 3) = hashs->D4R;
 
-    /* reset the context after we finish with the hash */
-    mctx->byte_count = 0;
-    memset(&mctx->hash[0], 0, sizeof(MD5_HASH_WORDS));
-    memset(&mctx->block[0], 0, sizeof(MD5_BLOCK_WORDS));
-    memset(&mctx->temp[0], 0, MD5_HMAC_DBN_TEMP_SIZE);
-
+    if (hash_final) {
+        /* reset the context after we finish with the hash */
+        mctx->byte_count = 0;
+        memset(&mctx->hash[0], 0, sizeof(MD5_HASH_WORDS));
+        memset(&mctx->block[0], 0, sizeof(MD5_BLOCK_WORDS));
+        memset(&mctx->temp[0], 0, MD5_HMAC_DBN_TEMP_SIZE);
+    } else {
+        mctx->dbn = 0;
+    }
     CRTCL_SECT_HASH_END;
 
 
diff --git a/package/kernel/lantiq/ltq-deu/src/ifxmips_sha1_hmac.c b/package/kernel/lantiq/ltq-deu/src/ifxmips_sha1_hmac.c
index a9e1d64d6f..72770a38e8 100644
--- a/package/kernel/lantiq/ltq-deu/src/ifxmips_sha1_hmac.c
+++ b/package/kernel/lantiq/ltq-deu/src/ifxmips_sha1_hmac.c
@@ -61,6 +61,7 @@
 
 #define SHA1_DIGEST_SIZE    20
 #define SHA1_BLOCK_WORDS    16
+#define SHA1_HASH_WORDS     5
 #define SHA1_HMAC_BLOCK_SIZE    64
 #define SHA1_HMAC_DBN_TEMP_SIZE 1024 // size in dword, needed for dbn workaround 
 #define HASH_START   IFX_HASH_CON
@@ -79,8 +80,9 @@ struct sha1_hmac_ctx {
 
     u8 buffer[SHA1_HMAC_BLOCK_SIZE];
     u8 key[SHA1_HMAC_MAX_KEYLEN];
-    u32 state[5];
+    u32 hash[SHA1_HASH_WORDS];
     u32 dbn;
+    int started;
     u64 count;
 
     struct shash_desc *desc;
@@ -89,6 +91,8 @@ struct sha1_hmac_ctx {
 
 extern int disable_deudma;
 
+static int sha1_hmac_final_impl(struct shash_desc *desc, u8 *out, bool hash_final);
+
 /*! \fn static void sha1_hmac_transform(struct crypto_tfm *tfm, u32 const *in)
  *  \ingroup IFX_SHA1_HMAC_FUNCTIONS
  *  \brief save input block to context   
@@ -99,14 +103,15 @@ static int sha1_hmac_transform(struct shash_desc *desc, u32 const *in)
 {
     struct sha1_hmac_ctx *sctx =  crypto_shash_ctx(desc->tfm);
 
-    memcpy(&sctx->temp[sctx->dbn], in, 64); //dbn workaround
-    sctx->dbn += 1;
-    
-    if ( (sctx->dbn<<4) > SHA1_HMAC_DBN_TEMP_SIZE )
+    if ( ((sctx->dbn<<4)+1) > SHA1_HMAC_DBN_TEMP_SIZE )
     {
-        printk("SHA1_HMAC_DBN_TEMP_SIZE exceeded\n");
+        //printk("SHA1_HMAC_DBN_TEMP_SIZE exceeded\n");
+        sha1_hmac_final_impl(desc, (u8 *)sctx->hash, false);
     }
-   
+
+    memcpy(&sctx->temp[sctx->dbn], in, 64); //dbn workaround
+    sctx->dbn += 1;
+
     return 0;
 }
 
@@ -186,6 +191,7 @@ static int sha1_hmac_init(struct shash_desc *desc)
 
     //printk("debug ln: %d, fn: %s\n", __LINE__, __func__);
     sctx->dbn = 0; //dbn workaround
+    sctx->started = 0;
 
     return 0;
 }
@@ -223,14 +229,26 @@ static int sha1_hmac_update(struct shash_desc *desc, const u8 *data,
     return 0;
 }
 
-/*! \fn static void sha1_hmac_final(struct crypto_tfm *tfm, u8 *out)
+/*! \fn static int sha1_hmac_final(struct crypto_tfm *tfm, u8 *out)
  *  \ingroup IFX_SHA1_HMAC_FUNCTIONS
- *  \brief ompute final sha1 hmac value   
+ *  \brief call sha1_hmac_final_impl with hash_final true   
  *  \param tfm linux crypto algo transform  
  *  \param out final sha1 hmac output value  
 */                                 
 static int sha1_hmac_final(struct shash_desc *desc, u8 *out)
 {
+    return sha1_hmac_final_impl(desc, out, true);
+}
+
+/*! \fn static int sha1_hmac_final_impl(struct crypto_tfm *tfm, u8 *out, bool hash_final)
+ *  \ingroup IFX_SHA1_HMAC_FUNCTIONS
+ *  \brief ompute final or intermediate sha1 hmac value   
+ *  \param tfm linux crypto algo transform  
+ *  \param out final sha1 hmac output value  
+ *  \param in finalize or intermediate processing  
+*/                                 
+static int sha1_hmac_final_impl(struct shash_desc *desc, u8 *out, bool hash_final)
+{
     struct sha1_hmac_ctx *sctx =  crypto_shash_ctx(desc->tfm);
     u32 index, padlen;
     u64 t;
@@ -241,40 +259,47 @@ static int sha1_hmac_final(struct shash_desc *desc, u8 *out)
     int i = 0;
     int dbn;
     u32 *in = sctx->temp[0];
-        
-    t = sctx->count + 512; // need to add 512 bit of the IPAD operation
-    bits[7] = 0xff & t;
-    t >>= 8;
-    bits[6] = 0xff & t;
-    t >>= 8;
-    bits[5] = 0xff & t;
-    t >>= 8;
-    bits[4] = 0xff & t;
-    t >>= 8;
-    bits[3] = 0xff & t;
-    t >>= 8;
-    bits[2] = 0xff & t;
-    t >>= 8;
-    bits[1] = 0xff & t;
-    t >>= 8;
-    bits[0] = 0xff & t;
-
-    /* Pad out to 56 mod 64 */
-    index = (sctx->count >> 3) & 0x3f;
-    padlen = (index < 56) ? (56 - index) : ((64 + 56) - index);
-    sha1_hmac_update (desc, padding, padlen);
-
-    /* Append length */
-    sha1_hmac_update (desc, bits, sizeof bits);
+
+    if (hash_final) {
+        t = sctx->count + 512; // need to add 512 bit of the IPAD operation
+        bits[7] = 0xff & t;
+        t >>= 8;
+        bits[6] = 0xff & t;
+        t >>= 8;
+        bits[5] = 0xff & t;
+        t >>= 8;
+        bits[4] = 0xff & t;
+        t >>= 8;
+        bits[3] = 0xff & t;
+        t >>= 8;
+        bits[2] = 0xff & t;
+        t >>= 8;
+        bits[1] = 0xff & t;
+        t >>= 8;
+        bits[0] = 0xff & t;
+
+        /* Pad out to 56 mod 64 */
+        index = (sctx->count >> 3) & 0x3f;
+        padlen = (index < 56) ? (56 - index) : ((64 + 56) - index);
+        sha1_hmac_update (desc, padding, padlen);
+
+        /* Append length */
+        sha1_hmac_update (desc, bits, sizeof bits);
+    }
 
     CRTCL_SECT_HASH_START;
 
     SHA_HASH_INIT;
 
     sha1_hmac_setkey_hw(sctx->key, sctx->keylen);
-    
-    hashs->DBN = sctx->dbn;
-    
+
+    if (hash_final) {
+        hashs->DBN = sctx->dbn;
+    } else {
+        hashs->DBN = sctx->dbn + 5;
+    }
+    asm("sync");
+
     //for vr9 change, ENDI = 1
     *IFX_HASH_CON = HASH_CON_VALUE; 
 
@@ -283,6 +308,16 @@ static int sha1_hmac_final(struct shash_desc *desc, u8 *out)
         // this will not take long
     }
 
+    if (sctx->started) {
+        hashs->D1R = *((u32 *) sctx->hash + 0);
+        hashs->D2R = *((u32 *) sctx->hash + 1);
+        hashs->D3R = *((u32 *) sctx->hash + 2);
+        hashs->D4R = *((u32 *) sctx->hash + 3);
+        hashs->D5R = *((u32 *) sctx->hash + 4);
+    } else {
+        sctx->started = 1;
+    }
+
     for (dbn = 0; dbn < sctx->dbn; dbn++)
     {
         for (i = 0; i < 16; i++) {
@@ -302,9 +337,11 @@ static int sha1_hmac_final(struct shash_desc *desc, u8 *out)
 
 
 #if 1
-    //wait for digest ready
-    while (! hashs->controlr.DGRY) {
-        // this will not take long
+    if (hash_final) {
+        //wait for digest ready
+        while (! hashs->controlr.DGRY) {
+            // this will not take long
+        }
     }
 #endif
 
@@ -314,9 +351,12 @@ static int sha1_hmac_final(struct shash_desc *desc, u8 *out)
     *((u32 *) out + 3) = hashs->D4R;
     *((u32 *) out + 4) = hashs->D5R;
 
-    memset(&sctx->buffer[0], 0, SHA1_HMAC_BLOCK_SIZE);
-    sctx->count = 0; 
- 
+    if (hash_final) {
+        memset(&sctx->buffer[0], 0, SHA1_HMAC_BLOCK_SIZE);
+        sctx->count = 0;
+    } else {
+        sctx->dbn = 0;
+    }
     //printk("debug ln: %d, fn: %s\n", __LINE__, __func__);
     CRTCL_SECT_HASH_END;