title: Different signatures when using C routines and openssl dgst, rsautl commands
body: <p>I am using following statement to create a RSA public and private key.</p>
  
  <p>openssl genrsa -out ksign_private.pem 1024
  openssl rsa -in ksign_private.pem -pubout > ksign_public.pem </p>
  
  <p>Then I have program that uses, PEM_read_RSAPrivateKey, EVP_PKEY_assign_RSA, EVP_SignInit, EVP_SignUpdate, EVP_SignFinal functions from openssl libcrypto to generate signature file.</p>
  
  <p>I also have routine that verifies that signature can be verified using  PEM_read_RSA_PUBKEY, EVP_PKEY_assign_RSA, EVP_VerifyInit, EVP_VerifyUpdate, EVP_VerifyFinal.
  Source code for these routines is attached below. </p>
  
  <p>When using these functions I can create SHA1 signature, encrypt it with private key, and decrypt it using public key. </p>
  
  <p>However I tried to use the same data file, same private, public key using the openssl rsautl and the signature that is getting created by openssl rsautl is vastly different. </p>
  
  <pre><code>openssl dgst -sha1 -binary &lt; myData &gt; testfile.sha1
  openssl rsautl -sign -in testfile.sha1 -inkey ksign_private.pem -keyform PEM -out testfile.sig
  </code></pre>
  
  <p>Can any one tell me what options I am using wrong when using openssl rsautl or dgst command?</p>
  
  <pre><code>#include &lt;stdio.h&gt;
  #include &lt;unistd.h&gt;
  #include &lt;stdlib.h&gt;
  #include &lt;ctype.h&gt;
  #include &lt;unistd.h&gt;
  #include &lt;string.h&gt;
  #include &lt;openssl/sha.h&gt;
  #include &lt;errno.h&gt;
  #include &lt;getopt.h&gt;
  #include &lt;sys/types.h&gt;
  #include &lt;sys/stat.h&gt;
  #include &lt;openssl/evp.h&gt;
  #include &lt;openssl/pem.h&gt;
  #include &lt;openssl/rsa.h&gt;
  
  int ksignEvpSign(FILE * private_key, FILE * inFileFP, FILE * outFileFP);
  int ksignEvpVerify(FILE * public_key, FILE * dataFileFP, FILE * signFileFP);
  
  int ksignEvpSign(FILE * privateKeyFP, FILE * inFileFP, FILE * outFileFP)
  {
      RSA *rsa_pkey = NULL;
      EVP_PKEY *pkey = EVP_PKEY_new();
      EVP_MD_CTX ctx;
      unsigned char buffer[4096];
      size_t len;
      unsigned char *sig;
      unsigned int siglen;
  
      if (!PEM_read_RSAPrivateKey(privateKeyFP, &amp;rsa_pkey, NULL, NULL)) {
          fprintf(stderr, "Error loading RSA Private Key File.\n");
          return 2;
      }
  
      if (!EVP_PKEY_assign_RSA(pkey, rsa_pkey)) {
          fprintf(stderr, "EVP_PKEY_assign_RSA: failed.\n");
          return 3;
      }
  
      EVP_MD_CTX_init(&amp;ctx);
  
      if (!EVP_SignInit(&amp;ctx, EVP_sha1())) {
          fprintf(stderr, "EVP_SignInit: failed.\n");
          EVP_PKEY_free(pkey);
          return 3;
      }
  
      while ((len = fread(buffer, 1, sizeof buffer, inFileFP)) &gt; 0) {
          if (!EVP_SignUpdate(&amp;ctx, buffer, len)) {
              fprintf(stderr, "EVP_SignUpdate: failed.\n");
              EVP_PKEY_free(pkey);
              return 3;
          }
      }
  
      if (ferror(inFileFP)) {
          perror("input file");
          EVP_PKEY_free(pkey);
          return 4;
      }
  
      sig = malloc(EVP_PKEY_size(pkey));
      if (!EVP_SignFinal(&amp;ctx, sig, &amp;siglen, pkey)) {
          fprintf(stderr, "EVP_SignFinal: failed.\n");
          free(sig);
          EVP_PKEY_free(pkey);
          return 3;
      }
      fwrite(sig, siglen, 1, outFileFP);
      free(sig);
      EVP_PKEY_free(pkey);
      return 0;
  }
  
  int ksignEvpVerify(FILE * publicKeyFP, FILE * dataFileFP, FILE * sigFileFP)
  {
      RSA *rsa_pkey = NULL;
      EVP_PKEY *pkey;
      EVP_MD_CTX ctx;
      unsigned char buffer[4096];
      size_t len;
      unsigned char *sig;
      unsigned int siglen;
      struct stat stat_buf;
  
      if (!PEM_read_RSA_PUBKEY(publicKeyFP, &amp;rsa_pkey, NULL, NULL)) {
          fprintf(stderr, "Error loading RSA public Key File.\n");
          return 2;
      }
      pkey = EVP_PKEY_new();
  
      if (!EVP_PKEY_assign_RSA(pkey, rsa_pkey)) {
          fprintf(stderr, "EVP_PKEY_assign_RSA: failed.\n");
          return 3;
      }
      /* Read the signature */
      if (fstat(fileno(sigFileFP), &amp;stat_buf) == -1) {
          fprintf(stderr, "Unable to read signature \n");
          return 4;
      }
      siglen = stat_buf.st_size;
      sig = (unsigned char *)malloc(siglen);
      if (sig == NULL) {
          fprintf(stderr, "Unable to allocated %d bytes for signature\n",
              siglen);
          return 5;
      }
      if ((fread(sig, 1, siglen, sigFileFP)) != siglen) {
          fprintf(stderr, "Unable to read %d bytes for signature\n",
              siglen);
          return 6;
      }
  /*
      printf("Signature:");
      for (i = 0; i &lt; siglen; i++) {
          fprintf(stdout, "%02x", sig[i]);
          if (i % 16 == 15)
              fprintf(stdout, "\n");
      }
      fprintf(stdout, "\n");
  */
  
      EVP_MD_CTX_init(&amp;ctx);
  
      if (!EVP_VerifyInit(&amp;ctx, EVP_sha1())) {
          fprintf(stderr, "EVP_SignInit: failed.\n");
          EVP_PKEY_free(pkey);
          return 7;
      }
  
      while ((len = fread(buffer, 1, sizeof buffer, dataFileFP)) &gt; 0) {
          if (!EVP_VerifyUpdate(&amp;ctx, buffer, len)) {
              fprintf(stderr, "EVP_SignUpdate: failed.\n");
              EVP_PKEY_free(pkey);
              return 8;
          }
      }
  
      if (ferror(dataFileFP)) {
          perror("input file");
          EVP_PKEY_free(pkey);
          return 9;
      }
  
      if (!EVP_VerifyFinal(&amp;ctx, sig, siglen, pkey)) {
          fprintf(stderr, "EVP_VerifyFinal: failed.\n");
          free(sig);
          EVP_PKEY_free(pkey);
          return 10;
      }
      free(sig);
      EVP_PKEY_free(pkey);
      return 0;
  }
  </code></pre>