Are ECDSA keys and RSA keys interchangeable? Is it possible to use an ES512 key pair for RS512 signatures?
Is it possible to use an RS512 key pair for ES512 signatures?
I've also posted an issue in the NodeJS jwa package.

I created a key pair for EC512 with the following commands:

openssl ecparam -genkey -name secp521r1 -noout -out ecdsa-p521-private.pem
openssl ec -in ecdsa-p521-private.pem -pubout -out ecdsa-p521-public.pem


Then, instead of using ECDSA as I should, I used RSA:

jwa = require('jwa');
rsa = jwa('RS512');
signature = rsa.sign('hello', fs.readFileSync('ecdsa-p521-private.pem').toString());
result = rsa.verify('hello', signature, fs.readFileSync('ecdsa-p521-public.pem').toString());


And the result was true!

Is this an expected behaviour? Is it possible to use ECDSA keys for RSA and vice versa?

Thanks

--- EDIT ---

Here's a list of possible algorithms to choose from (for my json-web-token implementation), so we will have common terminology:


HS256:   HMAC using SHA-256 hash algorithm
HS384:   HMAC using SHA-384 hash algorithm
HS512:   HMAC using SHA-512 hash algorithm
RS256:   RSASSA using SHA-256 hash algorithm
RS384:   RSASSA using SHA-384 hash algorithm
RS512:   RSASSA using SHA-512 hash algorithm
ES256:   ECDSA using P-256 curve and SHA-256 hash algorithm
ES384:   ECDSA using P-384 curve and SHA-384 hash algorithm
ES512:   ECDSA using P-521 curve and SHA-512 hash algorithm


--- EDIT ---

Following Squeamish Ossifrage's suggestion to use ed25519, here is my Node.JS code for signing and verifying "nonstandard ed25519 json-web-tokens".

const jwt = require('jsonwebtoken');
const base64url = require('base64url');
const ed25519 = require('ed25519');

const ed25519Header1 = base64url.encode(Buffer.from('{"alg":"Ed25519","typ":"JWT"}'));
const ed25519Header2 = base64url.encode(Buffer.from('{"typ":"JWT","alg":"Ed25519"}'));
const noneHeader = base64url.encode(Buffer.from('{"alg":"none","typ":"JWT"}'));

function sign(obj, privateKey, options) {
  const noneToken = jwt.sign(obj, '', { ...options, algorithm: "none" });
  if (jwt.decode(noneToken, {complete: true}).header.typ !== 'JWT') {
    throw new Error("Invalid object to sign");
  }
  const [, payload] = noneToken.split('.');
  const signature = base64url.encode(ed25519.Sign(
    Buffer.from(`${ed25519Header1}.${payload}`),
    { privateKey } // passing an object makes sure we want to sign with private-key, not seed.
  ));
  return `${ed25519Header1}.${payload}.${signature}`;
}


function verify(token, publicKey, options) {
  const [tokenHeader, payload, signature, ...leftover] = token.split('.');
  if ((tokenHeader !== ed25519Header1) &amp;&amp; (tokenHeader !== ed25519Header2)) {
    throw new Error("ed25519 jwt malformed - unexpected header");
  }
  const signatureBuffer = base64url.toBuffer(signature || '');
  if (base64url.encode(signatureBuffer) !== signature) {
    throw new Error("ed25519 jwt malformed - invalid signature format");
  }
  // signature is a string, therefore payload is a string (not undefined)
  if (!ed25519.Verify(
    Buffer.from(`${tokenHeader}.${payload}`),
    base64url.toBuffer(signature),
    publicKey
  )) {
    throw new Error("Invalid signature");
  }
  return jwt.verify([noneHeader, payload, '', ...leftover].join('.'), '', {
    ...options,
    algorithms: ['none']
  });
}


a pair of publicKey and privateKey can be generate with the MakeKeyPair(...) function of the ed25519 package (privateKey should be 64 bytes long and publicKey should be 32 bytes long).

The code allows to enjoy both ed25519 signing, and the features of the jsonwebtoken package.

For example:

var token = sign({hello:"world"}, privateKey, {expiresIn:'1 minute'})
console.log(verify(t, publicKey)); // prints { hello: 'world', iat: 1526078738, exp: 1526078798 }
// After 1 minute:
verify(t, publicKey); // Throws "TokenExpiredError: jwt expired"
From the source code:

function createKeySigner(bits) {
 return function sign(thing, privateKey) {
    if (!bufferOrString(privateKey) &amp;&amp; !(typeof privateKey === 'object'))
      throw typeError(MSG_INVALID_SIGNER_KEY);
    thing = normalizeInput(thing);
    // Even though we are specifying "RSA" here, this works with ECDSA
    // keys as well.
    var signer = crypto.createSign('RSA-SHA' + bits);
    var sig = (signer.update(thing), signer.sign(privateKey, 'base64'));
    return base64url.fromBase64(sig);
  }
}


See in particular the comment. What happens here is that even though you specified "RSA with SHA-512" as signature algorithm (the 'RS512' parameter, in that library's weird terminology), the code perfectly knows that the private key is for ECDSA, not RSA (look at the contents of the private key file, it starts with "BEGIN EC PRIVATE KEY"), and so it "helpfully" uses ECDSA instead of RSA, transparently.

No mystery here then, just some Node.js code doing Node.js stuff (i.e. doing what it can instead of what you asked for, and hoping for the best -- a rather dubious methodology for all things related to security, in particular cryptography).