Bitmap alpha bevel algorithm?
This can be accomplished with a convolution of the distance transform. 

Use a distance transform on the edge of the mask. Then threshold this distance transform to remove values beyond some distance. I think the secret to getting the shading is to convolve the distance transform result with a kernel that looks something like this:

[ -1.0  -1.0  -1.0
  -1.0   0.0   0.0
  -1.0   0.0   1.0 ]


This should get you started in the right direction:

#include "opencv/cv.h"
#include "opencv/highgui.h"

using namespace cv;
using namespace std;

int main() {
    Mat mask, dist, bevel;
    mask = Mat::zeros(200, 400, CV_8U);
    rectangle(mask, Point(30,30), Point(180,180), Scalar(255), -1);
    circle(mask, Point(30,30), 50, Scalar(0), -1);
    circle(mask, Point(180,180), 50, Scalar(0), -1);
    circle(mask, Point(300,100), 75, Scalar(255), -1);
    imshow("1",mask);




    //find edges and invert image for distance transform
    Canny(mask, dist, 50, 150);
    dist = 255-dist;
    distanceTransform(dist, dist, CV_DIST_L2, CV_DIST_MASK_5);
    threshold(dist, dist, 20, 20, CV_THRESH_TRUNC);
    blur(dist, dist, Size(3,3));
    dist.convertTo(bevel, CV_8U);
    equalizeHist(bevel, bevel);
    imshow("2",bevel);




    //convolve with secret sauce
    float d[] = {-1,-2,-3,
                 -2, 0, 0,
                 -3, 0, 1 };
    Mat kernel(3, 3, CV_32F, d);
    kernel = kernel - mean(kernel)[0];
    filter2D(dist, dist, CV_32F, kernel);

    //normalize filtering result to [-1, 1]
    double maxVal;
    minMaxLoc(dist, NULL, &amp;maxVal);
    dist = 128 * dist / maxVal;

    //convert and display result
    dist.convertTo(bevel, CV_8U, 1, 128);
    bevel = bevel.mul(mask)/255;
    imshow("3", bevel);




    waitKey(0);
}