Ray reflection with a triangle
Turns out I needed to normalize the normal first:

n.normalize();


full code for reference:

PVector[] face = new PVector[3];
float ai = TWO_PI/3;//angle increment
float r  = 300;//overall radius
float ro = 150;//random offset

PVector n;//normal
Ray r1;

void setup() {
  size(500, 500, P3D);
  for (int i = 0 ; i &lt; 3; i++) face[i] = new PVector(cos(ai * i) * r + random(ro), random(-50, 50), sin(ai * i) * r + random(ro));
  r1 = new Ray(new PVector(-100, -200, -300), new PVector(100, 200, 300));
}
void draw() {
  background(255);
  lights();
  translate(width/2, height/2, -500);
  rotateX(map(mouseY, 0, height, -PI, PI));
  rotateY(map(mouseX, 0, width, -PI, PI));
  //draw plane
  beginShape(TRIANGLES);
  for (PVector p : face) vertex(p.x, p.y, p.z);
  endShape();
  //normals
  PVector c = new PVector();//centroid
  for (PVector p : face) c.add(p);
  c.div(3.0);
  PVector cb = PVector.sub(face[2], face[1]);
  PVector ab = PVector.sub(face[0], face[1]);
  n = cb.cross(ab);//compute normal
  n.normalize();
  line(c.x, c.y, c.z, n.x, n.y, n.z);//draw normal

  pushStyle();
  //http://paulbourke.net/geometry/planeline/
  //line to plane intersection u = N dot ( P3 - P1 ) / N dot (P2 - P1), P = P1 + u (P2-P1), where P1,P2 are on the line and P3 is a point on the plane
  PVector P2SubP1 = PVector.sub(r1.end, r1.start);
  PVector P3SubP1 = PVector.sub(face[0], r1.start);
  float u = n.dot(P3SubP1) / n.dot(P2SubP1);
  PVector P = PVector.add(r1.start, PVector.mult(P2SubP1, u));
  strokeWeight(5);
  point(P.x, P.y, P.z);//point of ray-plane intersection

  //vector reflecting http://www.3dkingdoms.com/weekly/weekly.php?a=2
  //R = 2*(V dot N)*N - V
  //Vnew = -2*(V dot N)*N + V
  //PVector V = PVector.sub(r1.start,r1.end);
  PVector V = PVector.sub(r1.start, P);
  PVector R = PVector.sub(PVector.mult(n, 2 * (V.dot(n))), V);
  strokeWeight(1);
  stroke(0, 192, 0);
  line(P.x, P.y, P.z, R.x, R.y, R.z);
  stroke(192, 0, 0);
  line(r1.start.x, r1.start.y, r1.start.z, P.x, P.y, P.z);
  stroke(0, 0, 192);
  line(P.x, P.y, P.z, r1.end.x, r1.end.y, r1.end.z);
  popStyle();
}
void keyPressed() {   
  setup();
}//reset
class Ray {
  PVector start = new PVector(), end = new PVector();
  Ray(PVector s, PVector e) {   
    start = s ; 
    end = e;
  }
}