Random Weighted Classifier in R
Some improvements:

random_weighted_classifier2 &lt;- function(n = 1, weightA, weightB, weightC){
  x &lt;- sample(1:100, n, replace = T) / 100
  i1 &lt;- x &lt;= weightA
  i2 &lt;- x &gt; weightA &amp; x &lt;= (weightA + weightB)
  rez &lt;- rep('no_change', n)
  rez[i2] &lt;- "worse"
  rez[i1] &lt;- "better"
  rez
}

test &lt;- function(){

  betters &lt;- rep("better", 100)
  worses &lt;- rep("worse", 50)
  no_changes &lt;- rep("no_change", 10)
  reference &lt;- sample(c(betters, worses, no_changes))

  better &lt;- sum(reference == "better")
  worse &lt;- sum(reference == "worse")
  no_change &lt;- sum(reference == "no_change")
  total &lt;- length(reference)

  iterations &lt;- 1000
  f1_rwc &lt;- pres_rwc &lt;- rec_rwc &lt;- matrix(0., iterations, 3)
  kappa_rwc &lt;- vector("double", iterations)

  referenceF &lt;- factor(reference, levels = c("better","worse", "no_change"))
  for (i in seq(1:iterations)) {

    prediction_rwc &lt;- random_weighted_classifier2(total,
                                                  better/total,
                                                  worse/total,
                                                  no_change/total)
    prediction_rwc &lt;- 
      factor(prediction_rwc, levels = c("better","worse", "no_change"))
    conTable &lt;- table(prediction_rwc, referenceF)
    cm &lt;- confusionMatrix(conTable, positive = c("better", "worse"),
                          mode = "everything")

    f1_rwc[i, 1:3] &lt;-  cm$byClass[,"F1"]
    pres_rwc[i, 1:3] &lt;- cm$byClass[,"Precision"]
    rec_rwc[i, 1:3] &lt;- cm$byClass[,"Recall"]
    kappa_rwc[[i]] &lt;- round(cm$overall["Kappa"], 2)
  }

  print(list("f1" = colMeans(f1_rwc, na.rm = T),
             "precision" = colMeans(pres_rwc, na.rm = T),
             "recall" = colMeans(rec_rwc, na.rm = T),
             "kappa" = mean(kappa_rwc, na.rm = T)))
}


It should be around 50 % faster (16.80 vs 11.02 sec for 1k iterations).

You, possibly, could speed it up even more by removing the confusionMatrix function and calculation of all of the necessary parameters by yourself.

Update

Based on confusionMatrix I managed to extract relevant parts of code and wrap into a function:

statistics &lt;- function(data, beta = 1) {
  stat &lt;- sapply(rownames(data), function(relevant) {

    if (nrow(data) &gt; 2) {
      m &lt;- matrix(NA, 2, 2)
      colnames(m) &lt;- rownames(m) &lt;- c("rel", "irrel")
      irrelCol &lt;- which(!(colnames(data) %in% relevant))
      relCol &lt;- which(colnames(data) %in% relevant)
      m[1, 1] &lt;- sum(data[relCol, relCol])
      m[1, 2] &lt;- sum(data[relCol, irrelCol])
      m[2, 1] &lt;- sum(data[irrelCol, relCol])
      m[2, 2] &lt;- sum(data[irrelCol, irrelCol])
      m &lt;- as.table(m)
      relevant &lt;- "rel"
    }
    numer &lt;- m[relevant, relevant]
    denom &lt;- sum(m[relevant, ])
    prec &lt;- ifelse(denom &gt; 0, numer/denom, NA) # Precision

    denom &lt;- sum(m[, relevant])
    rec &lt;- ifelse(denom &gt; 0, numer / denom, NA) # Recall

    F1 &lt;- (1 + beta^2)*prec*rec/((beta^2 * prec) + rec) # F1
    c('Precision' = prec, 'Recall' = rec, 'F1' = F1)
  })

  k &lt;- unlist(e1071::classAgreement(data))["kappa"]
  list(stat, kappa = k)
}


and then test3 looks like:

test3 &lt;- function(iterations = 100){

  vals &lt;- c("better","worse", "no_change")
  betters &lt;- rep("better", 100)
  worses &lt;- rep("worse", 50)
  no_changes &lt;- rep("no_change", 10)
  reference &lt;- sample(c(betters, worses, no_changes))

  better &lt;- sum(reference == "better")
  worse &lt;- sum(reference == "worse")
  no_change &lt;- sum(reference == "no_change")
  n &lt;- length(reference)

  f1_rwc &lt;- pres_rwc &lt;- rec_rwc &lt;- matrix(0., iterations, 3)
  kappa_rwc &lt;- vector("double", iterations)
  referenceF &lt;- factor(reference, levels = vals)

  for (i in seq(1:iterations)) {

    prediction_rwc &lt;-
      random_weighted_classifier2(n, better/n, worse/n, no_change/n)
    prediction_rwc &lt;-  factor(prediction_rwc, levels = vals)
    conTable &lt;- table(prediction_rwc, referenceF)
    cm2 &lt;- statistics(conTable)
    f1_rwc[i, 1:3] &lt;-  cm2[[1]][3, ]
    pres_rwc[i, 1:3] &lt;- cm2[[1]][1, ]
    rec_rwc[i, 1:3] &lt;- cm2[[1]][2, ]
    kappa_rwc[[i]] &lt;- round(cm2[[2]], 2)
  }

  list("f1" = colMeans(f1_rwc, na.rm = T),
       "precision" = colMeans(pres_rwc, na.rm = T),
       "recall" = colMeans(rec_rwc, na.rm = T),
       "kappa" = mean(kappa_rwc, na.rm = T))
}


This should run under 1sec for 1k iterations.

p.s. kappa can be calculated with this:

  n &lt;- sum(data)
  ni &lt;- rowSums(data)
  nj &lt;- colSums(data)
  p0 &lt;- sum(diag(data, names = F))/n
  pc &lt;- sum((ni/n) * (nj/n))
  k &lt;- (p0 - pc)/(1 - pc)


p.s.s. when reducing to those formulas, I stripped the code which was used for testing, so if you change your data format, bugs may appear. I assumed that the setting doesn't change.