/**

• Compute the probabilities of each value on the given [[DataSet]]
*
• @param x single colum [[DataSet]]
• @return Sequence of probabilites for each value
*/

private[this] def probs(x: DataSet[Double]): Seq[Double] = {

``````    val counts = x.groupBy(_.doubleValue)
.reduceGroup(_.size.toDouble)
.name("X Probs")
.collect

val total = counts.sum

counts.map(_ / total)``````

}

/**

• Computes 'symmetrical uncertainty' (SU) - a symmetric mutual information measure.
*
• It is defined as SU(X, y) = 2 * (IG(X|Y) / (H(X) + H(Y)))
*
• @param xy [[DataSet]] with two features
• @return SU value
*/

def symmetricalUncertainty(xy: DataSet[(Double, Double)]): Double = {

``````val su = xy.mapPartitionWith {
case in ⇒
val x = in map (_._2)
val y = in map (_._1)

val mu = mutualInformation(x, y)
val Hx = entropy(x)
val Hy = entropy(y)

Some(2 * mu / (Hx + Hy))
}

}

1 条回答 写回答 取消 提交回答

它使用n级并行：

def symmetricalUncertainty(xy: DataSet[(Double, Double)]): Double = {

``````val su = xy.reduceGroup { in ⇒
val invec = in.toVector
val x = invec map (_._2)
val y = invec map (_._1)

val mu = mutualInformation(x, y)
val Hx = entropy(x)
val Hy = entropy(y)

2 * mu / (Hx + Hy)
}

}
您可以在InformationTheory.scala及其测试InformationTheorySpec.scala中查看整个代码

2019-07-17 23:19:15
赞同 展开评论 打赏

23
0
0
12
0
0
11
0
0
12
0
0
10
0
0
113
0
0
74
1
0
65
1
0
74
0
0
52
1
0 146 2 143 1 131 1 2938 10 562 29 510 3
{"ft_lang_label":"__label__zh","ft_lang_prob":0.5819761,"math_prob":0.9985069,"size":1357,"snap":"2022-40-2023-06","text_gpt3_token_len":674,"char_repetition_ratio":0.11825573,"word_repetition_ratio":0.2857143,"special_character_ratio":0.24834193,"punctuation_ratio":0.16831683,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.9949561,"pos_list":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],"im_url_duplicate_count":[null,null,null,4,null,2,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2023-02-08T14:28:32Z\",\"WARC-Record-ID\":\"<urn:uuid:0f83f01d-c19c-4bbd-8a9c-99cdd68f3d77>\",\"Content-Length\":\"76378\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:30cada73-d010-4332-b66f-f6f7bee3f615>\",\"WARC-Concurrent-To\":\"<urn:uuid:8a33c6b4-a95e-4541-8eec-7ebba84f60ba>\",\"WARC-IP-Address\":\"140.205.60.46\",\"WARC-Target-URI\":\"https://developer.aliyun.com/ask/130896?spm=a2c6h.13159736\",\"WARC-Payload-Digest\":\"sha1:4SP5U5LEYATLTLGD2N2RIY46GLU5WPFE\",\"WARC-Block-Digest\":\"sha1:NTOKIS57VAOTA7K4SSQ3LOQAN5Z7YXAY\",\"WARC-Identified-Payload-Type\":\"application/xhtml+xml\",\"warc_filename\":\"/cc_download/warc_2023/CC-MAIN-2023-06/CC-MAIN-2023-06_segments_1674764500813.58_warc_CC-MAIN-20230208123621-20230208153621-00396.warc.gz\"}"}