Spark MLlib特征处理：Normalizer 正则化---原理及实战

来源：互联网发布：淘宝店铺怎么复核认证编辑：程序博客网时间：2024/06/10 07:12

原理

向量a→=(x1,x2,x3...xn),ak是a→中的任意元素，k=1,2,3⋯n 例如:a→代表一个样本，ak代表特征。

p-Norm p范数

p=1

$p N o r m = \sum k = 1 n | a k |$

p=2

$p N o r m = \sum k = 1 n a 2 k - - - - - \sqrt$

p=inf

$p N o r m = max | a k |$

2<p<inf

$p N o r m = \sum k = 1 n | a k | p$

a→ 的 L1 正则化：ak 除以 pNorm p=1

$N o r m a l i z e r (a k) = a k \sum n k = 1 | a k |$

a→ 的 L2 正则化：ak 除以 pNorm p=2

$N o r m a l i z e r (a k) = a k \sum n k = 1 a 2 k - - - - - - - \sqrt$

实战

import org.apache.spark.ml.feature.Normalizerimport org.apache.spark.sql.SQLContextimport org.apache.spark.{SparkContext, SparkConf}object NormalizerExample {  def main(args: Array[String]) {    val conf = new SparkConf().setAppName("NormalizerExample").setMaster("local[6]")    val sc = new SparkContext(conf)    val sqlContext = new SQLContext(sc)    // LibSVMFile的格式：标签 索引：值    // 0 2:51 3:253 5:253    // 1 2:124 3:253 4:255    // 1 2:145 3:253 5:211    val dataFrame = sqlContext.read.format("libsvm").load("data/libsvm.txt")    // L1正则化    val normalizer = new Normalizer().setInputCol("features").setOutputCol("normFeatures")      // 设置 L1正则化      .setP(1.0)    // 正则化转换    val l1NormData = normalizer.transform(dataFrame)    //l1NormData.foreach(println)    // 结果：    // [1.0,(5,[1,2,4],[145.0,253.0,211.0]),(5,[1,2,4],[0.23809523809523808,0.4154351395730706,0.3464696223316913])]    // [0.0,(5,[1,2,4],[51.0,253.0,253.0]),(5,[1,2,4],[0.09156193895870736,0.4542190305206463,0.4542190305206463])]    // [1.0,(5,[1,2,3],[124.0,253.0,255.0]),(5,[1,2,3],[0.1962025316455696,0.40031645569620256,0.40348101265822783])]    // 结果分析：    // 如：向量(5,[1,2,4],[145.0,253.0,211.0])    // p-Norm=145.0+253.0+211.0=609.0   p=1    // L1正则化：(5,[1,2,4],[145.0/609.0,253.0/609.0,211.0/609.0])    // 正则化结果：(5,[1,2,4],[0.23809523809523808,0.4154351395730706,0.3464696223316913])]    // L2正则化     val l2InfNormData = normalizer.transform(dataFrame, normalizer.p -> 2)    l2InfNormData.foreach(println)    // 结果：    // [0.0,(5,[1,2,4],[51.0,253.0,253.0]),(5,[1,2,4],[0.1411129915384288,0.7000311148867153,0.7000311148867153])]    // [1.0,(5,[1,2,4],[145.0,253.0,211.0]),(5,[1,2,4],[0.40284772269065683,0.702899819591284,0.5862128930188178])]    // [1.0,(5,[1,2,3],[124.0,253.0,255.0]),(5,[1,2,3],[0.3263044910373298,0.6657664212293906,0.6710293968912829])]    // 结果分析：    // 如：向量(5,[1,2,4],[145.0,253.0,211.0])    // p-Norm=math.sqrt(145*145+253*253+211*211)=359.9374945737107 p=2    // L2正则化：(5,[1,2,4],[145.0/p-Norm,253.0/p-Norm,211.0/p-Norm])    // 正则化结果：(5,[1,2,4],[0.40284772269065683,0.702899819591284,0.5862128930188178])]    sc.stop()  }}

0 0