package zhouls.bigdata.DataFeatureSelection

import org.apache.spark.SparkConf

import org.apache.spark.SparkContext

import org.apache.spark.ml.feature.ChiSqSelector//导入mi里的特征选择里的ChiSqSelector算法

import org.apache.spark.mllib.linalg.Vectors//特征向量

/**

 * By  zhouls

 */

object ChiSqSelector extends App {

    val conf = new SparkConf().setMaster("local").setAppName("ChiSqSelector")

    val sc = new SparkContext(conf)

    val sqlContext = new org.apache.spark.sql.SQLContext(sc)

    import sqlContext.implicits._

    //构造数据集

    val data = Seq(

      (, Vectors.dense(0.0, 0.0, 18.0, 1.0), 1.0),

      (, Vectors.dense(0.0, 1.0, 12.0, 0.0), 0.0),

      (, Vectors.dense(1.0, 0.0, 15.0, 0.1), 0.0)

    )

    val df = sc.parallelize(data).toDF("id", "features", "clicked")//将构造的数据集，转成DF，即DataFrame

    df.select("id", "features","clicked").show()

    //使用卡方检验，将原始特征向量（特征数为4）降维（特征数为3）

    val selector = new ChiSqSelector().setNumTopFeatures().setFeaturesCol("features").setLabelCol("clicked").setOutputCol("selectedFeatures")

    val result = selector.fit(df).transform(df)

    result.show()

}