写给开发者的机器学习指南（九）

def getStopWords: List[String] = {
    val source =scala.io.Source
       .fromFile(newFile("/Users/../stopwords.txt"))("latin1")
    val lines =source.mkString.split("\n")
    source.close()
    lines.toList
   }

//Add to top:
val stopWords = getStopWords
val threadTermWeights = threadGroupedWithWeights
    .toArray
    .sortBy(x =>x._4)
    .flatMap(x=> x._1
            .replaceAll("[^a-zA-Z]", "")
           .toLowerCase.split(" ")
           .filter(_.nonEmpty)
            .map(y=> (y,x._4)))
val filteredThreadTermWeights = threadTermWeights
    .groupBy(x=> x._1)
    .map(x =>(x._1, x._2.maxBy(y => y._2)._2))
    .toArray.sortBy(x=> x._1)
    .filter(x =>!stopWords.contains(x._1))

val tdm = trainingData
      .flatMap(x=> x.body.split(" "))
      .filter(x=> x.nonEmpty && !stopWords.contains(x))
      .groupBy(x=> x)
      .map(x =>(x._1, Math.log10(x._2.length + 1)))
      .filter(x=> x._2 != 0)

val trainingRanks = trainingData.map(mail => {
      //Determinethe weight of the sender, if it is lower than 1, pick 1 instead
      //This isdone to prevent the feature from having a negative impact
      valsenderWeight = mailsGroupedBySender
       .collectFirst { case (mail.sender, x) =>  Math.max(x,1)}
       .getOrElse(1.0)
      //Determinethe weight of the subject
      valtermsInSubject = mail.subject
       .replaceAll("[^a-zA-Z ]", "")
       .toLowerCase.split(" ")
        .filter(x=>    x.nonEmpty &&
       !stopWords.contains(x)
        )
      valtermWeight = if (termsInSubject.size > 0)
       Math.max(termsInSubject
          .map(x=> {
         tdm.collectFirst { case (y, z) if y == x => z}
           .getOrElse(1.0)
        })
          .sum /termsInSubject.length,1)
      else 1.0
      //Determineif the email is from a thread,
      //and if itis the weight from this thread:
      valthreadGroupWeight: Double = threadGroupedWithWeights
       .collectFirst { case (mail.subject, _, _, weight) => weight}
       .getOrElse(1.0)
      //Determinethe commonly used terms in the email and the weight belonging to it:
      valtermsInMailBody = mail.body
       .replaceAll("[^a-zA-Z ]", "")
       .toLowerCase.split(" ")
        .filter(x=>    x.nonEmpty &&
       !stopWords.contains(x)
        )
      valcommonTermsWeight = if (termsInMailBody.size > 0)
       Math.max(termsInMailBody
          .map(x=> {
         tdm.collectFirst { case (y, z) if y == x => z}
           .getOrElse(1.0)
        })
          .sum /termsInMailBody.length,1)
      else 1.0
      val rank=    termWeight *
       threadGroupWeight *
        commonTermsWeight*
       senderWeight
      (mail, rank)
    })
    valsortedTrainingRanks = trainingRanks
      .sortBy(x=> x._2)
    val median =sortedTrainingRanks(sortedTrainingRanks.length / 2)._2
    val mean =  sortedTrainingRanks
      .map(x =>x._2).sum /
               sortedTrainingRanks.length

val testingRanks = trainingData.map(mail => {
      //mailcontains (full content, date, sender, subject, body)
      //Determinethe weight of the sender
      valsenderWeight = mailsGroupedBySender
       .collectFirst { case (mail.sender, x) =>  Math.max(x,1)}
       .getOrElse(1.0)
      //Determine the weight of the subject
      valtermsInSubject = mail.subject
       .replaceAll("[^a-zA-Z ]", "")
       .toLowerCase.split(" ")
        .filter(x=>    x.nonEmpty &&
       !stopWords.contains(x)
        )
      valtermWeight = if (termsInSubject.size > 0)
       Math.max(termsInSubject
        .map(x=> {
       tdm.collectFirst { case (y, z) if y == x => z}
         .getOrElse(1.0)
      })
        .sum /termsInSubject.length,1)
      else 1.0
      //Determineif the email is from a thread,
      //and if itis the weight from this thread:
      valthreadGroupWeight: Double = threadGroupedWithWeights
       .collectFirst { case (mail.subject, _, _, weight) => weight}
       .getOrElse(1.0)
      //Determinethe commonly used terms in the email and the weight belonging to it:
      valtermsInMailBody = mail.body
       .replaceAll("[^a-zA-Z ]", "")
       .toLowerCase.split(" ")
        .filter(x=>    x.nonEmpty &&
       !stopWords.contains(x)
        )
      valcommonTermsWeight = if (termsInMailBody.size > 0)
       Math.max(termsInMailBody
        .map(x=> {
       tdm.collectFirst { case (y, z) if y == x => z}
         .getOrElse(1.0)
      })
        .sum /termsInMailBody.length,1)
      else 1.0
      val rank=    termWeight *
       threadGroupWeight *
       commonTermsWeight *
       senderWeight
      (mail, rank)
    })
    valpriorityEmails = testingRanks
        .filter(x=> x._2 >= mean)
   println(priorityEmails.length + " ranked as priority")

val timsEmails = testingRanks
    .filter(x =>x._1.sender == "tim.one@...")
    .sortBy(x =>-x._2)
timsEmails
    .foreach(x=> println(  "| " +
                            x._1.emailDate +
                            " |"  +
                            x._1.subject + 
                            " | " +
                            df.format(x._2) + 
                            " |")
                        )

def getDataFromCSV(file: File): (Array[Array[Double]],Array[Double]) = {
    val source =scala.io.Source.fromFile(file)
    val data =source.getLines().drop(1).map(x => getDataFromString(x)).toArray
    source.close()
    var inputData =data.map(x => x._1)
    var resultData= data.map(x => x._2)
    return(inputData,resultData)
  }
  defgetDataFromString(dataString: String): (Array[Double], Double) = {
    //Split thecomma separated value string into an array of strings
    val dataArray:Array[String] = dataString.split(',')
    var person =1.0
    if(dataArray(0) == "\"Male\"") {
      person = 0.0
    }
    //Extract thevalues from the strings
    //Since the data is in US metrics 
    //inch andpounds we will recalculate this to cm and kilo's
    val data :Array[Double] = Array(person,dataArray(1).toDouble * 2.54)
    val weight:Double = dataArray(2).toDouble * 0.45359237
    //And return theresult in a format that can later easily be used to feed to Smile
    return (data,weight)
  }

object LinearRegressionExample extendsSimpleSwingApplication {
  def top = newMainFrame {
    title ="Linear Regression Example"
    val basePath ="/Users/.../OLS_Regression_Example_3.csv"
    val testData =getDataFromCSV(new File(basePath))
    val plotData =(testData._1 zip testData._2).map(x => Array(x._1(1) ,x._2))
    valmaleFemaleLabels = testData._1.map( x=> x(0).toInt)
    val plot =  ScatterPlot.plot(   plotData,
                                    maleFemaleLabels,
                                    '@',
                                   Array(Color.blue, Color.green)
                                 )
   plot.setTitle("Weight and heights for male and females")
    plot.setAxisLabel(0,"Heights")
   plot.setAxisLabel(1,"Weights")
   peer.setContentPane(plot)
    size = newDimension(400, 400)
  }

al olsModel = new OLS(testData._1,testData._2)
With this olsModel we can now predict someone's weight based on length andgender as follows: 
println("Prediction for Male of 1.7M: "+olsModel.predict(Array(0.0,170.0)))
println("Prediction for Female of 1.7M:" +olsModel.predict(Array(1.0,170.0)))
println("Model Error:" + olsModel.error())
and this will give the following results:
Prediction for Male of 1.7M: 79.14538559840447
    Prediction forFemale of 1.7M:70.35580395758966
    ModelError:4.5423150758157185