写给开发者的机器学习指南（七）

def getMessage(file : File)  : String =
  {
    //Note that theencoding of the example files is latin1,
    // thus thisshould be passed to the fromFile method.
    val source =scala.io.Source.fromFile(file)("latin1")
    val lines =source.getLines mkString "\n"
    source.close()
    //Find thefirst line break in the email, 
    //as thisindicates the message body
    valfirstLineBreak = lines.indexOf("\n\n")
    //Return themessage body filtered by only text from a-z and to lower case
    return lines
       .substring(firstLineBreak)
       .replace("\n"," ")
       .replaceAll("[^a-zA-Z ]","")
       .toLowerCase()
  }

def getFilesFromDir(path: String):List[File] = {
    val d = newFile(path)
    if (d.exists&& d.isDirectory) {
      //Remove themac os basic storage file, 
      //andalternatively for unix systems "cmds"
      d .listFiles
        .filter(x=> x  .isFile &&
                    !x .toString
                       .contains(".DS_Store") &&
                    !x .toString
                        .contains("cmds"))
                       .toList
    } 
    else {
      List[File]()
    }
  }

def main(args: Array[String]): Unit = {
    val basePath ="/Users/../Downloads/data"
    val spamPath =basePath + "/spam"
    val spam2Path =basePath + "/spam_2"
    val easyHamPath= basePath + "/easy_ham"
    valeasyHam2Path = basePath + "/easy_ham_2"
    valamountOfSamplesPerSet = 500
    valamountOfFeaturesToTake = 100
    //First get asubset of the filenames for the spam 
    // sample set(500 is the complete set in this case)
    vallistOfSpamFiles =  getFilesFromDir(spamPath)
       .take(amountOfSamplesPerSet)
    //Then get themessages that are contained in these files
    val spamMails =listOfSpamFiles.map(x => (x, getMessage(x)))
     //Get a subsetof the filenames from the ham sample set
     // (note thatin this case it is not necessary to randomly
     // sample asthe emails are already randomly ordered)
    vallistOfHamFiles =  getFilesFromDir(easyHamPath)
        .take(amountOfSamplesPerSet)
    //Get themessages that are contained in the ham files
    valhamMails  = listOfHamFiles
        .map{x=> (x,getMessage(x)) }
  }

val spamTDM  =spamMails
     .flatMap(email => email
       ._2.split(" ")
       .filter(word => word.nonEmpty)
        .map(word=> (email._1.getName,word)))
        .groupBy(x=> x._2)
        .map(x=> (x._1, x._2.groupBy(x => x._1)))
        .map(x=> (x._1, x._2.map( y => (y._1, y._2.length))))
        .toList
    //Sort thewords by occurrence rate descending 
    //(amount oftimes the word occurs among all documents)
  val sortedSpamTDM= spamTDM
    .sortBy(x=>  - (x._2.size.toDouble /spamMails.length))
  val hamTDM  = hamMails
      .flatMap(email=> email
     ._2.split(" ")
      .filter(word=> word.nonEmpty)
      .map(word=> (email._1.getName,word)))
      .groupBy(x=> x._2)
      .map(x =>(x._1, x._2.groupBy(x => x._1)))
      .map(x =>(x._1, x._2.map( y => (y._1, y._2.length))))
      .toList
    //Sort thewords by occurrence rate  descending 
    //(amount oftimes the word occurs among all documents)
    valsortedHamTDM = hamTDM
        .sortBy(x =>  - (x._2.size.toDouble / spamMails.length))

def getStopWords() : List[String] =
  {
    val source =scala.io.Source
       .fromFile(new File("/Users/.../.../ExampleData/stopwords.txt"))("latin1")
    val lines =source.mkString.split("\n")
    source.close()
    return  lines.toList
  }
现在我们增加TDM的代码以从结果中移去停止词
val stopWords = getStopWords
val spamTDM  =spamMails
     .flatMap(email => email
     ._2.split(" ")
      .filter(word=> word.nonEmpty && !stopWords.contains(word))
      .map(word=> (email._1.getName,word)))
      .groupBy(x=> x._2)
      .map(x =>(x._1, x._2.groupBy(x => x._1)))
      .map(x =>(x._1, x._2.map( y => (y._1, y._2.length))))
      .toList
 val hamTDM  = hamMails
     .flatMap(email => email
     ._2.split(" ")
      .filter(word=> word.nonEmpty && !stopWords.contains(word))
      .map(word=> (email._1.getName,word)))
      .groupBy(x=> x._2)
      .map(x =>(x._1, x._2.groupBy(x => x._1)))
      .map(x =>(x._1, x._2.map( y => (y._1, y._2.length))))
      .toList

//Add the code for getting the TDM data and combining itinto a feature bag.
val hamFeatures = hamTDM
    .records
   .take(amountOfFeaturesToTake)
    .map(x =>x.term)
val spamFeatures = spamTDM
    .records
   .take(amountOfFeaturesToTake)
    .map(x =>x.term)
//Now we have a set of ham and spam features,
// we group them and then remove the intersectingfeatures, as these are noise.
var data = (hamFeatures ++ spamFeatures).toSet
hamFeatures
    .intersect(spamFeatures)
    .foreach(x=> data = (data - x))
//Initialise a bag of words that takes the top x features
//from both spam and ham and combines them
var bag = new Bag[String] (data.toArray)
//Initialise the classifier array with first a set of 0(spam)
//and then a set of 1(ham) values that represent theemails
var classifiers =  Array.fill[Int](amountOfSamplesPerSet)(0) ++
                   Array.fill[Int](amountOfSamplesPerSet)(1)
//Get the trainingData in the right format for the spammails
var spamData = spamMails
    .map(x =>bag.feature(x._2.split(" ")))
    .toArray
//Get the trainingData in the right format for the hammails
var hamData = hamMails
    .map(x =>bag.feature(x._2.split(" ")))
    .toArray
//Combine the training data from both categories
var trainingData = spamData ++ hamData

//Create the bayes model as a multinomial with 2classification
// groups and the amount of features passed in theconstructor.
  var bayes = newNaiveBayes(NaiveBayes.Model.MULTINOMIAL, 2, data.size)
  //Now train thebayes instance with the training data,
  // which isrepresented in a specific format due to the 
  //bag.featuremethod, and the known classifiers.
 bayes.learn(trainingData, classifiers)

val listOfSpam2Files =  getFilesFromDir(spam2Path)
val spam2Mails = listOfSpam2Files
    .map{x =>(x,getMessage(x)) }
val spam2FeatureVectors = spam2Mails
    .map(x =>bag.feature(x._2.split(" ")))
val spam2ClassificationResults = spam2FeatureVectors
    .map(x =>bayes.predict(x))
//Correct classifications are those who resulted in aspam classification (0)
val correctClassifications = spam2ClassificationResults
    .count( x=>x == 0)
println (  correctClassifications + 
            "of " + 
           listOfSpam2Files.length + 
           "were correctly classified"
        )
println (( (correctClassifications.toDouble / 
            listOfSpam2Files.length) * 100)  +
             "% was correctly classified"
        )
//In case the algorithm could not decide which categorythe email 
//belongs to, it gives a -1 (unknown) rather than a 0(spam) or 1 (ham)
val unknownClassifications = spam2ClassificationResults
    .count( x=>x == -1)
println(   unknownClassifications + 
            "of " + 
           listOfSpam2Files.length +
           "were unknowingly classified"
        )
println(    (   (unknownClassifications.toDouble / 
                listOfSpam2Files.length) * 100)  +
                %was unknowingly classified"
        )