写给开发者的机器学习指南（八）

import java.awt.{Rectangle}
import java.io.File
import java.text.SimpleDateFormat
import java.util.Date
import smile.plot.BarPlot
import scala.swing.{MainFrame, SimpleSwingApplication}
import scala.util.Try
object RecommendationSystem extendsSimpleSwingApplication {
 case classEmailData(emailDate : Date, sender : String, subject : String, body : String)
  def top = newMainFrame {
    title ="Recommendation System Example"
    val basePath ="/Users/../data"
    val easyHamPath= basePath + "/easy_ham"
    val mails =getFilesFromDir(easyHamPath).map(x => getFullEmail(x))
    valtimeSortedMails = mails
      .map  (x => EmailData (   getDateFromEmail(x),
                               getSenderFromEmail(x), 
                               getSubjectFromEmail(x), 
                               getMessageBodyFromEmail(x)
                            )
            )
      .sortBy(x=> x.emailDate)
    val(trainingData, testingData) = timeSortedMails
           .splitAt(timeSortedMails.length / 2)
      }
  defgetFilesFromDir(path: String): List[File] = {
    val d = newFile(path)
    if (d.exists&& d.isDirectory) {
      //Remove themac os basic storage file, 
      //andalternatively for unix systems "cmds"
     d.listFiles.filter(x =>  x.isFile &&
                               !x.toString.contains(".DS_Store") &&
                                !x.toString.contains("cmds")).toList
    } else {
      List[File]()
    }
  }
  defgetFullEmail(file: File): String = {
    //Note that theencoding of the example files is latin1, 
    //thus thisshould be passed to the from file method.
    val source =scala.io.Source.fromFile(file)("latin1")
    val fullEmail =source.getLines mkString "\n"
    source.close()
    fullEmail
  }
  defgetSubjectFromEmail(email: String): String = {
    //Find theindex of the end of the subject line
    val subjectIndex =email.indexOf("Subject:")
    valendOfSubjectIndex = email   
   .substring(subjectIndex)                        .indexOf('\n') +subjectIndex
    //Extract thesubject: start of subject + 7 
    // (length ofSubject:) until the end of the line.
    val subject =email
   .substring(subjectIndex + 8, endOfSubjectIndex)
    .trim
    .toLowerCase
    //Additionally,we check whether the email was a response and 
    //remove the're: ' tag, to make grouping on topic easier:
   subject.replace("re: ", "")
  }
  defgetMessageBodyFromEmail(email: String): String = {
    valfirstLineBreak = email.indexOf("\n\n")
    //Return themessage body filtered by only text 
    //from a-z andto lower case
   email.substring(firstLineBreak)
   .replace("\n", " ")
   .replaceAll("[^a-zA-Z ]", "")
    .toLowerCase
  }
  defgetSenderFromEmail(email: String): String = {
    //Find theindex of the From: line
    valfromLineIndex = email
   .indexOf("From:")
    val endOfLine =email
   .substring(fromLineIndex)
    .indexOf('\n')+ fromLineIndex
    //Search forthe <> tags in this line, as if they are there,
    // the emailaddress is contained inside these tags
    valmailAddressStartIndex = email
   .substring(fromLineIndex, endOfLine)
   .indexOf('<') + fromLineIndex + 1
    valmailAddressEndIndex = email
   .substring(fromLineIndex, endOfLine)
   .indexOf('>') + fromLineIndex
    if(mailAddressStartIndex > mailAddressEndIndex) {
      //The emailaddress was not embedded in <> tags,
      // extractthe substring without extra spacing and to lower case
      varemailString = email
     .substring(fromLineIndex + 5, endOfLine)
      .trim
      .toLowerCase
      //Remove apossible name embedded in () at the end of the line,
      //for examplein test@test.com (tester) the name would be removed here
      valadditionalNameStartIndex = emailString.indexOf('(')
      if(additionalNameStartIndex == -1) {
        emailString
       .toLowerCase
      }
      else {
        emailString
       .substring(0, additionalNameStartIndex)
        .trim
       .toLowerCase
      }
    }
    else {
      //Extract theemail address from the tags. 
      //If these<> tags are there, there is no () with a name in
      // the From:string in our data
      email
     .substring(mailAddressStartIndex, mailAddressEndIndex)
      .trim
      .toLowerCase
    }
  }
  defgetDateFromEmail(email: String): Date = {
    //Find theindex of the Date: line in the complete email
    valdateLineIndex = email
   .indexOf("Date:")
    valendOfDateLine = email
   .substring(dateLineIndex)
    .indexOf('\n')+ dateLineIndex
    //All possibledate patterns in the emails.
    valdatePatterns = Array(   "EEE MMM ddHH:mm:ss yyyy",
                                "EEE, ddMMM yyyy HH:mm",
                                "dd MMMyyyy HH:mm:ss",
                                "EEE MMMdd yyyy HH:mm")
   datePatterns.foreach { x =>
      //Try todirectly return a date from the formatting.
      //when itfails on a pattern it continues with the next one
      // until oneworks
      Try(returnnew SimpleDateFormat(x)
                   .parse(email
                           .substring(dateLineIndex +5, endOfDateLine)
                            .trim.substring(0,x.length)))
    }
    //Finally, ifall failed return null 
    //(this willnot happen with our example data but without 
    //this returnthe code will not compile)
    null
  }
}

//Add to the top body:
//First we group the emails by Sender, then we extractonly the sender address 
//and amount of emails, and finally we sort them onamounts ascending
val mailsGroupedBySender = trainingData
.groupBy(x => x.sender)
.map(x => (x._1, x._2.length))
.toArray
.sortBy(x => x._2)
//In order to plot the data we split the values from theaddresses as 
//this is how the plotting library accepts the data.
val senderDescriptions = mailsGroupedBySender
    .map(x =>x._1)
val senderValues = mailsGroupedBySender
    .map(x =>x._2.toDouble)
val barPlot = BarPlot.plot("", senderValues,senderDescriptions)
//Rotate the email addresses by -80 degrees such that wecan read them
barPlot.getAxis(0).setRotation(-1.3962634)
barPlot.setAxisLabel(0, "")
barPlot.setAxisLabel(1, "Amount of emails received")
peer.setContentPane(barPlot)
bounds = new Rectangle(800, 600)

//Code changes:
val mailsGroupedBySender = trainingData
    .groupBy(x=> x.sender)
    .map(x =>(x._1, Math.log1p(x._2.length)))
    .toArray
    .sortBy(x =>x._2)
barPlot.setAxisLabel(1, "Amount of emails receivedon log Scale ")

//Add to 'def top' 
val mailsGroupedByThread = trainingData
    .groupBy(x=> x.subject)
//Create a list of tuples with (subject, list of emails)
val threadBarPlotData = mailsGroupedByThread
    .map(x =>(x._1, x._2.length))
    .toArray
    .sortBy(x =>x._2)
val threadDescriptions = threadBarPlotData
    .map(x =>x._1)
val threadValues = threadBarPlotData
    .map(x =>x._2.toDouble)
//Code changes in 'def top'
val barPlot = BarPlot.plot(threadValues,threadDescriptions)
barPlot.setAxisLabel(1, "Amount of emails persubject")

//Code change:
val threadBarPlotData = mailsGroupedByThread
    .map(x =>(x._1, Math.log1p(x._2.length)))
    .toArray       
.sortBy(x => x._2)

//Create a list of tuples with (subject, list of emails, 
//time difference between first and last email)
val mailGroupsWithMinMaxDates = mailsGroupedByThread
 .map(x =>(x._1, x._2,
             (x._2
               .maxBy(x => x.emailDate)
               .emailDate.getTime - 
             x._2
               .minBy(x => x.emailDate)
               .emailDate.getTime
                            ) / 1000
              )
      )
//turn into a list of tuples with (topic, list of emails,
// time difference, and weight) filtered that onlythreads occur
val threadGroupedWithWeights = mailGroupsWithMinMaxDates
 .filter(x =>x._3 != 0)
 .map(x =>(x._1, x._2, x._3, 10 + 
               Math.log10(x._2.length.toDouble / x._3)))
 .toArray
 .sortBy(x =>x._4)
val threadGroupValues = threadGroupedWithWeights
 .map(x => x._4)
val threadGroupDescriptions = threadGroupedWithWeights
 .map(x => x._1)
//Change the bar plot code to plot this data:
val barPlot = BarPlot.plot(threadGroupValues,threadGroupDescriptions)
barPlot.setAxisLabel(1, "Weighted amount of emailsper subject")