golang源码分析：推荐系统gorse（part I）

type User struct {
    UserId    string
}

type Item struct {
    ItemId    string
    Timestamp time.Time
    Labels    []string
}

type Feedback struct {
    FeedbackType string
    UserId       string
    ItemId       string
    Timestamp   time.Time
}

.deepsource.toml //放置一些配置项，熟悉整个代码后改配置可以快速配出一个系统
LICENSE
README.md
assets
base
bin
cmd //三个节点的入口文件
codecov.yml
config
docker //三个节点会编译成三个镜像，这里放置了他们的Dockerfile，
//以及docker-compose.yml，通过docker compose up 可以启动服务
floats
go.mod
go.sum
master //master节点的实现
misc
model  //模型相关的实现
protocol
server  //server 节点的实现
storage  //存储的实现，主要是mysql和redis
worker  //worker节点的实现

1，LoadLocalCache //通过缓存的数据初始化模型
2，初始化缓存和持久化存储（redis、mysql）
m.DataClient, err = data.Open(m.GorseConfig.Database.DataStore)
m.CacheClient, err = cache.Open(m.GorseConfig.Database.CacheStore)
3，err = m.loadRankingDataset()//加载排序用的数据集
  m.rankingTrainSet, m.rankingTestSet = rankingDataset.Split(0, 0)
4，err = m.loadClickDataset()//加载点击预测数据集
  m.clickTrainSet, m.clickTestSet = clickDataset.Split(0.2, 0)
5，go m.StartHttpServer()//启动一个httpserver主要用户数据的批量导入
  http.Handle("/", http.FileServer(&SinglePageAppFileSystem{statikFS}))
  http.HandleFunc("/api/bulk/items", m.importExportItems)
  http.HandleFunc("/api/bulk/feedback", m.importExportFeedback)
6，go m.FitLoop()//进入训练模型的大循环
7，go m.SearchLoop()//进入最佳模型参数筛选大循环
8，go m.AnalyzeLoop()//进行全局非个性化推荐，存入缓存
9，启动grpc服务
protocol.RegisterMasterServer(grpcServer, m)
  if err = grpcServer.Serve(lis); err != nil {

1，加载数据集
  m.hasFeedbackInserted()
  m.loadRankingDataset()
  m.loadClickDataset()
 2，训练排序模型、计算评分、存储
  m.fitRankingModelAndNonPersonalized(lastNumRankingUsers, lastNumRankingItems, lastNumRankingFeedback)
  
  score := rankingModel.Fit(m.rankingTrainSet, m.rankingTestSet, ranking.NewFitConfig().SetJobs(m.GorseConfig.Master.FitJobs))
  err := m.DataClient.InsertMeasurement(data.Measurement{Name: RankingTop10NDCG, Value: score.NDCG, Timestamp: time.Now()});
 3，训练点击预测模型、计算评分、存储
  m.fitClickModel(lastNumClickUsers, lastNumClickItems, lastNumClickFeedback)
  
  bestClickModel, bestClickScore := m.clickModelSearcher.GetBestModel()
  score := clickModel.Fit(m.clickTrainSet, m.clickTestSet, click.NewFitConfig().SetJobs(m.GorseConfig.Master.FitJobs))
  m.DataClient.InsertMeasurement(data.Measurement{Name: ClickPrecision, Value: score.Precision, Timestamp: time.Now()})

  // collect similar items
  m.similar(m.rankingItems, m.rankingFullSet, model.SimilarityDot)
  // collect popular items
  m.popItem(m.rankingItems, m.rankingFeedbacks)
  // collect latest items
   m.latest(m.rankingItems)

 1，排序模型
 m.searchRankingModel(lastNumRankingUsers, lastNumRankingItems, lastNumRankingFeedbacks)
 err = m.rankingModelSearcher.Fit(m.rankingTrainSet, m.rankingTestSet)
 2，点击预测模型    
 m.searchClickModel(lastNumClickUsers, lastNumClickItems, lastNumClickFeedbacks)
 err = m.clickModelSearcher.Fit(m.clickTrainSet, m.clickTestSet)

models := []string{"bpr", "ccd", "knn"}
r := RandomSearchCV(m, trainSet, valSet, m.GetParamsGrid(), searcher.numTrials, 0,NewFitConfig().SetJobs(searcher.numJobs))
score := estimator.Fit(trainSet, testSet, fitConfig)  

 RandomSearchCV(fm, trainSet, valSet, grid, searcher.numTrials*2, 0,NewFitConfig().SetJobs(searcher.numJobs))
 score := estimator.Fit(trainSet, testSet, fitConfig)

 m.analyze()
 clickThroughRates, err := m.DataClient.GetMeasurements(ClickThroughRate, 30)
 m.DataClient.GetClickThroughRate(date, m.GorseConfig.Database.ClickFeedbackTypes, m.GorseConfig.Database.ReadFeedbackType)
 m.DataClient.InsertMeasurement(data.Measurement{
        Name:      ClickThroughRate,
        Timestamp: date,
        Value:     float32(clickThroughRate),
      })

1，加载上次结果的缓存数据
state, err := LoadLocalCache(filepath.Join(os.TempDir(), "gorse-worker"))
2，获取master的grpc连接
w.masterClient = protocol.NewMasterClient(conn)
3，同步数据
go w.Sync()
4，拉取用户和模型相关的数据
go w.Pull()
主要包括三个部分：用户数据、排序模型、点击预测模型
  w.masterClient.GetUserIndex
  w.masterClient.GetRankingModel
  w.masterClient.GetClickModel
5，启动系统监控
go w.ServeMetrics()
  http.Handle("/metrics", promhttp.Handler())
6，进行个性化推荐
w.Recommend(w.rankingModel, workingUsers)

  1，从master节点获取的元数据
  w.masterClient.GetMeta
  2，持久化存储里的数据
  w.dataClient, err = data.Open(w.cfg.Database.DataStore)
  3，cache里的数据
  w.cacheClient, err = cache.Open(w.cfg.Database.CacheStore)

1，加载所有物料
  itemIds := m.GetItemIndex().GetNames()
 2，加载用户推荐的历史数据
  historyItems, err := loadUserHistoricalItems(w.dataClient, userId)
 3，加载用户正反馈的数据
  favoredItems, err := loadUserHistoricalItems(w.dataClient, userId, w.cfg.Database.PositiveFeedbackType...)
  4，获取候选物料和评分
  candidateItems, candidateScores := recItems.PopAll()
  5，加载最新数据
  latestItems, err := w.cacheClient.GetScores(cache.LatestItems, "", 0, w.cfg.Recommend.ExploreLatestNum-1)
  6，通过点击预测模型排序
  result, err = w.rankByClickTroughRate(userId, candidateItems)
  7，随机排序
  result = w.randomInsertLatestItem(candidateItems, candidateScores)
  8，缓存数据
  w.cacheClient.SetScores(cache.RecommendItems, userId, result)
  w.cacheClient.SetString(cache.LastUpdateRecommendTime, userId, base.Now())

1，加载缓存数据
state, err := LoadLocalCache(filepath.Join(os.TempDir(), "gorse-server"))
2，获取主节点的连接
s.masterClient = protocol.NewMasterClient(conn)
3，同步数据主要是元数据
go s.Sync()
  s.masterClient.GetMeta
4，提供http服务
s.StartHttpServer()

1，web服务
s.CreateWebService()
2，接口文档
  specConfig := restfulspec.Config{
    WebServices: restful.RegisteredWebServices(),
    APIPath:     "/apidocs.json",
  }
 3，监控
 http.Handle("/metrics", promhttp.Handler())