机器学习从零开始系列连载(10)——最优化原理（下）

// data structure of ftrl solver.
type FtrlSolver struct {
    Alpha   float64 `json:"Alpha"`
    Beta    float64 `json:"Beta"`
    L1      float64 `json:"L1"`
    L2      float64 `json:"L2"`
    Featnum int     `json:"Featnum"`
    Dropout float64 `json:"Dropout"`
    N []float64 `json:"N"`
    Z []float64 `json:"Z"`
    Weights util.Pvector `json:"Weights"`
    Init bool `json:"Init"`
}
// data structure of parameter server.
type FtrlParamServer struct {
    FtrlSolver
    ParamGroupNum int
    LockSlots     []sync.Mutex
    log           log4go.Logger
}
// fetch parameter group for update n and z value.
func (fps *FtrlParamServer) FetchParamGroup(n []float64, z []float64, group int) error {
    if !fps.FtrlSolver.Init {
        fps.log.Error("[FtrlParamServer-FetchParamGroup] Initialize fast ftrl solver error.")
        return errors.New("[FtrlParamServer-FetchParamGroup] Initialize fast ftrl solver error.")
    }
    var start int = group * ParamGroupSize
    var end int = util.MinInt((group+1)*ParamGroupSize, fps.FtrlSolver.Featnum)
    fps.LockSlots[group].Lock()
    for i := start; i < end; i++ {
        n[i] = fps.FtrlSolver.N[i]
        z[i] = fps.FtrlSolver.Z[i]
    }
    fps.LockSlots[group].Unlock()
    return nil
}
// fetch parameter from server.
func (fps *FtrlParamServer) FetchParam(n []float64, z []float64) error {
    if !fps.FtrlSolver.Init {
        fps.log.Error("[FtrlParamServer-FetchParam] Initialize fast ftrl solver error.")
        return errors.New("[FtrlParamServer-FetchParam] Initialize fast ftrl solver error.")
    }
    for i := 0; i < fps.ParamGroupNum; i++ {
        err := fps.FetchParamGroup(n, z, i)
        if err != nil {
            fps.log.Error(fmt.Sprintf("[FtrlParamServer-FetchParam] Initialize fast ftrl solver error.", err.Error()))
            return errors.New(fmt.Sprintf("[FtrlParamServer-FetchParam] Initialize fast ftrl solver error.", err.Error()))
        }
    }
    return nil
}
// push parameter group for upload n and z value.
func (fps *FtrlParamServer) PushParamGroup(n []float64, z []float64, group int) error {
    if !fps.FtrlSolver.Init {
        fps.log.Error("[FtrlParamServer-PushParamGroup] Initialize fast ftrl solver error.")
        return errors.New("[FtrlParamServer-PushParamGroup] Initialize fast ftrl solver error.")
    }
    var start int = group * ParamGroupSize
    var end int = util.MinInt((group+1)*ParamGroupSize, fps.FtrlSolver.Featnum)
    fps.LockSlots[group].Lock()
    for i := start; i < end; i++ {
        fps.FtrlSolver.N[i] += n[i]
        fps.FtrlSolver.Z[i] += z[i]
        n[i] = 0
        z[i] = 0
    }
    fps.LockSlots[group].Unlock()
    return nil
}
// push weight update to parameter server.
func (fw *FtrlWorker) PushParam(param_server *FtrlParamServer) error {
    if !fw.FtrlSolver.Init {
        fw.log.Error("[FtrlWorker-PushParam] Initialize fast ftrl solver error.")
        return errors.New("[FtrlWorker-PushParam] Initialize fast ftrl solver error.")
    }
    for i := 0; i < fw.ParamGroupNum; i++ {
        err := param_server.PushParamGroup(fw.NUpdate, fw.ZUpdate, i)
        if err != nil {
            fw.log.Error(fmt.Sprintf("[FtrlWorker-PushParam] Initialize fast ftrl solver error.", err.Error()))
            return errors.New(fmt.Sprintf("[FtrlWorker-PushParam] Initialize fast ftrl solver error.", err.Error()))
        }
    }
    return nil
}
// to do update for all weights.
func (fw *FtrlWorker) Update(
    x util.Pvector,
    y float64,
    param_server *FtrlParamServer) float64 {
    if !fw.FtrlSolver.Init {
        return 0.
    }
    var weights util.Pvector = make(util.Pvector, fw.FtrlSolver.Featnum)
    var gradients []float64 = make([]float64, fw.FtrlSolver.Featnum)
    var wTx float64 = 0.
    for i := 0; i < len(x); i++ {
        item := x[i]
        if util.UtilGreater(fw.FtrlSolver.Dropout, 0.0) {
            rand_prob := util.UniformDistribution()
            if rand_prob < fw.FtrlSolver.Dropout {
                continue
            }
        }
        var idx int = item.Index
        if idx >= fw.FtrlSolver.Featnum {
            continue
        }
        var val float64 = fw.FtrlSolver.GetWeight(idx)
        weights = append(weights, util.Pair{idx, val})
        gradients = append(gradients, item.Value)
        wTx += val * item.Value
    }
    var pred float64 = util.Sigmoid(wTx)
    var grad float64 = pred - y
    util.VectorMultiplies(gradients, grad)
    for k := 0; k < len(weights); k++ {
        var i int = weights[k].Index
        var g int = i / ParamGroupSize
        if fw.ParamGroupStep[g]%fw.FetchStep == 0 {
            param_server.FetchParamGroup(
                fw.FtrlSolver.N,
                fw.FtrlSolver.Z,
                g)
        }
        var w_i float64 = weights[k].Value
        var grad_i float64 = gradients[k]
        var sigma float64 = (math.Sqrt(fw.FtrlSolver.N[i]+grad_i*grad_i) - math.Sqrt(fw.FtrlSolver.N[i])) / fw.FtrlSolver.Alpha
        fw.FtrlSolver.Z[i] += grad_i - sigma*w_i
        fw.FtrlSolver.N[i] += grad_i * grad_i
        fw.ZUpdate[i] += grad_i - sigma*w_i
        fw.NUpdate[i] += grad_i * grad_i
        if fw.ParamGroupStep[g]%fw.PushStep == 0 {
            param_server.PushParamGroup(fw.NUpdate, fw.ZUpdate, g)
        }
        fw.ParamGroupStep[g] += 1
    }
    return pred
}