【总结】1086- 如何搭建适合自己团队的构建部署平台

  // 构建的流程
  async run() {
    const app = this.app;
    const processData = {};
    const pipeline = [{
      handler: context => app.fetchUpdate(context), // Git 更新代码
      name: 'codeUpdate',
      progress: 10 // 这里是当前构建的进度
    }, {
      handler: context => app.installDependency(context), // npm install 安装依赖
      name: 'dependency',
      progress: 30
    }, {
      handler: context => app.check(context), // 构建的前置校验（非必须）：代码检测，eslint，package.json 版本等
      name: 'check',
      progress: 40
    }, {
      handler: context => app.pack(context), // npm run build 的打包逻辑，如果有其他的项目类型，例如 gulp 之类，也可以在这一步进行处理
      name: 'pack', 
      progress: 70
    }, {
      handler: context => app.injectScript(context), // 构建的后置步骤（非必须）：打包后的资源注入
      name: 'injectRes',
      progress: 80
    }, { // docker image build
      handler: context => app.buildImage(context), // 生成 docker 镜像文件，镜像上传仓库，以及之后调用 K8S 能力进行部署
      name: 'buildImage',
      progress: 90
    }];
    // 循环执行每一步构建流程
    for (let i = 0; i < pipeline.length; i++) {
      const task = pipeline[i];
      const [ err, response ] = await to(this.execProcess({
        ...task,
        step: i
      }));
      if (response) {
        processData[task.name] = response;
      }
    }
    return Promise.resolve(processData);
  }
  // 执行构建中的 handler 操作
  async execProcess(task) {
    this.step(task.name, { status: 'start' });
    const result = await task.handler(this.buildContext);
    this.progress(task.progress);
    this.step(task.name, { status: 'end', taskMeta: result });
    return result;
  }

import { spawn } from 'child_process';
// git clone 
execCmd(`git clone ${url} ${dir}`, {
  cwd: this.root,
  verbose: this.verbose
});
// npm run build
const cmd = ['npm run build', cmdOption].filter(Boolean).join(' ');
execCmd(cmd, options);
// 执行 shell 命令
function execCmd(cmd: string, options:any = {}): Promise<any> {
  const [ shell, ...args ] = cmd.split(' ').filter(Boolean);
  const { verbose, ...others } = options;
  return new Promise((resolve, reject) => {
    let child: any = spawn(shell, args, others);
    let stdout = '';
    let stderr = '';
    child.stdout && child.stdout.on('data', (buf: Buffer) => {
      stdout = `${stdout}${buf}`;
      if (verbose) {
        logger.info(`${buf}`);
      }
    });
    child.stderr && child.stderr.on('data', (buf: Buffer) => {
      stderr = `${stderr}${buf}`;
      if (verbose) {
        logger.error(`${buf}`);
      }
    });
    child.on('exit', (code: number) => {
      if (code !== 0) {
        const reason = stderr || 'some unknown error';
        reject(`exited with code ${code} due to ${reason}`);
      } else {
        resolve({stdout,  stderr});
      }
      child.kill();
      child = null;
    });
    child.on('error', err => {
      reject(err.message);
      child.kill();
      child = null;
    });
  });
};

import { CLIEngine } from 'eslint';
export function lintOnFiles(context) {
  const { root } = context;
  const [ err ] = createPluginSymLink(root);
  if (err) {
    return [ err ];
  }
  const linter = new CLIEngine({
    envs: [ 'browser' ],
    useEslintrc: true,
    cwd: root,
    configFile: path.join(__dirname, 'LintConfig.js'),
    ignorePattern: ['**/router-config.js']
  });
  let report = linter.executeOnFiles(['src']);
  const errorReport = CLIEngine.getErrorResults(report.results);
  const errorList = errorReport.map(item => {
    const file = path.relative(root, item.filePath);
    return {
      file,
      errorCount: item.errorCount,
      warningCount: item.warningCount,
      messages: item.messages
    };
  });
  const result = {
    errorList,
    errorCount: report.errorCount,
    warningCount: report.warningCount
  }
  return [ null, result ];
};

import Docker = require('dockerode');
// 保证服务端中有一个基本的 dockerfile 镜像文件
const docker = new Docker({ socketPath: '/var/run/docker.sock' });
const image = '镜像打包名称'
let buildStream;
[ err, buildStream ] = await to(
  docker.buildImage({
    context: outputDir
  }, { t: image })
);
let pushStream;
// authconfig 镜像仓库的一些验证信息
const authconfig = {
  serveraddress: "镜像仓库地址"
};
// 向远端私有仓库推送镜像
const dockerImage = docker.getImage(image);
[ err, pushStream ] = await to(dockerImage.push({
  authconfig,
  tag
}));
// 3s 打印一次进度信息
const progressLog = _.throttle((msg) => logger.info(msg), 3000); 
const pushPromise = new Promise((resolve, reject) => {
  docker.modem.followProgress(pushStream, (err, res) => {
    err ? reject(err) : resolve(res);
  }, e => {
    if (e.error) {
      reject(e.error);
    } else {
      const { id, status, progressDetail } = e;
      if (progressDetail && !_.isEmpty(progressDetail)) {
        const { current, total } = progressDetail;
        const percent = Math.floor(current / total * 100);
        progressLog(`${id} : pushing progress ${percent}%`);
        if (percent === 100) { // 进度完成
          progressLog.flush();
        }
      } else if (id && status) {
        logger.info(`${id} : ${status}`);
      }
    }
  });
});
await to(pushPromise);

import * as child_process from 'child_process';
import { cpus } from 'os';
import { EventEmitter } from 'events';
import TaskQueue from './TaskQueue';
import TaskMap from './TaskMap';
import { to } from '../util/tool';
export default class ProcessPool extends EventEmitter {
  private jobQueue: TaskQueue;
  private depth: number;
  private processorFile: string;
  private workerPath: string;
  private runningJobMap: TaskMap;
  private idlePool: Array<number>;
  private workPool: Map<any, any>;
  constructor(options: any = {}) {
    super();
    this.jobQueue = new TaskQueue('fap_pack_task_queue');
    this.runningJobMap = new TaskMap('fap_running_pack_task');
    this.depth = options.depth || cpus().length; // 最大的实例进程数量
    this.workerPath = options.workerPath;
    this.idlePool = []; // 工作进程  pid 数组
    this.workPool = new Map();  // 工作实例进程池
    this.init();
  }
  /**
   * @func init 初始化进程，
   */
  init() {
    while (this.workPool.size < this.depth) {
      this.forkProcess();
    }
  }
  /**
   * @func forkProcess fork 子进程，创建任务实例
   */
  forkProcess() {
    let worker: any = child_process.fork(this.workerPath);
    const pid = worker.pid;
    this.workPool.set(pid, worker);
    worker.on('message', async (data) => {
      const { cmd } = data;
      // 根据 cmd 状态 返回日志状态或者结束后清理掉任务队列
      if (cmd === 'log') {
      }
      if (cmd === 'finish' || cmd === 'fail') {
        this.killProcess();//结束后清除任务
      }
    });
    worker.on('exit', () => {
      // 结束后，清理实例队列，开启下一个任务
      this.workPool.delete(pid);
      worker = null;
      this.forkProcess();
      this.startNextJob();
    });
    return worker;
  }
  // 根据任务队列，获取下一个要进行的实例，开始任务
  async startNextJob() {
    this.run();
  }
  /**
   * @func add 添加构建任务
   * @param task 运行的构建程序
   */
  async add(task) {
    const inJobQueue = await this.jobQueue.isInQueue(task.appId); // 任务队列
    const isRunningTask = await this.runningJobMap.has(task.appId); // 正在运行的任务
    const existed = inJobQueue || isRunningTask;
    if (!existed) {
      const len = await this.jobQueue.enqueue(task, task.appId);
      // 执行任务
      const [err] = await to(this.run());
      if (err) {
        return Promise.reject(err);
      }
    } else {
      return Promise.reject(new Error('DuplicateTask'));
    }
  }
  /**
   * @func initChild 开始构建任务
   * @param child 子进程引用
   * @param processFile 运行的构建程序文件
   */
  initChild(child, processFile) {
    return new Promise(resolve => {
      child.send({ cmd: 'init', value: processFile }, resolve);
    });
  }
  /**
   * @func startChild 开始构建任务
   * @param child 子进程引用
   * @param task 构建任务
   */
  startChild(child, task) {
    child.send({ cmd: 'start', task });
  }
  /**
   * @func run 开始队列任务运行
   */
  async run() {
    const jobQueue = this.jobQueue;
    const isEmpty = await jobQueue.isEmpty();
    // 有空闲资源并且任务队列不为空
    if (this.idlePool.length > 0 && !isEmpty) {
      // 获取空闲构建子进程实例
      const taskProcess = this.getFreeProcess();
      await this.initChild(taskProcess, this.processorFile);
      const task = await jobQueue.dequeue();
      if (task) {
        await this.runningJobMap.set(task.appId, task);
        this.startChild(taskProcess, task);
        return task;
      }
    } else {
      return Promise.reject(new Error('NoIdleResource'));
    }
  }
  /**
   * @func getFreeProcess 获取空闲构建子进程
   */
  getFreeProcess() {
    if (this.idlePool.length) {
      const pid = this.idlePool.shift();
      return this.workPool.get(pid);
    }
    return null;
  }
  
  /**
   * @func killProcess 杀死某个子进程，原因：释放构建运行时占用的内存
   * @param pid 进程 pid
   */
  killProcess(pid) {
    let child = this.workPool.get(pid);
    child.disconnect();
    child && child.kill();
    this.workPool.delete(pid);
    child = null;
  }
}

import ProcessPool from './ProcessPool';
import TaskMap from './TaskMap';
import * as path from 'path';
// 日志存储
const runningPackTaskLog = new TaskMap('fap_running_pack_task_log');
//初始化进程池
const packQueue = new ProcessPool({
  workerPath: path.join(__dirname, '../../task/func/worker'),
  depth: 3
});
// 初始化构建文件
packQueue.process(path.join(__dirname, '../../task/func/server-build'));
let key: string;
packQueue.on('message', async data => {
  // 根据项目 id，部署记录 id，以及用户 id 来设定 redis 缓存的 key 值，之后进行日志存储
  key = `${appId}_${deployId}_${deployer.userId}`;
  const { cmd, value } = data;
  if(cmd === 'log') { // 构建任务日志
    runningPackTaskLog.set(key,value);
  } else if (cmd === 'finish') { // 构建完成
    runningPackTaskLog.delete(key);
    // 后续日志可以进行数据库存储
  } else if (cmd === 'fail') { // 构建失败
    runningPackTaskLog.delete(key);
    // 后续日志可以进行数据库存储
  }
  // 可以通过 websocket 将进度同步给前台展示
});
//添加新的构建任务
let [ err ] = await to(packQueue.add({
  ...appAttrs, // 构建所需信息
}));