Flink(arm) on K8S 部署时的那些坑

apiVersion: v1
kind: ConfigMap
metadata:
  name: flink-config
  namespace: "flink-ns"
  labels:
    app: flink
data:
  flink-conf.yaml: |+
    jobmanager.rpc.address: flink-jobmanager
    taskmanager.numberOfTaskSlots: 2
    blob.server.port: 6124
    jobmanager.rpc.port: 6123
    taskmanager.rpc.port: 6122
    jobmanager.memory.process.size: 1600m
    taskmanager.memory.process.size: 1728m
    parallelism.default: 2
  log4j-console.properties: |+
    rootLogger.level = INFO
    rootLogger.appenderRef.console.ref = ConsoleAppender
    rootLogger.appenderRef.rolling.ref = RollingFileAppender
    logger.flink.name = org.apache.flink
    logger.flink.level = INFO
    logger.pekko.name = org.apache.pekko
    logger.pekko.level = INFO
    logger.kafka.name= org.apache.kafka
    logger.kafka.level = INFO
    logger.hadoop.name = org.apache.hadoop
    logger.hadoop.level = INFO
    logger.zookeeper.name = org.apache.zookeeper
    logger.zookeeper.level = INFO

    # 将所有 info 级别的日志输出到 console
    appender.console.name = ConsoleAppender
    appender.console.type = CONSOLE
    appender.console.layout.type = PatternLayout
    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n

    # 将所有 info 级别的日志输出到指定的 rolling file
    appender.rolling.name = RollingFileAppender
    appender.rolling.type = RollingFile
    appender.rolling.append = false
    appender.rolling.fileName = ${sys:log.file}
    appender.rolling.filePattern = ${sys:log.file}.%i
    appender.rolling.layout.type = PatternLayout
    appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
    appender.rolling.policies.type = Policies
    appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
    appender.rolling.policies.size.size=100MB
    appender.rolling.strategy.type = DefaultRolloverStrategy
    appender.rolling.strategy.max = 10

    # 关闭 Netty channel handler 中不相关的（错误）警告
    logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline
    logger.netty.level = OFF

apiVersion: v1
kind: Service
metadata:
  name: flink-jobmanager
  labels:
    app: flink
    component: jobmanager
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: flink
    component: jobmanager
  ports:
  - name: rpc
    port: 6123

apiVersion: v1
kind: Service
metadata:
  name: flink-jobmanager-rest
  namespace: "flink-ns"
spec:
  type: NodePort
  ports:
    - name: rest
      port: 8081
      targetPort: 8081
      nodePort: 30081
  selector:
    app: flink
    component: jobmanager

apiVersion: v1
kind: ServiceAccount
metadata:
  name: flink-service-account
  namespace: "flink-ns"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: configmap-manager
rules:
  - apiGroups: [ "" ]
    resources: [ "configmaps" ]
    verbs: [ "create", "delete", "get", "list", "patch", "update", "watch" ]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: configmap-manager-binding
  namespace: "flink-ns"
subjects:
  - kind: ServiceAccount
    name: flink-service-account
roleRef:
  kind: Role
  name: configmap-manager
  apiGroup: rbac.authorization.k8s.io

jobmanager-session-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: flink-jobmanager
  namespace: "
spec:
  replicas: 1 # 通过设置大于 1 的整型值来开启 Standby JobManager
  selector:
    matchLabels:
      app: flink
      component: jobmanager
  template:
    metadata:
      labels:
        app: flink
        component: jobmanager
    spec:
      containers:
        - name: jobmanager
          image: flink-arm:lastest
#          env:
#            - name: POD_IP
#              valueFrom:
#                fieldRef:
#                  apiVersion: v1
#                  fieldPath: status.podIP
#          # 下面的 args 参数会使用 POD_IP 对应的值覆盖 config map 中 jobmanager.rpc.address 的属性值。
#          args: [ "jobmanager", "$(POD_IP)" ]
          args: [ "jobmanager"]
          ports:
            - containerPort: 6123
              name: rpc
            - containerPort: 6124
              name: blob-server
            - containerPort: 8081
              name: webui
          livenessProbe:
            tcpSocket:
              port: 6123
            initialDelaySeconds: 30
            periodSeconds: 60
          volumeMounts:
            - name: flink-config-volume
              mountPath: /opt/flink/conf
          securityContext:
            runAsUser: 9999  # 参考官方 flink 镜像中的 _flink_ 用户，如有必要可以修改
      serviceAccountName: flink-service-account # 拥有创建、编辑、删除 ConfigMap 权限的 Service 账号
      volumes:
        - name: flink-config-volume
          configMap:
            name: flink-config
            items:
              - key: flink-conf.yaml
                path: flink-conf.yaml
              - key: log4j-console.properties
                path: log4j-console.properties

apiVersion: apps/v1
kind: Deployment
metadata:
  name: flink-taskmanager
  namespace: "flink-ns"
spec:
  replicas: 3
  selector:
    matchLabels:
      app: flink
      component: taskmanager
  template:
    metadata:
      labels:
        app: flink
        component: taskmanager
    spec:
      initContainers:
        - name: init-flink-conf
          image: flink-arm:latest
          command: [ "sh", "-c", "mkdir -p /opt/flink/conf /config ; cp /config/* /opt/flink/conf ; chmod 777 /opt/flink/conf/*" ]
          volumeMounts:
            - name: flink-config-volume
              mountPath: /config
            - name: local-storage
              mountPath: /opt/flink/conf
      containers:
        - name: taskmanager
          image: flink-arm:latest
          args: [ "taskmanager" ]
          ports:
            - containerPort: 6122
              name: rpc
          livenessProbe:
            tcpSocket:
              port: 6122
            initialDelaySeconds: 30
            periodSeconds: 60
          volumeMounts:
            - name: local-storage
              mountPath: /opt/flink/conf
          env:
            - name: JOB_MANAGER_RPC_ADDRESS
              value: flink-jobmanager
          securityContext:
            runAsUser: 9999  # 参考官方 flink 镜像中的 _flink_ 用户，如有必要可以修改
      volumes:
        - name: flink-config-volume
          configMap:
            name: flink-config
            items:
              - key: flink-conf.yaml
                path: flink-conf.yaml
              - key: log4j-console.properties
                path: log4j-console.properties
        - name: local-storage
          emptyDir: { }

WARNING: attempted to load jemalloc from /usr/lib/aarch64-linux-gnu/libjemalloc.so and /usr/lib/x86_64-linux-gnu/libjemalloc.so but the library couldn't be found. glibc will be used instead.
sed: couldn't open temporary file /opt/flink/conf/sedGGEvws: Read-only file system
sed: couldn't open temporary file /opt/flink/conf/sedL9ELTs: Read-only file system
/docker-entrypoint.sh: line 73: /opt/flink/conf/flink-conf.yaml: Permission denied
/docker-entrypoint.sh: line 89: /opt/flink/conf/flink-conf.yaml.tmp: Read-only file system
Starting Task Manager
Starting taskexecutor as a console application on host flink-taskmanager-b6c9fbb8c-hxc4h.
Error: VM option 'UseG1GC' is experimental and must be enabled via -XX:+UnlockExperimentalVMOptions.
Error: Could not create the Java Virtual Machine.
Error: A fatal exception has occurred. Program will exit.

# 修改 taskmanager 启动参数
RUN sed -i 's/-XX:+UseG1GC/-XX:+UnlockExperimentalVMOptions -XX:+UseG1GC/g' bin/taskmanager.sh