从pyspark应用程序记录到本地或hdfs文件

import logging

# 配置日志级别和输出格式
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')

# 记录日志
logging.info('This is an info message')
logging.error('This is an error message')

import logging

# 配置日志级别和输出格式
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')

# 创建FileHandler对象，指定日志文件路径和模式
file_handler = logging.FileHandler('app.log')

# 配置FileHandler的输出格式
file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))

# 将FileHandler添加到日志记录器中
logging.getLogger().addHandler(file_handler)

# 记录日志
logging.info('This is an info message')
logging.error('This is an error message')

from pyhdfs import HdfsClient
import logging

# 配置日志级别和输出格式
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')

# 创建HdfsClient对象，指定HDFS的地址和端口
client = HdfsClient(hosts='localhost', port=9000)

# 记录日志
logging.info('This is an info message')
logging.error('This is an error message')

# 将日志写入HDFS文件
client.create('/user/spark/app.log', 'app.log')