实战案例_Rust文件IO高性能完整解析

文件IO高性能演示/
├── Cargo.toml
└── src/
    └── main.rs  # 包含所有演示代码

cd 文件IO高性能演示
cargo run --release  # 使用 release 模式获得最佳性能

fn unbuffered_write_demo(file_path: &str, data: &[u8], iterations: usize) -> std::io::Result<()> {
    let start = Instant::now();
    
    for _ in 0..iterations {
        let mut file = File::create(file_path)?;
        file.write_all(data)?;
        file.sync_all()?;  // 强制刷新到磁盘
    }
    
    let duration = start.elapsed();
    println!("无缓冲写入 {} 次: {:?}", iterations, duration);
    Ok(())
}

fn buffered_write_demo(file_path: &str, data: &[u8], iterations: usize) -> std::io::Result<()> {
    let start = Instant::now();
    
    for _ in 0..iterations {
        let file = File::create(file_path)?;
        let mut writer = BufWriter::new(file);
        writer.write_all(data)?;
        writer.flush()?;
    }
    
    let duration = start.elapsed();
    println!("缓冲写入 {} 次: {:?}", iterations, duration);
    Ok(())
}

fn batch_write_demo(file_path: &str, data: &[u8], iterations: usize) -> std::io::Result<()> {
    let start = Instant::now();
    
    let file = File::create(file_path)?;
    let mut writer = BufWriter::new(file);
    
    for _ in 0..iterations {
        writer.write_all(data)?;
    }
    writer.flush()?;  // 只刷新一次
    
    let duration = start.elapsed();
    println!("批量写入 {} 次: {:?}", iterations, duration);
    Ok(())
}

fn byte_by_byte_read(file_path: &str) -> std::io::Result<usize> {
    let mut file = File::open(file_path)?;
    let mut buffer = [0u8; 1];  // 1 字节缓冲区
    let mut count = 0;
    
    loop {
        match file.read_exact(&mut buffer) {
            Ok(_) => count += 1,
            Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => break,
            Err(e) => return Err(e),
        }
    }
    
    Ok(count)
}

fn buffered_read_demo(file_path: &str) -> std::io::Result<usize> {
    let file = File::open(file_path)?;
    let mut reader = BufReader::new(file);
    let mut buffer = vec![0u8; 8192];  // 8KB 缓冲区
    let mut total = 0;
    
    loop {
        let bytes_read = reader.read(&mut buffer)?;
        if bytes_read == 0 {
            break;
        }
        total += bytes_read;
    }
    
    Ok(total)
}

fn read_all_demo(file_path: &str) -> std::io::Result<usize> {
    let data = std::fs::read(file_path)?;  // 一次性读取整个文件
    Ok(data.len())
}

fn zero_copy_read_demo(file_path: &str) -> std::io::Result<String> {
    let mut file = File::open(file_path)?;
    let mut content = String::new();
    file.read_to_string(&mut content)?;  // 零拷贝优化
    Ok(content)
}

fn line_by_line_read(file_path: &str) -> std::io::Result<usize> {
    let file = File::open(file_path)?;
    let reader = BufReader::new(file);
    let mut line_count = 0;
    
    for line in reader.lines() {
        let _ = line?;
        line_count += 1;
    }
    
    Ok(line_count)
}

fn sequential_process(files: &[&str]) -> std::io::Result<()> {
    let start = Instant::now();
    
    for file_path in files {
        let _ = std::fs::read_to_string(file_path)?;
    }
    
    let duration = start.elapsed();
    println!("单线程顺序处理 {} 个文件: {:?}", files.len(), duration);
    Ok(())
}

fn parallel_process(files: &[&str]) -> std::io::Result<()> {
    let start = Instant::now();
    let files = Arc::new(files.to_vec());
    let mut handles = vec![];
    
    for i in 0..files.len() {
        let files_clone = Arc::clone(&files);
        let handle = thread::spawn(move || {
            let file_path = files_clone[i];
            std::fs::read_to_string(file_path).ok()
        });
        handles.push(handle);
    }
    
    for handle in handles {
        let _ = handle.join().unwrap();
    }
    
    let duration = start.elapsed();
    println!("多线程并发处理 {} 个文件: {:?}", files.len(), duration);
    Ok(())
}

fn stream_process_large_file(file_path: &str, output_path: &str) -> std::io::Result<()> {
    let input_file = File::open(file_path)?;
    let output_file = File::create(output_path)?;
    
    let mut reader = BufReader::new(input_file);
    let mut writer = BufWriter::new(output_file);
    let mut buffer = vec![0u8; 64 * 1024];  // 64KB 缓冲区
    
    loop {
        let bytes_read = reader.read(&mut buffer)?;
        if bytes_read == 0 {
            break;
        }
        
        // 处理数据（转换、过滤等）
        writer.write_all(&buffer[..bytes_read])?;
    }
    
    writer.flush()?;
    Ok(())
}

fn buffer_size_comparison(file_path: &str) -> std::io::Result<()> {
    let buffer_sizes = [512, 1024, 4096, 8192, 16384, 65536];
    
    for &size in &buffer_sizes {
        let file = File::open(file_path)?;
        let mut reader = BufReader::with_capacity(size, file);
        let mut buffer = vec![0u8; size];
        // ... 读取测试
    }
    
    Ok(())
}

缓冲区 512 字节:   150ms (读取 1MB 文件)
缓冲区 1KB:        80ms
缓冲区 4KB:        25ms
缓冲区 8KB:        15ms  ← 推荐
缓冲区 16KB:       12ms
缓冲区 64KB:       10ms  ← 高性能

fn log_processing_demo(log_file: &str) -> std::io::Result<()> {
    let file = File::open(log_file)?;
    let reader = BufReader::new(file);
    
    let mut error_count = 0;
    let mut warning_count = 0;
    let mut info_count = 0;
    
    for line in reader.lines() {
        let line = line?;
        if line.contains("ERROR") {
            error_count += 1;
        } else if line.contains("WARNING") {
            warning_count += 1;
        } else if line.contains("INFO") {
            info_count += 1;
        }
    }
    
    println!("ERROR: {}, WARNING: {}, INFO: {}", 
             error_count, warning_count, info_count);
    Ok(())
}

fn file_copy_performance(source: &str, dest: &str) -> std::io::Result<()> {
    // 方法1：使用 std::fs::copy（系统调用优化）
    let start = Instant::now();
    std::fs::copy(source, dest)?;
    let duration1 = start.elapsed();
    
    // 方法2：使用 BufReader + BufWriter
    let start = Instant::now();
    let mut source_file = File::open(source)?;
    let mut dest_file = File::create(dest)?;
    let mut reader = BufReader::new(&mut source_file);
    let mut writer = BufWriter::new(&mut dest_file);
    io::copy(&mut reader, &mut writer)?;
    writer.flush()?;
    let duration2 = start.elapsed();
    
    Ok(())
}

// 高级 API
let data = std::fs::read("file.txt")?;

// 编译后等价于（简化）：
// let mut file = File::open("file.txt")?;
// let mut buffer = Vec::new();
// file.read_to_end(&mut buffer)?;
// buffer

用户请求读取 → BufReader 检查缓冲区
  ├─ 缓冲区有数据 → 直接返回（零系统调用）
  └─ 缓冲区空 → 调用 read() 填充缓冲区（一次系统调用读取 8KB）

磁盘 → 内核缓冲区 → 用户缓冲区 → String
      (拷贝1)      (拷贝2)      (拷贝3)

磁盘 → 内核缓冲区 → String（直接）
      (拷贝1)      (优化，减少拷贝)

单线程时间 / CPU核心数 ≈ 多线程时间
（理想情况，实际受 I/O 限制）

// ✅ 推荐
let reader = BufReader::new(file);
let writer = BufWriter::new(file);

// ✅ 推荐：内存高效
let mut buffer = vec![0u8; 64 * 1024];
loop {
    let bytes = reader.read(&mut buffer)?;
    // 处理数据
}

// ✅ 推荐：充分利用多核
for file in files {
    thread::spawn(move || process_file(file));
}

// ✅ 推荐：8KB 默认，64KB 高性能
BufReader::with_capacity(64 * 1024, file)

cd 文件IO高性能演示
cargo run --release  # 使用 release 模式