CS 144 Lab Seven -- putting it all together

./apps/lab7 server cs144.keithw.org 3000

./apps/lab7 client cs144.keithw.org 3001

EthernetAddress random_host_ethernet_address() {
    EthernetAddress addr;
    for (auto &byte : addr) {
        byte = rd();  // use a random local Ethernet address
    }
    addr.at(0) |= 0x02;  // "10" in last two binary digits marks a private Ethernet address
    addr.at(0) &= 0xfe;

    return addr;
}

EthernetAddress random_router_ethernet_address() {
    EthernetAddress addr;
    for (auto &byte : addr) {
        byte = rd();  // use a random local Ethernet address
    }
    addr.at(0) = 0x02;  // "10" in last two binary digits marks a private Ethernet address
    addr.at(1) = 0;
    addr.at(2) = 0;

    return addr;
}

//! A wrapper around [UDP sockets](\ref man7::udp)
// 对本机Linux网络子系统提供的UDP socket进行的包装
class UDPSocket : public Socket {
  protected:
    //  Construct from FileDescriptor (used by TCPOverUDPSocketAdapter)
    //  fd is the FileDescriptor from which to construct
    explicit UDPSocket(FileDescriptor &&fd) : Socket(std::move(fd), AF_INET, SOCK_DGRAM) {}

  public:
    //! Default: construct an unbound, unconnected UDP socket
    // 无参构造,默认创建出来的是UDP socket
    UDPSocket() : Socket(AF_INET, SOCK_DGRAM) {}

    //! Returned by UDPSocket::recv; carries received data and information about the sender
    // recv接收UDP数据报方法返回结果
    struct received_datagram {
        Address source_address;  // Address from which this datagram was received
        std::string payload;     // UDP datagram payload
    };

    //! Receive a datagram and the Address of its sender
    received_datagram recv(const size_t mtu = 65536);

    //! Receive a datagram and the Address of its sender (caller can allocate storage)
    void recv(received_datagram &datagram, const size_t mtu = 65536);

    //! Send a datagram to specified Address
    void sendto(const Address &destination, const BufferViewList &payload);

    //! Send datagram to the socket's connected address (must call connect() first)
    void send(const BufferViewList &payload);
};

// default constructor for socket of (subclassed) domain and type
//! \param[in] domain is as described in [socket(7)](\ref man7::socket), probably `AF_INET` or `AF_UNIX`
//! \param[in] type is as described in [socket(7)](\ref man7::socket)
Socket::Socket(const int domain, const int type) : FileDescriptor(SystemCall("socket", socket(domain, type, 0))) {}

Socket::Socket(FileDescriptor &&fd, const int domain, const int type) : FileDescriptor(move(fd)) {
    int actual_value;
    socklen_t len;

    // verify domain
    len = sizeof(actual_value);
    SystemCall("getsockopt", getsockopt(fd_num(), SOL_SOCKET, SO_DOMAIN, &actual_value, &len));
    if ((len != sizeof(actual_value)) or (actual_value != domain)) {
        throw runtime_error("socket domain mismatch");
    }

    // verify type
    len = sizeof(actual_value);
    SystemCall("getsockopt", getsockopt(fd_num(), SOL_SOCKET, SO_TYPE, &actual_value, &len));
    if ((len != sizeof(actual_value)) or (actual_value != type)) {
        throw runtime_error("socket type mismatch");
    }
}

//! \note If `mtu` is too small to hold the received datagram, this method throws a std::runtime_error
// 将接收到的UDP数据报存储到datagram中
void UDPSocket::recv(received_datagram &datagram, const size_t mtu) {
    // receive source address and payload
    // 用于接收数据报来源地址
    Address::Raw datagram_source_address;
    datagram.payload.resize(mtu);

    socklen_t fromlen = sizeof(datagram_source_address);
    
    // 通过系统调用,调用本机Linux网络子系统中socket提供的recvfrom接口
    const ssize_t recv_len = SystemCall(
        "recvfrom",
        ::recvfrom(
            // 哪个socket,接收的数据存储到哪里,接收缓冲区的大小，接收标志，表示如果数据报过大会截断，并返回截断后的数据。如果不指定这个标志，过大的数据报会被丢弃
            // 用于存储源地址的缓冲区,源地址缓冲区的大小 
            fd_num(), datagram.payload.data(), datagram.payload.size(), MSG_TRUNC, datagram_source_address, &fromlen));
    
    // 如果接收到的数据大小超过了mtu,则抛出异常
    if (recv_len > ssize_t(mtu)) {
        throw runtime_error("recvfrom (oversized datagram)");
    }

    register_read();
    // 记录数据包来源地址
    datagram.source_address = {datagram_source_address, fromlen};
    // 调整payload缓冲区大小为实际接收到的数据量
    datagram.payload.resize(recv_len);
}

UDPSocket::received_datagram UDPSocket::recv(const size_t mtu) {
    received_datagram ret{{nullptr, 0}, ""};
    recv(ret, mtu);
    return ret;
}

// 发送UDP数据报: socket描述符,存放目的地址的缓冲区,缓冲区大小,要发送的数据载荷
void sendmsg_helper(const int fd_num,
                    const sockaddr *destination_address,
                    const socklen_t destination_address_len,
                    const BufferViewList &payload) {
    auto iovecs = payload.as_iovecs();
    // 构建数据包
    msghdr message{};
    message.msg_name = const_cast<sockaddr *>(destination_address);
    message.msg_namelen = destination_address_len;
    message.msg_iov = iovecs.data();
    message.msg_iovlen = iovecs.size();
    // 通过系统调用sendmsg完成数据包的发送
    const ssize_t bytes_sent = SystemCall("sendmsg", ::sendmsg(fd_num, &message, 0));
    // 检验成功发送的字节数和payload大小是否一致,也就是数据包是否成功发送 
    if (size_t(bytes_sent) != payload.size()) {
        throw runtime_error("datagram payload too big for sendmsg()");
    }
}

// 发送时指明目的地址
void UDPSocket::sendto(const Address &destination, const BufferViewList &payload) {
    sendmsg_helper(fd_num(), destination, destination.size(), payload);
    register_write();
}

// 发送时不指定目的地址
void UDPSocket::send(const BufferViewList &payload) {
    sendmsg_helper(fd_num(), nullptr, 0, payload);
    register_write();
}

// 为了适配从通道读写IP数据报
class NetworkInterfaceAdapter : public TCPOverIPv4Adapter {
  private:
    // 网络接口
    NetworkInterface _interface;
    // 下一条IP地址
    Address _next_hop;
    // socket_pair系统调用创建出来的本地套接字双向通信通道 --> lab five的测试文件中,这里是写死为Tap设备
    // 但是此处我们利用双向通道进行解耦,这样数据可以来源于Tap设备,也可以来源于其他地方 -- 解耦
    pair<FileDescriptor, FileDescriptor> _data_socket_pair = socket_pair_helper(SOCK_DGRAM);
    
    // 将网络接口输出队列中等待输出的以太网帧取出,然后写入双向通信通道中
    void send_pending() {
        while (not _interface.frames_out().empty()) {
            _data_socket_pair.first.write(_interface.frames_out().front().serialize());
            _interface.frames_out().pop();
        }
    }

  public:
    NetworkInterfaceAdapter(const Address &ip_address, const Address &next_hop)
        :
        // 当前网络接口MAC地址采用随机生成,ip地址采用传入的 
        _interface(random_host_ethernet_address(), ip_address), 
        // 下一条IP地址也是采用传入的
        _next_hop(next_hop) {}
    
    // 从通道读取数据
    optional<TCPSegment> read() {
        EthernetFrame frame;
        // 解析从通道读取得到的以太网帧
        if (frame.parse(_data_socket_pair.first.read()) != ParseResult::NoError) {
            return {};
        }

        // Give the frame to the NetworkInterface. Get back an Internet datagram if frame was carrying one.
        // 交给网络接口处理,得到IP数据报
        optional<InternetDatagram> ip_dgram = _interface.recv_frame(frame);

        // The incoming frame may have caused the NetworkInterface to send a frame
        // 将网络接口中待发送的以太网帧一股脑发送到通道中
        send_pending();

        // Try to interpret IPv4 datagram as TCP
        // 如果是IP数据报,那么剥离得到TCP segment，然后返回
        if (ip_dgram) {
            return unwrap_tcp_in_ip(ip_dgram.value());
        }

        return {};
    }
    
    // 向网络接口写入TCP数据段,网络接口将处理完毕的数据段写入_segment_out输出队列
    // 然后调用send_pending将队列中带输出的以太网帧写入通道
    void write(TCPSegment &seg) {
        // tcp段加上IP头
        _interface.send_datagram(wrap_tcp_in_ip(seg), _next_hop);
        send_pending();
    }

    // _tcp_loop会不间断调用当前适配器的tick
    void tick(const size_t ms_since_last_tick) {
        _interface.tick(ms_since_last_tick);
        send_pending();
    }
    
    NetworkInterface &interface() { return _interface; }
    queue<EthernetFrame> frames_out() { return _interface.frames_out(); }
    // 运算符重载,用于事件循环判获取first,从而判断当前NetworkInterfaceAdapter是否可读可写
    operator FileDescriptor &() { return _data_socket_pair.first; }
    FileDescriptor &frame_fd() { return _data_socket_pair.second; }
};

class TCPSocketLab7 : public TCPSpongeSocket<NetworkInterfaceAdapter> {
    Address _local_address;

  public:
    TCPSocketLab7(const Address &ip_address, const Address &next_hop)
        : TCPSpongeSocket<NetworkInterfaceAdapter>(NetworkInterfaceAdapter(ip_address, next_hop))
        , _local_address(ip_address) {}

    // client建立连接--参数: 连接的server的地址
    void connect(const Address &address) {
        FdAdapterConfig multiplexer_config;
        // 客户端的启动端口随机采用
        _local_address = Address{_local_address.ip(), uint16_t(random_device()())};
        cerr << "DEBUG: Connecting from " << _local_address.to_string() << "...\n";
        // multiplexer_config保存源地址和目的地址
        multiplexer_config.source = _local_address;
        multiplexer_config.destination = address;
        // 调用父类的Connect方法
        TCPSpongeSocket<NetworkInterfaceAdapter>::connect({}, multiplexer_config);
    }
    
    // server绑定端口
    void bind(const Address &address) {
        // 我们只能指定port,ip是固定的
        if (address.ip() != _local_address.ip()) {
            throw runtime_error("Cannot bind to " + address.to_string());
        }
        _local_address = Address{_local_address.ip(), address.port()};
    }
    
    // server监听端口
    void listen_and_accept() {
        FdAdapterConfig multiplexer_config;
        multiplexer_config.source = _local_address;
        // 调用父类listen_and_accept方法
        TCPSpongeSocket<NetworkInterfaceAdapter>::listen_and_accept({}, multiplexer_config);
    }

    NetworkInterfaceAdapter &adapter() { return _datagram_adapter; }
};

int main(int argc, char *argv[]) {
    try {
        if (argc <= 0) {
            abort();  // For sticklers: don't try to access argv[0] if argc <= 0.
        }

        if (argc != 4 and argc != 5) {
            print_usage(argv[0]);
            return EXIT_FAILURE;
        }

        if (argv[1] != "client"s and argv[1] != "server"s) {
            print_usage(argv[0]);
            return EXIT_FAILURE;
        }
        // 启动程序主体 
        program_body(argv[1] == "client"s, argv[2], argv[3], argc == 5);
    } catch (const exception &e) {
        cerr << e.what() << "\n";
        return EXIT_FAILURE;
    }

    return EXIT_SUCCESS;
}

// lab7测试程序的主体
// 参数: 当前启动的是客户端和服务端,
void program_body(bool is_client, const string &bounce_host, const string &bounce_port, const bool debug) {
    // 连接外网的udp socket(通过本地linux网络子系统构建得到的udp socket)
    UDPSocket internet_socket;
    // 外部帮忙中转数据包的server
    Address bounce_address{bounce_host, bounce_port};

    /* let bouncer know where we are */
    // 让bouncer知道我们是谁
    internet_socket.sendto(bounce_address, "");
    internet_socket.sendto(bounce_address, "");
    internet_socket.sendto(bounce_address, "");

    /* set up the router */
    // client和server各自都有一个默认路由器 -- lab six我们实现的路由器
    Router router;
    
    // host_side代表的网络接口为连接主机所在内网的一端
    // internet_side代表的网络接口为连接外网的一端
    unsigned int host_side, internet_side;
    // 启动的是客户端
    if (is_client) {
        // 向路由器中添加两个网络接口
        // 1.该网络接口处于client主机所在子网
        host_side = router.add_interface({random_router_ethernet_address(), {"192.168.0.1"}});
        // 2.该网络接口连接广域网
        internet_side = router.add_interface({random_router_ethernet_address(), {"10.0.0.192"}});
        // 向路由器添加路由条目: 路由前缀,前缀长度,下一条IP地址,网络接口索引
        // 如果路由前缀所在子网与当前传入的网络接口处在同一个网络,则目的IP地址为空
        router.add_route(Address{"192.168.0.0"}.ipv4_numeric(), 16, {}, host_side);
        router.add_route(Address{"10.0.0.0"}.ipv4_numeric(), 8, {}, internet_side);
        router.add_route(Address{"172.16.0.0"}.ipv4_numeric(), 12, Address{"10.0.0.172"}, internet_side);
    } else {
    // 启动的是服务端    
        host_side = router.add_interface({random_router_ethernet_address(), {"172.16.0.1"}});
        internet_side = router.add_interface({random_router_ethernet_address(), {"10.0.0.172"}});
        router.add_route(Address{"172.16.0.0"}.ipv4_numeric(), 12, {}, host_side);
        router.add_route(Address{"10.0.0.0"}.ipv4_numeric(), 8, {}, internet_side);
        router.add_route(Address{"192.168.0.0"}.ipv4_numeric(), 16, Address{"10.0.0.192"}, internet_side);
    }

    /* set up the client */
    TCPSocketLab7 sock =
        is_client ? 
        // 客户端主机IP地址和下一条的IP地址(默认路由)
        TCPSocketLab7{{"192.168.0.50"}, {"192.168.0.1"}} : 
        // 服务端主机IP地址和下一条的IP地址(默认路由)
        TCPSocketLab7{{"172.16.0.100"}, {"172.16.0.1"}};

    atomic<bool> exit_flag{};

    /* set up the network */
    // 启动一个子线程
    thread network_thread([&]() {
         // 子线程需要干的事情
         ...
    });
    
    try {
        // 如果当前启动的是客户端,则调用sock的connect
        if (is_client) {
            sock.connect({"172.16.0.100", 1234});
        } else {
        // 如果启动的是服务端,向绑定ip和端口,然后开启监听    
            sock.bind({"172.16.0.100", 1234});
            sock.listen_and_accept();
        }
        // 开启标准输入,标准输出与socket之间的双向复制
        bidirectional_stream_copy(sock);
        sock.wait_until_closed();
    } catch (const exception &e) {
        cerr << "Exception: " << e.what() << "\n";
    }

    cerr << "Exiting... ";
    exit_flag = true;
    network_thread.join();
    cerr << "done.\n";
}

// 启动一个子线程
    thread network_thread([&]() {
        try {
            // 初始化事件循环
            EventLoop event_loop;
            // Frames from host to router
            // _data_socket_pair通道可读事件
            event_loop.add_rule(sock.adapter().frame_fd(), Direction::In, [&] {
                EthernetFrame frame;
                // 从通道读取以太网帧
                if (frame.parse(sock.adapter().frame_fd().read()) != ParseResult::NoError) {
                    return;
                }
                if (debug) {
                    cerr << "     Host->router:     " << summary(frame) << "\n";
                }
                // 交给对应路由器进行路由
                // 1.先找到当前主机端对应的网络接口,让其接收以太网帧,处理后暂存队列
                router.interface(host_side).recv_frame(frame);
                // 2.进行路由
                router.route();
            });

            // Frames from router to host
            // _data_socket_pair通道可写事件
            event_loop.add_rule(
                sock.adapter().frame_fd(),
                Direction::Out,
                [&] {
                    // 从路由器取出当前主机端的网络接口,获取其待输出队列
                    // 该输出队列暂存待发送以太网帧
                    // 由于lab seven构造的每个主机所在的局域网只有他自己,所以这里的以太网帧就是发送给当前主机的
                    auto &f = router.interface(host_side).frames_out();
                    if (debug) {
                        cerr << "     Router->host:     " << summary(f.front()) << "\n";
                    }
                    // 将数据包写入通道,即发送以太网帧给当前主机自己
                    sock.adapter().frame_fd().write(f.front().serialize());
                    f.pop();
                },
                [&] { return not router.interface(host_side).frames_out().empty(); });

            // Frames from router to Internet
            // internet_socket可写事件
            event_loop.add_rule(
                internet_socket,
                Direction::Out,
                [&] {
                    // 从当前路由器取出连接广域网的网络接口,然后获取它的待输出数据包队列
                    auto &f = router.interface(internet_side).frames_out();
                    if (debug) {
                        cerr << "     Router->Internet: " << summary(f.front()) << "\n";
                    }
                    // 将数据包发送到外部server服务器
                    internet_socket.sendto(bounce_address, f.front().serialize());
                    f.pop();
                },
                [&] { return not router.interface(internet_side).frames_out().empty(); });

            // Frames from Internet to router
            // internet_socket可读事件
            event_loop.add_rule(internet_socket, Direction::In, [&] {
                EthernetFrame frame;
                // 从internet_socket读取出以太网数据包
                if (frame.parse(internet_socket.read()) != ParseResult::NoError) {
                    return;
                }
                if (debug) {
                    cerr << "     Internet->router: " << summary(frame) << "\n";
                }
                // 然后将数据包交给连接广域网的网络接口进行接收,处理完后暂存队列
                router.interface(internet_side).recv_frame(frame);
                // 将队列中待路由的数据包取出进行路由发送
                router.route();
            });
            
            // 开启事件循环,并且一直轮询
            while (true) {
                // 每次最多等待50毫秒
                if (EventLoop::Result::Exit == event_loop.wait_next_event(50)) {
                    cerr << "Exiting...\n";
                    return;
                }
                // 定时调用tick方法
                router.interface(host_side).tick(50);
                router.interface(internet_side).tick(50);
                if (exit_flag) {
                    return;
                }
            }
        } catch (const exception &e) {
            cerr << "Thread ending from exception: " << e.what() << "\n";
        }
    });