一个简单的多的socket http 下载原型 perl

全栈程序员站长

发布于 2021-05-06 11:40:59

2550

发布于 2021-05-06 11:40:59

文章被收录于专栏：全栈程序员必看全栈程序员必看

基于perl,使用IO::Select实现，并非多线程。可指定分几部分下载。基本上没有作异常处理，没有处理redirect,甚至也没有判断对range头的响应是否为206.

还好的是它还可以工作，比wget快几倍地下载,挺好玩的.

perl module:

package HttpClient; use strict; use warnings; use IO::Socket::INET; use Data::Dumper; my $crlf = “/r/n”; my $buf_size = 8 * 1024; sub new { my class = shift; my %cnf = (@_); my self = { state => ‘init’, url => cnf{url}, ‘total_parts’ => cnf{part}, ‘content_length’ => url = self->{url}; my host = 1 if url =~ m{://([^/]*)}; my file = 1 if url =~ m{/([^/]*)}; if ( defined self->{part} ) { file .= “.part” . self->{part}; } self->{host} = self->{file} = port = 80; port = 1 if host =~ /:(/d+)/; my sock = IO::Socket::INET->new( PeerAddr => port, Proto => ‘tcp’, Blocking => 0, ) or die “can’t connect to server:sock); | = 1; select(STDOUT); self->{sock} = self, class; return sub sock { return shift->{sock}; } sub get_request_header { my self = shift; return self->{request} if defined self->{request}; my self->{url} HTTP/1.1self->{host}crlf”; if ( defined self->{’total_parts’} and defined self->{part} and defined length = self->{’content_length’}; my total_parts = self->{’total_parts’}; my part = self->{part}; my part_size = int( length / total_parts ); my start_pos = part_size * part; my recved = 0; if (-e self->{file}) { recved = -s self->{file}; start_pos+=recved; } my recv_size = ( part == total_parts – 1 ) ? length-part*part_size : part_size; self->{start_pos} = start_pos; self->{recv_size} = recv_size-recved; print “part self->{part} recv_size=start_pos,recved=recved,parts=total_parts,length=length/n”; request .= “Range: bytes=start_pos-” . ( start_pos + recv_size-1 ) . crlf; } request .= crlf; self->{request} = request; return request; } sub parse_header { my (self) = @_; my data = self->{data}; return 1 if data; return 0 unless data =~ m{^(.*?)(/r/n/r/n|/n/n)}s; my header_content = 1; my header_end = 2; print header_content, “/n”; my @headers = split //r?/n/, header_content; die “invalid header/n” unless scalar(@headers) > 0; my status_line = shift @headers; self->{status_line} = status_line; self->{code} = 2 if status_line =~ m{HTTP/1(.1)? (/d+)}; my last_header; my header = {}; foreach my line (@headers) { if ( line =~ /^/s+(.*)/ ) { header->{last_header} .= ” 1″; } elsif ( line =~ /^([^:]+): (.*)/ ) { last_header = 1; my value = 2; header->{value; } else { print “invalid header:line/n”; } } self->{header} = self->{’content_length’} = self->{’content_length’}; self->{’content_length’}; data,length(header_content)+length(header_end)); self->{state} = ‘body’; return 1; } sub recv_data { my ( self, data ) = @_; if ( defined self->{data} ) { data; } else { self->{data} = sub save_data { my ( self, read_select ) = @_; my fh = self->{fh}; if ( !defined fh ) { open fh, “>self->{file}” or die “can’t open file !/n”; binmode fh,”:bytes”; self->{fh} = write_len = self->{write_len} || 0; my recv_size = self->{recv_size}; my data = self->{data}; my max_len = length(data); return unless max_len > 0; if (write_len > max_len = recv_size – write_len ; my part = self->{part} || 0; print “part=max_len,write_len=write_len/n”; } if ( max_len == 0 ) { self->{done} = 1; close self->{fh}; self->sock ); close self->{file} recved self->{parent}->child_done(self->{parent}; return; } my fh, data, max_len ) or die “write data failed :!/n”; self->{data} = substr( len ); write_len += len; self->{write_len} = sub child_done { my ( self, child ) = @_; child->{done} = 1; return unless c ( @{ self->{children} } ) { return unless print “merge file/n”; open FH, “>>c ( @{ self->{children} } ) { print “c->{file} size:”,-s print “merge c->{file}/n”; my len = sysread( CFH, buf, buf_size ); last if !defined len || len == 0; syswrite( FH, buf, len ); } close CFH; unlink c->{file}; } close FH; } sub handle_read { my ( self, sock, read_select, write_select, my data; my len = sysread( sock, data, buf_size ); if ( len == 0 ) { print “sock sock finished/n”; read_select->remove(sock); close sock; print “self->{file} size=”,-s self->{parent}->child_done(self->{parent}; return; } data); if ( self->{state} !~ /body/ and self->parse_header ) { if ( !defined self->{content_length} ) { my self->{total_parts} || 5; length = self->{’content_length’}; my part_size = int( length / parts ); self->{recv_size} = part_size; print “parent recv_size=self->{recv_size}/n”; foreach my part ( 1 .. parts – 1 ) { my child = HttpClient->new( url => parts, part => self->{content_length}, ); child->sock } = child->{parent} = self->{children} }, read_select->add( write_select->add( self->save_data( sub handle_write { my ( self, sock, read_select, write_select, my offset = 0; offset = self->{request_offset} if defined request = self->get_request_header; if ( offset == 0 ) { print “try to send request/n”; print request; } print “offset=offset/n”; my len = syswrite( sock, request, length(request) – offset, offset ); if ( !defined len ) { print STDERR “write failed:!/n”; read_select->remove(sock); write_select->remove(sock); } else { offset += len; self->{request_offset} = offset; if ( offset == length(request) ) { write_select->remove( sub start { my (self) = @_; use IO::Select; my r = IO::Select->new; r->add( my w = IO::Select->new; my sock_client = { self->sock => use Time::HiRes qw(time); my start_time = time; for ( ; ; ) { last if ( r->count == 0 ); my ( rout, wout, eout ) = IO::Select->select( w, r ); last unless defined foreach my sock ( @{wout} ) { my c = sock_client->{sock}; if ( !defined c ) { die “oops,can’t find httpclient for sock/n”; } c->handle_write( sock, r, w, sock_client ); } foreach my sock ( @{rout} ) { my c = sock_client->{sock}; if ( !defined c ) { die “oops,can’t find httpclient for sock/n”; } c->handle_read( sock, r, w, sock_client ); } } my end_time = time; my used_time = end_time – start_time; my speed = self->{content_length} / used_time; print “Done,spend used_time seconds,speed: 1;

test perl script:

#!/usr/bin/perl use strict; use warnings; use lib ‘.’; use HttpClient; use Getopt::Long; | = 1; #my url = ‘http://eclipse.cdpa.nsysu.edu.tw/downloads/drops/R-3.2.1-200609210945/eclipse-SDK-3.2.1-linux-gtk.tar.gz’; my url = ”; my total_parts = 1; my result = GetOptions (”url|u=s” => /total_parts, ); unless (result and my client = HttpClient->new(url=>total_parts); client->start();

参考: