专栏首页全栈程序员必看一个简单的多的socket http 下载原型 perl

一个简单的多的socket http 下载原型 perl

基于perl,使用IO::Select实现,并非多线程。可指定分几部分下载。 基本上没有作异常处理,没有处理redirect,甚至也没有判断对range头的响应是否为206.

还好的是它还可以工作,比wget快几倍地下载,挺好玩的.

perl module:

package HttpClient; use strict; use warnings; use IO::Socket::INET; use Data::Dumper; my $crlf = “/r/n”; my $buf_size = 8 * 1024; sub new { my class = shift; my %cnf = (@_); my self = { state => ‘init’, url => cnf{url}, ‘total_parts’ => cnf{part}, ‘content_length’ => url = self->{url}; my host = 1 if url =~ m{://([^/]*)}; my file = 1 if url =~ m{/([^/]*)}; if ( defined self->{part} ) { file .= “.part” . self->{part}; } self->{host} = self->{file} = port = 80; port = 1 if host =~ /:(/d+)/; my sock = IO::Socket::INET->new( PeerAddr => port, Proto => ‘tcp’, Blocking => 0, ) or die “can’t connect to server:sock); | = 1; select(STDOUT); self->{sock} = self, class; return sub sock { return shift->{sock}; } sub get_request_header { my self = shift; return self->{request} if defined self->{request}; my self->{url} HTTP/1.1self->{host}crlf”; if ( defined self->{’total_parts’} and defined self->{part} and defined length = self->{’content_length’}; my total_parts = self->{’total_parts’}; my part = self->{part}; my part_size = int( length / total_parts ); my start_pos = part_size * part; my recved = 0; if (-e self->{file}) { recved = -s self->{file}; start_pos+=recved; } my recv_size = ( part == total_parts – 1 ) ? length-part*part_size : part_size; self->{start_pos} = start_pos; self->{recv_size} = recv_size-recved; print “part self->{part} recv_size=start_pos,recved=recved,parts=total_parts,length=length/n”; request .= “Range: bytes=start_pos-” . ( start_pos + recv_size-1 ) . crlf; } request .= crlf; self->{request} = request; return request; } sub parse_header { my (self) = @_; my data = self->{data}; return 1 if data; return 0 unless data =~ m{^(.*?)(/r/n/r/n|/n/n)}s; my header_content = 1; my header_end = 2; print header_content, “/n”; my @headers = split //r?/n/, header_content; die “invalid header/n” unless scalar(@headers) > 0; my status_line = shift @headers; self->{status_line} = status_line; self->{code} = 2 if status_line =~ m{HTTP/1(.1)? (/d+)}; my last_header; my header = {}; foreach my line (@headers) { if ( line =~ /^/s+(.*)/ ) { header->{last_header} .= ” 1″; } elsif ( line =~ /^([^:]+): (.*)/ ) { last_header = 1; my value = 2; header->{value; } else { print “invalid header:line/n”; } } self->{header} = self->{’content_length’} = self->{’content_length’}; self->{’content_length’}; data,length(header_content)+length(header_end)); self->{state} = ‘body’; return 1; } sub recv_data { my ( self, data ) = @_; if ( defined self->{data} ) { data; } else { self->{data} = sub save_data { my ( self, read_select ) = @_; my fh = self->{fh}; if ( !defined fh ) { open fh, “>self->{file}” or die “can’t open file !/n”; binmode fh,”:bytes”; self->{fh} = write_len = self->{write_len} || 0; my recv_size = self->{recv_size}; my data = self->{data}; my max_len = length(data); return unless max_len > 0; if (write_len > max_len = recv_size – write_len ; my part = self->{part} || 0; print “part=max_len,write_len=write_len/n”; } if ( max_len == 0 ) { self->{done} = 1; close self->{fh}; self->sock ); close self->{file} recved self->{parent}->child_done(self->{parent}; return; } my fh, data, max_len ) or die “write data failed :!/n”; self->{data} = substr( len ); write_len += len; self->{write_len} = sub child_done { my ( self, child ) = @_; child->{done} = 1; return unless c ( @{ self->{children} } ) { return unless print “merge file/n”; open FH, “>>c ( @{ self->{children} } ) { print “c->{file} size:”,-s print “merge c->{file}/n”; my len = sysread( CFH, buf, buf_size ); last if !defined len || len == 0; syswrite( FH, buf, len ); } close CFH; unlink c->{file}; } close FH; } sub handle_read { my ( self, sock, read_select, write_select, my data; my len = sysread( sock, data, buf_size ); if ( len == 0 ) { print “sock sock finished/n”; read_select->remove(sock); close sock; print “self->{file} size=”,-s self->{parent}->child_done(self->{parent}; return; } data); if ( self->{state} !~ /body/ and self->parse_header ) { if ( !defined self->{content_length} ) { my self->{total_parts} || 5; length = self->{’content_length’}; my part_size = int( length / parts ); self->{recv_size} = part_size; print “parent recv_size=self->{recv_size}/n”; foreach my part ( 1 .. parts – 1 ) { my child = HttpClient->new( url => parts, part => self->{content_length}, ); child->sock } = child->{parent} = self->{children} }, read_select->add( write_select->add( self->save_data( sub handle_write { my ( self, sock, read_select, write_select, my offset = 0; offset = self->{request_offset} if defined request = self->get_request_header; if ( offset == 0 ) { print “try to send request/n”; print request; } print “offset=offset/n”; my len = syswrite( sock, request, length(request) – offset, offset ); if ( !defined len ) { print STDERR “write failed:!/n”; read_select->remove(sock); write_select->remove(sock); } else { offset += len; self->{request_offset} = offset; if ( offset == length(request) ) { write_select->remove( sub start { my (self) = @_; use IO::Select; my r = IO::Select->new; r->add( my w = IO::Select->new; my sock_client = { self->sock => use Time::HiRes qw(time); my start_time = time; for ( ; ; ) { last if ( r->count == 0 ); my ( rout, wout, eout ) = IO::Select->select( w, r ); last unless defined foreach my sock ( @{wout} ) { my c = sock_client->{sock}; if ( !defined c ) { die “oops,can’t find httpclient for sock/n”; } c->handle_write( sock, r, w, sock_client ); } foreach my sock ( @{rout} ) { my c = sock_client->{sock}; if ( !defined c ) { die “oops,can’t find httpclient for sock/n”; } c->handle_read( sock, r, w, sock_client ); } } my end_time = time; my used_time = end_time – start_time; my speed = self->{content_length} / used_time; print “Done,spend used_time seconds,speed: 1;

test perl script:

#!/usr/bin/perl use strict; use warnings; use lib ‘.’; use HttpClient; use Getopt::Long; | = 1; #my url = ‘http://eclipse.cdpa.nsysu.edu.tw/downloads/drops/R-3.2.1-200609210945/eclipse-SDK-3.2.1-linux-gtk.tar.gz’; my url = ”; my total_parts = 1; my result = GetOptions (”url|u=s” => /total_parts,                         ); unless (result and my client = HttpClient->new(url=>total_parts); client->start();

参考:

RFC2616 – HTTP/1.1 Specification

technorati tags:perl, http, downloader, protocol

Blogged with Flock

固定链接

发布者:全栈程序员栈长,转载请注明出处:https://javaforall.cn/100597.html原文链接:

本文参与腾讯云自媒体分享计划,欢迎正在阅读的你也加入,一起分享。

我来说两句

0 条评论
登录 后参与评论

相关文章

  • 如何像黑客一样聊天 Mojo-Webqq

    电影里的黑客们聊天不想我们生活中的一样,用QQ、微信的客户端,都是通过命令行来进行聊天交流的,大概是为了提升逼格吧。

    C4rpeDime
  • 用Python socket实现一个简单的http服务器(post 与get 的区别)、CGIHTTPServer 简单应用

    废话不多说,前面实现过使用linux c 或者python 充当客户端来获取http 响应,也利用muduo库实现过一个简易http服务器,现在来实现一个py...

    bear_fish
  • 从两道靶场题中学习低权限如何反弹shell

    可以看出其他用户是没有高权限的,写入执行权限都没有,就不能在当前目录下写马;经尝试,bash反弹shell未能成功。

    Timeline Sec
  • 【生信菜鸟经】如何系统入门Perl

    Perl是典型的脚本语言,短小精悍,非常容易上手,尤其适合处理文本,数据,以及系统管理。它在老一辈的生物信息学分析人员中非常流行,出于历史遗留原因大家肯定会或多...

    生信技能树
  • Docker3-Dockerfile创建镜像的方法(推荐docker file这种方法

    前面两种方法已经介绍过了,这里介绍docker file,生成环境推荐使用这种方法

    Java帮帮
  • Python应用02 Python服务器进化

    **注意,在Python 3.x中,BaseHTTPServer, SimpleHTTPServer, CGIHTTPServer整合到http.server包...

    Vamei
  • MySql数据库大表添加字段的方法

    但是,线上的一张表如果数据量很大,执行加字段操作就会锁表,这个过程可能需要很长时间甚至导致服务崩溃,那么这样操作就有风险。

    剑行者
  • MySQL 数据备份与还原

      mysqldump命令将数据库中的数据备份成一个文本文件。表的结构和表中的数据将存储在生成的文本文件中。

    KEVINGUO_CN
  • Percona-toolkit的安装和配置(r8笔记第86天)

    pt工具是非常实用有效的一个工具集,对于诊断常规问题还是非常有效的,相比于Oracle的工具,MySQL中没有那么多复杂的数据字典,在实现方式上相对更加轻巧,主...

    jeanron100

扫码关注云+社区

领取腾讯云代金券