某管1个月迅速赚钱经验及colly实战爬取美女图片站点源码级细节分享，绝对不容错过golang技能buff加成！

#方法一:我上面说的，在Request中传一个参数
def start_requests(self):
    for url in self.start_urls:
        return Request(url=url,callback=self.parse,headers={"User-Agent":"xx"},meta={"proxy":"http:/154.112.82.262:8050"})
#方法二：创建自定义的代理中间件
from w3lib.http import basic_auth_header
class MyCustomProxyMiddleware(object):
    def process_proxy(self,request,spider):
        request.meta["proxy"] = "http://xxx.xx.x.x:8050"
        request.headers["Proxy-Authorization"] =
        basic_auth_header("<proxy_user>", "<proxy_pass>")
#然后在DOWNLOAD_MIDDLEWARES对于你自定义的中间件进行配置，这样爬虫运行时才会加载到这些配置，并保证在执行时你的配置可以被用到
DOWNLOADER_MIDDLEWARES = {
    'myproject.middlewares.CustomProxyMiddleware': 350,#优先级越高，会越先执行，
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
}

#方法三：在settings.py文件中把代理ip以数据结构lista或以文件的形式声明，同上步一样，也需要在DOWNLOAD_MIDDLEWARES中进行配置
ROTATING_PROXY_LIST = [
    'proxy1.com:8000',
    'proxy2.com:8031',
    'proxy3.com:8032',
]
# above is same as add this to settings.py

ROTATING_PROXY_LIST_PATH = '/my/path/proxies.txt'

#启用
DOWNLOADER_MIDDLEWARES = {
    'rotating_proxies.middlewares.RotatingProxyMiddleware':610,
    'rotating_proxies.middlewares.BanDetectionMiddleware':620
}
#方法四，几乎用这种方式都是针对隧道代理 
class MyCustomProxyMiddleware(object):
    @classmethod
    def from_crawl(cls,crawler):
        return cls(crawler.settings)
    def __init__(self,settings):
        self.username = settings.get('PROXY_USER')
        self.password = settings.get('PROXY_PASSWORD')
        self.url = settings.get('PROXY_URL')
        self.port = seetings.get('PROXY_PORT')
    def process_request(self,request,spider):
        host = f'http://{self.username}:{self.password}@{self.url}:{self.port}'
        request.meta['proxy'] = host

package main

import (
    "bytes"
    "log"
    "github.com/gocolly/colly"
)

func main() {
    // Instantiate default collector
    c := colly.NewCollector(colly.AllowURLRevisit())

    // 设置代理，与scrapy最简单的方式属实殊途同归了
    c.SetProxy("http://proxy.example.com:8080")
    ...

    // Request Page
    c.Visit("http://httpbin.org/ip")
}

package main

import (
    "bytes"
    "log"
    "github.com/gocolly/colly"
    "github.com/gocolly/colly/proxy"
)

func main() {
    // Instantiate default collector
    c := colly.NewCollector(colly.AllowURLRevisit())


    proxyList := []string{
        "http://Username:Password@85.237.57.198:20000",
      "http://Username:Password@85.237.57.198:21000",
      "http://Username:Password@85.237.57.198:22000",
      "http://Username:Password@85.237.57.198:23000",
    }

    // Create Rotating Proxy Switcher
    rp, err := proxy.RoundRobinProxySwitcher(proxyList...)
    if err != nil {
        log.Fatal(err)
    }

   // Set Collector To Use Proxy Switcher Function
    c.SetProxyFunc(rp)
    ...
    c.Visit("http://httpbin.org/ip")
}

package main

import (
    "bytes"
    "log"
    "github.com/gocolly/colly"
    "net/url"
)

func main() {
    // Instantiate default collector
    c := colly.NewCollector(colly.AllowURLRevisit())
    ...
    // Create Proxy API URL
    u, err := url.Parse("https://proxy.xx.xx/v1/")
    if err != nil {
        log.Fatal(err)
    }

    // Add Query Parameters
    q := u.Query()
    q.Set("api_key", "YOUR_API_KEY")
    q.Set("url", "http://httpbin.org/ip")
    u.RawQuery = q.Encode()
    // Request Page
    c.Visit(u.RawQuery)
}

type ProxyFunc func(*http.Request) (*url.URL, error)

func (c *Collector) SetProxyFunc(p ProxyFunc) {
	t, ok := c.backend.Client.Transport.(*http.Transport)
	if c.backend.Client.Transport != nil && ok {
		t.Proxy = p
	} else {
		c.backend.Client.Transport = &http.Transport{
			Proxy: p,
		}
	}
}
func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) {
	index := atomic.AddUint32(&r.index, 1) - 1
	u := r.proxyURLs[index%uint32(len(r.proxyURLs))]

	ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String())
	*pr = *pr.WithContext(ctx)
	return u, nil
}

func RoundRobinProxySwitcher(ProxyURLs ...string) (colly.ProxyFunc, error) {
	if len(ProxyURLs) < 1 {
		return nil, colly.ErrEmptyProxyURL
	}
	urls := make([]*url.URL, len(ProxyURLs))
	for i, u := range ProxyURLs {
		parsedU, err := url.Parse(u)
		if err != nil {
			return nil, err
		}
		urls[i] = parsedU
	}
	return (&roundRobinSwitcher{urls, 0}).GetProxy, nil
}

type Proxy struct{
    IP string
    Port int
    expireTime time.Time
    lastCalledTime time.Time
}

type ProxyHeap []*proxy
func (h ProxyHeap) Len() int { return len(h)}
func (h ProxyHeap) Less(i,j int) bool {
    return h[i].lastCalledTime.Before(h[j].lastCalledTime)
}
func (h ProxyHeap) Swap(i, j int) {
    h[i],h[j] = h[j],h[i]
}
func (h *ProxyHeap) Push(x interface{}){
    *h = append(*h,x.(*.Proxy))
}

func (h *ProxyHeap) Pop() interface(){
    old := *h
    n := len(old)
    x := old[n-1]
    *h = old[0:n-1]
    return x
}
type ProxyPool struct{
    proxies ProxyHeap
    delay time.Duration
    mu sync.Mutex
    one chan string
}

func NewProxyHeap(delay time.Duration) *ProxyPool{
    ch := make(chan string,0)
    return &ProxyPool{
        proxies: make(ProxyHeap,0)
        delay: delay,
        mu:sync.Mutext{},
        one:ch,
    }
}

func (p *ProxyPool) Add(proxy *Proxy){
    p.mu.Lock()
    defer p.mu.Unlock()
    heap.Push(&p.proxies,proxy)
}

func (p *ProxyPool) GetProxy(ctx context.Context) error{
    for {
        select{
            case <-ctx.Done():
                return ctx.Err()
            default:
                for p.proxies.Len() == 0{
                    return errors.New("No valid proxy")
                }
                for p.proxies.Len() > 0{
                    if time.Now().After(proxy.lastCalledTime.Add(p.delay)){
                        proxyURL := fmt.Sprintf("http://%s:%d",proxy.IP,proxy.Port)
                        p.one <- proxyURL
                        proxy.lastCalledTime = time.Now()
                        heap.Pop(&p.proxies)
                        heap.Push(&p.proxies,proxy)
                        break
                    }else{
                        time.Sleep(proxy.lastCalledTime.Sub(time.Now()))
                    }
                }

        }
    }
}
//如何在代码中使用呢？
func main() {
	pool := NewProxyPool(2 * time.Second)
    //我只是找了一个提供免费代理ip网站，下载了它们提供的txt格式文件，我本身也没打算采用这种方式，只是测试使用，因为我知道这种免费ip挺不靠谱的，还是得充钱。当然此处你也可以连mysql，redis，把免费ip代理站点的ip增量写进数据库，这边colly从数据库读取来使用，全自动化的流程就成形了
	fi, err := os.Open("./http_proxies.txt")
	if err != nil {
		fmt.Printf("Error: %s\n", err)
	}
	defer fi.Close()

	br := bufio.NewReader(fi)
	for {
		a, _, c := br.ReadLine()
		if c == io.EOF {
			break
		}
		final := strings.Split(string(a), ":")
		ipAddress := final[0]
		port := final[1]
		var portInt int
		if v, err := strconv.Atoi(port); err == nil {
			portInt = v	
		}else{
			panic(err)
		}
		fmt.Printf("####%v\n", ipAddress)
		fmt.Printf(">>>>%v\n", portInt)
		// fmt.Println(string(a))
		pool.Add(&Proxy{
			IP:             ipAddress,
			Port:           portInt,
			expireTime:     time.Now().Add(30 * time.Second),
			lastCalledTime: time.Now(),
		})
	}
	go func() {
		err := pool.GetProxy(ctx)
		if err != nil {
			fmt.Println("Error:", err)
		}
	}()



	url := "https://httpbin.org/ip"

	q, _ := queue.New(
		1, // Number of consumer threads
		&queue.InMemoryQueueStorage{MaxSize: 10000}, // Use default queue storage
	)

	c := colly.NewCollector()
	c.AllowURLRevisit = true
	c.SetProxyFunc(pool.ProxyFunc)

    ...
    c.Visit('xx')
}

q, _ := queue.New( 2, InMemoryQueueStorage{MaxSize: 10000},)