Python爬取Landsat云掩膜数据

from bs4 import BeautifulSoup
import pandas as pd
# from multiprocessing import Pool,Manager
from tqdm import tqdm
import requests
import os

def download_file(url,save_path):
    
    # 用流stream的方式获取url的数据
    resp = requests.get(url, stream=True)
    # 拿到文件的长度，并把total初始化为0
    total = int(resp.headers.get('content-length', 0))
    # 打开当前目录的fname文件(名字你来传入)
    # 初始化tqdm，传入总数，文件名等数据，接着就是写入，更新等操作了
    with open(save_path, 'wb') as file, tqdm(
        desc=save_path,
        total=total,
        unit='iB',
        unit_scale=True,
        unit_divisor=1024,
    ) as bar:
        for data in resp.iter_content(chunk_size=1024):
            size = file.write(data)
            bar.update(size)

url="https://landsat.usgs.gov/landsat-8-cloud-cover-assessment-validation-data"

response=requests.get(url).content
parse_content=BeautifulSoup(response)
table=parse_content.select('table')[2::]

root_path='/content/drive/MyDrive/landsat-8-cloud-cover-assessment-validation-data'

for land_cover in table:
    
    land_cover_name=land_cover.select('th')[0].select('strong')[0].string.replace('/','')
    land_cover_path=os.path.join(root_path,land_cover_name)
    if not os.path.exists(land_cover_path):
      os.makedirs(land_cover_path)

        
    tar_file_urls=land_cover.select('a')
    
    for file_url in tar_file_urls:
        url=file_url.get('href')
        file_name=url.split('/')[-1]
        if os.path.exists(os.path.join(land_cover_path,file_name)):
          continue
        download_file(url,os.path.join(land_cover_path,file_name))

import tarfile
from glob import glob
import os

input_path='/content/drive/MyDrive/dataset/landsat-8-cloud-cover-assessment-validation-data'
out_root_path='/content/drive/MyDrive/dataset/landsat_cloud_cover_unzip'

tar_files=glob(input_path+'/*/*.tar.gz')
for i in tar_files:
  tf = tarfile.open(i)
  out_path=os.path.join(out_root_path,i.split('/')[-2])
  if not os.path.exists(out_path):
    os.makedirs(out_path)
  tf.extractall(out_path)
  print("解压成功:{}".format(i))