问如何在Google Play搜索中抓取所有应用程序商店应用程序

EN

from bs4 import BeautifulSoup
import requests

url='https://play.google.com/store/search?q=weather%20app'
req=requests.get(url)

soup = BeautifulSoup(req.content, 'html.parser')

cards= soup.find_all("div",class_="vU6FJ p63iDd")

for card in cards:
    app_name= card.find("div",class_="WsMG1c nnK0zc").text
    company = card.find("div",class_="KoLSrc").text
    print("Name: " + app_name)
    print("Company: " + company)

Name: Weather app
Company: Accurate Weather Forecast & Weather Radar Map  
Name: AccuWeather: Weather Radar
Company: AccuWeather
Name: Weather Forecast - Accurate Local Weather & Widget
Company: Weather Forecast & Widget & Radar
Name: 1Weather Forecasts & Radar
Company: OneLouder Apps
Name: MyRadar Weather Radar
Company: ACME AtronOmatic LLC
Name: Weather data & microclimate : Weather Underground
Company: Weather Underground
Name: Weather & Widget - Weawow
Company: weawow weather app
Name: Weather forecast
Company: smart-pro android apps
Name: The Secret World of Weather: How to Read Signs in Every Cloud, Breeze, Hill, Street, Plant, Animal, and Dewdrop
Company: Tristan Gooley
Name: The Weather Machine: A Journey Inside the Forecast
Company: Andrew Blum
Name: The Mobile Mind Shift: Engineer Your Business to Win in the Mobile Moment
Company: Julie Ask
Name: Together: The Healing Power of Human Connection in a Sometimes Lonely World
Company: Vivek H. Murthy
Name: The Meadow
Company: James Galvin
Name: The Ancient Egyptian Culture Revealed, 2nd edition
Company: Moustafa Gadalla
Name: The Ancient Egyptian Culture Revealed, 2nd edition
Company: Moustafa Gadalla
Name: Chaos Theory
Company: Introbooks Team
Name: Survival Training: Killer Tips for Toughness and Secret Smart Survival Skills       
Company: Wesley Jones
Name: Kiasunomics 2: Economic Insights for Everyday Life
Company: Ang Swee Hoon
Name: Summary of We Are The Weather by Jonathan Safran Foer
Company: QuickRead
Name: Learn Swift by Building Applications: Explore Swift programming through iOS app development
Company: Emil Atanasov
Name: Weather Hazard Warning Application in Car-to-X Communication: Concepts, Implementations, and Evaluations
Company: Attila Jaeger
Name: Mobile App Development with Ionic, Revised Edition: Cross-Platform Apps with Ionic, 
Angular, and Cordova
Company: Chris Griffith
Name: Good Application Makes a Good Roof Better: A Simplified Guide: Installing Laminated 
Asphalt Shingles for Maximum Life & Weather Protection
Company: ARMA Asphalt Roofing Manufacturers Association
Name: The Secret World of Weather: How to Read Signs in Every Cloud, Breeze, Hill, Street, Plant, Animal, and Dewdrop
Company: Tristan Gooley
Name: The Weather Machine: A Journey Inside the Forecast
Company: Andrew Blum
Name: Space Physics and Aeronomy, Space Weather Effects and Applications
Company: Book 5
Name: How to Build Android Apps with Kotlin: A hands-on guide to developing, testing, and 
publishing your first apps with Android
Company: Alex Forrester
Name: Android 6 for Programmers: An App-Driven Approach, Edition 3
Company: Paul J. Deitel

from bs4 import BeautifulSoup
import requests, json, lxml, re


def bs4_scrape_google_play_store_search_apps(
    query: str, filter_by: str = "apps", country: str = "US"
):
    # https://docs.python-requests.org/en/master/user/quickstart/#passing-parameters-in-urls
    params = {
        "q": query,     # search query
        "gl": country,  # country of the search. Different country display different apps.
        "c": filter_by  # filter to display list of apps. Other filters: apps, books, movies
    }

    # https://docs.python-requests.org/en/master/user/quickstart/#custom-headers
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.79 Safari/537.36",
    }

    html = requests.get("https://play.google.com/store/search", params=params, headers=headers, timeout=30)
    soup = BeautifulSoup(html.text, "lxml")

    apps_data = []

    for app in soup.select("[jscontroller=tKHFxf]"):
        title = app.select_one(".DdYX5").text
        company = app.select_one(".wMUdtb").text
        app_icon = app.select_one(".j2FCNc img")["srcset"]

        try:
            thumbnail = app.select_one(".Shbxxd img")["srcset"]
        except:
            thumbnail = app.select_one(".Vc0mnc img")["src"]

        app_link = f'https://play.google.com{app.select_one(".Si6A0c.Gy4nib")["href"]}'
        app_id = app.select_one("a")["href"].split("id=")[1]
        
        try:
            # https://regex101.com/r/SZLPRp/1
            rating = re.search(r"\d{1}\.\d{1}", app.select_one(".ubGTjb div")["aria-label"]).group()
        except:
            rating = None
        
        
        apps_data.append({
            "title": title,
            "app_link": app_link,
            "company": company,
            "rating": float(rating) if rating else rating, # float if rating is not None else rating or None
            "app_id": app_id,
            "thumbnail": thumbnail,
            "icon": app_icon
        })        

    print(json.dumps(apps_data, indent=2, ensure_ascii=False))

bs4_scrape_google_play_store_search_apps(query="maps", filter_by="apps", country="US")

[
  {
    "title": "Google Maps",
    "app_link": "https://play.google.com/store/apps/details?id=com.google.android.apps.maps",
    "company": "Google LLC",
    "rating": 3.9,
    "app_id": "com.google.android.apps.maps",
    "thumbnail": "https://play-lh.googleusercontent.com/FQx43QTaAqeOtoTLylK3WIs7ySKuGS8AurXNA1Kj34m6w6CjavF4Oj3s5DB6xZZ7DS63=w832-h470-rw 2x",
    "icon": "https://play-lh.googleusercontent.com/Kf8WTct65hFJxBUDm5E-EpYsiDoLQiGGbnuyP6HBNax43YShXti9THPon1YKB6zPYpA=s128-rw 2x"
  }, ... other results
  {
    "title": "GPS, Maps, Voice Navigation & Directions",
    "app_link": "https://play.google.com/store/apps/details?id=com.maps.voice.navigation.traffic.gps.location.route.driving.directions",
    "company": "AppStar Studios",
    "rating": 4.0,
    "app_id": "com.maps.voice.navigation.traffic.gps.location.route.driving.directions",
    "thumbnail": "https://i.ytimg.com/vi/4E2NyVZlOjc/hqdefault.jpg",
    "icon": "https://play-lh.googleusercontent.com/NrK0b-e6cpj4yYkDuNZJHO9KUAl8pSj9TGi4Xw4GbPZ6UVsnAlLBH2AZuEMpb24Xig=s128-rw 2x"
  }
]

from serpapi import GoogleSearch
import json

params = {
    "api_key": "API KEY",      # your serpapi api key
    "engine": "google_play",   # search engine
    "hl": "en",                # language
    "store": "apps",           # apps search
    "gl": "us",                # country to search from. Different country displays different.
    "q": "weather"             # search query
}

search = GoogleSearch(params)  # where data extracts
results = search.get_dict()    # JSON -> Python dictionary

apps_data = []

for apps in results["organic_results"]:
    for app in apps["items"]:
        apps_data.append({
            "title": app.get("title"),
            "link": app.get("link"),
            "description": app.get("description"),
            "product_id": app.get("product_id"),
            "rating": app.get("rating"),
            "thumbnail": app.get("thumbnail"),
            })

print(json.dumps(apps_data, indent=2, ensure_ascii=False))

[
  {
    "title": "Weather app",
    "link": "https://play.google.com/store/apps/details?id=com.weather.forecast.weatherchannel",
    "description": "The weather channel, tiempo weather forecast, weather radar & weather map",
    "product_id": "com.weather.forecast.weatherchannel",
    "rating": 4.7,
    "thumbnail": "https://play-lh.googleusercontent.com/GdXjVGXQ90eVNpb1VoXWGT3pff2M9oe3yDdYGIsde7W9h3s2S6FDLfo1uO-gljBZ1QXO=s128-rw"
  },
  {
    "title": "The Weather Channel - Radar",
    "link": "https://play.google.com/store/apps/details?id=com.weather.Weather",
    "description": "Weather Forecast & Snow Radar: local rain tracker, weather maps & alerts",
    "product_id": "com.weather.Weather",
    "rating": 4.6,
    "thumbnail": "https://play-lh.googleusercontent.com/RV3DftXlA7WUV7w-BpE8zM0X7Y4RQd2vBvZVv6A01DEGb_eXFRjLmUhSqdbqrEl9klI=s128-rw"
  },
  {
    "title": "AccuWeather: Weather Radar",
    "link": "https://play.google.com/store/apps/details?id=com.accuweather.android",
    "description": "Your local weather forecast, storm tracker, radar maps & live weather news",
    "product_id": "com.accuweather.android",
    "rating": 4.0,
    "thumbnail": "https://play-lh.googleusercontent.com/EgDT3XrIaJbhZjINCWsiqjzonzqve7LgAbim8kHXWgg6fZnQebqIWjE6UcGahJ6yugU=s128-rw"
  },
  {
    "title": "Weather by WeatherBug",
    "link": "https://play.google.com/store/apps/details?id=com.aws.android",
    "description": "The Most Accurate Weather Forecast. Alerts, Radar, Maps & News from WeatherBug",
    "product_id": "com.aws.android",
    "rating": 4.7,
    "thumbnail": "https://play-lh.googleusercontent.com/_rZCkobaGZzXN3iquPr4u2KOe7C-ljnrSkBfw6sVL1kpUfq3sBl5MoRJEisBSnxaD-M=s128-rw"
  }, ... other results
]