问在web抓取时无法从文章中获取文本正文

EN

import json
from time import sleep
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

capabilities = DesiredCapabilities().CHROME

chrome_options = Options()
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.add_argument("--ignore-certificate-errors")

# disable the banner "Chrome is being controlled by automated test software"
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])

driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)

url_main = 'https://www.scmp.com/news/asia/east-asia/article/3199400/japan-asean-hold-summit-tokyo-around-december-2023-japanese-official'

driver.get(url_main)

driver.implicitly_wait(20)
element_has_bottom_message = WebDriverWait(driver, 120).until(EC.presence_of_element_located((By.CLASS_NAME, "has-bottom-messaging")))
if element_has_bottom_message:
    element_gdpr = WebDriverWait(driver, 120).until(
        EC.presence_of_element_located((By.CLASS_NAME, "gdpr-banner__accept")))
    if element_gdpr:
        gdrp_button = driver.find_element_by_xpath("//*[@class='gdpr-banner__accept']")
        driver.implicitly_wait(20)
        ActionChains(driver).move_to_element(gdrp_button).click(gdrp_button).perform()
        element_my_news_popup = WebDriverWait(driver, 120).until(
            EC.presence_of_element_located((By.CLASS_NAME, "my-news-landing-popup__icon-close")))
        if element_my_news_popup:
            my_news_popup = driver.find_element_by_xpath("//*[@class='my-news-landing-popup__icon-close']")
            ActionChains(driver).move_to_element(my_news_popup).click(my_news_popup).perform()
            driver.implicitly_wait(20)
            raw_soup = BeautifulSoup(driver.page_source, 'lxml')
            json_dictionaries = raw_soup.find_all(name='script', attrs={'type': 'application/ld+json'})
            if len(json_dictionaries) != 0:
                for json_dictionary in json_dictionaries:
                    dictionary = json.loads("".join(json_dictionary.contents), strict=False)
                    article_bool = bool([value for (key, value) in dictionary.items() if key == 'articleBody'])
                    if article_bool:
                        for key, value in dictionary.items():
                            if key == 'articleBody':
                                print(value)


sleep(30)
driver.close()
driver.quit()

The leaders of Japan and 10-member Asean on Saturday agreed to hold a summit in Tokyo 
in or around December next year to commemorate the 50th anniversary of their relationship, 
a Japanese official said. Japanese Prime Minister Fumio Kishida and his counterparts from 
the Association of Southeast Asian Nations also pledged to deepen their cooperative ties 
when they met in Phnom Penh, according to the official. Japan has been trying to boost 
relations with Asean at a time when some of its members are increasingly vigilant against 
China ’s assertive territorial claims in the East and South China seas . Why is Japan 
losing ground in Asean despite being a bigger investor than China? “Although concerns are 
growing over opaque and unfair development support, Japan will continue to back sustainable 
growth” of Southeast Asia , Kishida said at the outset of the meeting, which was open to 
the media, in a veiled reference to Beijing’s trade and economic practices. Leaders of 
several nations mentioned the importance of freedom of navigation and overflight in the 
South China Sea, and of the necessity of adhering to international law, the official said 
after the meeting. The agreement on the special summit in Tokyo came as the US and China 
have been intensifying their competition for influence in Southeast Asia. In November last 
year, China and Asean agreed to upgrade their ties to a “comprehensive strategic 
partnership” when the two sides held a special online summit commemorating the 30th 
anniversary of their dialogue, with Chinese President Xi Jinping making a rare appearance. 
China has stepped up efforts to expand its clout in the region as security tensions 
with the US escalate in nearby waters. After China’s move, the US in May declared with 
Asean that they had decided to elevate their relationship to a “comprehensive strategic 
partnership” as well. At the Asean-Japan gathering, Kishida also reiterated his support 
for the “Asean Outlook on the Indo-Pacific”, an initiative aimed at maintaining peace, 
freedom and prosperity in the region, the official said.