问硒网刮刀刮苹果过去三年的推特

EN

import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait


options = Options()
options.headless = True
options.add_argument('window-size=1920x1080')

web = "https://stocktwits.com/search"
driver = webdriver.Chrome(r"C:\Users\veron\Downloads\chromedriver\chromedriver.exe", options=options)
driver.get(web)
#driver.maximize_window()

username = driver.find_element_by_xpath('//input[@placeholder = "Symbol or @Username"]')
username.send_keys("AAPL")

time.sleep(2)

username.send_keys(Keys.ENTER)

time.sleep(2)


def get_tweet(element):
    try:
        user = element.find_element_by_xpath('.//span[@class = "st_2JY3sEE"]/a[contains(@href, "/")]/span[text()]').text
        text = element.find_element_by_xpath('.//div[@class="st_3SL2gug"]').text
        date = element.find_element_by_xpath('.//a[@class ="st_28bQfzV st_1E79qOs st_3TuKxmZ st_1VMMH6S"]').text
        # date = date_old.replace("\n", "")
        tweet_data = [user, text, date]
    except:
        tweet_data = ['user', 'text', 'date']
    return tweet_data


user_data = []
text_data = []
date_data = []
scrolling = True

while scrolling:

    tweets = WebDriverWait(driver, 5).until(
        EC.presence_of_all_elements_located((By.XPATH, '//div[@class = "st_2o0zabc st_jGV698i st_PLa30pM"]')))
    # print(len(tweets))
    for tweet in tweets:
        tweet_list = get_tweet(tweet)
        user_data.append(tweet_list[0])
        text_data.append(" ".join(tweet_list[1].split()))
        date_data.append(tweet_list[2])

    # Get the initial scroll height
    last_height = driver.execute_script("return document.body.scrollHeight")
    # Specified date
    str1 = "4/30/22"

    while True:
        # Scroll down to bottom
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        # Wait to load page
        time.sleep(2)
        # Calculate new scroll height and compare it with last scroll height
        new_height = driver.execute_script("return document.body.scrollHeight")
        # check if the date substring from above is in the date list, condition 1
        res = any(str1 in string for string in date_data)
        if res is True:
            scrolling = False
            break
        # condition 2
        if new_height == last_height:
            scrolling = False
            break

        else:
            last_height = new_height
            break

driver.quit()
df_tweets = pd.DataFrame({'user': user_data, 'text': text_data, 'date': date_data})  # , 'date': date_data
df_tweets.to_csv('stocktwits_tweets.csv', index=False)
print(df_tweets)