如何用Python抓取最便宜的机票信息（上）

 1from time import sleep, strftime
 2from random import randint
 3import pandas as pd
 4from selenium import webdriver
 5from selenium.webdriver.common.keys import Keys
 6import smtplib
 7from email.mime.multipart import MIMEMultipart
 8
 9# Change this to your own chromedriver path!
10chromedriver_path = 'C:/{YOUR PATH HERE}/chromedriver_win32/chromedriver.exe'
11
12driver = webdriver.Chrome(executable_path=chromedriver_path) # This will open the Chrome window
13sleep(2)

11 # This is what the copy method would return. Right click highlighted rows on the right side and select "copy > Copy XPath"
2//*[@id="wtKI-price_aTab"]/div[1]/div/div/div[1]/div/span/span2 # This is what I used to define the "Cheapest" button
3cheap_results = ‘//a[@data-code = “price”]’

 1# Load more results to maximize the scraping
 2def load_more():
 3try:
 4more_results = '//a[@class = "moreButton"]'
 5driver.find_element_by_xpath(more_results).click()
 6# Printing these notes during the program helps me quickly check what it is doing
 7print('sleeping.....')
 8sleep(randint(45,60))
 9except:
10pass

 1def page_scrape():
 2"""This function takes care of the scraping part"""
 3
 4xp_sections = '//*[@class="section duration"]'
 5sections = driver.find_elements_by_xpath(xp_sections)
 6sections_list = [value.text for value in sections]
 7section_a_list = sections_list[::2] # This is to separate the two flights
 8section_b_list = sections_list[1::2] # This is to separate the two flights
 9
10# if you run into a reCaptcha, you might want to do something about it
11# you will know there's a problem if the lists above are empty
12# this if statement lets you exit the bot or do something else
13# you can add a sleep here, to let you solve the captcha and continue scraping
14# i'm using a SystemExit because i want to test everything from the start
15if section_a_list == []:
16raise SystemExit
17
18# I'll use the letter A for the outbound flight and B for the inbound
19a_duration = []
20a_section_names = []
21for n in section_a_list:
22# Separate the time from the cities
23a_section_names.append(''.join(n.split()[2:5]))
24a_duration.append(''.join(n.split()[0:2]))
25b_duration = []
26b_section_names = []
27for n in section_b_list:
28# Separate the time from the cities
29b_section_names.append(''.join(n.split()[2:5]))
30b_duration.append(''.join(n.split()[0:2]))
31
32xp_dates = '//div[@class="section date"]'
33dates = driver.find_elements_by_xpath(xp_dates)
34dates_list = [value.text for value in dates]
35a_date_list = dates_list[::2]
36b_date_list = dates_list[1::2]
37# Separating the weekday from the day
38a_day = [value.split()[0] for value in a_date_list]
39a_weekday = [value.split()[1] for value in a_date_list]
40b_day = [value.split()[0] for value in b_date_list]
41b_weekday = [value.split()[1] for value in b_date_list]
42
43# getting the prices
44xp_prices = '//a[@class="booking-link"]/span[@class="price option-text"]'
45prices = driver.find_elements_by_xpath(xp_prices)
46prices_list = [price.text.replace('$','') for price in prices if price.text != '']
47prices_list = list(map(int, prices_list))
48
49# the stops are a big list with one leg on the even index and second leg on odd index
50xp_stops = '//div[@class="section stops"]/div[1]'
51stops = driver.find_elements_by_xpath(xp_stops)
52stops_list = [stop.text[0].replace('n','0') for stop in stops]
53a_stop_list = stops_list[::2]
54b_stop_list = stops_list[1::2]
55
56xp_stops_cities = '//div[@class="section stops"]/div[2]'
57stops_cities = driver.find_elements_by_xpath(xp_stops_cities)
58stops_cities_list = [stop.text for stop in stops_cities]
59a_stop_name_list = stops_cities_list[::2]
60b_stop_name_list = stops_cities_list[1::2]
61
62# this part gets me the airline company and the departure and arrival times, for both legs
63xp_schedule = '//div[@class="section times"]'
64schedules = driver.find_elements_by_xpath(xp_schedule)
65hours_list = []
66carrier_list = []
67for schedule in schedules:
68hours_list.append(schedule.text.split('\n')[0])
69carrier_list.append(schedule.text.split('\n')[1])
70# split the hours and carriers, between a and b legs
71a_hours = hours_list[::2]
72a_carrier = carrier_list[1::2]
73b_hours = hours_list[::2]
74b_carrier = carrier_list[1::2]
75
76cols = (['Out Day', 'Out Time', 'Out Weekday', 'Out Airline', 'Out Cities', 'Out Duration', 'Out Stops', 'Out Stop Cities',
77'Return Day', 'Return Time', 'Return Weekday', 'Return Airline', 'Return Cities', 'Return Duration', 'Return Stops', 'Return Stop Cities',
78'Price'])
79
80flights_df = pd.DataFrame({'Out Day': a_day,
81'Out Weekday': a_weekday,
82'Out Duration': a_duration,
83'Out Cities': a_section_names,
84'Return Day': b_day,
85'Return Weekday': b_weekday,
86'Return Duration': b_duration,
87'Return Cities': b_section_names,
88'Out Stops': a_stop_list,
89'Out Stop Cities': a_stop_name_list,
90'Return Stops': b_stop_list,
91'Return Stop Cities': b_stop_name_list,
92'Out Time': a_hours,
93'Out Airline': a_carrier,
94'Return Time': b_hours,
95'Return Airline': b_carrier, 
96'Price': prices_list})[cols]
97
98flights_df['timestamp'] = strftime("%Y%m%d-%H%M") # so we can know when it was scraped
99return flights_df