如何用Python抓取最便宜的机票信息（下）

 1def start_kayak(city_from, city_to, date_start, date_end):
 2"""City codes - it's the IATA codes!
 3Date format - YYYY-MM-DD"""
 4
 5kayak = ('https://www.kayak.com/flights/' + city_from + '-' + city_to +
 6'/' + date_start + '-flexible/' + date_end + '-flexible?sort=bestflight_a')
 7driver.get(kayak)
 8sleep(randint(8,10))
 9
10# sometimes a popup shows up, so we can use a try statement to check it and close
11try:
12xp_popup_close = '//button[contains(@id,"dialog-close") and contains(@class,"Button-No-Standard-Style close ")]'
13driver.find_elements_by_xpath(xp_popup_close)[5].click()
14except Exception as e:
15pass
16sleep(randint(60,95))
17print('loading more.....')
18
19# load_more()
20
21print('starting first scrape.....')
22df_flights_best = page_scrape()
23df_flights_best['sort'] = 'best'
24sleep(randint(60,80))
25
26# Let's also get the lowest prices from the matrix on top
27matrix = driver.find_elements_by_xpath('//*[contains(@id,"FlexMatrixCell")]')
28matrix_prices = [price.text.replace('$','') for price in matrix]
29matrix_prices = list(map(int, matrix_prices))
30matrix_min = min(matrix_prices)
31matrix_avg = sum(matrix_prices)/len(matrix_prices)
32
33print('switching to cheapest results.....')
34cheap_results = '//a[@data-code = "price"]'
35driver.find_element_by_xpath(cheap_results).click()
36sleep(randint(60,90))
37print('loading more.....')
38
39# load_more()
40
41print('starting second scrape.....')
42df_flights_cheap = page_scrape()
43df_flights_cheap['sort'] = 'cheap'
44sleep(randint(60,80))
45
46print('switching to quickest results.....')
47quick_results = '//a[@data-code = "duration"]'
48driver.find_element_by_xpath(quick_results).click() 
49sleep(randint(60,90))
50print('loading more.....')
51
52# load_more()
53
54print('starting third scrape.....')
55df_flights_fast = page_scrape()
56df_flights_fast['sort'] = 'fast'
57sleep(randint(60,80))
58
59# saving a new dataframe as an excel file. the name is custom made to your cities and dates
60final_df = df_flights_cheap.append(df_flights_best).append(df_flights_fast)
61final_df.to_excel('search_backups//{}_flights_{}-{}_from_{}_to_{}.xlsx'.format(strftime("%Y%m%d-%H%M"),
62city_from, city_to, 
63date_start, date_end), index=False)
64print('saved df.....')
65
66# We can keep track of what they predict and how it actually turns out!
67xp_loading = '//div[contains(@id,"advice")]'
68loading = driver.find_element_by_xpath(xp_loading).text
69xp_prediction = '//span[@class="info-text"]'
70prediction = driver.find_element_by_xpath(xp_prediction).text
71print(loading+'\n'+prediction)
72
73# sometimes we get this string in the loading variable, which will conflict with the email we send later
74# just change it to "Not Sure" if it happens
75weird = '¯\\_(ツ)_/¯'
76if loading == weird:
77loading = 'Not sure'
78
79username = 'YOUREMAIL@hotmail.com'
80password = 'YOUR PASSWORD'
81
82server = smtplib.SMTP('smtp.outlook.com', 587)
83server.ehlo()
84server.starttls()
85server.login(username, password)
86msg = ('Subject: Flight Scraper\n\n\
87Cheapest Flight: {}\nAverage Price: {}\n\nRecommendation: {}\n\nEnd of message'.format(matrix_min, matrix_avg, (loading+'\n'+prediction)))
88message = MIMEMultipart()
89message['From'] = 'YOUREMAIL@hotmail.com'
90message['to'] = 'YOUROTHEREMAIL@domain.com'
91server.sendmail('YOUREMAIL@hotmail.com', 'YOUROTHEREMAIL@domain.com', msg)
92print('sent email.....')

 1city_from = input('From which city? ')
 2city_to = input('Where to? ')
 3date_start = input('Search around which departure date? Please use YYYY-MM-DD format only ')
 4date_end = input('Return when? Please use YYYY-MM-DD format only ')
 5
 6# city_from = 'LIS'
 7# city_to = 'SIN'
 8# date_start = '2019-08-21'
 9# date_end = '2019-09-07'
10
11for n in range(0,5):
12start_kayak(city_from, city_to, date_start, date_end)
13print('iteration {} was complete @ {}'.format(n, strftime("%Y%m%d-%H%M")))
14
15# Wait 4 hours
16sleep(60*60*4)
17print('sleep finished.....')