我为学术目的创建了一个在Twitter数据上工作的项目。它应:
我使用django +普通HTML,但是当尝试一个只有230个联系人的帐户时,加载它需要3分钟。该应用程序旨在可伸缩以处理至少有100 k联系人的帐户。
如何提高该程序的可伸缩性?
from twitter_auth.apiKeys import *
import tweepy
from manager.models import Contact
def get_api(user):
access_tokens = user.social_auth \
.filter(provider='twitter')[0] \
.extra_data['access_token']
auth = tweepy.OAuthHandler(
SOCIAL_AUTH_TWITTER_KEY,
SOCIAL_AUTH_TWITTER_SECRET
)
auth.set_access_token(
access_tokens['oauth_token'],
access_tokens['oauth_token_secret']
)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
return api
def get_user_contacts(user):
# Setup Twitter API instance of logged User
api = get_api(user)
me = api.me()
# Set number of contacts fetched per call Max:200
count = 200
# Fetch logged User's friends into a list
friends = tweepy.Cursor(api.friends, me.screen_name, count=count).items()
# friends = api.friends(me.screen_name, count=10)
friend_list = []
for i in friends:
friend_list.append(i)
# Fetch logged User's followers into a list
followers = tweepy.Cursor(api.followers, me.screen_name, count=count).items()
# followers = api.followers(me.screen_name, count=10)
follower_list = []
for j in followers:
follower_list.append(j)
existing_contacts = Contact.objects.filter(user=user)
for contact in existing_contacts:
if contact not in friend_list and contact not in follower_list:
contact.delete()
# Start iterating friend list fetched from Twitter API
for friend in friend_list:
# Initialize Contact Object
temp_friend = Contact(
twitter_id=friend.id,
profile_image_url=friend.profile_image_url_https,
screen_name=friend.screen_name,
name=friend.name,
followers_count=friend.followers_count,
friends_count=friend.friends_count,
statuses_count=friend.statuses_count,
description=friend.description,
location=friend.location,
friendship_status=1,
protected_status=friend.protected,
user=user
)
# Check if entry is just Friend (1) or Friend & Follower (3)
if friend in follower_list:
temp_friend.friendship_status = 3
# Check if current friend already exists in DB
existing_friend = Contact.objects.filter(screen_name=friend.screen_name, user=user)
if existing_friend:
# Update Contact info
existing_friend.update(
profile_image_url=friend.profile_image_url_https,
name=friend.name,
followers_count=friend.followers_count,
friends_count=friend.friends_count,
statuses_count=friend.statuses_count,
description=friend.description,
location=friend.location,
protected_status=friend.protected,
)
# Check if existing Contact followed back
# Case: Have followed back so friendship status updates to (3)
if existing_friend in follower_list:
existing_friend.update(
friendship_status=3
)
# Case: Have not followed back so friendship status remains unchanged(1)
else:
temp_friend.save()
# Start iterating follower list fetched from Twitter API
for follower in follower_list:
# Initialize Contact Object
temp_follower = Contact(
twitter_id=follower.id,
profile_image_url=follower.profile_image_url_https,
screen_name=follower.screen_name,
name=follower.name,
followers_count=follower.followers_count,
friends_count=follower.friends_count,
statuses_count=follower.statuses_count,
description=follower.description,
location=follower.location,
friendship_status=2,
protected_status=follower.protected,
user=user
)
# Check if current follower already exists in DB
existing_follower = Contact.objects.filter(twitter_id=follower.id, user=user)
if existing_follower:
# Update Contact info
existing_follower.update(
profile_image_url=follower.profile_image_url_https,
name=follower.name,
followers_count=follower.followers_count,
friends_count=follower.friends_count,
statuses_count=follower.statuses_count,
description=follower.description,
location=follower.location,
protected_status=follower.protected,
)
# Check if user followed back existing the existing follower
# Case: Have followed back so friendship status updates to (3)
if existing_follower in friend_list:
existing_follower.update(
friendship_status=3
)
# Case: Have not followed back so friendship status remains unchanged(2)
else:
temp_follower.save()
def get_user_tweets(user, id):
api = get_api(user)
tweet_list = api.user_timeline(id=id, count=2)
return tweet_list发布于 2020-10-09 22:42:49
他们是有可能的,但却气馁。
access_tokens = user.social_auth \
.filter(provider='twitter')[0] \
.extra_data['access_token']会更好,根据大多数的指针,
access_tokens = (
user.social_auth
.filter(provider='twitter')[0]
.extra_data['access_token']
)friend_list = []
for i in friends:
friend_list.append(i)可能只是
friend_list = list(friends)但是,考虑到您的用法:
for contact in existing_contacts:
if contact not in friend_list and contact not in follower_list:
contact.delete()你最好用布景:
friend_list = set(friends)
follower_list = set(followers)
existing_contacts = set(Contact.objects.filter(user=user))
to_delete = existing_contacts - friend_list - follower_list
for contact in to_delete:
contact.delete()https://codereview.stackexchange.com/questions/250090
复制相似问题