Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
raise AttributeError("{cls!r} has no attribute {attr!r}".format(cls=self.__class__.__name__,
attr=attr))
return self._dict[attr]
def __setattr__(self, attr, value):
raise AttributeError("All attributes of DataRecord objects are read-only")
class Usage(Exception):
def __init__(self, msg):
self.msg = msg
def dict_mget(thedict, keylist, default=None):
result = reduce(lambda d, k: None if d is None else d.get(k), keylist, thedict)
return result if result is not None else default
class TweetListener(tweepy.streaming.StreamListener):
def __init__(self, queue, *args, **kwargs):
super(TweetListener, self).__init__(*args, **kwargs)
self.queue = queue
self.config = tweetsclient.Config().get()
self.database = MySQLdb.connect(
host=self.config.get('database', 'host'),
port=int(self.config.get('database', 'port')),
db=self.config.get('database', 'database'),
user=self.config.get('database', 'username'),
passwd=self.config.get('database', 'password'),
charset="utf8mb4",
use_unicode=True
)
self.database.autocommit(True) # needed if you're using InnoDB
self.database.cursor().execute('SET NAMES UTF8MB4')
self.users = self.get_users()
msg = "Tweet Error: {0}".format(err)
print(msg)
logError(msg)
def reply(self, message, reply_id):
try:
print(message[:200])
self.api.update_status(
status=message[:200], in_reply_to_status_id=reply_id)
except Exception as err:
msg = "Reply Error: {0}".format(err)
print(msg)
logError(msg)
class StListener(tweepy.streaming.StreamListener):
def on_data(self, data):
parsed_tweet = json.loads(data)
print(parsed_tweet)
text = parsed_tweet['text'].split()
user = "@" + parsed_tweet['user']['screen_name']
if text[0].upper() == USERNAME_STRING.upper():
if len(text)>1:
replyTweetSeed(text[1:4], parsed_tweet['id'], user)
else:
replyTweet(parsed_tweet['id'], user)
return True
def on_error(self, status):
logError(status)
print(status)
def handle(self, *args, **kwargs):
user_list = args
if not user_list:
user_list = UserSocialAuth.objects \
.filter(provider='twitter') \
.values_list('user__username', flat=True)
logger.info("Running Twitter URL collector for %s",
", ".join(user_list))
pool = Pool()
pool.map(run_process, user_list)
pool.close()
pool.join()
class TwitterListener(tweepy.streaming.StreamListener):
def __init__(self, *args, **kwargs):
self.user = kwargs.pop('user')
super(TwitterListener, self).__init__(*args, **kwargs)
self.queue = Queue('link_indexing',
connection=Redis(**settings.RQ_DATABASE))
def on_error(self, status_code):
logger.error("Twitter error with status code %s", status_code)
def on_status(self, status):
if hasattr(status, 'entities') and 'urls' in status.entities:
logger.info("Indexing tweet %s for %s",
str(status), self.user.username)
self.queue.enqueue_call(
func=index_url,
CONSUMER_KEY = config.get('auth','CONSUMER_KEY')
CONSUMER_SECRET = config.get('auth','CONSUMER_SECRET')
ACCESS_TOKEN = config.get('auth','ACCESS_TOKEN')
ACCESS_TOKEN_SECRET = config.get('auth','ACCESS_TOKEN_SECRET')
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
return auth
fn=sys.argv[1]
config = ConfigParser.RawConfigParser()
config.read(fn)
try:
auth = login(config)
streaming_api = tweepy.streaming.Stream(auth, Listener(), timeout=60)
# San Francisco area.
streaming_api.filter(follow=None, locations=[-122.75,36.8,-121.75,37.8])
except KeyboardInterrupt:
print "got keyboardinterrupt"
def on_error(self, status_code):
sys.stderr.write('Encountered error with status code: {}'.format(status_code))
return True # Don't kill the stream
def on_timeout(self):
sys.stderr.write('Timeout...')
return True # Don't kill the stream
if __name__ == '__main__':
# Setup authentication
auth = tweepy.OAuthHandler(secrets.CONSUMER_KEY, secrets.CONSUMER_SECRET)
auth.set_access_token(secrets.ACCESS_KEY, secrets.ACCESS_SECRET)
# Start listening to the streaming API
sapi = tweepy.streaming.Stream(auth, CustomStreamListener())
sapi.filter(track=["蜘蛛", "araña", "spider", "عنكبوت", "मकड़ी", "labah-labah",
"aranha", "паук", "クモ", "spinne", "araignée", "ragno"], stall_warnings=True)
from social_auth.models import UserSocialAuth
# rq
from redis import Redis
from rq import use_connection, Queue
from semantism.process import index_url
logger = logging.getLogger('tweet_collector')
CONSUMER_KEY = settings.TWITTER_CONSUMER_KEY
CONSUMER_SECRET = settings.TWITTER_CONSUMER_SECRET
class TwitterListener(tweepy.streaming.StreamListener):
def __init__(self, *args, **kwargs):
super(TwitterListener, self).__init__(*args, **kwargs)
self.q = Queue('link_indexing', connection=Redis('127.0.0.1', port=6379))
def on_error(self, status_code):
logger.error("Twitter error with status code %s", status_code)
def on_status(self, status):
if hasattr(status, 'entities') and 'urls' in status.entities:
logger.info("adding task called %s for %s" % (datetime.now(), sys.argv[1]))
self.q.enqueue_call(func=index_url, args=(status, self.user.pk, datetime.now(),
status.user.screen_name, "twitter"), timeout=60)
else:
logger.info("tweet ignored")
if not stream.running:
del streams[key]
query = Twitter.query(Twitter.status != 'disabled')
sources = {t.key: t for t in query.iter()}
stream_keys = set(streams.keys())
source_keys = set(sources.keys())
# Connect to new accounts
to_connect = source_keys - stream_keys
if to_connect:
logging.info('Connecting %d streams', len(to_connect))
for key in to_connect:
source = sources[key]
auth = twitter_auth.tweepy_auth(*source.auth_entity.get().access_token())
streams[key] = streaming.Stream(auth, Listener(source), secure=True)
# run stream in *non*-background thread, since app engine backends have a
# fixed limit of 10 background threads per instance. normal threads are only
# limited by memory, and since we're starting them from a background thread,
# they're not bound to an HTTP request.
# http://stackoverflow.com/a/20896720/186123
streams[key].userstream(async=True)
# Disconnect from deleted or disabled accounts
to_disconnect = stream_keys - source_keys
if to_disconnect:
logging.info('Disconnecting %d streams', len(to_disconnect))
for key in to_disconnect:
streams[key].disconnect()
del streams[key]
# They did not reply to the aaron-sort tweet
return True
def on_error(self, statusCode):
# There was a listener error
print("There was a listener error with the code", statusCode)
return True
def on_timeout(self):
# There was a listener timeout
print("There was a listener timeout")
return True
listener = tweepy.streaming.Stream(auth, ReplyListener())
# Listen for a response to the user
listener.filter(track=["@" + api.me().screen_name])