How to use the tweepy.streaming function in tweepy

To help you get started, we’ve selected a few tweepy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github propublica / politwoops-tweet-collector / bin / tweets-client.py View on Github external
raise AttributeError("{cls!r} has no attribute {attr!r}".format(cls=self.__class__.__name__,
                                                                        attr=attr))
        return self._dict[attr]

    def __setattr__(self, attr, value):
        raise AttributeError("All attributes of DataRecord objects are read-only")

class Usage(Exception):
    def __init__(self, msg):
        self.msg = msg

def dict_mget(thedict, keylist, default=None):
    result = reduce(lambda d, k: None if d is None else d.get(k), keylist, thedict)
    return result if result is not None else default

class TweetListener(tweepy.streaming.StreamListener):
    def __init__(self, queue, *args, **kwargs):
        super(TweetListener, self).__init__(*args, **kwargs)
        self.queue = queue
        self.config = tweetsclient.Config().get()
        self.database = MySQLdb.connect(
            host=self.config.get('database', 'host'),
            port=int(self.config.get('database', 'port')),
            db=self.config.get('database', 'database'),
            user=self.config.get('database', 'username'),
            passwd=self.config.get('database', 'password'),
            charset="utf8mb4",
            use_unicode=True
        )
        self.database.autocommit(True) # needed if you're using InnoDB
        self.database.cursor().execute('SET NAMES UTF8MB4')
        self.users = self.get_users()
github melizeche / comentaBOT / comment_bot.py View on Github external
msg = "Tweet Error: {0}".format(err)
            print(msg)
            logError(msg)

    def reply(self, message, reply_id):
        try:
            print(message[:200])
            self.api.update_status(
                status=message[:200], in_reply_to_status_id=reply_id)
        except Exception as err:
            msg = "Reply Error: {0}".format(err)
            print(msg)
            logError(msg)


class StListener(tweepy.streaming.StreamListener):

    def on_data(self, data):
        parsed_tweet = json.loads(data)
        print(parsed_tweet)
        text = parsed_tweet['text'].split()
        user = "@" + parsed_tweet['user']['screen_name']
        if text[0].upper() == USERNAME_STRING.upper():
            if len(text)>1:
                replyTweetSeed(text[1:4], parsed_tweet['id'], user)
            else:
                replyTweet(parsed_tweet['id'], user)
        return True

    def on_error(self, status):
        logError(status)
        print(status)
github Linkuist / linkuist / collectr / collector / management / commands / twitter_collector.py View on Github external
def handle(self, *args, **kwargs):
        user_list = args
        if not user_list:
            user_list = UserSocialAuth.objects \
                .filter(provider='twitter') \
                .values_list('user__username', flat=True)
        logger.info("Running Twitter URL collector for %s",
                    ", ".join(user_list))

        pool = Pool()
        pool.map(run_process, user_list)
        pool.close()
        pool.join()


class TwitterListener(tweepy.streaming.StreamListener):

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop('user')
        super(TwitterListener, self).__init__(*args, **kwargs)
        self.queue = Queue('link_indexing',
                           connection=Redis(**settings.RQ_DATABASE))

    def on_error(self, status_code):
        logger.error("Twitter error with status code %s", status_code)

    def on_status(self, status):
        if hasattr(status, 'entities') and 'urls' in status.entities:
            logger.info("Indexing tweet %s for %s",
                        str(status), self.user.username)
            self.queue.enqueue_call(
                func=index_url,
github telvis07 / twitter_mining / example.py View on Github external
CONSUMER_KEY = config.get('auth','CONSUMER_KEY')
    CONSUMER_SECRET = config.get('auth','CONSUMER_SECRET')
    ACCESS_TOKEN = config.get('auth','ACCESS_TOKEN')
    ACCESS_TOKEN_SECRET = config.get('auth','ACCESS_TOKEN_SECRET')

    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
    return auth


fn=sys.argv[1]
config = ConfigParser.RawConfigParser()
config.read(fn)
try:
    auth = login(config)
    streaming_api = tweepy.streaming.Stream(auth, Listener(), timeout=60)
    # San Francisco area.
    streaming_api.filter(follow=None, locations=[-122.75,36.8,-121.75,37.8]) 
except KeyboardInterrupt:
    print "got keyboardinterrupt"
github barentsen / tweet-tracking / spiders / track-spiders.py View on Github external
def on_error(self, status_code):
        sys.stderr.write('Encountered error with status code: {}'.format(status_code))
        return True # Don't kill the stream

    def on_timeout(self):
        sys.stderr.write('Timeout...')
        return True # Don't kill the stream


if __name__ == '__main__':
    # Setup authentication
    auth = tweepy.OAuthHandler(secrets.CONSUMER_KEY, secrets.CONSUMER_SECRET)
    auth.set_access_token(secrets.ACCESS_KEY, secrets.ACCESS_SECRET)
    # Start listening to the streaming API
    sapi = tweepy.streaming.Stream(auth, CustomStreamListener())
    sapi.filter(track=["蜘蛛", "araña", "spider", "عنكبوت", "मकड़ी", "labah-labah",
                       "aranha", "паук", "クモ", "spinne", "araignée", "ragno"], stall_warnings=True)
github Linkuist / linkuist / collectr / collector / twitterng.py View on Github external
from social_auth.models import UserSocialAuth

# rq
from redis import Redis
from rq import use_connection, Queue

from semantism.process import index_url

logger = logging.getLogger('tweet_collector')


CONSUMER_KEY = settings.TWITTER_CONSUMER_KEY
CONSUMER_SECRET = settings.TWITTER_CONSUMER_SECRET


class TwitterListener(tweepy.streaming.StreamListener):

    def __init__(self, *args, **kwargs):
        super(TwitterListener, self).__init__(*args, **kwargs)
        self.q = Queue('link_indexing', connection=Redis('127.0.0.1', port=6379))

    def on_error(self, status_code):
        logger.error("Twitter error with status code %s", status_code)

    def on_status(self, status):
        if hasattr(status, 'entities') and 'urls' in status.entities:
            logger.info("adding task called %s for %s" % (datetime.now(), sys.argv[1]))
            self.q.enqueue_call(func=index_url, args=(status, self.user.pk, datetime.now(),
                    status.user.screen_name, "twitter"), timeout=60)
        else:
            logger.info("tweet ignored")
github snarfed / bridgy / twitter_streaming.py View on Github external
if not stream.running:
      del streams[key]

  query = Twitter.query(Twitter.status != 'disabled')
  sources = {t.key: t for t in query.iter()}
  stream_keys = set(streams.keys())
  source_keys = set(sources.keys())

  # Connect to new accounts
  to_connect = source_keys - stream_keys
  if to_connect:
    logging.info('Connecting %d streams', len(to_connect))
  for key in to_connect:
    source = sources[key]
    auth = twitter_auth.tweepy_auth(*source.auth_entity.get().access_token())
    streams[key] = streaming.Stream(auth, Listener(source), secure=True)
    # run stream in *non*-background thread, since app engine backends have a
    # fixed limit of 10 background threads per instance. normal threads are only
    # limited by memory, and since we're starting them from a background thread,
    # they're not bound to an HTTP request.
    # http://stackoverflow.com/a/20896720/186123
    streams[key].userstream(async=True)

  # Disconnect from deleted or disabled accounts
  to_disconnect = stream_keys - source_keys
  if to_disconnect:
    logging.info('Disconnecting %d streams', len(to_disconnect))
  for key in to_disconnect:
    streams[key].disconnect()
    del streams[key]
github exPHAT / twitter-sort / main.py View on Github external
# They did not reply to the aaron-sort tweet
            return True

    def on_error(self, statusCode):
        # There was a listener error

        print("There was a listener error with the code", statusCode)
        return True

    def on_timeout(self):
        # There was a listener timeout

        print("There was a listener timeout")
        return True

listener = tweepy.streaming.Stream(auth, ReplyListener())
# Listen for a response to the user
listener.filter(track=["@" + api.me().screen_name])