Question

我正在通过建模this script将来自Twitter的Twitter数据从API流式传输到Postgres数据库中。使用这些确切的方法，我可以将数据成功地流式传输到两个表中（一个表包含user_id / user_name，另一个表包含数据）。我已经可以做一些小的改动以提取其他一些信息，但是使用这些方法，我只收集给定关键字列表的转发，而我希望收集给定列表的所有推特。基于原始脚本收集/存储转推的user_id和user_names的方式，我更改了尝试流式传输到新表而不引用任何转推代码的代码。不幸的是，这样做的结果是两个空表。否则，该代码可以正常运行，并且正在向终端打印语句，只是没有数据。为什么会这样呢？下面是我的代码：

import psycopg2
import tweepy
import json
import numpy as np

# Importing postgres credentials
import postgres_credentials

# Importing twitter credentials
import twitter_credentials


# Accesing twitter from the App created in my account
def autorize_twitter_api():
"""
This function gets the consumer key, consumer secret key, access token
and access token secret given by the app created in your Twitter account
and authenticate them with Tweepy.
"""
# Get access and costumer key and tokens
auth = tweepy.OAuthHandler(twitter_credentials.CONSUMER_KEY, twitter_credentials.CONSUMER_SECRET)
auth.set_access_token(twitter_credentials.ACCESS_TOKEN, twitter_credentials.ACCESS_TOKEN_SECRET)

return auth


def create_tweets_table(term_to_search):
"""
This function open a connection with an already created database and creates a new table to
store tweets related to a subject specified by the user
"""

# Connect to Twitter Database created in Postgres
conn_twitter = psycopg2.connect(dbname=postgres_credentials.dbname, user=postgres_credentials.user, password=postgres_credentials.password, host=postgres_credentials.host,
                                    port=postgres_credentials.port)

# Create a cursor to perform database operations
cursor_twitter = conn_twitter.cursor()

# with the cursor now, create two tables, users twitter and the corresponding table according to the selected topic
cursor_twitter.execute("CREATE TABLE IF NOT EXISTS test_twitter_users (user_id VARCHAR PRIMARY KEY, user_name VARCHAR);")

query_create = "CREATE TABLE IF NOT EXISTS %s (id SERIAL, created_at_utc timestamp, tweet text NOT NULL, user_id VARCHAR, user_name VARCHAR, PRIMARY KEY(id), FOREIGN KEY(user_id) REFERENCES twitter_users(user_id));" % (
            "test_tweet_text")
cursor_twitter.execute(query_create)

# Commit changes
conn_twitter.commit()

# Close cursor and the connection
cursor_twitter.close()
conn_twitter.close()
return


def store_tweets_in_table(term_to_search, created_at_utc, tweet, user_id, user_name):
"""
This function open a connection with an already created database and inserts into corresponding table
tweets related to the selected topic
"""

# Connect to Twitter Database created in Postgres
conn_twitter = psycopg2.connect(dbname=postgres_credentials.dbname, user=postgres_credentials.user, password=postgres_credentials.password, host=postgres_credentials.host,
                                    port=postgres_credentials.port)

# Create a cursor to perform database operations
cursor_twitter = conn_twitter.cursor()

# with the cursor now, insert tweet into table
cursor_twitter.execute(
    "INSERT INTO test_twitter_users (user_id, user_name) VALUES (%s, %s) ON CONFLICT(user_id) DO NOTHING;",
    (user_id, user_name))

cursor_twitter.execute(
    "INSERT INTO %s (created_at_utc, tweet, user_id, user_name) VALUES (%%s, %%s, %%s, %%s);" % (
                'test_tweet_text'),
    (created_at_utc, tweet, user_id, user_name))

# Commit changes
conn_twitter.commit()

# Close cursor and the connection
cursor_twitter.close()
conn_twitter.close()
return


class MyStreamListener(tweepy.StreamListener):
'''
def on_status(self, status):
    print(status.text)
'''

def on_data(self, raw_data):

    try:
        global term_to_search

        data = json.loads(raw_data)

        # Obtain all the variables to store in each column
        user_id = data['user']['id']
        user_name = data['user']['name']
        created_at_utc = data['created_at']
        tweet = data['text']

        # Store them in the corresponding table in the database
        store_tweets_in_table(term_to_search, created_at_utc, tweet, user_id, user_name)

    except Exception as e:
        print(e)

def on_error(self, status_code):
    if status_code == 420:
        # returning False in on_error disconnects the stream
        return False

########################################################################

while True:
if __name__ == "__main__":
    # Creates the table for storing the tweets
    term_to_search = ["donald trump","trump"]
    create_tweets_table(term_to_search)

    # Connect to the streaming twitter API
    api = tweepy.API(wait_on_rate_limit_notify=True)

    # Stream the tweets
    try:
        streamer = tweepy.Stream(auth=autorize_twitter_api(), listener=MyStreamListener(api=api),tweet_mode='extended')
        streamer.filter(track=term_to_search)
    except:
        continue

Answer 1

如果在此函数中打印值会怎样？你在那里有价值观吗？

def on_data(self, raw_data):

    try:
        global term_to_search

        data = json.loads(raw_data)

        # Obtain all the variables to store in each column
        user_id = data['user']['id']
        user_name = data['user']['name']
        created_at_utc = data['created_at']
        tweet = data['text']

        # Store them in the corresponding table in the database
        store_tweets_in_table(term_to_search, created_at_utc, tweet, user_id, user_name)

    except Exception as e:
        print(e)

打印sql语句时，可以看到没有数据的插入吗？

Answer 2

我发现了问题-我正在创建两个新表，但是将数据插入到两个不同的表中。

Python：tweepy / psycopg2未将数据插入表中

2 个答案: