SyntaxError:语法无效“异步定义websocketConnect_govahi():”

时间:2020-05-17 13:40:43

标签: python service async-await nameerror

以下代码在Jupyter上正常工作,但是我不知道为什么当我尝试将其应用于systemd服务时:

#cdn_crawl_mongo_gavahi_aati
import requests
import json
import asyncio
import websockets
import urllib
import random
import time
import datetime
import pandas as pd
from threading import Thread
import numpy as np
import matplotlib.pyplot as plt
import pymongo

myclient = pymongo.MongoClient("mongodb://localhost:27017/", username='mongo-admin', password='password')
mydb = myclient["admin"]

pd.set_option('display.max_columns', None)  # or 1000
pd.set_option('display.max_rows', None)

path_data_future = '/root/data_crwl/cdn/future_contract/'
#zaferan = ['SAFOR99', 'SAFTR99', 'SAFSH99']
zaferan = ['SAFTR99', 'SAFSH99']
zaferan_govahi = ['IRK1A0389983', 'IRK1A0449985']


def wrt_df_csv(dataframe, data_type = 'tahvil'):
    # filename            
    data_file_time_stamp = str(datetime.datetime.now()).replace(' ','--') + str(datetime.datetime.now().timestamp())
    data_file_time_stamp = data_file_time_stamp + '.csv'
    data_file_time_stamp = data_file_time_stamp.replace(':',"-")
    #write to file
    print(len(dataframe))
    if data_type == 'tahvil':
        data_file_time_stamp = path_data_future+ 'tahvil/' + data_file_time_stamp
    else:
        data_file_time_stamp = path_data_future+ 'govahi/' + data_file_time_stamp
    dataframe.to_csv(data_file_time_stamp, sep=',', encoding='utf-16')
    print(data_file_time_stamp)
    return 

def pars_dataframe(data_crawl):
    dataframe_initial = pd.DataFrame(columns=data_column_names)
    for i in range(len(data_crawl)):  
        if len(data_crawl[i][1])>0:
            item_data = json.loads(data_crawl[i][1].replace('\n',''))
            for item in item_data:
                if 'ContractCode' in item.keys() and item['ContractCode'] in zaferan:
                    data_ptem = pd.DataFrame([item])
                    dataframe_initial = pd.concat([dataframe_initial, data_ptem])
    return dataframe_initial

def pars_dataframe_govahi(data_crawl):
    dataframe_initial = pd.DataFrame(columns=data_column_names)
    for i in range(len(data_crawl)):  
        if len(data_crawl[i][1])>0:
            item_data = json.loads(data_crawl[i][1].replace('\n',''))
            for item in item_data:
                if 'Code' in item.keys() and item['Code'] in zaferan_govahi:
                    data_ptem = pd.DataFrame([item])
                    dataframe_initial = pd.concat([dataframe_initial, data_ptem])

    return dataframe_initial

def wrt_to_file(data_crwl, data_type = 'tahvil'):
    try:
        #print('write_to_file')
        # parse data to write in dataframe
        data_df = pars_dataframe(data_crwl)
        #write to file
        wrt_df_csv(data_df, data_type)
    except:
        pass  
    return 

str_column_names = ',AskPrice1,AskPrice2,AskPrice3,AskPrice4,AskPrice5,AskTotalVolume,AskVolume1,AskVolume2,AskVolume3,AskVolume4,AskVolume5,AverageTradedPrice,BidPrice1,BidPrice2,BidPrice3,BidPrice4,BidPrice5,BidTotalVolume,BidVolume1,BidVolume2,BidVolume3,BidVolume4,BidVolume5,ClosingPrice,ContractCode,ContractCurrencyDecimalPlaces,ContractCurrencyEnDesc,ContractCurrencyFaDesc,ContractDescription,ContractID,ContractSize,ContractSizeUnitEnDesc,ContractSizeUnitFaDesc,Expired,FirstTradedPrice,FirstTradedPriceChanges,FirstTradedPriceChangesPercent,FirstTradedPriceTime,HighTradedPrice,HighTradedPriceChanges,HighTradedPriceChangesPercent,InitialMargin,LastSettlementPrice,LastSettlementPriceDate,LastTradedPrice,LastTradedPriceChanges,LastTradedPriceChangesPercent,LastTradedPriceTime,LastTradingDate,LastUpdate,LowTradedPrice,LowTradedPriceChanges,LowTradedPriceChangesPercent,MaintenanceMargin,OpenInterests,OpenInterestsChanges,OpenInterestsChangesPercent,OpeningPrice,OrdersDateTime,PersianLastSettlementPriceDate,PersianOrdersDateTime,PesrsianLastTradingDate,TradesCount,TradesValue,TradesValueCurrencyEnDesc,TradesValueCurrencyFaDesc,TradesVolume,YesterdayOpenInterests'
data_column_names = str_column_names.split(',')

diagram_col = ['LastTradedPrice','LastTradedPriceTime']

connectionData_govahi = [{"name":"gavahihub"}]

async def websocketConnect_govahi():
    def startReceiving(arg):
        r = ''
        while r == '':
            try:
                r = requests.get("https://cdn.ime.co.ir/realTimeServer/start", params = wsParams)
                break
            except:
                print("Connection refused by the server..")
                #print("Let me sleep for 3 seconds")
                #print("ZZzzzz...")
                time.sleep(3)
                print("Was a nice sleep, now let me continue...")
                continue

    r = ''
    while r == '':
        try:
            r = requests.get("https://cdn.ime.co.ir/realTimeServer/negotiate", params = {
            "clientProtocol": "2.1",
            "connectionData": json.dumps(connectionData_govahi),
            })
            break
        except:
            print("Connection refused by the server..")
            #print("Let me sleep for 3 seconds")
            #print("ZZzzzz...")
            time.sleep(3)
            print("Was a nice sleep, now let me continue...")
            continue

    response = r.json()

    #print(response)
    #print(f'got connection token : {response["ConnectionToken"]}')

    wsParams = {
        "transport": "webSockets",
        "clientProtocol": "2.1",
        "connectionToken": response["ConnectionToken"],
        "connectionData": json.dumps(connectionData_govahi),
        "tid": random.randint(0,9)
    }

    websocketUri = f"wss://cdn.ime.co.ir/realTimeServer/connect?{urllib.parse.urlencode(wsParams)}"
    result = []
    global data_govahi
    #crawled_item = []
    async with websockets.connect(websocketUri, ping_interval=None) as websocket:

        #print(f'started websocket')
        thread = Thread(target = startReceiving, args = (0, ))
        thread.start()
        for i in range(0,5):
            #print("receiving")
            try:
                data = await websocket.recv()

                jsonData = json.loads(data)
                if ("M" in jsonData and len(jsonData["M"]) > 0 and "A" in jsonData["M"][0] and len(jsonData["M"][0]["A"]) > 0):
                    items = jsonData["M"][0]["A"][0]
                    if type(items) == list and len(items) > 0: 
                        result = items
                        #crawled_item = items
                        break
            except:
                print('Reconnecting')
                websocket = await websockets.connect(websocketUri, ping_interval=None)
        thread.join()

        #data_dic.append([datetime.datetime.now(), json.dumps(crawled_item, indent=4, sort_keys=True)])
        data_govahi.append([datetime.datetime.now(), json.dumps(result, indent=4, sort_keys=True)])
        #wrt_to_file([[datetime.datetime.now(), json.dumps(result, indent=4, sort_keys=True)]])

    #print('data_govahi')
    #print(len(data_govahi))
    return data_govahi

connectionData = [{"name":"marketshub"}]

async def websocketConnect():

    def startReceiving(arg):
        r = ''
        while r == '':
            try:
                r = requests.get("https://cdn.ime.co.ir/realTimeServer/start", params = wsParams)
                break
            except:
                print("Connection refused by the server..")
                #print("Let me sleep for 3 seconds")
                #print("ZZzzzz...")
                time.sleep(3)
                print("Was a nice sleep, now let me continue...")
                continue

    r = ''
    while r == '':
        try:
            r = requests.get("https://cdn.ime.co.ir/realTimeServer/negotiate", params = {
            "clientProtocol": "2.1",
            "connectionData": json.dumps(connectionData),
            })
            break
        except:
            print("Connection refused by the server..")
            #print("Let me sleep for 3 seconds")
            #print("ZZzzzz...")
            time.sleep(3)
            print("Was a nice sleep, now let me continue...")
            continue

    response = r.json()

    #print(response)
    #print(f'got connection token : {response["ConnectionToken"]}')

    wsParams = {
        "transport": "webSockets",
        "clientProtocol": "2.1",
        "connectionToken": response["ConnectionToken"],
        "connectionData": json.dumps(connectionData),
        "tid": random.randint(0,9)
    }

    websocketUri = f"wss://cdn.ime.co.ir/realTimeServer/connect?{urllib.parse.urlencode(wsParams)}"
    result = []
    global data_dic
    async with websockets.connect(websocketUri, ping_interval=None) as websocket:

        #print(f'started websocket')
        thread = Thread(target = startReceiving, args = (0, ))
        thread.start()
        for i in range(0,5):
            #print("receiving")
            try:
                data = await websocket.recv()

                jsonData = json.loads(data)
                if ("M" in jsonData and len(jsonData["M"]) > 0 and "A" in jsonData["M"][0] and len(jsonData["M"][0]["A"]) > 0):
                    items = jsonData["M"][0]["A"][0]
                    if type(items) == list and len(items) > 0: 
                        result = items
                        #crawled_item = items
                        break
            except:
                print('Reconnecting')
                websocket = await websockets.connect(websocketUri, ping_interval=None)
        thread.join()

        data_dic.append([datetime.datetime.now(), json.dumps(result, indent=4, sort_keys=True)])
    return data_dic

def scrap_data():
    asyncio.ensure_future(websocketConnect())
    #asyncio.run(websocketConnect())
    #client.send_message(data_dic, embed=e)
    #print(len(data_dic))
    return 

def wrt_to_mongo(data_type = 'tahvil'):
    global data_dic
    global data_govahi
    try:
        if data_type=='tahvil':
            data_dict_mongo = pars_dataframe_mongo(data_dic, data_type)   
        else:
            data_dict_mongo = pars_dataframe_mongo_govahi(data_govahi, data_type)  
        #print(data_dict_mongo)
        myclient = pymongo.MongoClient("mongodb://localhost:27017/", username='mongo-admin', password='password')
        mydb = myclient["admin"]
        posts = mydb.posts
        for item in data_dict_mongo:
            if '_id' in item.keys():
                del item['_id']
            post_id = posts.insert_one(item).inserted_id
        if data_type=='tahvil':
            data_dic = []  
        else:
            data_govahi = []
        print('write to mongo_db')
    except:
        print('unable to write on mongo_db')
        pass  
    return 

def pars_dataframe_mongo_govahi(data_crawl, data_type):
    data_dict_mongo = []
    for i in range(len(data_crawl)):  
        if len(data_crawl[i][1])>0:
            item_data = json.loads(data_crawl[i][1].replace('\n',''))
            for item in item_data:
                if 'Code' in item.keys() and item['Code'] in zaferan_govahi:
                    item['db_date'] = datetime.datetime.now()
                    item['type'] = 'govahi'
                    data_dict_mongo.append(item)
    return data_dict_mongo

def pars_dataframe_mongo(data_crawl, data_type):
    data_dict_mongo = []
    for i in range(len(data_crawl)):  
        if len(data_crawl[i][1])>0:
            item_data = json.loads(data_crawl[i][1].replace('\n',''))
            for item in item_data:
                if 'ContractCode' in item.keys() and item['ContractCode'] in zaferan:                    
                    item['db_date'] = datetime.datetime.now()
                    item['type'] = 'tahvil'
                    data_dict_mongo.append(item)
    return data_dict_mongo

mat = np.array([])
mat_2 = np.array([])
mat_g = np.array([])
mat_g_2 = np.array([])

data_dic  = []  
data_govahi  = []  



while(True):

    await (websocketConnect())
    await (websocketConnect_govahi())

    data_pars = pars_dataframe([data_dic[-1]])
    if len(data_pars) > 0:
        mat_append = np.array(data_pars.loc[data_pars['ContractCode'] == zaferan[0]]['LastTradedPrice'])
        mat_append_2 = np.array(data_pars.loc[data_pars['ContractCode'] == zaferan[1]]['LastTradedPrice'])
        if len(mat) == 0:
            mat = mat_append
        if len(mat_2) == 0:
            mat_2 = mat_append_2
        mat = np.append(mat, mat_append)
        mat_2 = np.append(mat_2, mat_append_2)

    data_pars_g = pars_dataframe_govahi([data_govahi[-1]])
    if len(data_pars_g) > 0:
        mat_append_g = np.array(data_pars_g.loc[data_pars_g['Code'] == 'IRK1A0389983']['FinalPrice'])
        mat_append_g_2 = np.array(data_pars_g.loc[data_pars_g['Code'] == 'IRK1A0449985']['FinalPrice'])
        if len(mat_g) == 0:
            mat_g = mat_append_g
        if len(mat_g_2) == 0:
            mat_g_2 = mat_append_g_2
        mat_g = np.append(mat_g, mat_append_g)
        mat_g_2 = np.append(mat_g_2, mat_append_g_2)

    print('...')
    print(len(data_dic))
    print(len(data_govahi))
    if len(data_dic)>2:
        wrt_to_mongo('tahvil')

    if len(data_govahi)>2:
        wrt_to_mongo('govahi')

运行systemctl status python-script时出现的完整错误是:

● python-script.service - Python Script Service
   Loaded: loaded (/etc/systemd/system/python-script.service; enabled; vendor preset: disabled)
   Active: failed (Result: exit-code) since Sun 2020-05-17 17:52:36 +0430; 5min ago
  Process: 850 ExecStart=/usr/bin/python /root/script.py (code=exited, status=1/FAILURE)
 Main PID: 850 (code=exited, status=1/FAILURE)

May 17 17:52:36 boiga.server1.more.com systemd[1]: Started Python Script Service.
May 17 17:52:36 boiga.server1.more.com python[850]: File "/root/script.py", line 84
May 17 17:52:36 boiga.server1.more.com python[850]: async def websocketConnect_govahi():
May 17 17:52:36 boiga.server1.more.com python[850]: ^
May 17 17:52:36 boiga.server1.more.com python[850]: SyntaxError: invalid syntax
May 17 17:52:36 boiga.server1.more.com systemd[1]: python-script.service: main process exited, code=exited, status=1/FAILURE
May 17 17:52:36 boiga.server1.more.com systemd[1]: Unit python-script.service entered failed state.
May 17 17:52:36 boiga.server1.more.com systemd[1]: python-script.service failed.

0 个答案:

没有答案