Streaming real time data, data mismatch issue.

userNameUnknown

Hi,

I am trying to stream and store the real time data to s3 using the code given below. The code runs without any error but the data OHLC value is different from the one available in tradingview. I guess something going wrong in pushing the data to s3, can someone help if something wrong in pushing data to s3 or some flaws in other part of the code.




import logging, json, datetime
import pandas as pd
from copy import copy
from kiteconnect import KiteConnect, KiteTicker

logging.basicConfig(level=logging.DEBUG)

api_key='api_key'
access_token='access_token'
client = KiteConnect(api_key='api_key', access_token='access_token')

ins_list = client.instruments()
ins_list = pd.DataFrame(ins_list)
nse_fno_list =ins_list[ins_list['exchange'].isin(['NFO'])]
nse_fno_list = nse_fno_list[nse_fno_list['name'].isin(['NIFTY',"NIFTY BANK","FINNIFTY"])]
nse_fno_tokens = nse_fno_list['instrument_token'].to_list()
ins_list = ins_list[ins_list['exchange'].isin(['NSE',"MCX"])]
ins_list_tokens = ins_list['instrument_token'].to_list()

tokens1 = nse_fno_tokens[0:3000]
tokens2 = ins_list_tokens[0:3000]
tokens3 = ins_list_tokens[3000:6000]
tokens4 = ins_list_tokens[6000:9000]
tokens5 = ins_list_tokens[9000:12000]
try:
data = []
c = 1
def on_ticks1(ws, ticks):
logging.debug("ticker 2: {}".format(len(ticks)))

def on_ticks2(ws, ticks):
# global data
# data.extend(ticks)
logging.debug("ticker 2: {}".format(len(ticks)))

def on_ticks3(ws, ticks):
# global data
# data.extend(ticks)
logging.debug("ticker 3: {}".format(len(ticks)))

def on_connect1(ws, response):
ws.subscribe(tokens3)
ws.set_mode(ws.MODE_FULL, tokens3)

def on_connect2(ws, response):
ws.subscribe(tokens4)
ws.set_mode(ws.MODE_FULL, tokens4)

def on_connect3(ws, response):
ws.subscribe(tokens5)
ws.set_mode(ws.MODE_FULL, tokens5)

except Exception as e:
print(e)

# Create three instances
kws1 = KiteTicker(api_key, access_token, debug=True)
kws2 = copy(kws1)
kws3 = copy(kws1)

kws1.on_ticks = on_ticks1
kws1.on_connect = on_connect1

kws2.on_ticks = on_ticks2
kws2.on_connect = on_connect2

kws3.on_ticks = on_ticks3
kws3.on_connect = on_connect3

kws1.connect(threaded=True)
kws2.connect(threaded=True)
kws3.connect(threaded=True)

# Block main thread
while True:
def on_ticks1(ws, ticks):
feed_data(ticks)
# print("i am from while True -> on ticks: feed_data module called!")

def on_ticks2(ws, ticks):
feed_data(ticks)
# print("i am from while True -> on ticks: feed_data module called!")

def on_ticks3(ws, ticks):
feed_data(ticks)
# print("i am from while True -> on ticks: feed_data module called!")

def feed_data(ticks):
data = pd.DataFrame(ticks)
dt = datetime.datetime.now()
data.to_csv(f'path/to/s3/TickData_{dt}.csv', index = False)
# print("i am from -> feed_data function")

# print("i am from main -> after kws.connect(threaded=True)")
kws1.on_ticks=on_ticks1
kws2.on_ticks=on_ticks2
kws3.on_ticks=on_ticks3


From the logs i could notice the below:

2023-03-21 07:32:45+0000 [-] will retry in 2 seconds
2023-03-21 07:32:45+0000 [-] Stopping factory
ERROR:kiteconnect.ticker:Connection error: 1006 - connection was closed uncleanly (None)
ERROR:kiteconnect.ticker:Connection closed: 1006 - connection was closed uncleanly (None)
2023-03-21 07:32:45+0000 [-] will retry in 2 seconds
2023-03-21 07:32:45+0000 [-] Stopping factory



DEBUG:kiteconnect.ticker:ping =>
DEBUG:kiteconnect.ticker:last ping was 2.6191964149475098 seconds back.
DEBUG:kiteconnect.ticker:pong => b''
DEBUG:root:ticker 2: 293
DEBUG:kiteconnect.ticker:last pong was 2.6649932861328125 seconds back.
DEBUG:kiteconnect.ticker:pong => b''
ERROR:kiteconnect.ticker:Connection error: 0 - Can't subscribe to more than 4000 instruments.
DEBUG:root:ticker 2: 316
DEBUG:kiteconnect.ticker:ping =>
DEBUG:kiteconnect.ticker:last ping was 2.5911364555358887 seconds back.

rakeshr

ERROR:kiteconnect.ticker:Connection error: 0 - Can't subscribe to more than 4000 instruments.

Your websocket connection is getting disconnected, because of exceeding instrument tokens. Maximum of 3000 tokens are allowed in an websocket connection. Go through the documentation here.

userNameUnknown

@rakeshr Thanks for the reply, but I have filtered only 3000 symbols per list.

tokens3 = ins_list_tokens[3000:6000]
tokens4 = ins_list_tokens[6000:9000]
tokens5 = ins_list_tokens[9000:12000]