Shioaji Quote Api with Tensorflow¶
In [1]:
import json
import datetime as dt
import numpy as np
import pandas as pd
import bqplot as bq
import shioaji as sj
from ipywidgets import HBox, VBox
In [2]:
import tensorflow as tf
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dropout, Dense, Input, Flatten
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import categorical_crossentropy
Build simple convolution neural network model that convolution through time¶
In [4]:
input_layer = Input(shape=(500, 6, 1), name='input')
conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
flat = Flatten(name='flat')(maxp4)
dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)
The Model Structure¶
In [5]:
model = Model(inputs=[input_layer], outputs=[fc2])
model.summary()
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= input (InputLayer) (None, 500, 6, 1) 0 _________________________________________________________________ conv1 (Conv2D) (None, 497, 6, 8) 40 _________________________________________________________________ conv2 (Conv2D) (None, 494, 6, 16) 528 _________________________________________________________________ maxp1 (MaxPooling2D) (None, 123, 6, 16) 0 _________________________________________________________________ conv3 (Conv2D) (None, 120, 6, 32) 2080 _________________________________________________________________ conv4 (Conv2D) (None, 117, 6, 64) 8256 _________________________________________________________________ maxp2 (MaxPooling2D) (None, 29, 6, 64) 0 _________________________________________________________________ conv5 (Conv2D) (None, 26, 6, 64) 16448 _________________________________________________________________ conv6 (Conv2D) (None, 23, 6, 128) 32896 _________________________________________________________________ maxp3 (MaxPooling2D) (None, 5, 6, 128) 0 _________________________________________________________________ conv7 (Conv2D) (None, 4, 6, 256) 65792 _________________________________________________________________ conv8 (Conv2D) (None, 3, 6, 256) 131328 _________________________________________________________________ maxp4 (MaxPooling2D) (None, 1, 6, 256) 0 _________________________________________________________________ flat (Flatten) (None, 1536) 0 _________________________________________________________________ dropout1 (Dropout) (None, 1536) 0 _________________________________________________________________ fc1 (Dense) (None, 256) 393472 _________________________________________________________________ dropout2 (Dropout) (None, 256) 0 _________________________________________________________________ fc2 (Dense) (None, 3) 771 ================================================================= Total params: 651,611 Trainable params: 651,611 Non-trainable params: 0 _________________________________________________________________
Load the trained model weights¶
In [6]:
model.load_weights('simplecnn.h5')
In [3]:
def load_model():
global model, graph
graph = tf.get_default_graph()
with graph.as_default():
input_layer = Input(shape=(500, 6, 1), name='input')
conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
flat = Flatten(name='flat')(maxp4)
dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)
model = Model(inputs=[input_layer], outputs=[fc2])
model.load_weights('simplecnn.h5')
Init shioaji api and login¶
In [4]:
api = sj.Shioaji(simulation=False)
In [5]:
with open('login.json', 'r') as f:
kw_login = json.loads(f.read())
api.login(**kw_login)
Init Charts dataset¶
In [6]:
tick_plot_length = 150
df_norm = pd.read_hdf('normalizer.h5')
norm_min = df_norm.loc['min'].values
norm_range = df_norm.loc['max'].values - norm_min
model_input_tick_num = 500
features_num = 6
np_features = np.zeros((model_input_tick_num, features_num))
pred_data = np.zeros((tick_plot_length, 3))
In [12]:
data_in = np.random.rand(2)
data_inner = np.random.rand(2)
data_outer = np.random.rand(4)
define init charts function¶
In [13]:
def init_model_chart():
global model_bar
x_ord = bq.LinearScale()
y_sc = bq.LinearScale()
model_bar = bq.Bars(x=x_tick_index, y=pred_data.T[:, :],
scales={'x': x_ord, 'y': y_sc},
colors=['#848484', '#f44242', '#00d100'], type='stacked')
ax_x = bq.Axis(scale=x_ord)
ax_y = bq.Axis(scale=y_sc, tick_format='0.2f', orientation='vertical')
fig = bq.Figure(marks=[model_bar], axes=[ax_x, ax_y],
fig_margin= {"top":60, "bottom":30, "left":60, "right":0},)
fig.layout.height = '200px'
fig.layout.width = '1200px'
return fig
define processing data function¶
In [18]:
def split_data(rec, index):
sp_rec = {}
[sp_rec.update({k: v[index] if isinstance(v, (list, tuple)) else v})
for k, v in rec.items()]
sp_rec['idx'] = index
return sp_rec
def proc_model_data(quote_msg):
global np_features, pred_data
global model, graph
quote_msgs = [split_data(quote_msg, ind) for ind, _ in enumerate(quote_msg['Close'])]
for msg in quote_msgs:
if msg.get('TickType', 0) == 1:
ask_vol, bid_vol = (msg.get('Volume', 0), 0)
elif msg.get('TickType', 0) == 2:
ask_vol, bid_vol = (0, msg.get('Volume', 0))
else:
ask_vol, bid_vol = (0, 0)
np_features[:-1] = np_features[1:]
np_features[-1] = [msg.get('Close', 0), msg.get('DiffPrice', 0),
msg.get('Volume', 0), msg.get('TargetKindPrice', 0),
ask_vol, bid_vol,]
input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
pred_data[:-1] = pred_data[1:]
with graph.as_default():
model.load_weights('simplecnn.h5')
pred_data[-1] = model.predict(input_x)[0]
define update chart function¶
In [23]:
def update_model_chart():
global model_bar
with model_bar.hold_sync():
model_bar.x = x_tick_index.copy()
model_bar.y = pred_data.T.copy()
define on quote callback¶
In [27]:
load_model()
@sj.on_quote
def quote_callback(topic, quote_msg):
global tick_plot_length
global line_chart, scatter_chart
global pie_outer, pie_inner, pie_in
global x_tick_index, y_price, y_vol, updown_color, askbid_color_data
global new_index, new_deal_price, new_vol, new_updown_color, new_askbid_color_data
global ask_price, bid_price
global large_ask_deal_volsum, small_ask_deal_volsum, large_bid_deal_volsum
global small_bid_deal_volsum, ask_deal_count, bid_deal_count
global ask_bid_static
global bar_bidask, bar_bidask_diff
global x_bar_data, y_bar_data, color_bar_data, y_data_diff
global ohlc_chart, df_min
global model, graph, init_model
vol_threshold = 10
#print(topic, quote_msg)
if topic.startswith('Q') and 'TXFD9' in topic:
proc_ask_bid_bardata(topic, quote_msg)
update_barchart(x_bar_data, y_bar_data,
color_bar_data, y_bar_data_diff)
elif topic.startswith('L') and 'TXFD9' in topic:
proc_tick_chartdata(topic, quote_msg)
proc_ask_bid_staticdata(topic, quote_msg, vol_threshold)
proc_ohlc_data(quote_msg, new_deal_price)
proc_model_data(quote_msg)
update_ohlc_chart()
update_tickandpie_chart(update_freq=1)
update_model_chart()
WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
define event callback¶
In [28]:
@sj.on_event
def event_callback(resp_code, event_code, event):
print("Respone Code: {} | Event Code: {} | Event: {}".format(resp_code, event_code, event))
set callback function¶
In [29]:
api.quote.set_callback(quote_callback)
api.quote.set_event_callback(event_callback)
Chart¶
In [30]:
VBox([init_model_chart(),
HBox([init_ohlc_chart(), init_tickchart(),]),
HBox([pies_chart(), bidask_bar_chart(), ]),
])
Subscribe Quote¶
In [31]:
TXFR1 = api.Contracts.Futures.TXF.TXF201904
TSE2330 = api.Contracts.Stocks.TSE.TSE2330
In [32]:
api.quote.subscribe(TXFR1)
api.quote.subscribe(TXFR1, quote_type='bidask')
api.quote.subscribe(TSE2330)
api.quote.subscribe(TSE2330, quote_type='bidask')
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Place order in quote callback?¶
- use global to let Shioaji Object work fine
- c++ call quote callback will borrow python interpreter only when interact with python
- when the process back to c++ thread will release the blocking
- It is not recommended to do that the thing spend too much time in the callback function
In [112]:
@sj.on_quote
def quote_callback(topic, quote_msg):
global api
CNN model inference performance¶
In [100]:
%%timeit
input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
28 µs ± 353 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
In [102]:
%%timeit
model.predict(input_x)
2.91 ms ± 98.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
Basic way for simplify the callback job for just push the quote msg to queue¶
In [114]:
storage_quote = {}
@sj.on_quote
def quote_callback(topic, quote_msg):
global storage_quote
if topic not in storage_quote:
storage_quote[topic] = [quote_msg]
else:
storage_quote[topic].append(quote_msg)
api.quote.set_callback(quote_callback)
Use redis for cross instance and cross language to get the quote¶
In [111]:
! pip install redis msgpack
Requirement already satisfied: redis in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (3.2.1) Requirement already satisfied: msgpack in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (0.6.1)
In [109]:
import redis
import msgpack
rs = redis.Redis(host='localhost', port=6666)
In [110]:
@sj.on_quote
def quote_callback(topic, quote_msg):
rs.rpush(topic, msgpack.dumps(quote_msg))
api.quote.set_callback(quote_callback)