Shioaji Quote Api with Tensorflow

shioaji-logotensorflow-logo

In [1]:
import json
import datetime as dt
import numpy as np
import pandas as pd
import bqplot as bq
import shioaji as sj
from ipywidgets import HBox, VBox
In [2]:
import tensorflow as tf
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dropout, Dense, Input, Flatten
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import categorical_crossentropy

Build simple convolution neural network model that convolution through time

In [4]:
input_layer = Input(shape=(500, 6, 1), name='input')
conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
flat = Flatten(name='flat')(maxp4)
dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)

The Model Structure

In [5]:
model = Model(inputs=[input_layer], outputs=[fc2])
model.summary()
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input (InputLayer)           (None, 500, 6, 1)         0         
_________________________________________________________________
conv1 (Conv2D)               (None, 497, 6, 8)         40        
_________________________________________________________________
conv2 (Conv2D)               (None, 494, 6, 16)        528       
_________________________________________________________________
maxp1 (MaxPooling2D)         (None, 123, 6, 16)        0         
_________________________________________________________________
conv3 (Conv2D)               (None, 120, 6, 32)        2080      
_________________________________________________________________
conv4 (Conv2D)               (None, 117, 6, 64)        8256      
_________________________________________________________________
maxp2 (MaxPooling2D)         (None, 29, 6, 64)         0         
_________________________________________________________________
conv5 (Conv2D)               (None, 26, 6, 64)         16448     
_________________________________________________________________
conv6 (Conv2D)               (None, 23, 6, 128)        32896     
_________________________________________________________________
maxp3 (MaxPooling2D)         (None, 5, 6, 128)         0         
_________________________________________________________________
conv7 (Conv2D)               (None, 4, 6, 256)         65792     
_________________________________________________________________
conv8 (Conv2D)               (None, 3, 6, 256)         131328    
_________________________________________________________________
maxp4 (MaxPooling2D)         (None, 1, 6, 256)         0         
_________________________________________________________________
flat (Flatten)               (None, 1536)              0         
_________________________________________________________________
dropout1 (Dropout)           (None, 1536)              0         
_________________________________________________________________
fc1 (Dense)                  (None, 256)               393472    
_________________________________________________________________
dropout2 (Dropout)           (None, 256)               0         
_________________________________________________________________
fc2 (Dense)                  (None, 3)                 771       
=================================================================
Total params: 651,611
Trainable params: 651,611
Non-trainable params: 0
_________________________________________________________________

Load the trained model weights

In [6]:
model.load_weights('simplecnn.h5')
In [3]:
def load_model():
    global model, graph
    graph = tf.get_default_graph()
    with graph.as_default():
        input_layer = Input(shape=(500, 6, 1), name='input')
        conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
        conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
        maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
        conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
        conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
        maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
        conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
        conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
        maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
        conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
        conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
        maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
        flat = Flatten(name='flat')(maxp4)
        dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
        fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
        dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
        fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)
        model = Model(inputs=[input_layer], outputs=[fc2])
        model.load_weights('simplecnn.h5')

Init shioaji api and login

In [4]:
api = sj.Shioaji(simulation=False)
In [5]:
with open('login.json', 'r') as f:
    kw_login = json.loads(f.read())
api.login(**kw_login)

Init Charts dataset

In [6]:
tick_plot_length = 150
df_norm = pd.read_hdf('normalizer.h5')
norm_min = df_norm.loc['min'].values
norm_range = df_norm.loc['max'].values - norm_min

model_input_tick_num = 500
features_num = 6
np_features = np.zeros((model_input_tick_num, features_num))
pred_data = np.zeros((tick_plot_length, 3))
In [12]:
data_in = np.random.rand(2)
data_inner = np.random.rand(2)
data_outer = np.random.rand(4)

define init charts function

In [13]:
def init_model_chart():
    global model_bar
    x_ord = bq.LinearScale()
    y_sc = bq.LinearScale()

    model_bar = bq.Bars(x=x_tick_index, y=pred_data.T[:, :], 
                  scales={'x': x_ord, 'y': y_sc},
                  colors=['#848484', '#f44242', '#00d100'], type='stacked')
    ax_x = bq.Axis(scale=x_ord)
    ax_y = bq.Axis(scale=y_sc, tick_format='0.2f', orientation='vertical')

    fig = bq.Figure(marks=[model_bar], axes=[ax_x, ax_y],
                   fig_margin= {"top":60, "bottom":30, "left":60, "right":0},)
    fig.layout.height = '200px'
    fig.layout.width = '1200px'
    return fig

define processing data function

In [18]:
def split_data(rec, index):
    sp_rec = {}
    [sp_rec.update({k: v[index] if isinstance(v, (list, tuple)) else v}) 
     for k, v in rec.items()]
    sp_rec['idx'] = index
    return sp_rec

def proc_model_data(quote_msg):
    global np_features, pred_data
    global model, graph
    quote_msgs = [split_data(quote_msg, ind) for ind, _ in enumerate(quote_msg['Close'])]
    for msg in quote_msgs:
        if msg.get('TickType', 0) == 1:
            ask_vol, bid_vol = (msg.get('Volume', 0), 0)
        elif msg.get('TickType', 0) == 2:
            ask_vol, bid_vol = (0, msg.get('Volume', 0))
        else:
            ask_vol, bid_vol = (0, 0)
        np_features[:-1] = np_features[1:]
        np_features[-1] = [msg.get('Close', 0), msg.get('DiffPrice', 0), 
                           msg.get('Volume', 0), msg.get('TargetKindPrice', 0),
                           ask_vol, bid_vol,]
        input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
        pred_data[:-1] = pred_data[1:]
        with graph.as_default():
            model.load_weights('simplecnn.h5')
            pred_data[-1] = model.predict(input_x)[0]

define update chart function

In [23]:
def update_model_chart():
    global model_bar
    with model_bar.hold_sync():
        model_bar.x = x_tick_index.copy()
        model_bar.y = pred_data.T.copy()

define on quote callback

In [27]:
load_model()
@sj.on_quote
def quote_callback(topic, quote_msg):
    global tick_plot_length
    global line_chart, scatter_chart
    global pie_outer, pie_inner, pie_in
    global x_tick_index, y_price, y_vol, updown_color, askbid_color_data
    global new_index, new_deal_price, new_vol, new_updown_color, new_askbid_color_data
    global ask_price, bid_price
    global large_ask_deal_volsum, small_ask_deal_volsum, large_bid_deal_volsum
    global small_bid_deal_volsum, ask_deal_count, bid_deal_count
    global ask_bid_static
    global bar_bidask, bar_bidask_diff
    global x_bar_data, y_bar_data, color_bar_data, y_data_diff
    global ohlc_chart, df_min
    global model, graph, init_model
    vol_threshold = 10
    #print(topic, quote_msg)
    if topic.startswith('Q') and 'TXFD9' in topic:
        proc_ask_bid_bardata(topic, quote_msg)
        update_barchart(x_bar_data, y_bar_data, 
                        color_bar_data, y_bar_data_diff)
        
    elif topic.startswith('L') and 'TXFD9' in topic:
        proc_tick_chartdata(topic, quote_msg)
        proc_ask_bid_staticdata(topic, quote_msg, vol_threshold)
        proc_ohlc_data(quote_msg, new_deal_price)
        proc_model_data(quote_msg)
        update_ohlc_chart()
        update_tickandpie_chart(update_freq=1)
        update_model_chart()
WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.

define event callback

In [28]:
@sj.on_event
def event_callback(resp_code, event_code, event):
    print("Respone Code: {} | Event Code: {} | Event: {}".format(resp_code, event_code, event))

set callback function

In [29]:
api.quote.set_callback(quote_callback)
api.quote.set_event_callback(event_callback)

Chart

In [30]:
VBox([init_model_chart(),
      HBox([init_ohlc_chart(), init_tickchart(),]),
      HBox([pies_chart(), bidask_bar_chart(), ]),
     ])

gif

Subscribe Quote

In [31]:
TXFR1 = api.Contracts.Futures.TXF.TXF201904
TSE2330 = api.Contracts.Stocks.TSE.TSE2330
In [32]:
api.quote.subscribe(TXFR1)
api.quote.subscribe(TXFR1, quote_type='bidask')
api.quote.subscribe(TSE2330)
api.quote.subscribe(TSE2330, quote_type='bidask')
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok

Place order in quote callback?

  • use global to let Shioaji Object work fine
  • c++ call quote callback will borrow python interpreter only when interact with python
  • when the process back to c++ thread will release the blocking
  • It is not recommended to do that the thing spend too much time in the callback function
In [112]:
@sj.on_quote
def quote_callback(topic, quote_msg):
    global api

CNN model inference performance

In [100]:
%%timeit
input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
28 µs ± 353 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
In [102]:
%%timeit
model.predict(input_x)
2.91 ms ± 98.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

Basic way for simplify the callback job for just push the quote msg to queue

In [114]:
storage_quote = {}
@sj.on_quote
def quote_callback(topic, quote_msg):
    global storage_quote
    if topic not in storage_quote:
        storage_quote[topic] = [quote_msg]
    else:
        storage_quote[topic].append(quote_msg)

api.quote.set_callback(quote_callback)

Use redis for cross instance and cross language to get the quote

In [111]:
! pip install redis msgpack
Requirement already satisfied: redis in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (3.2.1)
Requirement already satisfied: msgpack in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (0.6.1)
In [109]:
import redis
import msgpack
rs = redis.Redis(host='localhost', port=6666)
In [110]:
@sj.on_quote
def quote_callback(topic, quote_msg):
    rs.rpush(topic, msgpack.dumps(quote_msg))
    
api.quote.set_callback(quote_callback)

Some advanced alternative