Shioaji Quote Api with Tensorflow

import json
import datetime as dt
import numpy as np
import pandas as pd
import bqplot as bq
import shioaji as sj
from ipywidgets import HBox, VBox

import tensorflow as tf
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dropout, Dense, Input, Flatten
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import categorical_crossentropy

Build simple convolution neural network model that convolution through time

input_layer = Input(shape=(500, 6, 1), name='input')
conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
flat = Flatten(name='flat')(maxp4)
dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)

The Model Structure

model = Model(inputs=[input_layer], outputs=[fc2])
model.summary()

_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input (InputLayer)           (None, 500, 6, 1)         0         
_________________________________________________________________
conv1 (Conv2D)               (None, 497, 6, 8)         40        
_________________________________________________________________
conv2 (Conv2D)               (None, 494, 6, 16)        528       
_________________________________________________________________
maxp1 (MaxPooling2D)         (None, 123, 6, 16)        0         
_________________________________________________________________
conv3 (Conv2D)               (None, 120, 6, 32)        2080      
_________________________________________________________________
conv4 (Conv2D)               (None, 117, 6, 64)        8256      
_________________________________________________________________
maxp2 (MaxPooling2D)         (None, 29, 6, 64)         0         
_________________________________________________________________
conv5 (Conv2D)               (None, 26, 6, 64)         16448     
_________________________________________________________________
conv6 (Conv2D)               (None, 23, 6, 128)        32896     
_________________________________________________________________
maxp3 (MaxPooling2D)         (None, 5, 6, 128)         0         
_________________________________________________________________
conv7 (Conv2D)               (None, 4, 6, 256)         65792     
_________________________________________________________________
conv8 (Conv2D)               (None, 3, 6, 256)         131328    
_________________________________________________________________
maxp4 (MaxPooling2D)         (None, 1, 6, 256)         0         
_________________________________________________________________
flat (Flatten)               (None, 1536)              0         
_________________________________________________________________
dropout1 (Dropout)           (None, 1536)              0         
_________________________________________________________________
fc1 (Dense)                  (None, 256)               393472    
_________________________________________________________________
dropout2 (Dropout)           (None, 256)               0         
_________________________________________________________________
fc2 (Dense)                  (None, 3)                 771       
=================================================================
Total params: 651,611
Trainable params: 651,611
Non-trainable params: 0
_________________________________________________________________

Load the trained model weights

model.load_weights('simplecnn.h5')

def load_model():
    global model, graph
    graph = tf.get_default_graph()
    with graph.as_default():
        input_layer = Input(shape=(500, 6, 1), name='input')
        conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
        conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
        maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
        conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
        conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
        maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
        conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
        conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
        maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
        conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
        conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
        maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
        flat = Flatten(name='flat')(maxp4)
        dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
        fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
        dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
        fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)
        model = Model(inputs=[input_layer], outputs=[fc2])
        model.load_weights('simplecnn.h5')

Init shioaji api and login

api = sj.Shioaji(simulation=False)

with open('login.json', 'r') as f:
    kw_login = json.loads(f.read())
api.login(**kw_login)

Init Charts dataset

tick_plot_length = 150
df_norm = pd.read_hdf('normalizer.h5')
norm_min = df_norm.loc['min'].values
norm_range = df_norm.loc['max'].values - norm_min

model_input_tick_num = 500
features_num = 6
np_features = np.zeros((model_input_tick_num, features_num))
pred_data = np.zeros((tick_plot_length, 3))

data_in = np.random.rand(2)
data_inner = np.random.rand(2)
data_outer = np.random.rand(4)

define init charts function

def init_model_chart():
    global model_bar
    x_ord = bq.LinearScale()
    y_sc = bq.LinearScale()

    model_bar = bq.Bars(x=x_tick_index, y=pred_data.T[:, :], 
                  scales={'x': x_ord, 'y': y_sc},
                  colors=['#848484', '#f44242', '#00d100'], type='stacked')
    ax_x = bq.Axis(scale=x_ord)
    ax_y = bq.Axis(scale=y_sc, tick_format='0.2f', orientation='vertical')

    fig = bq.Figure(marks=[model_bar], axes=[ax_x, ax_y],
                   fig_margin= {"top":60, "bottom":30, "left":60, "right":0},)
    fig.layout.height = '200px'
    fig.layout.width = '1200px'
    return fig

define processing data function

def split_data(rec, index):
    sp_rec = {}
    [sp_rec.update({k: v[index] if isinstance(v, (list, tuple)) else v}) 
     for k, v in rec.items()]
    sp_rec['idx'] = index
    return sp_rec

def proc_model_data(quote_msg):
    global np_features, pred_data
    global model, graph
    quote_msgs = [split_data(quote_msg, ind) for ind, _ in enumerate(quote_msg['Close'])]
    for msg in quote_msgs:
        if msg.get('TickType', 0) == 1:
            ask_vol, bid_vol = (msg.get('Volume', 0), 0)
        elif msg.get('TickType', 0) == 2:
            ask_vol, bid_vol = (0, msg.get('Volume', 0))
        else:
            ask_vol, bid_vol = (0, 0)
        np_features[:-1] = np_features[1:]
        np_features[-1] = [msg.get('Close', 0), msg.get('DiffPrice', 0), 
                           msg.get('Volume', 0), msg.get('TargetKindPrice', 0),
                           ask_vol, bid_vol,]
        input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
        pred_data[:-1] = pred_data[1:]
        with graph.as_default():
            model.load_weights('simplecnn.h5')
            pred_data[-1] = model.predict(input_x)[0]

define update chart function

def update_model_chart():
    global model_bar
    with model_bar.hold_sync():
        model_bar.x = x_tick_index.copy()
        model_bar.y = pred_data.T.copy()

define on quote callback

load_model()
@sj.on_quote
def quote_callback(topic, quote_msg):
    global tick_plot_length
    global line_chart, scatter_chart
    global pie_outer, pie_inner, pie_in
    global x_tick_index, y_price, y_vol, updown_color, askbid_color_data
    global new_index, new_deal_price, new_vol, new_updown_color, new_askbid_color_data
    global ask_price, bid_price
    global large_ask_deal_volsum, small_ask_deal_volsum, large_bid_deal_volsum
    global small_bid_deal_volsum, ask_deal_count, bid_deal_count
    global ask_bid_static
    global bar_bidask, bar_bidask_diff
    global x_bar_data, y_bar_data, color_bar_data, y_data_diff
    global ohlc_chart, df_min
    global model, graph, init_model
    vol_threshold = 10
    #print(topic, quote_msg)
    if topic.startswith('Q') and 'TXFD9' in topic:
        proc_ask_bid_bardata(topic, quote_msg)
        update_barchart(x_bar_data, y_bar_data, 
                        color_bar_data, y_bar_data_diff)
        
    elif topic.startswith('L') and 'TXFD9' in topic:
        proc_tick_chartdata(topic, quote_msg)
        proc_ask_bid_staticdata(topic, quote_msg, vol_threshold)
        proc_ohlc_data(quote_msg, new_deal_price)
        proc_model_data(quote_msg)
        update_ohlc_chart()
        update_tickandpie_chart(update_freq=1)
        update_model_chart()

WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.

define event callback

@sj.on_event
def event_callback(resp_code, event_code, event):
    print("Respone Code: {} | Event Code: {} | Event: {}".format(resp_code, event_code, event))

set callback function

api.quote.set_callback(quote_callback)
api.quote.set_event_callback(event_callback)

Chart

VBox([init_model_chart(),
      HBox([init_ohlc_chart(), init_tickchart(),]),
      HBox([pies_chart(), bidask_bar_chart(), ]),
     ])

Subscribe Quote

TXFR1 = api.Contracts.Futures.TXF.TXF201904
TSE2330 = api.Contracts.Stocks.TSE.TSE2330

api.quote.subscribe(TXFR1)
api.quote.subscribe(TXFR1, quote_type='bidask')
api.quote.subscribe(TSE2330)
api.quote.subscribe(TSE2330, quote_type='bidask')

Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok

Place order in quote callback?

• use global to let Shioaji Object work fine
• c++ call quote callback will borrow python interpreter only when interact with python
• when the process back to c++ thread will release the blocking
• It is not recommended to do that the thing spend too much time in the callback function

@sj.on_quote
def quote_callback(topic, quote_msg):
    global api

CNN model inference performance

%%timeit
input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]

28 µs ± 353 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

%%timeit
model.predict(input_x)

2.91 ms ± 98.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

Basic way for simplify the callback job for just push the quote msg to queue

storage_quote = {}
@sj.on_quote
def quote_callback(topic, quote_msg):
    global storage_quote
    if topic not in storage_quote:
        storage_quote[topic] = [quote_msg]
    else:
        storage_quote[topic].append(quote_msg)

api.quote.set_callback(quote_callback)

Use redis for cross instance and cross language to get the quote

! pip install redis msgpack

Requirement already satisfied: redis in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (3.2.1)
Requirement already satisfied: msgpack in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (0.6.1)

import redis
import msgpack
rs = redis.Redis(host='localhost', port=6666)

@sj.on_quote
def quote_callback(topic, quote_msg):
    rs.rpush(topic, msgpack.dumps(quote_msg))
    
api.quote.set_callback(quote_callback)

Some advanced alternative

• Arrow
• GCP DataFlow
• Spark Streaming
• Flink
• Hive