import json
import datetime as dt
import numpy as np
import pandas as pd
import bqplot as bq
import shioaji as sj
from ipywidgets import HBox, VBox
import tensorflow as tf
from tensorflow.keras.layers import Conv2D, MaxPool2D, Dropout, Dense, Input, Flatten
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import categorical_crossentropy
input_layer = Input(shape=(500, 6, 1), name='input')
conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
flat = Flatten(name='flat')(maxp4)
dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)
model = Model(inputs=[input_layer], outputs=[fc2])
model.summary()
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= input (InputLayer) (None, 500, 6, 1) 0 _________________________________________________________________ conv1 (Conv2D) (None, 497, 6, 8) 40 _________________________________________________________________ conv2 (Conv2D) (None, 494, 6, 16) 528 _________________________________________________________________ maxp1 (MaxPooling2D) (None, 123, 6, 16) 0 _________________________________________________________________ conv3 (Conv2D) (None, 120, 6, 32) 2080 _________________________________________________________________ conv4 (Conv2D) (None, 117, 6, 64) 8256 _________________________________________________________________ maxp2 (MaxPooling2D) (None, 29, 6, 64) 0 _________________________________________________________________ conv5 (Conv2D) (None, 26, 6, 64) 16448 _________________________________________________________________ conv6 (Conv2D) (None, 23, 6, 128) 32896 _________________________________________________________________ maxp3 (MaxPooling2D) (None, 5, 6, 128) 0 _________________________________________________________________ conv7 (Conv2D) (None, 4, 6, 256) 65792 _________________________________________________________________ conv8 (Conv2D) (None, 3, 6, 256) 131328 _________________________________________________________________ maxp4 (MaxPooling2D) (None, 1, 6, 256) 0 _________________________________________________________________ flat (Flatten) (None, 1536) 0 _________________________________________________________________ dropout1 (Dropout) (None, 1536) 0 _________________________________________________________________ fc1 (Dense) (None, 256) 393472 _________________________________________________________________ dropout2 (Dropout) (None, 256) 0 _________________________________________________________________ fc2 (Dense) (None, 3) 771 ================================================================= Total params: 651,611 Trainable params: 651,611 Non-trainable params: 0 _________________________________________________________________
model.load_weights('simplecnn.h5')
def load_model():
global model, graph
graph = tf.get_default_graph()
with graph.as_default():
input_layer = Input(shape=(500, 6, 1), name='input')
conv1 = Conv2D(filters=8, kernel_size=(4, 1), activation='relu', name='conv1')(input_layer)
conv2 = Conv2D(filters=16, kernel_size=(4, 1), activation='relu', name='conv2')(conv1)
maxp1 = MaxPool2D(pool_size=(4, 1), name='maxp1')(conv2)
conv3 = Conv2D(filters=32, kernel_size=(4, 1), activation='relu', name='conv3')(maxp1)
conv4 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv4')(conv3)
maxp2 = MaxPool2D(pool_size=(4, 1), name='maxp2')(conv4)
conv5 = Conv2D(filters=64, kernel_size=(4, 1), activation='relu', name='conv5')(maxp2)
conv6 = Conv2D(filters=128, kernel_size=(4, 1), activation='relu', name='conv6')(conv5)
maxp3 = MaxPool2D(pool_size=(4, 1), name='maxp3')(conv6)
conv7 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv7')(maxp3)
conv8 = Conv2D(filters=256, kernel_size=(2, 1), activation='relu', name='conv8')(conv7)
maxp4 = MaxPool2D(pool_size=(2, 1), name='maxp4')(conv8)
flat = Flatten(name='flat')(maxp4)
dropout1 = Dropout(rate=0.4, name='dropout1')(flat)
fc1 = Dense(256, activation='relu', name='fc1')(dropout1)
dropout2 = Dropout(rate=0.4, name='dropout2')(fc1)
fc2 = Dense(3, activation='softmax', name='fc2')(dropout2)
model = Model(inputs=[input_layer], outputs=[fc2])
model.load_weights('simplecnn.h5')
api = sj.Shioaji(simulation=False)
with open('login.json', 'r') as f:
kw_login = json.loads(f.read())
api.login(**kw_login)
tick_plot_length = 150
df_norm = pd.read_hdf('normalizer.h5')
norm_min = df_norm.loc['min'].values
norm_range = df_norm.loc['max'].values - norm_min
model_input_tick_num = 500
features_num = 6
np_features = np.zeros((model_input_tick_num, features_num))
pred_data = np.zeros((tick_plot_length, 3))
data_in = np.random.rand(2)
data_inner = np.random.rand(2)
data_outer = np.random.rand(4)
def init_model_chart():
global model_bar
x_ord = bq.LinearScale()
y_sc = bq.LinearScale()
model_bar = bq.Bars(x=x_tick_index, y=pred_data.T[:, :],
scales={'x': x_ord, 'y': y_sc},
colors=['#848484', '#f44242', '#00d100'], type='stacked')
ax_x = bq.Axis(scale=x_ord)
ax_y = bq.Axis(scale=y_sc, tick_format='0.2f', orientation='vertical')
fig = bq.Figure(marks=[model_bar], axes=[ax_x, ax_y],
fig_margin= {"top":60, "bottom":30, "left":60, "right":0},)
fig.layout.height = '200px'
fig.layout.width = '1200px'
return fig
def split_data(rec, index):
sp_rec = {}
[sp_rec.update({k: v[index] if isinstance(v, (list, tuple)) else v})
for k, v in rec.items()]
sp_rec['idx'] = index
return sp_rec
def proc_model_data(quote_msg):
global np_features, pred_data
global model, graph
quote_msgs = [split_data(quote_msg, ind) for ind, _ in enumerate(quote_msg['Close'])]
for msg in quote_msgs:
if msg.get('TickType', 0) == 1:
ask_vol, bid_vol = (msg.get('Volume', 0), 0)
elif msg.get('TickType', 0) == 2:
ask_vol, bid_vol = (0, msg.get('Volume', 0))
else:
ask_vol, bid_vol = (0, 0)
np_features[:-1] = np_features[1:]
np_features[-1] = [msg.get('Close', 0), msg.get('DiffPrice', 0),
msg.get('Volume', 0), msg.get('TargetKindPrice', 0),
ask_vol, bid_vol,]
input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
pred_data[:-1] = pred_data[1:]
with graph.as_default():
model.load_weights('simplecnn.h5')
pred_data[-1] = model.predict(input_x)[0]
def update_model_chart():
global model_bar
with model_bar.hold_sync():
model_bar.x = x_tick_index.copy()
model_bar.y = pred_data.T.copy()
load_model()
@sj.on_quote
def quote_callback(topic, quote_msg):
global tick_plot_length
global line_chart, scatter_chart
global pie_outer, pie_inner, pie_in
global x_tick_index, y_price, y_vol, updown_color, askbid_color_data
global new_index, new_deal_price, new_vol, new_updown_color, new_askbid_color_data
global ask_price, bid_price
global large_ask_deal_volsum, small_ask_deal_volsum, large_bid_deal_volsum
global small_bid_deal_volsum, ask_deal_count, bid_deal_count
global ask_bid_static
global bar_bidask, bar_bidask_diff
global x_bar_data, y_bar_data, color_bar_data, y_data_diff
global ohlc_chart, df_min
global model, graph, init_model
vol_threshold = 10
#print(topic, quote_msg)
if topic.startswith('Q') and 'TXFD9' in topic:
proc_ask_bid_bardata(topic, quote_msg)
update_barchart(x_bar_data, y_bar_data,
color_bar_data, y_bar_data_diff)
elif topic.startswith('L') and 'TXFD9' in topic:
proc_tick_chartdata(topic, quote_msg)
proc_ask_bid_staticdata(topic, quote_msg, vol_threshold)
proc_ohlc_data(quote_msg, new_deal_price)
proc_model_data(quote_msg)
update_ohlc_chart()
update_tickandpie_chart(update_freq=1)
update_model_chart()
WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer. WARNING:tensorflow:From /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
@sj.on_event
def event_callback(resp_code, event_code, event):
print("Respone Code: {} | Event Code: {} | Event: {}".format(resp_code, event_code, event))
api.quote.set_callback(quote_callback)
api.quote.set_event_callback(event_callback)
VBox([init_model_chart(),
HBox([init_ohlc_chart(), init_tickchart(),]),
HBox([pies_chart(), bidask_bar_chart(), ]),
])
TXFR1 = api.Contracts.Futures.TXF.TXF201904
TSE2330 = api.Contracts.Stocks.TSE.TSE2330
api.quote.subscribe(TXFR1)
api.quote.subscribe(TXFR1, quote_type='bidask')
api.quote.subscribe(TSE2330)
api.quote.subscribe(TSE2330, quote_type='bidask')
Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok Respone Code: 200 | Event Code: 16 | Event: Subscribe or Unsubscribe ok
@sj.on_quote
def quote_callback(topic, quote_msg):
global api
%%timeit
input_x = ((np_features - norm_min) / norm_range)[np.newaxis,:,:,np.newaxis]
28 µs ± 353 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)
%%timeit
model.predict(input_x)
2.91 ms ± 98.8 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
storage_quote = {}
@sj.on_quote
def quote_callback(topic, quote_msg):
global storage_quote
if topic not in storage_quote:
storage_quote[topic] = [quote_msg]
else:
storage_quote[topic].append(quote_msg)
api.quote.set_callback(quote_callback)
! pip install redis msgpack
Requirement already satisfied: redis in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (3.2.1) Requirement already satisfied: msgpack in /Users/Tanni/.pyenv/versions/miniconda3-latest/lib/python3.7/site-packages (0.6.1)
import redis
import msgpack
rs = redis.Redis(host='localhost', port=6666)
@sj.on_quote
def quote_callback(topic, quote_msg):
rs.rpush(topic, msgpack.dumps(quote_msg))
api.quote.set_callback(quote_callback)