In [3]:
import pandas as pd
import numpy as np
from tqdm import tqdm
import gc
import matplotlib.pyplot as plt
import talib as ta
import datetime as dt
pd.set_option('display.max_rows', 16)
import statsmodels.api as sm
In [4]:
plt.rcParams['figure.figsize'] = (16.0, 9.0)
In [5]:
ta.__version__
Out[5]:
Data¶
In [6]:
START = '2007-01-01'
END = '2025-04-30'
In [7]:
index_info = DataAPI.SecIDGet(assetClass="IDX",pandas="1")
In [8]:
index_id = index_info[index_info['secShortName'].isin(['上证综指','深证综指','创业板指','沪深300','中证500','中证1000'])].drop_duplicates('secShortName').secID.values
index_df = DataAPI.MktIdxdGet(indexID=index_id,beginDate=START,endDate=END,field=['indexID','secShortName','tradeDate','openIndex','closeIndex','turnoverVol','turnoverValue','CHGPct'],pandas="1")
In [9]:
index_df
Out[9]:
In [10]:
# Security Id
stk_info = DataAPI.SecIDGet(assetClass="E",pandas="1")
cond1 = (stk_info['exchangeCD'] == 'XSHE') | (stk_info['exchangeCD'] == 'XSHG')
cond2 = (stk_info['listStatusCD'] == 'L') | (stk_info['listStatusCD'] == 'DE')
cond3 = stk_info['transCurrCD']=='CNY'
stk_info = stk_info[cond1 & cond2 & cond3].copy()
stk_id = stk_info['secID']
# ST
st_df = DataAPI.SecSTGet(beginDate=START,endDate=END,secID=stk_id,field=['secID','tradeDate','STflg'],pandas="1")
st_df['tradeDate'] = pd.to_datetime(st_df['tradeDate'],format="%Y-%m-%d")
In [11]:
# %%time
# # About 8 mins
# # # 从优矿下载股票信息，时间较长。由于优矿的限制，每次下载3年的数据
# stk_dict = {}
# begin_ = dt.datetime.strptime(START, '%Y-%m-%d').year
# end_ = dt.datetime.strptime(START, '%Y-%m-%d').year+3
# field = ["secID","tradeDate",'preClosePrice',"closePrice",'openPrice',"negMarketValue","turnoverValue",'turnoverRate']
# while begin_ <= 2025:
# if begin_ == 2025:
# yesterday = dt.datetime.today() - dt.timedelta(days=1)
# yesterday.strftime('%Y%m%d')
# stk_dict[begin_] = DataAPI.MktEqudAdjAfGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=yesterday,
# field=field,pandas="1")
# else:
# stk_dict[begin_] = DataAPI.MktEqudAdjAfGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=f'{end_}1231',
# field=field,pandas="1")
# begin_ = end_ + 1
# end_ = begin_ + 3
# for i in range(len(stk_dict)):
# stk_df = pd.DataFrame(np.vstack([_df for _df in stk_dict.values()]),columns=field)
# stk_df.to_pickle('./data/stk_df.pkl')
In [12]:
# %%time
# stk_df = DataAPI.MktEqudAdjAfGet(secID=stk_id,beginDate=START,endDate=END,isOpen=1,
# field=["secID","tradeDate",
# 'preClosePrice',"closePrice",
# 'openPrice','highestPrice','lowestPrice',
# "negMarketValue",
# "turnoverValue",'turnoverRate'],pandas="1")
# stk_df.to_pickle('./data/stk_df.pkl')
# Takes about 6 mins
In [13]:
stk_df = pd.read_pickle('./data/stk_df.pkl')
In [14]:
stk_df.info()
In [15]:
num_cols = ['preClosePrice','closePrice','openPrice',
'negMarketValue','turnoverValue','turnoverRate']
In [16]:
for col in num_cols:
stk_df[col] = pd.to_numeric(stk_df[col])
In [17]:
stk_df['tradeDate'] = pd.to_datetime(stk_df['tradeDate'], format='%Y-%m-%d')
stk_df.sort_values(['secID','tradeDate'],inplace=True)
# drop ST stocks
print(stk_df.shape)
stk_df = pd.merge(stk_df, st_df, on=['secID','tradeDate'],how='left')
stk_df = stk_df[stk_df['STflg'].isna()].copy()
stk_df.drop('STflg',axis=1,inplace=True)
print(stk_df.shape)
不填充停牌值比较合理，因为技术分析只看量价，直接计算量价关系较为合适
沪深300¶
In [18]:
hs300_df = index_df[index_df['secShortName']=='沪深300'].reset_index(drop=True)
hs300_df.rename(columns={'CHGPct':'close_ret'},inplace=True)
hs300_df['open_ret'] = hs300_df['openIndex']/hs300_df['openIndex'].shift()-1
hs300_df
Out[18]:
In [19]:
hs300_df.info()
In [20]:
# 过滤市场趋势
hs300_df['close_ret_demean'] = hs300_df['close_ret'] - hs300_df['close_ret'].mean()
hs300_df['open_ret_demean'] = hs300_df['open_ret'] - hs300_df['open_ret'].mean()
In [21]:
(hs300_df['close_ret']- hs300_df['close_ret_demean']).describe()
Out[21]:
In [22]:
hs300_cols = hs300_df.columns
Technical indicators¶
Moving average¶
MA30¶
In [23]:
ta.SMA?
In [24]:
ta.SMA(hs300_df['closeIndex'], 5)
Out[24]:
In [25]:
hs300_df['closeIndex'].rolling(5).mean()
Out[25]:
In [26]:
MA_df = hs300_df.copy()
MA_df['MA30'] = ta.SMA(MA_df['closeIndex'], 30)
In [27]:
MA_df[['tradeDate','closeIndex','MA30']].set_index('tradeDate').loc[:'2007-12'].plot()
Out[27]:
- 当收盘价高于均线时买入，低于均线时卖出
Case 1:
- t日：收盘价高于MA30，假设可以以当日收盘价买入
- t+k日：收盘价低于MA30，假设可以以当日收盘价卖出
Case 2:
- t日：收盘价高于MA30，假设不能以当日收盘价买入。以次日开盘价买入
- t+k日：收盘价低于MA30，假设不能以当日收盘价卖出。以次日开盘价卖出
In [28]:
MA_df['signal'] = np.nan
MA_df.loc[MA_df['closeIndex'] > MA_df['MA30'], 'signal'] = 1
MA_df.loc[MA_df['closeIndex'] < MA_df['MA30'], 'signal'] = 0
In [29]:
MA_df[~MA_df['signal'].isna()]
Out[29]:
In [30]:
MA_df['position_close'] = MA_df['signal'] # 第一天收盘价生成signal后，立即对应于第一天收盘价形成的头寸
MA_df['position_open'] = MA_df['signal'].shift() # 第一天用收盘价生成signal。对应于第二天的开盘价形成的头寸
In [31]:
# 再次滞后的原因：position变换的时候实现的收益
MA_df['position_close_ret'] = MA_df['position_close'].shift() * MA_df['close_ret']
MA_df['position_open_ret'] = MA_df['position_open'].shift() * MA_df['open_ret']
MA_df['position_close_ret_demean'] = MA_df['position_close'].shift() * MA_df['close_ret_demean']
MA_df['position_open_ret_demean'] = MA_df['position_open'].shift() * MA_df['open_ret_demean']
In [32]:
MA_df['MA30_close_cumret'] = (MA_df['position_close_ret']+1).cumprod()
MA_df['MA30_open_cumret'] = (MA_df['position_open_ret']+1).cumprod()
In [33]:
MA_df['signal'].unique()
Out[33]:
In [34]:
MA_df[MA_df['signal']==0]
Out[34]:
In [35]:
MA_df.loc[96:99]
Out[35]:
In [36]:
3511.43/3803.95 - 1
Out[36]:
In [37]:
3407.00 / 3804.96 - 1
Out[37]:
In [38]:
## Example
temp = MA_df.loc[99:113,['tradeDate','openIndex','closeIndex','close_ret','open_ret','signal','MA30',
'position_close','position_open','position_close_ret','position_open_ret',
'position_close_ret_demean','position_open_ret_demean']].copy()
temp['MA30_close_cumret'] = (temp['position_close_ret']+1).cumprod()
temp['MA30_open_cumret'] = (temp['position_open_ret']+1).cumprod()
display(temp)
# close
print(3877.59 / 3802.30)
# open
print(3804.41 / 3814.19)
In [39]:
MA30_ret_df = MA_df[['tradeDate','openIndex','closeIndex','open_ret','close_ret','MA30',
'signal','position_close','position_open','position_close_ret','position_open_ret',
'position_close_ret_demean','position_open_ret_demean',
'MA30_close_cumret','MA30_open_cumret']].copy()
In [40]:
MA30_ret_df.set_index('tradeDate',inplace=True)
In [41]:
# Close price cumret
fig, axes = plt.subplots(3,1)
MA30_ret_df[['closeIndex','MA30']].plot(ax=axes[0],grid=True)
MA30_ret_df[['position_close']].plot(ax=axes[1],grid=True)
MA30_ret_df[['MA30_close_cumret']].plot(ax=axes[2],grid=True)
Out[41]:
In [42]:
# open price cumret
fig, axes = plt.subplots(3,1)
MA30_ret_df[['openIndex','MA30']].plot(ax=axes[0],grid=True)
MA30_ret_df[['position_open']].plot(ax=axes[1], grid=True)
MA30_ret_df[['MA30_open_cumret']].plot(ax=axes[2], grid=True)
Out[42]:
MA20¶
In [43]:
hs300_df
Out[43]:
In [44]:
MA_df = hs300_df.copy()
In [45]:
ma_length = 20
MA_df[f'MA{ma_length}'] = ta.SMA(MA_df['closeIndex'], ma_length)
MA_df['signal'] = 0
ndays = MA_df.shape[0]
MA_df.loc[MA_df['closeIndex'] > MA_df[f'MA{ma_length}'], 'signal'] = 1
MA_df['open_ret'] = MA_df['openIndex']/MA_df['openIndex'].shift()-1
MA_df['position_close'] = MA_df['signal']
MA_df['position_open'] = MA_df['signal'].shift()
MA_df.rename(columns={'CHGPct':'close_ret'},inplace=True)
MA_df['position_close'] = MA_df['signal']
MA_df['position_close_ret'] = MA_df['position_close'].shift() * MA_df['close_ret']
MA_df['position_open_ret'] = MA_df['position_open'].shift() * MA_df['open_ret']
MA_df['position_close_ret_demean'] = MA_df['position_close'].shift() * MA_df['close_ret_demean']
MA_df['position_open_ret_demean'] = MA_df['position_open'].shift() * MA_df['open_ret_demean']
MA_df[f'MA{ma_length}_close_cumret'] = (MA_df['position_close_ret']+1).cumprod()
MA_df[f'MA{ma_length}_open_cumret'] = (MA_df['position_open_ret']+1).cumprod()
MA_ret_df = MA_df[['tradeDate','openIndex','closeIndex','open_ret','close_ret',f'MA{ma_length}',
'signal','position_close','position_open','position_close_ret','position_open_ret',
'position_close_ret_demean','position_open_ret_demean',
f'MA{ma_length}_close_cumret',f'MA{ma_length}_open_cumret']].copy()
MA_ret_df.set_index('tradeDate',inplace=True)
# Close price cumret
fig, axes = plt.subplots(3,1)
MA_ret_df[['closeIndex',f'MA{ma_length}']].plot(ax=axes[0],grid=True)
MA_ret_df[['position_close']].plot(ax=axes[1],grid=True)
MA_ret_df[[f'MA{ma_length}_close_cumret']].plot(ax=axes[2],grid=True)
Out[45]:
In [46]:
# open price cumret
fig, axes = plt.subplots(3,1)
MA_ret_df[['openIndex',f'MA{ma_length}']].plot(ax=axes[0],grid=True)
MA_ret_df[['position_open']].plot(ax=axes[1],grid=True)
MA_ret_df[[f'MA{ma_length}_open_cumret']].plot(ax=axes[2],grid=True)
Out[46]:
Exponential moving average¶
In [47]:
ta.EMA?
\begin{aligned}
E M A_{\text {Today }}=&\left(\text { Value }_{\text {Today }} *\left(\frac{\text { Smoothing }}{1+\text { Days }}\right)\right) \\
&+E M A_{\text {Yesterday }} *\left(1-\left(\frac{\text { Smoothing }}{1+\text { Days }}\right)\right)
\end{aligned}
Smoothing = 2
EMA20¶
In [48]:
ema_length = 20
EMA_df = hs300_df.copy()
EMA_df['EMA'] = ta.EMA(EMA_df['closeIndex'], ema_length)
In [49]:
EMA_df[['tradeDate','closeIndex','EMA']].set_index('tradeDate').plot()
Out[49]:
In [50]:
EMA_df['EMA'] = ta.EMA(EMA_df['closeIndex'], ema_length)
EMA_df['signal'] = 0
EMA_df.loc[EMA_df['closeIndex'] > EMA_df['EMA'], 'signal'] = 1
EMA_df['open_ret'] = EMA_df['openIndex']/EMA_df['openIndex'].shift()-1
EMA_df['position_close'] = EMA_df['signal']
EMA_df['position_open'] = EMA_df['signal'].shift()
EMA_df.rename(columns={'CHGPct':'close_ret'},inplace=True)
EMA_df['position_close_ret'] = EMA_df['position_close'].shift() * EMA_df['close_ret']
EMA_df['position_open_ret'] = EMA_df['position_open'].shift() * EMA_df['open_ret']
EMA_df['position_close_ret_demean'] = EMA_df['position_close'].shift() * EMA_df['close_ret_demean']
EMA_df['position_open_ret_demean'] = EMA_df['position_open'].shift() * EMA_df['open_ret_demean']
EMA_df['EMA_close_cumret'] = (EMA_df['position_close_ret']+1).cumprod()
EMA_df['EMA_open_cumret'] = (EMA_df['position_open_ret']+1).cumprod()
EMA_ret_df = EMA_df[['tradeDate','openIndex','closeIndex','open_ret','close_ret','EMA',
'signal','position_close','position_open','position_close_ret','position_open_ret',
'position_close_ret_demean','position_open_ret_demean',
'EMA_close_cumret','EMA_open_cumret']].copy()
EMA_ret_df.set_index('tradeDate',inplace=True)
# open price cumret
fig, axes = plt.subplots(3,1)
EMA_ret_df[['openIndex','EMA']].plot(ax=axes[0], grid=True)
EMA_ret_df[['position_open']].plot(ax=axes[1], grid=True)
EMA_ret_df[['EMA_open_cumret']].plot(ax=axes[2], grid=True)
Out[50]:
In [51]:
EMA_ret_df
Out[51]:
Statistical Inferences¶
Cross-sectional test¶
In [52]:
def rule_return(df, demean=True, open_ret=True):
"""
df should contain these columns:
signal: the signal generated by the rule
close_ret: return calculated by close price
open_ret: return calculated by open price
close_ret_demean is demeaned return of close_ret, i.e. close_ret - close_ret.mean.
open_ret_demean is similarly defined. The use of demeaned return series is to adjust the
bias created by bullish or bearish markets.
"""
df['close_ret_demean'] = df['close_ret'] - df['close_ret'].mean()
df['open_ret_demean'] = df['open_ret'] - df['open_ret'].mean()
df['position_close'] = df['signal']
df['position_open'] = df['signal'].shift()
df['position_close_ret'] = df['position_close'].shift() * df['close_ret']
df['position_open_ret'] = df['position_open'].shift() * df['open_ret']
df['position_close_ret_demean'] = df['position_close'].shift() * df['close_ret_demean']
df['position_open_ret_demean'] = df['position_open'].shift() * df['open_ret_demean']
df['close_cumret'] = (df['position_close_ret']+1).cumprod()
df['open_cumret'] = (df['position_open_ret']+1).cumprod()
if open_ret:
if demean:
return pd.DataFrame({'tradeDate':df['tradeDate'].values,
'position_open_ret_demean':df['position_open_ret_demean'].values,
'open_cumret':df['open_cumret'].values})
else:
return pd.DataFrame({'tradeDate':df['tradeDate'].values,
'position_open_ret':df['position_open_ret'].values,
'open_cumret':df['open_cumret'].values})
else:
if demean:
return pd.DataFrame({'tradeDate':df['tradeDate'].values,
'position_close_ret_demean':df['position_close_ret_demean'].values,
'close_cumret':df['close_cumret'].values})
else:
return pd.DataFrame({'tradeDate':df['tradeDate'].values,
'position_close_ret':df['position_close_ret'].values,
'close_cumret':df['close_cumret'].values})
In [53]:
ema_length = 20
In [54]:
stk_df
Out[54]:
In [55]:
stk_df['EMA'] = stk_df.groupby('secID')['closePrice'].apply(ta.EMA, 20)
In [56]:
stk_df.drop(stk_df.loc[stk_df['openPrice']==0].index, inplace=True) # drop 停牌但有收盘价的数据
In [57]:
stk_df.loc[stk_df['openPrice']==0]
Out[57]:
In [58]:
stk_df['open_ret'] = stk_df.groupby('secID')['openPrice'].apply(lambda x: x / x.shift() - 1)
In [59]:
stk_df['close_ret'] = stk_df['closePrice']/stk_df['preClosePrice'] - 1
In [60]:
stk_df['signal'] = 0
stk_df.loc[stk_df['closePrice'] > stk_df['EMA'], 'signal'] = 1
In [61]:
stk_df
Out[61]:
In [62]:
%%time
rule_ret_df = stk_df.groupby('secID').apply(rule_return)
In [63]:
rule_ret_df.reset_index(inplace=True)
In [64]:
rule_ret_df.drop('level_1',axis=1,inplace=True)
In [65]:
rule_ret_df
Out[65]:
Cross-sectional test of cumulative return¶
In [66]:
rule_cumret_by_crs = rule_ret_df.groupby('secID')['open_cumret'].last()
In [67]:
rule_cumret_by_crs.describe()
Out[67]:
In [68]:
rule_cumret_by_crs.hist(bins=200)
Out[68]:
In [69]:
rule_cumret_by_crs.dropna(inplace=True)
y = rule_cumret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y-const, const).fit().get_robustcov_results(cov_type='HC0')
mean_values = reg.params[0]
t_values = reg.tvalues[0]
pd.DataFrame([mean_values,t_values],index=['ret_mean','t_values'],columns=['rule_cumret'])
Out[69]:
In [70]:
rule_cumret_by_crs.dropna(inplace=True)
y = rule_cumret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit()
print(reg.t_test('const = 1'))
平均年化收益：
In [71]:
# # 1.12/(2022-2007+1)
# 2.12**(1/(2022-2007+1)) - 1
In [72]:
2.1069**(1/(2024-2007+1)) - 1
Out[72]:
这个统计检验不够好：
- 累积收益率可能在区间最后一段短时间内冲高
- 累积收益率右侧可以无限大，左侧有界(0)。日收益率原则上也是这样，但没有累积收益率这么严重
In [73]:
rule_ret_df.loc[rule_ret_df['tradeDate']>='2018-01-01']
Out[73]:
In [74]:
rule_cumret_by_crs = rule_ret_df.loc[rule_ret_df['tradeDate']>='2022-01-01'].groupby('secID')['open_cumret'].last()
rule_cumret_by_crs.dropna(inplace=True)
y = rule_cumret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y-const, const).fit().get_robustcov_results(cov_type='HC0')
mean_values = reg.params[0]
t_values = reg.tvalues[0]
pd.DataFrame([mean_values,t_values],index=['ret_mean','t_values'],columns=['rule_cumret'])
Out[74]:
Cross-sectional test of mean daily return¶
In [75]:
# time-series mean of daily return
rule_tsmean_ret_by_crs = rule_ret_df.groupby('secID')['position_open_ret_demean'].mean()
rule_tsmean_ret_by_crs
Out[75]:
In [76]:
temp = stk_df[stk_df['secID']==np.random.choice(stk_df['secID'].unique(),1)[0]].copy()
temp['signal'] = 0
temp.loc[temp['closePrice'] > temp['EMA'], 'signal'] = 1
display(temp)
rule_return(temp)['position_open_ret_demean'].mean()
Out[76]:
In [77]:
rule_tsmean_ret_by_crs['002976.XSHE']
Out[77]:
In [78]:
rule_tsmean_ret_by_crs.dropna(inplace=True)
y = rule_tsmean_ret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HC0')
mean_values = reg.params[0]
t_values = reg.tvalues[0]
In [79]:
pd.DataFrame([mean_values,t_values],index=['ret_mean','t_values'],columns=['rule_daily_ret'])
Out[79]:
Time series test. Bootstrapping: White's Reality Check¶
- 原始index return去掉均值的目的：过滤掉牛市或者熊市时，随机策略的正或负的收益的偏误
- position return 去掉均值的目的：bootstrap 的原假设是：策略期望收益是0。以此生成抽样分布。
In [80]:
EMA_ret_df
Out[80]:
In [81]:
rule_ret_series = EMA_ret_df['position_open_ret_demean'].dropna() # position_open_ret_demean: (raw return demeaned)*position
rule_ret_series_for_bootstrap = rule_ret_series - rule_ret_series.mean() # demean here: H0: the average of the rule's return is zero
n_sample = rule_ret_series.shape[0]
n_boostrap = 1000
In [82]:
n_sample
Out[82]:
In [83]:
rule_ret_series.mean()
Out[83]:
In [84]:
rule_ret_mean_distr = []
for i in range(n_boostrap):
rule_ret_mean_distr.append(np.random.choice(rule_ret_series_for_bootstrap, n_sample).mean())
In [85]:
rule_ret_mean_distr = pd.Series(rule_ret_mean_distr)
rule_ret_mean_distr.hist(bins=50)
Out[85]:
In [86]:
(rule_ret_mean_distr > rule_ret_series.mean()).sum()
Out[86]:
In [87]:
(rule_ret_mean_distr > rule_ret_series.mean()).sum() / n_boostrap
Out[87]:
In [88]:
np.mean(rule_ret_series)
Out[88]:
In [89]:
def white_reality_test(rule_ret, n_boostrap=1000):
n_sample = len(rule_ret)
if n_sample < 100:
return None
else:
mean_rule_ret = np.mean(rule_ret)
rule_ret_for_bootstrap = rule_ret - mean_rule_ret
rule_ret_mean_distr = []
for i in range(n_boostrap):
rule_ret_mean_distr.append(np.random.choice(rule_ret_for_bootstrap, n_sample).mean())
rule_ret_mean_distr = pd.Series(rule_ret_mean_distr)
pvalue = (rule_ret_mean_distr > mean_rule_ret).sum() / n_boostrap
return pvalue
In [90]:
# The p value of the rule's return series
white_reality_test(rule_ret_series)
Out[90]:
Test another stock¶
In [91]:
one_stk_id = np.random.choice(rule_ret_df['secID'].unique(),1)[0]
In [92]:
rule_ret_series = rule_ret_df.loc[rule_ret_df['secID'] == one_stk_id,'position_open_ret_demean']
In [93]:
rule_ret_series.plot()
Out[93]:
In [94]:
rule_ret_series.dropna(inplace=True)
In [95]:
white_reality_test(rule_ret_series)
Out[95]:
Test 300 stocks¶
In [96]:
stk_id_300 = np.random.choice(rule_ret_df['secID'].unique(),300,replace=False)
In [97]:
temp = rule_ret_df.loc[rule_ret_df['secID'].isin(stk_id_300)].copy()
In [98]:
temp.dropna(inplace=True)
In [99]:
temp
Out[99]:
In [100]:
%%time
stk_white_p = temp.groupby('secID')['position_open_ret_demean'].apply(white_reality_test)
In [101]:
stk_white_p
Out[101]:
In [102]:
stk_white_p.describe()
Out[102]:
In [103]:
stk_white_p.hist(bins=50)
Out[103]:
In [104]:
stk_white_p.loc[stk_white_p < 0.10]
Out[104]:
In [105]:
good_EMA_stks = stk_white_p.loc[stk_white_p < 0.10].index
In [106]:
rule_ret_df.loc[rule_ret_df['secID']=='600086.XSHG',['tradeDate','open_cumret']].set_index('tradeDate').plot()
Out[106]:
In [107]:
rule_ret_df.loc[rule_ret_df['secID'].isin(good_EMA_stks[0:5])].set_index('tradeDate',inplace=True)
In [108]:
temp2 = rule_ret_df.loc[rule_ret_df['secID'].isin(good_EMA_stks)].copy()
In [109]:
temp2.pivot(index='tradeDate',columns='secID',values='open_cumret').plot()
Out[109]:
MACD¶
Moving Average Convergence Divergence (MACD)
$$MACD = EMA_{\text{fast period}} - EMA_{\text{slow period}}$$（也叫DIF）
$$E M A_t(\text{Value})= \left(\text { Value }_t *\left(\frac{\text { Smoothing }}{1+\text { Days }}\right)\right) + E M A_{t-1} *\left(1-\left(\frac{\text { Smoothing }}{1+\text { Days }}\right)\right)$$$Smoothing = 2$, $\text{fast period} = 12$, $\text{slow period} = 26$. 第一个$EMA$用简单平均。比如，计算$EMA_{\text{12 period}}$，那么就等有12个观测值之后，取简单平均得到$EMA_1$，然后$EMA_2$用上述公式计算。
按照级数展开可发现，越靠近当前的价格，权重越大。Smoothing越大，越靠近当前价格的权重越大。
得到 MACD 以后，再计算 MACD 的 Signal (也叫DEA)，
$$Signal = EMA_{\text{9 period}}(MACD) $$判断标准：当 MACD 上穿 Signal 时，处于上升趋势。当 MACD 下穿 Signal 时，处于下降趋势。
MACD 的核心逻辑是：
趋势反转时，短期均线（如12日EMA）会比长期均线（如26日EMA）更快地反应价格变化；
（MACD）DIF 上穿（Signal） DEA（“金叉”）被视为买入信号；
（MACD）DIF 下穿（Signal）DEA（“死叉”）被视为卖出信号；
In [110]:
MACD_df = hs300_df.copy()
In [111]:
fastperiod = 12
slowperiod = 26
signalperiod = 9
In [112]:
MACD_df
Out[112]:
In [113]:
MACD_df['MACD'], MACD_df['MACD_signal'], _ = ta.MACD(MACD_df['closeIndex'], fastperiod=fastperiod, slowperiod=slowperiod, signalperiod=signalperiod)
In [114]:
MACD_df.loc[MACD_df['tradeDate']<='2007-12-31',['MACD','MACD_signal']].plot()
Out[114]:
In [115]:
MACD_df['signal'] = 0
MACD_df.loc[MACD_df['MACD'] > MACD_df['MACD_signal'], 'signal'] = 1
MACD_df['open_ret'] = MACD_df['openIndex']/MACD_df['openIndex'].shift()-1
MACD_df['position_close'] = MACD_df['signal']
MACD_df['position_open'] = MACD_df['signal'].shift()
MACD_df.rename(columns={'CHGPct':'close_ret'},inplace=True)
MACD_df['position_close_ret'] = MACD_df['position_close'].shift() * MACD_df['close_ret']
MACD_df['position_open_ret'] = MACD_df['position_open'].shift() * MACD_df['open_ret']
MACD_df['position_close_ret_demean'] = MACD_df['position_close'].shift() * MACD_df['close_ret_demean']
MACD_df['position_open_ret_demean'] = MACD_df['position_open'].shift() * MACD_df['open_ret_demean']
MACD_df['MACD_close_cumret'] = (MACD_df['position_close_ret']+1).cumprod()
MACD_df['MACD_open_cumret'] = (MACD_df['position_open_ret']+1).cumprod()
MACD_ret_df = MACD_df[['tradeDate','openIndex','closeIndex','open_ret','close_ret','MACD','MACD_signal',
'signal','position_close','position_open','position_close_ret','position_open_ret',
'position_close_ret_demean','position_open_ret_demean',
'MACD_close_cumret','MACD_open_cumret']].copy()
MACD_ret_df.set_index('tradeDate',inplace=True)
# open price cumret
fig, axes = plt.subplots(3,1)
MACD_ret_df[['MACD','MACD_signal']].plot(ax=axes[0], grid=True)
MACD_ret_df[['openIndex']].plot(secondary_y=True,ax=axes[0],grid=True)
MACD_ret_df[['position_open']].plot(ax=axes[1], grid=True)
MACD_ret_df[['MACD_open_cumret']].plot(ax=axes[2], grid=True)
Out[115]:
In [116]:
fig, axes = plt.subplots(3,1)
MACD_ret_df.loc[:'2009',['MACD','MACD_signal']].plot(ax=axes[0], grid=True)
MACD_ret_df.loc[:'2009',['openIndex']].plot(secondary_y=True,ax=axes[0],grid=True)
MACD_ret_df.loc[:'2009',['position_open']].plot(ax=axes[1], grid=True)
MACD_ret_df.loc[:'2009',['MACD_open_cumret']].plot(ax=axes[2], grid=True)
Out[116]:
Cross-sectional test¶
In [117]:
stk_df
Out[117]:
In [118]:
stk_df.drop('EMA',axis=1,inplace=True)
In [119]:
stk_df['MACD'] = stk_df.groupby('secID')['closePrice'].apply(lambda x: ta.MACD(x)[0])
stk_df['MACD_signal'] = stk_df.groupby('secID')['closePrice'].apply(lambda x: ta.MACD(x)[1])
In [120]:
stk_df.loc[stk_df['secID']=='000001.XSHE',['tradeDate','MACD','MACD_signal']].set_index('tradeDate').loc['2020':].plot()
Out[120]:
In [121]:
stk_df['signal'] = 0
stk_df.loc[stk_df['MACD'] > stk_df['MACD_signal'], 'signal'] = 1
In [122]:
stk_df
Out[122]:
In [123]:
%%time
rule_ret_df = stk_df.groupby('secID').apply(rule_return)
In [124]:
rule_ret_df.reset_index(inplace=True)
In [125]:
rule_ret_df.drop('level_1',axis=1,inplace=True)
In [126]:
rule_ret_df
Out[126]:
Cross-sectional test of cumulative return¶
In [127]:
rule_cumret_by_crs = rule_ret_df.groupby('secID')['open_cumret'].last()
In [128]:
rule_cumret_by_crs
Out[128]:
In [129]:
rule_cumret_by_crs.describe()
Out[129]:
In [130]:
rule_cumret_by_crs.hist(bins=75)
Out[130]:
In [131]:
rule_cumret_by_crs.dropna(inplace=True)
y = rule_cumret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y-const, const).fit().get_robustcov_results(cov_type='HC0')
mean_values = reg.params[0]
t_values = reg.tvalues[0]
pd.DataFrame([mean_values,t_values],index=['ret_mean','t_values'],columns=['rule_cumret'])
Out[131]:
In [132]:
rule_cumret_by_crs.dropna(inplace=True)
y = rule_cumret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit()
print(reg.t_test('const = 1'))
平均年化收益：
In [133]:
# 2.1293**(1/(2022-2007+1))-1
In [134]:
1.9272**(1/(2023-2007+1)) - 1
Out[134]:
In [135]:
2.1518**(1/(2024-2007+1)) - 1
Out[135]:
Cross-sectional test of mean daily return¶
In [136]:
# time-series mean of daily return
rule_tsmean_ret_by_crs = rule_ret_df.groupby('secID')['position_open_ret_demean'].mean()
rule_tsmean_ret_by_crs
Out[136]:
In [137]:
rule_tsmean_ret_by_crs.dropna(inplace=True)
y = rule_tsmean_ret_by_crs.values
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HC0')
mean_values = reg.params[0]
t_values = reg.tvalues[0]
In [138]:
pd.DataFrame([mean_values,t_values],index=['ret_mean','t_values'],columns=['rule_daily_ret'])
Out[138]:
Time series test. Bootstrapping: White's Reality Check¶
- 原始index return去掉均值的目的：过滤掉牛市或者熊市时，随机策略的正或负的收益的偏误
- position return 去掉均值的目的：bootstrap 的原假设是：策略期望收益是0。以此生成抽样分布。
In [139]:
rule_ret_series = MACD_ret_df['position_open_ret_demean'].dropna() # position_open_ret_demean: (raw return demeaned)*position
rule_ret_series_for_bootstrap = rule_ret_series - rule_ret_series.mean() # demean here: H0: the average of the rule's return is zero
n_sample = rule_ret_series.shape[0]
n_boostrap = 1000
In [140]:
n_sample
Out[140]:
In [141]:
rule_ret_series.mean()
Out[141]:
In [142]:
rule_ret_mean_distr = []
for i in range(n_boostrap):
rule_ret_mean_distr.append(np.random.choice(rule_ret_series_for_bootstrap, n_sample).mean())
In [143]:
rule_ret_mean_distr = pd.Series(rule_ret_mean_distr)
rule_ret_mean_distr.hist(bins=50)
Out[143]:
In [144]:
(rule_ret_mean_distr > rule_ret_series.mean()).sum() / n_boostrap
Out[144]:
In [145]:
np.mean(rule_ret_series)
Out[145]:
In [146]:
def white_reality_test(rule_ret, n_boostrap=1000):
n_sample = len(rule_ret)
if n_sample < 100:
return None
else:
mean_rule_ret = np.mean(rule_ret)
rule_ret_for_bootstrap = rule_ret - mean_rule_ret
rule_ret_mean_distr = []
for i in range(n_boostrap):
rule_ret_mean_distr.append(np.random.choice(rule_ret_for_bootstrap, n_sample).mean())
rule_ret_mean_distr = pd.Series(rule_ret_mean_distr)
pvalue = (rule_ret_mean_distr > mean_rule_ret).sum() / n_boostrap
return pvalue
In [147]:
# The p value of the rule's return series
white_reality_test(rule_ret_series)
Out[147]:
Compare this when there are multiple strategies
Test another stock¶
In [148]:
one_stk_id = np.random.choice(rule_ret_df['secID'].unique(),1)[0]
In [149]:
rule_ret_series = rule_ret_df.loc[rule_ret_df['secID'] == one_stk_id,'position_open_ret_demean']
In [150]:
rule_ret_series.dropna(inplace=True)
In [151]:
white_reality_test(rule_ret_series)
Out[151]:
Test 300 stocks¶
In [152]:
stk_id_300 = np.random.choice(rule_ret_df['secID'].unique(),300,replace=False)
In [153]:
temp = rule_ret_df.loc[rule_ret_df['secID'].isin(stk_id_300)].copy()
In [154]:
temp.dropna(inplace=True)
In [155]:
temp
Out[155]:
In [156]:
%%time
stk_white_p = temp.groupby('secID')['position_open_ret_demean'].apply(white_reality_test)
In [157]:
stk_white_p
Out[157]:
In [158]:
stk_white_p.describe()
Out[158]:
In [159]:
stk_white_p.hist(bins=50)
Out[159]:
In [160]:
stk_white_p.loc[stk_white_p < 0.10]
Out[160]:
In [161]:
good_MACD_stks = stk_white_p.loc[stk_white_p < 0.10].index
In [162]:
rule_ret_df.loc[rule_ret_df['secID']=='600146.XSHG','open_cumret']
Out[162]:
In [163]:
rule_ret_df.loc[rule_ret_df['secID'].isin(good_MACD_stks[0:5])].set_index('tradeDate',inplace=True)
In [164]:
temp2 = rule_ret_df.loc[rule_ret_df['secID'].isin(good_MACD_stks)].copy()
In [165]:
temp2.pivot(index='tradeDate',columns='secID',values='open_cumret').plot()
Out[165]:
White's Reality Check for multiple strategies¶
In [166]:
def rule_return_open(df, demean=True):
"""
df should contain these columns:
signal: the signal generated by the rule
close_ret: return calculated by close price
open_ret: return calculated by open price
close_ret_demean is demeaned return of close_ret, i.e. close_ret - close_ret.mean.
open_ret_demean is similarly defined. The use of demeaned return series is to adjust the
bias created by bullish or bearish markets.
"""
df['position_open'] = df['signal'].shift()
df['position_open_ret'] = df['position_open'].shift() * df['open_ret']
df['position_open_ret_demean'] = df['position_open'].shift() * df['open_ret_demean']
if demean==True:
return df['position_open_ret_demean']
else:
return df['position_open_ret']
In [167]:
cols = ['tradeDate','openIndex','closeIndex','open_ret','close_ret','open_ret_demean','close_ret_demean']
ta_rules_df = hs300_df[cols].copy()
In [168]:
# MA
ma_params = [30, 20, 10]
for ma_param in ma_params:
ta_rules_df[f'MA{ma_param}'] = ta.SMA(ta_rules_df['closeIndex'], ma_param)
ta_rules_df['signal'] = np.nan
ta_rules_df.loc[ta_rules_df['closeIndex'] > ta_rules_df[f'MA{ma_param}'], 'signal'] = 1
ta_rules_df.loc[ta_rules_df['closeIndex'] < ta_rules_df[f'MA{ma_param}'], 'signal'] = 0
ta_rules_df[f'ret_MA{ma_param}'] = rule_return_open(ta_rules_df)
ta_rules_df = ta_rules_df[cols+[f'ret_MA{ma_param}' for ma_param in ma_params]].copy()
In [169]:
ta_rules_df
Out[169]:
In [170]:
cols = ta_rules_df.columns.tolist()
In [171]:
#EMA
ema_params = [30, 20, 10]
for ema_param in ema_params:
ta_rules_df[f'EMA{ema_param}'] = ta.EMA(ta_rules_df['closeIndex'], ema_param)
ta_rules_df['signal'] = np.nan
ta_rules_df.loc[ta_rules_df['closeIndex'] > ta_rules_df[f'EMA{ema_param}'], 'signal'] = 1
ta_rules_df.loc[ta_rules_df['closeIndex'] < ta_rules_df[f'EMA{ema_param}'], 'signal'] = 0
ta_rules_df[f'ret_EMA{ema_param}'] = rule_return_open(ta_rules_df)
ta_rules_df = ta_rules_df[cols+[f'ret_EMA{ema_param}' for ema_param in ema_params]].copy()
In [172]:
ta_rules_df
Out[172]:
In [173]:
cols = ta_rules_df.columns.tolist()
In [174]:
# MACD
macd_models = {'MACD1': {'fastperiod':12, 'slowperiod':26, 'signalperiod':9},
'MACD2': {'fastperiod':10, 'slowperiod':20, 'signalperiod':5}}
for macd, param in macd_models.items():
ta_rules_df[macd], ta_rules_df[f'{macd}_signal'], _ = ta.MACD(ta_rules_df['closeIndex'], fastperiod=param['fastperiod'], slowperiod=param['slowperiod'], signalperiod=param['signalperiod'])
ta_rules_df['signal'] = 0
ta_rules_df.loc[ta_rules_df[macd] > ta_rules_df[f'{macd}_signal'], 'signal'] = 1
ta_rules_df[f'ret_{macd}'] = rule_return_open(ta_rules_df)
In [175]:
ta_rules_df.dropna(inplace=True)
ta_rules_df.reset_index(inplace=True,drop=True)
In [176]:
ta_rules_df
Out[176]:
In [177]:
multi_rule_cols = ta_rules_df.columns[ta_rules_df.columns.str.startswith('ret')]
multi_rule_ret_df = ta_rules_df[multi_rule_cols].copy()
multi_rule_ret_df
Out[177]:
In [178]:
def multi_white_reality_test(target_rule, multi_rule_ret_df, n_bootstrap=500):
"""
target_rule: the name of the rule under consideration. target_rule should be one column of multi_rule_ret_df.
multi_rule_ret_df: the df of multiple rule returns.
"""
n_sample = multi_rule_ret_df.shape[0]
if n_sample < 100:
return None
else:
max_bs_distr = np.full(shape=n_bootstrap, fill_value=np.nan)
multi_rule_ret_df_mean = multi_rule_ret_df - multi_rule_ret_df.mean()
for i in tqdm(range(n_bootstrap)):
idx = np.random.choice(multi_rule_ret_df.index, n_sample)
max_bs_distr[i] = multi_rule_ret_df_mean.loc[idx].mean().max()
pvalue = (max_bs_distr > multi_rule_ret_df[target_rule].mean()).sum() / n_bootstrap
return pvalue, max_bs_distr
In [179]:
p_MACD1, max_bs_distr_MACD1 = multi_white_reality_test(target_rule='ret_MACD1', multi_rule_ret_df=multi_rule_ret_df)
In [180]:
p_MACD1
Out[180]:
Compare this with the WRC when there is only one