In [1]:
import numpy as np
import pandas as pd
import datetime as dt
pd.set_option('display.max_rows', 16)
import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams['figure.figsize'] = (16.0, 9.0)
import seaborn as sns
import statsmodels.api as sm
from sklearn.linear_model import LinearRegression
import gc
In [2]:
plt.rcParams['figure.figsize'] = (16.0, 9.0)
Data¶
In [3]:
START = '2008-01-01'
END = '2025-04-30'
In [4]:
# Security Id
stk_info = DataAPI.SecIDGet(assetClass="E",pandas="1")
cond1 = (stk_info['exchangeCD'] == 'XSHE') | (stk_info['exchangeCD'] == 'XSHG')
cond2 = (stk_info['listStatusCD'] == 'L') | (stk_info['listStatusCD'] == 'DE')
cond3 = stk_info['transCurrCD']=='CNY'
stk_info = stk_info[cond1 & cond2 & cond3].copy()
stk_id = stk_info['secID']
In [5]:
stk_info
Out[5]:
ST¶
In [6]:
st_df = DataAPI.SecSTGet(beginDate=START,endDate=END,secID=stk_id,field=['secID','tradeDate','STflg'],pandas="1")
In [7]:
st_df['tradeDate'] = pd.to_datetime(st_df['tradeDate'],format="%Y-%m-%d")
Risk free rate¶
In [8]:
DataAPI.MktIborGet(secID="Shibor1M.IRCN",beginDate=START,endDate=END,field=['secID','tradeDate','rate'],pandas="1")
Out[8]:
In [9]:
shibor_df = DataAPI.MktIborGet(secID="Shibor1M.IRCN",beginDate=START,endDate=END,field=['secID','tradeDate','rate'],pandas="1")
shibor_df['rate'] = shibor_df['rate']*0.01/12
shibor_df['tradeDate'] = pd.to_datetime(shibor_df['tradeDate'])
shibor_df.drop('secID',axis=1,inplace=True)
shibor_df.rename(columns={'rate':'rf'},inplace=True)
shibor_df['ym'] = shibor_df['tradeDate'].dt.to_period('M')
shibor_df.sort_values('tradeDate',inplace=True)
shibor_df_m = shibor_df.groupby('ym',as_index=False).last()
shibor_df_m.drop('tradeDate',axis=1,inplace=True)
shibor_df_m
Out[9]:
In [10]:
shibor_df_m.set_index('ym').plot()
Out[10]:
Beta¶
In [11]:
# %%time
# # About 4 min 41s
# # # 从优矿下载 beta，时间较长。由于优矿的限制，每次下载3年的数据
# beta = {}
# begin_ = 2008
# end_ = 2010
# while begin_ <= 2025:
# if begin_ == 2025:
# yesterday = dt.datetime.today() - dt.timedelta(days=1)
# yesterday.strftime('%Y%m%d')
# beta[begin_] = DataAPI.MktStockFactorsDateRangeProGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=yesterday,
# field=['secID','tradeDate','Beta252'],pandas="1")
# else:
# beta[begin_] = DataAPI.MktStockFactorsDateRangeProGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=f'{end_}1231',
# field=['secID','tradeDate','Beta252'],pandas="1")
# begin_ = end_ + 1
# end_ = begin_ + 3
# for i in range(len(beta)):
# beta_df = pd.DataFrame(np.vstack([_df for _df in beta.values()]),columns=['secID','tradeDate','Beta252'])
In [12]:
# beta_df.to_pickle('./data/beta_df.pkl')
In [13]:
beta_df = pd.read_pickle('./data/beta_df.pkl')
beta_df['tradeDate'] = pd.to_datetime(beta_df['tradeDate'], format="%Y-%m-%d")
beta_df['ym'] = beta_df['tradeDate'].dt.to_period('M')
beta_df['Beta252'] = pd.to_numeric(beta_df['Beta252'])
# Winsorization
# up_q = 0.99999
# lower_q = 0.00001
# beta_df['Beta252_winsor'] = beta_df['Beta252'].clip(lower=beta_df['Beta252'].quantile(lower_q),upper=beta_df['Beta252'].quantile(up_q))
# Monthly
beta_df_m = beta_df.groupby(['secID','ym'],as_index=False)['Beta252'].last()
beta_df_m.rename(columns={'Beta252':'beta'},inplace=True)
beta_df_m
Out[13]:
Trading data¶
In [14]:
# %%time
# # About 12 mins
# # # 从优矿下载股票信息，时间较长。由于优矿的限制，每次下载3年的数据
# stk_dict = {}
# begin_ = dt.datetime.strptime(START, '%Y-%m-%d').year
# end_ = dt.datetime.strptime(START, '%Y-%m-%d').year+3
# field = ['secID','tradeDate','preClosePrice','closePrice','negMarketValue','turnoverValue','turnoverRate']
# while begin_ <= 2025:
# if begin_ == 2025:
# yesterday = dt.datetime.today() - dt.timedelta(days=1)
# yesterday.strftime('%Y%m%d')
# stk_dict[begin_] = DataAPI.MktEqudAdjAfGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=yesterday,
# field=field,pandas="1")
# else:
# stk_dict[begin_] = DataAPI.MktEqudAdjAfGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=f'{end_}1231',
# field=field,pandas="1")
# begin_ = end_ + 1
# end_ = begin_ + 3
# for i in range(len(stk_dict)):
# stk_df = pd.DataFrame(np.vstack([_df for _df in stk_dict.values()]),columns=field)
# stk_df.to_pickle('./data/stk_df.pkl')
In [15]:
# stk_df
In [16]:
stk_df = pd.read_pickle('./data/stk_df.pkl')
stk_df['tradeDate'] = pd.to_datetime(stk_df['tradeDate'], format='%Y-%m-%d')
stk_df['ym'] = stk_df['tradeDate'].dt.to_period('M')
stk_df.sort_values(['secID','tradeDate'],inplace=True)
# drop ST stocks
print(stk_df.shape)
stk_df = pd.merge(stk_df, st_df, on=['secID','tradeDate'],how='left')
stk_df = stk_df[stk_df['STflg'].isna()].copy()
stk_df.drop('STflg',axis=1,inplace=True)
print(stk_df.shape)
In [17]:
# Monthly
stk_df_m = stk_df.groupby(['secID','ym'],as_index=False).last()
stk_df_m
Out[17]:
新股上市后首月交易不考虑¶
需要考虑2008年后上市的股票。因为数据从2008开始，有一些在此之前上市的股票无需处理。
思路：找到每只股票的上市日期。用最早的数据时间跟其比较
In [18]:
stk_info
Out[18]:
In [19]:
list_date = stk_info[['secID','listDate']].copy()
list_date['listDate'] = pd.to_datetime(list_date['listDate'],format="%Y-%m-%d")
list_date['listMonth'] = list_date['listDate'].dt.to_period('M')
# list_date
stk_df_m = pd.merge(stk_df_m,list_date[['secID','listMonth']],on='secID')
stk_df_m
Out[19]:
In [20]:
# 随机检查
stk_df_m[stk_df_m['secID']==np.random.choice(stk_df_m['secID'],1)[0]]
Out[20]:
In [21]:
stk_listMonth = stk_df_m.groupby('secID')[['ym','listMonth']].first()
stk_listMonth['listm_dist'] = stk_listMonth['ym'] - stk_listMonth['listMonth']
# pandas 版本太久
stk_listMonth['listm_dist'] = stk_listMonth['listm_dist'].apply(lambda x: x.n)
In [22]:
stk_listMonth['listm_dist'].value_counts()
Out[22]:
In [23]:
# # 上市日期在样本内的新上市股票
# stk_listMonth.loc[stk_listMonth['listm_dist'] < 1]
# newlist_secID = stk_listMonth.loc[stk_listMonth['listm_dist'] < 1].index
# newlist_secID
# stk_df_m.loc[stk_df_m['secID'].isin(newlist_secID)]
# temp = stk_df_m.loc[stk_df_m['secID'].isin(newlist_secID)].copy()
# temp['ret'] = temp.groupby('secID',as_index=False)['closePrice'].apply(lambda x: x / x.shift() - 1).values
# temp.dropna().groupby('secID')['ret'].first().mean()
In [24]:
stk_listMonth
Out[24]:
In [25]:
stk_df_m_inclIPO = stk_df_m.copy()
In [26]:
# 上市日期在样本内的新上市股票
stk_listMonth.loc[stk_listMonth['listm_dist'] < 1]
newlist_secID = stk_listMonth.loc[stk_listMonth['listm_dist'] < 1].index
newlist_secID
# 每只股票丢掉第一个观测值
temp = stk_df_m.loc[stk_df_m['secID'].isin(newlist_secID)].groupby('secID').apply(lambda x: x.iloc[1:]).drop('secID',axis=1).reset_index().drop('level_1',axis=1)
stk_df_m = pd.concat([stk_df_m.loc[~stk_df_m['secID'].isin(newlist_secID)],temp],axis=0).reset_index(drop=True)
# stk_df_m.drop('listMonth',axis=1,inplace=True)
stk_df_m = stk_df_m.sort_values(['secID', 'ym']).reset_index(drop=True)
stk_df_m
Out[26]:
In [27]:
stk_df_m.info()
In [28]:
# 随机检查
stk_df_m[stk_df_m['secID']==np.random.choice(stk_df_m['secID'],1)[0]]
Out[28]:
In [29]:
# stk_df_m.drop('listMonth',axis=1,inplace=True)
Momentum¶
Momentum 介绍：
Momentum即动量，指的是过去一段时间表现较好的股票，在未来一段时间内也会较好。“过去”和“未来”有很多定义，比较常见的：
- 过去：t月前的t-12:t-2个月的累积收益率
- 未来: t月的收益
也即，中间跳过了一个月，t-1。 这是因为，短期发现了反转，reversal：上个月表现好的，这个月表现差
Momentum 的计算涉及到pandas groupby rolling。如果是自定义的函数，apply会比较慢。但新版本(pandas version >= 1)中，apply中可以指定用numba作为计算引擎，速度会快非常多。由于优矿的pandas版本很低, 没有这个选项。
另外，按照月来做rolling时，pandas rolling的选项不可以用"MonthEnd"这样长度不固定的时间作为window size。因此，如果想做得很精确的话，需要用一些其他的办法。一种比较容易的思路是把停牌的日期(用MultiIndex)填上，对应的ret值留空。窗口长度就可以固定（通过指定observation个数，而不是月份数）。
- 注意：应当先计算收益率，再填充空值。原因：
- 如果先填充空值，刚恢复交易时的第一个月，ret会是NaN。若用0填充来得到1+ret==1，会有一定程度的失真。
- 先计算ret，则刚恢复交易时的第一个月，ret是从刚停牌时的价格和这个月的价格相除计算得到的，较真实。
例：
In [30]:
stk_df.loc[(stk_df['secID']=='000001.XSHE') & (stk_df['tradeDate']>='2010-06-20') & (stk_df['tradeDate']<='2010-07-10')]
Out[30]:
In [31]:
stk_df[(stk_df['secID']=='000007.XSHE') & (stk_df['tradeDate']>='2021-04-20')&(stk_df['tradeDate']<='2022-07-05')]
Out[31]:
In [32]:
stk_df_m.groupby('secID',as_index=False)['closePrice'].apply(lambda x: x / x.shift() - 1)
Out[32]:
In [33]:
stk_df_m
Out[33]:
In [34]:
stk_df_m.shape
Out[34]:
In [35]:
stk_df_m.groupby('secID',as_index=False)['closePrice'].apply(lambda x: x / x.shift() - 1).reset_index()['closePrice']
Out[35]:
In [36]:
stk_df_m['ret_mom'] = stk_df_m.groupby('secID',as_index=False)['closePrice'].apply(lambda x: x / x.shift() - 1).values #这个ret_mom不用作后面ret的计算，后面仍保留monthly ret
In [37]:
stk_df_m[(stk_df_m['secID']=='000007.XSHE') & (stk_df_m['tradeDate']>='2021-04-20')&(stk_df_m['tradeDate']<='2022-09-05')]
Out[37]:
In [38]:
stk_df_m.sort_values(['secID','ym'],inplace=True)
In [39]:
stk_df_m
Out[39]:
In [40]:
stk_df_m['1+ret_mom'] = stk_df_m['ret_mom'] + 1
Fill na months¶
In [41]:
def fill_missing(df, full_dates, id_col='secID', date_col='ym'):
"""
This function fills the missing dates for stocks.
Parameters:
df: The dataframe. Could be a sub-dataframe created by "groupby".
The dataframe must be sorted on the "date_col".
full_dates: the unique dates covering all securities in the full dataframe.
Need to be sorted.
id_col: the security id.
date_col: the dates column for the security
Returns:
A dataframe with the missing dates filled with NA.
"""
one_stk_id = df[id_col].unique()
date_start = np.where(full_dates == df[date_col].min())[0][0]
date_end = np.where(full_dates == df[date_col].max())[0][0]
dates = full_dates[date_start:date_end+1]
idx = pd.MultiIndex.from_product([one_stk_id,dates],
names=(id_col,date_col))
df = df.set_index([id_col,date_col]).reindex(idx).reset_index()
return df
In [42]:
%%time
full_dates = np.sort(stk_df['ym'].unique())
stk_df_m = stk_df_m.groupby('secID').apply(fill_missing, full_dates=full_dates)
stk_df_m.reset_index(drop=True, inplace=True)
In [43]:
stk_df_m.loc[stk_df_m['1+ret_mom'].isna()]
Out[43]:
In [44]:
stk_df_m.loc[(stk_df_m['secID']=='000001.XSHE') & (stk_df_m['ym']>='2010-06') & (stk_df_m['ym']<='2010-11')]
Out[44]:
In [45]:
stk_df_m[(stk_df_m['secID']=='000007.XSHE')&(stk_df_m['ym']>='2021-02')]
Out[45]:
In [46]:
stk_df_m[(stk_df_m['secID']=='000007.XSHE')&(stk_df_m['ym']>='2021-02')&(~stk_df_m['tradeDate'].isna())]
Out[46]:
In [47]:
stk_df_m.loc[stk_df_m['1+ret_mom'].isna(),'1+ret_mom'] = 1 # 缺失位置填充为1，以便连乘。
In [48]:
stk_df_m['mom'] = stk_df_m.groupby('secID').rolling(11,min_periods=11)['1+ret_mom'].apply(np.prod, raw=True).values - 1
stk_df_m['mom_6m'] = stk_df_m.groupby('secID').rolling(6,min_periods=6)['1+ret_mom'].apply(np.prod, raw=True).values - 1
# 当只用numpy function时可以选raw=True，只用ndarray格式的数据，运算速度快很多。
In [49]:
stk_df_m
Out[49]:
In [50]:
stk_df_m['ret'] = stk_df_m.groupby('secID')['closePrice'].apply(lambda x: x / x.shift() - 1)
In [51]:
stk_df_m['size'] = np.log(stk_df_m['negMarketValue'])
stk_df_m.drop(['tradeDate','closePrice'],axis=1,inplace=True)
stk_df_m = pd.merge(stk_df_m, shibor_df_m, on='ym')
stk_df_m['exret'] = stk_df_m['ret'] - stk_df_m['rf']
In [52]:
# 把日期对齐。
# 例：
# ret_date == 2020.03
# size_date == 2020.02
# cumret_date == 2020.01
stk_df_m['exret'] = stk_df_m.groupby(['secID'])['exret'].shift(-1)
stk_df_m['ret_date'] = stk_df_m.groupby('secID')['ym'].shift(-1)
stk_df_m['mom'] = stk_df_m.groupby(['secID'])['mom'].shift()
stk_df_m['mom_6m'] = stk_df_m.groupby(['secID'])['mom_6m'].shift()
stk_df_m['mom_date'] = stk_df_m.groupby('secID')['ym'].shift()
In [53]:
stk_df_m.dropna(inplace=True)
In [54]:
stk_df_m = stk_df_m[['secID','ret_date','exret','mom_date','mom','mom_6m','ym','negMarketValue','size']]
In [55]:
stk_df_m.rename(columns={'negMarketValue':'mktcap'},inplace=True)
In [56]:
stk_df_m.sort_values(['secID','ym'],inplace=True)
In [57]:
stk_df_m
Out[57]:
In [58]:
stk_df_m[['mom','mom_6m']].describe()
Out[58]:
BM¶
In [59]:
# # # 从优矿下载 PB，时间较长。由于优矿的限制，每次下载3年的数据
# # %%time
# pb = {}
# begin_ = 2008
# end_ = 2011
# while begin_ <= 2025:
# if begin_ == 2025:
# yesterday = dt.datetime.today() - dt.timedelta(days=1)
# yesterday.strftime('%Y%m%d')
# pb[begin_] = DataAPI.MktStockFactorsDateRangeProGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=yesterday,
# field=['secID','tradeDate','PB'],pandas="1")
# else:
# pb[begin_] = DataAPI.MktStockFactorsDateRangeProGet(secID=stk_id,
# beginDate=f'{begin_}0101',
# endDate=f'{end_}1231',
# field=['secID','tradeDate','PB'],pandas="1")
# begin_ = end_ + 1
# end_ = begin_ + 3
# for i in range(len(pb)):
# pb_df = pd.DataFrame(np.vstack([_df for _df in pb.values()]),columns=['secID','tradeDate','PB'])
# pb_df.to_pickle('./data/pb_df.pkl')
# # About 5mins
In [60]:
pb_df = pd.read_pickle('./data/pb_df.pkl')
pb_df['tradeDate'] = pd.to_datetime(pb_df['tradeDate'])
pb_df['PB'] = pd.to_numeric(pb_df['PB'])
pb_df['ym'] = pb_df['tradeDate'].dt.to_period('M')
pb_df.sort_values(['secID','tradeDate'],inplace=True)
pb_df = pb_df.groupby(['secID','ym'],as_index=False).last()
pb_df['bm'] = 1 / pb_df['PB']
pb_df.drop(['tradeDate','PB'],axis=1,inplace=True)
pb_df = pb_df[pb_df['bm'] >= 0]
pb_df
Out[60]:
Merge¶
In [61]:
stk_df_m
Out[61]:
In [62]:
beta_df_m
Out[62]:
In [63]:
ret_df = pd.merge(stk_df_m, beta_df_m, on=['secID','ym'],how='left') # beta 的 na 值不管它，不是重点，保留左边的dataframe
In [64]:
ret_df = pd.merge(ret_df, pb_df,on=['secID','ym'] ,how='left')
In [65]:
ret_df
Out[65]:
In [66]:
# ret_df_full = ret_df.copy()
# ret_df = ret_df[ret_df['ret_date']>='2015'].copy()
# ret_df = ret_df_full.copy()
Momentum single sort¶
In [67]:
q = dict()
keys = ['q'+str(i) for i in range(1, 10)]
values = np.arange(0.1, 1.0, 0.1)
q.update(zip(keys,values))
quantile_df = pd.DataFrame()
for key, value in q.items():
quantile_df[key] = ret_df.groupby(['mom_date'])['mom'].quantile(value)
ret_df_q = pd.merge(ret_df, quantile_df, on='mom_date')
In [68]:
# ret_df_q = ret_df_q.loc[ret_df_q['ret_date']<='2024-03'].copy()
In [69]:
ret_df_q
Out[69]:
In [70]:
portfolios = dict()
drop_cols = [col for col in ret_df_q.columns if col[0]=='q']
portfolios['p1'] = ret_df_q.loc[ret_df_q['mom'] <= ret_df_q['q1']].copy().drop(drop_cols, axis=1)
for i in range(2,10):
idx = (ret_df_q[f'q{i-1}'] <= ret_df_q['mom']) & (ret_df_q['mom'] <= ret_df_q[f'q{i}'])
portfolios[f'p{i}'] = ret_df_q.loc[idx].copy().drop(drop_cols, axis=1)
portfolios['p10'] = ret_df_q.loc[ret_df_q['mom'] >= ret_df_q['q9']].copy().drop(drop_cols, axis=1)
portfolios_crs_mean = dict()
for k in portfolios.keys():
portfolios_crs_mean[k] = portfolios[k].groupby(['ret_date'])['exret'].mean()
mean_values = {}
t_values = {}
for k in portfolios_crs_mean.keys():
y = portfolios_crs_mean[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
# Portfolio 10-1
y = portfolios_crs_mean['p10'] - portfolios_crs_mean['p1']
const = np.full(shape=len(y), fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values['p10-p1'] = reg.params[0]
t_values['p10-p1'] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['mean','t-value'],
columns=mean_values.keys())
Out[70]:
In [71]:
q_size = dict()
keys = ['q_size_1']
values = [0.5]
q_size.update(zip(keys,values))
q_mom = dict()
keys = ['q_mom_1','q_mom_2']
values = [0.3, 0.7]
q_mom.update(zip(keys,values))
q_size_df = pd.DataFrame()
for key, value in q_size.items():
q_size_df[key] = ret_df.groupby(['ym'])['size'].quantile(value)
q_mom_df = pd.DataFrame()
for key, value in q_mom.items():
q_mom_df[key] = ret_df.groupby(['mom_date'])['mom'].quantile(value)
ret_df_q = pd.merge(ret_df, q_size_df, on='ym')
ret_df_q = pd.merge(ret_df_q, q_mom_df, on='mom_date')
portfolios_size = dict()
portfolios_size['size1'] = ret_df_q.loc[ret_df_q['size'] <= ret_df_q['q_size_1'],
['secID','ym','ret_date','exret','size','mktcap']]
portfolios_size['size2'] = ret_df_q.loc[ret_df_q['size'] >= ret_df_q['q_size_1'],
['secID','ym','ret_date','exret','size','mktcap']]
portfolios_mom = dict()
portfolios_mom['mom1'] = ret_df_q.loc[ret_df_q['mom'] <= ret_df_q['q_mom_1'],
['secID','ym','ret_date','exret','mom']]
portfolios_mom['mom2'] = ret_df_q.loc[(ret_df_q['mom'] >= ret_df_q['q_mom_1']) & \
(ret_df_q['mom'] <= ret_df_q['q_mom_2']),
['secID','ym','ret_date','exret','mom']]
portfolios_mom['mom3'] = ret_df_q.loc[ret_df_q['mom'] >= ret_df_q['q_mom_2'],
['secID','ym','ret_date','exret','mom']]
portfolios = dict()
for mom_group in portfolios_mom.keys():
for size_group in portfolios_size.keys():
portfolios[f'{mom_group}_{size_group}'] = pd.merge(portfolios_size[size_group],
portfolios_mom[mom_group][['secID','ret_date','mom']],
on=['secID','ret_date'])
mean_portfolios_ret = dict()
for pf in portfolios.keys():
mean_portfolios_ret[pf] = portfolios[pf].groupby('ret_date')['exret'].mean()
print(mean_portfolios_ret[pf].shape) # print 看一下会不会存在某个月份上没有mom和size分组没有任何交叉
# Fast merge by stacking
mean_portfolios_ret_df = pd.DataFrame(np.vstack([pf for pf in mean_portfolios_ret.values()])).T
mean_portfolios_ret_df.columns = mean_portfolios_ret.keys()
mean_portfolios_ret_df.index = mean_portfolios_ret['mom1_size1'].index
# Newey-West adjustment
mean_values = {}
t_values = {}
for k in mean_portfolios_ret.keys():
y = mean_portfolios_ret[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=4)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[71]:
Fama MacBeth regression¶
In [72]:
ret_df['exret100'] = ret_df['exret'] * 100
def fm_reg(df,cols):
df_ = df.dropna()
if df_.shape[0] < 15:
return [None]*(len(cols)+1)
reg = LinearRegression(fit_intercept=True).fit(y=df_.loc[:,'exret100'], X=df_.loc[:,cols])
return np.insert(reg.coef_, 0, reg.intercept_)
In [73]:
ret_df_ = ret_df
# ret_df_ = ret_df.loc[ret_df['ret_date'] <= '2023-12'].copy()
In [74]:
cols = ['mom']
temp = ret_df_.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[74]:
In [75]:
cols = ['beta','size','bm','mom']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[75]:
Reversal¶
In [76]:
ret_df['rev'] = ret_df.groupby('secID')['exret'].shift()
In [77]:
ret_df
Out[77]:
Reversal single sort¶
In [78]:
q = dict()
keys = ['q'+str(i) for i in range(1, 10)]
values = np.arange(0.1, 1.0, 0.1)
q.update(zip(keys,values))
quantile_df = pd.DataFrame()
for key, value in q.items():
quantile_df[key] = ret_df.groupby(['ym'])['rev'].quantile(value)
ret_df_q = pd.merge(ret_df, quantile_df, on='ym')
portfolios = dict()
drop_cols = [col for col in ret_df_q.columns if col[0]=='q']
portfolios['p1'] = ret_df_q.loc[ret_df_q['rev'] <= ret_df_q['q1']].copy().drop(drop_cols, axis=1)
for i in range(2,10):
idx = (ret_df_q[f'q{i-1}'] <= ret_df_q['rev']) & (ret_df_q['rev'] <= ret_df_q[f'q{i}'])
portfolios[f'p{i}'] = ret_df_q.loc[idx].copy().drop(drop_cols, axis=1)
portfolios['p10'] = ret_df_q.loc[ret_df_q['rev'] >= ret_df_q['q9']].copy().drop(drop_cols, axis=1)
portfolios_crs_mean = dict()
for k in portfolios.keys():
portfolios_crs_mean[k] = portfolios[k].groupby(['ret_date'])['exret'].mean()
mean_values = {}
t_values = {}
for k in portfolios_crs_mean.keys():
y = portfolios_crs_mean[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
# Portfolio 10-1
y = portfolios_crs_mean['p10'] - portfolios_crs_mean['p1']
const = np.full(shape=len(y), fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values['p10-p1'] = reg.params[0]
t_values['p10-p1'] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['mean','t-value'],
columns=mean_values.keys())
Out[78]:
Double Sorting on Size and Reversal¶
In [79]:
q_size = dict()
keys = ['q_size_1']
values = [0.5]
q_size.update(zip(keys,values))
q_rev = dict()
keys = ['q_rev_1','q_rev_2']
values = [0.3, 0.7]
q_rev.update(zip(keys,values))
q_size_df = pd.DataFrame()
for key, value in q_size.items():
q_size_df[key] = ret_df.groupby(['ym'])['size'].quantile(value)
q_rev_df = pd.DataFrame()
for key, value in q_rev.items():
q_rev_df[key] = ret_df.groupby(['ym'])['rev'].quantile(value)
ret_df_q = pd.merge(ret_df, q_size_df, on='ym')
ret_df_q = pd.merge(ret_df_q, q_rev_df, on='ym')
portfolios_size = dict()
portfolios_size['size1'] = ret_df_q.loc[ret_df_q['size'] <= ret_df_q['q_size_1'],
['secID','ym','ret_date','exret','size','mktcap']]
portfolios_size['size2'] = ret_df_q.loc[ret_df_q['size'] >= ret_df_q['q_size_1'],
['secID','ym','ret_date','exret','size','mktcap']]
portfolios_rev = dict()
portfolios_rev['rev1'] = ret_df_q.loc[ret_df_q['rev'] <= ret_df_q['q_rev_1'],
['secID','ym','ret_date','exret','rev']]
portfolios_rev['rev2'] = ret_df_q.loc[(ret_df_q['rev'] >= ret_df_q['q_rev_1']) & \
(ret_df_q['rev'] <= ret_df_q['q_rev_2']),
['secID','ym','ret_date','exret','rev']]
portfolios_rev['rev3'] = ret_df_q.loc[ret_df_q['rev'] >= ret_df_q['q_rev_2'],
['secID','ym','ret_date','exret','rev']]
portfolios = dict()
for rev_group in portfolios_rev.keys():
for size_group in portfolios_size.keys():
portfolios[f'{rev_group}_{size_group}'] = pd.merge(portfolios_size[size_group],
portfolios_rev[rev_group][['secID','ret_date','rev']],
on=['secID','ret_date'])
mean_portfolios_ret = dict()
for pf in portfolios.keys():
mean_portfolios_ret[pf] = portfolios[pf].groupby('ret_date')['exret'].mean()
print(mean_portfolios_ret[pf].shape) # print 看一下会不会存在某个月份上没有rev和size分组没有任何交叉
# Fast merge by stacking
mean_portfolios_ret_df = pd.DataFrame(np.vstack([pf for pf in mean_portfolios_ret.values()])).T
mean_portfolios_ret_df.columns = mean_portfolios_ret.keys()
mean_portfolios_ret_df.index = mean_portfolios_ret['rev1_size1'].index
# Newey-West adjustment
mean_values = {}
t_values = {}
for k in mean_portfolios_ret.keys():
y = mean_portfolios_ret[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=4)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[79]:
FM regression on Reversal¶
In [80]:
cols = ['beta','size','bm','mom','rev']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[80]:
Reversal factor¶
In [81]:
portfolios_vwret = {}
for pf in portfolios.keys():
temp = portfolios[pf].groupby('ret_date')['mktcap'].agg({'mktcap_sum':np.sum})
portfolios[pf] = pd.merge(portfolios[pf], temp, on='ret_date')
portfolios[pf]['weight'] = portfolios[pf]['mktcap'] / portfolios[pf]['mktcap_sum']
portfolios[pf]['weighted_exret'] = portfolios[pf]['exret'] * portfolios[pf]['weight']
portfolios_vwret[pf] = portfolios[pf].groupby('ret_date')['weighted_exret'].sum()
portfolios_vwret_df = pd.DataFrame(np.vstack([pf for pf in portfolios_vwret.values()])).T
portfolios_vwret_df.index = portfolios_vwret['rev1_size1'].index
portfolios_vwret_df.columns = portfolios_vwret.keys()
portfolios_vwret_df.rename(columns={"rev1_size1": "Small_LowRet",
"rev2_size1": "Small_MedRet",
"rev3_size1": "Small_HighRet",
"rev1_size2": "Big_LowRet",
"rev2_size2": "Big_MedRet",
"rev3_size2": "Big_HighRet"},
inplace=True)
In [82]:
portfolios_vwret_df
Out[82]:
In [83]:
rev_df = (portfolios_vwret_df['Small_LowRet'] + portfolios_vwret_df['Big_LowRet']) / 2 - \
(portfolios_vwret_df['Small_HighRet'] + portfolios_vwret_df['Big_HighRet']) / 2
In [84]:
rev_df = rev_df.reset_index()
rev_df.columns = ['ret_date','rev']
rev_df
Out[84]:
Combine with FF3¶
In [85]:
factors_df = pd.read_csv('./output_data/factors/ff3.csv')
In [86]:
factors_df
Out[86]:
In [87]:
factors_df['ret_date'] = pd.to_datetime(factors_df['ret_date'])
factors_df['ret_date'] = factors_df['ret_date'].dt.to_period('M')
factors_df = pd.merge(factors_df, rev_df, on='ret_date')
factors_df['ret_date'] = factors_df['ret_date'].dt.to_timestamp(freq='day',how='end').dt.normalize()
factors_df.set_index('ret_date',inplace=True)
In [88]:
factors_df
Out[88]:
In [89]:
factors_df.to_csv('./output_data/factors/ff3_rev.csv')
In [90]:
factors_df.to_pickle('./output_data/factors/ff3_rev.pkl')
In [91]:
((1 + factors_df).cumprod()*100).plot()
Out[91]:
In [92]:
((1 + factors_df['2018':]).cumprod()*100).plot()
Out[92]:
Long-only factor¶
In [93]:
rev_long_df = (portfolios_vwret_df['Small_LowRet'] + portfolios_vwret_df['Big_LowRet']) / 2
rev_long_df = rev_long_df.reset_index()
rev_long_df.columns=['ret_date','rev_long']
rev_long_df
Out[93]:
In [94]:
factors_long_df = pd.read_csv('./output_data/factors/ff3_long_only.csv')
factors_long_df['ret_date'] = pd.to_datetime(factors_long_df['ret_date'])
factors_long_df['ret_date'] = factors_long_df['ret_date'].dt.to_period('M')
factors_long_df = pd.merge(factors_long_df, rev_long_df, on='ret_date')
factors_long_df['ret_date'] = factors_long_df['ret_date'].dt.to_timestamp(freq='day',how='end').dt.normalize()
factors_long_df.set_index('ret_date',inplace=True)
In [95]:
factors_long_df
Out[95]:
In [96]:
((1 + factors_long_df).cumprod()*100).plot()
Out[96]:
In [97]:
((1 + factors_long_df['2018':]).cumprod()*100).plot()
Out[97]:
In [98]:
factors_long_df.to_csv('./output_data/factors/ff3_rev_long_only.csv')
In [99]:
factors_long_df.to_pickle('./output_data/factors/ff3_rev_long_only.pkl')
In [ ]: