In [1]:
import numpy as np
import pandas as pd
import datetime as dt
pd.set_option('display.max_rows', 16)
import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams['figure.figsize'] = (16.0, 9.0)
import seaborn as sns
import statsmodels.api as sm
from sklearn.linear_model import LinearRegression
import gc
In [2]:
plt.rcParams['figure.figsize'] = (16.0, 9.0)
Data¶
In [3]:
START = '2007-01-01'
END = '2022-03-31'
In [4]:
# Security Id
stk_info = DataAPI.SecIDGet(assetClass="E",pandas="1")
cond1 = (stk_info['exchangeCD'] == 'XSHE') | (stk_info['exchangeCD'] == 'XSHG')
cond2 = (stk_info['listStatusCD'] == 'L') | (stk_info['listStatusCD'] == 'DE')
stk_info = stk_info[cond1 & cond2].copy()
stk_id = stk_info['secID']
In [5]:
stk_info
Out[5]:
ST¶
In [6]:
st_df = DataAPI.SecSTGet(beginDate=START,endDate=END,secID=stk_id,field=['secID','tradeDate','STflg'],pandas="1")
In [7]:
st_df['tradeDate'] = pd.to_datetime(st_df['tradeDate'],format="%Y-%m-%d")
Risk free rate¶
In [8]:
shibor_df = DataAPI.MktIborGet(secID="Shibor1M.IRCN",beginDate=START,endDate=END,field=['secID','tradeDate','rate'],pandas="1")
shibor_df['rate'] = shibor_df['rate']*0.01/12
shibor_df['tradeDate'] = pd.to_datetime(shibor_df['tradeDate'])
shibor_df.drop('secID',axis=1,inplace=True)
shibor_df.rename(columns={'rate':'rf'},inplace=True)
shibor_df['ym'] = shibor_df['tradeDate'].dt.to_period('M')
shibor_df.sort_values('tradeDate',inplace=True)
shibor_df_m = shibor_df.groupby('ym',as_index=False).last()
shibor_df_m.drop('tradeDate',axis=1,inplace=True)
shibor_df_m
Out[8]:
Beta¶
In [9]:
beta_df = pd.read_pickle('./data/beta_df.pkl')
beta_df['tradeDate'] = pd.to_datetime(beta_df['tradeDate'], format="%Y-%m-%d")
beta_df['ym'] = beta_df['tradeDate'].dt.to_period('M')
beta_df.drop(['Beta60','Beta120'],axis=1,inplace=True)
beta_df['Beta252'] = pd.to_numeric(beta_df['Beta252'])
# Winsorization
# up_q = 0.99999
# lower_q = 0.00001
# beta_df['Beta252_winsor'] = beta_df['Beta252'].clip(lower=beta_df['Beta252'].quantile(lower_q),upper=beta_df['Beta252'].quantile(up_q))
# Monthly
beta_df_m = beta_df.groupby(['secID','ym'],as_index=False)['Beta252'].last()
beta_df_m.rename(columns={'Beta252':'beta'},inplace=True)
beta_df_m
Out[9]:
BM¶
In [10]:
pb_df = pd.read_pickle('./data/pb_df.pkl')
pb_df['tradeDate'] = pd.to_datetime(pb_df['tradeDate'])
pb_df['PB'] = pd.to_numeric(pb_df['PB'])
pb_df['ym'] = pb_df['tradeDate'].dt.to_period('M')
pb_df.sort_values(['secID','tradeDate'],inplace=True)
pb_df = pb_df.groupby(['secID','ym'],as_index=False).last()
pb_df['bm'] = 1 / pb_df['PB']
pb_df.drop(['tradeDate','PB'],axis=1,inplace=True)
pb_df = pb_df[pb_df['bm'] >= 0]
pb_df
Out[10]:
Trading data¶
In [11]:
# stk_df = DataAPI.MktEqudAdjAfGet(secID=stk_id,beginDate=START,endDate=END,isOpen=1,
# field=["secID","tradeDate",
# 'preClosePrice',"closePrice",
# "negMarketValue",
# "turnoverValue",'turnoverRate'],pandas="1")
# stk_df.to_pickle('./data/stk_df.pkl')
In [12]:
stk_df = pd.read_pickle('./data/stk_df.pkl')
stk_df['tradeDate'] = pd.to_datetime(stk_df['tradeDate'], format='%Y-%m-%d')
stk_df.sort_values(['secID','tradeDate'],inplace=True)
# drop ST stocks
print(stk_df.shape)
stk_df = pd.merge(stk_df, st_df, on=['secID','tradeDate'],how='left')
stk_df = stk_df[stk_df['STflg'].isna()].copy()
stk_df.drop('STflg',axis=1,inplace=True)
print(stk_df.shape)
In [13]:
stk_df
Out[13]:
In [14]:
# # If the trading days are required to be consecutive, fill missing days first. This could possibly produce a much larger df when using
## daily data, and if the missing dates are a lot for some securities
def fill_missing(df, full_dates, id_col='secID', date_col='tradeDate'):
"""
This function fills the missing dates for stocks.
Parameters:
df: The dataframe. Could be a sub-dataframe created by "groupby".
The dataframe must be sorted on the "date_col".
full_dates: the unique dates covering all securities in the full dataframe.
Need to be sorted.
id_col: the security id.
date_col: the dates column for the security
Returns:
A dataframe with the missing dates filled with NA.
"""
stk_id = df[id_col].unique()
# Newer version of pandas will allow comparison between "Timestamp" and "datetime64"
# date_start = np.where(full_dates == df[date_col].min())[0][0]
# date_end = np.where(full_dates == df[date_col].max())[0][0]
date_start = np.where(full_dates == df[date_col].min().to_datetime64())[0][0]
date_end = np.where(full_dates == df[date_col].max().to_datetime64())[0][0]
dates = full_dates[date_start:date_end+1]
idx = pd.MultiIndex.from_product([stk_id,dates],
names=(id_col,date_col))
df = df.set_index([id_col,date_col]).reindex(idx).reset_index()
return df
In [15]:
full_dates = np.sort(stk_df['tradeDate'].unique())
In [16]:
full_dates
Out[16]:
In [17]:
# %%time
# stk_df = stk_df.groupby('secID').apply(fill_missing, full_dates=full_dates)
Amihud (2002) liquidity¶
Amihud (2002)'s liquidity measure is perhaps the most widely known.
$$ illiq_i = \frac{1}{D} \sum^D_{d=1} \frac{|R_{i,d}|}{VOLD_{i,d}} $$VOLD: the dollar volume, measured as "close_price * shares_traded"
Intuition: the effect of volume on moving prices. If a small volume moves price significantly (nonsignificantly), the stock is quite illiquid (liquid).
Period choices: Amihud(2002) uses t-11 month through t month (12 months) daily data as the measure for month t.
Let's use the two measures:
- Amihud's original measure, but we do not stick to calendar month. Let's use 250 trading days.
- a simple one: one-month daily data.
Also, let's use the actual dollar volume within one day
- Acharya and Pedersen (2005): liquidity should be positively correlated with contemporaneous returns, and negatively with future returns
- 其他相关文献， Bali, Peng, Shen, and Tang (2014)，Chordia, Roll and Subrahmanyam (2000)
- Pastor and Stambaugh (2003): Aggregate liquidity factor.
In [18]:
stk_df
Out[18]:
In [19]:
# stk_df.reset_index(drop=True, inplace=True)
# stk_df['ret_daily'] = stk_df['closePrice'] / stk_df['preClosePrice'] - 1
# stk_df['illiq_daily'] = abs(stk_df['ret_daily']) / stk_df['turnoverValue']
# stk_df['ym'] = stk_df['tradeDate'].dt.to_period('M')
# stk_df.to_pickle('./data/stk_df_filled.pkl')
In [20]:
stk_df = pd.read_pickle('./data/stk_df_filled.pkl')
In [21]:
stk_df
Out[21]:
In [22]:
stk_df[(stk_df['secID']=='000001.XSHE') & (stk_df['tradeDate']>='2010-06-01') & (stk_df['tradeDate']<='2010-09-05')]
Out[22]:
Monthly trading data¶
In [23]:
stk_df_m = stk_df.groupby(['secID','ym'],as_index=False).last()
stk_df_m['ret'] = stk_df_m.groupby('secID')['closePrice'].apply(lambda x: x / x.shift() - 1)
stk_df_m['size'] = np.log(stk_df_m['negMarketValue'])
stk_df_m.drop(['tradeDate','preClosePrice'],axis=1,inplace=True)
stk_df_m = pd.merge(stk_df_m, shibor_df_m, on='ym')
stk_df_m['exret'] = stk_df_m['ret'] - stk_df_m['rf']
stk_df_m.sort_values(['secID','ym'],inplace=True)
stk_df_m.rename(columns={'negMarketValue':'mktcap'},inplace=True)
stk_df_m
Out[23]:
Momentum and reversal¶
Using unfilled data¶
In [24]:
stk_unfilled_df = pd.read_pickle('./data/stk_df.pkl')
stk_unfilled_df['tradeDate'] = pd.to_datetime(stk_unfilled_df['tradeDate'], format='%Y-%m-%d')
stk_unfilled_df['ym'] = stk_unfilled_df['tradeDate'].dt.to_period('M')
stk_unfilled_df.sort_values(['secID','tradeDate'],inplace=True)
# drop ST stocks
print(stk_unfilled_df.shape)
stk_unfilled_df = pd.merge(stk_unfilled_df, st_df, on=['secID','tradeDate'],how='left')
stk_unfilled_df = stk_unfilled_df[stk_unfilled_df['STflg'].isna()].copy()
stk_unfilled_df.drop('STflg',axis=1,inplace=True)
print(stk_unfilled_df.shape)
# Monthly
stk_unfilled_df_m = stk_unfilled_df.groupby(['secID','ym'],as_index=False).last()
In [25]:
stk_unfilled_df_m['ret_mom'] = stk_unfilled_df_m.groupby('secID')['closePrice'].apply(lambda x: x / x.shift() - 1) #这个ret_mom不用作后面ret的计算，后面仍保留monthly ret
stk_unfilled_df_m.sort_values(['secID','ym'],inplace=True)
In [26]:
stk_unfilled_df_m['1+ret_mom'] = stk_unfilled_df_m['ret_mom'] + 1
In [27]:
stk_unfilled_df_m
Out[27]:
In [28]:
stk_unfilled_df_m['mom'] = stk_unfilled_df_m.groupby('secID').rolling(11,min_periods=9)['1+ret_mom'].apply(np.prod, raw=True).values - 1
In [29]:
stk_unfilled_df_m
Out[29]:
In [30]:
stk_df_m = pd.merge(stk_df_m, stk_unfilled_df_m[['secID','ym','1+ret_mom']],on=['secID','ym'],how='left')
In [31]:
stk_df_m[(stk_df_m['secID']=='000001.XSHE') & (stk_df_m['ym']>='2010-06') & (stk_df_m['ym']<='2010-10')]
Out[31]:
In [32]:
stk_df_m.loc[stk_df_m['1+ret_mom'].isna(),'1+ret_mom'] = 1 # 缺失位置填充为1，以便连乘。
In [33]:
stk_df_m['mom'] = stk_df_m.groupby('secID').rolling(11,min_periods=11)['1+ret_mom'].apply(np.prod, raw=True).values - 1
In [34]:
stk_df_m
Out[34]:
In [35]:
stk_df_m['rev'] = stk_df_m['exret'].values
stk_df_m['ret'] = stk_df_m.groupby(['secID'])['ret'].shift(-1)
stk_df_m['rf'] = stk_df_m.groupby(['secID'])['rf'].shift(-1)
stk_df_m['exret'] = stk_df_m.groupby(['secID'])['exret'].shift(-1)
stk_df_m['ret_date'] = stk_df_m.groupby('secID')['ym'].shift(-1)
stk_df_m['mom'] = stk_df_m.groupby(['secID'])['mom'].shift()
stk_df_m['mom_date'] = stk_df_m.groupby('secID')['ym'].shift()
In [36]:
stk_df_m.drop(['ret_daily','turnoverValue','turnoverRate','illiq_daily','1+ret_mom'],axis=1,inplace=True)
In [37]:
stk_df_m
Out[37]:
1 month Amihud Illiquidity¶
In [38]:
MIN_NOBS = 15
illiq_df = stk_df.groupby(['secID','ym'],as_index=False)['illiq_daily'].mean()
In [39]:
idx = stk_df.groupby(['secID','ym'],as_index=False)['ret_daily'].count()['ret_daily'] < MIN_NOBS
illiq_df.loc[idx, 'illiq_daily'] = np.nan
illiq_df.rename(columns={'illiq_daily':'illiq_1m'},inplace=True)
In [40]:
illiq_df
Out[40]:
In [41]:
illiq_df['illiq_1m'] = illiq_df['illiq_1m'] * 1e6
In [42]:
illiq_df
Out[42]:
In [43]:
illiq_df['illiq_1m'].describe()
Out[43]:
12 months Amihud Illiquidity¶
In [44]:
# # This code choses 12 months, instead of trading days.
# def amihud_illiq(df, len_periods='12m', min_nobs=200):
# """
# df: year_month is set as the index, this is faster than using mask.
# """
# year_months = df.index.unique()
# n = len(year_months)
# illiq = np.full(shape=n, fill_value=np.nan)
# start_notnan_month = int(len_periods[:-1])-1
# for i in range(start_notnan_month,n):
# df_ = df[str(year_months[i]-int(len_periods[:-1])+1) : str(year_months[i])].to_numpy()
# if len(df_) < min_nobs:
# continue
# else:
# illiq[i] = df_[:,-1].mean()
# illiq_df = pd.DataFrame({'year_month': year_months, f'illiq{len_periods}': illiq})
# return illiq_df
In [45]:
stk_df['illiq_250d'] = stk_df.groupby(['secID']).rolling(250, min_periods=200)['illiq_daily'].mean().values
In [46]:
illiq_df['illiq_12m'] = stk_df.groupby(['secID','ym'])['illiq_250d'].last().values
In [47]:
illiq_df['illiq_12m'] = illiq_df['illiq_12m']*1e6
In [48]:
illiq_df
Out[48]:
Merge¶
In [49]:
pd.merge(stk_df_m[['secID','ret_date','ret','rf','exret','ym','mktcap','size','rev','mom_date','mom']],
beta_df_m[['secID','ym','beta']], on=['secID','ym'],how='left')
Out[49]:
In [50]:
ret_df = pd.merge(stk_df_m[['secID','ret_date','ret','rf','exret','ym','mktcap','size','rev','mom_date','mom']],
beta_df_m[['secID','ym','beta']], on=['secID','ym'],how='left')
In [51]:
ret_df = pd.merge(ret_df, pb_df, on=['secID','ym'],how='left')
In [52]:
ret_df = pd.merge(ret_df, illiq_df, on=['secID','ym'],how='left')
In [53]:
ret_df
Out[53]:
In [54]:
ret_df.columns.drop(['mom_date','mom'])
Out[54]:
In [55]:
cols = list(ret_df.columns.drop(['mom_date','mom']))
cols = cols+['mom_date','mom']
cols
Out[55]:
In [56]:
ret_df = ret_df[cols].copy()
In [57]:
ret_df[ret_df['secID']=='000004.XSHE']
Out[57]:
In [58]:
gc.collect()
Out[58]:
In [59]:
q = dict()
keys = ['q'+str(i) for i in range(1, 10)]
values = np.arange(0.1, 1.0, 0.1)
q.update(zip(keys,values))
quantile_df = pd.DataFrame()
for key, value in q.items():
quantile_df[key] = ret_df.groupby(['ym'])['illiq_12m'].quantile(value)
# quantile_df[key] = ret_df.dropna().groupby(['ym'])['illiq_12m'].quantile(value)
ret_df_q = pd.merge(ret_df, quantile_df, on='ym')
# ret_df_q = pd.merge(ret_df.dropna(), quantile_df, on='ym')
portfolios = dict()
drop_cols = [col for col in ret_df_q.columns if col[0]=='q']
portfolios['p1'] = ret_df_q.loc[ret_df_q['illiq_12m'] <= ret_df_q['q1']].copy().drop(drop_cols, axis=1)
for i in range(2,10):
idx = (ret_df_q[f'q{i-1}'] <= ret_df_q['illiq_12m']) & (ret_df_q['illiq_12m'] <= ret_df_q[f'q{i}'])
portfolios[f'p{i}'] = ret_df_q.loc[idx].copy().drop(drop_cols, axis=1)
portfolios['p10'] = ret_df_q.loc[ret_df_q['illiq_12m'] >= ret_df_q['q9']].copy().drop(drop_cols, axis=1)
portfolios_crs_mean = dict()
for k in portfolios.keys():
portfolios_crs_mean[k] = portfolios[k].groupby(['ret_date'])['exret'].mean()
mean_values = {}
t_values = {}
for k in portfolios_crs_mean.keys():
y = portfolios_crs_mean[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
# Portfolio 10-1
y = portfolios_crs_mean['p10'] - portfolios_crs_mean['p1']
const = np.full(shape=len(y), fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values['p10-p1'] = reg.params[0]
t_values['p10-p1'] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['mean','t-value'],
columns=mean_values.keys())
Out[59]:
In [60]:
q = dict()
keys = ['q'+str(i) for i in range(1, 10)]
values = np.arange(0.1, 1.0, 0.1)
q.update(zip(keys,values))
quantile_df = pd.DataFrame()
for key, value in q.items():
quantile_df[key] = ret_df.groupby(['ym'])['illiq_1m'].quantile(value)
ret_df_q = pd.merge(ret_df, quantile_df, on='ym')
portfolios = dict()
drop_cols = [col for col in ret_df_q.columns if col[0]=='q']
portfolios['p1'] = ret_df_q.loc[ret_df_q['illiq_1m'] <= ret_df_q['q1']].copy().drop(drop_cols, axis=1)
for i in range(2,10):
idx = (ret_df_q[f'q{i-1}'] <= ret_df_q['illiq_1m']) & (ret_df_q['illiq_1m'] <= ret_df_q[f'q{i}'])
portfolios[f'p{i}'] = ret_df_q.loc[idx].copy().drop(drop_cols, axis=1)
portfolios['p10'] = ret_df_q.loc[ret_df_q['illiq_1m'] >= ret_df_q['q9']].copy().drop(drop_cols, axis=1)
portfolios_crs_mean = dict()
for k in portfolios.keys():
portfolios_crs_mean[k] = portfolios[k].groupby(['ret_date'])['exret'].mean()
mean_values = {}
t_values = {}
for k in portfolios_crs_mean.keys():
y = portfolios_crs_mean[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
# Portfolio 10-1
y = portfolios_crs_mean['p10'] - portfolios_crs_mean['p1']
const = np.full(shape=len(y), fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values['p10-p1'] = reg.params[0]
t_values['p10-p1'] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['mean','t-value'],
columns=mean_values.keys())
Out[60]:
In [61]:
pd.DataFrame(portfolios_crs_mean)[['p2','p9']].plot()
Out[61]:
Double sort¶
In [62]:
ret_df.rename(columns={'illiq_1m':'illiq'},inplace=True)
In [63]:
from myutils.factor_func import double_sort, factor, daily_factor
In [64]:
double_sort?
In [65]:
portfolios = double_sort(ret_df, sort1='illiq')
In [66]:
portfolios
Out[66]:
In [67]:
mean_portfolios_ret = dict()
for pf in portfolios.keys():
mean_portfolios_ret[pf] = portfolios[pf].groupby('ret_date')['exret'].mean()
# print(mean_portfolios_ret[pf].shape) # print 看一下会不会存在某个月份上没有illiq和size分组没有任何交叉
# Fast merge by stacking
mean_portfolios_ret_df = pd.DataFrame(np.vstack([pf for pf in mean_portfolios_ret.values()])).T
mean_portfolios_ret_df.columns = mean_portfolios_ret.keys()
mean_portfolios_ret_df.index = mean_portfolios_ret['illiq1_size1'].index
# Newey-West adjustment
mean_values = {}
t_values = {}
for k in mean_portfolios_ret.keys():
y = mean_portfolios_ret[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=4)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[67]:
In [68]:
mean_portfolios_ret_df.plot()
Out[68]:
In [69]:
# Within mktcap1, any difference in illiq groups?
pfs = mean_portfolios_ret_df.columns
cols = list(pfs[pfs.str[-5:] == 'size1'])
mean_portfolios_ret_df[cols].plot()
Out[69]:
In [70]:
# Within mktcap1, any difference in illiq groups?
pfs = mean_portfolios_ret_df.columns
cols = list(pfs[pfs.str[-5:] == 'size2'])
mean_portfolios_ret_df[cols].plot()
Out[70]:
FM regression¶
In [71]:
def fm_reg(df,cols):
df_ = df.dropna()
if df_.shape[0] < 15:
return [None]*(len(cols)+1)
reg = LinearRegression(fit_intercept=True).fit(y=df_.loc[:,'exret'], X=df_.loc[:,cols])
return np.insert(reg.coef_, 0, reg.intercept_)
In [72]:
cols = ['illiq']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[72]:
In [73]:
cols = ['beta','size','bm','mom','rev','illiq']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[73]:
ST?¶
以前做上面内容时，先不考虑去除ST，这部分再去除。结论是差不多的。
In [74]:
# # st_df = DataAPI.SecSTGet(beginDate=START,endDate=END,secID=stk_id,field=['secID','tradeDate','STflg'],pandas="1")
# # st_df.to_pickle('./data/st_df.pkl')
# ret_df
# st_df = pd.read_pickle('./data/st_df.pkl')
# st_df['ym'] = pd.to_datetime(st_df['tradeDate']).dt.to_period('M')
# st_df.sort_values(['secID','tradeDate'],inplace=True)
# st_df = st_df.groupby(['secID','ym'],as_index=False).last()
# st_df.drop('tradeDate',axis=1,inplace=True)
# st_df
# st_df.loc[st_df['secID']=='000004.XSHE','STflg']
# st_df['STflg'].unique()
# st_df[st_df['STflg'] == '*']
# st_df[~st_df['STflg'].isin(['S'])]
# st_df = st_df[~st_df['STflg'].isin(['S'])]
# st_df
# ret_df[ret_df['secID']=='000004.XSHE']
# ret_df = pd.merge(ret_df,st_df,on=['secID','ym'],how='left')
# ret_df[(ret_df['secID'] =='000004.XSHE') & (ret_df['ym']=='2007-02')]
# ret_df[~ret_df['STflg'].isna()]
# ret_df_exST = ret_df[ret_df['STflg'].isna()]
# ret_df_ST = ret_df[~ret_df['STflg'].isna()]
# ret_df_exST.drop('STflg',axis=1,inplace=True)
# ret_df_ST.drop('STflg',axis=1,inplace=True)
# ret_df.drop('STflg',axis=1,inplace=True)
Single sort¶
In [75]:
# q = dict()
# keys = ['q'+str(i) for i in range(1, 10)]
# values = np.arange(0.1, 1.0, 0.1)
# q.update(zip(keys,values))
# quantile_df = pd.DataFrame()
# for key, value in q.items():
# quantile_df[key] = ret_df_exST.groupby(['ym'])['illiq'].quantile(value)
# ret_df_exST_q = pd.merge(ret_df_exST, quantile_df, on='ym')
# portfolios = dict()
# drop_cols = [col for col in ret_df_exST_q.columns if col[0]=='q']
# portfolios['p1'] = ret_df_exST_q.loc[ret_df_exST_q['illiq'] <= ret_df_exST_q['q1']].copy().drop(drop_cols, axis=1)
# for i in range(2,10):
# idx = (ret_df_exST_q[f'q{i-1}'] <= ret_df_exST_q['illiq']) & (ret_df_exST_q['illiq'] <= ret_df_exST_q[f'q{i}'])
# portfolios[f'p{i}'] = ret_df_exST_q.loc[idx].copy().drop(drop_cols, axis=1)
# portfolios['p10'] = ret_df_exST_q.loc[ret_df_exST_q['illiq'] >= ret_df_exST_q['q9']].copy().drop(drop_cols, axis=1)
# portfolios_crs_mean = dict()
# for k in portfolios.keys():
# portfolios_crs_mean[k] = portfolios[k].groupby(['ret_date'])['exret'].mean()
# mean_values = {}
# t_values = {}
# for k in portfolios_crs_mean.keys():
# y = portfolios_crs_mean[k]
# const = np.full(shape=len(y),fill_value=1)
# reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
# mean_values[k] = reg.params[0]
# t_values[k] = reg.tvalues[0]
# # Portfolio 10-1
# y = portfolios_crs_mean['p10'] - portfolios_crs_mean['p1']
# const = np.full(shape=len(y), fill_value=1)
# reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
# mean_values['p10-p1'] = reg.params[0]
# t_values['p10-p1'] = reg.tvalues[0]
# pd.DataFrame([mean_values.values(),t_values.values()],index=['mean','t-value'],
# columns=mean_values.keys())
Double sort¶
In [76]:
# portfolios = double_sort(ret_df_exST, sort1='illiq')
# mean_portfolios_ret = dict()
# for pf in portfolios.keys():
# mean_portfolios_ret[pf] = portfolios[pf].groupby('ret_date')['exret'].mean()
# print(mean_portfolios_ret[pf].shape) # print 看一下会不会存在某个月份上没有illiq和size分组没有任何交叉
In [77]:
# # Fast merge by stacking
# mean_portfolios_ret_df = pd.DataFrame(np.vstack([pf for pf in mean_portfolios_ret.values()])).T
# mean_portfolios_ret_df.columns = mean_portfolios_ret.keys()
# mean_portfolios_ret_df.index = mean_portfolios_ret['illiq1_size1'].index
# # Newey-West adjustment
# mean_values = {}
# t_values = {}
# for k in mean_portfolios_ret.keys():
# y = mean_portfolios_ret[k]
# const = np.full(shape=len(y),fill_value=1)
# reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=4)
# mean_values[k] = reg.params[0]
# t_values[k] = reg.tvalues[0]
# pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
FM regression¶
In [78]:
# cols = ['beta','size','bm','mom','rev','illiq']
# temp = ret_df_exST.groupby('ret_date').apply(fm_reg, cols=cols)
# reg_result_df = pd.DataFrame(temp.values.tolist())
# reg_result_df.index=temp.index
# reg_result_df.columns = ['intercept'] + cols
# reg_result_df.dropna(inplace=True)
# # Mean of coefs with NW adjustment
# mean_values = {}
# t_values = {}
# for k in reg_result_df.columns:
# y = reg_result_df[k]
# const = np.full(shape=len(y),fill_value=1)
# reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
# mean_values[k] = reg.params[0]
# t_values[k] = reg.tvalues[0]
# pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Illiquidity factor¶
Long-short factor¶
In [79]:
illiq_df = factor(ret_df, sort1='illiq', long_only=False)
In [80]:
illiq_df
Out[80]:
In [81]:
factors_df = pd.read_pickle('./data/factors/ff3_rev.pkl')
In [82]:
factors_df.index = factors_df.index.to_period('M')
In [83]:
factors_df = pd.merge(factors_df,illiq_df,on='ret_date')
In [84]:
factors_df
Out[84]:
In [85]:
rf_df = ret_df[['ret_date','rf']].drop_duplicates().sort_values('ret_date').dropna()
In [86]:
factors_df = pd.merge(rf_df,factors_df,on='ret_date').set_index('ret_date')
In [87]:
((factors_df+1).cumprod()*100).plot()
Out[87]:
In [88]:
((factors_df['2015':]+1).cumprod()*100).plot()
Out[88]:
In [89]:
(np.log(1 + factors_df).cumsum()*100).plot()
Out[89]:
Long-only factor¶
In [90]:
illiq_long_df = factor(ret_df, sort1='illiq')
In [91]:
illiq_long_df
Out[91]:
In [92]:
factors_long_df = pd.read_pickle('./data/factors/ff3_rev_long_only.pkl')
In [93]:
factors_long_df.index = factors_long_df.index.to_period('M')
In [94]:
factors_long_df = pd.merge(factors_long_df,illiq_long_df,on='ret_date')
In [95]:
factors_long_df = pd.merge(rf_df,factors_long_df,on='ret_date').set_index('ret_date')
In [96]:
factors_long_df.rename(columns={'illiq':'illiq_long'},inplace=True)
In [97]:
factors_long_df
Out[97]:
In [98]:
((factors_long_df+1).cumprod()*100).plot()
Out[98]:
In [99]:
((1 + factors_long_df['2018':]).cumprod()*100).plot()
Out[99]:
Volality¶
Raw volatility¶
In [100]:
stk_df
Out[100]:
In [101]:
vol_df = stk_df.groupby(['secID','ym'])['ret_daily'].std()
In [102]:
vol_df
Out[102]:
In [103]:
vol_df = vol_df.to_frame()
In [104]:
vol_df.columns = ['vol']
vol_df.reset_index(inplace=True)
MIN_NOBS = 15
idx = stk_df.groupby(['secID','ym'],as_index=False)['tradeDate'].count()['tradeDate'] < MIN_NOBS
vol_df.loc[idx, 'vol'] = np.nan
In [105]:
vol_df
Out[105]:
Idiosyncratic volatility¶
Idiosyncratic volatility 指的是被已有的因子收益率回归之后，剩下的残差的波动率。传统金融理论认为残差部分属于可分散的风险，不应该被定价。即使认为因子模型还不够好，残差部分有剩余未被解释的系统性风险的部分，这部分的风险溢价(factor risk premium)也应该是正的。然而，美国市场的数据发现，波动率越低，未来的收益越高。这是一个很有意思的异象。
最广为人知的文献是 Ang, Hodrick, Xing, and Zhang (2006)。
- Bali and Cakici (2008): 数据处理的细节对结果影响很大，包括数据频率、portfolio加权的方式、分位点的计算等等
- Huang et al. (2010 RFS): 这个异象和 reversal 有关
- 其他部分文献：Han and Lesmond (2011), Bali, Cakici, and Whitelaw (2011), Baker, Bradley, and Wurgler (2011), Baker and Wurgler (2014)
我们每个月用因子模型计算一下ivol，用来排序。
Daily factors¶
In [106]:
pd.read_pickle('./data/factors/factors_daily.pkl')
Out[106]:
In [107]:
shibor1d = DataAPI.MktIborGet(secID="Shibor1D.IRCN",beginDate=START,endDate=END,field=['tradeDate','rate'],pandas="1")
shibor1d['tradeDate'] = pd.to_datetime(shibor1d['tradeDate'])
shibor1d['rate'] = shibor1d['rate'] * 0.01 / 365
shibor1d.rename(columns={'rate':'rf'},inplace=True)
In [108]:
factors_daily = pd.read_pickle('./data/factors/factors_daily.pkl')
In [109]:
illiq_daily = daily_factor(ret_df, stock_df=stk_df,sort1='illiq',long_only=False)
In [110]:
illiq_daily
Out[110]:
In [111]:
illiq_daily.name = 'illiq'
In [112]:
factors_daily = pd.merge(factors_daily, illiq_daily, on='tradeDate')
In [113]:
factors_daily
Out[113]:
计算月度的 idiosyncratic volatility¶
In [114]:
def idiovol_np(df, factor_cols, len_periods='1m', min_nobs=15):
"""
df: year_month is set as index
"""
year_months = df.index.unique()
n = len(year_months)
idiovol = np.full(shape=n, fill_value=np.nan)
start_notnan_month = int(len_periods[:-1])-1
for i in range(start_notnan_month,n):
df_ = df.loc[str(year_months[i]-int(len_periods[:-1])+1) : str(year_months[i]),['exret']+factor_cols].dropna().to_numpy()
if len(df_) < min_nobs:
continue
else:
y_ = df_[:,0] #
X_ = df_[:,1:]
reg = LinearRegression().fit(y=y_, X=X_)
res = y_ - reg.predict(X_)
idiovol[i] = np.std(res)
idiovol_df = pd.DataFrame({'ym': year_months, f'idiovol{len_periods}': idiovol})
return idiovol_df
In [115]:
def idiovol(df,factor_cols,min_nobs=15):
df = df.dropna()
y = df.loc[:,'exret']
X = df.loc[:,factor_cols]
nobs = len(X)
if nobs < min_nobs:
idiovol = np.nan
else:
reg = LinearRegression().fit(y=y,X=X)
res = y - reg.predict(X)
idiovol = np.std(res)
return idiovol
In [116]:
reg_df = pd.merge(stk_df[['secID','tradeDate','ret_daily']],factors_daily,
on=['tradeDate'])
In [117]:
reg_df
Out[117]:
In [118]:
reg_df.sort_values(['secID','tradeDate'],inplace=True)
In [119]:
reg_df
Out[119]:
In [120]:
reg_df['exret'] = reg_df['ret_daily'] - reg_df['rf']
In [121]:
reg_df['ym'] = reg_df['tradeDate'].dt.to_period('M')
代码测试¶
In [122]:
# Testing the time consuming
ids = reg_df['secID'].unique()
ids_ = np.random.choice(ids, 50)
temp = reg_df[reg_df['secID'].isin(ids_)]
In [123]:
temp.set_index('ym',inplace=True)
In [124]:
%%time
result1 = temp.groupby(['secID','ym']).apply(idiovol, factor_cols=['exmktret','size','bm','mom','rev','illiq'])
In [125]:
%%time
result2 = temp.groupby(['secID']).apply(idiovol_np, factor_cols=['exmktret','size','bm','mom','rev','illiq'])
In [126]:
result1
Out[126]:
In [127]:
result2
Out[127]:
In [128]:
result1 = result1.to_frame().reset_index()
result1.columns = ['secID','ym','r1']
In [129]:
result2 = result2.reset_index().drop('level_1',axis=1)
In [130]:
result = pd.merge(result1, result2, on=['secID','ym'])
In [131]:
result
Out[131]:
In [132]:
(result['r1'] - result['idiovol1m']).sum()
Out[132]:
In [133]:
result['r1'].to_numpy()
Out[133]:
In [134]:
result['idiovol1m'].to_numpy()
Out[134]:
In [135]:
np.array_equal(result['r1'].to_numpy(), result['idiovol1m'].to_numpy())
Out[135]:
In [136]:
np.allclose(result['r1'].to_numpy(), result['idiovol1m'].to_numpy())
Out[136]:
In [137]:
result.fillna(0, inplace=True)
In [138]:
np.array_equal(result['r1'].to_numpy(), result['idiovol1m'].to_numpy())
Out[138]:
In [139]:
np.allclose(result['r1'].to_numpy(), result['idiovol1m'].to_numpy())
Out[139]:
实际计算¶
In [140]:
reg_df.set_index('ym',inplace=True)
In [141]:
reg_df
Out[141]:
In [142]:
# %%time
# ivol = reg_df.groupby(['secID']).apply(idiovol_np, factor_cols=['exmktret','size','bm','mom','rev','illiq'])
# # Takes about 26min 40s to execute
In [143]:
# ivol = ivol.reset_index().drop('level_1',axis=1)
In [144]:
# ivol.to_pickle('./data/idiovol1m.pkl')
In [145]:
ivol = pd.read_pickle('./data/idiovol1m.pkl')
In [146]:
vol_df = pd.merge(vol_df,ivol,on=['secID','ym'])
In [147]:
vol_df
Out[147]:
In [148]:
vol_df.loc[vol_df['ym']>='2018',['ym','vol']].boxplot(by='ym',showmeans=True)
Out[148]:
In [149]:
vol_df['vol_clip'] = vol_df['vol'].clip(lower=vol_df['vol'].quantile(0.001),
upper=vol_df['vol'].quantile(0.999))
vol_df['ivol_clip'] = vol_df['idiovol1m'].clip(lower=vol_df['idiovol1m'].quantile(0.001),
upper=vol_df['idiovol1m'].quantile(0.999))
In [150]:
vol_df.loc[vol_df['ym']>='2018',['ym','vol_clip']].boxplot(by='ym',showmeans=True,rot=45)
Out[150]:
In [151]:
vol_df.loc[vol_df['ym']>='2018',['ym','ivol_clip']].boxplot(by='ym',showmeans=True,rot=45)
Out[151]:
In [152]:
vol_df.loc[:,'vol':'ivol_clip'].describe()
Out[152]:
In [153]:
vol_df.rename(columns={'idiovol1m':'ivol'},inplace=True)
Sorting and Factors¶
In [154]:
ret_df = pd.merge(ret_df,vol_df,
on=['secID','ym'])
In [155]:
ret_df
Out[155]:
In [156]:
def single_sort(df, sort):
q = dict()
keys = ['q'+str(i) for i in range(1, 10)]
values = np.arange(0.1, 1.0, 0.1)
q.update(zip(keys,values))
quantile_df = pd.DataFrame()
for key, value in q.items():
quantile_df[key] = df.groupby(['ym'])[sort].quantile(value)
df_q = pd.merge(df, quantile_df, on='ym')
portfolios = dict()
drop_cols = [col for col in df_q.columns if col[0]=='q']
portfolios['p1'] = df_q.loc[df_q[sort] <= df_q['q1']].copy().drop(drop_cols, axis=1)
for i in range(2,10):
idx = (df_q[f'q{i-1}'] <= df_q[sort]) & (df_q[sort] <= df_q[f'q{i}'])
portfolios[f'p{i}'] = df_q.loc[idx].copy().drop(drop_cols, axis=1)
portfolios['p10'] = df_q.loc[df_q[sort] >= df_q['q9']].copy().drop(drop_cols, axis=1)
portfolios_crs_mean = dict()
for k in portfolios.keys():
portfolios_crs_mean[k] = portfolios[k].groupby(['ret_date'])['exret'].mean()
mean_values = {}
t_values = {}
for k in portfolios_crs_mean.keys():
y = portfolios_crs_mean[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
# Portfolio 10-1
y = portfolios_crs_mean['p10'] - portfolios_crs_mean['p1']
const = np.full(shape=len(y), fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values['p10-p1'] = reg.params[0]
t_values['p10-p1'] = reg.tvalues[0]
display(pd.DataFrame([mean_values.values(),t_values.values()],index=['mean','t-value'],
columns=mean_values.keys()))
Raw volatility¶
In [157]:
single_sort(ret_df, 'vol')
In [158]:
portfolios = double_sort(ret_df, sort1='vol')
mean_portfolios_ret = dict()
for pf in portfolios.keys():
mean_portfolios_ret[pf] = portfolios[pf].groupby('ret_date')['exret'].mean()
print(mean_portfolios_ret[pf].shape) # print 看一下会不会存在某个月份上没有vol和size分组没有任何交叉
# Fast merge by stacking
mean_portfolios_ret_df = pd.DataFrame(np.vstack([pf for pf in mean_portfolios_ret.values()])).T
mean_portfolios_ret_df.columns = mean_portfolios_ret.keys()
mean_portfolios_ret_df.index = mean_portfolios_ret['vol1_size1'].index
# Newey-West adjustment
mean_values = {}
t_values = {}
for k in mean_portfolios_ret.keys():
y = mean_portfolios_ret[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=4)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[158]:
In [159]:
cols = ['beta','size','bm','mom','rev','illiq','vol']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[159]:
In [160]:
cols = ['beta','size','bm','mom','rev','illiq','vol_clip']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[160]:
Idiosyncratic volatility¶
In [161]:
single_sort(ret_df, 'ivol')
In [162]:
portfolios = double_sort(ret_df, sort1='ivol')
mean_portfolios_ret = dict()
for pf in portfolios.keys():
mean_portfolios_ret[pf] = portfolios[pf].groupby('ret_date')['exret'].mean()
print(mean_portfolios_ret[pf].shape) # print 看一下会不会存在某个月份上没有vol和size分组没有任何交叉
# Fast merge by stacking
mean_portfolios_ret_df = pd.DataFrame(np.vstack([pf for pf in mean_portfolios_ret.values()])).T
mean_portfolios_ret_df.columns = mean_portfolios_ret.keys()
mean_portfolios_ret_df.index = mean_portfolios_ret['ivol1_size1'].index
# Newey-West adjustment
mean_values = {}
t_values = {}
for k in mean_portfolios_ret.keys():
y = mean_portfolios_ret[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=4)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[162]:
In [163]:
cols = ['beta','size','bm','mom','rev','illiq','ivol_clip']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[163]:
In [164]:
cols = ['beta','size','bm','mom','rev','illiq','ivol_clip']
temp = ret_df[ret_df['ret_date']>='2015'].groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[164]:
In [165]:
cols = ['beta','size','bm','mom','rev','illiq','ivol_clip']
temp = ret_df[ret_df['ret_date']>='2018'].groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[165]:
In [166]:
ivol_factor = factor(ret_df, sort1='ivol',long_high=False,long_only=False)
In [167]:
ivol_factor
Out[167]:
In [168]:
factors_df = pd.merge(factors_df,ivol_factor,on='ret_date',how='left')
In [169]:
((factors_df+1).cumprod()*100).plot()
Out[169]:
In [170]:
((factors_df['2018':]+1).cumprod()*100).plot()
Out[170]:
In [171]:
factors_df.to_pickle('./data/factors/factors_all.pkl')
Output factor exposures¶
In [172]:
ret_df
Out[172]:
In [173]:
ret_df.to_pickle('./data/factor_exposure/all_exposure.pkl')
In [210]:
cols = ['size','bm','rev','illiq','ivol']
In [211]:
monthend = stk_df[['secID','tradeDate','ym']].groupby(['secID','ym'],as_index=False).last()
monthend.rename(columns={'tradeDate':'month_end'},inplace=True)
In [212]:
ret_df = pd.merge(ret_df, monthend,on=['secID','ym'],how='left')
In [213]:
ret_df['month_end'] = ret_df['ym'].dt.to_timestamp() + pd.tseries.offsets.MonthEnd(0)
In [216]:
ret_df[['secID','month_end','rev']].dropna().pivot(index='month_end',
columns='secID',values='rev')
Out[216]:
In [226]:
temp = ret_df[['secID','month_end','size']].dropna().pivot(index='month_end',
columns='secID',values='size').reset_index()
temp['month_end'] = temp['month_end'].dt.strftime('%Y/%m/%d')
temp.set_index('month_end')
Out[226]:
In [231]:
for c in cols:
temp = ret_df[['secID','month_end',c]].dropna().pivot(index='month_end',columns='secID',values=c)
temp.reset_index(inplace=True)
temp['month_end'] = temp['month_end'].dt.strftime('%Y/%m/%d')
temp.set_index('month_end',inplace=True)
temp.to_csv(f'./data/factor_exposure/{c}_exposure.csv')
Long-only ivol factor¶
In [181]:
factors_daily = pd.read_pickle('./data/factors/factors_daily_long_only.pkl')
illiq_daily = daily_factor(ret_df, stock_df=stk_df,sort1='illiq',long_only=True)
illiq_daily.name = 'illiq_long'
factors_daily = pd.merge(factors_daily, illiq_daily, on='tradeDate')
In [182]:
reg_df = pd.merge(stk_df[['secID','tradeDate','ret_daily']],factors_daily,
on=['tradeDate'])
reg_df.sort_values(['secID','tradeDate'],inplace=True)
reg_df['exret'] = reg_df['ret_daily'] - reg_df['rf']
reg_df['ym'] = reg_df['tradeDate'].dt.to_period('M')
In [183]:
reg_df.set_index('ym',inplace=True)
In [184]:
reg_df
Out[184]:
In [185]:
gc.collect()
Out[185]:
In [186]:
# import sys
# def sizeof_fmt(num, suffix='B'):
# ''' by Fred Cirera, https://stackoverflow.com/a/1094933/1870254, modified'''
# for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
# if abs(num) < 1024.0:
# return "%3.1f %s%s" % (num, unit, suffix)
# num /= 1024.0
# return "%.1f %s%s" % (num, 'Yi', suffix)
# for name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),
# key= lambda x: -x[1])[:10]:
# print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
In [187]:
# del stk_unfilled_df, beta_df
In [188]:
# %%time
# ivol = reg_df.groupby(['secID']).apply(idiovol_np, factor_cols=['exmktret','size_long','bm_long','mom_long','rev_long','illiq_long'])
# # Takes about 27min 25s to execute
In [189]:
# ivol = ivol.reset_index().drop('level_1',axis=1)
# display(ivol)
# ivol.to_pickle('./data/idiovol1m_long_only.pkl')
In [190]:
vol_df = stk_df.groupby(['secID','ym'])['ret_daily'].std()
vol_df = vol_df.to_frame()
vol_df.columns = ['vol']
vol_df.reset_index(inplace=True)
MIN_NOBS = 15
idx = stk_df.groupby(['secID','ym'],as_index=False)['tradeDate'].count()['tradeDate'] < MIN_NOBS
vol_df.loc[idx, 'vol'] = np.nan
In [191]:
ivol = pd.read_pickle('./data/idiovol1m_long_only.pkl')
vol_df = pd.merge(vol_df,ivol,on=['secID','ym'])
vol_df['vol_clip'] = vol_df['vol'].clip(lower=vol_df['vol'].quantile(0.001),
upper=vol_df['vol'].quantile(0.999))
vol_df['ivol_clip'] = vol_df['idiovol1m'].clip(lower=vol_df['idiovol1m'].quantile(0.001),
upper=vol_df['idiovol1m'].quantile(0.999))
vol_df.rename(columns={'idiovol1m':'ivol'},inplace=True)
In [192]:
try:
ret_df.drop(['vol','ivol','vol_clip','ivol_clip'],axis=1,inplace=True)
except KeyError:
pass
In [193]:
ret_df = pd.merge(ret_df,vol_df,
on=['secID','ym'])
In [194]:
ret_df
Out[194]:
In [195]:
single_sort(ret_df, 'ivol')
In [196]:
cols = ['beta','size','bm','mom','rev','illiq','ivol_clip']
temp = ret_df.groupby('ret_date').apply(fm_reg, cols=cols)
reg_result_df = pd.DataFrame(temp.values.tolist())
reg_result_df.index=temp.index
reg_result_df.columns = ['intercept'] + cols
reg_result_df.dropna(inplace=True)
# Mean of coefs with NW adjustment
mean_values = {}
t_values = {}
for k in reg_result_df.columns:
y = reg_result_df[k]
const = np.full(shape=len(y),fill_value=1)
reg = sm.OLS(y, const).fit().get_robustcov_results(cov_type='HAC', maxlags=6)
mean_values[k] = reg.params[0]
t_values[k] = reg.tvalues[0]
pd.DataFrame([mean_values.values(),t_values.values()],index=['ret_mean','t_values'],columns=mean_values.keys())
Out[196]:
In [197]:
ivol_factor = factor(ret_df, sort1='ivol',long_high=False,long_only=True)
In [198]:
factors_long_df = pd.merge(factors_long_df,ivol_factor,on='ret_date',how='left')
In [199]:
((factors_long_df+1).cumprod()*100).plot()
Out[199]:
In [200]:
((factors_long_df['2018':]+1).cumprod()*100).plot()
Out[200]:
In [201]:
factors_long_df.rename(columns={'ivol':'ivol_long'},inplace=True)
In [202]:
factors_long_df
Out[202]:
In [203]:
factors_long_df.to_pickle('./data/factors/factors_all_long_only.pkl')
In [ ]:
In [ ]:
In [ ]:
In [ ]: