In [ ]:
#4201458 石怀宇
引用数据库¶
In [1]:
import re
import pdfplumber
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pylab import mpl
mpl.rcParams['font.sans-serif']=['SimHei']
mpl.rcParams['axes.unicode_minus']=False
import tushare as ts
爬取公司基本信息¶
In [2]:
ts.set_token('4a22e1fcdbba73b7336b8aa1f7f6e5f2eef0cf6fee943619d48d3bfb')
pro = ts.pro_api()
df1 = pro.query('stock_basic', exchange='', list_status='L', fields='ts_code,name')
df1=df1.sort_values(by='ts_code')
df1
df2=pro.stock_company(exchange='', fields='ts_code,chairman,office,website,email')
df2=df2.sort_values(by='ts_code')
df2
df2['name']=df1['name']
df2
Out[2]:
文件路径¶
In [3]:
os.chdir('C:\\Users\\桌面\\2019')
#读取文件夹文件
path = 'C:\\Users\\桌面\\2019'
path1 = 'C:\\Users\\桌面\\2019txt'
files = os.listdir(path)
files1 = os.listdir(path1)
PDF转换为txt¶
In [4]:
for file in files:
print(file)
with pdfplumber.open(file) as pdf:
f = open(path1+'\\'+file+'.txt', 'w', encoding='utf-8')
for page in pdf.pages:
try:
text = page.extract_text()
if text != None:
f.write(text)
except :
print('error')
pass
最近三年归属于公司股东的净利润（元）¶
In [54]:
最近三年归属于上市公司股东的净利润 = []
for file in files1:
print(file)
with open(path1+'\\'+file, 'r', encoding='utf-8') as f:
data = f.read()
f.close
data = data.replace(' ', '')
profit = re.findall(r'归属于上市公司股东的净利润（元）(.*?)\n', data)
profit = " ".join(str(i) for i in profit)
prof1 = re.findall(r'\d*\,?\d*\,?\d*\,?\d+\.?\d\d', profit)
del prof1[2]
gsmc = re.search('(.*?)有限公司', data)
公司名称 = gsmc.group(1)
porf1.insert(0,公司名称)
最近三年归属于上市公司股东的净利润.append(incom1)
最近三年归属于公司股东的净利润（元）¶
表格形式输出¶
In [56]:
df3 = pd.DataFrame(最近三年归属于上市公司股东的净利润)
df3
Out[56]:
获取基本面数据¶
In [1]:
#先引入后面可能用到的包（package）
import pandas as pd
import numpy as np
from scipy import stats
import tushare as ts
import matplotlib.pyplot as plt
%matplotlib inline
#正常显示画图时出现的中文和负号
from pylab import mpl
mpl.rcParams['font.sans-serif']=['SimHei']
mpl.rcParams['axes.unicode_minus']=False
#调取股票基本面数据和行情数据
#基本面数据
basics_data=pro.query('stock_basic', exchange='', list_status='L', fields='ts_code,symbol,name,area,industry,list_date')
basics_data
分布¶
In [58]:
#使用groupby对上市公司归属地进行汇总，
#统计每个省份（直辖市）上市公司的总数
area=basics_data.groupby('area')['name'].count()
#从大到小进行排序，取出前十名
area.sort_values(ascending=False)[:10]
Out[58]:
把深圳的数据加入到广东当中¶
In [59]:
#将“广东”与深圳数据合并成新的广东
area['广东']=area['广东']+area['深圳']
area.drop(['深圳'],inplace=True)
area.sort_values(ascending=False)[:10]
Out[59]:
分布地图¶
In [61]:
#导入作图包
from pyecharts import Map
# 将数据转化为字典，不转也可以
d =dict(area)
province=list(d.keys())
value=list(d.values())
#maptype='china' 只显示全国直辖市和省级
#数据只能是省名和直辖市的名称
map = Map("中国上市公司分布", title_color="#fff",
title_pos="center", width=1200,
height=600,background_color='#404a59')
map.add("", province, value,visual_range=[min(value),max(value)],
is_label_show=True,maptype='china',
visual_text_color='#000',label_pos="center",
is_visualmap=True)
map.render(path="中国上市公司分布.html")
#到所在的目录查找文件：中国上市公司分布.html
使用python对时间序列模型进行建模与分析¶
In [15]:
import pandas as pd
import numpy as np
import statsmodels.tsa.api as smt
#tsa为Time Series analysis缩写
import statsmodels.api as sm
import scipy.stats as scs
from arch import arch_model
#画图
import matplotlib.pyplot as plt
import matplotlib as mpl
%matplotlib inline
#正常显示画图时出现的中文和负号
from pylab import mpl
mpl.rcParams['font.sans-serif']=['SimHei']
mpl.rcParams['axes.unicode_minus']=False
In [16]:
#先定义一个画图函数，后面都会用到
def ts_plot(data, lags=None,title=''):
if not isinstance(data, pd.Series):
data = pd.Series(data)
#matplotlib官方提供了五种不同的图形风格，
#包括bmh、ggplot、dark_background、fivethirtyeight和grayscale
with plt.style.context('ggplot'):
fig = plt.figure(figsize=(10, 8))
layout = (3, 2)
ts_ax = plt.subplot2grid(layout, (0, 0), colspan=2)
acf_ax = plt.subplot2grid(layout, (1, 0))
pacf_ax = plt.subplot2grid(layout, (1, 1))
qq_ax = plt.subplot2grid(layout, (2, 0))
pp_ax = plt.subplot2grid(layout, (2, 1))
data.plot(ax=ts_ax)
ts_ax.set_title(title+'时序图')
smt.graphics.plot_acf(data, lags=lags, ax=acf_ax, alpha=0.5)
acf_ax.set_title('自相关系数')
smt.graphics.plot_pacf(data, lags=lags, ax=pacf_ax, alpha=0.5)
pacf_ax.set_title('偏自相关系数')
sm.qqplot(data, line='s', ax=qq_ax)
qq_ax.set_title('QQ 图')
scs.probplot(data, sparams=(data.mean(),
data.std()), plot=pp_ax)
pp_ax.set_title('PP 图')
plt.tight_layout()
return
# 模拟AR(1) 过程
#设置随机种子（括号里数字无意义）
np.random.seed(1)
#模拟次数
n=5000
#AR模型的参数
a = 0.8
#扰动项为正态分布
x = w = np.random.normal(size=n)
for t in range(1,n):
x[t] = a*x[t-1] + w[t]
#画图
ts_plot(x, lags=30)
In [20]:
#估计数据的AR模型参数和滞后阶数
def simu_ar(data,a,maxlag=30,true_order = 1):
'''data:要拟合的数据；a为参数,可以为列表；maxlag:最大滞后阶数'''
# 拟合AR(p)模型
result = smt.AR(data).fit(maxlag=maxlag, ic='aic', trend='nc')
#选择滞后阶数
est_order = smt.AR(data).select_order(maxlag=maxlag,
ic='aic', trend='nc')
#参数选择标准ic : 有四个选择 {‘aic’,’bic’,’hqic’,’t-stat’}
#趋势项：trend：c是指包含常数项，nc为不含常数项
#打印结果
print(f'参数估计值：{result.params.round(2)}，估计的滞后阶数：{est_order}')
print(f'真实参数值：{a}，真实滞后阶数 {true_order}')
simu_ar(x,a=0.8)
以MFI指标编写的python回测框架¶
In [39]:
#先引入后面可能用到的包（package）
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
#正常显示画图时出现的中文和负号
from pylab import mpl
mpl.rcParams['font.sans-serif']=['SimHei']
mpl.rcParams['axes.unicode_minus']=False
#引入TA-Lib库
import talib as ta
import tushare as ts
df=ts.get_k_data('sh',start='2000-01-01')
df.index=pd.to_datetime(df.date)
df=df.sort_index()
df['ret']=df.close/df.close.shift(1)-1
df.head()
high,low,close,volume=df.high.values,df.low.values,df.close.values,df.volume.values
df['mfi']=ta.MFI(high, low, close, volume, timeperiod=14)
plt.figure(figsize=(16,14))
plt.subplot(211)
df['close'].plot(color='r')
plt.xlabel('')
plt.title('上证综指走势',fontsize=15)
plt.subplot(212)
df['mfi'].plot()
plt.title('MFI指标',fontsize=15)
plt.xlabel('')
plt.show()
# In[186]:
#当前日的MFI<20，而当日的MFI>20时，买入信号设置为1
for i in range(15,len(df)):
if df['mfi'][i]>20 and df['mfi'][i-1]<20:
df.loc[df.index[i],'收盘信号']=1
if df['mfi'][i]<80 and df['mfi'][i-1]>80:
df.loc[df.index[i],'收盘信号']=0
#计算每天的仓位，当天持有上证指数时，仓位为1，当天不持有上证指数时，仓位为0
pd.options.mode.chained_assignment = None
df['当天仓位']=df['收盘信号'].shift(1)
df['当天仓位'].fillna(method='ffill',inplace=True)
from datetime import datetime,timedelta
d=df[df['当天仓位']==1].index[0]-timedelta(days=1)
df_new=df.loc[d:]
df_new['ret'][0]=0
df_new['当天仓位'][0]=0
#当仓位为1时，买入上证指数，当仓位为0时，空仓，计算资金指数
df_new['资金指数']=(df_new.ret*df['当天仓位']+1.0).cumprod()
df_new['指数净值']=(df_new.ret+1.0).cumprod()
df.close.plot(figsize=(16,7))
for i in range(len(df)):
if df['收盘信号'][i]==1:
plt.annotate('买',xy=(df.index[i],df.close[i]),arrowprops=dict(facecolor='r',shrink=0.05))
if df['收盘信号'][i]==0:
plt.annotate('卖',xy=(df.index[i],df.close[i]),arrowprops=dict(facecolor='g',shrink=0.1))
plt.title('上证指数2000-2019年MFI买卖信号',size=15)
plt.xlabel('')
ax=plt.gca()
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
plt.show()
#查看最近四年情况
df1=df.loc['2016-01-01':,]
df1.close.plot(figsize=(16,7))
for i in range(len(df1)):
if df1['收盘信号'][i]==1:
plt.annotate('买',xy=(df1.index[i],df1.close[i]),arrowprops=dict(facecolor='r',shrink=0.05))
if df1['收盘信号'][i]==0:
plt.annotate('卖',xy=(df1.index[i],df1.close[i]),arrowprops=dict(facecolor='g',shrink=0.1))
plt.title('上证指数2016-2019年MFI买卖信号',fontsize=15)
plt.xlabel('')
ax=plt.gca()
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
plt.show()
df1['策略净值']=(df1.ret*df1['当天仓位']+1.0).cumprod()
df1['指数净值']=(df1.ret+1.0).cumprod()
df1['策略收益率']=df1['策略净值']/df1['策略净值'].shift(1)-1
df1['指数收益率']=df1.ret
total_ret=df1[['策略净值','指数净值']].iloc[-1]-1
annual_ret=pow(1+total_ret,250/len(df_new))-1
dd=(df1[['策略净值','指数净值']].cummax()-df1[['策略净值','指数净值']])/df1[['策略净值','指数净值']].cummax()
d=dd.max()
beta=df1[['策略收益率','指数收益率']].cov().iat[0,1]/df1['指数收益率'].var()
alpha=(annual_ret['策略净值']-annual_ret['指数净值']*beta)
exReturn=df1['策略收益率']-0.03/250
sharper_atio=np.sqrt(len(exReturn))*exReturn.mean()/exReturn.std()
TA1=round(total_ret['策略净值']*100,2)
TA2=round(total_ret['指数净值']*100,2)
AR1=round(annual_ret['策略净值']*100,2)
AR2=round(annual_ret['指数净值']*100,2)
MD1=round(d['策略净值']*100,2)
MD2=round(d['指数净值']*100,2)
S=round(sharper_atio,2)
print(f'累计收益率：策略{TA1}%，指数{TA2}%;\n年化收益率：策略{AR1}%，指数{AR2}%；\n最大回撤： 策略{MD1}%，指数{MD2}%;\n策略alpha: {round(alpha,2)}，策略beta：{round(beta,2)}; \n夏普比率： {S}')
df1[['策略净值','指数净值']].plot(figsize=(15,7))
plt.title('上证指数与MFI指标策略\n2016年1月1日至今',size=15)
bbox = dict(boxstyle="round", fc="w", ec="0.5", alpha=0.9)
plt.text('2017-05-01', 0.75, f'累计收益率：策略{TA1}%，指数{TA2}%;\n年化收益率：策略{AR1}%，指数{AR2}%；\n最大回撤： 策略{MD1}%，指数{MD2}%;\n策略alpha: {round(alpha,2)}，策略beta：{round(beta,2)}; \n夏普比率： {S}', size=13,bbox=bbox)
plt.xlabel('')
ax=plt.gca()
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
plt.show()
def get_data(code,date):
df=ts.get_k_data(code,start=date)
df.index=pd.to_datetime(df.date)
df=df.sort_index()
df['ret']=df.close/df.close.shift(1)-1
return df
#关掉pandas的warnings
pd.options.mode.chained_assignment = None
def strategy(code,date,L,H):
df=get_data(code,date)
high,low,close,volume=df.high.values,df.low.values,df.close.values,df.volume.values
df['mfi']=ta.MFI(high, low, close, volume, timeperiod=14)
for i in range(14,len(df)):
if df['mfi'][i]>L and df['mfi'][i-1]<L:
df.loc[df.index[i],'收盘信号']=1
if df['mfi'][i]<H and df['mfi'][i-1]>H:
df.loc[df.index[i],'收盘信号']=0
df['当天仓位']=df['收盘信号'].shift(1)
df['当天仓位'].fillna(method='ffill',inplace=True)
d=df[df['当天仓位']==1].index[0]-timedelta(days=1)
df1=df.loc[d:]
df1['ret'][0]=0
df1['当天仓位'][0]=0
#当仓位为1时，买入上证指数，当仓位为0时，空仓，计算资金指数
df1['策略净值']=(df1.ret.values*df1['当天仓位'].values+1.0).cumprod()
df1['指数净值']=(df1.ret.values+1.0).cumprod()
df1['策略收益率']=df1['策略净值']/df1['策略净值'].shift(1)-1
df1['指数收益率']=df1.ret
total_ret=df1[['策略净值','指数净值']].iloc[-1]-1
annual_ret=pow(1+total_ret,250/len(df_new))-1
dd=(df1[['策略净值','指数净值']].cummax()-df1[['策略净值','指数净值']])/df1[['策略净值','指数净值']].cummax()
d=dd.max()
beta=df1[['策略收益率','指数收益率']].cov().iat[0,1]/df1['指数收益率'].var()
alpha=(annual_ret['策略净值']-annual_ret['指数净值']*beta)
exReturn=df1['策略收益率']-0.03/250
sharper_atio=np.sqrt(len(exReturn))*exReturn.mean()/exReturn.std()
TA1=round(total_ret['策略净值']*100,2)
TA2=round(total_ret['指数净值']*100,2)
AR1=round(annual_ret['策略净值']*100,2)
AR2=round(annual_ret['指数净值']*100,2)
MD1=round(d['策略净值']*100,2)
MD2=round(d['指数净值']*100,2)
S=round(sharper_atio,2)
df1[['策略净值','指数净值']].plot(figsize=(15,7))
plt.title('上证指数与MFI指标策略\n'+date+'至今',size=15)
bbox = dict(boxstyle="round", fc="w", ec="0.5", alpha=0.9)
plt.text(df1.index[int(len(df1)/5)], df1['指数净值'].max()/1.5, f'累计收益率：策略{TA1}%，指数{TA2}%;\n年化收益率：策略{AR1}%，指数{AR2}%；\n最大回撤： 策略{MD1}%，指数{MD2}%;\n策略alpha: {round(alpha,2)}，策略beta：{round(beta,2)}; \n夏普比率： {S}',size=13,bbox=bbox)
plt.xlabel('')
ax=plt.gca()
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
plt.show()
strategy('sh','2009-05-12',20,80)
strategy('sh','2009-04-12',20,90)
strategy('sh','2009-04-12',20,95)
strategy('sh','2009-04-12',30,95)
strategy('sh','2009-04-12',15,95)
strategy('sh','2016-01-01',20,90)
strategy('sh','2000-01-01',20,80)
strategy('sh','2000-01-01',20,92)
strategy('sh','2017-04-12',20,80)
strategy('sh','2017-04-12',20,92)
