陈予同的实验报告
第一步 下载年报
from time import sleep
from urllib import parse
import os
import json
import requests
def get_address(bank_code):
url = "http://www.cninfo.com.cn/new/information/topSearch/detailOfQuery"
data = {
'keyWord': bank_code,
'maxSecNum': 10,
'maxListNum': 5,
}
hd = {
'Host': 'www.cninfo.com.cn',
'Origin': 'http://www.cninfo.com.cn',
'Pragma': 'no-cache',
'Accept-Encoding': 'gzip,deflate',
'Connection': 'keep-alive',
'Content-Length': '70',
'User-Agent': 'Mozilla/5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 75.0.3770.100Safari / 537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Accept': 'application/json,text/plain,*/*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
r = requests.post(url, headers=hd, data=data)
print(r.text)
r = r.content
m = str(r, encoding="utf-8")
pk = json.loads(m)
orgId = pk["keyBoardList"][0]["orgId"] # 获取参数
plate = pk["keyBoardList"][0]["plate"]
code = pk["keyBoardList"][0]["code"]
print(orgId, plate, code)
return orgId, plate, code
def download_PDF(url, file_name):
url = url
r = requests.get(url)
f = open(code + "/" + file_name + ".pdf", "wb")
f.write(r.content)
def get_PDF(orgId, plate, code):
url = "http://www.cninfo.com.cn/new/hisAnnouncement/query"
data = {
'stock': '{},{}'.format(code, orgId),
'tabName': 'fulltext',
'pageSize': 20,
'pageNum': 1,
'column': plate,
'category': 'category_ndbg_szsh;',
'plate': '',
'seDate': '',
'searchkey': '',
'secid': '',
'sortName': '',
'sortType': '',
'isHLtitle': 'true',
}
hd = {
'Host': 'www.cninfo.com.cn',
'Origin': 'http://www.cninfo.com.cn',
'Pragma': 'no-cache',
'Accept-Encoding': 'gzip,deflate',
'Connection': 'keep-alive',
# 'Content-Length': '216',
'User-Agent': 'User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Accept': 'application/json,text/plain,*/*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'X-Requested-With': 'XMLHttpRequest',
# 'Cookie': cookies
}
data = parse.urlencode(data)
print(data)
r = requests.post(url, headers=hd, data=data)
print(r.text)
r = str(r.content, encoding="utf-8")
r = json.loads(r)
reports_list = r['announcements']
for report in reports_list:
if '摘要' in report['announcementTitle'] or "20" not in report['announcementTitle']:
continue
if 'H' in report['announcementTitle']:
continue
else: # http://static.cninfo.com.cn/finalpage/2019-03-29/1205958883.PDF
pdf_url = "http://static.cninfo.com.cn/" + report['adjunctUrl']
file_name = report['announcementTitle']
print("正在下载:" + pdf_url, "存放在当前目录:/" + code + "/" + file_name)
download_PDF(pdf_url, file_name)
sleep(2)
if __name__ == '__main__':
#code list
code_list = ['002286','002311', '002330', '002385', '002515', '002548', '002557', '002567', '002582', '002695']
for code in code_list:
os.mkdir(code)
orgId, plate, code = get_adress(code)
get_PDF(orgId, plate, code)
print("next bank")
print("Finished")
结果展示
第二步 提取公司基本信息
import pdfplumber
import pandas as pd
#以002286为例
filename="C:/Users/Administrator/Desktop/大作业/002286/2013年年度报告.pdf"
pdf = pdfplumber.open(filename)
page1 = pdf.pages[5]
table1 = page1.extract_tables()[0]
df1 = pd.DataFrame(table1)
stock1=pd.concat([df1], ignore_index=True)
stock1.set_index([0],inplace=True)
stock1=stock1.iloc[:,0]
stock1=pd.DataFrame(stock1)
stock1.columns=["002286"]
allinfo= pd.concat([stock1], axis=1, ignore_index=False)
allinfo=allinfo.iloc[1:]
结果展示
第三步 获取数据并画图
import matplotlib.pyplot as plt
from pylab import mpl
import os
import re
file_dir = "C:\\Users\\Administrator\\Desktop\\大作业"
filepath=file_dir
filelist=[]
pdfpath=[]
list1= os.listdir(filepath)
for file1 in list1:
path = os.path.join(filepath, file1)
# print(file1)
if os.path.isfile(path):
filelist.append(file1)
pdfpath.append(path)
else:
list2=os.listdir(path)
for file2 in list2:
path2 = os.path.join(path, file2)
# print(file2)
if os.path.isfile(path2):
filelist.append(file2)
pdfpath.append(path2)
#start
def get_subtxt(doc,bounds=('主要会计数据和财务指标','总资产')):
#默认设置为首尾页码
start_pageno=0
end_pageno=len(doc)-1
#
lb,ub=bounds
#获取左界页码
for n in range(len(doc)):
page=doc[n]
txt=page.get_text()
if lb in txt:
start_pageno=n
break
#获取右界页码
for n in range(start_pageno,len(doc)):
if ub in doc[n].get_text():
end_pageno=n
break
#获取小范围内字符串
txt=''
for n in range(start_pageno,end_pageno+1):
page=doc[n]
txt += page.get_text()
return(txt)
#获取表头
def get_th_span(txt):
nianfen='(20\d\d|199\d)\s*?年' #2016和年之间是空格,而2016年和2015年之间是空格
s=f'{nianfen}\s*{nianfen}.*?{nianfen}'
p=re.compile(s,re.DOTALL) #re.DOTALL指.遇到换行符也是可以的
matchobj=p.search(txt)
#
end=matchobj.end()
year1=matchobj.group(1)
year2=matchobj.group(2)
year3=matchobj.group(3)
#
flag=(int(year1)-int(year2) == 1) and (int(year2)-int(year3) == 1)
#
while (not flag):
matchobj=p.search(txt[end:])
end=matchobj.end()
year1=matchobj.group(1)
year2=matchobj.group(2)
year3=matchobj.group(3)
flag=(int(year1)-int(year2) == 1)
flag=flag and (int(year2)-int(year3) ==1)
#获取表格边界
def get_bounds(txt):
th_span_1st=get_th_span(txt)
end=th_span_1st[1]
th_span_2nd=get_th_span(txt[end:])
th_span_2nd=(end+th_span_2nd[0],end+th_span_2nd[1])
#
s=th_span_1st[1]
e=th_span_2nd[0]-1
#
while (txt[e] not in '0123456789'): #如果最后一个不是数字
e=e-1
return(s,e+1)
#获取‘营业收入’和‘归属于上市公司股\n东的净利润’
def get_account_data(account,txt):
p_txt='%s\D*?(\d{1,3}(?:,\d{3})*(?:\.\d+)?)' % account #%s是占位符,用‘account’替换,\D是非数字,\d{1,3}是数字1或2或3个,*可重复,?非贪婪,()内是所要的数字,小数点后\d+表示小数点后至少一位数字
p=re.compile(p_txt)
matchobj=p.search(txt)
amt=matchobj.group(1)
return(amt)
#获取整张表格
# subtxt=txt[txt.find('营业收入'):txt.find('总资产')]
def get_keywords(txt):
p=re.compile(r's\D*?(\d{1,3}(?:,\d{3})*(?:\.\d+)?)')
keywords=p.findall(txt)
return(keywords)
###
def parse_key_fin_data(subtxt,keywords):
# keywords=['营业收入','营业成本','毛利','归属于上市','归属于上市','经营活动']
ss=[]
s=0
for kw in keywords:
n=subtxt.find(kw,s)
ss.append(n)
s=n+len(kw)
ss.append(len(subtxt))
data=[]
p=re.compile('\D+(?:\s+\D*)?(?:(.*)|\(.*\))?')
p2=re.compile('\s')
for n in range(len(ss)-1):
s=ss[n]
e=ss[n+1]
line=subtxt[s:e]
#获取可能换行的账户名称
matchobj=p.search(line)
account_name=p2.sub('',matchobj.group())
#获取三年数据
amnts=line[matchobj.end():].split()
#加上账户名称
amnts.insert(0,account_name)
#追加到总数据
data.append(amnts)
return data
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pylab import mpl
import datetime
mpl.rcParams['font.sans-serif']=['FangSong']
mpl.rcParams['axes.unicode_minus']=False
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
import fitz
#以002286为例
stock1_sr=[]
stock1_jlr=[]
filename='C:\\Users\\Administrator\\Desktop\\大作业\\002286\\2013年年度报告.pdf'
doc=fitz.open(filename)
txt=get_subtxt(doc)
span=get_bounds(txt)
subtxt=txt[txt.find('营业收入'):txt.find('总资产')]
keywords=['营业收入','归属于上市公司股东的净利润']
data=parse_key_fin_data(subtxt, keywords)
data_get=data[0]
stock1_sr.append(data_get[4])
stock1_jlr.append(data_get[10])
stock1_sr.append(data_get[2])
stock1_jlr.append(data_get[8])
stock1_sr.append(data_get[1])
stock1_jlr.append(data_get[7])
filename='C:\\Users\\Administrator\\Desktop\\大作业\\002286\\2016年年度报告.pdf'
doc=fitz.open(filename)
txt=get_subtxt(doc)
span=get_bounds(txt)
subtxt=txt[txt.find('营业收入'):txt.find('总资产')]
keywords=['营业收入','归属于上市公司股东的净利润']
data=parse_key_fin_data(subtxt, keywords)
data_get=data[0]
stock1_sr.append(data_get[5])
stock1_jlr.append(data_get[13])
stock1_sr.append(data_get[2])
stock1_jlr.append(data_get[10])
stock1_sr.append(data_get[1])
stock1_jlr.append(data_get[9])
filename='C:\\Users\\Administrator\\Desktop\\大作业\\002286\\2019年年度报告.pdf'
doc=fitz.open(filename)
txt=get_subtxt(doc)
span=get_bounds(txt)
subtxt=txt[txt.find('营业收入'):txt.find('总资产')]
keywords=['营业收入','归属于上市公司股东的净利润']
data=parse_key_fin_data(subtxt, keywords)
data_get=data[0]
stock1_sr.append(data_get[4])
stock1_jlr.append(data_get[10])
stock1_sr.append(data_get[2])
stock1_jlr.append(data_get[8])
stock1_sr.append(data_get[1])
stock1_jlr.append(data_get[7])
filename='C:\\Users\\Administrator\\Desktop\\大作业\\002286\\2022年年度报告.pdf'
doc=fitz.open(filename)
txt=get_subtxt(doc)
span=get_bounds(txt)
subtxt=txt[txt.find('营业收入'):txt.find('总资产')]
keywords=['营业收入','归属于上市公司股东的净利润']
data=parse_key_fin_data(subtxt, keywords)
data_get=data[0]
stock1_sr.append(data_get[2])
stock1_jlr.append(data_get[8])
stock1_sr.append(data_get[1])
stock1_jlr.append(data_get[7])
# year=['2013年','2016年','2019年','2022年']
#stock1
stock11=[]
tran(stock1_sr,stock11)
stock12=[]
tran(stock1_jlr,stock12)
ykzj=['2013年','2014年','2015年','2016年','2017年','2018年','2019年','2020年','2021年','2022年']
plt.figure(figsize=(9, 6))
plt.plot(ykzj,stock11)
plt.bar(ykzj,stock11)
plt.title(u'002286营业收入')
plt.legend()
plt.show()
plt.figure(figsize=(9, 6))
plt.plot(ykzj,stock12)
plt.bar(ykzj,stock12)
plt.title(u'002285归属于上市公司股东的净利润')
plt.show()
结果展示(以002286为例)
多公司比较
import matplotlib.pyplot as plt
#多公司比较
#营业收入
#form={'东鹏饮料':dp1}
a = ['2013','2014','2015','2016','2017','2018','2019','2020','2021','2022']
b_1 =stock11
b_2 = [None,None,None]+stock21
b_3 = stock31
b_4 = stock41
b_5 = [None]+stock51
b_6 = stock61
b_7 = stock71
b_8 = [None]+stock81
b_9 = stock91
b_10 = stock101
#设置图片尺寸与清晰度
plt.figure(figsize=(10, 8), dpi=80)
#导入数据,绘制条形图
plt.plot( a, b_1, marker = "o", mfc = "white", label='002286')
plt.plot( a,b_2, marker = "o", mfc = "white",label='002311')
plt.plot( a,b_3, marker = "o", mfc = "white",label='002330')
plt.plot( a,b_4, marker = "o", mfc = "white", label='002385')
plt.plot( a,b_5, marker = "o", mfc = "white",label='002515')
plt.plot( a,b_6, marker = "o", mfc = "white",label='002548')
plt.plot( a,b_7, marker = "o", mfc = "white",label='002557')
plt.plot( a,b_8, marker = "o", mfc = "white", label='002567')
plt.plot( a,b_9, marker = "o", mfc = "white", label='002582')
plt.plot( a,b_10, marker = "o", mfc = "white", label='002695')
#添加标题
plt.title('每一年的营业收入对比图', size=20)
#添加xy轴
plt.xlabel('时间')
plt.ylabel('营业收入(元)')
#x轴刻度
plt.grid()
plt.legend()
#展示效果图
plt.show()
结果
心得
这次大作业对我来说很有难度,但学到的知识技能非常有用。虽然课程结束了,但我还需多多练习,直至能够熟练运用。