仅用于个人记录，方便以后自己查阅
import requests
import os
import pymysql
import time
import json
def get_file_last_line(path):
file_size = os.path.getsize(path)
block_size = 50000
with open(path, 'rb') as f:
last_line = ""
if file_size > block_size:
max_seek_point = (file_size // block_size)
f.seek((max_seek_point - 1) * block_size)
elif file_size:
f.seek(0, 0)
lines = f.readlines()
if lines:
lineno = 1
while last_line == "":
last_line = lines[-lineno].strip()
lineno += 1
return last_line
def update(headers,Updateurl):
responseUpdate = requests.get(Updateurl, headers=headers)
def post_data(conn, subject, message):
cursor = conn.cursor()
uid = 1
fid = 37
view = 1
# subject="hello"
# message="1"
try:
cursor.execute('SELECT username FROM pre_common_member WHERE uid = ' + str(uid) + ";")
author = cursor.fetchone()[0] # 用户name
# print(author)
# 第一步给pre_forum_post_tableid表插入空数据进行自增，然后获取自增pid
cursor.execute('INSERT INTO pre_forum_post_tableid VALUES (NULL);')
cursor.execute('SELECT max(pid) FROM pre_forum_post_tableid;')
pid = cursor.fetchone()[0]
# print(pid)
# 第二步给pre_forum_thread表插入帖子标题数据，然后获取自增tid
sql_thread = "INSERT INTO pre_forum_thread SET fid=" + str(fid) + ",author='" + author + "',authorid=" + str(
uid) + ",subject='" + subject + "',dateline=" + str(
int(time.time())) + ",lastposter='" + author + "',lastpost=" + str(int(time.time())) + ",views=" + str(
view) + ";"
cursor.execute(sql_thread)
cursor.execute('SELECT max(tid) FROM pre_forum_thread')
tid = int(cursor.fetchone()[0])
# print(tid)
# 第三步给pre_forum_post表插入帖子的标题、内容等，pid、tid用上两步获得的数据 如要增加附件需修改attachment
sql_post = "INSERT INTO pre_forum_post SET pid=" + str(pid) + ",fid=" + str(fid) + ",tid=" + str(
tid) + ",first=1,author='" + author + "', authorid=" + str(
uid) + ", subject='" + subject + "' ,dateline=" + str(int(
time.time())) + ", message='" + message + "' , useip='192.168.31.1' ,port=11560 , invisible = 0, anonymous = 0 , usesig = 1 , htmlon = 1 , bbcodeoff =-1 , smileyoff =-1 , parseurloff =0 , attachment = 0 , tags='' , replycredit=0 , status=0;"
# print(sql_post)
cursor.execute(sql_post)
# cursor.execute('SELECT max(aid) FROM pre_forum_attachment')
# aid = int(cursor.fetchone()[0]) + 1
# 第四步给pre_forum_forum版块表进行更新帖子数量
sql_forum = 'UPDATE pre_forum_forum SET threads=threads+1, posts=posts+1, todayposts=todayposts+1 , allowsmilies = 1,allowbbcode = 1, allowimgcode =1 ,allowspecialonly = 1,allowglobalstick = 1,alloweditpost = 1 ,recyclebin =1 WHERE fid=' + str(
fid) + ';'
# print(sql_forum)
cursor.execute(sql_forum)
# 第五步给pre_common_member_count表更新用户帖子数量信息
sql_count = 'UPDATE pre_common_member_count SET threads = threads+1 WHERE uid =' + str(uid) + ';'
cursor.execute(sql_count)
# cursor.execute('INSERT INTO pre_forum_attachment_" + str(aid % 10) + " SET readperm='0' , price='10' , tid='" + str(tid) + "' , pid=' + pid+ ',uid=1 , description=, aid=' + str(aid) + ' ,dateline='+ str(int(time.time())) + ',filename="' + att_name + '", filesize=4, attachment="upload/' + att_name + '",remote=0, isimage=0, width=0, thumb=0;')
# cursor.execute('INSERT INTO pre_forum_attachment SET tid=' + str(tid) + ', pid=' +str(pid)+ ', tableid='+ str(aid % 10) + ', aid=' + str(aid) + ';')
# 提交，不然无法保存新建或者修改的数据
conn.commit()
except:
print("写入数据库失败，事物回滚！")
conn.rollback()
finally:
cursor.close()
def updateForum():
conn = pymysql.connect(host="192.168.31.134", user="jcy1998", passwd="12345678", db="ultrax")
# post_data(conn,'mysql测试','mysql')
file = open("update.txt",'r')
for line in file:
content = json.loads(line)
# print(content)
subject = str(content['TitleNum']) + str(' ') + content['Comments']['data']['content'] + str(' 学号') + \
content['Comments']['data']['stucode'] + str(' 姓名') + content['Comments']['data']['username']
subjectlen = len(subject)
if subjectlen > 75:
subject = subject[0:75]
ReplyLength = len(content['Reply']['data'])
message = ''
message = message + str(content['TitleNum']) + str(' ') + content['Comments']['data']['content'] + str(
' 学号') + content['Comments']['data']['stucode'] + str(' 姓名') + content['Comments']['data']['username']+ str(' 时间') + content['Comments']['data']['addtime']
if len(content['Comments']['data']['photo']) > 1:
photoList = content['Comments']['data']['photo'].split(',')
for photo in photoList:
message = message + "\n" + '[img]' + photo + '[/img]' + '\n'
if isinstance(content['Comments']['data']['imglist'], str):
temp = content['Comments']['data']['imglist']
photoList = temp.split(',')
for photo in photoList:
message = message + "\n" + '[img]' + photo + '[/img]' + '\n'
# if type(content['Comments']['data']['imglist']) == 'str':
# photoList = content['Comments']['data']['imglist'].split(',')
# for photo in photoList:
# message = message + "\n" + '[img]' + photo + '[/img]' + '\n'
# for photo in photoList:
# message=message+"\n"+'[img]'+content['Comments']['data']['photo']+'[/img]'
for i in range(0, ReplyLength, 1):
if type(content['Reply']['data'][i]['username']) != 'str':
content['Reply']['data'][i]['username'] = '*********'
message = message + content['Reply']['data'][i]['replaycontent'] + ' 回复时间' + content['Reply']['data'][i][
'replytime'] + ' 姓名' + content['Reply']['data'][i]['username'] + ' 学号' + \
content['Reply']['data'][i]['stucode'] + "\n"
if len(content['Comments']['data']['photo']) > 1:
photoList = content['Reply']['data'][i]['photo'].split(',')
for photo in photoList:
message = message + "\n" + '[img]' + photo + '[/img]' + '\n'
if isinstance(content['Reply']['data'][i]['imglist'], str):
photoList = content['Reply']['data'][i]['imglist'].split(',')
for photo in photoList:
message = message + "\n" + '[img]' + photo + '[/img]' + '\n'
# if type(content['Reply']['data'][i]['imglist']) == 'str':
# photoList = content['Comments']['data']['imglist'].split(',')
# for photo in photoList:
# message = message + "\n" + '[img]' + photo + '[/img]' + '\n'
# message=content['Reply']['data']
# pass # do something
# print(message)
post_data(conn, subject, message)
cursor = conn.cursor()
sqlstr = 'UPDATE pre_forum_post SET bbcodeoff = "0"'
cursor.execute(sqlstr)
file.close()
urlComment='https://lightapp.m.dlut.edu.cn/api/complainpraise/complainpraise/commentDetail?'
urlReply='https://lightapp.m.dlut.edu.cn/api/complainpraise/complainpraise/replyList?'
Updateurl='https://lightapp.m.dlut.edu.cn/api/complainpraise/complainpraise/latest?schcode=dlut&pagenum=1&pagesize=20&departmentid=&stucode='
headers = {
'authority': 'lightapp.m.dlut.edu.cn',
'cache-control': 'max-age=0',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'dnt': '1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'sec-fetch-site': 'none',
'sec-fetch-mode': 'navigate',
'sec-fetch-user': '?1',
'sec-fetch-dest': 'document',
'accept-language': 'en,zh;q=0.9,zh-CN;q=0.8,en-IN;q=0.7,hi;q=0.6,ru;q=0.5',
'cookie': 'Hm_lvt_2897656ea377e58fb1af08554ed019b4=1646746257; webapp_spa=s%3A3RjqyEu2oE65oM0GhpcUrRtYugXEmQHR.JAlwbHwrnvq8uhy1mLeXX%2F9N1NJOPInFd9JQ1F74E68',
'if-none-match': 'W/"3c7-1797d866bc0"',
'if-modified-since': 'Tue, 18 May 2021 03:30:32 GMT',
}
# 找最后一行编号
# print(get_file_last_line('json.txt'))
while True:
fUpdate = open("update.txt", 'w')
str1 = get_file_last_line('json.txt')
print(str1)
str1 = str1.decode()
print(str1)
lastSJson = json.loads(str1)
startNum = lastSJson['TitleNum']
print(lastSJson['TitleNum'])
responseUpdate = requests.get(Updateurl, headers=headers)
UpdateJson = json.loads(responseUpdate.content.decode('utf-8'))
UpdateNewestNum = UpdateJson['data'][0]['infoid']
print(UpdateNewestNum)
for i in range(startNum + 1, UpdateNewestNum + 1):
info = {}
dataSaveJson = json.loads(json.dumps(info))
titlenum = i
paramsComment = (
('infoid', titlenum),
('schcode', 'dlut'),
)
paramsReply = (
('schcode', 'dlut'),
('pagenum', 1),
('pagesize', 1000),
('infoid', titlenum),
)
responseComments = requests.get(urlComment, headers=headers, params=paramsComment)
responseReply = requests.get(urlReply, headers=headers, params=paramsReply)
# Comments=responseComments.content.decode('utf-8')
# print(Comments['data'])
# Comments=json.loads(Comments)
Comments = json.loads(responseComments.content.decode('utf-8'))
Reply = json.loads(responseReply.content.decode('utf-8'))
# 获得回复的长度
Replylength = len(Reply['data'])
if Comments['errcode'] == 0:
# TopicContent = Comments['data']['content']
# TopicAuthor = Comments['data']['stuname']
# TopicAuthorStuNum = Comments['data']['stucode']
# TopicTimeStamp = Comments['data']['addtime']
# 照片待补充
# for i in range(0, Replylength, 1):
# ReplyContent = Reply['data'][i]['replaycontent']
# ReplyTimeStamp = Reply['data'][i]['replytime']
# ReplyAuthor = Reply['data'][i]['stucode']
dataSaveJson['TitleNum'] = titlenum
dataSaveJson['Comments'] = Comments
dataSaveJson['Reply'] = Reply
write = json.dumps(dataSaveJson)
# print(write)
f = open("json.txt", 'a')
f.write(write + '\n')
# json.dump(dataSaveJson,f)
f.close()
fUpdate.write(write + '\n')
# print(Replylength)
# print(Comments)
# print(Reply['data'])
else:
print("该主题不存在")
print(i)
fUpdate.close()
updateForum()
for i in range(1, 30):
update(headers, Updateurl)
print("成功")
time.sleep(120)
# Note: original query string below. It seems impossible to parse and
# reproduce query strings 100% accurately so the one below is given
# in case the reproduced version is not "correct".
#response = requests.get('https://lightapp.m.dlut.edu.cn/complainpraise/commentdetail?infoid=20390', headers=headers)