提交文件造数据代码
This commit is contained in:
parent
96cab1dddb
commit
13fe7b0a49
|
|
@ -8,18 +8,21 @@ from common import global_var
|
|||
from pathlib import Path
|
||||
|
||||
from PySide6.QtUiTools import QUiLoader
|
||||
from PySide6.QtCore import Signal,QThread,QObject,QEventLoop,QTimer
|
||||
from PySide6.QtCore import Signal, QThread, QObject, QEventLoop, QTimer
|
||||
from PySide6.QtWidgets import QApplication
|
||||
from PySide6.QtGui import QTextCursor
|
||||
|
||||
# 声明全局变量
|
||||
global_var._init()
|
||||
global_var.set_value('FromGUI', True)
|
||||
|
||||
|
||||
# 控制台重定向到GUI
|
||||
class EmittingStr(QObject):
|
||||
# 定义信号
|
||||
textWritten = Signal(str)
|
||||
# 将控制台的内容输出到QTextBrowser
|
||||
|
||||
def write(self, text):
|
||||
# 调用 emit方法 发信号时，传入参数 必须是这里指定的 参数类型
|
||||
self.textWritten.emit(str(text))
|
||||
|
|
@ -29,6 +32,8 @@ class EmittingStr(QObject):
|
|||
QApplication.processEvents()
|
||||
|
||||
# logging重定向到print
|
||||
|
||||
|
||||
class PrintHandler(logging.Handler):
|
||||
def emit(self, record):
|
||||
pass
|
||||
|
|
@ -40,6 +45,7 @@ class FileThread(QThread):
|
|||
def __init__(self, parent=None):
|
||||
super().__init__(parent=parent)
|
||||
# 覆盖原来run去调用文件造数据脚本
|
||||
|
||||
def run(self):
|
||||
fileDataCreating.main()
|
||||
|
||||
|
|
@ -48,11 +54,11 @@ class FileThread(QThread):
|
|||
class Window:
|
||||
|
||||
def __init__(self):
|
||||
#载入ui
|
||||
# 载入ui
|
||||
path = Path(__file__)
|
||||
ui_file_patch = path.parent.parent / "ui" / "ui_main.ui"
|
||||
self.ui = QUiLoader().load(str(ui_file_patch))
|
||||
#指定按钮和槽函数
|
||||
# 指定按钮和槽函数
|
||||
self.ui.startButton.clicked.connect(self.start)
|
||||
self.ui.stopButton.clicked.connect(self.stop)
|
||||
self.ui.conn_Button.clicked.connect(self.conntest)
|
||||
|
|
@ -71,11 +77,12 @@ class Window:
|
|||
self.logger.setLevel(logging.INFO)
|
||||
# 创建并配置处理程序
|
||||
handler = PrintHandler()
|
||||
handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
||||
handler.setFormatter(logging.Formatter(
|
||||
'%(asctime)s - %(levelname)s - %(message)s'))
|
||||
self.logger.addHandler(handler)
|
||||
|
||||
|
||||
# 输出到GUI，信号处理函数
|
||||
|
||||
def outputWritten(self, text):
|
||||
cursor = self.ui.Log_Output.textCursor()
|
||||
cursor.movePosition(QTextCursor.End)
|
||||
|
|
@ -83,22 +90,28 @@ class Window:
|
|||
self.ui.Log_Output.setTextCursor(cursor)
|
||||
self.ui.Log_Output.ensureCursorVisible()
|
||||
|
||||
|
||||
def start(self):
|
||||
self.logger.info("点击了开始按钮")
|
||||
# 按钮置灰
|
||||
self.ui.startButton.setEnabled(False)
|
||||
# 输入框参数校验
|
||||
num = self.ui.tabWidget.currentIndex()
|
||||
if num == 0:
|
||||
self.check_file_input()
|
||||
else:
|
||||
self.conntest()
|
||||
# 开始运行
|
||||
if num == 0:
|
||||
if not self.check_file_input():
|
||||
self.logger.info("参数校验失败，请检查输入的值")
|
||||
self.ui.startButton.setEnabled(True)
|
||||
return
|
||||
self.logger.info("参数校验通过，开始执行")
|
||||
# 持续性生成和一次性的
|
||||
if self.ui.checkBox.isChecked():
|
||||
self.start_thread()
|
||||
else:
|
||||
fileDataCreating.new()
|
||||
self.ui.startButton.setEnabled(True)
|
||||
self.logger.info("*"*5 + "一次性任务执行完成" + "*"*5 )
|
||||
else:
|
||||
self.conntest()
|
||||
self.start_run_db()
|
||||
# 按钮置灰
|
||||
self.ui.startButton.setEnabled(False)
|
||||
|
||||
|
||||
def stop(self):
|
||||
|
|
@ -112,20 +125,17 @@ class Window:
|
|||
# self.startButton.isEnabled() 获取按钮的状态
|
||||
self.ui.startButton.setEnabled(True)
|
||||
|
||||
|
||||
def conntest(self):
|
||||
self.logger.info("点击了测试连接按钮")
|
||||
self.check_db_input()
|
||||
# TODO(MH):校验连接信息
|
||||
self.logger.info("数据库连接成功")
|
||||
|
||||
|
||||
def setup_thread(self):
|
||||
self.thread = FileThread()
|
||||
global_var.set_value('start_flag',True)
|
||||
global_var.set_value('start_flag', True)
|
||||
self.thread_running = global_var.get_value('start_flag')
|
||||
|
||||
|
||||
def start_thread(self):
|
||||
if self.thread_running:
|
||||
self.thread.start()
|
||||
|
|
@ -133,35 +143,51 @@ class Window:
|
|||
self.setup_thread()
|
||||
self.thread.start()
|
||||
|
||||
def check_common(self,elemnet):
|
||||
# 为空标红处理
|
||||
elemnet.setStyleSheet("")
|
||||
if not elemnet.text():
|
||||
elemnet.setStyleSheet("border: 1px solid red;")
|
||||
# 改变状态
|
||||
self.check_flag = False
|
||||
|
||||
return elemnet.text()
|
||||
|
||||
|
||||
def check_file_input(self):
|
||||
self.logger.info("文件参数校验通过")
|
||||
self.check_flag = True
|
||||
# 获取界面输入
|
||||
file_dict = {}
|
||||
file_dict['filename'] = self.check_common(self.ui.fileNameText)
|
||||
file_dict['outputpath'] = self.check_common(self.ui.fileOutputText)
|
||||
file_dict['fileinit'] = self.check_common(self.ui.fileInitRowsText)
|
||||
# 是否持续
|
||||
file_dict['filecreating'] = self.ui.checkBox.isChecked()
|
||||
if file_dict['filecreating']:
|
||||
file_dict['fileinsert'] = self.check_common(self.ui.fileInsertRowsText)
|
||||
file_dict['filedel'] = self.check_common(self.ui.fileDelRowsText)
|
||||
# 将值放到全局变量中
|
||||
global_var.set_value('file_args', file_dict)
|
||||
|
||||
return self.check_flag
|
||||
|
||||
def check_db_input(self):
|
||||
self.logger.info("数据库参数校验通过，开始连接数据库测试。。。")
|
||||
|
||||
|
||||
def start_run_file(self):
|
||||
|
||||
# 调用文件处理脚本
|
||||
self.logger.info("开始生成文件。。。")
|
||||
|
||||
|
||||
|
||||
def start_run_db(self):
|
||||
pass
|
||||
|
||||
|
||||
def stop_run_file(self):
|
||||
# 终止线程的事件循环
|
||||
global_var.set_value('start_flag',False)
|
||||
global_var.set_value('start_flag', False)
|
||||
self.thread.quit()
|
||||
# 标记线程停止
|
||||
self.thread_running = global_var.get_value('start_flag')
|
||||
|
||||
|
||||
|
||||
def stop_run_db(self):
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
def _init(): # 初始化
|
||||
global _global_dict
|
||||
_global_dict = {}
|
||||
_global_dict = {"FromGUI":False}
|
||||
|
||||
def set_value(key, value):
|
||||
#定义一个全局变量
|
||||
|
|
|
|||
|
|
@ -39,31 +39,55 @@ logger.addHandler(handler)
|
|||
|
||||
# 定义公共的部分
|
||||
path = Path(__file__)
|
||||
etc_dir = path.parent.parent.parent / "etc"
|
||||
config_file_patch = etc_dir / "file_config.yml"
|
||||
|
||||
# 获取配置文件
|
||||
with open(config_file_patch, "r", encoding='utf-8') as fy:
|
||||
def upconfig():
|
||||
# 输出路径默认为当前的目录中
|
||||
data_dir = path.parent.parent.parent / "data"
|
||||
InsertRows=200
|
||||
DeleteRows=100
|
||||
# 默认从全局参数中获取参数
|
||||
whorun = global_var.get_value('FromGUI')
|
||||
filedict=global_var.get_value('file_args')
|
||||
fname = filedict['filename']
|
||||
initRows = int(filedict['fileinit'])
|
||||
# 界面如果有输入就取界面的值
|
||||
if filedict['outputpath']:
|
||||
data_dir = Path(filedict['outputpath'])
|
||||
if filedict['filecreating']:
|
||||
InsertRows = int(filedict['fileinsert'])
|
||||
DeleteRows = int(filedict['filedel'])
|
||||
|
||||
# 兼容脚本处理
|
||||
if not whorun:
|
||||
# 从配置文件中获取参数
|
||||
etc_dir = path.parent.parent.parent / "etc"
|
||||
config_file_patch = etc_dir / "file_config.yml"
|
||||
# 获取配置文件
|
||||
with open(config_file_patch, "r", encoding='utf-8') as fy:
|
||||
config = yaml.safe_load(fy)
|
||||
|
||||
# 默认为当前的目录
|
||||
data_dir = path.parent.parent.parent / "data"
|
||||
if 'DATADIR' in config.keys() and config['DATADIR']:
|
||||
fname = config['filename']
|
||||
if 'DATADIR' in config.keys() and config['DATADIR']:
|
||||
data_dir = Path(config['DATADIR'])
|
||||
initRows = config['initRows']
|
||||
InsertRows = config['InsertRows']
|
||||
DeleteRows = config['DeleteRows']
|
||||
|
||||
# 初始化目录
|
||||
if not data_dir.exists():
|
||||
# 初始化目录
|
||||
if not data_dir.exists():
|
||||
# shutil.rmtree(data_dir)
|
||||
data_dir.mkdir()
|
||||
return [fname,data_dir,initRows,InsertRows,DeleteRows]
|
||||
|
||||
# 定义文件的信息
|
||||
def file_info():
|
||||
varlist = upconfig()
|
||||
file_info_list = []
|
||||
now = datetime.now()
|
||||
now_day = now.strftime("%Y-%m-%d")
|
||||
now_hours = now.strftime("%Y-%m-%d-%H")
|
||||
today_dir = data_dir/now_day
|
||||
filename = "filetest_" + now_hours + ".csv"
|
||||
today_dir = varlist[1]/now_day
|
||||
filename = varlist[0] + "_" + now_hours + ".csv"
|
||||
filepath = today_dir/filename
|
||||
|
||||
file_info_list.append(now_day)
|
||||
|
|
@ -77,22 +101,24 @@ def file_info():
|
|||
# 每小时生成一个csv文件
|
||||
# @repeat(every(1).hours)--MH:这个是基于运行的时间，不是基于系统时间
|
||||
def new():
|
||||
logger.info('new...')
|
||||
varlist = upconfig()
|
||||
# 创建文件夹
|
||||
finfo = file_info()
|
||||
if not finfo[2].exists():
|
||||
finfo[2].mkdir()
|
||||
# 创建文件
|
||||
faker_data.save_data_csv(finfo[4], lines=100)
|
||||
logger.info("开始创建文件：{}".format(finfo[4]))
|
||||
faker_data.save_data_csv(finfo[4], lines=varlist[2])
|
||||
|
||||
|
||||
# 每1秒插入N条数据
|
||||
@repeat(every(1).seconds)
|
||||
def inserting():
|
||||
varlist = upconfig()
|
||||
finfo = file_info()
|
||||
if finfo[4].exists():
|
||||
logger.info('insert...')
|
||||
datas = faker_data.faker_data(lines=config['InsertRows'])[1:]
|
||||
datas = faker_data.faker_data(lines=varlist[3])[1:]
|
||||
logger.info("正在向{0}中插入{1}条数据...".format(finfo[3],varlist[3]))
|
||||
with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv:
|
||||
writer = csv.writer(file_csv, delimiter=',',
|
||||
quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
|
|
@ -106,34 +132,37 @@ def inserting():
|
|||
# 每隔10分钟删除100条数据
|
||||
@repeat(every(10).minutes)
|
||||
def deleting_data():
|
||||
varlist = upconfig()
|
||||
finfo = file_info()
|
||||
# 获取所有文件
|
||||
files = list(finfo[2].glob('*.csv'))
|
||||
if len(files) > 1:
|
||||
file = choice(files[:-1])
|
||||
logger.info(str(file) + "start delete data ....")
|
||||
# 删除掉前N条数据
|
||||
with open(file, 'rb') as fr:
|
||||
data = fr.readlines()
|
||||
new_data = data[config['DeleteRows']:]
|
||||
new_data = data[varlist[4]:]
|
||||
# 少于100条的不删除
|
||||
if len(new_data) > 100:
|
||||
with open(file, 'wb') as fw:
|
||||
fw.writelines(new_data)
|
||||
logger.info("已经从文件：{0} 中删除前{1}条".format(file,varlist[4]) )
|
||||
else:
|
||||
logger.info("file number is less 1,wait next time.")
|
||||
logger.info("文件总数小于1跳过本次删除，等待下一次。")
|
||||
|
||||
|
||||
# 每隔6小时删除1个文件,低于3个不删除
|
||||
@repeat(every(6).hours)
|
||||
# @repeat(every(2).seconds)
|
||||
def deleting_file():
|
||||
varlist = upconfig()
|
||||
logger.info("deleting file ....")
|
||||
# 从data目录中随机选一个
|
||||
files = list(data_dir.rglob('*.csv'))
|
||||
files = list(varlist[1].rglob('*.csv'))
|
||||
if len(files) > 3:
|
||||
file = choice(files[:-1])
|
||||
file.unlink()
|
||||
logger.info("文件：{0} 已删除".format(file) )
|
||||
else:
|
||||
logger.info("file num is less 3, not delete. wait next time.")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
#生成文件的配置文件
|
||||
vsersion : 1.3
|
||||
|
||||
filename : "filetest"
|
||||
# 生成csv输出的目录，默认为程序的运行目录。需要自定义就打开配置
|
||||
#DATADIR : "D:\\data"
|
||||
|
||||
#初始数据
|
||||
initRows: 101
|
||||
# 每次插入多少条数据
|
||||
InsertRows: 200
|
||||
InsertRows: 201
|
||||
# 每次删除多少条数据
|
||||
DeleteRows: 100
|
||||
DeleteRows: 101
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue