From e78d291936ecdfbce1f37d0dee87c28b43abf04f Mon Sep 17 00:00:00 2001 From: halliday2023 Date: Mon, 12 Feb 2024 22:19:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=96=87=E4=BB=B6=E9=80=A0?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9A=84=E5=BC=80=E5=A7=8B=E5=92=8C=E5=81=9C?= =?UTF-8?q?=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/GUIWindows.py | 20 ++++-- bin/faker_data.py | 81 +++++++++++++++++++++++ bin/fileDataCreating.py | 138 ++++++++++++++++++++++++++++++++++++++++ bin/global_var.py | 17 +++++ 4 files changed, 250 insertions(+), 6 deletions(-) create mode 100644 bin/faker_data.py create mode 100644 bin/fileDataCreating.py create mode 100644 bin/global_var.py diff --git a/bin/GUIWindows.py b/bin/GUIWindows.py index 6ceaf9f..d93c3e8 100644 --- a/bin/GUIWindows.py +++ b/bin/GUIWindows.py @@ -1,20 +1,22 @@ # -*- coding:utf-8 -*- +import os import sys import fileDataCreating +import global_var from pathlib import Path from PySide6.QtUiTools import loadUiType from PySide6.QtCore import Slot,Signal,QThread,QMutex,QWaitCondition,QMutexLocker - +global_var._init() # 定义公共路径 path = Path(__file__) ui_file_patch = path.parent.parent / "ui" / "ui_main.ui" # 从文件中加载UI定义 formType, baseType = loadUiType(str(ui_file_patch)) -class MyThread(QThread): +class FileThread(QThread): def __init__(self, parent=None): super().__init__(parent=parent) @@ -69,12 +71,17 @@ class Window(formType, baseType): def setup_thread(self): - self.thread = MyThread() - self.thread_running = True + self.thread = FileThread() + global_var.set_value('start_flag',True) + self.thread_running = global_var.get_value('start_flag') + def start_thread(self): if self.thread_running: self.thread.start() + else: + self.setup_thread() + self.thread.start() def check_file_input(self): @@ -97,8 +104,9 @@ class Window(formType, baseType): def stop_run_file(self): - # 终止线程的事件循环 - self.thread.quit() + global_var.set_value('start_flag',False) + self.thread.quit() # 终止线程的事件循环 + self.thread_running = False # 标记线程停止 diff --git a/bin/faker_data.py b/bin/faker_data.py new file mode 100644 index 0000000..868c1ad --- /dev/null +++ b/bin/faker_data.py @@ -0,0 +1,81 @@ +#encoding=utf-8 +import csv +from faker import Faker +from datetime import datetime + +""" +生成尽可能真实的假数据 + +使用方法： +1.安装依赖包 + pip install faker +2.执行脚本 + python faker_data.py +3.当前目录，查看生成的数据 +""" + +#保存为csv文件 +def save_data_csv(file_name,lines=100): + #获取数据 + datas = faker_data(lines) + #保存 + with open(file_name,'w+',encoding='utf-8',newline='') as file_csv: + writer = csv.writer(file_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) + writer.writerows(datas) + +#生成数据,默认10条 +def faker_data(lines=10): + #指定数据的国家地区 + f = Faker('zh-CN') + #定义一个列表，用来存放所有数据 + datas = [] + #标题 + title = ["uuid","id","name","mobile","ssn","sex","email","job","address","actime_time"] + #title2 = ["唯一标识","编号","姓名","手机号","身份证号","性别","邮箱","职业","家庭地址","获取时间"] + #添加标题到列表中 + datas.append(title) + #datas.append(title2) + #开始按照标题的顺序，生成N条数据 + for i in range(0,lines): + #定义一个列表，用来存一行数据 + data = [] + #uuid + data.append(f.uuid4()) + #编号，001，不足3位的左边用0来补齐 + data.append(str(i+1).rjust(3,'0')) + #姓名 + data.append(f.name()) + #手机号 + data.append(f.phone_number()) + #身份证 + ssn = f.ssn() + data.append(ssn) + #性别，根据身份证的第17位来判断 + ssn_sex = int(ssn[16:17]) + #01：男，02：女 + if ssn_sex % 2: + sex = "01" + else: + sex = "02" + data.append(sex) + #邮箱 + data.append(f.email()) + #职业 + data.append(f.job()) + #地址，让其更加复合中国的地址 + address = f.address()[:-9] + str(f.pyint(min_value=0, max_value=999))+ "号" + data.append(address) + #获取当前时间 + actime_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') + data.append(actime_time) + #将这一行数据添加到datas中 + datas.append(data) + #返回所有的数据 + return datas + + +if __name__ == '__main__': + #文件名 + file_name = 'test.csv' + save_data_csv(file_name) + diff --git a/bin/fileDataCreating.py b/bin/fileDataCreating.py new file mode 100644 index 0000000..a21276f --- /dev/null +++ b/bin/fileDataCreating.py @@ -0,0 +1,138 @@ +# -*- coding:utf-8 -*- +import os +import _load + +import faker_data +import csv +import yaml +import time +import global_var + +from schedule import every, repeat, run_pending +from pathlib import Path +from datetime import datetime +from random import choice + +""" +说明： +1. 用于文件的持续创建、写入、修改和创建 +2. 输出目录为data，会在里面创建当天的子目录 +3. 每1小时，生成一个CSV文件 +4. 每1秒，向上面的文件中写入N条数据 +5. 每10分钟，从当天的文件中随机找个1个文件，删除前N条数据 +6. 每6小时，从所有的文件中随机删除1个文件 +""" + + +# 定义公共的部分 +path = Path(__file__) +etc_dir = path.parent.parent / "etc" +config_file_patch = etc_dir / "file_config.yml" + +# 获取配置文件 +with open(config_file_patch, "r", encoding='utf-8') as fy: + config = yaml.safe_load(fy) + +# 默认为当前的目录 +data_dir = path.parent.parent / "data" +if 'DATADIR' in config.keys() and config['DATADIR']: + data_dir = Path(config['DATADIR']) + +# 初始化目录 +if not data_dir.exists(): + # shutil.rmtree(data_dir) + data_dir.mkdir() + +# 定义文件的信息 +def file_info(): + file_info_list = [] + now = datetime.now() + now_day = now.strftime("%Y-%m-%d") + now_hours = now.strftime("%Y-%m-%d-%H") + today_dir = data_dir/now_day + filename = "filetest_" + now_hours + ".csv" + filepath = today_dir/filename + + file_info_list.append(now_day) + file_info_list.append(now_hours) + file_info_list.append(today_dir) + file_info_list.append(filename) + file_info_list.append(filepath) + + return (file_info_list) + +# 每小时生成一个csv文件 +# @repeat(every(1).hours)--MH:这个是基于运行的时间，不是基于系统时间 +def new(): + print('new...') + # 创建文件夹 + finfo = file_info() + if not finfo[2].exists(): + finfo[2].mkdir() + # 创建文件 + faker_data.save_data_csv(finfo[4], lines=100) + + +# 每2秒插入N条数据 +@repeat(every(2).seconds) +def inserting(): + finfo = file_info() + if finfo[4].exists(): + print('insert...') + datas = faker_data.faker_data(lines=config['InsertRows'])[1:] + with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv: + writer = csv.writer(file_csv, delimiter=',', + quotechar='"', quoting=csv.QUOTE_ALL) + writer.writerows(datas) + else: + print(str(finfo[4])+" is not exists,wating") + # 手动去调一下，让其整点创建 + new() + + +# 每隔10分钟删除100条数据 +@repeat(every(10).minutes) +def deleting_data(): + finfo = file_info() + # 获取所有文件 + files = list(finfo[2].glob('*.csv')) + if len(files) > 1: + file = choice(files[:-1]) + print(str(file) + "start delete data ....") + # 删除掉前N条数据 + with open(file, 'rb') as fr: + data = fr.readlines() + new_data = data[config['DeleteRows']:] + # 少于100条的不删除 + if len(new_data) > 100: + with open(file, 'wb') as fw: + fw.writelines(new_data) + else: + print("file number is less 1,wait next time.") + + +# 每隔6小时删除1个文件,低于3个不删除 +@repeat(every(6).hours) +# @repeat(every(2).seconds) +def deleting_file(): + print("deleting file ....") + # 从data目录中随机选一个 + files = list(data_dir.rglob('*.csv')) + if len(files) > 3: + file = choice(files[:-1]) + file.unlink() + else: + print("file num is less 3, not delete. wait next time.") + +def main(): + start_flag = True + while start_flag: + start_flag = global_var.get_value('start_flag') + run_pending() + time.sleep(0.3) + print("已经停止") + + +if __name__ == '__main__': + main() + \ No newline at end of file diff --git a/bin/global_var.py b/bin/global_var.py new file mode 100644 index 0000000..f4bc466 --- /dev/null +++ b/bin/global_var.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +def _init(): # 初始化 + global _global_dict + _global_dict = {} + +def set_value(key, value): + #定义一个全局变量 + _global_dict[key] = value + +def get_value(key): + #获得一个全局变量，不存在则提示读取对应变量失败 + try: + return _global_dict[key] + except: + print('读取'+key+'失败\r\n') + \ No newline at end of file