提交文件造数据的开始和停止
This commit is contained in:
parent
ed2dd9cce1
commit
e78d291936
|
|
@ -1,20 +1,22 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import os
|
||||
import sys
|
||||
import fileDataCreating
|
||||
import global_var
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from PySide6.QtUiTools import loadUiType
|
||||
from PySide6.QtCore import Slot,Signal,QThread,QMutex,QWaitCondition,QMutexLocker
|
||||
|
||||
|
||||
global_var._init()
|
||||
# 定义公共路径
|
||||
path = Path(__file__)
|
||||
ui_file_patch = path.parent.parent / "ui" / "ui_main.ui"
|
||||
# 从文件中加载UI定义
|
||||
formType, baseType = loadUiType(str(ui_file_patch))
|
||||
|
||||
class MyThread(QThread):
|
||||
class FileThread(QThread):
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent=parent)
|
||||
|
|
@ -69,12 +71,17 @@ class Window(formType, baseType):
|
|||
|
||||
|
||||
def setup_thread(self):
|
||||
self.thread = MyThread()
|
||||
self.thread_running = True
|
||||
self.thread = FileThread()
|
||||
global_var.set_value('start_flag',True)
|
||||
self.thread_running = global_var.get_value('start_flag')
|
||||
|
||||
|
||||
def start_thread(self):
|
||||
if self.thread_running:
|
||||
self.thread.start()
|
||||
else:
|
||||
self.setup_thread()
|
||||
self.thread.start()
|
||||
|
||||
|
||||
def check_file_input(self):
|
||||
|
|
@ -97,8 +104,9 @@ class Window(formType, baseType):
|
|||
|
||||
|
||||
def stop_run_file(self):
|
||||
# 终止线程的事件循环
|
||||
self.thread.quit()
|
||||
global_var.set_value('start_flag',False)
|
||||
self.thread.quit() # 终止线程的事件循环
|
||||
self.thread_running = False # 标记线程停止
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
#encoding=utf-8
|
||||
import csv
|
||||
from faker import Faker
|
||||
from datetime import datetime
|
||||
|
||||
"""
|
||||
生成尽可能真实的假数据
|
||||
|
||||
使用方法：
|
||||
1.安装依赖包
|
||||
pip install faker
|
||||
2.执行脚本
|
||||
python faker_data.py
|
||||
3.当前目录，查看生成的数据
|
||||
"""
|
||||
|
||||
#保存为csv文件
|
||||
def save_data_csv(file_name,lines=100):
|
||||
#获取数据
|
||||
datas = faker_data(lines)
|
||||
#保存
|
||||
with open(file_name,'w+',encoding='utf-8',newline='') as file_csv:
|
||||
writer = csv.writer(file_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
writer.writerows(datas)
|
||||
|
||||
#生成数据,默认10条
|
||||
def faker_data(lines=10):
|
||||
#指定数据的国家地区
|
||||
f = Faker('zh-CN')
|
||||
#定义一个列表，用来存放所有数据
|
||||
datas = []
|
||||
#标题
|
||||
title = ["uuid","id","name","mobile","ssn","sex","email","job","address","actime_time"]
|
||||
#title2 = ["唯一标识","编号","姓名","手机号","身份证号","性别","邮箱","职业","家庭地址","获取时间"]
|
||||
#添加标题到列表中
|
||||
datas.append(title)
|
||||
#datas.append(title2)
|
||||
#开始按照标题的顺序，生成N条数据
|
||||
for i in range(0,lines):
|
||||
#定义一个列表，用来存一行数据
|
||||
data = []
|
||||
#uuid
|
||||
data.append(f.uuid4())
|
||||
#编号，001，不足3位的左边用0来补齐
|
||||
data.append(str(i+1).rjust(3,'0'))
|
||||
#姓名
|
||||
data.append(f.name())
|
||||
#手机号
|
||||
data.append(f.phone_number())
|
||||
#身份证
|
||||
ssn = f.ssn()
|
||||
data.append(ssn)
|
||||
#性别，根据身份证的第17位来判断
|
||||
ssn_sex = int(ssn[16:17])
|
||||
#01：男，02：女
|
||||
if ssn_sex % 2:
|
||||
sex = "01"
|
||||
else:
|
||||
sex = "02"
|
||||
data.append(sex)
|
||||
#邮箱
|
||||
data.append(f.email())
|
||||
#职业
|
||||
data.append(f.job())
|
||||
#地址，让其更加复合中国的地址
|
||||
address = f.address()[:-9] + str(f.pyint(min_value=0, max_value=999))+ "号"
|
||||
data.append(address)
|
||||
#获取当前时间
|
||||
actime_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
|
||||
data.append(actime_time)
|
||||
#将这一行数据添加到datas中
|
||||
datas.append(data)
|
||||
#返回所有的数据
|
||||
return datas
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
#文件名
|
||||
file_name = 'test.csv'
|
||||
save_data_csv(file_name)
|
||||
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import os
|
||||
import _load
|
||||
|
||||
import faker_data
|
||||
import csv
|
||||
import yaml
|
||||
import time
|
||||
import global_var
|
||||
|
||||
from schedule import every, repeat, run_pending
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from random import choice
|
||||
|
||||
"""
|
||||
说明：
|
||||
1. 用于文件的持续创建、写入、修改和创建
|
||||
2. 输出目录为data，会在里面创建当天的子目录
|
||||
3. 每1小时，生成一个CSV文件
|
||||
4. 每1秒，向上面的文件中写入N条数据
|
||||
5. 每10分钟，从当天的文件中随机找个1个文件，删除前N条数据
|
||||
6. 每6小时，从所有的文件中随机删除1个文件
|
||||
"""
|
||||
|
||||
|
||||
# 定义公共的部分
|
||||
path = Path(__file__)
|
||||
etc_dir = path.parent.parent / "etc"
|
||||
config_file_patch = etc_dir / "file_config.yml"
|
||||
|
||||
# 获取配置文件
|
||||
with open(config_file_patch, "r", encoding='utf-8') as fy:
|
||||
config = yaml.safe_load(fy)
|
||||
|
||||
# 默认为当前的目录
|
||||
data_dir = path.parent.parent / "data"
|
||||
if 'DATADIR' in config.keys() and config['DATADIR']:
|
||||
data_dir = Path(config['DATADIR'])
|
||||
|
||||
# 初始化目录
|
||||
if not data_dir.exists():
|
||||
# shutil.rmtree(data_dir)
|
||||
data_dir.mkdir()
|
||||
|
||||
# 定义文件的信息
|
||||
def file_info():
|
||||
file_info_list = []
|
||||
now = datetime.now()
|
||||
now_day = now.strftime("%Y-%m-%d")
|
||||
now_hours = now.strftime("%Y-%m-%d-%H")
|
||||
today_dir = data_dir/now_day
|
||||
filename = "filetest_" + now_hours + ".csv"
|
||||
filepath = today_dir/filename
|
||||
|
||||
file_info_list.append(now_day)
|
||||
file_info_list.append(now_hours)
|
||||
file_info_list.append(today_dir)
|
||||
file_info_list.append(filename)
|
||||
file_info_list.append(filepath)
|
||||
|
||||
return (file_info_list)
|
||||
|
||||
# 每小时生成一个csv文件
|
||||
# @repeat(every(1).hours)--MH:这个是基于运行的时间，不是基于系统时间
|
||||
def new():
|
||||
print('new...')
|
||||
# 创建文件夹
|
||||
finfo = file_info()
|
||||
if not finfo[2].exists():
|
||||
finfo[2].mkdir()
|
||||
# 创建文件
|
||||
faker_data.save_data_csv(finfo[4], lines=100)
|
||||
|
||||
|
||||
# 每2秒插入N条数据
|
||||
@repeat(every(2).seconds)
|
||||
def inserting():
|
||||
finfo = file_info()
|
||||
if finfo[4].exists():
|
||||
print('insert...')
|
||||
datas = faker_data.faker_data(lines=config['InsertRows'])[1:]
|
||||
with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv:
|
||||
writer = csv.writer(file_csv, delimiter=',',
|
||||
quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
writer.writerows(datas)
|
||||
else:
|
||||
print(str(finfo[4])+" is not exists,wating")
|
||||
# 手动去调一下，让其整点创建
|
||||
new()
|
||||
|
||||
|
||||
# 每隔10分钟删除100条数据
|
||||
@repeat(every(10).minutes)
|
||||
def deleting_data():
|
||||
finfo = file_info()
|
||||
# 获取所有文件
|
||||
files = list(finfo[2].glob('*.csv'))
|
||||
if len(files) > 1:
|
||||
file = choice(files[:-1])
|
||||
print(str(file) + "start delete data ....")
|
||||
# 删除掉前N条数据
|
||||
with open(file, 'rb') as fr:
|
||||
data = fr.readlines()
|
||||
new_data = data[config['DeleteRows']:]
|
||||
# 少于100条的不删除
|
||||
if len(new_data) > 100:
|
||||
with open(file, 'wb') as fw:
|
||||
fw.writelines(new_data)
|
||||
else:
|
||||
print("file number is less 1,wait next time.")
|
||||
|
||||
|
||||
# 每隔6小时删除1个文件,低于3个不删除
|
||||
@repeat(every(6).hours)
|
||||
# @repeat(every(2).seconds)
|
||||
def deleting_file():
|
||||
print("deleting file ....")
|
||||
# 从data目录中随机选一个
|
||||
files = list(data_dir.rglob('*.csv'))
|
||||
if len(files) > 3:
|
||||
file = choice(files[:-1])
|
||||
file.unlink()
|
||||
else:
|
||||
print("file num is less 3, not delete. wait next time.")
|
||||
|
||||
def main():
|
||||
start_flag = True
|
||||
while start_flag:
|
||||
start_flag = global_var.get_value('start_flag')
|
||||
run_pending()
|
||||
time.sleep(0.3)
|
||||
print("已经停止")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
def _init(): # 初始化
|
||||
global _global_dict
|
||||
_global_dict = {}
|
||||
|
||||
def set_value(key, value):
|
||||
#定义一个全局变量
|
||||
_global_dict[key] = value
|
||||
|
||||
def get_value(key):
|
||||
#获得一个全局变量，不存在则提示读取对应变量失败
|
||||
try:
|
||||
return _global_dict[key]
|
||||
except:
|
||||
print('读取'+key+'失败\r\n')
|
||||
|
||||
Loading…
Reference in New Issue