HA
/
dataCreating
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

提交修改

This commit is contained in:
halliday 2024-01-08 14:27:58 +08:00
parent 1d3bc9455a
commit 847dea4a8e
5 changed files with 73 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Changes.txt
View File

@ -1,5 +1,14 @@
changes log changes log
====================================== ======================================
------------------------------
1.0.2 2024-01-08
1.增加配置文件，配置信息参数化
2.按周创建表
[mh]
------------------------------ ------------------------------
1.0.1 2023-11-28 1.0.1 2023-11-28

43
bin/DBConfig.py
View File

@ -1,5 +1,7 @@
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
import faker_data import faker_data
import yaml
from pathlib import Path from pathlib import Path
from datetime import datetime from datetime import datetime
@ -8,13 +10,18 @@ from datetime import datetime
# 定义公共的部分 # 定义公共的部分
path = Path(__file__) path = Path(__file__)
etc_dir = path.parent.parent / "etc" etc_dir = path.parent.parent / "etc"
config_file_patch = path.parent.parent / "test_config.yml"
# def get_now(): # 获取配置文件
# now = datetime.now() with open(config_file_patch, "r", encoding='utf-8') as fy:
# #now_day = now.strftime("%Y-%m-%d") config = yaml.safe_load(fy)
# now_hours = now.strftime("%Y%m%d%H") # print(config)
# return(now_hours)
def get_now():
week = str(datetime.now().isocalendar()[1])
return(week)
def mysql_config(): def mysql_config():
@ -24,14 +31,13 @@ def mysql_config():
driver_jar_path = etc_dir / "driver" / mysql_info_dict['driver_jar'] driver_jar_path = etc_dir / "driver" / mysql_info_dict['driver_jar']
mysql_info_dict['driver_jar_path'] = str(driver_jar_path) mysql_info_dict['driver_jar_path'] = str(driver_jar_path)
#这里要连接到具体的库 #这里要连接到具体的库
mysql_info_dict['jdbc_url'] = "jdbc:mysql://10.10.14.69:3306/sys" mysql_info_dict['jdbc_url'] = "jdbc:mysql://" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES']
mysql_info_dict['db_user'] = "root" mysql_info_dict['db_user'] = config['DBUSER']
mysql_info_dict['db_password'] = "111111" mysql_info_dict['db_password'] = config['DBPASSWD']
#create table sql #create table sql
# now_hours = get_now() week = get_now()
#TODO(MH):根据时间生成不同的表 table_name = "mysql_test_table_w" + week
table_name = "mysql_test_table"
create_sql_string = "CREATE TABLE IF NOT EXISTS `" create_sql_string = "CREATE TABLE IF NOT EXISTS `"
create_sql_string += table_name create_sql_string += table_name
create_sql_string += "` (" create_sql_string += "` ("
@ -50,7 +56,7 @@ def mysql_config():
# insert into sql # insert into sql
insert_list = [] insert_list = []
data = faker_data.faker_data(lines=200)[1:] data = faker_data.faker_data(lines=config['InsertRows'])[1:]
for a in data: for a in data:
#拼接sql #拼接sql
insert_sql_string = "INSERT INTO " + table_name insert_sql_string = "INSERT INTO " + table_name
@ -81,14 +87,13 @@ def oracle_config():
driver_jar_path = etc_dir / "driver" / oracle_info_dict['driver_jar'] driver_jar_path = etc_dir / "driver" / oracle_info_dict['driver_jar']
oracle_info_dict['driver_jar_path'] = str(driver_jar_path) oracle_info_dict['driver_jar_path'] = str(driver_jar_path)
#这里要连接到具体的实例 #这里要连接到具体的实例
oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@10.10.29.10:1521/unary" oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES']
oracle_info_dict['db_user'] = "Test01" oracle_info_dict['db_user'] = config['DBUSER']
oracle_info_dict['db_password'] = "test123" oracle_info_dict['db_password'] = config['DBPASSWD']
#create table sql #create table sql
# now_hours = get_now() week = get_now()
#TODO(MH):根据时间生成不同的表 table_name = "ORACLE_TEST_w" + week
table_name = "ORACLE_TEST"
create_sql_string = 'CREATE TABLE "' create_sql_string = 'CREATE TABLE "'
create_sql_string += table_name + '" ' create_sql_string += table_name + '" '
create_sql_string += '("UUID" VARCHAR2(50),' create_sql_string += '("UUID" VARCHAR2(50),'
@ -106,7 +111,7 @@ def oracle_config():
# insert into sql # insert into sql
insert_list = [] insert_list = []
data = faker_data.faker_data(lines=200)[1:] data = faker_data.faker_data(lines=config['InsertRows'])[1:]
for a in data: for a in data:
#拼接sql #拼接sql
insert_sql_string = "INSERT INTO " + table_name insert_sql_string = "INSERT INTO " + table_name

34
bin/SQLDataCreating.py
View File

@ -12,10 +12,10 @@ from schedule import every, repeat, run_pending
说明： 说明：
1. 用于数据库的持续增、删、改，目前支持MySQL和Oracle 1. 用于数据库的持续增、删、改，目前支持MySQL和Oracle
2. DBConfig.py 为数据库连接配置，使用时需要修改连接信息 2. DBConfig.py 为数据库连接配置，使用时需要修改连接信息
3. 脚本运行会创建一个测试表 - 每周一建立一张表
4. 每秒往测试表中插入200条数据 - 每秒往测试表中插入200条数据
5. 每5分钟删除100条数据 - 每5分钟删除100条数据
6. 每1小时，修改一下数据 - 每1小时，修改一下数据
""" """
# 目前支持 MySQL和Oracle两种数据库，其他的数据库需要做适配 # 目前支持 MySQL和Oracle两种数据库，其他的数据库需要做适配
@ -50,20 +50,20 @@ def execute_sql(sql):
with conn.cursor() as curs: with conn.cursor() as curs:
curs.execute(sql) curs.execute(sql)
#@repeat(every(1).hours) @repeat(every().monday)
#TODO(MH):按小时创建表，不太好控制，后面再去搞
def create_table(): def create_table():
print("start create table...") print("start create table...")
sql_string = db_config["create_sql_string"] sql_string = db_config["create_sql_string"]
try: try:
execute_sql(sql_string) execute_sql(sql_string)
except: except:
print("table maybe exists,continue this create table.") print("table maybe exists,continue this create table task.")
#每秒插入1条数据 #每秒插入200条数据
@repeat(every(2).seconds) @repeat(every(2).seconds)
def insert_data(): def insert_data():
create_table()
print("insert data ....") print("insert data ....")
#重新获取一下，确保每次的插入的数据都不一样 #重新获取一下，确保每次的插入的数据都不一样
db_config2 = get_db_info() db_config2 = get_db_info()
@ -77,21 +77,27 @@ def insert_data():
#每5分钟删除100条数据 #每5分钟删除100条数据
@repeat(every(5).minutes) @repeat(every(5).minutes)
def dalete_data(): def dalete_data():
db_config2 = get_db_info()
print("dalete data....") print("dalete data....")
sql_string = db_config["del_sql"] sql_string = db_config2["del_sql"]
execute_sql(sql_string) try:
execute_sql(sql_string)
except:
print("table not exists,continue this delete task.")
#每1小时，修改一下数据 #每1小时，修改一下数据
@repeat(every(1).hours) @repeat(every(1).hours)
def updata_data(): def updata_data():
db_config2 = get_db_info()
print("updata data ....") print("updata data ....")
sql_string = db_config["updata_sql"] sql_string = db_config2["updata_sql"]
execute_sql(sql_string) try:
execute_sql(sql_string)
except:
print("table not exists,continue this updata task.")
if __name__ == '__main__': if __name__ == '__main__':
create_table()
while True: while True:
run_pending() run_pending()

10
bin/fileDataCreating.py
View File

@ -13,8 +13,8 @@ from random import choice
说明： 说明：
1. 用于文件的持续创建、写入、修改和创建 1. 用于文件的持续创建、写入、修改和创建
2. 输出目录为data，会在里面创建当天的子目录 2. 输出目录为data，会在里面创建当天的子目录
3. 每1小时，生成一个10M大小左右的csv文件 3. 每1小时，生成一个sv文件
4. 每2秒，向上面的文件中写入10条数据 4. 每1秒，向上面的文件中写入100条数据
5. 每10分钟，从当天的文件中随机找个1个文件，删除前100条数据 5. 每10分钟，从当天的文件中随机找个1个文件，删除前100条数据
6. 每6小时，从所有的文件中随机删除1个文件 6. 每6小时，从所有的文件中随机删除1个文件
""" """
@ -47,7 +47,7 @@ def file_info():
return (file_info_list) return (file_info_list)
# 每小时生成一个10M大小的csv文件 # 每小时生成一个csv文件
# @repeat(every(1).hours) TODO(MH):这个是基于运行的时间，不是基于系统时间 # @repeat(every(1).hours) TODO(MH):这个是基于运行的时间，不是基于系统时间
def new(): def new():
print('new...') print('new...')
@ -56,7 +56,7 @@ def new():
if not finfo[2].exists(): if not finfo[2].exists():
finfo[2].mkdir() finfo[2].mkdir()
# 创建文件 # 创建文件
faker_data.save_data_csv(finfo[4], lines=200000) faker_data.save_data_csv(finfo[4], lines=100)
# 每2秒插入10条数据 # 每2秒插入10条数据
@ -65,7 +65,7 @@ def inserting():
finfo = file_info() finfo = file_info()
if finfo[4].exists(): if finfo[4].exists():
print('insert...') print('insert...')
datas = faker_data.faker_data(lines=10)[1:] datas = faker_data.faker_data(lines=200)[1:]
with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv: with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv:
writer = csv.writer(file_csv, delimiter=',', writer = csv.writer(file_csv, delimiter=',',
quotechar='"', quoting=csv.QUOTE_ALL) quotechar='"', quoting=csv.QUOTE_ALL)

20
etc/test_config.yml
View File

@ -1,7 +1,17 @@
#数据库配置文件 #数据库配置文件
driver_name: "com.mysql.jdbc.Driver"
driver_jar: "mysql-connector-java-8.0.29.jar"
jdbc_url: "jdbc:mysql://10.10.29.9:3306/test2" # 数据库类型，目前仅支持MySQL和Oracle
db_user: "root" DBType: MySQL
db_password: "Unary@2023" # 数据库地址
IP: "10.10.13.167"
PROT: "3306"
# 数据库用户名
DBUSER: "root"
# 数据库密码
DBPASSWD: "111111"
# 库名
DATABASES: "test"
# 每秒中插入多少条数据
InsertRows: 200