提交修改
This commit is contained in:
parent
1d3bc9455a
commit
847dea4a8e
|
|
@ -1,5 +1,14 @@
|
|||
changes log
|
||||
======================================
|
||||
|
||||
------------------------------
|
||||
1.0.2 2024-01-08
|
||||
|
||||
1.增加配置文件，配置信息参数化
|
||||
2.按周创建表
|
||||
|
||||
[mh]
|
||||
|
||||
------------------------------
|
||||
1.0.1 2023-11-28
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import faker_data
|
||||
import yaml
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
|
@ -8,13 +10,18 @@ from datetime import datetime
|
|||
# 定义公共的部分
|
||||
path = Path(__file__)
|
||||
etc_dir = path.parent.parent / "etc"
|
||||
config_file_patch = path.parent.parent / "test_config.yml"
|
||||
|
||||
|
||||
# def get_now():
|
||||
# now = datetime.now()
|
||||
# #now_day = now.strftime("%Y-%m-%d")
|
||||
# now_hours = now.strftime("%Y%m%d%H")
|
||||
# return(now_hours)
|
||||
# 获取配置文件
|
||||
with open(config_file_patch, "r", encoding='utf-8') as fy:
|
||||
config = yaml.safe_load(fy)
|
||||
# print(config)
|
||||
|
||||
|
||||
def get_now():
|
||||
week = str(datetime.now().isocalendar()[1])
|
||||
return(week)
|
||||
|
||||
|
||||
def mysql_config():
|
||||
|
|
@ -24,14 +31,13 @@ def mysql_config():
|
|||
driver_jar_path = etc_dir / "driver" / mysql_info_dict['driver_jar']
|
||||
mysql_info_dict['driver_jar_path'] = str(driver_jar_path)
|
||||
#这里要连接到具体的库
|
||||
mysql_info_dict['jdbc_url'] = "jdbc:mysql://10.10.14.69:3306/sys"
|
||||
mysql_info_dict['db_user'] = "root"
|
||||
mysql_info_dict['db_password'] = "111111"
|
||||
mysql_info_dict['jdbc_url'] = "jdbc:mysql://" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES']
|
||||
mysql_info_dict['db_user'] = config['DBUSER']
|
||||
mysql_info_dict['db_password'] = config['DBPASSWD']
|
||||
|
||||
#create table sql
|
||||
# now_hours = get_now()
|
||||
#TODO(MH):根据时间生成不同的表
|
||||
table_name = "mysql_test_table"
|
||||
week = get_now()
|
||||
table_name = "mysql_test_table_w" + week
|
||||
create_sql_string = "CREATE TABLE IF NOT EXISTS `"
|
||||
create_sql_string += table_name
|
||||
create_sql_string += "` ("
|
||||
|
|
@ -50,7 +56,7 @@ def mysql_config():
|
|||
|
||||
# insert into sql
|
||||
insert_list = []
|
||||
data = faker_data.faker_data(lines=200)[1:]
|
||||
data = faker_data.faker_data(lines=config['InsertRows'])[1:]
|
||||
for a in data:
|
||||
#拼接sql
|
||||
insert_sql_string = "INSERT INTO " + table_name
|
||||
|
|
@ -81,14 +87,13 @@ def oracle_config():
|
|||
driver_jar_path = etc_dir / "driver" / oracle_info_dict['driver_jar']
|
||||
oracle_info_dict['driver_jar_path'] = str(driver_jar_path)
|
||||
#这里要连接到具体的实例
|
||||
oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@10.10.29.10:1521/unary"
|
||||
oracle_info_dict['db_user'] = "Test01"
|
||||
oracle_info_dict['db_password'] = "test123"
|
||||
oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES']
|
||||
oracle_info_dict['db_user'] = config['DBUSER']
|
||||
oracle_info_dict['db_password'] = config['DBPASSWD']
|
||||
|
||||
#create table sql
|
||||
# now_hours = get_now()
|
||||
#TODO(MH):根据时间生成不同的表
|
||||
table_name = "ORACLE_TEST"
|
||||
week = get_now()
|
||||
table_name = "ORACLE_TEST_w" + week
|
||||
create_sql_string = 'CREATE TABLE "'
|
||||
create_sql_string += table_name + '" '
|
||||
create_sql_string += '("UUID" VARCHAR2(50),'
|
||||
|
|
@ -106,7 +111,7 @@ def oracle_config():
|
|||
|
||||
# insert into sql
|
||||
insert_list = []
|
||||
data = faker_data.faker_data(lines=200)[1:]
|
||||
data = faker_data.faker_data(lines=config['InsertRows'])[1:]
|
||||
for a in data:
|
||||
#拼接sql
|
||||
insert_sql_string = "INSERT INTO " + table_name
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ from schedule import every, repeat, run_pending
|
|||
说明：
|
||||
1. 用于数据库的持续增、删、改，目前支持MySQL和Oracle
|
||||
2. DBConfig.py 为数据库连接配置，使用时需要修改连接信息
|
||||
3. 脚本运行会创建一个测试表
|
||||
4. 每秒往测试表中插入200条数据
|
||||
5. 每5分钟删除100条数据
|
||||
6. 每1小时，修改一下数据
|
||||
- 每周一建立一张表
|
||||
- 每秒往测试表中插入200条数据
|
||||
- 每5分钟删除100条数据
|
||||
- 每1小时，修改一下数据
|
||||
"""
|
||||
|
||||
# 目前支持 MySQL和Oracle两种数据库，其他的数据库需要做适配
|
||||
|
|
@ -50,20 +50,20 @@ def execute_sql(sql):
|
|||
with conn.cursor() as curs:
|
||||
curs.execute(sql)
|
||||
|
||||
#@repeat(every(1).hours)
|
||||
#TODO(MH):按小时创建表，不太好控制，后面再去搞
|
||||
@repeat(every().monday)
|
||||
def create_table():
|
||||
print("start create table...")
|
||||
sql_string = db_config["create_sql_string"]
|
||||
try:
|
||||
execute_sql(sql_string)
|
||||
except:
|
||||
print("table maybe exists,continue this create table.")
|
||||
print("table maybe exists,continue this create table task.")
|
||||
|
||||
|
||||
#每秒插入1条数据
|
||||
#每秒插入200条数据
|
||||
@repeat(every(2).seconds)
|
||||
def insert_data():
|
||||
create_table()
|
||||
print("insert data ....")
|
||||
#重新获取一下，确保每次的插入的数据都不一样
|
||||
db_config2 = get_db_info()
|
||||
|
|
@ -77,21 +77,27 @@ def insert_data():
|
|||
#每5分钟删除100条数据
|
||||
@repeat(every(5).minutes)
|
||||
def dalete_data():
|
||||
db_config2 = get_db_info()
|
||||
print("dalete data....")
|
||||
sql_string = db_config["del_sql"]
|
||||
execute_sql(sql_string)
|
||||
sql_string = db_config2["del_sql"]
|
||||
try:
|
||||
execute_sql(sql_string)
|
||||
except:
|
||||
print("table not exists,continue this delete task.")
|
||||
|
||||
|
||||
#每1小时，修改一下数据
|
||||
@repeat(every(1).hours)
|
||||
def updata_data():
|
||||
db_config2 = get_db_info()
|
||||
print("updata data ....")
|
||||
sql_string = db_config["updata_sql"]
|
||||
execute_sql(sql_string)
|
||||
|
||||
sql_string = db_config2["updata_sql"]
|
||||
try:
|
||||
execute_sql(sql_string)
|
||||
except:
|
||||
print("table not exists,continue this updata task.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
create_table()
|
||||
while True:
|
||||
run_pending()
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ from random import choice
|
|||
说明：
|
||||
1. 用于文件的持续创建、写入、修改和创建
|
||||
2. 输出目录为data，会在里面创建当天的子目录
|
||||
3. 每1小时，生成一个10M大小左右的csv文件
|
||||
4. 每2秒，向上面的文件中写入10条数据
|
||||
3. 每1小时，生成一个sv文件
|
||||
4. 每1秒，向上面的文件中写入100条数据
|
||||
5. 每10分钟，从当天的文件中随机找个1个文件，删除前100条数据
|
||||
6. 每6小时，从所有的文件中随机删除1个文件
|
||||
"""
|
||||
|
|
@ -47,7 +47,7 @@ def file_info():
|
|||
|
||||
return (file_info_list)
|
||||
|
||||
# 每小时生成一个10M大小的csv文件
|
||||
# 每小时生成一个csv文件
|
||||
# @repeat(every(1).hours) TODO(MH):这个是基于运行的时间，不是基于系统时间
|
||||
def new():
|
||||
print('new...')
|
||||
|
|
@ -56,7 +56,7 @@ def new():
|
|||
if not finfo[2].exists():
|
||||
finfo[2].mkdir()
|
||||
# 创建文件
|
||||
faker_data.save_data_csv(finfo[4], lines=200000)
|
||||
faker_data.save_data_csv(finfo[4], lines=100)
|
||||
|
||||
|
||||
# 每2秒插入10条数据
|
||||
|
|
@ -65,7 +65,7 @@ def inserting():
|
|||
finfo = file_info()
|
||||
if finfo[4].exists():
|
||||
print('insert...')
|
||||
datas = faker_data.faker_data(lines=10)[1:]
|
||||
datas = faker_data.faker_data(lines=200)[1:]
|
||||
with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv:
|
||||
writer = csv.writer(file_csv, delimiter=',',
|
||||
quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,17 @@
|
|||
#数据库配置文件
|
||||
driver_name: "com.mysql.jdbc.Driver"
|
||||
driver_jar: "mysql-connector-java-8.0.29.jar"
|
||||
|
||||
jdbc_url: "jdbc:mysql://10.10.29.9:3306/test2"
|
||||
db_user: "root"
|
||||
db_password: "Unary@2023"
|
||||
# 数据库类型，目前仅支持MySQL和Oracle
|
||||
DBType: MySQL
|
||||
# 数据库地址
|
||||
IP: "10.10.13.167"
|
||||
PROT: "3306"
|
||||
# 数据库用户名
|
||||
DBUSER: "root"
|
||||
# 数据库密码
|
||||
DBPASSWD: "111111"
|
||||
# 库名
|
||||
DATABASES: "test"
|
||||
# 每秒中插入多少条数据
|
||||
InsertRows: 200
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue