From 847dea4a8e252b86fd1e602fa75bc615810a2663 Mon Sep 17 00:00:00 2001 From: halliday Date: Mon, 8 Jan 2024 14:27:58 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Changes.txt | 9 +++++++++ bin/DBConfig.py | 43 +++++++++++++++++++++++------------------ bin/SQLDataCreating.py | 34 ++++++++++++++++++-------------- bin/fileDataCreating.py | 10 +++++----- etc/test_config.yml | 20 ++++++++++++++----- 5 files changed, 73 insertions(+), 43 deletions(-) diff --git a/Changes.txt b/Changes.txt index d61e7b4..4c982c5 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,5 +1,14 @@ changes log ====================================== + +------------------------------ +1.0.2 2024-01-08 + +1.增加配置文件，配置信息参数化 +2.按周创建表 + +[mh] + ------------------------------ 1.0.1 2023-11-28 diff --git a/bin/DBConfig.py b/bin/DBConfig.py index 5df63ca..c65656f 100644 --- a/bin/DBConfig.py +++ b/bin/DBConfig.py @@ -1,5 +1,7 @@ # -*- coding:utf-8 -*- import faker_data +import yaml + from pathlib import Path from datetime import datetime @@ -8,13 +10,18 @@ from datetime import datetime # 定义公共的部分 path = Path(__file__) etc_dir = path.parent.parent / "etc" +config_file_patch = path.parent.parent / "test_config.yml" -# def get_now(): -# now = datetime.now() -# #now_day = now.strftime("%Y-%m-%d") -# now_hours = now.strftime("%Y%m%d%H") -# return(now_hours) +# 获取配置文件 +with open(config_file_patch, "r", encoding='utf-8') as fy: + config = yaml.safe_load(fy) +# print(config) + + +def get_now(): + week = str(datetime.now().isocalendar()[1]) + return(week) def mysql_config(): @@ -24,14 +31,13 @@ def mysql_config(): driver_jar_path = etc_dir / "driver" / mysql_info_dict['driver_jar'] mysql_info_dict['driver_jar_path'] = str(driver_jar_path) #这里要连接到具体的库 - mysql_info_dict['jdbc_url'] = "jdbc:mysql://10.10.14.69:3306/sys" - mysql_info_dict['db_user'] = "root" - mysql_info_dict['db_password'] = "111111" + mysql_info_dict['jdbc_url'] = "jdbc:mysql://" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES'] + mysql_info_dict['db_user'] = config['DBUSER'] + mysql_info_dict['db_password'] = config['DBPASSWD'] #create table sql - # now_hours = get_now() - #TODO(MH):根据时间生成不同的表 - table_name = "mysql_test_table" + week = get_now() + table_name = "mysql_test_table_w" + week create_sql_string = "CREATE TABLE IF NOT EXISTS `" create_sql_string += table_name create_sql_string += "` (" @@ -50,7 +56,7 @@ def mysql_config(): # insert into sql insert_list = [] - data = faker_data.faker_data(lines=200)[1:] + data = faker_data.faker_data(lines=config['InsertRows'])[1:] for a in data: #拼接sql insert_sql_string = "INSERT INTO " + table_name @@ -81,14 +87,13 @@ def oracle_config(): driver_jar_path = etc_dir / "driver" / oracle_info_dict['driver_jar'] oracle_info_dict['driver_jar_path'] = str(driver_jar_path) #这里要连接到具体的实例 - oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@10.10.29.10:1521/unary" - oracle_info_dict['db_user'] = "Test01" - oracle_info_dict['db_password'] = "test123" + oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES'] + oracle_info_dict['db_user'] = config['DBUSER'] + oracle_info_dict['db_password'] = config['DBPASSWD'] #create table sql - # now_hours = get_now() - #TODO(MH):根据时间生成不同的表 - table_name = "ORACLE_TEST" + week = get_now() + table_name = "ORACLE_TEST_w" + week create_sql_string = 'CREATE TABLE "' create_sql_string += table_name + '" ' create_sql_string += '("UUID" VARCHAR2(50),' @@ -106,7 +111,7 @@ def oracle_config(): # insert into sql insert_list = [] - data = faker_data.faker_data(lines=200)[1:] + data = faker_data.faker_data(lines=config['InsertRows'])[1:] for a in data: #拼接sql insert_sql_string = "INSERT INTO " + table_name diff --git a/bin/SQLDataCreating.py b/bin/SQLDataCreating.py index 51b0142..bbce3a8 100644 --- a/bin/SQLDataCreating.py +++ b/bin/SQLDataCreating.py @@ -12,10 +12,10 @@ from schedule import every, repeat, run_pending 说明： 1. 用于数据库的持续增、删、改，目前支持MySQL和Oracle 2. DBConfig.py 为数据库连接配置，使用时需要修改连接信息 -3. 脚本运行会创建一个测试表 -4. 每秒往测试表中插入200条数据 -5. 每5分钟删除100条数据 -6. 每1小时，修改一下数据 + - 每周一建立一张表 + - 每秒往测试表中插入200条数据 + - 每5分钟删除100条数据 + - 每1小时，修改一下数据 """ # 目前支持 MySQL和Oracle两种数据库，其他的数据库需要做适配 @@ -50,20 +50,20 @@ def execute_sql(sql): with conn.cursor() as curs: curs.execute(sql) -#@repeat(every(1).hours) -#TODO(MH):按小时创建表，不太好控制，后面再去搞 +@repeat(every().monday) def create_table(): print("start create table...") sql_string = db_config["create_sql_string"] try: execute_sql(sql_string) except: - print("table maybe exists,continue this create table.") + print("table maybe exists,continue this create table task.") -#每秒插入1条数据 +#每秒插入200条数据 @repeat(every(2).seconds) def insert_data(): + create_table() print("insert data ....") #重新获取一下，确保每次的插入的数据都不一样 db_config2 = get_db_info() @@ -77,21 +77,27 @@ def insert_data(): #每5分钟删除100条数据 @repeat(every(5).minutes) def dalete_data(): + db_config2 = get_db_info() print("dalete data....") - sql_string = db_config["del_sql"] - execute_sql(sql_string) + sql_string = db_config2["del_sql"] + try: + execute_sql(sql_string) + except: + print("table not exists,continue this delete task.") #每1小时，修改一下数据 @repeat(every(1).hours) def updata_data(): + db_config2 = get_db_info() print("updata data ....") - sql_string = db_config["updata_sql"] - execute_sql(sql_string) - + sql_string = db_config2["updata_sql"] + try: + execute_sql(sql_string) + except: + print("table not exists,continue this updata task.") if __name__ == '__main__': - create_table() while True: run_pending() diff --git a/bin/fileDataCreating.py b/bin/fileDataCreating.py index 3e7bca8..0b8282d 100644 --- a/bin/fileDataCreating.py +++ b/bin/fileDataCreating.py @@ -13,8 +13,8 @@ from random import choice 说明： 1. 用于文件的持续创建、写入、修改和创建 2. 输出目录为data，会在里面创建当天的子目录 -3. 每1小时，生成一个10M大小左右的csv文件 -4. 每2秒，向上面的文件中写入10条数据 +3. 每1小时，生成一个sv文件 +4. 每1秒，向上面的文件中写入100条数据 5. 每10分钟，从当天的文件中随机找个1个文件，删除前100条数据 6. 每6小时，从所有的文件中随机删除1个文件 """ @@ -47,7 +47,7 @@ def file_info(): return (file_info_list) -# 每小时生成一个10M大小的csv文件 +# 每小时生成一个csv文件 # @repeat(every(1).hours) TODO(MH):这个是基于运行的时间，不是基于系统时间 def new(): print('new...') @@ -56,7 +56,7 @@ def new(): if not finfo[2].exists(): finfo[2].mkdir() # 创建文件 - faker_data.save_data_csv(finfo[4], lines=200000) + faker_data.save_data_csv(finfo[4], lines=100) # 每2秒插入10条数据 @@ -65,7 +65,7 @@ def inserting(): finfo = file_info() if finfo[4].exists(): print('insert...') - datas = faker_data.faker_data(lines=10)[1:] + datas = faker_data.faker_data(lines=200)[1:] with open(finfo[4], 'a+', encoding='utf-8', newline='') as file_csv: writer = csv.writer(file_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) diff --git a/etc/test_config.yml b/etc/test_config.yml index b7065c1..65e51fd 100644 --- a/etc/test_config.yml +++ b/etc/test_config.yml @@ -1,7 +1,17 @@ #数据库配置文件 -driver_name: "com.mysql.jdbc.Driver" -driver_jar: "mysql-connector-java-8.0.29.jar" -jdbc_url: "jdbc:mysql://10.10.29.9:3306/test2" -db_user: "root" -db_password: "Unary@2023" +# 数据库类型，目前仅支持MySQL和Oracle +DBType: MySQL +# 数据库地址 +IP: "10.10.13.167" +PROT: "3306" +# 数据库用户名 +DBUSER: "root" +# 数据库密码 +DBPASSWD: "111111" +# 库名 +DATABASES: "test" +# 每秒中插入多少条数据 +InsertRows: 200 + +