From fc17a0206b384afd456ad0f448f25d52cf0cc92d Mon Sep 17 00:00:00 2001 From: halliday2023 Date: Tue, 13 Feb 2024 17:25:03 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=9B=AE=E5=BD=95=E7=BB=93?= =?UTF-8?q?=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/{main.py => DataCreating_Click.py} | 0 bin/GUIWindows.py | 4 +- bin/{ => common}/faker_data.py | 0 bin/{ => common}/global_var.py | 0 bin/db/DBConfig.py | 136 +++++++++++++++++++++++++ bin/db/SQLDataCreating.py | 109 ++++++++++++++++++++ bin/{ => file}/fileDataCreating.py | 10 +- etc/db_config.yml | 20 ++++ etc/file_config.yml | 11 ++ 9 files changed, 282 insertions(+), 8 deletions(-) rename bin/{main.py => DataCreating_Click.py} (100%) rename bin/{ => common}/faker_data.py (100%) rename bin/{ => common}/global_var.py (100%) create mode 100644 bin/db/DBConfig.py create mode 100644 bin/db/SQLDataCreating.py rename bin/{ => file}/fileDataCreating.py (96%) create mode 100644 etc/db_config.yml create mode 100644 etc/file_config.yml diff --git a/bin/main.py b/bin/DataCreating_Click.py similarity index 100% rename from bin/main.py rename to bin/DataCreating_Click.py diff --git a/bin/GUIWindows.py b/bin/GUIWindows.py index 4be7bf2..bb34762 100644 --- a/bin/GUIWindows.py +++ b/bin/GUIWindows.py @@ -3,8 +3,8 @@ import os import sys import logging -import fileDataCreating -import global_var +from file import fileDataCreating +from common import global_var from pathlib import Path diff --git a/bin/faker_data.py b/bin/common/faker_data.py similarity index 100% rename from bin/faker_data.py rename to bin/common/faker_data.py diff --git a/bin/global_var.py b/bin/common/global_var.py similarity index 100% rename from bin/global_var.py rename to bin/common/global_var.py diff --git a/bin/db/DBConfig.py b/bin/db/DBConfig.py new file mode 100644 index 0000000..0379c09 --- /dev/null +++ b/bin/db/DBConfig.py @@ -0,0 +1,136 @@ +# -*- coding:utf-8 -*- +import faker_data +import yaml + + +from pathlib import Path +from datetime import datetime + + +# 定义公共的部分 +path = Path(__file__) +etc_dir = path.parent.parent / "etc" +config_file_patch = etc_dir / "db_config.yml" + + +# 获取配置文件 +with open(config_file_patch, "r", encoding='utf-8') as fy: + config = yaml.safe_load(fy) +# print(config) + + +def get_now(): + week = str(datetime.now().isocalendar()[1]) + return(week) + + +def mysql_config(): + mysql_info_dict={} + mysql_info_dict['driver_name'] = "com.mysql.cj.jdbc.Driver" + mysql_info_dict['driver_jar'] = "mysql-connector-java-8.0.29.jar" + driver_jar_path = etc_dir / "driver" / mysql_info_dict['driver_jar'] + mysql_info_dict['driver_jar_path'] = str(driver_jar_path) + #这里要连接到具体的库 + mysql_info_dict['jdbc_url'] = "jdbc:mysql://" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES'] + mysql_info_dict['db_user'] = config['DBUSER'] + mysql_info_dict['db_password'] = config['DBPASSWD'] + + #create table sql + week = get_now() + table_name = "mysql_test_table_w" + week + create_sql_string = "CREATE TABLE IF NOT EXISTS `" + create_sql_string += table_name + create_sql_string += "` (" + create_sql_string += "`uuid` varchar(50) DEFAULT NULL," + create_sql_string += "`id` bigint DEFAULT NULL," + create_sql_string += "`name` varchar(50) DEFAULT NULL," + create_sql_string += "`mobile` varchar(50) DEFAULT NULL," + create_sql_string += "`ssn` varchar(50) DEFAULT NULL," + create_sql_string += "`sex` int DEFAULT NULL," + create_sql_string += "`email` varchar(50) DEFAULT NULL," + create_sql_string += "`job` varchar(50) DEFAULT NULL," + create_sql_string += "`address` varchar(50) DEFAULT NULL," + create_sql_string += "`actime_time` varchar(50) NULL DEFAULT NULL" + create_sql_string += ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" + mysql_info_dict['create_sql_string'] = create_sql_string + + # insert into sql + insert_list = [] + data = faker_data.faker_data(lines=config['InsertRows'])[1:] + for a in data: + #拼接sql + insert_sql_string = "INSERT INTO " + table_name + insert_sql_string += "(uuid, id, name, mobile, ssn, sex, email, job, address, actime_time) VALUES(" + str_data = ','.join("'{0}'".format(x) for x in a) + insert_sql_string += str_data + ");" + + insert_list.append(insert_sql_string) + # 返回所有的插入语句 + mysql_info_dict['insert_list'] = insert_list + + # delete sql + del_sql = "delete FROM "+ table_name + " where 1=1 limit 100" + mysql_info_dict['del_sql'] = del_sql + + + # uptata sql + updata_sql = "UPDATE "+ table_name + " t SET t.job='unary_测试' WHERE t.name like '李%' AND sex = 1 AND id = 1 " + mysql_info_dict['updata_sql'] = updata_sql + + return(mysql_info_dict) + + +def oracle_config(): + oracle_info_dict={} + oracle_info_dict['driver_name'] = "oracle.jdbc.OracleDriver" + oracle_info_dict['driver_jar'] = "ojdbc8-12.2.0.1.jar" + driver_jar_path = etc_dir / "driver" / oracle_info_dict['driver_jar'] + oracle_info_dict['driver_jar_path'] = str(driver_jar_path) + #这里要连接到具体的实例 + oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@" + config['IP'] + ":" + config['PROT'] + "/" + config['SERVER'] + oracle_info_dict['db_user'] = config['DBUSER'] + oracle_info_dict['db_password'] = config['DBPASSWD'] + + #create table sql + week = get_now() + table_name = "ORACLE_TEST_w" + week + create_sql_string = 'CREATE TABLE "' + create_sql_string += table_name + '" ' + create_sql_string += '("UUID" VARCHAR2(50),' + create_sql_string += '"ID" NUMBER(38,0), ' + create_sql_string += '"NAME" VARCHAR2(50),' + create_sql_string += '"MOBILE" VARCHAR2(50),' + create_sql_string += '"SSN" VARCHAR2(50),' + create_sql_string += '"SEX" NUMBER(*,0), ' + create_sql_string += '"EMAIL" VARCHAR2(50), ' + create_sql_string += '"JOB" VARCHAR2(50),' + create_sql_string += '"ADDRESS" VARCHAR2(100),' + create_sql_string += '"ACTIME_TIME" VARCHAR2(50))' + oracle_info_dict['create_sql_string'] = create_sql_string + + + # insert into sql + insert_list = [] + data = faker_data.faker_data(lines=config['InsertRows'])[1:] + for a in data: + #拼接sql + insert_sql_string = 'INSERT INTO ' + config['DATABASES'] + '."' + table_name + '" ' + insert_sql_string += " t (uuid, id, name, mobile, ssn, sex, email, job, address, actime_time) VALUES(" + str_data = ','.join("'{0}'".format(x) for x in a) + insert_sql_string += str_data + ")" + + insert_list.append(insert_sql_string) + # 返回所有的插入语句 + oracle_info_dict['insert_list'] = insert_list + + # delete sql + del_sql = 'delete FROM '+ config['DATABASES'] + '."' + table_name + '" WHERE rownum<=100' + oracle_info_dict['del_sql'] = del_sql + + + # uptata sql + updata_sql = "UPDATE "+ config['DATABASES'] + '."' + table_name + '" '+ " t SET t.job='unary_测试' WHERE t.name like '李%' AND sex = 1 AND id = 1 " + oracle_info_dict['updata_sql'] = updata_sql + + return(oracle_info_dict) + diff --git a/bin/db/SQLDataCreating.py b/bin/db/SQLDataCreating.py new file mode 100644 index 0000000..10dea48 --- /dev/null +++ b/bin/db/SQLDataCreating.py @@ -0,0 +1,109 @@ +# -*- coding:utf-8 -*- +import _load + +import DBConfig + +import jaydebeapi + +from schedule import every, repeat, run_pending + + +""" +说明： +1. 用于数据库的持续增、删、改，目前支持MySQL和Oracle +2. DBConfig.py 为数据库连接配置，使用时需要修改连接信息 + - 每周一建立一张表 + - 每秒往测试表中插入N条数据 + - 每5分钟删除N条数据 + - 每1小时，修改一下数据 +""" + +# 目前支持 MySQL和Oracle两种数据库，其他的数据库需要做适配 +DBType = DBConfig.config['DBType'] + + +#获取连接信息 +def get_db_info(dbtype=DBType): + db_config = {} + if dbtype == "MySQL": + db_config = DBConfig.mysql_config() + elif dbtype == "Oracle": + db_config = DBConfig.oracle_config() + else: + assert False ,"DBType is not support." + return(db_config) + +#处理连接信息 +db_config = get_db_info() +OAuth = [str(db_config["db_user"]), str(db_config["db_password"])] +if not db_config["db_user"] and not db_config["db_password"]: + OAuth = None +# conn to db by jaydebeapi +conn = jaydebeapi.connect(db_config["driver_name"], + db_config["jdbc_url"], + OAuth, + db_config["driver_jar_path"]) +# conn.close() + +#执行sql +def execute_sql(sql): + with conn.cursor() as curs: + curs.execute(sql) + +# 按周创建，这里不采用这种方式，通过插入来触发 +#@repeat(every().monday) +def create_table(): + print("start create table...") + #重新获取一下 + db_config2 = get_db_info() + sql_string = db_config2["create_sql_string"] + print("create table ...." + sql_string) + try: + execute_sql(sql_string) + except: + print("table maybe exists,continue this create table task.") + + +#每秒插入N条数据 +@repeat(every(2).seconds) +def insert_data(): + print("insert data ....") + #重新获取一下，确保每次的插入的数据都不一样 + db_config2 = get_db_info() + print("insert data ...."+db_config2["jdbc_url"]) + insert_list = db_config2["insert_list"] + #执行插入 + try: + for sql_string in insert_list: + execute_sql(sql_string) + # 出错可能是表不存在，这里触发一下建表 + except: + create_table() + +#每5分钟删除N条数据 +@repeat(every(5).minutes) +def dalete_data(): + db_config2 = get_db_info() + print("dalete data....") + sql_string = db_config2["del_sql"] + try: + execute_sql(sql_string) + except: + print("table not exists,continue this delete task.") + + +#每1小时，修改一下数据 +@repeat(every(1).hours) +def updata_data(): + db_config2 = get_db_info() + print("updata data ....") + sql_string = db_config2["updata_sql"] + try: + execute_sql(sql_string) + except: + print("table not exists,continue this updata task.") + + +if __name__ == '__main__': + while True: + run_pending() diff --git a/bin/fileDataCreating.py b/bin/file/fileDataCreating.py similarity index 96% rename from bin/fileDataCreating.py rename to bin/file/fileDataCreating.py index 8bae273..0b2d02c 100644 --- a/bin/fileDataCreating.py +++ b/bin/file/fileDataCreating.py @@ -1,12 +1,10 @@ # -*- coding:utf-8 -*- -import os -import _load -import faker_data +from common import faker_data +from common import global_var import csv import yaml import time -import global_var import logging @@ -41,7 +39,7 @@ logger.addHandler(handler) # 定义公共的部分 path = Path(__file__) -etc_dir = path.parent.parent / "etc" +etc_dir = path.parent.parent.parent / "etc" config_file_patch = etc_dir / "file_config.yml" # 获取配置文件 @@ -49,7 +47,7 @@ with open(config_file_patch, "r", encoding='utf-8') as fy: config = yaml.safe_load(fy) # 默认为当前的目录 -data_dir = path.parent.parent / "data" +data_dir = path.parent.parent.parent / "data" if 'DATADIR' in config.keys() and config['DATADIR']: data_dir = Path(config['DATADIR']) diff --git a/etc/db_config.yml b/etc/db_config.yml new file mode 100644 index 0000000..796af9b --- /dev/null +++ b/etc/db_config.yml @@ -0,0 +1,20 @@ +#数据库配置文件 + +# 通用配置 +# 数据库类型，目前仅支持MySQL和Oracle,注意大小写 +DBType: Oracle +# 数据库地址 +IP: "10.10.29.41" +PROT: "1521" +# 数据库用户名 +DBUSER: "test" +# 数据库密码 +DBPASSWD: "test123" +# 库名 +DATABASES: "TEST" +# 每秒中插入多少条数据 +InsertRows: 200 + + +# 服务名-Oracle +SERVER: "orcl" diff --git a/etc/file_config.yml b/etc/file_config.yml new file mode 100644 index 0000000..cb63a61 --- /dev/null +++ b/etc/file_config.yml @@ -0,0 +1,11 @@ +#生成文件的配置文件 +vsersion : 1.3 + +# 生成csv输出的目录，默认为程序的运行目录。需要自定义就打开配置 +#DATADIR : "D:\\data" + +# 每次插入多少条数据 +InsertRows: 200 +# 每次删除多少条数据 +DeleteRows: 100 +