调整目录结构
This commit is contained in:
parent
45a904d8c7
commit
fc17a0206b
|
|
@ -3,8 +3,8 @@ import os
|
|||
import sys
|
||||
import logging
|
||||
|
||||
import fileDataCreating
|
||||
import global_var
|
||||
from file import fileDataCreating
|
||||
from common import global_var
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import faker_data
|
||||
import yaml
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
# 定义公共的部分
|
||||
path = Path(__file__)
|
||||
etc_dir = path.parent.parent / "etc"
|
||||
config_file_patch = etc_dir / "db_config.yml"
|
||||
|
||||
|
||||
# 获取配置文件
|
||||
with open(config_file_patch, "r", encoding='utf-8') as fy:
|
||||
config = yaml.safe_load(fy)
|
||||
# print(config)
|
||||
|
||||
|
||||
def get_now():
|
||||
week = str(datetime.now().isocalendar()[1])
|
||||
return(week)
|
||||
|
||||
|
||||
def mysql_config():
|
||||
mysql_info_dict={}
|
||||
mysql_info_dict['driver_name'] = "com.mysql.cj.jdbc.Driver"
|
||||
mysql_info_dict['driver_jar'] = "mysql-connector-java-8.0.29.jar"
|
||||
driver_jar_path = etc_dir / "driver" / mysql_info_dict['driver_jar']
|
||||
mysql_info_dict['driver_jar_path'] = str(driver_jar_path)
|
||||
#这里要连接到具体的库
|
||||
mysql_info_dict['jdbc_url'] = "jdbc:mysql://" + config['IP'] + ":" + config['PROT'] + "/" + config['DATABASES']
|
||||
mysql_info_dict['db_user'] = config['DBUSER']
|
||||
mysql_info_dict['db_password'] = config['DBPASSWD']
|
||||
|
||||
#create table sql
|
||||
week = get_now()
|
||||
table_name = "mysql_test_table_w" + week
|
||||
create_sql_string = "CREATE TABLE IF NOT EXISTS `"
|
||||
create_sql_string += table_name
|
||||
create_sql_string += "` ("
|
||||
create_sql_string += "`uuid` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`id` bigint DEFAULT NULL,"
|
||||
create_sql_string += "`name` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`mobile` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`ssn` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`sex` int DEFAULT NULL,"
|
||||
create_sql_string += "`email` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`job` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`address` varchar(50) DEFAULT NULL,"
|
||||
create_sql_string += "`actime_time` varchar(50) NULL DEFAULT NULL"
|
||||
create_sql_string += ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"
|
||||
mysql_info_dict['create_sql_string'] = create_sql_string
|
||||
|
||||
# insert into sql
|
||||
insert_list = []
|
||||
data = faker_data.faker_data(lines=config['InsertRows'])[1:]
|
||||
for a in data:
|
||||
#拼接sql
|
||||
insert_sql_string = "INSERT INTO " + table_name
|
||||
insert_sql_string += "(uuid, id, name, mobile, ssn, sex, email, job, address, actime_time) VALUES("
|
||||
str_data = ','.join("'{0}'".format(x) for x in a)
|
||||
insert_sql_string += str_data + ");"
|
||||
|
||||
insert_list.append(insert_sql_string)
|
||||
# 返回所有的插入语句
|
||||
mysql_info_dict['insert_list'] = insert_list
|
||||
|
||||
# delete sql
|
||||
del_sql = "delete FROM "+ table_name + " where 1=1 limit 100"
|
||||
mysql_info_dict['del_sql'] = del_sql
|
||||
|
||||
|
||||
# uptata sql
|
||||
updata_sql = "UPDATE "+ table_name + " t SET t.job='unary_测试' WHERE t.name like '李%' AND sex = 1 AND id = 1 "
|
||||
mysql_info_dict['updata_sql'] = updata_sql
|
||||
|
||||
return(mysql_info_dict)
|
||||
|
||||
|
||||
def oracle_config():
|
||||
oracle_info_dict={}
|
||||
oracle_info_dict['driver_name'] = "oracle.jdbc.OracleDriver"
|
||||
oracle_info_dict['driver_jar'] = "ojdbc8-12.2.0.1.jar"
|
||||
driver_jar_path = etc_dir / "driver" / oracle_info_dict['driver_jar']
|
||||
oracle_info_dict['driver_jar_path'] = str(driver_jar_path)
|
||||
#这里要连接到具体的实例
|
||||
oracle_info_dict['jdbc_url'] = "jdbc:oracle:thin:@" + config['IP'] + ":" + config['PROT'] + "/" + config['SERVER']
|
||||
oracle_info_dict['db_user'] = config['DBUSER']
|
||||
oracle_info_dict['db_password'] = config['DBPASSWD']
|
||||
|
||||
#create table sql
|
||||
week = get_now()
|
||||
table_name = "ORACLE_TEST_w" + week
|
||||
create_sql_string = 'CREATE TABLE "'
|
||||
create_sql_string += table_name + '" '
|
||||
create_sql_string += '("UUID" VARCHAR2(50),'
|
||||
create_sql_string += '"ID" NUMBER(38,0), '
|
||||
create_sql_string += '"NAME" VARCHAR2(50),'
|
||||
create_sql_string += '"MOBILE" VARCHAR2(50),'
|
||||
create_sql_string += '"SSN" VARCHAR2(50),'
|
||||
create_sql_string += '"SEX" NUMBER(*,0), '
|
||||
create_sql_string += '"EMAIL" VARCHAR2(50), '
|
||||
create_sql_string += '"JOB" VARCHAR2(50),'
|
||||
create_sql_string += '"ADDRESS" VARCHAR2(100),'
|
||||
create_sql_string += '"ACTIME_TIME" VARCHAR2(50))'
|
||||
oracle_info_dict['create_sql_string'] = create_sql_string
|
||||
|
||||
|
||||
# insert into sql
|
||||
insert_list = []
|
||||
data = faker_data.faker_data(lines=config['InsertRows'])[1:]
|
||||
for a in data:
|
||||
#拼接sql
|
||||
insert_sql_string = 'INSERT INTO ' + config['DATABASES'] + '."' + table_name + '" '
|
||||
insert_sql_string += " t (uuid, id, name, mobile, ssn, sex, email, job, address, actime_time) VALUES("
|
||||
str_data = ','.join("'{0}'".format(x) for x in a)
|
||||
insert_sql_string += str_data + ")"
|
||||
|
||||
insert_list.append(insert_sql_string)
|
||||
# 返回所有的插入语句
|
||||
oracle_info_dict['insert_list'] = insert_list
|
||||
|
||||
# delete sql
|
||||
del_sql = 'delete FROM '+ config['DATABASES'] + '."' + table_name + '" WHERE rownum<=100'
|
||||
oracle_info_dict['del_sql'] = del_sql
|
||||
|
||||
|
||||
# uptata sql
|
||||
updata_sql = "UPDATE "+ config['DATABASES'] + '."' + table_name + '" '+ " t SET t.job='unary_测试' WHERE t.name like '李%' AND sex = 1 AND id = 1 "
|
||||
oracle_info_dict['updata_sql'] = updata_sql
|
||||
|
||||
return(oracle_info_dict)
|
||||
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import _load
|
||||
|
||||
import DBConfig
|
||||
|
||||
import jaydebeapi
|
||||
|
||||
from schedule import every, repeat, run_pending
|
||||
|
||||
|
||||
"""
|
||||
说明：
|
||||
1. 用于数据库的持续增、删、改，目前支持MySQL和Oracle
|
||||
2. DBConfig.py 为数据库连接配置，使用时需要修改连接信息
|
||||
- 每周一建立一张表
|
||||
- 每秒往测试表中插入N条数据
|
||||
- 每5分钟删除N条数据
|
||||
- 每1小时，修改一下数据
|
||||
"""
|
||||
|
||||
# 目前支持 MySQL和Oracle两种数据库，其他的数据库需要做适配
|
||||
DBType = DBConfig.config['DBType']
|
||||
|
||||
|
||||
#获取连接信息
|
||||
def get_db_info(dbtype=DBType):
|
||||
db_config = {}
|
||||
if dbtype == "MySQL":
|
||||
db_config = DBConfig.mysql_config()
|
||||
elif dbtype == "Oracle":
|
||||
db_config = DBConfig.oracle_config()
|
||||
else:
|
||||
assert False ,"DBType is not support."
|
||||
return(db_config)
|
||||
|
||||
#处理连接信息
|
||||
db_config = get_db_info()
|
||||
OAuth = [str(db_config["db_user"]), str(db_config["db_password"])]
|
||||
if not db_config["db_user"] and not db_config["db_password"]:
|
||||
OAuth = None
|
||||
# conn to db by jaydebeapi
|
||||
conn = jaydebeapi.connect(db_config["driver_name"],
|
||||
db_config["jdbc_url"],
|
||||
OAuth,
|
||||
db_config["driver_jar_path"])
|
||||
# conn.close()
|
||||
|
||||
#执行sql
|
||||
def execute_sql(sql):
|
||||
with conn.cursor() as curs:
|
||||
curs.execute(sql)
|
||||
|
||||
# 按周创建，这里不采用这种方式，通过插入来触发
|
||||
#@repeat(every().monday)
|
||||
def create_table():
|
||||
print("start create table...")
|
||||
#重新获取一下
|
||||
db_config2 = get_db_info()
|
||||
sql_string = db_config2["create_sql_string"]
|
||||
print("create table ...." + sql_string)
|
||||
try:
|
||||
execute_sql(sql_string)
|
||||
except:
|
||||
print("table maybe exists,continue this create table task.")
|
||||
|
||||
|
||||
#每秒插入N条数据
|
||||
@repeat(every(2).seconds)
|
||||
def insert_data():
|
||||
print("insert data ....")
|
||||
#重新获取一下，确保每次的插入的数据都不一样
|
||||
db_config2 = get_db_info()
|
||||
print("insert data ...."+db_config2["jdbc_url"])
|
||||
insert_list = db_config2["insert_list"]
|
||||
#执行插入
|
||||
try:
|
||||
for sql_string in insert_list:
|
||||
execute_sql(sql_string)
|
||||
# 出错可能是表不存在，这里触发一下建表
|
||||
except:
|
||||
create_table()
|
||||
|
||||
#每5分钟删除N条数据
|
||||
@repeat(every(5).minutes)
|
||||
def dalete_data():
|
||||
db_config2 = get_db_info()
|
||||
print("dalete data....")
|
||||
sql_string = db_config2["del_sql"]
|
||||
try:
|
||||
execute_sql(sql_string)
|
||||
except:
|
||||
print("table not exists,continue this delete task.")
|
||||
|
||||
|
||||
#每1小时，修改一下数据
|
||||
@repeat(every(1).hours)
|
||||
def updata_data():
|
||||
db_config2 = get_db_info()
|
||||
print("updata data ....")
|
||||
sql_string = db_config2["updata_sql"]
|
||||
try:
|
||||
execute_sql(sql_string)
|
||||
except:
|
||||
print("table not exists,continue this updata task.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
while True:
|
||||
run_pending()
|
||||
|
|
@ -1,12 +1,10 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import os
|
||||
import _load
|
||||
|
||||
import faker_data
|
||||
from common import faker_data
|
||||
from common import global_var
|
||||
import csv
|
||||
import yaml
|
||||
import time
|
||||
import global_var
|
||||
import logging
|
||||
|
||||
|
||||
|
|
@ -41,7 +39,7 @@ logger.addHandler(handler)
|
|||
|
||||
# 定义公共的部分
|
||||
path = Path(__file__)
|
||||
etc_dir = path.parent.parent / "etc"
|
||||
etc_dir = path.parent.parent.parent / "etc"
|
||||
config_file_patch = etc_dir / "file_config.yml"
|
||||
|
||||
# 获取配置文件
|
||||
|
|
@ -49,7 +47,7 @@ with open(config_file_patch, "r", encoding='utf-8') as fy:
|
|||
config = yaml.safe_load(fy)
|
||||
|
||||
# 默认为当前的目录
|
||||
data_dir = path.parent.parent / "data"
|
||||
data_dir = path.parent.parent.parent / "data"
|
||||
if 'DATADIR' in config.keys() and config['DATADIR']:
|
||||
data_dir = Path(config['DATADIR'])
|
||||
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#数据库配置文件
|
||||
|
||||
# 通用配置
|
||||
# 数据库类型，目前仅支持MySQL和Oracle,注意大小写
|
||||
DBType: Oracle
|
||||
# 数据库地址
|
||||
IP: "10.10.29.41"
|
||||
PROT: "1521"
|
||||
# 数据库用户名
|
||||
DBUSER: "test"
|
||||
# 数据库密码
|
||||
DBPASSWD: "test123"
|
||||
# 库名
|
||||
DATABASES: "TEST"
|
||||
# 每秒中插入多少条数据
|
||||
InsertRows: 200
|
||||
|
||||
|
||||
# 服务名-Oracle
|
||||
SERVER: "orcl"
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#生成文件的配置文件
|
||||
vsersion : 1.3
|
||||
|
||||
# 生成csv输出的目录，默认为程序的运行目录。需要自定义就打开配置
|
||||
#DATADIR : "D:\\data"
|
||||
|
||||
# 每次插入多少条数据
|
||||
InsertRows: 200
|
||||
# 每次删除多少条数据
|
||||
DeleteRows: 100
|
||||
|
||||
Loading…
Reference in New Issue