168 lines
6.2 KiB
Python
168 lines
6.2 KiB
Python
import pandas as pd
|
|
from sshtunnel import SSHTunnelForwarder
|
|
import pymysql
|
|
|
|
|
|
class JumpHostDatabaseConnector:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
|
|
def connect_to_jump_host(self):
|
|
server = SSHTunnelForwarder(
|
|
(self.config['jump_host_ip'], self.config['jump_host_port']),
|
|
ssh_username=self.config['jump_host_user'],
|
|
ssh_password=self.config['jump_host_password'],
|
|
remote_bind_address=(self.config['database_ip'], self.config['database_port'])
|
|
)
|
|
server.start()
|
|
return server
|
|
|
|
def connect_to_database(self, server):
|
|
conn = pymysql.connect(
|
|
host='127.0.0.1', # 这里必须填127.0.0.1
|
|
port=server.local_bind_port,
|
|
user=self.config['database_user'],
|
|
password=self.config['database_password'],
|
|
db=self.config['database_name']
|
|
)
|
|
return conn
|
|
|
|
def query_database(self, query):
|
|
with self.connect_to_jump_host() as server:
|
|
with self.connect_to_database(server) as conn:
|
|
with conn.cursor() as cursor:
|
|
cursor.execute(query)
|
|
results = cursor.fetchall()
|
|
return results
|
|
|
|
def query_database_select_field_names(self, query):
|
|
with self.connect_to_jump_host() as server:
|
|
with self.connect_to_database(server) as conn:
|
|
with conn.cursor() as cursor:
|
|
cursor.execute(query)
|
|
results = cursor.fetchall()
|
|
field_names = [desc[0] for desc in cursor.description]
|
|
return {'field_names': field_names, 'results': results}
|
|
|
|
def query_database_select_to_dataframe(self, query):
|
|
with self.connect_to_jump_host() as server:
|
|
with self.connect_to_database(server) as conn:
|
|
df = pd.read_sql_query(query, conn)
|
|
return df.to_dict(orient='records')
|
|
|
|
|
|
# 配置字典
|
|
config_common = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysqlc684e4271035.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_ucx_common"
|
|
}
|
|
|
|
config_task = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysql94eab4cb64ee.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_task_ucx"
|
|
}
|
|
|
|
config_baidu_task = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysqlf546d0debe54.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_task_baidu"
|
|
}
|
|
|
|
config_material1 = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysql7f735bf46c55.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_material"
|
|
}
|
|
|
|
config_material2 = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysql58c5730efa71.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_material"
|
|
}
|
|
|
|
config_async_create_ad_batch = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysqlf43ac72e50a0.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_task"
|
|
}
|
|
# 生产魔剪业务库配置
|
|
config_chuangliang_ad_magic_cut = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysql2d9941b86eb8.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_magic_cut"
|
|
}
|
|
# 生产魔剪任务库配置
|
|
config_chuangliang_ad_magic_task_skit = {
|
|
'jump_host_ip': "180.184.103.38",
|
|
'jump_host_port': 2222,
|
|
'jump_host_user': "cl_ad",
|
|
'jump_host_password': "4CGbdPW2zkbewcp^",
|
|
'database_ip': "mysqlc00f5833fdad.rds.ivolces.com",
|
|
'database_port': 3306,
|
|
'database_user': "cl_readOnly",
|
|
'database_password': "Chuangliang@2023",
|
|
'database_name': "chuangliang_ad_task_skit"
|
|
}
|
|
|
|
# 实例化数据库连接器
|
|
UC_common_db = JumpHostDatabaseConnector(config_common)
|
|
UC_task_db = JumpHostDatabaseConnector(config_task)
|
|
Bd_task_db = JumpHostDatabaseConnector(config_baidu_task)
|
|
Material1_db = JumpHostDatabaseConnector(config_material1)
|
|
Material2_db = JumpHostDatabaseConnector(config_material2)
|
|
Async_create_ad_batch_db = JumpHostDatabaseConnector(config_async_create_ad_batch)
|
|
Chuangliang_ad_magic_cut = JumpHostDatabaseConnector(config_chuangliang_ad_magic_cut)
|
|
Chuangliang_ad_magic_task_skit = JumpHostDatabaseConnector(config_chuangliang_ad_magic_task_skit)
|
|
|
|
if __name__ == '__main__':
|
|
sql = "SELECT * FROM chuangliang_ad_magic_cut.config_apollo WHERE `key` = 'client_task_num_day_limit'"
|
|
magic_sql = "SELECT task_id,batch_id,task_param,result_data,main_user_id,create_user_id FROM chuangliang_ad_task_skit.task_log_client_mix_shear WHERE task_id = 30821620"
|
|
material_sql = "SELECT new_material_id,material_id,material_id_start,material_id_start_end,start_time,end_time,use_duration,rule_id,task_id,is_effect FROM chuangliang_ad_magic_cut.client_material_relation WHERE task_id = 30821691"
|
|
# 执行查询
|
|
results = Chuangliang_ad_magic_cut.query_database_select_to_dataframe(material_sql)
|
|
print(results)
|