支持了基础的黑名单功能。下一个版本会提供黑名单文档和接口。

Author: white-shiro-bai

README.md

@@ -31,6 +31,8 @@ ghost_sa(鬼策)可以理解为不带前端界面的神策服务端。
 
 9.召回信息发送
 
+10.支持黑名单功能，便于提高一些免费邮件如阿里的授信，也减少其他渠道的无效召回浪费
+
 使用了flask框架，可以通过uwsgi部署。数据库建议使用TiDB，实测1天200万事件量，单次查询当天事件在10毫秒左右，查询1个月范围的数据，返回在30-60秒左右。
 实际使用在TiDB最低的配置3x8c_32g的情况下，每天可以支持500万的事件量。
 如果只是体验和测试功能，也可以用MySQL 5.7（含）以上的版本，不过性能很差。
@@ -106,24 +108,31 @@ https://space.bilibili.com/920446/channel/detail?cid=124583
 
 # 2020年12月28日版本升级注意事项：
 
-1.鬼策老用户如需升级2020年12月28日及以后的版本，须文件覆盖后，运行/tools/update_20201015.py程序升级数据表。该操作不可逆，但升级后的数据表与老版本鬼策兼容。但程序与老版本鬼策不兼容，需要完整覆盖安装目录才能继续运行。建议新老版本分开两个目录放置。clone新目录->升级->运行新目录的程序即可。
+1.鬼策老用户如需升级2020年12月28日及以后的版本，须文件覆盖后，运行/tools/update_base_to_20201015.py程序升级数据表。该操作不可逆，但升级后的数据表与老版本鬼策兼容。但程序与老版本鬼策不兼容，需要完整覆盖安装目录才能继续运行。建议新老版本分开两个目录放置。clone新目录->升级->运行新目录的程序即可。
 
 2.由于TiDB v4的新特性，鬼策缩短了一些之前预留较长的字段，以提高兼容性，mysql版本也不需要额外使用安装程序，统一使用/component/setup.py安装即可。缩短后我已经在生产环境使用超过半个月，尚无不良影响。如果您的应用环境特殊，请谨慎升级TiDB到v4及以后版本，并自行修改建表程序中的字段长度。如果您的鬼策是在TiDB v3环境下安装的，那么升级到TiDB v4后，字段长度不会改变，依然有效。
 
+# 2021年03月15日版本升级注意事项：
+
+1.2020年12月28日之前版本的鬼策老用户如需升级2021年03月15日及以后的版本，须文件覆盖后，运行/tools/update_base_to_20210226.py程序升级数据表。或2020年12月28日之后版本的鬼策老用户如需升级2021年03月15日及以后的版本，须文件覆盖后，运行/tools/update_20201015_to_20210226.py程序升级数据表。该操作不可逆，但升级后的数据表与老版本鬼策兼容。但程序与老版本鬼策不兼容，需要完整覆盖安装目录才能继续运行。建议新老版本分开两个目录放置。clone新目录->升级->运行新目录的程序即可。
+
+2.黑名单开关可以在configs/admin.py里配置
 
 
 # 近期大版本迭代规划：
 
-2021年02月<-补齐用户分群及召回功能的文档和交互用接口。ghost_console（管理端）功能补齐
+2021年03月<-黑名单支持外部接口
 
-2021年03月<-召回支持黑名单功能，便于提高一些免费邮件如阿里的授信，也减少其他渠道的无效召回浪费
+2021年04月<-补齐用户分群及召回功能的文档和交互用接口。ghost_console（管理端）功能补齐
 
-2021年04月<-召回功能的消息中心支持非模板消息
+2021年05月<-增加连接池，提高高并发性能
 
 2021年06月<-召回功能适配更多的渠道
 
 2021年08月<-支持广告位管理及第三方监测（如秒针等。这个是用来给第三方监测的。不是自己用CPC广告的那些，那个早就支持了。）
 
 2021年10月<-支持项目管理和神策的crc校验功能，减少恶意攻击和恶意刷量。
 
-2022年4月<-支持oCPC广告位管理，支持第三方回调，简单的效果管理
+2022年4月<-支持oCPC广告位管理，支持第三方回调，简单的效果管理
+
+2022年6月<-支持Hadoop等其他类型数据库

component/api_noti.py

@@ -18,7 +18,6 @@
 from scheduler_jobs.etl_model import apply_temple
 
 
-
 def show_usergroup_plan():
     #查询用户分群计划列表
     start_time = time.time()
@@ -411,7 +410,6 @@ def create_scheduler_jobs_manual():
             returnjson = {'result':'fail','error':error}
             return jsonify(returnjson)
 
-
 def create_manual_temple_noti():
     #外部触发模板消息
     project = get_url_params('project')

component/db_func.py

@@ -403,10 +403,10 @@ def select_usergroupdata_data(project,data_id):
     result = do_tidb_select(sql=sql)
     return result[0],result[1]
 
-def insert_noti(project,type_1,created_at,updated_at,distinct_id,content,send_at,plan_id=0,list_id=0,data_id=0,temple_id=0,noti_group_id=0,priority=13,status=9,owner='noti',recall_result=None):
+def insert_noti(project,type_1,created_at,updated_at,distinct_id,content,send_at,plan_id=0,list_id=0,data_id=0,temple_id=0,noti_group_id=0,priority=13,status=9,owner='noti',recall_result=None,key=None,level=None):
     import json
-    sql = """insert ignore {project}_noti (`distinct_id`,`plan_id`,`list_id`,`data_id`,`temple_id`,`noti_group_id`,`priority`,`status`,`owner`,`type`,`content`,`send_at`,`recall_result`,`created_at`,`updated_at`) values (%(distinct_id)s,%(plan_id)s,%(list_id)s,%(data_id)s,%(temple_id)s,%(noti_group_id)s,%(priority)s,%(status)s,%(owner)s,%(type)s,%(content)s,%(send_at)s,%(recall_result)s,%(created_at)s,%(updated_at)s)""".format(project=project)
-    key = {'distinct_id':distinct_id,'type':type_1,'plan_id':plan_id,'list_id':list_id,'data_id':data_id,'temple_id':temple_id,'noti_group_id':noti_group_id,'priority':priority,'status':status,'owner':owner,'content':json.dumps(content),'recall_result':recall_result,'send_at':send_at if send_at else int(time.time()),'created_at':created_at if created_at else int(time.time()),'updated_at':updated_at if updated_at else int(time.time())}
+    sql = """insert ignore {project}_noti (`distinct_id`,`plan_id`,`list_id`,`data_id`,`temple_id`,`noti_group_id`,`priority`,`status`,`owner`,`type`,`content`,`send_at`,`recall_result`,`created_at`,`updated_at`,`key`,`level`) values (%(distinct_id)s,%(plan_id)s,%(list_id)s,%(data_id)s,%(temple_id)s,%(noti_group_id)s,%(priority)s,%(status)s,%(owner)s,%(type)s,%(content)s,%(send_at)s,%(recall_result)s,%(created_at)s,%(updated_at)s,%(key)s,%(level)s)""".format(project=project)
+    key = {'key':key,'distinct_id':distinct_id,'type':type_1,'plan_id':plan_id,'list_id':list_id,'data_id':data_id,'temple_id':temple_id,'noti_group_id':noti_group_id,'priority':priority,'status':status,'owner':owner,'content':json.dumps(content),'recall_result':recall_result,'send_at':send_at if send_at else int(time.time()),'created_at':created_at if created_at else int(time.time()),'updated_at':updated_at if updated_at else int(time.time()),'level':level}
     result = do_tidb_exe(sql=sql, args=key)
     return result[0],result[1]
 
@@ -776,4 +776,133 @@ def show_scheduler_jobs_count_db():
     sql = f"""select count(*)
     from scheduler_jobs """
     result = do_tidb_select(sql=sql)
+    return result
+
+def insert_update_recall_blacklist(project,key,type_id,status,reason_id,latest_owner,distinct_id='',timenow=None):
+    sql="""INSERT INTO `recall_blacklist` ( `project`, `distinct_id`, `key`, `type_id`, `reason_id`, `owner`, `latest_owner`, `status`, `created_at` ,`updated_at`)
+VALUES
+	(%(project)s,%(distinct_id)s,%(key)s,%(type_id)s,%(reason_id)s,%(owner)s,%(latest_owner)s,%(status)s,%(created_at)s,%(updated_at)s) 
+	ON DUPLICATE KEY UPDATE `reason_id` =IF(%(reason_id)s != 0 ,%(reason_id)s, reason_id ),`status` =IF(%(status)s != 0,%(status)s, status ),`distinct_id` = IF(%(distinct_id)s != '',%(distinct_id)s, distinct_id ),latest_owner =%(latest_owner)s,updated_at =%(updated_at)s;"""
+    timenow = int(time.time()) if not timenow else timenow
+    key = {'project':project,'distinct_id':distinct_id,'key':key,'type_id':type_id,'reason_id':reason_id,'owner':latest_owner,'latest_owner':latest_owner,'status':status,'created_at':timenow,'updated_at':timenow}
+    result = do_tidb_exe(sql=sql, args=key)
+    return result
+
+def insert_recall_blacklist(project,key,type_id,status,reason_id,latest_owner,distinct_id='',timenow=None):
+    sql="""INSERT INTO `recall_blacklist` ( `project`, `distinct_id`, `key`, `type_id`, `reason_id`, `owner`, `latest_owner`, `status`, `created_at` ,`updated_at`)
+VALUES
+	(%(project)s,%(distinct_id)s,%(key)s,%(type_id)s,%(reason_id)s,%(owner)s,%(latest_owner)s,%(status)s,%(created_at)s,%(updated_at)s);"""
+    timenow = int(time.time()) if not timenow else timenow
+    key = {'project':project,'distinct_id':distinct_id,'key':key,'type_id':type_id,'reason_id':reason_id,'owner':latest_owner,'latest_owner':latest_owner,'status':status,'created_at':timenow,'updated_at':timenow}
+    result = do_tidb_exe(sql=sql, args=key)
+    return result
+
+def update_recall_blacklist(project,key,type_id,status,reason_id,latest_owner,distinct_id='',timenow=None):
+    sql="""UPDATE `recall_blacklist` set `reason_id` =IF(%(reason_id)s != 0 ,%(reason_id)s, reason_id ),`status` =IF(%(status)s != 0,%(status)s, status ),`distinct_id` = IF(%(distinct_id)s != '',%(distinct_id)s, distinct_id ),latest_owner =%(latest_owner)s,updated_at =%(updated_at)s;"""
+    timenow = int(time.time()) if not timenow else timenow
+    key = {'project':project,'distinct_id':distinct_id,'key':key,'type_id':type_id,'reason_id':reason_id,'owner':latest_owner,'latest_owner':latest_owner,'status':status,'created_at':timenow,'updated_at':timenow}
+    result = do_tidb_exe(sql=sql, args=key)
+    return result
+
+
+def insert_recall_blacklist_reason(rbid,reason_id,reason_owner,final_status_id,reason_comment='',timenow=None):
+    sql="""insert into `recall_blacklist_reason` (`rbid`,`reason_id`,`reason_owner`,`reason_comment`,`final_status_id`,`created_at`) VALUES (%(rbid)s,%(reason_id)s,%(reason_owner)s,%(reason_comment)s,%(final_status_id)s,%(created_at)s)"""
+    timenow = int(time.time()) if not timenow else timenow
+    key = {'rbid':rbid,'reason_id':reason_id,'reason_owner':reason_owner,'reason_comment':reason_comment,'final_status_id':final_status_id,'created_at':timenow}
+    result = do_tidb_exe(sql=sql, args=key)
+    return result
+
+def select_recall_blacklist_id(type_id,distinct_id=None,key=None,project=None,status=None,limit=None):
+    add_on_status =''
+    add_limit =''
+    add_fliter = ''
+    if limit :
+        add_limit = 'limit '+str(int(limit))
+    if distinct_id :
+        add_fliter = add_fliter + f' and recall_blacklist.`distinct_id` = "{distinct_id}"'
+    if key :
+        add_fliter = add_fliter + f' and recall_blacklist.`key` = "{key}"'
+    if project :
+        add_fliter = add_fliter + f' and recall_blacklist.`project` = "{project}"'
+    if status :
+        status_int = []
+        for s in status:
+            status_int.append(str(int(s)))
+        add_on_status = "and recall_blacklist.status in ("+",".join(status_int)+")"
+    
+    sql="""SELECT
+	recall_blacklist.id,
+	recall_blacklist.reason_id,
+	r_id.`desc`,
+	recall_blacklist.`owner`,
+	recall_blacklist.latest_owner,
+	recall_blacklist.`status`,
+	s_id.`desc`,
+	cast(FROM_UNIXTIME(recall_blacklist.created_at) as char),
+	cast(FROM_UNIXTIME(recall_blacklist.updated_at) as char),
+	recall_blacklist_reason.final_status_id,
+	s2_id.`desc`,
+	recall_blacklist_reason.reason_id,
+	r2_id.`desc`,
+	recall_blacklist_reason.reason_owner,
+	recall_blacklist_reason.reason_comment,
+	cast(FROM_UNIXTIME(recall_blacklist_reason.created_at) as char)
+FROM
+	recall_blacklist
+	JOIN recall_blacklist_reason ON recall_blacklist.id = recall_blacklist_reason.rbid 
+	join status_code as r_id on recall_blacklist.reason_id = r_id.id
+	join status_code as s_id on recall_blacklist.`status` = s_id.id
+	join status_code as r2_id on recall_blacklist_reason.`reason_id` = r2_id.id
+	join status_code as s2_id on recall_blacklist_reason.`final_status_id` = s2_id.id
+	where recall_blacklist.type_id = {type_id} {add_fliter} {add_on_status}
+ORDER BY
+	recall_blacklist_reason.created_at DESC {add_limit} ;""".format(type_id=type_id,add_fliter=add_fliter,add_on_status=add_on_status,add_limit=add_limit)
+    result = do_tidb_select(sql=sql)
+    return result
+
+def select_recall_blacklist_list(type_id,distinct_id=None,project=None,status=None,start=None,limit=None):
+    add_on_status =''
+    add_limit =''
+    add_fliter = ''
+    if not start:
+        start = 0
+    if limit :
+        add_limit = f'limit {start},{limit}'
+    if distinct_id :
+        add_fliter = add_fliter + f' and recall_blacklist.`distinct_id` = "{distinct_id}"'
+    if project :
+        add_fliter = add_fliter + f' and recall_blacklist.`project` = "{project}"'
+    if status :
+        status_int = []
+        for s in status:
+            status_int.append(str(int(s)))
+        add_on_status = "and recall_blacklist.status in ("+",".join(status_int)+")"
+    
+    sql="""SELECT
+	recall_blacklist.id,
+	recall_blacklist.reason_id,
+	r_id.`desc`,
+	recall_blacklist.`owner`,
+	recall_blacklist.latest_owner,
+	recall_blacklist.`status`,
+	s_id.`desc`,
+	cast(FROM_UNIXTIME(recall_blacklist.created_at) as char),
+	cast(FROM_UNIXTIME(recall_blacklist.updated_at) as char)
+FROM
+	recall_blacklist
+	join status_code as r_id on recall_blacklist.`reason_id` = r_id.id
+	join status_code as s_id on recall_blacklist.`status` = s_id.id
+	where recall_blacklist.type_id = {type_id} {add_fliter} {add_on_status} {add_limit} ;""".format(type_id=type_id,add_fliter=add_fliter,add_on_status=add_on_status,add_limit=add_limit)
+    result = do_tidb_select(sql=sql)
+    return result
+
+
+
+
+
+def insert_recall_blacklist_history(rbid,checker,result_status_id,result_reason_id,timenow=None):
+    sql="""insert into `recall_blacklist_history` (`rbid`,`checker`,`result_status_id`,`result_reason_id`,`created_at`) VALUES (%(rbid)s,%(checker)s,%(result_status_id)s,%(result_reason_id)s,%(created_at)s)"""
+    timenow = int(time.time()) if not timenow else timenow
+    key = {'rbid':rbid,'checker':checker,'result_status_id':result_status_id,'result_reason_id':result_reason_id,'created_at':timenow}
+    result = do_tidb_exe(sql=sql, args=key)
     return result