# coding: UTF-8 import sys import os import hashlib import traceback sys.path.append('/var/local/mode2/') # Recommend Modules from lib.WebRecommend.searchtrend import SearchTrend from lib.WebRecommend.accesstrend import AccessTrend from lib.WebRecommend.siteflow import SiteFlow from lib.WebRecommend.userDevice import userDevice from lib.WebRecommend.userRegion import userRegion from functions.mwDB import mwDB def insert_data(conn, dataset, recommend_category): try: print(f'recommend_category: {recommend_category}') print(f'dataset: {dataset}') for i in dataset: body_hash = hashlib.md5(i['text'].encode('utf-8')).hexdigest() # 一週間以内に同じレコメンド文が生成された場合はインサートをスキップする sql = "SELECT COUNT(*) as cnt FROM web_recommend WHERE recommend_body_hash='%s' AND date > date_add(current_date, interval -7 day)" % (body_hash) res = conn.fetchone(sql) if int(res['cnt']) == 0: sql = "INSERT INTO web_recommend (date, recommend_category, recommend_body_hash, recommend_body) VALUES (CURDATE(), '%s', '%s', '%s') " \ "ON DUPLICATE KEY UPDATE date=CURDATE(), recommend_category='%s', recommend_body_hash='%s', recommend_body='%s';" % \ (recommend_category, body_hash, i['text'], recommend_category, body_hash, i['text']) # print sql conn.execute(sql) conn.commit() except Exception as e: raise e if __name__ == "__main__": try: settings_list = [] target_path = f'/var/tmp/' for x in os.listdir(target_path): if x.startswith('mode2_webanalysis_settings'): if x == 'mode2_webanalysis_settings': settings_list.append(False) else: domain = x.replace("mode2_webanalysis_settings_", "") settings_list.append(domain) for domain in settings_list: try: conn = mwDB(domain, DictCursor=True) try: at = AccessTrend(domain) insert_data(conn, at.getRecommend(), 'access_trend') except Exception as e: raise e try: st = SearchTrend(domain) insert_data(conn, st.getRecommend(), 'search_trend') except Exception as e: raise e # page analysis try: sf = SiteFlow(domain) insert_data(conn, sf.getRecommend(), 'site_flow') except Exception as e: raise e # user device try: ud = userDevice(domain) insert_data(conn, ud.getRecommend(), 'user_device') except Exception as e: raise e # user region try: ur = userRegion(domain) insert_data(conn, ur.getRecommend(), 'user_region') except Exception as e: raise e except Exception as e: raise e finally: conn.close() except Exception as e: print(traceback(e))