import re from datetime import datetime import pymysql import sys import os sys.path.append('/var/local/mode2/') from functions.apch_httpd_cf_action import get_vhostconfig from functions.utility import get_ctrinfo from functions.mwDB import get_settings DATE_FORMAT = "%a %b %d %H:%M:%S.%f %Y" HTTPD_ERROR_LOG = "/usr/home/%s/www/logs/error_log" MYSQL_SETTINGS_PATH = "/var/tmp/waf_settings" BATCH_LAST_RUN_TIME_PATH = "/var/tmp/waf_batch_lastruntime" WAF_BATCH_LOCK_PATH = "/var/tmp/waf_batch_lock" def main(): ctrid = get_ctrinfo()["ctrid"] vhostconfig = get_vhostconfig(ctrid) if len(vhostconfig) != 0: # プレミアムの場合 for domain in vhostconfig.keys(): errorlog = vhostconfig[domain]["errorlog"] #テンポラリドメインの場合 if errorlog == "": errorlog = HTTPD_ERROR_LOG % ctrid last_import_date = get_last_import_date(domain) import_log(domain, errorlog, last_import_date) else: # ビジネスの場合 domain = get_ctrinfo()["tmpdomain"] errorlog = HTTPD_ERROR_LOG % ctrid last_import_date = get_last_import_date(domain) import_log(domain, errorlog, last_import_date) def make_sql(col_name): sql = "INSERT INTO audit_log (" for idx, s in enumerate(col_name): if idx == 0: sql += s else: sql += "," + s sql += ") VALUE (" for idx, s in enumerate(col_name): if idx == 0: sql += "%s" else: sql += ",%s" sql += ")" return sql def insert_data(col_name, data): try: settings = get_settings(MYSQL_SETTINGS_PATH) connection = pymysql.connect( host='localhost', user=settings["mysql_user"], password=settings["mysql_password"], database='mod_security' ) with connection.cursor() as cursor: sql_query = make_sql(col_name) cursor.execute(sql_query, data) connection.commit() except pymysql.Error as e: print("Error while connecting to MySQL", e) del_batch_exec_status() exit(1) finally: if connection: connection.close() def get_last_import_date(domain): try: settings = get_settings(MYSQL_SETTINGS_PATH) connection = pymysql.connect( host='localhost', user=settings["mysql_user"], password=settings["mysql_password"], database='mod_security' ) with connection.cursor() as cursor: sql = f'SELECT MAX(access_date) FROM audit_log WHERE domain = \'{domain}\';' cursor.execute(sql) result = cursor.fetchall()[0][0] if result is None: result = datetime.strptime("2000-01-01 00:00:00", '%Y-%m-%d %H:%M:%S') return result except pymysql.Error as e: print("Error while connecting to MySQL", e) del_batch_exec_status() exit(1) finally: if connection: connection.close() def right_str(str, label): index = len(str) - len(label) return str[-index:] def cut_str(str, start_val, end_val): start_index = str.find(start_val) + len(start_val) end_index = str.find(end_val) if start_index != -1 and end_index != -1: result_value = str[start_index:end_index] else: result_value = str return result_value def import_log(domain, logfile, last_import_date): print(f'domain:{domain}') if not os.path.isfile(logfile): print(f'no file:{logfile}') return try: with open(logfile, "r") as f: lines = f.readlines() ins_cnt = 0 skip_cnt = 0 for line in lines: if "ModSecurity:" in line and "client " in line: try: st_pos = re.search(r'ModSecurity:*', line).start() end_pos = re.search(r'\[file', line).end() - 5 details = "" if st_pos and end_pos: details = line[st_pos:end_pos] except Exception as e: details = "" line = line.split("[") col_name = [] data = [] tag_cnt = 1 for index, s in enumerate(line): val = s.replace("]", "").replace("\"","").strip() if index == 1: col_name.append("access_date") val = datetime.strptime(val, DATE_FORMAT) access_date = val elif "client " in val and " ModSecurity:" in val: col_name.append("client") val = cut_str(val, "client ", " ModSecurity:") elif "data Matched Data: " in val: col_name.append("data") val = right_str(val, "data Matched Data: ") elif "msg " in val: col_name.append("msg") val = right_str(val, "msg ") elif "hostname " in val: col_name.append("hostname") val = right_str(val, "hostname ") elif "uri " in val: col_name.append("uri") val = right_str(val, "uri ") elif "severity " in val: col_name.append("severity") val = right_str(val, "severity ") elif re.match(r'^id*', val): col_name.append("crs_id") val = right_str(val, "id ") elif "file " in val: col_name.append("conf_file") val = right_str(val, "file ").split("/")[-1] elif "line " in val: col_name.append("line") val = right_str(val, "line ") elif "ver " in val: col_name.append("ver") val = right_str(val, "ver ") elif "unique_id " in val: col_name.append("unique_id") val = right_str(val, "unique_id ").split(",")[0] elif "tag " in val: col_name.append(f'tag_{tag_cnt}') val = right_str(val, "tag ") tag_cnt += 1 else: continue data.append(val) if len(data) == 1: continue col_name.append("domain") data.append(domain) col_name.append("details") data.append(details) ins_data = tuple(data) if access_date > last_import_date: insert_data(col_name, ins_data) ins_cnt += 1 else: skip_cnt += 1 print(f'ins_cnt: {ins_cnt}') print(f'skip_cnt: {skip_cnt}') except Exception as e: print(f'could not open file:{e}') return def setLastExecTime(): try: with open(BATCH_LAST_RUN_TIME_PATH, "w") as f: f.write(str(datetime.now())) except Exception as e: print(f'Could not open file:{e}') del_batch_exec_status() exit(1) def check_batch_exec_status(): if os.path.isfile(WAF_BATCH_LOCK_PATH): with open(WAF_BATCH_LOCK_PATH, "r") as f: t = f.readline().strip() try: last_run_time = datetime.strptime(t, '%Y-%m-%d %H:%M:%S.%f') now = datetime.now() if (now - last_run_time).total_seconds() <= 1800.0: exit(2) except Exception as e: print(e) del_batch_exec_status() make_batch_exec_status() else: make_batch_exec_status() def make_batch_exec_status(): with open(WAF_BATCH_LOCK_PATH, "w") as f: f.write(str(datetime.now())) def del_batch_exec_status(): os.remove(WAF_BATCH_LOCK_PATH) if __name__ == "__main__": # 起動チェック check_batch_exec_status() # メイン処理 main() # 実行完了時間をファイルに書き込む setLastExecTime() # バッチ実行管理ファイル削除 del_batch_exec_status() exit(0)