from settings import get_connection_event class contentPipeline: def process_item(self, item, spider): print("entered process_item:") print("item: ", item) with get_connection_event() as conn: with conn.cursor() as cur: try: sql_statement = f""" select * from unloze_event.event e where e.event_title = %s """ cur.execute(sql_statement, [item['event_title']]) res = cur.fetchone() if res is None: sql_statement = f""" delete from unloze_event.event where event_server like '%{item['event_server'].split(":27")[1]}%' """ #very cheap way of replacing rows print("deleting thread: ", sql_statement) cur.execute(sql_statement) sql_statement = f""" insert into unloze_event.event (event_title, event_server, event_maps, event_date, event_time, event_reward, event_url) VALUES (%s, %s, %s, %s, %s, %s, %s) """ cur.execute(sql_statement, [item['event_title'], item['event_server'], item['event_maps'], item['event_date'], item['event_time'], item['event_reward'], item['event_url']]) #context manager does not seem to work with this mysql library so manual commiting seems needed conn.commit() except Exception: import traceback error_msg = traceback.format_exc() print("error_msg: ", error_msg) return item