38 lines
1.8 KiB
Python
38 lines
1.8 KiB
Python
from settings import get_connection_event
|
|
|
|
class contentPipeline:
|
|
def process_item(self, item, spider):
|
|
print("entered process_item:")
|
|
print("item: ", item)
|
|
with get_connection_event() as conn:
|
|
with conn.cursor() as cur:
|
|
try:
|
|
sql_statement = f"""
|
|
select * from unloze_event.event e
|
|
where e.event_title = %s
|
|
"""
|
|
cur.execute(sql_statement, [item['event_title']])
|
|
res = cur.fetchone()
|
|
if res is None:
|
|
sql_statement = f"""
|
|
delete from unloze_event.event
|
|
where event_server like '%{item['event_server'].split(":27")[1]}%'
|
|
"""
|
|
#very cheap way of replacing rows
|
|
print("deleting thread: ", sql_statement)
|
|
cur.execute(sql_statement)
|
|
sql_statement = f"""
|
|
insert into unloze_event.event
|
|
(event_title, event_server, event_maps, event_date, event_time, event_reward, event_url)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
|
"""
|
|
cur.execute(sql_statement, [item['event_title'], item['event_server'], item['event_maps'], item['event_date'], item['event_time'], item['event_reward'], item['event_url']])
|
|
#context manager does not seem to work with this mysql library so manual commiting seems needed
|
|
conn.commit()
|
|
except Exception:
|
|
import traceback
|
|
error_msg = traceback.format_exc()
|
|
print("error_msg: ", error_msg)
|
|
|
|
return item
|