Skip to content

Commit

Permalink
Merge pull request #118 from NIAEFEUP/fix/sqldump-and-teacher-id-as-s…
Browse files Browse the repository at this point in the history
…tring

SQL dump now has column names and fixed cases where `professor_id` was NULL
  • Loading branch information
tomaspalma authored Aug 14, 2024
2 parents 796e4ae + e7bcb29 commit f55a788
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 4 deletions.
4 changes: 2 additions & 2 deletions src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ upload:
@python ./scripts/upload.py
@echo "Upload completed!"

convert_mysql:
@echo "Converting dump to mysql..."
convert_postgres:
@echo "Converting dump to postgres..."
@bash ./scripts/sqlite3-to-mysql.sh ./scripts/dump/data/dump_sqlite3.sql > ./scripts/dump/data/01_data.sql
@echo "Convertion completed!"

Expand Down
7 changes: 6 additions & 1 deletion src/scrapper/spiders/slot_spider.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,11 @@ def get_professor_info(self, teacher):
[professor_sigarra_id, professor_name,
*_] = teacher["name"].split("-", 1)

return (professor_sigarra_id.strip(), professor_name.strip())
try:
professor_sigarra_id = int(professor_sigarra_id.strip())
except:
professor_sigarra_id = teacher["id"]

return (professor_sigarra_id, professor_name.strip())

return (teacher["sigarra_id"], teacher["name"])
9 changes: 8 additions & 1 deletion src/scripts/dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,16 @@ def dump(self):
f.close()

def dump_table(self, table, con, f):
cursor = con.cursor()
cursor.execute("PRAGMA table_info({})".format(table))

columns = [row[1] for row in cursor.fetchall()]

for line in con.iterdump():
if line.startswith("INSERT") and "\"{}\"".format(table) in line:
f.write('%s\n' % line)
insert_stmt = "INSERT INTO {} ({}) VALUES ".format(table, ", ".join(columns))
values = line.split("VALUES")[1].strip()
f.write(insert_stmt + values + "\n")


if __name__ == '__main__':
Expand Down

0 comments on commit f55a788

Please sign in to comment.