csvfile = os.path.join(csvdir, table_name + ".csv")
df.to_csv(csvfile, index=False)
tbl = pudl.models.entities.PUDLBase.metadata.tables[table_name]
with open(csvfile, "r", encoding="utf8") as f:
postgres_copy.copy_from(f, tbl, engine, columns=tuple(df.columns),
format="csv", header=True, delimiter=",")
// TODO: For the CEMS, this function is called many times, but the CSV
After Change
// max_size is in bytes; spill to disk if >2GB
with tempfile.SpooledTemporaryFile(max_size=2*1024**3, encoding="utf8") as f:
df.to_csv(f, index=False)
f.seek(0)
postgres_copy.copy_from(f, tbl, engine, columns=tuple(df.columns),
format="csv", header=True, delimiter=",")
if keep_csv:
import shutil
