Skip to content

Commit 5ee1499

Browse files
TomSteenbergentsisodia10
authored andcommitted
fix: Remove redundant batching in PostgreSQLOnlineStore.online_write_batch and fix progress bar (feast-dev#4331)
* Remove batching and fix tqdm progress bar Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com> * Comment Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com> * Remove test changes Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com> * Update comment Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com> --------- Signed-off-by: TomSteenbergen <tomsteenbergen1995@gmail.com>
1 parent 8de8f78 commit 5ee1499

File tree

1 file changed

+5
-8
lines changed
  • sdk/python/feast/infra/online_stores/contrib

1 file changed

+5
-8
lines changed

sdk/python/feast/infra/online_stores/contrib/postgres.py

+5-8
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ def online_write_batch(
7575
Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]
7676
],
7777
progress: Optional[Callable[[int], Any]],
78-
batch_size: int = 5000,
7978
) -> None:
8079
# Format insert values
8180
insert_values = []
@@ -118,15 +117,13 @@ def online_write_batch(
118117
"""
119118
).format(sql.Identifier(_table_id(config.project, table)))
120119

121-
# Push data in batches to online store
120+
# Push data into the online store
122121
with self._get_conn(config) as conn, conn.cursor() as cur:
123-
for i in range(0, len(insert_values), batch_size):
124-
cur_batch = insert_values[i : i + batch_size]
125-
cur.executemany(sql_query, cur_batch)
126-
conn.commit()
122+
cur.executemany(sql_query, insert_values)
123+
conn.commit()
127124

128-
if progress:
129-
progress(len(cur_batch))
125+
if progress:
126+
progress(len(data))
130127

131128
def online_read(
132129
self,

0 commit comments

Comments
 (0)