config.ini

[default]
max_lines_to_read = 0
force = false
aggregate_n_rows = 1000
aggregate_all_min_rows = 500
dropna = true
add_fields_id_field = id_of_max_pos_removed_item
# extra_fields must be the same order as specified by the DB in 5-load-db.py 'columns' variable
# order is, id, [extra_fields], score
extra_fields_comments = body,created_utc,link_id
extra_fields_posts = created_utc,num_comments,title
remote_file_sizes = remote_file_sizes.txt

[normal]
pushshift_raw_dir = ./data/0-pushshift_raw/
pushshift_slim_dir = ./data/1-pushshift_slim/
aggregate_dir = ./data/2-aggregate_monthly/
aggregate_all_dir = ./data/3-aggregate_all/
add_fields_dir = ./data/4-add_fields/
unprocessable_files_log = ./unprocessable_files_log.txt
skippable_files_log = ./skippable_files.txt
inaccessible_ids_file_comments = ./inaccessible_ids_comments.txt
inaccessible_ids_file_posts = ./inaccessible_ids_posts.txt

[test]
pushshift_raw_dir = ./test/0-pushshift_raw/
pushshift_slim_dir = ./test/1-pushshift_slim/
aggregate_dir = ./test/2-aggregate_monthly/
aggregate_all_dir = ./test/3-aggregate_all/
add_fields_dir = ./test/4-add_fields/
unprocessable_files_log = ./test/unprocessable_files_log.txt
skippable_files_log = ./test/skippable_files.txt
inaccessible_ids_file_comments = ./test/inaccessible_ids_comments.txt
inaccessible_ids_file_posts = ./test/inaccessible_ids_posts.txt
max_lines_to_read = 100000
force = true
aggregate_n_rows = 100
aggregate_all_min_rows = 50
dropna = true

[test2]
pushshift_raw_dir = ./data/test2/0-pushshift_raw/
pushshift_slim_dir = ./data/test2/1-pushshift_slim/
aggregate_dir = ./data/test2/2-aggregate_monthly/
aggregate_all_dir = ./data/test2/3-aggregate_all/
add_fields_dir = ./data/test2/4-add_fields/
unprocessable_files_log = ./data/test2/unprocessable_files_log.txt
skippable_files_log = ./data/test2/skippable_files.txt
inaccessible_ids_file_comments = ./data/test2/inaccessible_ids_comments.txt
inaccessible_ids_file_posts = ./data/test2/inaccessible_ids_posts.txt
max_lines_to_read = 0
force = false
aggregate_n_rows = 1000
aggregate_all_min_rows = 500
dropna = true