Index

A

any_command_failed() (in module openwpm.utilities.db_utils)

append_command() (openwpm.command_sequence.CommandSequence method) ArrowProvider (class in openwpm.storage.arrow_storage)

B

BaseCommand (class in openwpm.commands.types) BaseCookie (class in openwpm.utilities.cookie) bot_mitigation (openwpm.config.BrowserParams attribute) bot_mitigation() (in module openwpm.commands.browser_commands) browse() (openwpm.command_sequence.CommandSequence method) (openwpm.task_manager.TaskManager method) BrowseCommand (class in openwpm.commands.browser_commands) browser (openwpm.config.BrowserParams attribute)

browser_id (openwpm.config.BrowserParamsInternal attribute) browser_manager (openwpm.browser_manager.BrowserManagerHandle attribute) BrowserConfigError BrowserCrashError BrowserManager (class in openwpm.browser_manager) BrowserManagerHandle (class in openwpm.browser_manager) BrowserParams (class in openwpm.config) BrowserParamsInternal (class in openwpm.config)

C

cache1 (openwpm.storage.in_memory_storage.MemoryStructuredProvider attribute) cache2 (openwpm.storage.in_memory_storage.MemoryStructuredProvider attribute) callstack_instrument (openwpm.config.BrowserParams attribute) check_expired_leases() (openwpm.utilities.rediswq.RedisWQ method) check_previous_tags() (in module openwpm.commands.utils.XPathUtil) clean_js_instrumentation_settings() (in module openwpm.js_instrumentation) cleaned_js_instrument_settings (openwpm.config.BrowserParamsInternal attribute) click_to_element() (in module openwpm.commands.utils.webdriver_utils) ClientSocket (class in openwpm.socket_interface) ClientSocketHandler (class in openwpm.mp_logger) close() (openwpm.mp_logger.MPLogger method) (openwpm.socket_interface.ClientSocket method) (openwpm.socket_interface.ServerSocket method) (openwpm.storage.storage_controller.DataSocket method) (openwpm.task_manager.TaskManager method) close_browser_manager() (openwpm.browser_manager.BrowserManagerHandle method)

close_other_windows() (in module openwpm.commands.browser_commands) command_queue (openwpm.browser_manager.BrowserManagerHandle attribute) command_thread (openwpm.browser_manager.BrowserManagerHandle attribute) CommandExecutionError CommandSequence (class in openwpm.command_sequence) complete() (openwpm.utilities.rediswq.RedisWQ method) ConfigEncoder (class in openwpm.config) ConfigError connect() (openwpm.socket_interface.ClientSocket method) content_batch (openwpm.storage.leveldb.LevelDbProvider attribute) convert_size() (in module openwpm.utilities.storage_watchdog) Cookie (in module openwpm.utilities.cookie) cookie_instrument (openwpm.config.BrowserParams attribute) CookieError cur (openwpm.storage.sql_provider.SQLiteStorageProvider attribute) current_timeout (openwpm.browser_manager.BrowserManagerHandle attribute) custom_params (openwpm.config.BrowserParams attribute)

D

data_directory (openwpm.config.ManagerParams attribute) DataSocket (class in openwpm.storage.storage_controller) db (openwpm.storage.sql_provider.SQLiteStorageProvider attribute) default() (openwpm.config.ConfigEncoder method) deploy_firefox() (in module openwpm.deploy_browsers.deploy_firefox) display_mode (openwpm.config.BrowserParams attribute) display_pid (openwpm.browser_manager.BrowserManagerHandle attribute)

display_port (openwpm.browser_manager.BrowserManagerHandle attribute) dns_instrument (openwpm.config.BrowserParams attribute) donottrack (openwpm.config.BrowserParams attribute) dump_page_source() (openwpm.command_sequence.CommandSequence method) dump_profile() (in module openwpm.commands.profile_commands) (openwpm.command_sequence.CommandSequence method) DumpPageSourceCommand (class in openwpm.commands.browser_commands) DumpProfileCommand (class in openwpm.commands.profile_commands)

E

empty() (openwpm.utilities.rediswq.RedisWQ method) execute() (openwpm.commands.browser_commands.BrowseCommand method) (openwpm.commands.browser_commands.DumpPageSourceCommand method) (openwpm.commands.browser_commands.FinalizeCommand method) (openwpm.commands.browser_commands.GetCommand method) (openwpm.commands.browser_commands.InitializeCommand method) (openwpm.commands.browser_commands.RecursiveDumpPageSourceCommand method) (openwpm.commands.browser_commands.SaveScreenshotCommand method) (openwpm.commands.browser_commands.ScreenshotFullPageCommand method) (openwpm.commands.profile_commands.DumpProfileCommand method) (openwpm.commands.types.BaseCommand method)

execute_command_sequence() (openwpm.browser_manager.BrowserManagerHandle method) (openwpm.task_manager.TaskManager method) execute_in_all_frames() (in module openwpm.commands.utils.webdriver_utils) execute_script_with_retry() (in module openwpm.commands.utils.webdriver_utils) execute_statement() (openwpm.storage.sql_provider.SQLiteStorageProvider method) extension_enabled (openwpm.config.BrowserParams attribute) ExtractXPath() (in module openwpm.commands.utils.XPathUtil) ExtractXPathError

F

failure_limit (openwpm.config.ManagerParams property) file_name_cache (openwpm.storage.cloud_storage.gcp_storage.GcsUnstructuredProvider attribute) (openwpm.storage.cloud_storage.s3_storage.S3UnstructuredProvider attribute) file_system (openwpm.storage.cloud_storage.gcp_storage.GcsStructuredProvider attribute) (openwpm.storage.cloud_storage.gcp_storage.GcsUnstructuredProvider attribute) (openwpm.storage.cloud_storage.s3_storage.S3StructuredProvider attribute) (openwpm.storage.cloud_storage.s3_storage.S3UnstructuredProvider attribute) finalize_tasks (openwpm.storage.storage_controller.StorageController attribute) finalize_visit_id() (openwpm.storage.arrow_storage.ArrowProvider method) (openwpm.storage.in_memory_storage.MemoryStructuredProvider method) (openwpm.storage.sql_provider.SQLiteStorageProvider method) (openwpm.storage.storage_controller.DataSocket method) (openwpm.storage.storage_controller.StorageController method) (openwpm.storage.storage_providers.StructuredStorageProvider method)

FinalizeCommand (class in openwpm.commands.browser_commands) FirefoxLogInterceptor (class in openwpm.deploy_browsers.selenium_firefox) flush_cache() (openwpm.storage.arrow_storage.ArrowProvider method) (openwpm.storage.cloud_storage.gcp_storage.GcsUnstructuredProvider method) (openwpm.storage.cloud_storage.s3_storage.S3UnstructuredProvider method) (openwpm.storage.in_memory_storage.MemoryStructuredProvider method) (openwpm.storage.in_memory_storage.MemoryUnstructuredProvider method) (openwpm.storage.leveldb.LevelDbProvider method) (openwpm.storage.local_storage.LocalGzipProvider method) (openwpm.storage.sql_provider.SQLiteStorageProvider method) (openwpm.storage.storage_providers.StorageProvider method)

G

GcsStructuredProvider (class in openwpm.storage.cloud_storage.gcp_storage) GcsUnstructuredProvider (class in openwpm.storage.cloud_storage.gcp_storage) geckodriver_pid (openwpm.browser_manager.BrowserManagerHandle attribute) get() (openwpm.command_sequence.CommandSequence method) (openwpm.task_manager.TaskManager method) get_button_text() (in module openwpm.commands.utils.webdriver_utils) get_commands_with_timeout() (openwpm.command_sequence.CommandSequence method) get_configuration_string() (in module openwpm.utilities.platform_utils) get_content() (in module openwpm.utilities.db_utils) get_default_log_settings() (in module openwpm.js_instrumentation) get_firefox_binary_path() (in module openwpm.utilities.platform_utils)

get_intra_links() (in module openwpm.commands.utils.webdriver_utils) get_javascript_entries() (in module openwpm.utilities.db_utils) get_message_from_reader() (in module openwpm.socket_interface) get_most_recent_status() (openwpm.storage.storage_controller.StorageControllerHandle method) get_new_completed_visits() (openwpm.storage.storage_controller.StorageControllerHandle method) get_next_browser_id() (openwpm.storage.storage_controller.StorageControllerHandle method) get_next_visit_id() (openwpm.storage.storage_controller.StorageControllerHandle method) get_retry_number() (openwpm.utilities.rediswq.RedisWQ method) get_status() (openwpm.storage.storage_controller.StorageControllerHandle method) get_version() (in module openwpm.utilities.platform_utils) GetCommand (class in openwpm.commands.browser_commands)

H

handler() (openwpm.storage.storage_controller.StorageController method)

http_instrument (openwpm.config.BrowserParams attribute)

I

init() (openwpm.storage.arrow_storage.ArrowProvider method) (openwpm.storage.cloud_storage.gcp_storage.GcsStructuredProvider method) (openwpm.storage.cloud_storage.gcp_storage.GcsUnstructuredProvider method) (openwpm.storage.cloud_storage.s3_storage.S3StructuredProvider method) (openwpm.storage.cloud_storage.s3_storage.S3UnstructuredProvider method) (openwpm.storage.in_memory_storage.MemoryStructuredProvider method) (openwpm.storage.in_memory_storage.MemoryUnstructuredProvider method) (openwpm.storage.leveldb.LevelDbProvider method) (openwpm.storage.local_storage.LocalGzipProvider method) (openwpm.storage.sql_provider.SQLiteStorageProvider method) (openwpm.storage.storage_providers.StorageProvider method)

InitializeCommand (class in openwpm.commands.browser_commands) is_active() (in module openwpm.commands.utils.webdriver_utils) is_clickable() (in module openwpm.commands.utils.webdriver_utils) (in module openwpm.commands.utils.XPathUtil) is_displayed() (in module openwpm.commands.utils.webdriver_utils) is_found() (in module openwpm.commands.utils.webdriver_utils) is_fresh (openwpm.browser_manager.BrowserManagerHandle attribute) is_loaded() (in module openwpm.commands.utils.webdriver_utils) is_visible() (in module openwpm.commands.utils.webdriver_utils) iter_frames() (in module openwpm.commands.utils.webdriver_utils)

J

js_instrument (openwpm.config.BrowserParams attribute)

js_instrument_settings (openwpm.config.BrowserParams attribute) js_output() (openwpm.utilities.cookie.BaseCookie method)

K

kill_browser_manager() (openwpm.browser_manager.BrowserManagerHandle method)

kill_process_and_children() (in module openwpm.utilities.multiprocess_utils)

L

launch() (openwpm.storage.storage_controller.StorageControllerHandle method) launch_browser_manager() (openwpm.browser_manager.BrowserManagerHandle method) ldb (openwpm.storage.leveldb.LevelDbProvider attribute) lease() (openwpm.utilities.rediswq.RedisWQ method) LevelDbProvider (class in openwpm.storage.leveldb) load() (openwpm.utilities.cookie.BaseCookie method)

load_profile() (in module openwpm.commands.profile_commands) LocalArrowProvider (class in openwpm.storage.local_storage) LocalGzipProvider (class in openwpm.storage.local_storage) lock (openwpm.storage.in_memory_storage.MemoryStructuredProvider attribute) log_path (openwpm.config.ManagerParams attribute) logger_address (openwpm.config.ManagerParamsInternal attribute)

M

main() (in module openwpm.commands.utils.XPathUtil) (in module openwpm.socket_interface) makePickle() (openwpm.mp_logger.ClientSocketHandler method) ManagerParams (class in openwpm.config) ManagerParamsInternal (class in openwpm.config) mark_done() (openwpm.command_sequence.CommandSequence method) maximum_profile_size (openwpm.config.BrowserParams attribute) memory_watchdog (openwpm.config.ManagerParams attribute) MemoryArrowProvider (class in openwpm.storage.in_memory_storage) MemoryProviderHandle (class in openwpm.storage.in_memory_storage) MemoryStructuredProvider (class in openwpm.storage.in_memory_storage) MemoryUnstructuredProvider (class in openwpm.storage.in_memory_storage) module openwpm openwpm.browser_manager openwpm.command_sequence openwpm.commands openwpm.commands.browser_commands openwpm.commands.profile_commands openwpm.commands.types openwpm.commands.utils openwpm.commands.utils.firefox_profile openwpm.commands.utils.webdriver_utils openwpm.commands.utils.XPathUtil openwpm.config openwpm.deploy_browsers openwpm.deploy_browsers.configure_firefox openwpm.deploy_browsers.deploy_firefox openwpm.deploy_browsers.selenium_firefox openwpm.errors openwpm.js_instrumentation openwpm.mp_logger openwpm.socket_interface openwpm.storage openwpm.storage.arrow_storage openwpm.storage.cloud_storage openwpm.storage.cloud_storage.gcp_storage openwpm.storage.cloud_storage.s3_storage openwpm.storage.in_memory_storage openwpm.storage.leveldb openwpm.storage.local_storage openwpm.storage.parquet_schema openwpm.storage.sql_provider openwpm.storage.storage_controller openwpm.storage.storage_providers openwpm.task_manager openwpm.types openwpm.utilities openwpm.utilities.cookie openwpm.utilities.db_utils openwpm.utilities.multiprocess_utils openwpm.utilities.platform_utils openwpm.utilities.rediswq openwpm.utilities.storage_watchdog

move_to_and_click() (in module openwpm.commands.utils.webdriver_utils) move_to_element() (in module openwpm.commands.utils.webdriver_utils) MPLogger (class in openwpm.mp_logger)

N

navigation_instrument (openwpm.config.BrowserParams attribute)

num_browsers (openwpm.config.ManagerParams attribute)

O

openwpm module openwpm.browser_manager module openwpm.command_sequence module openwpm.commands module openwpm.commands.browser_commands module openwpm.commands.profile_commands module openwpm.commands.types module openwpm.commands.utils module openwpm.commands.utils.firefox_profile module openwpm.commands.utils.webdriver_utils module openwpm.commands.utils.XPathUtil module openwpm.config module openwpm.deploy_browsers module openwpm.deploy_browsers.configure_firefox module openwpm.deploy_browsers.deploy_firefox module openwpm.deploy_browsers.selenium_firefox module openwpm.errors module openwpm.js_instrumentation module openwpm.mp_logger module openwpm.socket_interface module openwpm.storage module

openwpm.storage.arrow_storage module openwpm.storage.cloud_storage module openwpm.storage.cloud_storage.gcp_storage module openwpm.storage.cloud_storage.s3_storage module openwpm.storage.in_memory_storage module openwpm.storage.leveldb module openwpm.storage.local_storage module openwpm.storage.parquet_schema module openwpm.storage.sql_provider module openwpm.storage.storage_controller module openwpm.storage.storage_providers module openwpm.task_manager module openwpm.types module openwpm.utilities module openwpm.utilities.cookie module openwpm.utilities.db_utils module openwpm.utilities.multiprocess_utils module openwpm.utilities.platform_utils module openwpm.utilities.rediswq module openwpm.utilities.storage_watchdog module optimize_prefs() (in module openwpm.deploy_browsers.configure_firefox) output() (openwpm.utilities.cookie.BaseCookie method)

P

parse_config_from_env() (in module openwpm.mp_logger) parse_http_stack_trace_str() (in module openwpm.utilities.platform_utils) parse_neterror() (in module openwpm.commands.utils.webdriver_utils) parse_traceback_for_sentry() (in module openwpm.utilities.multiprocess_utils) path_to_str() (in module openwpm.config) poll_queue() (openwpm.storage.in_memory_storage.MemoryProviderHandle method) prefs (openwpm.config.BrowserParams attribute)

privacy() (in module openwpm.deploy_browsers.configure_firefox) Process (class in openwpm.utilities.multiprocess_utils) process_watchdog (openwpm.config.ManagerParams attribute) profile_archive_dir (openwpm.config.BrowserParams attribute) profile_path (openwpm.config.BrowserParamsInternal attribute) profile_size_exceeds_max_size() (in module openwpm.utilities.storage_watchdog) ProfileLoadError

Q

query_db() (in module openwpm.utilities.db_utils)

R

ready() (openwpm.browser_manager.BrowserManagerHandle method) receive_msg() (openwpm.socket_interface.ServerSocket method) recovery_tar (openwpm.config.BrowserParams attribute) recursive_dump_page_source() (openwpm.command_sequence.CommandSequence method) RecursiveDumpPageSourceCommand (class in openwpm.commands.browser_commands) RedisWQ (class in openwpm.utilities.rediswq) renew_lease() (openwpm.utilities.rediswq.RedisWQ method)

restart_browser_manager() (openwpm.browser_manager.BrowserManagerHandle method) restart_required (openwpm.browser_manager.BrowserManagerHandle attribute) run() (openwpm.browser_manager.BrowserManager method) (openwpm.deploy_browsers.selenium_firefox.FirefoxLogInterceptor method) (openwpm.storage.storage_controller.StorageController method) (openwpm.utilities.multiprocess_utils.Process method) (openwpm.utilities.storage_watchdog.StorageLogger method)

S

S3StructuredProvider (class in openwpm.storage.cloud_storage.s3_storage) S3UnstructuredProvider (class in openwpm.storage.cloud_storage.s3_storage) save_batch_if_past_timeout() (openwpm.storage.storage_controller.StorageController method) save_configuration() (openwpm.storage.storage_controller.StorageControllerHandle method) save_content (openwpm.config.BrowserParams attribute) save_screenshot() (openwpm.command_sequence.CommandSequence method) SaveScreenshotCommand (class in openwpm.commands.browser_commands) screenshot_full_page() (openwpm.command_sequence.CommandSequence method) screenshot_path (openwpm.config.ManagerParamsInternal attribute) ScreenshotFullPageCommand (class in openwpm.commands.browser_commands) scroll_down() (in module openwpm.commands.utils.webdriver_utils) scroll_to_bottom() (in module openwpm.commands.utils.webdriver_utils) scroll_to_element() (in module openwpm.commands.utils.webdriver_utils) seed_tar (openwpm.config.BrowserParams attribute) send() (openwpm.socket_interface.ClientSocket method) SerialCookie (class in openwpm.utilities.cookie) ServerSocket (class in openwpm.socket_interface) sessionID() (openwpm.utilities.rediswq.RedisWQ method) set_start_time() (openwpm.commands.types.BaseCommand method) set_visit_browser_id() (openwpm.commands.types.BaseCommand method) set_visit_id() (openwpm.browser_manager.BrowserManagerHandle method) should_shutdown() (openwpm.storage.storage_controller.StorageController method) shutdown() (openwpm.storage.arrow_storage.ArrowProvider method) (openwpm.storage.cloud_storage.gcp_storage.GcsStructuredProvider method) (openwpm.storage.cloud_storage.gcp_storage.GcsUnstructuredProvider method) (openwpm.storage.cloud_storage.s3_storage.S3UnstructuredProvider method) (openwpm.storage.in_memory_storage.MemoryArrowProvider method) (openwpm.storage.in_memory_storage.MemoryStructuredProvider method) (openwpm.storage.in_memory_storage.MemoryUnstructuredProvider method) (openwpm.storage.leveldb.LevelDbProvider method) (openwpm.storage.local_storage.LocalGzipProvider method) (openwpm.storage.sql_provider.SQLiteStorageProvider method) (openwpm.storage.storage_controller.StorageController method) (openwpm.storage.storage_controller.StorageControllerHandle method) (openwpm.storage.storage_providers.StorageProvider method)

shutdown_browser() (openwpm.browser_manager.BrowserManagerHandle method) ShutdownSignal (class in openwpm.commands.types) SimpleCookie (class in openwpm.utilities.cookie) sleep_until_sqlite_checkpoint() (in module openwpm.commands.utils.firefox_profile) SmartCookie (class in openwpm.utilities.cookie) source_dump_path (openwpm.config.ManagerParamsInternal attribute) SQLiteStorageProvider (class in openwpm.storage.sql_provider) start_accepting() (openwpm.socket_interface.ServerSocket method) status_queue (openwpm.browser_manager.BrowserManagerHandle attribute) storage_controller_address (openwpm.config.ManagerParamsInternal attribute) StorageController (class in openwpm.storage.storage_controller) StorageControllerHandle (class in openwpm.storage.storage_controller) StorageLogger (class in openwpm.utilities.storage_watchdog) StorageProvider (class in openwpm.storage.storage_providers) store_blob() (openwpm.storage.cloud_storage.gcp_storage.GcsUnstructuredProvider method) (openwpm.storage.cloud_storage.s3_storage.S3UnstructuredProvider method) (openwpm.storage.in_memory_storage.MemoryUnstructuredProvider method) (openwpm.storage.leveldb.LevelDbProvider method) (openwpm.storage.local_storage.LocalGzipProvider method) (openwpm.storage.storage_providers.UnstructuredStorageProvider method) store_record() (openwpm.storage.arrow_storage.ArrowProvider method) (openwpm.storage.in_memory_storage.MemoryStructuredProvider method) (openwpm.storage.sql_provider.SQLiteStorageProvider method) (openwpm.storage.storage_controller.DataSocket method) (openwpm.storage.storage_controller.StorageController method) (openwpm.storage.storage_providers.StructuredStorageProvider method) store_record_tasks (openwpm.storage.storage_controller.StorageController attribute) storing_lock (openwpm.storage.arrow_storage.ArrowProvider attribute) str_to_path() (in module openwpm.config) StructuredStorageProvider (class in openwpm.storage.storage_providers) switch_to_parent_frame() (in module openwpm.commands.utils.webdriver_utils)

T

tab_restart_browser() (in module openwpm.commands.browser_commands) TaskManager (class in openwpm.task_manager) testing (openwpm.config.ManagerParams attribute) title_contains() (in module openwpm.commands.utils.webdriver_utils) title_is() (in module openwpm.commands.utils.webdriver_utils)

tmp_profile_dir (openwpm.config.BrowserParams attribute) tmp_sqlite_files_exist() (in module openwpm.commands.utils.firefox_profile) total_folder_size() (in module openwpm.utilities.storage_watchdog) tp_cookies (openwpm.config.BrowserParams attribute) tracking_protection (openwpm.config.BrowserParams attribute)

U

UnstructuredStorageProvider (class in openwpm.storage.storage_providers)

update_completion_queue() (openwpm.storage.storage_controller.StorageController method) update_status_queue() (openwpm.storage.storage_controller.StorageController method)

V

validate_browser_params() (in module openwpm.config) validate_crawl_configs() (in module openwpm.config) validate_manager_params() (in module openwpm.config) value_decode() (openwpm.utilities.cookie.BaseCookie method) (openwpm.utilities.cookie.SerialCookie method) (openwpm.utilities.cookie.SimpleCookie method) (openwpm.utilities.cookie.SmartCookie method)

value_encode() (openwpm.utilities.cookie.BaseCookie method) (openwpm.utilities.cookie.SerialCookie method) (openwpm.utilities.cookie.SimpleCookie method) (openwpm.utilities.cookie.SmartCookie method)

W

wait_and_find() (in module openwpm.commands.utils.webdriver_utils) wait_until_loaded() (in module openwpm.commands.utils.webdriver_utils) write_table() (openwpm.storage.arrow_storage.ArrowProvider method) (openwpm.storage.cloud_storage.gcp_storage.GcsStructuredProvider method) (openwpm.storage.cloud_storage.s3_storage.S3StructuredProvider method) (openwpm.storage.in_memory_storage.MemoryArrowProvider method) (openwpm.storage.local_storage.LocalArrowProvider method)

X

xp1_lowercase() (in module openwpm.commands.utils.XPathUtil)

xp1_wildcard() (in module openwpm.commands.utils.XPathUtil)